From 80cc5b20982696bac0c45305f5f4b98a1e709e5e Mon Sep 17 00:00:00 2001 From: mpariente Date: Wed, 5 Aug 2026 10:26:57 -0700 Subject: [PATCH] Add FP8 vision encoder quantization support and recipes for Qwen3-VL and Qwen3.5 Signed-off-by: mpariente --- CHANGELOG.rst | 5 + examples/hf_ptq/README.md | 57 ++++++- examples/hf_ptq/hf_ptq.py | 25 +-- .../qwen3_5/ptq/fp8_vision-kv_none.yaml | 33 ++++ .../ptq/fp8_vision_lm-kv_fp8_cast.yaml | 41 +++++ .../qwen3_vl/ptq/fp8_vision-kv_none.yaml | 33 ++++ .../ptq/fp8_vision_lm-kv_fp8_cast.yaml | 41 +++++ .../qwen3_vl/ptq/vision_fp8.quant_cfg.yaml | 35 +++++ modelopt_recipes/ptq.md | 14 +- .../_test_utils/torch/transformers_models.py | 34 ++++- .../hf_ptq/test_hf_ptq_vision_quantization.py | 144 ++++++++++++++++++ .../export/test_qwen_vision_recipe_export.py | 104 +++++++++++++ tests/unit/recipe/test_qwen_vision_recipe.py | 92 +++++++++++ 13 files changed, 637 insertions(+), 21 deletions(-) create mode 100644 modelopt_recipes/huggingface/qwen3_5/ptq/fp8_vision-kv_none.yaml create mode 100644 modelopt_recipes/huggingface/qwen3_5/ptq/fp8_vision_lm-kv_fp8_cast.yaml create mode 100644 modelopt_recipes/huggingface/qwen3_vl/ptq/fp8_vision-kv_none.yaml create mode 100644 modelopt_recipes/huggingface/qwen3_vl/ptq/fp8_vision_lm-kv_fp8_cast.yaml create mode 100644 modelopt_recipes/huggingface/qwen3_vl/ptq/vision_fp8.quant_cfg.yaml create mode 100644 tests/examples/hf_ptq/test_hf_ptq_vision_quantization.py create mode 100644 tests/gpu/torch/export/test_qwen_vision_recipe_export.py create mode 100644 tests/unit/recipe/test_qwen_vision_recipe.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7a188669cb8..0894a23c3c5 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,7 @@ Changelog *Quantization* +- Add opt-in FP8 Vision Encoder recipes under the ``qwen3_vl`` and ``qwen3_5`` model types. The vision-only recipe keeps the language model and KV cache in high precision; the joint recipe quantizes Vision Encoder and language-model Linears and uses FP8 KV-cache cast. Both quantize primary and deepstack merger Linears where present, while leaving patch embedding and vision-attention BMMs in high precision. Exported checkpoints require an inference runtime that supports quantized Vision Encoder Linears. - Add the ``nvfp4_act_headroom`` calibration algorithm for NVFP4 **activation** global scales. Plain ``max`` sets a tensor's global scale from the largest per-block amax seen during calibration, leaving no room above it, so any activation larger than the calibration max saturates. ``nvfp4_act_headroom`` instead anchors the global scale to a low percentile of the per-block amax distribution, ``amax = max(rho * anchor, upper)``, placing the calibrated blocks in the lower part of the FP8 block-scale range and leaving the rest as headroom. ``upper`` is the top of the range the scale commits to representing and defaults to the 99.99th percentile rather than the literal maximum: chasing a lone freak block would drag the global scale up until every other block's FP8 block scale falls below subnormal and flushes to zero, so the rarest blocks are clipped instead. Set ``upper_percentile=100`` to use the literal observed max, which guarantees no calibration data is clipped. The calibrator warns when the per-block range is too wide for ``rho`` to clear any headroom. Tunable via ``anchor_percentile`` (default 1), ``upper_percentile`` (default 99.99) and ``rho`` (default 16384). Applies only to NVFP4 dynamic-block input quantizers. Weight scales are an orthogonal axis, selected by a nested ``weight_scale_algorithm`` (``max`` by default, or ``mse`` / ``local_hessian`` with that algorithm's own options), so one recipe can combine a weight calibration with this activation policy in a single pass. ``SequentialQuantizer`` activation quantizers are not supported and raise. Ships ``modelopt_recipes/general/ptq/nvfp4_act_headroom-kv_fp8_cast.yaml``, which mirrors ``nvfp4_default-kv_fp8_cast`` (dynamic NVFP4 W4A4 plus FP8 KV-cache cast) with only the calibration algorithm swapped, so it exports a standard NVFP4 checkpoint. *Misc* @@ -17,10 +18,14 @@ Changelog **Backward Breaking Changes** +- Image-text calibration with ``--recipe`` now applies recipe rules to the complete VLM. Custom recipes must explicitly exclude the vision branch when it should remain in high precision. The deprecated ``--auto_quantize_bits`` path also retains the complete VLM, although AutoQuantize continues to reject image-text calibration before quantization. + **Deprecations** **Bug Fixes** +- Fix image-text calibration for non-Nemotron VLMs by forwarding multimodal batches through the complete VLM while instrumenting only the selected quantization target. + 0.46 (2026-08-xx) ^^^^^^^^^^^^^^^^^ diff --git a/examples/hf_ptq/README.md b/examples/hf_ptq/README.md index e80926bea91..a45fd104207 100755 --- a/examples/hf_ptq/README.md +++ b/examples/hf_ptq/README.md @@ -121,6 +121,7 @@ Please reference our [framework scripts](#framework-scripts) and our [docs](http | Llava (VLM)11 | ✅ | ✅12 | ✅ | ✅ | - | | Phi-3-vision, Phi-4-multimodal (VLM)11 | ✅ | ✅12 | ✅ | ✅ | ✅ | | Qwen2, 2.5-VL (VLM)11 | ✅ | ✅12 | ✅ | ✅ | ✅ | +| Qwen3-VL, Qwen3.5 (VLM)11,14 | ✅ | - | - | - | - | | Gemma 3 (VLM)11 | ✅ | - | - | - | - | | Nemotron VL (VLM)11,13 | ✅ | - | - | - | ✅ | @@ -136,18 +137,20 @@ Please reference our [framework scripts](#framework-scripts) and our [docs](http > *8.GLM-4.7 has MTP (Multi-Token Prediction) layers that are automatically loaded and excluded from quantization.* \ > *9.Running Whisper model with transformers>=5.0 requires [torchcodec](https://github.com/meta-pytorch/torchcodec?tab=readme-ov-file#installing-cuda-enabled-torchcodec) and other system packages (e.g. ffmpeg).* \ > *10.GPT-OSS ships with native MXFP4 weights; NVFP4 export is produced via the closed-form `--cast_mxfp4_to_nvfp4` cast (see [MXFP4 → NVFP4 cast](#mxfp4--nvfp4-cast-for-gpt-oss)).* \ -> *11.Vision-language model (VLM): only the language model is quantized while the vision encoder is kept in high precision. Pass `--vlm` to the shell script (see [VLM quantization](#vlm-quantization)).* \ +> *11.Vision-language model (VLM): by default, only the language model is quantized while the vision encoder is kept in high precision. Pass `--vlm` to the shell script (see [VLM quantization](#vlm-quantization)).* \ > *12.For VLMs, `int8_sq` only supports TensorRT-LLM checkpoint export and is not compatible with the TensorRT-LLM torch backend.* \ -> *13.Nemotron VL automatically calibrates with image-text pairs; see [VLM calibration with image-text pairs](#vlm-calibration-with-image-text-pairs-eg-nemotron-vl).* +> *13.Nemotron VL automatically calibrates with image-text pairs; see [VLM calibration with image-text pairs](#vlm-calibration-with-image-text-pairs-eg-nemotron-vl).* \ +> *14.Qwen3-VL and dense Qwen3.5 VLM checkpoints support opt-in FP8 vision encoder quantization through model-specific recipes. Vision Linear layers, including primary and deepstack merger Linears where present, are quantized; patch embedding and vision-attention BMMs remain in high precision. MoE variants are not validated by these recipes.* > *The accuracy loss after PTQ may vary depending on the actual model and the quantization method. Different models may have different accuracy loss and usually the accuracy loss is more significant when the base model is small. If the accuracy after PTQ is not meeting the requirement, please try either modifying [hf_ptq.py](./hf_ptq.py) and disabling the KV cache quantization or using the [QAT](./../llm_qat/README.md) instead. For NVFP4 quantization specifically, we recommend `nvfp4_mlp_only`, `nvfp4_experts_only`, or `nvfp4_omlp_only` to achieve higher accuracy by restricting quantization to the MLP/expert layers (and optionally the `o_proj` layer) while keeping the attention QKV projections unquantized.* > You can also create your own custom config using [this](https://nvidia.github.io/Model-Optimizer/guides/_pytorch_quantization.html#custom-calibration-algorithm) guide. > *Vision-language models (VLMs) are listed in the support matrix above (rows marked `(VLM)`). PTQ for -> VLMs is handled by the same `hf_ptq.py` entry point and shell script as LLMs — the language model is -> quantized while the vision encoder is kept in high precision. Pass `--vlm` to the shell script (see -> [VLM quantization](#vlm-quantization)). For detailed TensorRT-LLM torch backend multimodal support, +> VLMs is handled by the same `hf_ptq.py` entry point and shell script as LLMs. By default, the +> language model is quantized while the vision branch remains in high precision. Dense Qwen3-VL +> and dense Qwen3.5 VLM checkpoints additionally support the opt-in FP8 recipes documented under +> [VLM quantization](#vlm-quantization). For detailed TensorRT-LLM torch backend multimodal support, > please refer to [this doc](https://github.com/NVIDIA/TensorRT-LLM/blob/main/docs/source/models/supported-models.md#multimodal-feature-support-matrix-pytorch-backend).* ## Framework Scripts @@ -269,6 +272,46 @@ scripts/huggingface_example.sh --model - Supported `--quant` values for VLMs are `fp8`, `nvfp4`, `int8_sq`, `int4_awq`, and `w4a8_awq` (see the `(VLM)` rows in the [Support Matrix](#hugging-face-supported-models)). +By default, `hf_ptq.py` applies `--qformat` only to the language model. Model-specific recipes +add FP8 quantization of the vision branch for validated Qwen3-VL and dense Qwen3.5 checkpoints. +Use the recipe directory matching the checkpoint's `model_type`: `qwen3_vl` or `qwen3_5`. + +```bash +# Vision encoder only: FP8 vision Linears and merger, BF16 LLM and KV cache. +python hf_ptq.py \ + --pyt_ckpt_path \ + --recipe huggingface/qwen3_vl/ptq/fp8_vision-kv_none \ + --calib_with_images \ + --calib_size 512 \ + --skip_generate \ + --export_path + +# Joint vision encoder + language model FP8 with FP8 KV-cache cast. +python hf_ptq.py \ + --pyt_ckpt_path \ + --recipe huggingface/qwen3_vl/ptq/fp8_vision_lm-kv_fp8_cast \ + --calib_with_images \ + --calib_size 512 \ + --skip_generate \ + --export_path +``` + +`fp8_vision-kv_none` starts from all quantizers disabled and enables FP8 only for `nn.Linear` +weights and inputs under the `visual` branch, including the primary merger and any deepstack +mergers. The language model, KV cache, patch embedding, and vision-attention QK/softmax/AV +operations stay in high precision. +`fp8_vision_lm-kv_fp8_cast` applies the standard FP8 model recipe to both model branches and enables FP8 +KV-cache cast, while keeping patch embedding and vision-attention operands in high precision. +Other precision combinations can be expressed by composing the same recipe units in a custom +recipe; no model-specific Python path is required. + +The exported checkpoint requires an inference runtime that supports quantized vision encoder +Linears. Runtime-specific vision-attention quantization is separate from this ModelOpt checkpoint. +Use the direct `hf_ptq.py` commands above for these recipes; the generic multimodal quickstart is +not a serving validation for a vision-quantized checkpoint. Both examples pass `--skip_generate` +because the script's text-only preview does not forward the image tensors used during calibration. +For a Qwen3.5 checkpoint, replace `qwen3_vl` with `qwen3_5` in the recipe path. + > *This consolidates the former `examples/vlm_ptq` example, which now forwards here.* #### VLM calibration with image-text pairs (e.g., Nemotron VL) @@ -291,6 +334,10 @@ The same flag is exposed by the shell script: scripts/huggingface_example.sh --model --quant nvfp4 --vlm --calib_with_images --trust_remote_code ``` +With `--calib_with_images`, calibration batches always pass through the complete VLM so image +features reach the component selected by the preset or recipe. This also applies to the default +language-model-only path: its quantizers are exercised by the complete multimodal forward. + > Note: when `--calib_with_images` is set, `--calib_size` must be a single value, and the calibration dataset is nvidia/nemotron_vlm_dataset_v2. This functionality is currently in beta and has been tested on `nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16`. diff --git a/examples/hf_ptq/hf_ptq.py b/examples/hf_ptq/hf_ptq.py index 6ddfba6614c..c577cc9bb7c 100755 --- a/examples/hf_ptq/hf_ptq.py +++ b/examples/hf_ptq/hf_ptq.py @@ -663,11 +663,15 @@ def load_model(args: argparse.Namespace): default_padding_side = tokenizer.padding_side tokenizer.padding_side = "left" - # Quantize only the language model, but keep the full_model for calibration forward. - extracted_lm, extracted_model_type = extract_and_prepare_language_model_from_vl(full_model) - if extracted_lm is not None: - language_model = extracted_lm - model_type = extracted_model_type + # Plain PTQ quantizes only the language model. Recipes and AutoQuantize keep the complete + # VLM so their quantizer rules can target vision and language components in one state. + if args.recipe is None and args.auto_quantize_bits is None: + extracted_lm, extracted_model_type = extract_and_prepare_language_model_from_vl( + full_model + ) + if extracted_lm is not None: + language_model = extracted_lm + model_type = extracted_model_type else: if args.specdec_offline_dataset is not None: language_model = full_model @@ -762,7 +766,7 @@ def mono_quantize( calib_dataloader: DataLoader, is_nemotron_vl_model: bool, ): - """Plain quantization of the given language model to a single quantization configuration.""" + """Plain quantization of the selected model target to one quantization configuration.""" model_is_already_quantized = is_quantized(language_model) @@ -781,9 +785,10 @@ def mono_quantize( warnings.warn("Dynamic quantization. Calibration skipped.") calibrate_loop = None if use_calibration: - # For Nemotron VL image calibration, the dataloader yields multimodal kwargs (e.g., pixel_values). - # Those kwargs must be consumed by the *full* VLM model, not the extracted language_model. - if args.calib_with_images and is_nemotron_vl_model: + # Image calibration batches contain multimodal kwargs (for example pixel_values). + # They must be consumed by the complete VLM even when only a nested component is the + # quantization target; the full forward still exercises that component's quantizers. + if args.calib_with_images: calibrate_loop = create_vlm_calibration_loop(full_model, calib_dataloader) else: calibrate_loop = create_forward_loop( @@ -801,7 +806,7 @@ def mono_quantize( language_model = mtq.quantize(language_model, quant_cfg, forward_loop=calibrate_loop) # For VL models, update full_model to use the quantized language model - if is_nemotron_vl_model: + if is_nemotron_vl_model and language_model is not full_model: language_model_lineage = get_language_model_from_vl(full_model) if language_model_lineage is not None: print("Updating full_model with quantized language_model...") diff --git a/modelopt_recipes/huggingface/qwen3_5/ptq/fp8_vision-kv_none.yaml b/modelopt_recipes/huggingface/qwen3_5/ptq/fp8_vision-kv_none.yaml new file mode 100644 index 00000000000..59105e4d804 --- /dev/null +++ b/modelopt_recipes/huggingface/qwen3_5/ptq/fp8_vision-kv_none.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dense Qwen3.5 Vision Encoder FP8 with a high-precision language model and KV cache. + +imports: + base_disable_all: configs/ptq/units/base_disable_all + vision_fp8: huggingface/qwen3_vl/ptq/vision_fp8.quant_cfg + +metadata: + recipe_type: ptq + description: >- + FP8 quantization of dense Qwen3.5 Vision Encoder Linear layers, including any merger Linears; + the language model, KV cache, patch embedding, and vision-attention operands remain in high + precision. + +quantize: + algorithm: max + quant_cfg: + - $import: base_disable_all + - $import: vision_fp8 diff --git a/modelopt_recipes/huggingface/qwen3_5/ptq/fp8_vision_lm-kv_fp8_cast.yaml b/modelopt_recipes/huggingface/qwen3_5/ptq/fp8_vision_lm-kv_fp8_cast.yaml new file mode 100644 index 00000000000..ff82963d6fe --- /dev/null +++ b/modelopt_recipes/huggingface/qwen3_5/ptq/fp8_vision_lm-kv_fp8_cast.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dense Qwen3.5 Vision Encoder and language model FP8 with FP8 KV-cache cast. + +imports: + base_disable_all: configs/ptq/units/base_disable_all + default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers + kv_fp8_cast: configs/ptq/units/kv_fp8_cast + vision_fp8: huggingface/qwen3_vl/ptq/vision_fp8.quant_cfg + w8a8_fp8_fp8: configs/ptq/units/w8a8_fp8_fp8 + +metadata: + recipe_type: ptq + description: >- + W8A8 FP8 quantization of dense Qwen3.5 Vision Encoder and language-model Linear layers, + including any merger Linears, with FP8 KV-cache cast; patch embedding and vision-attention + operands remain in high precision. + +quantize: + algorithm: max + quant_cfg: + - $import: base_disable_all + - $import: w8a8_fp8_fp8 + - $import: kv_fp8_cast + - $import: default_disabled_quantizers + # Re-enable visual Linears after the standard vision exclusion. The imported snippet also + # keeps vision-attention BMM quantizers disabled as a final defense-in-depth rule. + - $import: vision_fp8 diff --git a/modelopt_recipes/huggingface/qwen3_vl/ptq/fp8_vision-kv_none.yaml b/modelopt_recipes/huggingface/qwen3_vl/ptq/fp8_vision-kv_none.yaml new file mode 100644 index 00000000000..6daec3751ef --- /dev/null +++ b/modelopt_recipes/huggingface/qwen3_vl/ptq/fp8_vision-kv_none.yaml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Qwen3-VL Vision Encoder FP8 with a high-precision language model and KV cache. + +imports: + base_disable_all: configs/ptq/units/base_disable_all + vision_fp8: huggingface/qwen3_vl/ptq/vision_fp8.quant_cfg + +metadata: + recipe_type: ptq + description: >- + FP8 quantization of Qwen3-VL Vision Encoder Linear layers, including primary and deepstack + merger Linears; the language model, KV cache, patch embedding, and vision-attention operands + remain in high precision. + +quantize: + algorithm: max + quant_cfg: + - $import: base_disable_all + - $import: vision_fp8 diff --git a/modelopt_recipes/huggingface/qwen3_vl/ptq/fp8_vision_lm-kv_fp8_cast.yaml b/modelopt_recipes/huggingface/qwen3_vl/ptq/fp8_vision_lm-kv_fp8_cast.yaml new file mode 100644 index 00000000000..e3c78ba3ba6 --- /dev/null +++ b/modelopt_recipes/huggingface/qwen3_vl/ptq/fp8_vision_lm-kv_fp8_cast.yaml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Qwen3-VL Vision Encoder and language model FP8 with FP8 KV-cache cast. + +imports: + base_disable_all: configs/ptq/units/base_disable_all + default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers + kv_fp8_cast: configs/ptq/units/kv_fp8_cast + vision_fp8: huggingface/qwen3_vl/ptq/vision_fp8.quant_cfg + w8a8_fp8_fp8: configs/ptq/units/w8a8_fp8_fp8 + +metadata: + recipe_type: ptq + description: >- + W8A8 FP8 quantization of Qwen3-VL Vision Encoder and language-model Linear layers, including + primary and deepstack merger Linears, with FP8 KV-cache cast; patch embedding and + vision-attention operands remain in high precision. + +quantize: + algorithm: max + quant_cfg: + - $import: base_disable_all + - $import: w8a8_fp8_fp8 + - $import: kv_fp8_cast + - $import: default_disabled_quantizers + # Re-enable visual Linears after the standard vision exclusion. The imported snippet also + # keeps vision-attention BMM quantizers disabled as a final defense-in-depth rule. + - $import: vision_fp8 diff --git a/modelopt_recipes/huggingface/qwen3_vl/ptq/vision_fp8.quant_cfg.yaml b/modelopt_recipes/huggingface/qwen3_vl/ptq/vision_fp8.quant_cfg.yaml new file mode 100644 index 00000000000..9486d5a7fa2 --- /dev/null +++ b/modelopt_recipes/huggingface/qwen3_vl/ptq/vision_fp8.quant_cfg.yaml @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# FP8 Linear weights and inputs in the Qwen visual branch. Patch embedding +# and vision-attention operands remain in high precision. + +# modelopt-schema: modelopt.torch.quantization.config.QuantizerCfgListConfig +imports: + fp8: configs/numerics/fp8 +--- + - quantizer_name: '*visual.*weight_quantizer' + parent_class: 'nn.Linear' + cfg: + $import: fp8 + - quantizer_name: '*visual.*input_quantizer' + parent_class: 'nn.Linear' + cfg: + $import: fp8 + - quantizer_name: '*visual.*patch_embed*' + enable: false + # Keep this defense-in-depth rule last when the snippet follows a KV-cache unit. + - quantizer_name: '*visual.*_bmm_quantizer' + enable: false diff --git a/modelopt_recipes/ptq.md b/modelopt_recipes/ptq.md index 0b932a35673..40d3349a6c7 100644 --- a/modelopt_recipes/ptq.md +++ b/modelopt_recipes/ptq.md @@ -231,22 +231,30 @@ that baseline. The deviations come in four kinds: | Kind | What changes vs. the general recipe | Examples | |------|-------------------------------------|----------| -| **Architecture-aware `quant_cfg`** | Per-sub-module format choices a single wildcard scheme can't express | `minimax_m3_vl`, `qwen3_5`, `qwen3_5_moe`, `vit`, `nemotron_llama` | +| **Architecture-aware `quant_cfg`** | Per-sub-module format choices a single wildcard scheme can't express | `minimax_m3_vl`, `qwen3_vl`, `qwen3_5`, `qwen3_5_moe`, `vit`, `nemotron_llama` | | **Algorithm override** | Same numerics & scope, but the *calibration algorithm* is tweaked because the default breaks or regresses | `gemma`, `gemma4`, `mpt` | | **Extra exclusions** | Adds disabled-quantizer patterns so non-language branches stay full precision | `nemotron_vl`, `phi4mm`, `diffusion_gemma` | | **Checkpoint mirror** | A mixed-precision map reproducing one published checkpoint exactly | `models/nvidia/Nemotron-3-*`, `models/nvidia/Mistral-Medium-3.5-128B-NVFP4` | The numerics and standard exclusions are still inherited from `configs/` wherever possible — the model folder captures *only* the delta. Each `/` -folder carries a `README.md` spelling out that delta. +folder may carry a `README.md` spelling out that delta. -### Architecture-aware `quant_cfg` — `minimax_m3_vl`, `qwen3_5`, `qwen3_5_moe`, `vit`, `nemotron_llama` +### Architecture-aware `quant_cfg` — `minimax_m3_vl`, `qwen3_vl`, `qwen3_5`, `qwen3_5_moe`, `vit`, `nemotron_llama` **`minimax_m3_vl/ptq/mxfp8_nvfp4_experts`** applies MXFP8 to the language-model linear layers and MSE-calibrated NVFP4 to routed experts, with expert `input_scale` fixed to 1.0. The vision branch, routers, `lm_head`, and KV cache remain unquantized. +**`qwen3_vl/ptq`** and **`qwen3_5/ptq`** provide FP8 recipes for the `visual` branch of +validated Qwen3-VL and dense Qwen3.5 checkpoints. `fp8_vision-kv_none` enables only Vision +Encoder `nn.Linear` weights and inputs, including the primary merger and any deepstack mergers. +`fp8_vision_lm-kv_fp8_cast` combines that visual configuration with the standard W8A8 FP8 model +and FP8 KV-cache-cast units. Both keep patch embedding and vision-attention BMM operands in high +precision. The shared visual snippet lives under `qwen3_vl`; thin wrappers remain discoverable +under each exact Hugging Face `model_type`. + `huggingface/qwen3_5/ptq/w4a16_nvfp4-fp8_attn-kv_fp8_cast` (and its MoE twin, which shares the same `quant_cfg` snippet) is a **mixed scheme no single general body covers**: NVFP4 **W4A16** on MLP / expert projection weights and `lm_head`, diff --git a/tests/_test_utils/torch/transformers_models.py b/tests/_test_utils/torch/transformers_models.py index cf75e50e107..895d1769b68 100644 --- a/tests/_test_utils/torch/transformers_models.py +++ b/tests/_test_utils/torch/transformers_models.py @@ -41,6 +41,7 @@ Qwen3Config, Qwen3MoeConfig, Qwen3VLConfig, + Qwen3VLTextConfig, T5Config, T5ForConditionalGeneration, ViTConfig, @@ -221,6 +222,14 @@ def get_tiny_qwen3vl(**config_kwargs) -> PreTrainedModel: "max_position_embeddings": 32, "vocab_size": 32, } + # Transformers 4.x names this field rope_scaling; 5.x renamed it to rope_parameters. + # Supplying it avoids the 4.57 Qwen3-VL constructor dereferencing a None rope config. + rope_field = ( + "rope_parameters" + if "rope_parameters" in getattr(Qwen3VLTextConfig, "__annotations__", {}) + else "rope_scaling" + ) + text_kwargs[rope_field] = {"rope_type": "default", "mrope_section": [1, 1, 2]} text_kwargs.update(config_kwargs) # Pass as dicts — transformers 5.3.0 Qwen3VLConfig.__init__ only handles # vision_config/text_config when they are dicts or None, not instances. @@ -324,14 +333,28 @@ def create_tiny_gemma3vl_dir( QWEN3_5_VL_REF = "Qwen/Qwen3.5-0.8B" -def _get_tiny_qwen3_5_vl(moe: bool = False, **config_kwargs) -> PreTrainedModel: +class _TinyQwen35Tokenizer: + """Minimal offline tokenizer contract required to construct a tiny Qwen3.5-VL model.""" + + image_token_id = 1 + video_token_id = 2 + vision_bos_token_id = 3 + vision_eos_token_id = 4 + + def __len__(self): + return 32 + + +def _get_tiny_qwen3_5_vl(moe: bool = False, *, tokenizer=None, **config_kwargs) -> PreTrainedModel: # Lazy imports — Qwen3.5-VL requires a recent transformers version. from transformers import Qwen3_5Config, Qwen3_5MoeConfig set_seed(SEED) - # Vocab + vision token ids derive from the ref tokenizer to match the saved processor. - tokenizer = _get_tiny_vlm_tokenizer(AutoTokenizer.from_pretrained(QWEN3_5_VL_REF)) + # Vocab + vision token ids normally derive from the reference tokenizer. Tests that do not + # exercise processing may inject the minimal offline contract above instead. + if tokenizer is None: + tokenizer = _get_tiny_vlm_tokenizer(AutoTokenizer.from_pretrained(QWEN3_5_VL_REF)) # Hybrid GatedDeltaNet (linear attention) + gated full-attention (layer_types auto-generated). text_kwargs = { @@ -415,6 +438,11 @@ def _create_tiny_qwen3_5_vl_dir( create_tiny_qwen3_5_moe_vl_dir = partial(_create_tiny_qwen3_5_vl_dir, moe=True) +def get_tiny_qwen3_5_vl_offline(**config_kwargs) -> PreTrainedModel: + """Build a dense tiny Qwen3.5-VL without downloading its reference tokenizer.""" + return _get_tiny_qwen3_5_vl(moe=False, tokenizer=_TinyQwen35Tokenizer(), **config_kwargs) + + ##### NEMOTRON ##### def get_tiny_nemotron(**config_kwargs) -> PreTrainedModel: set_seed(SEED) diff --git a/tests/examples/hf_ptq/test_hf_ptq_vision_quantization.py b/tests/examples/hf_ptq/test_hf_ptq_vision_quantization.py new file mode 100644 index 00000000000..504182a1405 --- /dev/null +++ b/tests/examples/hf_ptq/test_hf_ptq_vision_quantization.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch + +_EXAMPLES_DIR = Path(__file__).resolve().parents[3] / "examples" / "hf_ptq" + + +@pytest.fixture +def hf_ptq(monkeypatch): + monkeypatch.syspath_prepend(str(_EXAMPLES_DIR)) + return importlib.import_module("hf_ptq") + + +@pytest.mark.parametrize( + ("recipe", "extracts_language_model"), + [ + (None, True), + ("huggingface/qwen3_vl/ptq/fp8_vision-kv_none", False), + ], +) +def test_image_calibration_model_target_follows_recipe( + hf_ptq, monkeypatch, recipe, extracts_language_model +): + """Plain PTQ extracts the LM; a recipe keeps the complete VLM as its target.""" + full_model = SimpleNamespace(device=torch.device("cpu")) + extracted_language_model = object() + tokenizer = SimpleNamespace(pad_token="", eos_token="", padding_side="right") + extraction_calls = [] + + args = SimpleNamespace( + use_fsdp2=False, + specdec_offline_dataset=None, + low_memory_mode=False, + pyt_ckpt_path="dummy", + dist_state=SimpleNamespace(device=torch.device("cpu")), + gpu_max_mem_percentage=0.8, + trust_remote_code=False, + use_seq_device_map=False, + attn_implementation=None, + calib_with_images=True, + recipe=recipe, + auto_quantize_bits=None, + ) + + monkeypatch.setattr(hf_ptq, "get_model", lambda *args, **kwargs: full_model) + monkeypatch.setattr(hf_ptq, "get_model_type", lambda model: "qwen3_vl") + monkeypatch.setattr(hf_ptq, "is_nemotron_vl", lambda model: False) + monkeypatch.setattr( + hf_ptq.AutoProcessor, + "from_pretrained", + lambda *args, **kwargs: SimpleNamespace(tokenizer=tokenizer), + ) + + def extract_language_model(model): + extraction_calls.append(model) + return extracted_language_model, "qwen3" + + monkeypatch.setattr( + hf_ptq, "extract_and_prepare_language_model_from_vl", extract_language_model + ) + + loaded = hf_ptq.load_model(args) + quantization_target = loaded[1] + + assert extraction_calls == ([full_model] if extracts_language_model else []) + assert quantization_target is ( + extracted_language_model if extracts_language_model else full_model + ) + + +def test_image_calibration_uses_full_vlm_forward(hf_ptq, monkeypatch): + args = SimpleNamespace( + qformat="fp8", + calib_with_images=True, + specdec_offline_dataset=None, + ) + full_model = torch.nn.Module() + calib_dataloader = object() + calibration_loop = object() + calls = {} + + monkeypatch.setattr(hf_ptq, "is_quantized", lambda model: False) + monkeypatch.setattr(hf_ptq, "need_calibration", lambda quant_cfg: True) + + def create_vlm_loop(model, dataloader): + calls["loop_model"] = model + calls["dataloader"] = dataloader + return calibration_loop + + def quantize(model, quant_cfg, forward_loop): + calls["quantization_target"] = model + calls["forward_loop"] = forward_loop + return model + + monkeypatch.setattr(hf_ptq, "create_vlm_calibration_loop", create_vlm_loop) + monkeypatch.setattr( + hf_ptq, + "get_language_model_from_vl", + lambda model: pytest.fail("The complete VLM must not be reinserted as its language model"), + ) + monkeypatch.setattr( + hf_ptq, + "create_forward_loop", + lambda **kwargs: pytest.fail("Text-only calibration loop must not be selected"), + ) + monkeypatch.setattr(hf_ptq.mtq, "quantize", quantize) + + # Force the Nemotron reinsertion branch: a full-model recipe target must not be assigned as + # its own nested language model. + hf_ptq.mono_quantize( + args, + {"algorithm": "max"}, + full_model, + full_model, + model_type="qwen3_vl", + calibration_only=False, + calib_dataloader=calib_dataloader, + is_nemotron_vl_model=True, + ) + + assert calls == { + "loop_model": full_model, + "dataloader": calib_dataloader, + "quantization_target": full_model, + "forward_loop": calibration_loop, + } diff --git a/tests/gpu/torch/export/test_qwen_vision_recipe_export.py b/tests/gpu/torch/export/test_qwen_vision_recipe_export.py new file mode 100644 index 00000000000..78e873266a8 --- /dev/null +++ b/tests/gpu/torch/export/test_qwen_vision_recipe_export.py @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +import pytest +import torch +import transformers +from _test_utils.torch import transformers_models +from safetensors import safe_open + +import modelopt.torch.quantization as mtq +from modelopt.recipe import load_recipe +from modelopt.torch.export import export_hf_checkpoint + +_VISION_RECIPE = "fp8_vision-kv_none" +_JOINT_RECIPE = "fp8_vision_lm-kv_fp8_cast" + + +def _get_tiny_qwen_vlm(model_type): + if model_type == "qwen3_vl": + return transformers_models.get_tiny_qwen3vl() + if not all(hasattr(transformers, name) for name in ("Qwen3_5Config", "Qwen3_5MoeConfig")): + pytest.skip("Qwen3.5-VL requires a newer Transformers release") + return transformers_models.get_tiny_qwen3_5_vl_offline() + + +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9), + reason="FP8 vision encoder export requires compute capability 8.9 or newer", +) +@pytest.mark.parametrize( + ("model_type", "architecture"), + [ + ("qwen3_vl", "Qwen3VLForConditionalGeneration"), + ("qwen3_5", "Qwen3_5ForConditionalGeneration"), + ], +) +@pytest.mark.parametrize( + ("recipe", "quantizes_language", "quantizes_kv"), + [ + (_VISION_RECIPE, False, False), + (_JOINT_RECIPE, True, True), + ], +) +def test_qwen_vision_recipe_calibrates_and_exports( + tmp_path, model_type, architecture, recipe, quantizes_language, quantizes_kv +): + model = _get_tiny_qwen_vlm(model_type).to("cuda").eval() + model.config.architectures = [architecture] + quant_cfg = load_recipe(f"huggingface/{model_type}/ptq/{recipe}").quantize.model_dump() + vision_config = model.config.vision_config + pixel_width = ( + vision_config.in_channels * vision_config.temporal_patch_size * vision_config.patch_size**2 + ) + pixel_values = torch.randn(4, pixel_width, dtype=torch.bfloat16, device="cuda") + image_grid_thw = torch.tensor([[1, 2, 2]], device="cuda") + input_ids = torch.randint(5, 16, (1, 4), device="cuda") + + def calibration_forward(_model): + model.model.visual(pixel_values, image_grid_thw) + if quantizes_language: + model(input_ids=input_ids) + + mtq.quantize(model, quant_cfg, forward_loop=calibration_forward) + + export_path = tmp_path / f"{model_type}-{recipe}" + export_hf_checkpoint(model, export_dir=export_path) + + with (export_path / "hf_quant_config.json").open() as config_file: + quantization = json.load(config_file)["quantization"] + assert quantization["quant_algo"] == "FP8" + assert (quantization["kv_cache_quant_algo"] == "FP8") is quantizes_kv + assert ("model.language_model*" in quantization["exclude_modules"]) is not quantizes_language + + tensor_names = set() + for checkpoint_path in export_path.glob("*.safetensors"): + with safe_open(str(checkpoint_path), framework="pt") as checkpoint: + tensor_names.update(checkpoint.keys()) + scale_names = {name for name in tensor_names if name.endswith(("input_scale", "weight_scale"))} + assert scale_names + assert any(name.startswith("model.visual.") for name in scale_names) + assert any(".attn.qkv.weight_scale" in name for name in scale_names) + assert any(".merger.linear_fc1.weight_scale" in name for name in scale_names) + if model_type == "qwen3_vl": + assert any( + ".deepstack_merger_list.0.linear_fc1.weight_scale" in name for name in scale_names + ) + assert not any("patch_embed" in name for name in scale_names) + assert ( + any(name.startswith("model.language_model.") for name in scale_names) is quantizes_language + ) diff --git a/tests/unit/recipe/test_qwen_vision_recipe.py b/tests/unit/recipe/test_qwen_vision_recipe.py new file mode 100644 index 00000000000..74b65b5ee7e --- /dev/null +++ b/tests/unit/recipe/test_qwen_vision_recipe.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import transformers +from _test_utils.torch import transformers_models + +import modelopt.torch.quantization as mtq +from modelopt.recipe import load_recipe + +_VISION_RECIPE = "fp8_vision-kv_none" +_JOINT_RECIPE = "fp8_vision_lm-kv_fp8_cast" + + +def _get_tiny_qwen_vlm(model_type): + if model_type == "qwen3_vl": + return transformers_models.get_tiny_qwen3vl() + if not all(hasattr(transformers, name) for name in ("Qwen3_5Config", "Qwen3_5MoeConfig")): + pytest.skip("Qwen3.5-VL requires a newer Transformers release") + return transformers_models.get_tiny_qwen3_5_vl_offline() + + +@pytest.mark.parametrize("model_type", ["qwen3_vl", "qwen3_5"]) +@pytest.mark.parametrize( + ("recipe", "quantizes_language", "quantizes_kv"), + [ + (_VISION_RECIPE, False, False), + (_JOINT_RECIPE, True, True), + ], +) +def test_qwen_vision_recipes_select_expected_quantizers( + model_type, recipe, quantizes_language, quantizes_kv +): + model = _get_tiny_qwen_vlm(model_type) + assert model.config.model_type == model_type + quant_cfg = load_recipe(f"huggingface/{model_type}/ptq/{recipe}").quantize.model_dump() + + mtq.quantize(model, quant_cfg, forward_loop=None) + modules = dict(model.named_modules()) + enabled = {name for name, module in modules.items() if getattr(module, "is_enabled", False)} + + expected_visual_suffixes = { + "blocks.0.attn.qkv.weight_quantizer", + "blocks.0.attn.qkv.input_quantizer", + "blocks.0.attn.proj.weight_quantizer", + "blocks.0.attn.proj.input_quantizer", + "blocks.0.mlp.linear_fc1.weight_quantizer", + "blocks.0.mlp.linear_fc1.input_quantizer", + "blocks.0.mlp.linear_fc2.weight_quantizer", + "blocks.0.mlp.linear_fc2.input_quantizer", + "merger.linear_fc1.weight_quantizer", + "merger.linear_fc1.input_quantizer", + "merger.linear_fc2.weight_quantizer", + "merger.linear_fc2.input_quantizer", + } + if model_type == "qwen3_vl": + expected_visual_suffixes.update( + { + "deepstack_merger_list.0.linear_fc1.weight_quantizer", + "deepstack_merger_list.0.linear_fc1.input_quantizer", + "deepstack_merger_list.0.linear_fc2.weight_quantizer", + "deepstack_merger_list.0.linear_fc2.input_quantizer", + } + ) + assert {f"model.visual.{suffix}" for suffix in expected_visual_suffixes} <= enabled + assert not any("patch_embed" in name for name in enabled) + + vision_weight = modules["model.visual.blocks.0.attn.qkv.weight_quantizer"] + vision_input = modules["model.visual.blocks.0.attn.qkv.input_quantizer"] + assert vision_weight.num_bits == (4, 3) + assert vision_input.num_bits == (4, 3) + + language_quantizers = {name for name in enabled if name.startswith("model.language_model.")} + assert bool(language_quantizers) is quantizes_language + if not quantizes_language: + assert all(name.startswith("model.visual.") for name in enabled) + + enabled_bmm = {name for name in enabled if name.endswith("_bmm_quantizer")} + assert bool(enabled_bmm) is quantizes_kv + assert all(name.startswith("model.language_model.") for name in enabled_bmm)