From 90c984e00a1ffe0a913aa9001357a248a19c7027 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Fri, 7 Aug 2026 07:44:16 +0000 Subject: [PATCH 1/5] [NVBug: 6563509] Don't abort model load when the meta-device skeleton fails get_model() builds a throwaway skeleton under init_empty_weights(include_buffers=True) purely to size the model for infer_auto_device_map. include_buffers=True makes accelerate push a global torch.device("meta") context, so every tensor constructed in __init__ lands on meta -- not just parameters and buffers. Remote-code checkpoints written before Transformers v5 routinely derive scalar hyperparameters from real tensors there (Phi-4-multimodal's conformer subsampling does int(torch.tensor(...))), which raises "Tensor.item() cannot be called on meta tensors" and killed the whole run before from_pretrained was ever reached. Retry the skeleton without the global meta context, and if that also fails, warn and skip the memory estimate instead of aborting -- from_pretrained can still map the model on its own. Losing the estimate only costs the automatic max_memory shrink, which --use_seq_device_map and --gpu_max_memory_percentage already cover. Note this does not by itself make Phi-4-multimodal-instruct loadable: its remote code additionally needs transformers <4.52 (Phi4MMModel relies on PreTrainedModel inheriting GenerationMixin, which peft's get_peft_model calls into) and declares _tied_weights_keys as a list, which Transformers 5.x rejects. Both are outside ModelOpt. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- examples/hf_ptq/example_utils.py | 45 ++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index 740a09b2267..b3842c4181a 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -873,17 +873,40 @@ def has_pack_quantized_config(config): hf_config, auto_model_module, ckpt_path, config_kwargs ) - with init_empty_weights(include_buffers=True): - # When computing the device_map, assuming bfloat16 precision by default, - # unless specified by the hf_config. - config_dtype = _get_config_dtype(config_for_init) - model_kwargs2 = _apply_dtype_to_config( - model_kwargs, config_dtype, architecture, apply_config_dtype=True + # When computing the device_map, assuming bfloat16 precision by default, + # unless specified by the hf_config. + config_dtype = _get_config_dtype(config_for_init) + model_kwargs2 = _apply_dtype_to_config( + model_kwargs, config_dtype, architecture, apply_config_dtype=True + ) + if auto_model_module not in [AutoModelForCausalLM, AutoModel]: + model_kwargs2.pop("trust_remote_code", None) + model_kwargs2.pop("max_memory", None) + + # This skeleton is only a sizing aid for ``infer_auto_device_map``; it is thrown + # away right after. ``include_buffers=True`` makes accelerate push a global + # ``torch.device("meta")`` context, so *every* tensor built in ``__init__`` lands + # on meta -- not just parameters and buffers. Remote-code checkpoints written + # before Transformers v5 often compute scalar hyperparameters from real tensors + # there (Phi-4-multimodal's conformer subsampling does ``int(torch.tensor(...))``), + # which raises on meta. Retry without the global context, then give up on the + # estimate rather than failing a load that ``from_pretrained`` can still do. + model = None + skeleton_errors = [] + for include_buffers in (True, False): + try: + with init_empty_weights(include_buffers=include_buffers): + model = from_config(config_for_init, **model_kwargs2) + break + except Exception as e: + skeleton_errors.append(f"include_buffers={include_buffers}: {e!r}") + if model is None: + warnings.warn( + f"Could not build a meta-device skeleton of {architecture} " + f"({'; '.join(skeleton_errors)}). Skipping the device-map memory estimate " + "and letting from_pretrained map the model; if you hit GPU OOM, pass " + "--use_seq_device_map or lower --gpu_max_memory_percentage." ) - if auto_model_module not in [AutoModelForCausalLM, AutoModel]: - model_kwargs2.pop("trust_remote_code", None) - model_kwargs2.pop("max_memory", None) - model = from_config(config_for_init, **model_kwargs2) max_memory = get_max_memory() @@ -903,7 +926,7 @@ def has_pack_quantized_config(config): f"Offload folder: {offload_folder}\n" "Weights exceeding GPU+CPU budgets will be streamed from disk." ) - else: + elif model is not None: inferred_device_map = infer_auto_device_map(model, max_memory=max_memory) if "cpu" in inferred_device_map.values(): for _device in max_memory: From d94911be1feef3bfcc245f3f040090b7371c6688 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Fri, 7 Aug 2026 20:38:47 +0000 Subject: [PATCH 2/5] [NVBug: 6563509] Drop Phi-3-vision / Phi-4-multimodal PTQ support Both ship remote code that predates Transformers v5 and no longer loads on any version this repo supports (transformers>=4.57,<5.15): - Phi-4-multimodal needs transformers<4.52. Its __init__ calls peft.get_peft_model on Phi4MMModel, which reads prepare_inputs_for_generation -- present only while PreTrainedModel still inherited GenerationMixin. Verified loading at 4.48.2 / 4.49.0 / 4.50.0 / 4.51.3, failing at 4.53.3 / 4.56.2 / 4.57.1 with AttributeError. - Both declare _tied_weights_keys as a list; Transformers 5.x calls .keys() on it in post_init and raises AttributeError. - Phi-4-multimodal additionally computes int(torch.tensor(...)) in __init__, which Transformers 5.x's meta-device from_pretrained cannot evaluate. The model card pins transformers==4.48.2 / peft==0.13.2, so there is no overlap with our floor and nothing on our side can bridge it. Removes the support-matrix row, the phi4mm model type, the multimodal-detection heuristics that only ever matched these two (vision_lora, audio_processor, embd_layer.image_embd_layer), the Phi3Image/PhiImage embedding-export exclusions, and modelopt_recipes/huggingface/phi4mm/. Text-only Phi-3/Phi-4 and Phi-3.5-MoE are natively supported by transformers and are untouched. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 3 ++ examples/hf_ptq/README.md | 8 ++++- examples/hf_ptq/example_utils.py | 6 ---- examples/hf_ptq/hf_ptq.py | 3 -- modelopt/torch/export/layer_utils.py | 7 +--- modelopt/torch/export/model_utils.py | 15 -------- .../huggingface/phi4mm/ptq/README.md | 13 ------- .../phi4mm/ptq/disabled_quantizers.yaml | 34 ------------------ .../phi4mm/ptq/nvfp4-kv_fp8_cast.yaml | 36 ------------------- modelopt_recipes/ptq.md | 6 ++-- 10 files changed, 13 insertions(+), 118 deletions(-) delete mode 100644 modelopt_recipes/huggingface/phi4mm/ptq/README.md delete mode 100644 modelopt_recipes/huggingface/phi4mm/ptq/disabled_quantizers.yaml delete mode 100644 modelopt_recipes/huggingface/phi4mm/ptq/nvfp4-kv_fp8_cast.yaml diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 632b5532ebf..a7d522d213b 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,9 +19,12 @@ Changelog **Deprecations** +- Drop PTQ support for **Phi-3-vision** and **Phi-4-multimodal**. Their bundled remote code predates Transformers v5 and no longer loads on the versions this repo requires (``transformers>=4.57``): Phi-4-multimodal needs ``transformers<4.52`` because it reaches ``prepare_inputs_for_generation`` through ``peft``, which requires ``PreTrainedModel`` to still inherit ``GenerationMixin``, and both models declare ``_tied_weights_keys`` as a list, which Transformers 5.x rejects. Removes the ``phi4mm`` model type, its multimodal-detection heuristics (``vision_lora`` / ``audio_processor`` / ``embd_layer.image_embd_layer``), the ``Phi3Image`` / ``PhiImage`` embedding-export exclusions, and the ``modelopt_recipes/huggingface/phi4mm/`` recipes. + **Bug Fixes** - Fix EAGLE-3 training with context parallelism (``--cp_size > 1`` in ``examples/speculative_decoding``), which failed to start on ``accelerate >= 1.13`` and then raised ``got mixed torch.Tensor and DTensor``. +- ``examples/hf_ptq/hf_ptq.py`` no longer aborts the whole load when the throwaway meta-device skeleton it builds to size ``infer_auto_device_map`` cannot be constructed. ``init_empty_weights(include_buffers=True)`` pushes a global ``torch.device("meta")`` context, so remote-code checkpoints that derive scalar hyperparameters from real tensors in ``__init__`` raised ``Tensor.item() cannot be called on meta tensors``. The skeleton is now retried without the global meta context, and if that also fails the memory estimate is skipped with a warning instead of failing the run. 0.46 (2026-08-17) ^^^^^^^^^^^^^^^^^ diff --git a/examples/hf_ptq/README.md b/examples/hf_ptq/README.md index e80926bea91..903d708de9b 100755 --- a/examples/hf_ptq/README.md +++ b/examples/hf_ptq/README.md @@ -119,13 +119,19 @@ Please reference our [framework scripts](#framework-scripts) and our [docs](http | Whisper9 | ✅ | ❌ | ❌ | ❌ | - | | Nemotron-3 | ✅ | ❌ | ❌ | ❌ | ✅ | | Llava (VLM)11 | ✅ | ✅12 | ✅ | ✅ | - | -| Phi-3-vision, Phi-4-multimodal (VLM)11 | ✅ | ✅12 | ✅ | ✅ | ✅ | | Qwen2, 2.5-VL (VLM)11 | ✅ | ✅12 | ✅ | ✅ | ✅ | | Gemma 3 (VLM)11 | ✅ | - | - | - | - | | Nemotron VL (VLM)11,13 | ✅ | - | - | - | ✅ | > *This is a subset of the models supported. For the full list please check the [TensorRT-LLM support matrix](https://nvidia.github.io/TensorRT-LLM/reference/precision.html#support-matrix)* +> *Phi-3-vision and Phi-4-multimodal were dropped from this matrix: their bundled +> remote code predates Transformers v5 and no longer loads on the versions this repo +> requires (`transformers>=4.57`). Phi-4-multimodal needs `transformers<4.52` — it +> reaches `prepare_inputs_for_generation` through `peft`, which requires +> `PreTrainedModel` to still inherit `GenerationMixin` — and both models declare +> `_tied_weights_keys` as a list, which Transformers 5.x rejects.* + > *1.The w4a8_awq is an experimental quantization scheme that may result in a higher accuracy penalty.* \ > *2.For some models, there is only support for exporting quantized checkpoints.* \ > *3.W4A8_AWQ is only available on some models but not all* \ diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index b3842c4181a..6f9fe51063e 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -177,12 +177,6 @@ def _is_multimodal_config(config): """Check if a config indicates a multimodal model (config-only version of is_multimodal_model).""" return ( hasattr(config, "vision_config") # Standard vision config (e.g., Qwen2.5-VL) - or getattr(config, "model_type", "") == "phi4mm" # Phi-4 multimodal - or hasattr(config, "vision_lora") # Vision LoRA configurations - or hasattr(config, "audio_processor") # Audio processing capabilities - or ( - hasattr(config, "embd_layer") and hasattr(config.embd_layer, "image_embd_layer") - ) # Image embedding layers or getattr(config, "is_encoder_decoder", False) # Encoder-decoder VL models or any( # Architecture-based detection for custom VL models (e.g., Nemotron-Parse) "conditionalgeneration" in arch.lower() for arch in getattr(config, "architectures", []) diff --git a/examples/hf_ptq/hf_ptq.py b/examples/hf_ptq/hf_ptq.py index 7a2328d10f7..0790f644308 100755 --- a/examples/hf_ptq/hf_ptq.py +++ b/examples/hf_ptq/hf_ptq.py @@ -705,9 +705,6 @@ def load_model(args: argparse.Namespace): # Left padding usually provides better calibration result. tokenizer.padding_side = "left" - if model_type == "phi4mm": - warnings.warn("Please set the default input_mode to InputMode.LANGUAGE before quantizing.") - return ( full_model, language_model, diff --git a/modelopt/torch/export/layer_utils.py b/modelopt/torch/export/layer_utils.py index d5f1fb2330d..de136fcd378 100755 --- a/modelopt/torch/export/layer_utils.py +++ b/modelopt/torch/export/layer_utils.py @@ -222,12 +222,7 @@ def is_conv(module: nn.Module) -> bool: def is_embedding(module: nn.Module) -> bool: """Returns whether the module is an embedding layer.""" module_type_name = type(module).__name__ - return ( - "Embedding" in module_type_name - and "Rotary" not in module_type_name - and "PhiImage" not in module_type_name - and "Phi3Image" not in module_type_name - ) + return "Embedding" in module_type_name and "Rotary" not in module_type_name def build_embedding_config(module: nn.Module, normalization_constant: float = 1) -> EmbeddingConfig: diff --git a/modelopt/torch/export/model_utils.py b/modelopt/torch/export/model_utils.py index 307ea9aac51..1729dbfffcf 100755 --- a/modelopt/torch/export/model_utils.py +++ b/modelopt/torch/export/model_utils.py @@ -44,7 +44,6 @@ "phi3small": "phi3small", "phi3": "phi3", "PhiMoEForCausalLM": "phi3", - "Phi4MMForCausalLM": "phi4mm", "phi": "phi", "TLGv4ForCausalLM": "phi", "MixtralForCausalLM": "llama", @@ -88,10 +87,6 @@ def is_multimodal_model(model): This function detects various multimodal model architectures by checking for: - Standard vision configurations (vision_config) - Language model attributes (language_model) - - Specific multimodal model types (phi4mm) - - Vision LoRA configurations - - Audio processing capabilities - - Image embedding layers - Nemotron-Parse conditional generation models Args: @@ -104,10 +99,6 @@ def is_multimodal_model(model): >>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-VL-7B-Instruct") >>> is_multimodal_model(model) True - - >>> model = AutoModelForCausalLM.from_pretrained("microsoft/Phi-4-multimodal-instruct") - >>> is_multimodal_model(model) - True """ config = model.config @@ -118,12 +109,6 @@ def is_multimodal_model(model): return ( hasattr(config, "vision_config") # Standard vision config (e.g., Qwen2.5-VL) or hasattr(model, "language_model") # Language model attribute (e.g., LLaVA) - or getattr(config, "model_type", "") == "phi4mm" # Phi-4 multimodal - or hasattr(config, "vision_lora") # Vision LoRA configurations - or hasattr(config, "audio_processor") # Audio processing capabilities - or ( - hasattr(config, "embd_layer") and hasattr(config.embd_layer, "image_embd_layer") - ) # Image embedding layers or is_nemotron_parse # Nemotron-Parse conditional generation model ) diff --git a/modelopt_recipes/huggingface/phi4mm/ptq/README.md b/modelopt_recipes/huggingface/phi4mm/ptq/README.md deleted file mode 100644 index bedaf1fcb6b..00000000000 --- a/modelopt_recipes/huggingface/phi4mm/ptq/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Phi-4-Multimodal PTQ recipes - -Phi-4-Multimodal is a multimodal model. Quantization should be applied only to -the language model; the speech, audio, image, and vision branches are kept in -full precision to avoid accuracy regressions on those modalities. - -| File | What's model-specific | -|------|-----------------------| -| `disabled_quantizers.yaml` | Reusable unit (`QuantizerCfgListConfig`). Merges the standard `default_disabled_quantizers` exclusions with Phi-4-MM ones (`*speech*`, `*audio*`, `*image*`, `*vision*`). Imported by recipes below as the single `disabled_quantizers` slot so they don't pull in two disabled-quantizer sets. | -| `nvfp4-kv_fp8_cast.yaml` | NVFP4 W4A4 model quantization + FP8 KV-cache cast (constant amax, no KV calibration). Identical numerics to the general `nvfp4` preset / `kv_fp8_cast` unit; what makes it model-specific is that it imports `disabled_quantizers.yaml` from this folder to skip the non-language branches. | - -Additional `-kv_fp8_cast.yaml` recipes can be generated for other formats -if needed; only `nvfp4-kv_fp8_cast.yaml` is shipped by default. diff --git a/modelopt_recipes/huggingface/phi4mm/ptq/disabled_quantizers.yaml b/modelopt_recipes/huggingface/phi4mm/ptq/disabled_quantizers.yaml deleted file mode 100644 index 1c6089f087f..00000000000 --- a/modelopt_recipes/huggingface/phi4mm/ptq/disabled_quantizers.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# QuantizerCfgList snippet of disabled quantizers for Phi-4-Multimodal. -# Splices in the standard `default_disabled_quantizers` exclusions and appends -# Phi-4-MM-specific ones so that only the language model is quantized; -# speech/audio/image/vision branches are skipped. Recipes that import this -# should NOT also import `default_disabled_quantizers`. - -# modelopt-schema: modelopt.torch.quantization.config.QuantizerCfgListConfig -imports: - default_disabled_quantizers: configs/ptq/units/default_disabled_quantizers ---- - - $import: default_disabled_quantizers - - quantizer_name: '*speech*' - enable: false - - quantizer_name: '*audio*' - enable: false - - quantizer_name: '*image*' - enable: false - - quantizer_name: '*vision*' - enable: false diff --git a/modelopt_recipes/huggingface/phi4mm/ptq/nvfp4-kv_fp8_cast.yaml b/modelopt_recipes/huggingface/phi4mm/ptq/nvfp4-kv_fp8_cast.yaml deleted file mode 100644 index dfb1be1778d..00000000000 --- a/modelopt_recipes/huggingface/phi4mm/ptq/nvfp4-kv_fp8_cast.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Phi-4-Multimodal-specific PTQ recipe for the `nvfp4` quantization format. -# Equivalent to the general `nvfp4` preset with quantization disabled -# on non-language branches. - -imports: - base_disable_all: configs/ptq/units/base_disable_all - w4a4_nvfp4_nvfp4: configs/ptq/units/w4a4_nvfp4_nvfp4 - disabled_quantizers: huggingface/phi4mm/ptq/disabled_quantizers - kv_fp8_cast: configs/ptq/units/kv_fp8_cast - -metadata: - recipe_type: ptq - description: 'Phi-4-Multimodal PTQ recipe (nvfp4): same numerics as the general nvfp4 preset, applied to the language model only (speech, audio, image, - and vision branches are skipped).' -quantize: - algorithm: max - quant_cfg: - - $import: base_disable_all - - $import: w4a4_nvfp4_nvfp4 - - $import: kv_fp8_cast - - $import: disabled_quantizers diff --git a/modelopt_recipes/ptq.md b/modelopt_recipes/ptq.md index 54ff6511489..3d56c761adf 100644 --- a/modelopt_recipes/ptq.md +++ b/modelopt_recipes/ptq.md @@ -234,7 +234,7 @@ that baseline. The deviations come in four kinds: |------|-------------------------------------|----------| | **Architecture-aware `quant_cfg`** | Per-sub-module format choices a single wildcard scheme can't express | `minimax_m3_vl`, `qwen3_5`, `qwen3_5_moe`, `vit`, `nemotron_llama` | | **Algorithm override** | Same numerics & scope, but the *calibration algorithm* is tweaked because the default breaks or regresses | `gemma`, `gemma4`, `mpt` | -| **Extra exclusions** | Adds disabled-quantizer patterns so non-language branches stay full precision | `nemotron_vl`, `phi4mm`, `diffusion_gemma` | +| **Extra exclusions** | Adds disabled-quantizer patterns so non-language branches stay full precision | `nemotron_vl`, `diffusion_gemma` | | **Checkpoint mirror** | A mixed-precision map reproducing one published checkpoint exactly | `models/nvidia/Nemotron-3-*`, `models/nvidia/Mistral-Medium-3.5-128B-NVFP4` | The numerics and standard exclusions are still inherited from `configs/` @@ -314,7 +314,7 @@ These quantize the **same layers** as the general recipes; only the *Why special:* identical scope/numerics to a general scheme, but a general recipe's default algorithm would overflow or regress here. -### Extra exclusions — `nemotron_vl`, `phi4mm`, `diffusion_gemma` +### Extra exclusions — `nemotron_vl`, `diffusion_gemma` Each of these is **numerically identical** to a general recipe. What makes them special is a model-local `disabled_quantizers.yaml` unit that *extends* the @@ -324,8 +324,6 @@ standard exclusions so a model-specific branch stays in full precision: `nvfp4_default-kv_fp8_cast` numerics, adding `*vision*`, `*image*`, `*radio*`, `*visual*`, `*encoder*`, `*model_encoder*` so only the language decoder is quantized. -- **`phi4mm`** (Phi-4-Multimodal) — general `nvfp4_default-kv_fp8_cast` - numerics, adding `*speech*`, `*audio*`, `*image*`, `*vision*`. - **`diffusion_gemma`** (block-diffusion encoder-decoder text LLM on a Gemma4 MoE backbone) — general `nvfp4_experts_only-kv_fp8_cast` numerics, adding `*self_conditioning*`: the self-conditioning network is text-only and never From 876dd8c6f75a99f0e8aa1b02c5ecae6022df898c Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Fri, 7 Aug 2026 22:09:32 +0000 Subject: [PATCH 3/5] Move NVBug 6563509 changelog entries into 0.46 0.46 is the release these land in, not the still-open 0.47 section. The Phi drop goes under Backward Breaking Changes next to the VILA / NVILA entry, which removed model support for the same reason (bundled remote code pinned below our transformers floor); the skeleton fallback goes under Bug Fixes. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a7d522d213b..69e501580ac 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,12 +19,9 @@ Changelog **Deprecations** -- Drop PTQ support for **Phi-3-vision** and **Phi-4-multimodal**. Their bundled remote code predates Transformers v5 and no longer loads on the versions this repo requires (``transformers>=4.57``): Phi-4-multimodal needs ``transformers<4.52`` because it reaches ``prepare_inputs_for_generation`` through ``peft``, which requires ``PreTrainedModel`` to still inherit ``GenerationMixin``, and both models declare ``_tied_weights_keys`` as a list, which Transformers 5.x rejects. Removes the ``phi4mm`` model type, its multimodal-detection heuristics (``vision_lora`` / ``audio_processor`` / ``embd_layer.image_embd_layer``), the ``Phi3Image`` / ``PhiImage`` embedding-export exclusions, and the ``modelopt_recipes/huggingface/phi4mm/`` recipes. - **Bug Fixes** - Fix EAGLE-3 training with context parallelism (``--cp_size > 1`` in ``examples/speculative_decoding``), which failed to start on ``accelerate >= 1.13`` and then raised ``got mixed torch.Tensor and DTensor``. -- ``examples/hf_ptq/hf_ptq.py`` no longer aborts the whole load when the throwaway meta-device skeleton it builds to size ``infer_auto_device_map`` cannot be constructed. ``init_empty_weights(include_buffers=True)`` pushes a global ``torch.device("meta")`` context, so remote-code checkpoints that derive scalar hyperparameters from real tensors in ``__init__`` raised ``Tensor.item() cannot be called on meta tensors``. The skeleton is now retried without the global meta context, and if that also fails the memory estimate is skipped with a warning instead of failing the run. 0.46 (2026-08-17) ^^^^^^^^^^^^^^^^^ @@ -86,6 +83,7 @@ Changelog - Remove the deprecated ``examples/llm_autodeploy`` example (deprecated in 0.45). Use TensorRT-LLM's `AutoDeploy `_ directly together with ModelOpt PTQ in ``examples/hf_ptq``. - Remove the deprecated ``examples/llm_qad`` Megatron-LM QAD example (deprecated in 0.45). Use the `megatron_bridge QAD example `_ instead, which provides a simpler Python-based interface and better model coverage. - Dropped VILA / NVILA vision-language model support in ``examples/hf_ptq``. VILA's modeling code requires ``transformers<=4.50.0``, which conflicts with ModelOpt's minimum supported ``transformers`` version. The VILA-specific bootstrap (repo clone, ``requirements-vila.txt``) and loading paths in ``example_utils.py`` have been removed. +- Dropped **Phi-3-vision** and **Phi-4-multimodal** PTQ support in ``examples/hf_ptq`` (NVBug 6563509). Their bundled remote code predates Transformers v5 and no longer loads on any version ModelOpt supports (``transformers>=4.57``): Phi-4-multimodal requires ``transformers<4.52`` because it reaches ``prepare_inputs_for_generation`` through ``peft``, which needs ``PreTrainedModel`` to still inherit ``GenerationMixin``, and both models declare ``_tied_weights_keys`` as a list, which Transformers 5.x rejects. The support-matrix row, the ``phi4mm`` model type, the multimodal-detection heuristics that only ever matched these two (``vision_lora`` / ``audio_processor`` / ``embd_layer.image_embd_layer``), the ``Phi3Image`` / ``PhiImage`` embedding-export exclusions, and the ``modelopt_recipes/huggingface/phi4mm/`` recipes have been removed. Text-only Phi-3/Phi-4 and Phi-3.5-MoE are unaffected. **Deprecations** @@ -108,6 +106,7 @@ Changelog - Fix ``examples/vllm_serve`` serving shared experts uncalibrated: their ``gate_proj``/``up_proj`` quantizer keys were not merged into ``gate_up_proj`` on reload, so they matched no module and were dropped. - Fix Qwen3-VL MoE PTQ failing on ``transformers>=5.12`` with ``AttributeError: 'QuantQwen3VLMoeTextExperts' object has no attribute 'hidden_size'`` (NVBug 6518551). transformers 5.12 moved ``Qwen3VLMoeTextExperts`` onto the standard ``@use_experts_implementation`` fused layout (``hidden_size``/``expert_dim`` renamed to ``hidden_dim``/``intermediate_dim``, ``gate_up_proj`` transposed to ``(num_experts, 2*intermediate_dim, hidden_dim)``, two ``F.linear`` calls per expert), but the legacy ``_QuantQwen3VLMoeTextExperts`` wrapper stayed statically registered and shadowed on-the-fly detection. The new layout is now left to ``register_fused_experts_on_the_fly``, which claims it with the generic ``_QuantFusedExperts``; the legacy wrapper is still registered on ``transformers<5.12``, whose ``torch.bmm``-based forward the generic wrapper cannot intercept. - Fix ``examples/hf_ptq`` multi-node FSDP2 export (``--use_fsdp2``) failing with ``RuntimeError: Cannot set version_counter for inference tensor``. ``export_quantized`` wrapped its whole body in ``torch.inference_mode()``, so the full params gathered by ``get_model_state_dict(full_state_dict=True)`` were inference tensors and the subsequent ``state_dict()`` -> ``param.detach()`` could not set their version counter. The export context is now ``torch.no_grad()``, which still disables autograd but keeps the gathered params as normal tensors. +- Fix ``examples/hf_ptq`` aborting the whole model load when the throwaway meta-device skeleton it builds to size ``infer_auto_device_map`` cannot be constructed (NVBug 6563509). ``init_empty_weights(include_buffers=True)`` pushes a global ``torch.device("meta")`` context, so remote-code checkpoints that derive scalar hyperparameters from real tensors in ``__init__`` failed with ``Tensor.item() cannot be called on meta tensors`` before ``from_pretrained`` was ever reached. The skeleton is now retried without the global meta context, and if that also fails the memory estimate is skipped with a warning instead of failing the run; ``--use_seq_device_map`` and ``--gpu_max_memory_percentage`` cover the lost heuristic. - Fix unified HF export of multimodal models whose vision tower carries its own ``PrefixChange`` conversion (``LlavaForConditionalGeneration`` on ``transformers>=5.12`` — NVBug 6525511). transformers collects conversion mappings recursively and scopes each sub-model's transforms to that sub-module via ``scope_prefix``, matching only keys under that prefix. ModelOpt's quant-aware reverse conversion read the raw patterns and ignored ``scope_prefix``, so the vision tower's "add a ``vision_model.`` prefix" rule was applied to *every* key in the state dict: an exported llava-1.5-13b checkpoint had all 758 tensors moved under a bogus top-level ``vision_model.`` namespace (``vision_model.language_model.*``, ``vision_model.lm_head.*``), and vLLM rejected it with ``ValueError: There is no module or parameter named 'vision_model' in LlavaForConditionalGeneration``. Reverse rename rules now carry their scope and are applied only to keys under it, matching transformers' own ``WeightTransform._scoped_match`` semantics. ``Gemma3ForConditionalGeneration`` was affected identically and is fixed by the same change. 0.45 (2026-07-02) From 8444c11f6bba343e25830658571aefe53fc7672e Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Sat, 8 Aug 2026 05:55:59 +0000 Subject: [PATCH 4/5] Address PR review: extract skeleton helper, fix OOM hint, drop README note - Extract the two-attempt skeleton build into `_build_meta_skeleton()` next to the other `get_model` helpers (@cjluo-nv). `get_model` now reads as one line, and the fallback logic is unit-testable in isolation. - Fix the OOM hint in the fallback warning. It named `--gpu_max_memory_percentage`, which does not exist -- the flag is `--gpu_max_mem_percentage` -- and on its own that flag has no effect when the skeleton fails: `model_kwargs["max_memory"]` is only set by the `_disk_offload` and `use_seq_device_map` paths, so the skipped `infer_auto_device_map` branch never applies it. Point at `--use_seq_device_map` (which does apply the percentage) and `--batch_size` instead (@coderabbitai). Not adopting the suggested unconditional cap: the original code only shrinks `max_memory` when `infer_auto_device_map` reports a CPU spill, so capping whenever the skeleton fails would force CPU offload onto models that currently fit entirely on GPU. - Drop the support-matrix explanation note; the CHANGELOG covers it (@cjluo-nv). Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- examples/hf_ptq/README.md | 7 ---- examples/hf_ptq/example_utils.py | 57 ++++++++++++++++++-------------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/examples/hf_ptq/README.md b/examples/hf_ptq/README.md index 903d708de9b..dc852408c7a 100755 --- a/examples/hf_ptq/README.md +++ b/examples/hf_ptq/README.md @@ -125,13 +125,6 @@ Please reference our [framework scripts](#framework-scripts) and our [docs](http > *This is a subset of the models supported. For the full list please check the [TensorRT-LLM support matrix](https://nvidia.github.io/TensorRT-LLM/reference/precision.html#support-matrix)* -> *Phi-3-vision and Phi-4-multimodal were dropped from this matrix: their bundled -> remote code predates Transformers v5 and no longer loads on the versions this repo -> requires (`transformers>=4.57`). Phi-4-multimodal needs `transformers<4.52` — it -> reaches `prepare_inputs_for_generation` through `peft`, which requires -> `PreTrainedModel` to still inherit `GenerationMixin` — and both models declare -> `_tied_weights_keys` as a list, which Transformers 5.x rejects.* - > *1.The w4a8_awq is an experimental quantization scheme that may result in a higher accuracy penalty.* \ > *2.For some models, there is only support for exporting quantized checkpoints.* \ > *3.W4A8_AWQ is only available on some models but not all* \ diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index 6f9fe51063e..cb02fbb3408 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -657,6 +657,36 @@ def _resolve_init_config(hf_config, auto_model_module, ckpt_path, config_kwargs) return hf_config +def _build_meta_skeleton(from_config, config_for_init, model_kwargs, architecture): + """Build a throwaway meta-device model used only to size ``infer_auto_device_map``. + + ``include_buffers=True`` makes accelerate push a global ``torch.device("meta")`` + context, so *every* tensor built in ``__init__`` lands on meta -- not just parameters + and buffers. Remote-code checkpoints written before Transformers v5 routinely derive + scalar hyperparameters from real tensors there, which raises on meta, so fall back to + the parameter-only patching of ``include_buffers=False``. + + Returns ``None`` (after warning) when neither attempt works; the caller then skips the + memory estimate rather than failing a load ``from_pretrained`` can still do. + """ + skeleton_errors = [] + for include_buffers in (True, False): + try: + with init_empty_weights(include_buffers=include_buffers): + return from_config(config_for_init, **model_kwargs) + except Exception as e: # noqa: PERF203 -- at most two attempts, error path only + skeleton_errors.append(f"include_buffers={include_buffers}: {e!r}") + + warnings.warn( + f"Could not build a meta-device skeleton of {architecture} " + f"({'; '.join(skeleton_errors)}). Skipping the device-map memory estimate and " + "letting from_pretrained map the model. If you hit GPU OOM, rerun with " + "--use_seq_device_map (which applies --gpu_max_mem_percentage) or lower " + "--batch_size." + ) + return None + + def _get_config_dtype(config): config_dtype = ( getattr(config, "dtype", None) or getattr(config, "torch_dtype", None) or torch.bfloat16 @@ -877,30 +907,9 @@ def has_pack_quantized_config(config): model_kwargs2.pop("trust_remote_code", None) model_kwargs2.pop("max_memory", None) - # This skeleton is only a sizing aid for ``infer_auto_device_map``; it is thrown - # away right after. ``include_buffers=True`` makes accelerate push a global - # ``torch.device("meta")`` context, so *every* tensor built in ``__init__`` lands - # on meta -- not just parameters and buffers. Remote-code checkpoints written - # before Transformers v5 often compute scalar hyperparameters from real tensors - # there (Phi-4-multimodal's conformer subsampling does ``int(torch.tensor(...))``), - # which raises on meta. Retry without the global context, then give up on the - # estimate rather than failing a load that ``from_pretrained`` can still do. - model = None - skeleton_errors = [] - for include_buffers in (True, False): - try: - with init_empty_weights(include_buffers=include_buffers): - model = from_config(config_for_init, **model_kwargs2) - break - except Exception as e: - skeleton_errors.append(f"include_buffers={include_buffers}: {e!r}") - if model is None: - warnings.warn( - f"Could not build a meta-device skeleton of {architecture} " - f"({'; '.join(skeleton_errors)}). Skipping the device-map memory estimate " - "and letting from_pretrained map the model; if you hit GPU OOM, pass " - "--use_seq_device_map or lower --gpu_max_memory_percentage." - ) + # Only a sizing aid for ``infer_auto_device_map`` below; ``None`` when the model + # cannot be built on meta, in which case the estimate is skipped. + model = _build_meta_skeleton(from_config, config_for_init, model_kwargs2, architecture) max_memory = get_max_memory() From 1b062da365e3029dae96a876683f9177956487b3 Mon Sep 17 00:00:00 2001 From: Chenjie Luo Date: Sat, 8 Aug 2026 22:15:10 +0000 Subject: [PATCH 5/5] Simplify sizing skeleton to a single include_buffers=False build The probe exists only to answer one boolean -- will this model spill to CPU, so should max_memory get the gpu_max_mem_percentage haircut. `infer_auto_device_map` gets there via `compute_module_sizes`, which reads `tensor.numel()` and `tensor.dtype` and never touches storage. So the probe only has to reproduce the module tree, and it must be no stricter than the loader it predicts. `include_buffers=True` was stricter than both loaders: - Transformers 4.x `from_pretrained` builds under `init_empty_weights()`, whose `include_buffers` defaults to False (modeling_utils.py:4378). - Transformers 5.x builds under `torch.device("meta")` plus `meta_device_safe_creation_ops()`, which redirects `torch.linspace` to CPU precisely so pre-v5 remote code that derives scalars in `__init__` keeps working. accelerate implements `include_buffers=True` as a bare global `torch.device("meta")` context, which also captures scratch arithmetic in `__init__` that has nothing to do with weights. A model doing `int(torch.tensor(...))` there loads fine through `from_pretrained` on both 4.x and 5.x but died in our probe -- an optimization killing runs it was only meant to speed up. Dropping to a single `include_buffers=False` build removes the retry loop, the PERF203 waiver, and the two-attempt error accumulation. Measured cost on real checkpoints is nil: Qwen3-8B and DeepSeek-R1-Distill-Llama-70B both report 0.0 MiB retained and identical sized totals (15.256 / 131.417 GiB) either way, for +0.09s and +0.19s respectively. Adds the coverage requested in review: the probe uses permissive patching, it survives a meta-hostile `__init__`, it returns None and warns on failure, and `get_model` skips `infer_auto_device_map` while still calling `from_pretrained` without inventing a max_memory cap. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Chenjie Luo --- CHANGELOG.rst | 2 +- examples/hf_ptq/example_utils.py | 53 ++++++------ tests/examples/hf_ptq/test_example_utils.py | 90 +++++++++++++++++++++ 3 files changed, 120 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 69e501580ac..c3c3976e51a 100755 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -106,7 +106,7 @@ Changelog - Fix ``examples/vllm_serve`` serving shared experts uncalibrated: their ``gate_proj``/``up_proj`` quantizer keys were not merged into ``gate_up_proj`` on reload, so they matched no module and were dropped. - Fix Qwen3-VL MoE PTQ failing on ``transformers>=5.12`` with ``AttributeError: 'QuantQwen3VLMoeTextExperts' object has no attribute 'hidden_size'`` (NVBug 6518551). transformers 5.12 moved ``Qwen3VLMoeTextExperts`` onto the standard ``@use_experts_implementation`` fused layout (``hidden_size``/``expert_dim`` renamed to ``hidden_dim``/``intermediate_dim``, ``gate_up_proj`` transposed to ``(num_experts, 2*intermediate_dim, hidden_dim)``, two ``F.linear`` calls per expert), but the legacy ``_QuantQwen3VLMoeTextExperts`` wrapper stayed statically registered and shadowed on-the-fly detection. The new layout is now left to ``register_fused_experts_on_the_fly``, which claims it with the generic ``_QuantFusedExperts``; the legacy wrapper is still registered on ``transformers<5.12``, whose ``torch.bmm``-based forward the generic wrapper cannot intercept. - Fix ``examples/hf_ptq`` multi-node FSDP2 export (``--use_fsdp2``) failing with ``RuntimeError: Cannot set version_counter for inference tensor``. ``export_quantized`` wrapped its whole body in ``torch.inference_mode()``, so the full params gathered by ``get_model_state_dict(full_state_dict=True)`` were inference tensors and the subsequent ``state_dict()`` -> ``param.detach()`` could not set their version counter. The export context is now ``torch.no_grad()``, which still disables autograd but keeps the gathered params as normal tensors. -- Fix ``examples/hf_ptq`` aborting the whole model load when the throwaway meta-device skeleton it builds to size ``infer_auto_device_map`` cannot be constructed (NVBug 6563509). ``init_empty_weights(include_buffers=True)`` pushes a global ``torch.device("meta")`` context, so remote-code checkpoints that derive scalar hyperparameters from real tensors in ``__init__`` failed with ``Tensor.item() cannot be called on meta tensors`` before ``from_pretrained`` was ever reached. The skeleton is now retried without the global meta context, and if that also fails the memory estimate is skipped with a warning instead of failing the run; ``--use_seq_device_map`` and ``--gpu_max_memory_percentage`` cover the lost heuristic. +- Fix ``examples/hf_ptq`` aborting the whole model load when the throwaway meta-device skeleton it builds to size ``infer_auto_device_map`` cannot be constructed (NVBug 6563509). The skeleton used ``init_empty_weights(include_buffers=True)``, which accelerate implements as a bare global ``torch.device("meta")`` context: *every* tensor built in ``__init__`` lands on meta, so remote-code checkpoints that derive scalar hyperparameters from real tensors there failed with ``Tensor.item() cannot be called on meta tensors`` before ``from_pretrained`` was ever reached. That made the probe stricter than the loader it predicts -- Transformers 4.x builds under ``include_buffers=False`` and Transformers 5.x under a meta context patched by ``meta_device_safe_creation_ops()`` -- so it could kill loads that would otherwise have succeeded. The skeleton now uses ``include_buffers=False``, which patches only ``nn.Module.register_parameter`` and leaves ``__init__`` arithmetic on a real device (module sizes are unchanged: ``compute_module_sizes`` reads shape and dtype, not storage). If it still cannot be built, the memory estimate is skipped with a warning instead of failing the run; ``--use_seq_device_map`` (which applies ``--gpu_max_mem_percentage``) and ``--batch_size`` cover the lost heuristic. - Fix unified HF export of multimodal models whose vision tower carries its own ``PrefixChange`` conversion (``LlavaForConditionalGeneration`` on ``transformers>=5.12`` — NVBug 6525511). transformers collects conversion mappings recursively and scopes each sub-model's transforms to that sub-module via ``scope_prefix``, matching only keys under that prefix. ModelOpt's quant-aware reverse conversion read the raw patterns and ignored ``scope_prefix``, so the vision tower's "add a ``vision_model.`` prefix" rule was applied to *every* key in the state dict: an exported llava-1.5-13b checkpoint had all 758 tensors moved under a bogus top-level ``vision_model.`` namespace (``vision_model.language_model.*``, ``vision_model.lm_head.*``), and vLLM rejected it with ``ValueError: There is no module or parameter named 'vision_model' in LlavaForConditionalGeneration``. Reverse rename rules now carry their scope and are applied only to keys under it, matching transformers' own ``WeightTransform._scoped_match`` semantics. ``Gemma3ForConditionalGeneration`` was affected identically and is fixed by the same change. 0.45 (2026-07-02) diff --git a/examples/hf_ptq/example_utils.py b/examples/hf_ptq/example_utils.py index cb02fbb3408..d14784f8dcd 100755 --- a/examples/hf_ptq/example_utils.py +++ b/examples/hf_ptq/example_utils.py @@ -660,31 +660,36 @@ def _resolve_init_config(hf_config, auto_model_module, ckpt_path, config_kwargs) def _build_meta_skeleton(from_config, config_for_init, model_kwargs, architecture): """Build a throwaway meta-device model used only to size ``infer_auto_device_map``. - ``include_buffers=True`` makes accelerate push a global ``torch.device("meta")`` - context, so *every* tensor built in ``__init__`` lands on meta -- not just parameters - and buffers. Remote-code checkpoints written before Transformers v5 routinely derive - scalar hyperparameters from real tensors there, which raises on meta, so fall back to - the parameter-only patching of ``include_buffers=False``. - - Returns ``None`` (after warning) when neither attempt works; the caller then skips the - memory estimate rather than failing a load ``from_pretrained`` can still do. + ``compute_module_sizes`` needs shapes and dtypes, never values or storage, so this + probe only has to reproduce the module tree. It must also be *no stricter than the + loader it predicts*, or it kills runs ``from_pretrained`` would have completed: + + - Transformers 4.x builds under ``init_empty_weights()``, i.e. ``include_buffers=False``. + - Transformers 5.x builds under ``torch.device("meta")`` plus + ``meta_device_safe_creation_ops()``, which redirects ``torch.linspace`` to CPU so + remote code that derives scalars from it in ``__init__`` keeps working. + + ``include_buffers=True`` is stricter than both: accelerate implements it as a bare + global ``torch.device("meta")`` context, so *every* tensor built in ``__init__`` lands + on meta and any ``int(torch.tensor(...))`` raises. ``include_buffers=False`` patches + only ``nn.Module.register_parameter``, leaving ``__init__`` arithmetic on a real + device; buffers stay materialized, which costs nothing here because their shape and + dtype size the same either way. + + Returns ``None`` (after warning) if the model cannot be built at all; the caller then + skips the estimate rather than failing a load ``from_pretrained`` can still do. """ - skeleton_errors = [] - for include_buffers in (True, False): - try: - with init_empty_weights(include_buffers=include_buffers): - return from_config(config_for_init, **model_kwargs) - except Exception as e: # noqa: PERF203 -- at most two attempts, error path only - skeleton_errors.append(f"include_buffers={include_buffers}: {e!r}") - - warnings.warn( - f"Could not build a meta-device skeleton of {architecture} " - f"({'; '.join(skeleton_errors)}). Skipping the device-map memory estimate and " - "letting from_pretrained map the model. If you hit GPU OOM, rerun with " - "--use_seq_device_map (which applies --gpu_max_mem_percentage) or lower " - "--batch_size." - ) - return None + try: + with init_empty_weights(include_buffers=False): + return from_config(config_for_init, **model_kwargs) + except Exception as e: + warnings.warn( + f"Could not build a meta-device skeleton of {architecture} ({e!r}). " + "Skipping the device-map memory estimate and letting from_pretrained map the " + "model. If you hit GPU OOM, rerun with --use_seq_device_map (which applies " + "--gpu_max_mem_percentage) or lower --batch_size." + ) + return None def _get_config_dtype(config): diff --git a/tests/examples/hf_ptq/test_example_utils.py b/tests/examples/hf_ptq/test_example_utils.py index 4b9120b0441..6175066f5c2 100644 --- a/tests/examples/hf_ptq/test_example_utils.py +++ b/tests/examples/hf_ptq/test_example_utils.py @@ -471,3 +471,93 @@ def from_pretrained(*args, **kwargs): example_utils.get_model("checkpoint", device="cpu", trust_remote_code=trust_remote_code) assert used["path"] == ("bundled" if expect_bundled_code else "builtin") + + +# ---------- _build_meta_skeleton ---------------------------------------------- + + +def test_build_meta_skeleton_uses_permissive_patching(): + """The probe must be no stricter than the loader it predicts. + + ``include_buffers=True`` is a bare global ``torch.device("meta")`` context, which + also captures scratch arithmetic in ``__init__``; ``from_pretrained`` never does + that, so the probe must not either. + """ + seen = {} + + def fake_init_empty_weights(include_buffers): + seen["include_buffers"] = include_buffers + return nullcontext() + + sentinel = object() + with patch.object(example_utils, "init_empty_weights", fake_init_empty_weights): + out = example_utils._build_meta_skeleton( + lambda cfg, **kw: sentinel, "cfg", {"dtype": torch.bfloat16}, "Arch" + ) + + assert out is sentinel + assert seen["include_buffers"] is False + + +def test_build_meta_skeleton_survives_meta_hostile_init(): + """Remote code that reads a real scalar in ``__init__`` must still build.""" + + def from_config(cfg, **kwargs): + # Mirrors Phi-4-multimodal's conformer: int() on a freshly built tensor. This + # raises under a global meta context but works with parameter-only patching. + return SimpleNamespace(width=int(torch.tensor(80.0))) + + model = example_utils._build_meta_skeleton(from_config, "cfg", {}, "Arch") + + assert model.width == 80 + + +def test_build_meta_skeleton_returns_none_and_warns_on_failure(): + def from_config(cfg, **kwargs): + raise RuntimeError("boom") + + with pytest.warns(UserWarning, match="Could not build a meta-device skeleton of Arch"): + assert example_utils._build_meta_skeleton(from_config, "cfg", {}, "Arch") is None + + +def test_get_model_skips_device_map_estimate_when_skeleton_fails(monkeypatch): + """A failed probe must not abort the load: skip sizing, still call from_pretrained.""" + calls = {} + hf_config = SimpleNamespace( + architectures=["LlamaForCausalLM"], + dtype=torch.float16, + model_type="llama", + torch_dtype=torch.bfloat16, + ) + + class FakeModel: + def eval(self): + calls["eval"] = True + + class FakeLlamaForCausalLM: + @staticmethod + def _from_config(config, **kwargs): + raise RuntimeError("Tensor.item() cannot be called on meta tensors") + + @staticmethod + def from_pretrained(*args, **kwargs): + calls["from_pretrained"] = kwargs + return FakeModel() + + def _boom_infer(model, max_memory): + raise AssertionError("infer_auto_device_map must be skipped without a skeleton") + + monkeypatch.setattr(example_utils.AutoConfig, "from_pretrained", lambda *a, **kw: hf_config) + monkeypatch.setattr(example_utils.transformers, "LlamaForCausalLM", FakeLlamaForCausalLM) + monkeypatch.setattr(example_utils, "is_nemotron_vl", lambda config: False) + monkeypatch.setattr(example_utils, "is_speculative", lambda config: False) + monkeypatch.setattr(example_utils, "get_max_memory", lambda: {0: 1024}) + monkeypatch.setattr(example_utils, "infer_auto_device_map", _boom_infer) + + with pytest.warns(UserWarning, match="Skipping the device-map memory estimate"): + model = example_utils.get_model("checkpoint", device="cpu", trust_remote_code=True) + + assert isinstance(model, FakeModel) + assert calls["eval"] + # The estimate is the only thing lost; no memory cap is invented for the load. + assert "max_memory" not in calls["from_pretrained"]