From 01739e656e580afa8bd635f425d9cd829c9199ed Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Wed, 17 Jul 2024 11:21:04 +0000 Subject: [PATCH 1/5] add quark quantizer --- optimum/amd/quantization/__init__.py | 7 + .../quark/algo_config_constants.py | 195 ++++++++++ .../amd/quantization/quark/configuration.py | 367 ++++++++++++++++++ optimum/amd/quantization/quark/quantizer.py | 226 +++++++++++ 4 files changed, 795 insertions(+) create mode 100644 optimum/amd/quantization/__init__.py create mode 100644 optimum/amd/quantization/quark/algo_config_constants.py create mode 100644 optimum/amd/quantization/quark/configuration.py create mode 100644 optimum/amd/quantization/quark/quantizer.py diff --git a/optimum/amd/quantization/__init__.py b/optimum/amd/quantization/__init__.py new file mode 100644 index 00000000..4177cd14 --- /dev/null +++ b/optimum/amd/quantization/__init__.py @@ -0,0 +1,7 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# Licensed under the MIT License. + +from .quark import ( + QuantizationMethod, + QuarkQuantizationConfig, +) diff --git a/optimum/amd/quantization/quark/algo_config_constants.py b/optimum/amd/quantization/quark/algo_config_constants.py new file mode 100644 index 00000000..24283625 --- /dev/null +++ b/optimum/amd/quantization/quark/algo_config_constants.py @@ -0,0 +1,195 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# Licensed under the MIT License. + + +ALGO_CONFIG_PARAMS = { + "llama": { + "scaling_layers": [ + { + "prev_op": "input_layernorm", + "layers": ["self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj"], + "inp": "self_attn.q_proj", + "module2inspect": "self_attn", + "has_kwargs": True, + "help": "attention input", + }, + { + "prev_op": "self_attn.v_proj", + "layers": ["self_attn.o_proj"], + "inp": "self_attn.o_proj", + "module2inspect": None, + "has_kwargs": False, + "help": "attention out, Please refer to https://github.com/mit-han-lab/llm-awq/pull/67#issue-1850622696, if module.self_attn.v_proj.weight.shape == module.self_attn.o_proj.weight.shape", + "condition": "module.self_attn.v_proj.weight.shape == module.self_attn.o_proj.weight.shape", + }, + { + "prev_op": "post_attention_layernorm", + "layers": ["mlp.gate_proj", "mlp.up_proj"], + "inp": "mlp.gate_proj", + "module2inspect": "mlp", + "has_kwargs": False, + "help": "linear 1", + }, + { + "prev_op": "mlp.up_proj", + "layers": ["mlp.down_proj"], + "inp": "mlp.down_proj", + "module2inspect": None, + "has_kwargs": False, + "help": "linear 2", + }, + ], + "inside_layer_modules": [ + "self_attn.k_proj", + "self_attn.v_proj", + "self_attn.q_proj", + "self_attn.o_proj", + "mlp.up_proj", + "mlp.gate_proj", + "mlp.down_proj", + ], + "model_decoder_layers": "model.layers", + "embedding_layers": ["model.embed_tokens"], + }, + "mistral": { + "scaling_layers": [ + { + "prev_op": "input_layernorm", + "layers": ["self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj"], + "inp": "self_attn.q_proj", + "module2inspect": "self_attn", + "has_kwargs": True, + "help": "attention input", + }, + { + "prev_op": "self_attn.v_proj", + "layers": ["self_attn.o_proj"], + "inp": "self_attn.o_proj", + "module2inspect": None, + "has_kwargs": False, + "help": "attention out, Please refer to https://github.com/mit-han-lab/llm-awq/pull/67#issue-1850622696, if module.self_attn.v_proj.weight.shape == module.self_attn.o_proj.weight.shape", + "condition": "module.self_attn.v_proj.weight.shape == module.self_attn.o_proj.weight.shape", + }, + { + "prev_op": "post_attention_layernorm", + "layers": ["mlp.gate_proj", "mlp.up_proj"], + "inp": "mlp.gate_proj", + "module2inspect": "mlp", + "has_kwargs": False, + "help": "linear 1", + }, + { + "prev_op": "mlp.up_proj", + "layers": ["mlp.down_proj"], + "inp": "mlp.down_proj", + "module2inspect": None, + "has_kwargs": False, + "help": "linear 2", + }, + ], + "inside_layer_modules": [ + "self_attn.k_proj", + "self_attn.v_proj", + "self_attn.q_proj", + "self_attn.o_proj", + "mlp.up_proj", + "mlp.gate_proj", + "mlp.down_proj", + ], + "model_decoder_layers": "model.layers", + "embedding_layers": ["model.embed_tokens"], + }, + "opt": { + "scaling_layers": [ + { + "prev_op": "self_attn_layer_norm", + "layers": ["self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj"], + "inp": "self_attn.q_proj", + "module2inspect": "self_attn", + "has_kwargs": True, + "help": "attention input", + }, + { + "prev_op": "self_attn.v_proj", + "layers": ["self_attn.out_proj"], + "inp": "self_attn.out_proj", + "module2inspect": None, + "has_kwargs": False, + "help": "attention out", + }, + { + "prev_op": "final_layer_norm", + "layers": ["fc1"], + "inp": "fc1", + "module2inspect": None, + "has_kwargs": False, + "help": "linear 1", + }, + { + "prev_op": "fc1", + "layers": ["fc2"], + "inp": "fc2", + "module2inspect": None, + "has_kwargs": False, + "help": "linear 2", + }, + ], + "inside_layer_modules": [ + "self_attn.k_proj", + "self_attn.v_proj", + "self_attn.q_proj", + "self_attn.out_proj", + "fc1", + "fc2", + ], + "model_decoder_layers": "model.decoder.layers", + "embedding_layers": ["model.decoder.embed_tokens", "model.decoder.embed_positions"], + }, + "qwen2": { + "scaling_layers": [ + { + "prev_op": "input_layernorm", + "layers": ["self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj"], + "inp": "self_attn.q_proj", + "module2inspect": "self_attn", + "has_kwargs": True, + "help": "attention input", + }, + { + "prev_op": "self_attn.v_proj", + "layers": ["self_attn.o_proj"], + "inp": "self_attn.o_proj", + "module2inspect": None, + "has_kwargs": False, + "help": "attention out, Please refer to https://github.com/mit-han-lab/llm-awq/pull/67#issue-1850622696, if module.self_attn.v_proj.weight.shape == module.self_attn.o_proj.weight.shape", + }, + { + "prev_op": "post_attention_layernorm", + "layers": ["mlp.gate_proj", "mlp.up_proj"], + "inp": "mlp.gate_proj", + "module2inspect": "mlp", + "has_kwargs": False, + "help": "linear 1", + }, + { + "prev_op": "mlp.up_proj", + "layers": ["mlp.down_proj"], + "inp": "mlp.down_proj", + "module2inspect": None, + "has_kwargs": False, + "help": "linear 2", + }, + ], + "inside_layer_modules": [ + "self_attn.k_proj", + "self_attn.v_proj", + "self_attn.q_proj", + "self_attn.o_proj", + "mlp.up_proj", + "mlp.gate_proj", + "mlp.down_proj", + ], + "model_decoder_layers": "model.layers", + "embedding_layers": ["model.embed_tokens"], + }, +} diff --git a/optimum/amd/quantization/quark/configuration.py b/optimum/amd/quantization/quark/configuration.py new file mode 100644 index 00000000..677ec308 --- /dev/null +++ b/optimum/amd/quantization/quark/configuration.py @@ -0,0 +1,367 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# Licensed under the MIT License. +"""Configuration classes for quantization with AMD Quark.""" + +from enum import Enum +from typing import Dict, List, Optional + +from quark.torch.quantization.config.config import ( + AlgoConfig, + AWQConfig, + Config, + GPTQConfig, + QuantizationConfig, + QuantizationSpec, + SmoothQuantConfig, +) +from quark.torch.quantization.config.custom_config import ( + DEFAULT_AWQ_CONFIG, + DEFAULT_FLOAT16_CONFIG, + DEFAULT_GPTQ_CONFIG, + DEFAULT_SMOOTH_QUANT_CONFIG, + DEFAULT_W_FP8_A_FP8_OFP8_PER_TENSOR_CONFIG, + DEFAULT_W_FP8_A_FP8_PER_TENSOR_CONFIG, + DEFAULT_W_INT4_PER_CHANNEL_CONFIG, + DEFAULT_W_INT4_PER_GROUP_SYM_CONFIG, + DEFAULT_W_INT4_PER_TENSOR_CONFIG, + DEFAULT_W_INT8_A_INT8_PER_TENSOR_CONFIG, + DEFAULT_W_INT8_A_INT8_PER_TENSOR_DYNAMIC_CONFIG, + DEFAULT_W_UINT4_A_BFLOAT16_PER_GROUP_CONFIG, + DEFAULT_W_UINT4_PER_GROUP_CONFIG, + FP8_PER_TENSOR_SPEC, +) + +from .algo_config_constants import ALGO_CONFIG_PARAMS + + +QuarkQuantizationConfig = Config + +__all__ = [ + "QuarkQuantizationConfig", + "QuantizationSpec", + "SmoothQuantConfig", + "AWQConfig", + "GPTQConfig", + "AlgoConfig", +] + + +class KVCacheDType(Enum): + FP8 = "fp8" + + +class AutoQuantizationConfig: + @staticmethod + def _apply_fp8_config(layer_quant_config): + KV_CACHE_CFG = { + "*.v_proj": QuantizationConfig( + input_tensors=FP8_PER_TENSOR_SPEC, output_tensors=FP8_PER_TENSOR_SPEC, weight=FP8_PER_TENSOR_SPEC + ), + "*.k_proj": QuantizationConfig( + input_tensors=FP8_PER_TENSOR_SPEC, output_tensors=FP8_PER_TENSOR_SPEC, weight=FP8_PER_TENSOR_SPEC + ), + } + return {**layer_quant_config, **KV_CACHE_CFG} + + @staticmethod + def _validate_kv_cache_dtype(kv_cache_dtype: str): + if kv_cache_dtype.lower() != "fp8": + raise ValueError(f"Invalid value for kv_cache_dtype: {kv_cache_dtype}. Expected 'fp8'.") + + @staticmethod + def _validate_model_type(model_type: str): + SUPPORTED_MODEL_TYPES = ["llama", "mistral", "opt", "qwen2"] + + if model_type not in SUPPORTED_MODEL_TYPES: + raise ValueError( + f"Invalid value for model_type for AutoQuantizationConfig: {model_type}. Expected one of {SUPPORTED_MODEL_TYPES}." + ) + + return model_type + + @staticmethod + def w_fp8_a_fp8( + layer_type_quant_config: Dict[str, QuantizationConfig] = {}, + layer_quant_config: Dict[str, QuantizationConfig] = {}, + exclude: List[str] = None, + pre_quant_opt_config: Optional[QuantizationConfig] = None, + kv_cache_dtype: Optional[KVCacheDType] = None, + ): + if kv_cache_dtype: + AutoQuantizationConfig._validate_kv_cache_dtype(kv_cache_dtype) + layer_quant_config = AutoQuantizationConfig._apply_fp8_config(layer_quant_config) + + return QuarkQuantizationConfig( + global_quant_config=DEFAULT_W_FP8_A_FP8_PER_TENSOR_CONFIG, + layer_type_quant_config=layer_type_quant_config, + layer_quant_config=layer_quant_config, + exclude=exclude, + pre_quant_opt_config=pre_quant_opt_config, + ) + + @staticmethod + def w_fp8_a_fp8_o_fp8( + layer_type_quant_config: Dict[str, QuantizationConfig] = {}, + layer_quant_config: Dict[str, QuantizationConfig] = {}, + exclude: List[str] = None, + pre_quant_opt_config: Optional[QuantizationConfig] = None, + kv_cache_dtype: Optional[KVCacheDType] = None, + ): + if kv_cache_dtype: + AutoQuantizationConfig._validate_kv_cache_dtype(kv_cache_dtype) + layer_quant_config = AutoQuantizationConfig._apply_fp8_config(layer_quant_config) + + return QuarkQuantizationConfig( + global_quant_config=DEFAULT_W_FP8_A_FP8_OFP8_PER_TENSOR_CONFIG, + layer_type_quant_config=layer_type_quant_config, + layer_quant_config=layer_quant_config, + exclude=exclude, + pre_quant_opt_config=pre_quant_opt_config, + ) + + @staticmethod + def w_int4_per_tensor( + layer_type_quant_config: Dict[str, QuantizationConfig] = {}, + layer_quant_config: Dict[str, QuantizationConfig] = {}, + exclude: List[str] = None, + pre_quant_opt_config: Optional[QuantizationConfig] = None, + kv_cache_dtype: Optional[KVCacheDType] = None, + ): + if kv_cache_dtype: + AutoQuantizationConfig._validate_kv_cache_dtype(kv_cache_dtype) + layer_quant_config = AutoQuantizationConfig._apply_fp8_config(layer_quant_config) + + return QuarkQuantizationConfig( + global_quant_config=DEFAULT_W_INT4_PER_TENSOR_CONFIG, + layer_type_quant_config=layer_type_quant_config, + layer_quant_config=layer_quant_config, + exclude=exclude, + pre_quant_opt_config=pre_quant_opt_config, + ) + + @staticmethod + def w_int4_per_channel_sym( + layer_type_quant_config: Dict[str, QuantizationConfig] = {}, + layer_quant_config: Dict[str, QuantizationConfig] = {}, + exclude: List[str] = None, + pre_quant_opt_config: Optional[QuantizationConfig] = None, + kv_cache_dtype: Optional[KVCacheDType] = None, + ): + if kv_cache_dtype: + AutoQuantizationConfig._validate_kv_cache_dtype(kv_cache_dtype) + layer_quant_config = AutoQuantizationConfig._apply_fp8_config(layer_quant_config) + + return QuarkQuantizationConfig( + global_quant_config=DEFAULT_W_INT4_PER_CHANNEL_CONFIG, + layer_type_quant_config=layer_type_quant_config, + layer_quant_config=layer_quant_config, + exclude=exclude, + pre_quant_opt_config=pre_quant_opt_config, + ) + + @staticmethod + def w_int4_per_group_sym( + layer_type_quant_config: Dict[str, QuantizationConfig] = {}, + layer_quant_config: Dict[str, QuantizationConfig] = {}, + exclude: List[str] = None, + pre_quant_opt_config: Optional[QuantizationConfig] = None, + kv_cache_dtype: Optional[KVCacheDType] = None, + ): + if kv_cache_dtype: + AutoQuantizationConfig._validate_kv_cache_dtype(kv_cache_dtype) + layer_quant_config = AutoQuantizationConfig._apply_fp8_config(layer_quant_config) + + return QuarkQuantizationConfig( + global_quant_config=DEFAULT_W_INT4_PER_GROUP_SYM_CONFIG, + layer_type_quant_config=layer_type_quant_config, + layer_quant_config=layer_quant_config, + exclude=exclude, + pre_quant_opt_config=pre_quant_opt_config, + ) + + @staticmethod + def w_uint4_per_group_asym_awq( + model_type: str, + layer_type_quant_config: Dict[str, QuantizationConfig] = {}, + layer_quant_config: Dict[str, QuantizationConfig] = {}, + exclude: List[str] = None, + pre_quant_opt_config: Optional[QuantizationConfig] = None, + kv_cache_dtype: Optional[KVCacheDType] = None, + ): + if kv_cache_dtype: + AutoQuantizationConfig._validate_kv_cache_dtype(kv_cache_dtype) + layer_quant_config = AutoQuantizationConfig._apply_fp8_config(layer_quant_config) + + model_type = AutoQuantizationConfig._validate_model_type(model_type) + algo_config_info = ALGO_CONFIG_PARAMS[model_type] + + quant_config = DEFAULT_AWQ_CONFIG + quant_config.algo_config.scaling_layers = algo_config_info["scaling_layers"] + quant_config.algo_config.model_decoder_layers = algo_config_info["model_decoder_layers"] + quant_config.algo_config.embedding_layers = algo_config_info["embedding_layers"] + return QuarkQuantizationConfig( + global_quant_config=quant_config.global_quant_config, + layer_type_quant_config=layer_type_quant_config, + layer_quant_config=layer_quant_config, + exclude=exclude, + algo_config=quant_config.algo_config, + pre_quant_opt_config=pre_quant_opt_config, + ) + + @staticmethod + def w_uint4_per_group_asym_smoothquant( + model_type: str, + layer_type_quant_config: Dict[str, QuantizationConfig] = {}, + layer_quant_config: Dict[str, QuantizationConfig] = {}, + exclude: List[str] = None, + pre_quant_opt_config: Optional[QuantizationConfig] = None, + kv_cache_dtype: Optional[KVCacheDType] = None, + ): + if kv_cache_dtype: + AutoQuantizationConfig._validate_kv_cache_dtype(kv_cache_dtype) + layer_quant_config = AutoQuantizationConfig._apply_fp8_config(layer_quant_config) + + model_type = AutoQuantizationConfig._validate_model_type(model_type) + algo_config_info = ALGO_CONFIG_PARAMS[model_type] + + quant_config = DEFAULT_SMOOTH_QUANT_CONFIG + quant_config.algo_config.scaling_layers = algo_config_info["scaling_layers"] + quant_config.algo_config.model_decoder_layers = algo_config_info["model_decoder_layers"] + quant_config.algo_config.embedding_layers = algo_config_info["embedding_layers"] + return QuarkQuantizationConfig( + global_quant_config=quant_config.global_quant_config, + layer_type_quant_config=layer_type_quant_config, + layer_quant_config=layer_quant_config, + exclude=exclude, + algo_config=quant_config.algo_config, + pre_quant_opt_config=pre_quant_opt_config, + ) + + @staticmethod + def w_uint4_per_group_asym_gptq( + model_type: str, + layer_type_quant_config: Dict[str, QuantizationConfig] = {}, + layer_quant_config: Dict[str, QuantizationConfig] = {}, + exclude: List[str] = None, + pre_quant_opt_config: Optional[QuantizationConfig] = None, + kv_cache_dtype: Optional[KVCacheDType] = None, + ): + if kv_cache_dtype: + AutoQuantizationConfig._validate_kv_cache_dtype(kv_cache_dtype) + layer_quant_config = AutoQuantizationConfig._apply_fp8_config(layer_quant_config) + + model_type = AutoQuantizationConfig._validate_model_type(model_type) + algo_config_info = ALGO_CONFIG_PARAMS[model_type] + + quant_config = DEFAULT_GPTQ_CONFIG + quant_config.algo_config.inside_layer_modules = algo_config_info["inside_layer_modules"] + quant_config.algo_config.model_decoder_layers = algo_config_info["model_decoder_layers"] + quant_config.algo_config.embedding_layers = algo_config_info["embedding_layers"] + return QuarkQuantizationConfig( + global_quant_config=quant_config.global_quant_config, + layer_type_quant_config=layer_type_quant_config, + layer_quant_config=layer_quant_config, + exclude=exclude, + algo_config=quant_config.algo_config, + pre_quant_opt_config=pre_quant_opt_config, + ) + + @staticmethod + def w_uint4_per_group_asym( + layer_type_quant_config: Dict[str, QuantizationConfig] = {}, + layer_quant_config: Dict[str, QuantizationConfig] = {}, + exclude: List[str] = None, + pre_quant_opt_config: Optional[QuantizationConfig] = None, + kv_cache_dtype: Optional[KVCacheDType] = None, + ): + if kv_cache_dtype: + AutoQuantizationConfig._validate_kv_cache_dtype(kv_cache_dtype) + layer_quant_config = AutoQuantizationConfig._apply_fp8_config(layer_quant_config) + + return QuarkQuantizationConfig( + global_quant_config=DEFAULT_W_UINT4_PER_GROUP_CONFIG, + layer_type_quant_config=layer_type_quant_config, + layer_quant_config=layer_quant_config, + exclude=exclude, + pre_quant_opt_config=pre_quant_opt_config, + ) + + @staticmethod + def w_uint4_a_bfloat16_per_group_asym( + layer_type_quant_config: Dict[str, QuantizationConfig] = {}, + layer_quant_config: Dict[str, QuantizationConfig] = {}, + exclude: List[str] = None, + pre_quant_opt_config: Optional[QuantizationConfig] = None, + kv_cache_dtype: Optional[KVCacheDType] = None, + ): + if kv_cache_dtype: + AutoQuantizationConfig._validate_kv_cache_dtype(kv_cache_dtype) + layer_quant_config = AutoQuantizationConfig._apply_fp8_config(layer_quant_config) + + return QuarkQuantizationConfig( + global_quant_config=DEFAULT_W_UINT4_A_BFLOAT16_PER_GROUP_CONFIG, + layer_type_quant_config=layer_type_quant_config, + layer_quant_config=layer_quant_config, + exclude=exclude, + pre_quant_opt_config=pre_quant_opt_config, + ) + + @staticmethod + def w_int8_a_int8_per_tensor_sym( + layer_type_quant_config: Dict[str, QuantizationConfig] = {}, + layer_quant_config: Dict[str, QuantizationConfig] = {}, + exclude: List[str] = None, + pre_quant_opt_config: Optional[QuantizationConfig] = None, + kv_cache_dtype: Optional[KVCacheDType] = None, + ): + if kv_cache_dtype: + AutoQuantizationConfig._validate_kv_cache_dtype(kv_cache_dtype) + layer_quant_config = AutoQuantizationConfig._apply_fp8_config(layer_quant_config) + + return QuarkQuantizationConfig( + global_quant_config=DEFAULT_W_INT8_A_INT8_PER_TENSOR_CONFIG, + layer_type_quant_config=layer_type_quant_config, + layer_quant_config=layer_quant_config, + exclude=exclude, + pre_quant_opt_config=pre_quant_opt_config, + ) + + @staticmethod + def w_int8_a_int8_per_tensor_sym_dynamic( + layer_type_quant_config: Dict[str, QuantizationConfig] = {}, + layer_quant_config: Dict[str, QuantizationConfig] = {}, + exclude: List[str] = None, + pre_quant_opt_config: Optional[QuantizationConfig] = None, + kv_cache_dtype: Optional[KVCacheDType] = None, + ): + if kv_cache_dtype: + AutoQuantizationConfig._validate_kv_cache_dtype(kv_cache_dtype) + layer_quant_config = AutoQuantizationConfig._apply_fp8_config(layer_quant_config) + + return QuarkQuantizationConfig( + global_quant_config=DEFAULT_W_INT8_A_INT8_PER_TENSOR_DYNAMIC_CONFIG, + layer_type_quant_config=layer_type_quant_config, + layer_quant_config=layer_quant_config, + exclude=exclude, + pre_quant_opt_config=pre_quant_opt_config, + ) + + @staticmethod + def float16( + layer_type_quant_config: Dict[str, QuantizationConfig] = {}, + layer_quant_config: Dict[str, QuantizationConfig] = {}, + exclude: List[str] = None, + pre_quant_opt_config: Optional[QuantizationConfig] = None, + kv_cache_dtype: Optional[KVCacheDType] = None, + ): + if kv_cache_dtype: + AutoQuantizationConfig._validate_kv_cache_dtype(kv_cache_dtype) + layer_quant_config = AutoQuantizationConfig._apply_fp8_config(layer_quant_config) + + return QuarkQuantizationConfig( + global_quant_config=DEFAULT_FLOAT16_CONFIG, + layer_type_quant_config=layer_type_quant_config, + layer_quant_config=layer_quant_config, + exclude=exclude, + pre_quant_opt_config=pre_quant_opt_config, + ) diff --git a/optimum/amd/quantization/quark/quantizer.py b/optimum/amd/quantization/quark/quantizer.py new file mode 100644 index 00000000..19e0fcb9 --- /dev/null +++ b/optimum/amd/quantization/quark/quantizer.py @@ -0,0 +1,226 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# Licensed under the MIT License. +"""AMD Quark Quantizer""" + +import logging +from typing import Dict, Optional, Union + +import torch +from quark.torch import ModelExporter, ModelQuantizer +from quark.torch.export.config.custom_config import DEFAULT_EXPORTER_CONFIG, EMPTY_EXPORTER_CONFIG + +from optimum.exporters import TasksManager +from optimum.quantization_base import OptimumQuantizer +from transformers import PretrainedConfig + + +logger = logging.getLogger(__name__) + + +class QuarkQuantizer(OptimumQuantizer): + """ + Handles the quantization process for models shared on huggingface.co/models. + """ + + def __init__( + self, + model: torch.nn.Module, + quantization_config, + model_name_or_path: Optional[str], + config: Optional[PretrainedConfig] = None, + ): + super().__init__() + self.model_name_or_path = model_name_or_path + + self.model = model + self.model_type = model.config.type + + # Initialize the quantizer + self.quantizer = ModelQuantizer(config) + + @classmethod + def from_pretrained( + cls, + model_name_or_path: str, + subfolder: str = "", + revision: Optional[str] = None, + cache_dir: Optional[str] = None, + trust_remote_code: bool = False, + force_download: bool = False, + local_files_only: bool = False, + use_auth_token: Optional[Union[bool, str]] = None, + device_map: Optional[Union[Dict, str, torch.device]] = None, + **model_kwargs, + ): + """ + Loads the BrevitasQuantizer and model. + + Arguments: + model_name_or_path (`Union[str, Path]`): + Can be either the model id of a model repo on the Hugging Face Hub, or a path to a local directory + containing a model. + subfolder (`str`, defaults to `""`): + In case the model files are located inside a subfolder of the model directory / repo on the Hugging + Face Hub, you can specify the subfolder name here. + revision (`Optional[str]`, *optional*, defaults to `None`): + Revision is the specific model version to use. It can be a branch name, a tag name, or a commit id. + cache_dir (`Optional[str]`, *optional*): + Path to a directory in which a downloaded pretrained model weights have been cached if the standard cache should not be used. + trust_remote_code (`bool`, defaults to `False`): + Allows to use custom code for the modeling hosted in the model repository. This option should only be set for repositories + you trust and in which you have read the code, as it will execute on your local machine arbitrary code present in the + model repository. + force_download (`bool`, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + local_files_only (`Optional[bool]`, defaults to `False`): + Whether or not to only look at local files (i.e., do not try to download the model). + use_auth_token (`Optional[str]`, defaults to `None`): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `transformers-cli login` (stored in `~/.huggingface`). + """ + + # TODO: fix + # task = TasksManager.infer_task_from_model(model_name_or_path) + task = "text-generation" + + device = None + if not isinstance(device_map, dict) and device_map not in ["auto", "balanced"]: + device = device_map + device_map = None + + model = TasksManager.get_model_from_task( + task, + model_name_or_path, + subfolder=subfolder, + revision=revision, + cache_dir=cache_dir, + use_auth_token=use_auth_token, + local_files_only=local_files_only, + force_download=force_download, + trust_remote_code=trust_remote_code, + device_map=device_map, + device=device, + framework="pt", + **model_kwargs, + ) + + return cls(model, model_name_or_path) + + def quantize( + self, + dataset: Dataset, + ) -> torch.nn.Module: + """ + Quantizes the model using Brevitas according to the `quantization_config`. + + Arguments: + quantization_config (`BrevitasQuantizationConfig`): + Quantization configuration to use to quantize the model. + calibration_dataset (`Optional[List[Dict]]`, defaults to `None`): + In case the quantization involves a calibration phase, this argument needs to be specified as a list of inputs to the model. + Example: `calibration_dataset = [{"input_ids": torch.tensor([[1, 2, 3, 4]])}, {"input_ids": torch.tensor([[6, 7, 3, 4]])}]` which is a dataset for a model taking `input_ids` as an argument, and which has two samples. + """ + calib_dataloader = get_calib_dataloader(dataset) + + self.model = self.quantizer.quantize_model(self.model, calib_dataloader) + + return self.model + + def save_pretrained(self, save_directory: str, export_config=None): + """ + Save the quantized model to the specified directory. + + Arguments: + save_directory (`str`): + Directory to save the quantized model to. + """ + model = self.quantizer.freeze(self.model) + + with torch.inference_mode(): + export_config = EMPTY_EXPORTER_CONFIG if args.no_weight_matrix_merge else DEFAULT_EXPORTER_CONFIG + + exporter = ModelExporter(config=config, export_dir=save_directory) + exporter.export_model_info(model, model_type, model_dtype, export_type="native") + + + def get_calibration_dataset( + self, + dataset_name: str, + num_samples: int = 100, + dataset_config_name: Optional[str] = None, + dataset_split: Optional[str] = None, + preprocess_function: Optional[Callable] = None, + preprocess_batch: bool = True, + seed: int = 2016, + use_auth_token: Optional[Union[bool, str]] = None, + token: Optional[Union[bool, str]] = None, + ) -> Dataset: + """ + Creates the calibration `datasets.Dataset` to use for the post-training static quantization calibration step. + + Args: + dataset_name (`str`): + The dataset repository name on the Hugging Face Hub or path to a local directory containing data files + to load to use for the calibration step. + num_samples (`int`, defaults to 100): + The maximum number of samples composing the calibration dataset. + dataset_config_name (`Optional[str]`, defaults to `None`): + The name of the dataset configuration. + dataset_split (`Optional[str]`, defaults to `None`): + Which split of the dataset to use to perform the calibration step. + preprocess_function (`Optional[Callable]`, defaults to `None`): + Processing function to apply to each example after loading dataset. + preprocess_batch (`bool`, defaults to `True`): + Whether the `preprocess_function` should be batched. + seed (`int`, defaults to 2016): + The random seed to use when shuffling the calibration dataset. + use_auth_token (`Optional[Union[bool,str]]`, defaults to `None`): + Deprecated. Please use the `token` argument instead. + token (`Optional[Union[bool,str]]`, defaults to `None`): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `huggingface-cli login` (stored in `huggingface_hub.constants.HF_TOKEN_PATH`). + + Returns: + The calibration `datasets.Dataset` to use for the post-training static quantization calibration + step. + """ + + if use_auth_token is not None: + warnings.warn( + "The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.", + FutureWarning, + ) + if token is not None: + raise ValueError("You cannot use both `use_auth_token` and `token` arguments at the same time.") + token = use_auth_token + + if dataset_name is None: + raise ValueError( + "ORTQuantizer: Static quantization calibration step requires a dataset_name if no calib_dataset is " + "provided." + ) + + calib_dataset = load_dataset( + dataset_name, + name=dataset_config_name, + split=dataset_split, + token=token, + ) + + if num_samples is not None: + num_samples = min(num_samples, len(calib_dataset)) + calib_dataset = calib_dataset.shuffle(seed=seed).select(range(num_samples)) + + if preprocess_function is not None: + processed_calib_dataset = calib_dataset.map(preprocess_function, batched=preprocess_batch) + else: + processed_calib_dataset = calib_dataset + + return self.clean_calibration_dataset(processed_calib_dataset) + + def clean_calibration_dataset(self, dataset: Dataset) -> Dataset: + model = onnx.load(self.onnx_model_path) + model_inputs = {input.name for input in model.graph.input} + ignored_columns = list(set(dataset.column_names) - model_inputs) + return dataset.remove_columns(ignored_columns) \ No newline at end of file From 0a677ee42322599e651d6f86a2ee43d3c7509415 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Wed, 17 Jul 2024 13:01:42 +0000 Subject: [PATCH 2/5] fix --- docker/quark-quantization/amd/Dockerfile | 39 +++++++ optimum/amd/quantization/__init__.py | 3 +- .../amd/quantization/quark/configuration.py | 26 ++--- optimum/amd/quantization/quark/quantizer.py | 101 +++++++++--------- 4 files changed, 102 insertions(+), 67 deletions(-) create mode 100644 docker/quark-quantization/amd/Dockerfile diff --git a/docker/quark-quantization/amd/Dockerfile b/docker/quark-quantization/amd/Dockerfile new file mode 100644 index 00000000..218fffb5 --- /dev/null +++ b/docker/quark-quantization/amd/Dockerfile @@ -0,0 +1,39 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# Licensed under the MIT License. + +FROM rocm/dev-ubuntu-22.04:6.1 + +LABEL maintainer="Hugging Face" + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + sudo \ + python3.10 \ + python3.10-dev \ + python3-pip \ + git \ + wget \ + unzip \ + libsndfile1-dev \ + tesseract-ocr \ + espeak-ng \ + rocthrust-dev \ + hipsparse-dev \ + hipblas-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 && \ + python -m pip install -U pip + +RUN pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.0 --no-cache-dir + +WORKDIR /quark +RUN wget -O quark.zip https://www.xilinx.com/bin/public/openDownload?filename=quark-0.1.0+a9827f5.zip && \ + unzip quark.zip -d quark && \ + pip install quark/quark-0.1.0+a9827f5/quark-0.1.0+a9827f5-py39-none-any.whl && \ + rm -rf quark.zip quark + +RUN python -c "import quark.torch.kernel" + +ENTRYPOINT ["bash"] diff --git a/optimum/amd/quantization/__init__.py b/optimum/amd/quantization/__init__.py index 4177cd14..afddd61b 100644 --- a/optimum/amd/quantization/__init__.py +++ b/optimum/amd/quantization/__init__.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. from .quark import ( - QuantizationMethod, + AutoQuantizationConfig, QuarkQuantizationConfig, + QuarkQuantizer, ) diff --git a/optimum/amd/quantization/quark/configuration.py b/optimum/amd/quantization/quark/configuration.py index 677ec308..b93c7787 100644 --- a/optimum/amd/quantization/quark/configuration.py +++ b/optimum/amd/quantization/quark/configuration.py @@ -83,7 +83,7 @@ def _validate_model_type(model_type: str): def w_fp8_a_fp8( layer_type_quant_config: Dict[str, QuantizationConfig] = {}, layer_quant_config: Dict[str, QuantizationConfig] = {}, - exclude: List[str] = None, + exclude: List[str] = [], pre_quant_opt_config: Optional[QuantizationConfig] = None, kv_cache_dtype: Optional[KVCacheDType] = None, ): @@ -103,7 +103,7 @@ def w_fp8_a_fp8( def w_fp8_a_fp8_o_fp8( layer_type_quant_config: Dict[str, QuantizationConfig] = {}, layer_quant_config: Dict[str, QuantizationConfig] = {}, - exclude: List[str] = None, + exclude: List[str] = [], pre_quant_opt_config: Optional[QuantizationConfig] = None, kv_cache_dtype: Optional[KVCacheDType] = None, ): @@ -123,7 +123,7 @@ def w_fp8_a_fp8_o_fp8( def w_int4_per_tensor( layer_type_quant_config: Dict[str, QuantizationConfig] = {}, layer_quant_config: Dict[str, QuantizationConfig] = {}, - exclude: List[str] = None, + exclude: List[str] = [], pre_quant_opt_config: Optional[QuantizationConfig] = None, kv_cache_dtype: Optional[KVCacheDType] = None, ): @@ -143,7 +143,7 @@ def w_int4_per_tensor( def w_int4_per_channel_sym( layer_type_quant_config: Dict[str, QuantizationConfig] = {}, layer_quant_config: Dict[str, QuantizationConfig] = {}, - exclude: List[str] = None, + exclude: List[str] = [], pre_quant_opt_config: Optional[QuantizationConfig] = None, kv_cache_dtype: Optional[KVCacheDType] = None, ): @@ -163,7 +163,7 @@ def w_int4_per_channel_sym( def w_int4_per_group_sym( layer_type_quant_config: Dict[str, QuantizationConfig] = {}, layer_quant_config: Dict[str, QuantizationConfig] = {}, - exclude: List[str] = None, + exclude: List[str] = [], pre_quant_opt_config: Optional[QuantizationConfig] = None, kv_cache_dtype: Optional[KVCacheDType] = None, ): @@ -184,7 +184,7 @@ def w_uint4_per_group_asym_awq( model_type: str, layer_type_quant_config: Dict[str, QuantizationConfig] = {}, layer_quant_config: Dict[str, QuantizationConfig] = {}, - exclude: List[str] = None, + exclude: List[str] = [], pre_quant_opt_config: Optional[QuantizationConfig] = None, kv_cache_dtype: Optional[KVCacheDType] = None, ): @@ -213,7 +213,7 @@ def w_uint4_per_group_asym_smoothquant( model_type: str, layer_type_quant_config: Dict[str, QuantizationConfig] = {}, layer_quant_config: Dict[str, QuantizationConfig] = {}, - exclude: List[str] = None, + exclude: List[str] = [], pre_quant_opt_config: Optional[QuantizationConfig] = None, kv_cache_dtype: Optional[KVCacheDType] = None, ): @@ -242,7 +242,7 @@ def w_uint4_per_group_asym_gptq( model_type: str, layer_type_quant_config: Dict[str, QuantizationConfig] = {}, layer_quant_config: Dict[str, QuantizationConfig] = {}, - exclude: List[str] = None, + exclude: List[str] = [], pre_quant_opt_config: Optional[QuantizationConfig] = None, kv_cache_dtype: Optional[KVCacheDType] = None, ): @@ -270,7 +270,7 @@ def w_uint4_per_group_asym_gptq( def w_uint4_per_group_asym( layer_type_quant_config: Dict[str, QuantizationConfig] = {}, layer_quant_config: Dict[str, QuantizationConfig] = {}, - exclude: List[str] = None, + exclude: List[str] = [], pre_quant_opt_config: Optional[QuantizationConfig] = None, kv_cache_dtype: Optional[KVCacheDType] = None, ): @@ -290,7 +290,7 @@ def w_uint4_per_group_asym( def w_uint4_a_bfloat16_per_group_asym( layer_type_quant_config: Dict[str, QuantizationConfig] = {}, layer_quant_config: Dict[str, QuantizationConfig] = {}, - exclude: List[str] = None, + exclude: List[str] = [], pre_quant_opt_config: Optional[QuantizationConfig] = None, kv_cache_dtype: Optional[KVCacheDType] = None, ): @@ -310,7 +310,7 @@ def w_uint4_a_bfloat16_per_group_asym( def w_int8_a_int8_per_tensor_sym( layer_type_quant_config: Dict[str, QuantizationConfig] = {}, layer_quant_config: Dict[str, QuantizationConfig] = {}, - exclude: List[str] = None, + exclude: List[str] = [], pre_quant_opt_config: Optional[QuantizationConfig] = None, kv_cache_dtype: Optional[KVCacheDType] = None, ): @@ -330,7 +330,7 @@ def w_int8_a_int8_per_tensor_sym( def w_int8_a_int8_per_tensor_sym_dynamic( layer_type_quant_config: Dict[str, QuantizationConfig] = {}, layer_quant_config: Dict[str, QuantizationConfig] = {}, - exclude: List[str] = None, + exclude: List[str] = [], pre_quant_opt_config: Optional[QuantizationConfig] = None, kv_cache_dtype: Optional[KVCacheDType] = None, ): @@ -350,7 +350,7 @@ def w_int8_a_int8_per_tensor_sym_dynamic( def float16( layer_type_quant_config: Dict[str, QuantizationConfig] = {}, layer_quant_config: Dict[str, QuantizationConfig] = {}, - exclude: List[str] = None, + exclude: List[str] = [], pre_quant_opt_config: Optional[QuantizationConfig] = None, kv_cache_dtype: Optional[KVCacheDType] = None, ): diff --git a/optimum/amd/quantization/quark/quantizer.py b/optimum/amd/quantization/quark/quantizer.py index 19e0fcb9..6c1fd76e 100644 --- a/optimum/amd/quantization/quark/quantizer.py +++ b/optimum/amd/quantization/quark/quantizer.py @@ -3,15 +3,18 @@ """AMD Quark Quantizer""" import logging -from typing import Dict, Optional, Union +from typing import Callable, Dict, List, Optional, Union import torch +from datasets import Dataset, load_dataset from quark.torch import ModelExporter, ModelQuantizer from quark.torch.export.config.custom_config import DEFAULT_EXPORTER_CONFIG, EMPTY_EXPORTER_CONFIG +from torch.utils.data import DataLoader from optimum.exporters import TasksManager from optimum.quantization_base import OptimumQuantizer -from transformers import PretrainedConfig + +from .configuration import QuarkQuantizationConfig logger = logging.getLogger(__name__) @@ -25,23 +28,28 @@ class QuarkQuantizer(OptimumQuantizer): def __init__( self, model: torch.nn.Module, - quantization_config, + quantization_config: QuarkQuantizationConfig, model_name_or_path: Optional[str], - config: Optional[PretrainedConfig] = None, ): super().__init__() self.model_name_or_path = model_name_or_path self.model = model - self.model_type = model.config.type + self.model_type = model.config.model_type + self.model_dtype = next(model.parameters()).dtype # Initialize the quantizer - self.quantizer = ModelQuantizer(config) + self.quantizer = ModelQuantizer(quantization_config) + + from pdb import set_trace + + set_trace() @classmethod def from_pretrained( cls, model_name_or_path: str, + quantization_config: QuarkQuantizationConfig, subfolder: str = "", revision: Optional[str] = None, cache_dir: Optional[str] = None, @@ -53,9 +61,9 @@ def from_pretrained( **model_kwargs, ): """ - Loads the BrevitasQuantizer and model. + Loads the QuarkQuantizer and model. - Arguments: + Args: model_name_or_path (`Union[str, Path]`): Can be either the model id of a model repo on the Hugging Face Hub, or a path to a local directory containing a model. @@ -103,48 +111,53 @@ def from_pretrained( device=device, framework="pt", **model_kwargs, - ) + ).eval() - return cls(model, model_name_or_path) + return cls(model, quantization_config, model_name_or_path) def quantize( self, - dataset: Dataset, + dataloader: Optional[Dataset] = None, ) -> torch.nn.Module: + """_summary_ + + Args: + dataloader (`Optional[Union[DataLoader[torch.Tensor], DataLoader[List[Dict[str, torch.Tensor]]], + DataLoader[Dict[str, torch.Tensor]]]]`, defaults to `None`): + The DataLoader providing data that the quantization process will use for calibration. This can be a simple DataLoader returning + tensors, or a more complex structure returning either a list of dictionaries or a dictionary of tensors. + + Returns: + torch.nn.Module: Quantized model """ - Quantizes the model using Brevitas according to the `quantization_config`. - - Arguments: - quantization_config (`BrevitasQuantizationConfig`): - Quantization configuration to use to quantize the model. - calibration_dataset (`Optional[List[Dict]]`, defaults to `None`): - In case the quantization involves a calibration phase, this argument needs to be specified as a list of inputs to the model. - Example: `calibration_dataset = [{"input_ids": torch.tensor([[1, 2, 3, 4]])}, {"input_ids": torch.tensor([[6, 7, 3, 4]])}]` which is a dataset for a model taking `input_ids` as an argument, and which has two samples. - """ - calib_dataloader = get_calib_dataloader(dataset) + if not self.quantizer.is_all_dynamic and dataloader is None: + raise ValueError("A calibration dataset is required for the quantization method.") - self.model = self.quantizer.quantize_model(self.model, calib_dataloader) + self.model = self.quantizer.quantize_model(self.model, dataloader) return self.model - def save_pretrained(self, save_directory: str, export_config=None): + def save_pretrained(self, save_directory: str, no_weight_matrix_merge=False): """ Save the quantized model to the specified directory. - Arguments: + Args: save_directory (`str`): Directory to save the quantized model to. + no_weight_matrix_merge (`bool`, defaults to `False`): + Whether to merge weight matrix when dump quantized model """ + if self.model_dtype != "llama": + raise ValueError("Only models with dtype `llama` can be saved.") model = self.quantizer.freeze(self.model) with torch.inference_mode(): - export_config = EMPTY_EXPORTER_CONFIG if args.no_weight_matrix_merge else DEFAULT_EXPORTER_CONFIG - - exporter = ModelExporter(config=config, export_dir=save_directory) - exporter.export_model_info(model, model_type, model_dtype, export_type="native") + export_config = EMPTY_EXPORTER_CONFIG if no_weight_matrix_merge else DEFAULT_EXPORTER_CONFIG + exporter = ModelExporter(config=export_config, export_dir=save_directory) + exporter.export_model_info(model, self.model_type, self.model_dtype, export_type="vllm-adopt") - def get_calibration_dataset( + def get_calibration_data( self, dataset_name: str, num_samples: int = 100, @@ -153,9 +166,11 @@ def get_calibration_dataset( preprocess_function: Optional[Callable] = None, preprocess_batch: bool = True, seed: int = 2016, - use_auth_token: Optional[Union[bool, str]] = None, token: Optional[Union[bool, str]] = None, - ) -> Dataset: + batch_size: int = 1, + ) -> Union[ + DataLoader[torch.Tensor], DataLoader[List[Dict[str, torch.Tensor]]], DataLoader[Dict[str, torch.Tensor]] + ]: """ Creates the calibration `datasets.Dataset` to use for the post-training static quantization calibration step. @@ -185,22 +200,6 @@ def get_calibration_dataset( The calibration `datasets.Dataset` to use for the post-training static quantization calibration step. """ - - if use_auth_token is not None: - warnings.warn( - "The `use_auth_token` argument is deprecated and will be removed soon. Please use the `token` argument instead.", - FutureWarning, - ) - if token is not None: - raise ValueError("You cannot use both `use_auth_token` and `token` arguments at the same time.") - token = use_auth_token - - if dataset_name is None: - raise ValueError( - "ORTQuantizer: Static quantization calibration step requires a dataset_name if no calib_dataset is " - "provided." - ) - calib_dataset = load_dataset( dataset_name, name=dataset_config_name, @@ -217,10 +216,6 @@ def get_calibration_dataset( else: processed_calib_dataset = calib_dataset - return self.clean_calibration_dataset(processed_calib_dataset) + dataloader = DataLoader(processed_calib_dataset, batch_size=batch_size, shuffle=False) - def clean_calibration_dataset(self, dataset: Dataset) -> Dataset: - model = onnx.load(self.onnx_model_path) - model_inputs = {input.name for input in model.graph.input} - ignored_columns = list(set(dataset.column_names) - model_inputs) - return dataset.remove_columns(ignored_columns) \ No newline at end of file + return dataloader From 99ba7ad891dc466256050cf4d3e08083076fb922 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Wed, 17 Jul 2024 13:45:29 +0000 Subject: [PATCH 3/5] remove pdb --- optimum/amd/quantization/quark/quantizer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/optimum/amd/quantization/quark/quantizer.py b/optimum/amd/quantization/quark/quantizer.py index 6c1fd76e..42777120 100644 --- a/optimum/amd/quantization/quark/quantizer.py +++ b/optimum/amd/quantization/quark/quantizer.py @@ -41,10 +41,6 @@ def __init__( # Initialize the quantizer self.quantizer = ModelQuantizer(quantization_config) - from pdb import set_trace - - set_trace() - @classmethod def from_pretrained( cls, From a8925a7a7c547abfefddb4111e41474152b5eabb Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Wed, 17 Jul 2024 13:48:13 +0000 Subject: [PATCH 4/5] fix save_pretrained --- optimum/amd/quantization/quark/quantizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/amd/quantization/quark/quantizer.py b/optimum/amd/quantization/quark/quantizer.py index 42777120..f84f7aac 100644 --- a/optimum/amd/quantization/quark/quantizer.py +++ b/optimum/amd/quantization/quark/quantizer.py @@ -143,7 +143,7 @@ def save_pretrained(self, save_directory: str, no_weight_matrix_merge=False): no_weight_matrix_merge (`bool`, defaults to `False`): Whether to merge weight matrix when dump quantized model """ - if self.model_dtype != "llama": + if self.model_type != "llama": raise ValueError("Only models with dtype `llama` can be saved.") model = self.quantizer.freeze(self.model) From 876efedc6545eb3f2da54e8bebb47e550d16e46d Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Wed, 17 Jul 2024 14:02:57 +0000 Subject: [PATCH 5/5] fix save_pretrained --- optimum/amd/quantization/quark/quantizer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/optimum/amd/quantization/quark/quantizer.py b/optimum/amd/quantization/quark/quantizer.py index f84f7aac..27a0740b 100644 --- a/optimum/amd/quantization/quark/quantizer.py +++ b/optimum/amd/quantization/quark/quantizer.py @@ -153,6 +153,8 @@ def save_pretrained(self, save_directory: str, no_weight_matrix_merge=False): exporter = ModelExporter(config=export_config, export_dir=save_directory) exporter.export_model_info(model, self.model_type, self.model_dtype, export_type="vllm-adopt") + model.config.save_pretrained(save_directory) + def get_calibration_data( self, dataset_name: str,