From 8c54971aa12a60c1b8a15b0dd3a3cbe63e78cdab Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Wed, 13 Mar 2024 19:47:08 +0530 Subject: [PATCH 01/28] Add ryzen model --- optimum/amd/ryzenai/modeling_decoder.py | 280 ++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 optimum/amd/ryzenai/modeling_decoder.py diff --git a/optimum/amd/ryzenai/modeling_decoder.py b/optimum/amd/ryzenai/modeling_decoder.py new file mode 100644 index 00000000..18dd806e --- /dev/null +++ b/optimum/amd/ryzenai/modeling_decoder.py @@ -0,0 +1,280 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# Licensed under the MIT License. +"""RyzenAIModelForXXX classes, allowing to run ONNX Models with ONNX Runtime VITIS-AI EP using the same API as Transformers.""" + +import logging +import os +import shutil +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, Dict, List, Optional, Tuple, Union + +import onnx +import onnxruntime as ort +import torch +from huggingface_hub import HfApi, HfFolder, hf_hub_download +from huggingface_hub.utils import EntryNotFoundError +from onnx import shape_inference +from onnx.tools import update_model_dims + +from optimum.exporters import TasksManager +from optimum.modeling_base import FROM_PRETRAINED_START_DOCSTRING, OptimizedModel +from optimum.onnx.utils import _get_external_data_paths +from optimum.utils.save_utils import maybe_load_preprocessors +from transformers import ( + AutoConfig, + AutoModel, + AutoModelForImageClassification, + PretrainedConfig, +) +from transformers.file_utils import add_start_docstrings +from transformers.modeling_outputs import ImageClassifierOutput, ModelOutput + +from .utils import ( + ONNX_WEIGHTS_NAME, + ONNX_WEIGHTS_NAME_STATIC, + validate_provider_availability, +) + + +logger = logging.getLogger(__name__) + +CONFIG_NAME = "config.json" + +class RyzenAIModelForCausalLM(RyzenAIModel, GenerationMixin): + """ + Runs model with causal language modeling head using ONNX Runtime VITIS-AI EP. + """ + + model_type = "onnx_model" + auto_model_class = AutoModel + + def __init__( + self, + model: ort.InferenceSession, + config: PretrainedConfig, + vaip_config: Union[str, Path] = None, + model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, + preprocessors: Optional[List] = None, + generation_config: Optional[GenerationConfig] = None, + use_cache: Optional[bool] = None, + **kwargs, + ): + super().__init__(model, config, vaip_config, model_save_dir, preprocessors, **kwargs) + + self.num_pkv = 2 + self.normalized_config = NormalizedConfigManager.get_normalized_config_class(config.model_type)(config) + self.key_value_input_names = [key for key in self.inputs_names if (".key" in key) or (".value" in key)] + self.key_value_output_names = [key for key in self.output_names if (".key" in key) or (".value" in key)] + self.use_cache = len(self.key_value_input_names) > 0 + + if generation_config is None: + generation_config = GenerationConfig.from_model_config(config) + self.generation_config = generation_config + + if use_cache ^ self.use_cache: + raise ValueError( + f"`use_cache` was set to `{use_cache}` but the loaded model only supports `use_cache={self.use_cache}`. " + f"Please load your current model with `use_cache={self.use_cache}` or export the original model " + f"once again with `use_cache={use_cache}` when calling the `from_pretrained` method. " + "To export your model, simply set `export=True`." + ) + + def forward( + self, + input_ids: torch.LongTensor, + attention_mask: Optional[torch.FloatTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, + **kwargs, + ) -> CausalLMOutputWithPast: + use_torch = isinstance(input_ids, torch.Tensor) + + inputs = {} + known_output_shapes = {} + + if self.use_cache: + # Create dummy past_key_values for decoder first generation step if none given + use_cache_branch, past_key_values, known_output_shapes = self.prepare_past_key_values( + input_ids, past_key_values, use_torch + ) + + inputs["input_ids"] = input_ids.cpu().detach().numpy() if use_torch else input_ids + + if "attention_mask" in self.inputs_names: + inputs["attention_mask"] = attention_mask.cpu().detach().numpy() if use_torch else attention_mask + + if "position_ids" in self.inputs_names: + if position_ids is None: + raise ValueError("position_ids was not passed but is a required input for this ONNX model.") + inputs["position_ids"] = position_ids.cpu().detach().numpy() if use_torch else position_ids + + # Add the past_key_values to the decoder inputs + if past_key_values is not None: + for input_name, past_key_value in zip(self.key_value_input_names, past_key_values): + inputs[input_name] = past_key_value.cpu().detach().numpy() if use_torch else past_key_value + + outputs = self.model.run(None, inputs) + + if self.use_cache: + # Tuple of length equal to : number of layer * number of past_key_value per decoder layer (2 for the self-attention) + past_key_values = tuple( + torch.from_numpy(outputs[self.output_names[key]]).to(self.device) + for key in self.key_value_output_names + ) + + logits = torch.from_numpy(outputs[self.output_names["logits"]]).to(self.device) + + if self.use_cache and self.model_type != "gpt_bigcode": + # Tuple of tuple of length `n_layers`, with each tuple of length equal to the number of self-attention and + # per decoder layer + past_key_values = tuple( + past_key_values[i : i + self.num_pkv] for i in range(0, len(past_key_values), self.num_pkv) + ) + + return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=past_key_values) + + def prepare_past_key_values( + self, + input_ids: Union[None, torch.LongTensor, np.ndarray], + past_key_values: Union[None, Tuple[torch.FloatTensor], Tuple[np.ndarray]], + use_torch: bool, + ): + if past_key_values is not None: + return past_key_values, {} + + sequence_length = input_ids.shape[1] + + constructor = torch if use_torch else np + + pkv_output_shape = {} + # Generate dummy past for the first forward + batch_size = input_ids.shape[0] + embed_size_per_head = self.normalized_config.hidden_size // self.normalized_config.num_attention_heads + if self.model_type == "gemma": + num_attention_heads = self.normalized_config.num_key_value_heads + embed_size_per_head = self.normalized_config.head_dim + elif self.model_type in {"mistral", "llama"}: + num_attention_heads = self.normalized_config.num_key_value_heads + else: + num_attention_heads = self.normalized_config.num_attention_heads + + dtype = constructor.float16 if self.use_fp16 else constructor.float32 + + num_key_value_heads = num_attention_heads + + shape = (batch_size, num_key_value_heads, 0, embed_size_per_head) + key_or_value = constructor.zeros(shape, dtype=dtype) + + if use_torch: + key_or_value = key_or_value.to(self.device) + + past_key_values = tuple(key_or_value for _ in range(len(self.key_value_input_names))) + + for name, value in zip(self.key_value_output_names, past_key_values): + shape = [*value.shape] + shape[2] += sequence_length + pkv_output_shape[name] = shape + + return past_key_values, pkv_output_shape + + @classmethod + def _from_pretrained( + cls, + model_id: Union[str, Path], + config: PretrainedConfig, + vaip_config: Optional[str] = None, + use_auth_token: Optional[Union[bool, str]] = None, + revision: Optional[str] = None, + force_download: bool = False, + cache_dir: Optional[str] = None, + file_name: Optional[str] = None, + subfolder: str = "", + local_files_only: bool = False, + provider: str = "VitisAIExecutionProvider", + session_options: Optional[ort.SessionOptions] = None, + provider_options: Optional[Dict[str, Any]] = None, + model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, + init_cls: Optional[RyzenAIModel] = None, + **kwargs, + ) -> "RyzenAIModelForCausalLM": + if config.model_type == "bloom": + init_cls = RyzenAIBloomForCausalLM + elif config.model_type == "falcon": + init_cls = RyzenAIFalconForCausalLM + elif config.model_type == "mpt": + init_cls = RyzenAIMPTForCausalLM + elif config.model_type == "opt": + init_cls = RyzenAIOptForCausalLM + elif config.model_type == "gpt_bigcode": + init_cls = RyzenAIGPTBigCodeForCausalLM + else: + init_cls = RyzenAIModelForCausalLM + + model = super()._from_pretrained( + model_id, + config, + use_auth_token, + revision, + force_download, + cache_dir, + file_name, + subfolder, + use_cache, + local_files_only, + use_merged, + provider, + session_options, + provider_options, + use_io_binding, + model_save_dir, + init_cls, + **kwargs, + ) + + + return model + + # Adapted from transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel.prepare_inputs_for_generation + def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs): + if past_key_values is not None: + past_length = past_key_values[0][0].shape[2] + # Some generation methods already pass only the last input ID + if input_ids.shape[1] > past_length: + remove_prefix_length = past_length + else: + # Default to old behavior: keep only final ID + remove_prefix_length = input_ids.shape[1] - 1 + input_ids = input_ids[:, remove_prefix_length:] + + # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly + attention_mask = kwargs.get("attention_mask", None) + use_cache = kwargs.get("use_cache", None) + + position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -1].unsqueeze(-1) + + return { + "input_ids": input_ids, + "past_key_values": past_key_values, + "use_cache": use_cache, + "position_ids": position_ids, + "attention_mask": attention_mask, + } + + # Copied from transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel._reorder_cache + @staticmethod + def _reorder_cache(past: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor) -> Tuple[Tuple[torch.Tensor]]: + return tuple( + tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past) + for layer_past in past + ) + + def can_generate(self): + """Returns True to validate the check that the model using `GenerationMixin.generate()` can indeed generate.""" + return True From a4c5326e38f8864bac53122730fe39ccfdd6a90b Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Wed, 13 Mar 2024 20:27:59 +0530 Subject: [PATCH 02/28] restructure --- optimum/amd/ryzenai/modeling.py | 15 +- optimum/amd/ryzenai/modeling_decoder.py | 182 ++++++++++++------------ 2 files changed, 94 insertions(+), 103 deletions(-) diff --git a/optimum/amd/ryzenai/modeling.py b/optimum/amd/ryzenai/modeling.py index 8ccfc6e9..8c739803 100644 --- a/optimum/amd/ryzenai/modeling.py +++ b/optimum/amd/ryzenai/modeling.py @@ -42,14 +42,6 @@ CONFIG_NAME = "config.json" -class classproperty: - def __init__(self, getter): - self.getter = getter - - def __get__(self, instance, owner): - return self.getter(owner) - - class RyzenAIModel(OptimizedModel): """ Base class for implementing models using ONNX Runtime. @@ -126,6 +118,7 @@ def __init__( self.model_path = Path(model._model_path) self.model_name = self.model_path.name + self.model_type = config.model_type self.vaip_config = Path(vaip_config) if vaip_config else None self.shared_attributes_init( @@ -246,6 +239,7 @@ def _from_pretrained( session_options: Optional[ort.SessionOptions] = None, provider_options: Optional[Dict[str, Any]] = None, model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, + init_cls: Optional["RyzenAIModel"] = None, **kwargs, ) -> "RyzenAIModel": model_path = Path(model_id) @@ -353,7 +347,10 @@ def _from_pretrained( if model_save_dir is None: model_save_dir = new_model_save_dir - return cls( + if init_cls is None: + init_cls = cls + + return init_cls( model=model, config=config, vaip_config=vaip_config, diff --git a/optimum/amd/ryzenai/modeling_decoder.py b/optimum/amd/ryzenai/modeling_decoder.py index 18dd806e..7e13b29b 100644 --- a/optimum/amd/ryzenai/modeling_decoder.py +++ b/optimum/amd/ryzenai/modeling_decoder.py @@ -2,44 +2,43 @@ # Licensed under the MIT License. """RyzenAIModelForXXX classes, allowing to run ONNX Models with ONNX Runtime VITIS-AI EP using the same API as Transformers.""" -import logging -import os -import shutil from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union -import onnx +import numpy as np import onnxruntime as ort import torch -from huggingface_hub import HfApi, HfFolder, hf_hub_download -from huggingface_hub.utils import EntryNotFoundError -from onnx import shape_inference -from onnx.tools import update_model_dims - -from optimum.exporters import TasksManager -from optimum.modeling_base import FROM_PRETRAINED_START_DOCSTRING, OptimizedModel -from optimum.onnx.utils import _get_external_data_paths -from optimum.utils.save_utils import maybe_load_preprocessors + +from optimum.utils import NormalizedConfigManager, check_if_transformers_greater from transformers import ( - AutoConfig, AutoModel, - AutoModelForImageClassification, + GenerationConfig, PretrainedConfig, ) -from transformers.file_utils import add_start_docstrings -from transformers.modeling_outputs import ImageClassifierOutput, ModelOutput +from transformers.modeling_outputs import CausalLMOutputWithPast -from .utils import ( - ONNX_WEIGHTS_NAME, - ONNX_WEIGHTS_NAME_STATIC, - validate_provider_availability, -) +from .modeling import RyzenAIModel + + +if TYPE_CHECKING: + from transformers import PretrainedConfig +if check_if_transformers_greater("4.25.0"): + from transformers.generation import GenerationMixin +else: + from transformers.generation_utils import GenerationMixin -logger = logging.getLogger(__name__) -CONFIG_NAME = "config.json" +model_types = {} +# model_types = { +# "bloom": RyzenAIBloomForCausalLM, +# "falcon": RyzenAIFalconForCausalLM, +# "mpt": RyzenAIMPTForCausalLM, +# "opt": RyzenAIOptForCausalLM, +# "gpt_bigcode": RyzenAIGPTBigCodeForCausalLM, +# } + class RyzenAIModelForCausalLM(RyzenAIModel, GenerationMixin): """ @@ -61,78 +60,91 @@ def __init__( **kwargs, ): super().__init__(model, config, vaip_config, model_save_dir, preprocessors, **kwargs) - - self.num_pkv = 2 - self.normalized_config = NormalizedConfigManager.get_normalized_config_class(config.model_type)(config) - self.key_value_input_names = [key for key in self.inputs_names if (".key" in key) or (".value" in key)] - self.key_value_output_names = [key for key in self.output_names if (".key" in key) or (".value" in key)] - self.use_cache = len(self.key_value_input_names) > 0 - if generation_config is None: - generation_config = GenerationConfig.from_model_config(config) - self.generation_config = generation_config + self._initialize_params(use_cache, generation_config) if use_cache ^ self.use_cache: raise ValueError( f"`use_cache` was set to `{use_cache}` but the loaded model only supports `use_cache={self.use_cache}`. " f"Please load your current model with `use_cache={self.use_cache}` or export the original model " f"once again with `use_cache={use_cache}` when calling the `from_pretrained` method. " - "To export your model, simply set `export=True`." ) - def forward( - self, - input_ids: torch.LongTensor, - attention_mask: Optional[torch.FloatTensor] = None, - position_ids: Optional[torch.LongTensor] = None, - past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None, - **kwargs, - ) -> CausalLMOutputWithPast: + def _get_key_value_names(self): + key_names = [key for key in self.inputs_names if (".key" in key) or (".value" in key)] + value_names = [key for key in self.output_names if (".key" in key) or (".value" in key)] + return key_names, value_names + + def _initialize_params(self, use_cache, generation_config): + self.num_pkv = 2 + self.normalized_config = NormalizedConfigManager.get_normalized_config_class(self.config.model_type)( + self.config + ) + self.key_value_input_names, self.key_value_output_names = self._get_key_value_names() + self.use_cache = len(self.key_value_input_names) > 0 + self.generation_config = generation_config or GenerationConfig.from_model_config(self.config) + self._validate_cache_param(use_cache) + + def forward(self, input_ids, attention_mask=None, position_ids=None, past_key_values=None, **kwargs): use_torch = isinstance(input_ids, torch.Tensor) - inputs = {} - known_output_shapes = {} + inputs, known_output_shapes = self._prepare_inputs( + input_ids, attention_mask, position_ids, past_key_values, use_torch + ) + + # run inference + outputs = self.model.run(None, inputs) + + logits, past_key_values = self._process_outputs(outputs, use_torch) + return CausalLMOutputWithPast(loss=None, logits=logits, past_key_values=past_key_values) + + def _prepare_inputs(self, input_ids, attention_mask, position_ids, past_key_values, use_torch): + inputs = {"input_ids": self._convert_to_numpy(input_ids, use_torch)} if self.use_cache: - # Create dummy past_key_values for decoder first generation step if none given - use_cache_branch, past_key_values, known_output_shapes = self.prepare_past_key_values( - input_ids, past_key_values, use_torch - ) + use_cache_branch, past_key_values = self.prepare_past_key_values(input_ids, past_key_values, use_torch) - inputs["input_ids"] = input_ids.cpu().detach().numpy() if use_torch else input_ids + for name in ["attention_mask", "position_ids"]: + if name in self.inputs_names: + inputs[name] = self._convert_to_numpy(locals()[name], use_torch) if "attention_mask" in self.inputs_names: - inputs["attention_mask"] = attention_mask.cpu().detach().numpy() if use_torch else attention_mask + inputs["attention_mask"] = self._convert_to_numpy(attention_mask, use_torch) if "position_ids" in self.inputs_names: if position_ids is None: raise ValueError("position_ids was not passed but is a required input for this ONNX model.") - inputs["position_ids"] = position_ids.cpu().detach().numpy() if use_torch else position_ids + inputs["position_ids"] = self._convert_to_numpy(attention_mask, position_ids) - # Add the past_key_values to the decoder inputs if past_key_values is not None: for input_name, past_key_value in zip(self.key_value_input_names, past_key_values): - inputs[input_name] = past_key_value.cpu().detach().numpy() if use_torch else past_key_value + inputs[input_name] = self._convert_to_numpy(past_key_value, use_torch) - outputs = self.model.run(None, inputs) + return inputs + + def _process_outputs(self, outputs, use_torch): + logits = self._convert_to_tensor(outputs[self.output_names["logits"]], use_torch) if self.use_cache: - # Tuple of length equal to : number of layer * number of past_key_value per decoder layer (2 for the self-attention) past_key_values = tuple( - torch.from_numpy(outputs[self.output_names[key]]).to(self.device) + self._convert_to_tensor(outputs[self.output_names[key]], use_torch) for key in self.key_value_output_names ) - logits = torch.from_numpy(outputs[self.output_names["logits"]]).to(self.device) + if self.model_type != "gpt_bigcode": + past_key_values = tuple( + past_key_values[i : i + self.num_pkv] for i in range(0, len(past_key_values), self.num_pkv) + ) + else: + past_key_values = None - if self.use_cache and self.model_type != "gpt_bigcode": - # Tuple of tuple of length `n_layers`, with each tuple of length equal to the number of self-attention and - # per decoder layer - past_key_values = tuple( - past_key_values[i : i + self.num_pkv] for i in range(0, len(past_key_values), self.num_pkv) - ) + return logits, past_key_values - return CausalLMOutputWithPast(loss=loss, logits=logits, past_key_values=past_key_values) + def _convert_to_numpy(self, value, use_torch): + return value.cpu().detach().numpy() if use_torch else value + + def _convert_to_tensor(self, value, use_torch): + return torch.from_numpy(value).to(self.device) if use_torch else torch.from_numpy(value) def prepare_past_key_values( self, @@ -141,13 +153,12 @@ def prepare_past_key_values( use_torch: bool, ): if past_key_values is not None: - return past_key_values, {} + return past_key_values sequence_length = input_ids.shape[1] constructor = torch if use_torch else np - pkv_output_shape = {} # Generate dummy past for the first forward batch_size = input_ids.shape[0] embed_size_per_head = self.normalized_config.hidden_size // self.normalized_config.num_attention_heads @@ -174,9 +185,8 @@ def prepare_past_key_values( for name, value in zip(self.key_value_output_names, past_key_values): shape = [*value.shape] shape[2] += sequence_length - pkv_output_shape[name] = shape - return past_key_values, pkv_output_shape + return past_key_values @classmethod def _from_pretrained( @@ -195,44 +205,32 @@ def _from_pretrained( session_options: Optional[ort.SessionOptions] = None, provider_options: Optional[Dict[str, Any]] = None, model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, - init_cls: Optional[RyzenAIModel] = None, + init_cls: Optional["RyzenAIModel"] = None, **kwargs, - ) -> "RyzenAIModelForCausalLM": - if config.model_type == "bloom": - init_cls = RyzenAIBloomForCausalLM - elif config.model_type == "falcon": - init_cls = RyzenAIFalconForCausalLM - elif config.model_type == "mpt": - init_cls = RyzenAIMPTForCausalLM - elif config.model_type == "opt": - init_cls = RyzenAIOptForCausalLM - elif config.model_type == "gpt_bigcode": - init_cls = RyzenAIGPTBigCodeForCausalLM - else: - init_cls = RyzenAIModelForCausalLM + ) -> RyzenAIModel: + if init_cls is None: + init_cls = model_types.get(config.model_type, RyzenAIModelForCausalLM) model = super()._from_pretrained( + cls, model_id, config, + vaip_config, use_auth_token, revision, force_download, cache_dir, file_name, subfolder, - use_cache, local_files_only, - use_merged, provider, session_options, provider_options, - use_io_binding, model_save_dir, init_cls, **kwargs, ) - return model # Adapted from transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel.prepare_inputs_for_generation @@ -247,22 +245,18 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwarg remove_prefix_length = input_ids.shape[1] - 1 input_ids = input_ids[:, remove_prefix_length:] - # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly attention_mask = kwargs.get("attention_mask", None) - use_cache = kwargs.get("use_cache", None) - position_ids = kwargs.get("position_ids", None) + if attention_mask is not None and position_ids is None: - # create position_ids on the fly for batch generation position_ids = attention_mask.long().cumsum(-1) - 1 position_ids.masked_fill_(attention_mask == 0, 1) - if past_key_values: - position_ids = position_ids[:, -1].unsqueeze(-1) + position_ids = position_ids[:, -1].unsqueeze(-1) if past_key_values else position_ids return { "input_ids": input_ids, "past_key_values": past_key_values, - "use_cache": use_cache, + "use_cache": kwargs.get("use_cache", None), "position_ids": position_ids, "attention_mask": attention_mask, } From 449e32bdaed6e42fb9a0e30f8eb0b64fec83e1de Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Thu, 14 Mar 2024 18:33:30 +0530 Subject: [PATCH 03/28] update decoder --- optimum/amd/ryzenai/modeling.py | 6 + optimum/amd/ryzenai/modeling_decoder.py | 225 +++++++++++++++++------- 2 files changed, 169 insertions(+), 62 deletions(-) diff --git a/optimum/amd/ryzenai/modeling.py b/optimum/amd/ryzenai/modeling.py index 8c739803..a27b9963 100644 --- a/optimum/amd/ryzenai/modeling.py +++ b/optimum/amd/ryzenai/modeling.py @@ -566,6 +566,12 @@ def reshape( return model_path + def _convert_to_numpy(self, value, use_torch): + return value.cpu().detach().numpy() if use_torch else value + + def _convert_to_tensor(self, value, use_torch): + return torch.from_numpy(value) if use_torch else torch.from_numpy(value) + class RyzenAIModelForCustomTasks(RyzenAIModel): def forward(self, **kwargs): diff --git a/optimum/amd/ryzenai/modeling_decoder.py b/optimum/amd/ryzenai/modeling_decoder.py index 7e13b29b..dc6349f6 100644 --- a/optimum/amd/ryzenai/modeling_decoder.py +++ b/optimum/amd/ryzenai/modeling_decoder.py @@ -7,12 +7,12 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import numpy as np -import onnxruntime as ort +import ort as ort import torch from optimum.utils import NormalizedConfigManager, check_if_transformers_greater from transformers import ( - AutoModel, + AutoModelForCausalLM, GenerationConfig, PretrainedConfig, ) @@ -30,23 +30,13 @@ from transformers.generation_utils import GenerationMixin -model_types = {} -# model_types = { -# "bloom": RyzenAIBloomForCausalLM, -# "falcon": RyzenAIFalconForCausalLM, -# "mpt": RyzenAIMPTForCausalLM, -# "opt": RyzenAIOptForCausalLM, -# "gpt_bigcode": RyzenAIGPTBigCodeForCausalLM, -# } - - class RyzenAIModelForCausalLM(RyzenAIModel, GenerationMixin): """ Runs model with causal language modeling head using ONNX Runtime VITIS-AI EP. """ model_type = "onnx_model" - auto_model_class = AutoModel + auto_model_class = AutoModelForCausalLM def __init__( self, @@ -83,14 +73,11 @@ def _initialize_params(self, use_cache, generation_config): self.key_value_input_names, self.key_value_output_names = self._get_key_value_names() self.use_cache = len(self.key_value_input_names) > 0 self.generation_config = generation_config or GenerationConfig.from_model_config(self.config) - self._validate_cache_param(use_cache) def forward(self, input_ids, attention_mask=None, position_ids=None, past_key_values=None, **kwargs): use_torch = isinstance(input_ids, torch.Tensor) - inputs, known_output_shapes = self._prepare_inputs( - input_ids, attention_mask, position_ids, past_key_values, use_torch - ) + inputs = self._prepare_inputs(input_ids, attention_mask, position_ids, past_key_values, use_torch) # run inference outputs = self.model.run(None, inputs) @@ -101,22 +88,17 @@ def forward(self, input_ids, attention_mask=None, position_ids=None, past_key_va def _prepare_inputs(self, input_ids, attention_mask, position_ids, past_key_values, use_torch): inputs = {"input_ids": self._convert_to_numpy(input_ids, use_torch)} - if self.use_cache: - use_cache_branch, past_key_values = self.prepare_past_key_values(input_ids, past_key_values, use_torch) - - for name in ["attention_mask", "position_ids"]: - if name in self.inputs_names: - inputs[name] = self._convert_to_numpy(locals()[name], use_torch) - if "attention_mask" in self.inputs_names: inputs["attention_mask"] = self._convert_to_numpy(attention_mask, use_torch) if "position_ids" in self.inputs_names: if position_ids is None: - raise ValueError("position_ids was not passed but is a required input for this ONNX model.") + raise ValueError("`position_ids` was not passed but is a required input for this ONNX model.") inputs["position_ids"] = self._convert_to_numpy(attention_mask, position_ids) - if past_key_values is not None: + if self.use_cache: + past_key_values = self.prepare_past_key_values(input_ids, past_key_values, use_torch) + for input_name, past_key_value in zip(self.key_value_input_names, past_key_values): inputs[input_name] = self._convert_to_numpy(past_key_value, use_torch) @@ -125,27 +107,19 @@ def _prepare_inputs(self, input_ids, attention_mask, position_ids, past_key_valu def _process_outputs(self, outputs, use_torch): logits = self._convert_to_tensor(outputs[self.output_names["logits"]], use_torch) + past_key_values = None if self.use_cache: past_key_values = tuple( self._convert_to_tensor(outputs[self.output_names[key]], use_torch) for key in self.key_value_output_names ) - if self.model_type != "gpt_bigcode": - past_key_values = tuple( - past_key_values[i : i + self.num_pkv] for i in range(0, len(past_key_values), self.num_pkv) - ) - else: - past_key_values = None + past_key_values = tuple( + past_key_values[i : i + self.num_pkv] for i in range(0, len(past_key_values), self.num_pkv) + ) return logits, past_key_values - def _convert_to_numpy(self, value, use_torch): - return value.cpu().detach().numpy() if use_torch else value - - def _convert_to_tensor(self, value, use_torch): - return torch.from_numpy(value).to(self.device) if use_torch else torch.from_numpy(value) - def prepare_past_key_values( self, input_ids: Union[None, torch.LongTensor, np.ndarray], @@ -153,38 +127,37 @@ def prepare_past_key_values( use_torch: bool, ): if past_key_values is not None: - return past_key_values + constructor = torch if use_torch else np + dtype = constructor.float16 if self.use_fp16 else constructor.float32 + + params = self.get_params_from_normalized_config() + params["batch_size"] = input_ids.shape[0] + params["sequence_length"] = input_ids.shape[1] - sequence_length = input_ids.shape[1] + # Generate dummy past for the first forward + past_key_values = self.generate_past_key_values(constructor, params, dtype) - constructor = torch if use_torch else np + return past_key_values - # Generate dummy past for the first forward - batch_size = input_ids.shape[0] + def get_params_from_normalized_config(self): + num_attention_heads = self.normalized_config.num_attention_heads embed_size_per_head = self.normalized_config.hidden_size // self.normalized_config.num_attention_heads - if self.model_type == "gemma": - num_attention_heads = self.normalized_config.num_key_value_heads - embed_size_per_head = self.normalized_config.head_dim - elif self.model_type in {"mistral", "llama"}: - num_attention_heads = self.normalized_config.num_key_value_heads - else: - num_attention_heads = self.normalized_config.num_attention_heads - dtype = constructor.float16 if self.use_fp16 else constructor.float32 + return { + "num_key_value_heads": num_attention_heads, + "embed_size_per_head": embed_size_per_head, + } - num_key_value_heads = num_attention_heads + def generate_past_key_values(self, constructor: Any, params: Dict[str, int], dtype: Any): + shapes = self.get_params_from_normalized_config() + shape = (params["batch_size"], shapes["num_key_value_heads"], 0, shapes["embed_size_per_head"]) - shape = (batch_size, num_key_value_heads, 0, embed_size_per_head) key_or_value = constructor.zeros(shape, dtype=dtype) - - if use_torch: - key_or_value = key_or_value.to(self.device) - past_key_values = tuple(key_or_value for _ in range(len(self.key_value_input_names))) - for name, value in zip(self.key_value_output_names, past_key_values): + for _, value in zip(self.key_value_output_names, past_key_values): shape = [*value.shape] - shape[2] += sequence_length + shape[2] += params["sequence_length"] return past_key_values @@ -192,7 +165,7 @@ def prepare_past_key_values( def _from_pretrained( cls, model_id: Union[str, Path], - config: PretrainedConfig, + config: "PretrainedConfig", vaip_config: Optional[str] = None, use_auth_token: Optional[Union[bool, str]] = None, revision: Optional[str] = None, @@ -209,7 +182,7 @@ def _from_pretrained( **kwargs, ) -> RyzenAIModel: if init_cls is None: - init_cls = model_types.get(config.model_type, RyzenAIModelForCausalLM) + init_cls = model_type_to_class.get(config.model_type, RyzenAIModelForCausalLM) model = super()._from_pretrained( cls, @@ -265,10 +238,138 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwarg @staticmethod def _reorder_cache(past: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor) -> Tuple[Tuple[torch.Tensor]]: return tuple( - tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past) + tuple(past_state.index_select(0, beam_idx.to(past_state)) for past_state in layer_past) for layer_past in past ) def can_generate(self): """Returns True to validate the check that the model using `GenerationMixin.generate()` can indeed generate.""" return True + + +class RyzenAIMistralForCausalLM(RyzenAIModelForCausalLM): + def get_params_from_normalized_config(self): + num_key_value_heads = self.normalized_config.num_key_value_heads + embed_size_per_head = self.normalized_config.hidden_size // self.normalized_config.num_attention_heads + return { + "num_key_value_heads": num_key_value_heads, + "embed_size_per_head": embed_size_per_head, + } + + +class RyzenAILlamaForCausalLM(RyzenAIModelForCausalLM): + def get_params_from_normalized_config(self): + num_key_value_heads = self.normalized_config.num_key_value_heads + embed_size_per_head = self.normalized_config.hidden_size // self.normalized_config.num_attention_heads + return { + "num_key_value_heads": num_key_value_heads, + "embed_size_per_head": embed_size_per_head, + } + + +class RyzenAIOPTForCausalLM(RyzenAIModelForCausalLM): + # Adapted from transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel.prepare_inputs_for_generation + def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs): + if past_key_values is not None: + past_length = past_key_values[0][0].shape[2] + # Some generation methods already pass only the last input ID + if input_ids.shape[1] > past_length: + remove_prefix_length = past_length + else: + # Default to old behavior: keep only final ID + remove_prefix_length = input_ids.shape[1] - 1 + input_ids = input_ids[:, remove_prefix_length:] + + attention_mask = kwargs.get("attention_mask", None) + use_cache = kwargs.get("use_cache", None) + + return { + "input_ids": input_ids, + "past_key_values": past_key_values, + "use_cache": use_cache, + "position_ids": None, + "attention_mask": attention_mask, + } + + +class RyzenAIGPTBigCodeForCausalLM(RyzenAIModelForCausalLM): + def _process_outputs(self, outputs, use_torch): + logits = self._convert_to_tensor(outputs[self.output_names["logits"]], use_torch) + + past_key_values = None + if self.use_cache: + past_key_values = tuple( + self._convert_to_tensor(outputs[self.output_names[key]], use_torch) + for key in self.key_value_output_names + ) + + return logits, past_key_values + + def generate_past_key_values(self, constructor: Any, params: Dict[str, int], dtype: Any): + # GPT BigCode uses muti-query attention, and has the specificity of putting both key and value in the same cache tensor. + shape_key_and_value = (params["batch_size"], 0, params["embed_size_per_head"] * 2) + key_and_value = constructor.zeros(shape_key_and_value, dtype=dtype) + + past_key_values = tuple(key_and_value for _ in range(len(self.key_value_input_names))) + + for _, value in zip(self.key_value_output_names, past_key_values): + shape = [*value.shape] + shape[1] += params["sequence_length"] + + return past_key_values + + # Adapted from transformers.models.gpt_bigcode.modeling_gpt_bigcode.GPTBigCodeForCausalLM.prepare_inputs_for_generation + def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs): + # Omit tokens covered by past_key_values + if past_key_values: + if self.config.multi_query: + past_length = past_key_values[0].shape[1] + else: + past_length = past_key_values[0].shape[2] + + # Some generation methods already pass only the last input ID + if input_ids.shape[1] > past_length: + remove_prefix_length = past_length + else: + # Default to old behavior: keep only final ID + remove_prefix_length = input_ids.shape[1] - 1 + + input_ids = input_ids[:, remove_prefix_length:] + + attention_mask = kwargs.get("attention_mask", None) + position_ids = kwargs.get("position_ids", None) + + if attention_mask is not None and position_ids is None: + # create position_ids on the fly for batch generation + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + if past_key_values: + position_ids = position_ids[:, -input_ids.shape[1] :] + else: + position_ids = None + + model_inputs = {"input_ids": input_ids} + model_inputs.update( + { + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + "position_ids": position_ids, + "attention_mask": attention_mask, + } + ) + return model_inputs + + # Copied from transformers.models.gpt_bigcode.modeling_gpt_bigcode.GPTBigCodeForCausalLM._reorder_cache + @staticmethod + def _reorder_cache( + past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor + ) -> Tuple[Tuple[torch.Tensor]]: + return tuple(layer_past.index_select(0, beam_idx) for layer_past in past_key_values) + + +model_type_to_class = { + "opt": RyzenAIOPTForCausalLM, + "gpt_bigcode": RyzenAIGPTBigCodeForCausalLM, + "mistral": RyzenAIMistralForCausalLM, + "llama": RyzenAILlamaForCausalLM, +} From 1ca2eb7a21efc67faa28063891ee7f4db44931cd Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Thu, 14 Mar 2024 18:43:20 +0530 Subject: [PATCH 04/28] update decoder --- optimum/amd/ryzenai/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/optimum/amd/ryzenai/__init__.py b/optimum/amd/ryzenai/__init__.py index 79e98198..9cd8a0c6 100644 --- a/optimum/amd/ryzenai/__init__.py +++ b/optimum/amd/ryzenai/__init__.py @@ -16,6 +16,9 @@ "RyzenAIModelForImageToImage", "RyzenAIModelForObjectDetection", ], + "modeling_decoder": [ + "RyzenAIDecoder", + ], "quantization": ["RyzenAIOnnxQuantizer"], "pipelines": ["pipeline"], "version": ["__version__"], @@ -33,9 +36,9 @@ RyzenAIModelForImageToImage, RyzenAIModelForObjectDetection, ) + from .modeling_decoder import RyzenAIDecoder from .pipelines import pipeline from .quantization import RyzenAIOnnxQuantizer - from .version import __version__ else: import sys From 3e93153f96feca1fef6e4ff87f33a45d35d0b251 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Thu, 14 Mar 2024 18:55:37 +0530 Subject: [PATCH 05/28] fix import --- optimum/amd/ryzenai/modeling_decoder.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/optimum/amd/ryzenai/modeling_decoder.py b/optimum/amd/ryzenai/modeling_decoder.py index dc6349f6..d4f67a59 100644 --- a/optimum/amd/ryzenai/modeling_decoder.py +++ b/optimum/amd/ryzenai/modeling_decoder.py @@ -14,7 +14,6 @@ from transformers import ( AutoModelForCausalLM, GenerationConfig, - PretrainedConfig, ) from transformers.modeling_outputs import CausalLMOutputWithPast @@ -41,7 +40,7 @@ class RyzenAIModelForCausalLM(RyzenAIModel, GenerationMixin): def __init__( self, model: ort.InferenceSession, - config: PretrainedConfig, + config: "PretrainedConfig", vaip_config: Union[str, Path] = None, model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, preprocessors: Optional[List] = None, From df893ab9df99c0859bc21b3aca6a736e76a0c52e Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Thu, 14 Mar 2024 19:04:19 +0530 Subject: [PATCH 06/28] fix tests --- optimum/amd/ryzenai/modeling.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/optimum/amd/ryzenai/modeling.py b/optimum/amd/ryzenai/modeling.py index a27b9963..e295a5b3 100644 --- a/optimum/amd/ryzenai/modeling.py +++ b/optimum/amd/ryzenai/modeling.py @@ -118,7 +118,6 @@ def __init__( self.model_path = Path(model._model_path) self.model_name = self.model_path.name - self.model_type = config.model_type self.vaip_config = Path(vaip_config) if vaip_config else None self.shared_attributes_init( @@ -128,6 +127,8 @@ def __init__( **kwargs, ) + self.model_type = config.model_type + self.inputs_names = {input_key.name: idx for idx, input_key in enumerate(model.get_inputs())} self.output_names = {output_key.name: idx for idx, output_key in enumerate(model.get_outputs())} From 29dd6d563cce0a4d73e4748e6f3844c9b7c3bb52 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Fri, 15 Mar 2024 20:57:07 +0530 Subject: [PATCH 07/28] fixed errors --- .../quantization/brevitas/quantize_llm.py | 9 +- optimum/amd/ryzenai/__init__.py | 5 +- optimum/amd/ryzenai/modeling.py | 10 +-- optimum/amd/ryzenai/modeling_decoder.py | 89 ++++++++++++------- tests/ryzenai/vaip_config.json | 23 ++++- 5 files changed, 89 insertions(+), 47 deletions(-) diff --git a/examples/quantization/brevitas/quantize_llm.py b/examples/quantization/brevitas/quantize_llm.py index 7f4ebcf1..9e441306 100644 --- a/examples/quantization/brevitas/quantize_llm.py +++ b/examples/quantization/brevitas/quantize_llm.py @@ -1,8 +1,10 @@ +import os from argparse import ArgumentParser import torch from brevitas.export.onnx.standard.qcdq.manager import StdQCDQONNXManager from brevitas_examples.llm.llm_quant.export import brevitas_proxy_export_mode +from rewriter import find_and_insert_matmulinteger from optimum.amd import BrevitasQuantizationConfig, BrevitasQuantizer from optimum.amd.brevitas.accelerate_utils import calc_cpu_device_map, calc_gpu_device_map, offload_model, remove_hooks @@ -81,15 +83,16 @@ def main(args): # Export to ONNX through optimum.exporters. export_manager = StdQCDQONNXManager - export_manager.change_weight_export(export_weight_q_node=True) + # export_manager.change_weight_export(export_weight_q_node=True) with torch.no_grad(), brevitas_proxy_export_mode(quantized_model, export_manager=export_manager): onnx_export_from_model( quantized_model, args.onnx_output_path, task="text-generation-with-past", do_validation=False, - no_post_process=True, - ) + no_post_process=True) + + find_and_insert_matmulinteger(os.path.join(args.onnx_output_path, 'model.onnx')) return return_val diff --git a/optimum/amd/ryzenai/__init__.py b/optimum/amd/ryzenai/__init__.py index 9cd8a0c6..ee185c5f 100644 --- a/optimum/amd/ryzenai/__init__.py +++ b/optimum/amd/ryzenai/__init__.py @@ -17,11 +17,10 @@ "RyzenAIModelForObjectDetection", ], "modeling_decoder": [ - "RyzenAIDecoder", + "RyzenAIModelForCausalLM", ], "quantization": ["RyzenAIOnnxQuantizer"], "pipelines": ["pipeline"], - "version": ["__version__"], } @@ -36,7 +35,7 @@ RyzenAIModelForImageToImage, RyzenAIModelForObjectDetection, ) - from .modeling_decoder import RyzenAIDecoder + from .modeling_decoder import RyzenAIModelForCausalLM from .pipelines import pipeline from .quantization import RyzenAIOnnxQuantizer else: diff --git a/optimum/amd/ryzenai/modeling.py b/optimum/amd/ryzenai/modeling.py index e295a5b3..49ae6af0 100644 --- a/optimum/amd/ryzenai/modeling.py +++ b/optimum/amd/ryzenai/modeling.py @@ -174,11 +174,11 @@ def load_model( else: providers_options = None - is_dynamic = RyzenAIModel._check_uses_static_shape(path) - if is_dynamic and provider == "VitisAIExecutionProvider": - raise ValueError( - "The model provided has dynamic axes in input/output. Please provide model with static shapes for inference with RyzenAI." - ) + # is_dynamic = RyzenAIModel._check_uses_static_shape(path) + # if is_dynamic and provider == "VitisAIExecutionProvider": + # raise ValueError( + # "The model provided has dynamic axes in input/output. Please provide model with static shapes for inference with RyzenAI." + # ) return ort.InferenceSession( path, diff --git a/optimum/amd/ryzenai/modeling_decoder.py b/optimum/amd/ryzenai/modeling_decoder.py index d4f67a59..9d8cfbe2 100644 --- a/optimum/amd/ryzenai/modeling_decoder.py +++ b/optimum/amd/ryzenai/modeling_decoder.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import numpy as np -import ort as ort +import onnxruntime as ort import torch from optimum.utils import NormalizedConfigManager, check_if_transformers_greater @@ -34,7 +34,7 @@ class RyzenAIModelForCausalLM(RyzenAIModel, GenerationMixin): Runs model with causal language modeling head using ONNX Runtime VITIS-AI EP. """ - model_type = "onnx_model" + main_input_name = "input_ids" auto_model_class = AutoModelForCausalLM def __init__( @@ -52,6 +52,12 @@ def __init__( self._initialize_params(use_cache, generation_config) + self.device = torch.device("cpu") + + use_cache = True + + self.use_fp16 = False + if use_cache ^ self.use_cache: raise ValueError( f"`use_cache` was set to `{use_cache}` but the loaded model only supports `use_cache={self.use_cache}`. " @@ -96,7 +102,12 @@ def _prepare_inputs(self, input_ids, attention_mask, position_ids, past_key_valu inputs["position_ids"] = self._convert_to_numpy(attention_mask, position_ids) if self.use_cache: - past_key_values = self.prepare_past_key_values(input_ids, past_key_values, use_torch) + if past_key_values is None: + # Generate dummy past for the first forward + batch_size, sequence_length = input_ids.shape + past_key_values = self.prepare_past_key_values(batch_size, sequence_length, use_torch) + else: + past_key_values = self.process_past_key_values(past_key_values) for input_name, past_key_value in zip(self.key_value_input_names, past_key_values): inputs[input_name] = self._convert_to_numpy(past_key_value, use_torch) @@ -119,26 +130,15 @@ def _process_outputs(self, outputs, use_torch): return logits, past_key_values - def prepare_past_key_values( - self, - input_ids: Union[None, torch.LongTensor, np.ndarray], - past_key_values: Union[None, Tuple[torch.FloatTensor], Tuple[np.ndarray]], - use_torch: bool, - ): - if past_key_values is not None: - constructor = torch if use_torch else np - dtype = constructor.float16 if self.use_fp16 else constructor.float32 - - params = self.get_params_from_normalized_config() - params["batch_size"] = input_ids.shape[0] - params["sequence_length"] = input_ids.shape[1] - - # Generate dummy past for the first forward - past_key_values = self.generate_past_key_values(constructor, params, dtype) + def process_past_key_values(self, past_key_values: Tuple[torch.Tensor]): + past_key_values = tuple( + past_key_value for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer + ) return past_key_values - def get_params_from_normalized_config(self): + def get_shape_params_from_normalized_config(self): + # Override this method in subclasses to adapt to the specific model's configuration num_attention_heads = self.normalized_config.num_attention_heads embed_size_per_head = self.normalized_config.hidden_size // self.normalized_config.num_attention_heads @@ -147,19 +147,31 @@ def get_params_from_normalized_config(self): "embed_size_per_head": embed_size_per_head, } - def generate_past_key_values(self, constructor: Any, params: Dict[str, int], dtype: Any): - shapes = self.get_params_from_normalized_config() - shape = (params["batch_size"], shapes["num_key_value_heads"], 0, shapes["embed_size_per_head"]) + def prepare_past_key_values( + self, + batch_size: int, + sequence_length: int, + use_torch: bool, + ): + # Override this method in subclasses to adapt to the specific model's configuration + params = self.get_shape_params_from_normalized_config() + key_or_value_shape = (batch_size, params["num_key_value_heads"], 0, params["embed_size_per_head"]) + + constructor, dtype = self.get_constructor(use_torch) + key_or_value = constructor.zeros(key_or_value_shape, dtype=dtype) - key_or_value = constructor.zeros(shape, dtype=dtype) past_key_values = tuple(key_or_value for _ in range(len(self.key_value_input_names))) - for _, value in zip(self.key_value_output_names, past_key_values): shape = [*value.shape] - shape[2] += params["sequence_length"] + shape[2] += sequence_length return past_key_values + def get_constructor(self, use_torch): + constructor = torch if use_torch else np + dtype = constructor.float16 if self.use_fp16 else constructor.float32 + return constructor, dtype + @classmethod def _from_pretrained( cls, @@ -184,7 +196,6 @@ def _from_pretrained( init_cls = model_type_to_class.get(config.model_type, RyzenAIModelForCausalLM) model = super()._from_pretrained( - cls, model_id, config, vaip_config, @@ -247,7 +258,7 @@ def can_generate(self): class RyzenAIMistralForCausalLM(RyzenAIModelForCausalLM): - def get_params_from_normalized_config(self): + def get_shape_params_from_normalized_config(self): num_key_value_heads = self.normalized_config.num_key_value_heads embed_size_per_head = self.normalized_config.hidden_size // self.normalized_config.num_attention_heads return { @@ -257,7 +268,7 @@ def get_params_from_normalized_config(self): class RyzenAILlamaForCausalLM(RyzenAIModelForCausalLM): - def get_params_from_normalized_config(self): + def get_shape_params_from_normalized_config(self): num_key_value_heads = self.normalized_config.num_key_value_heads embed_size_per_head = self.normalized_config.hidden_size // self.normalized_config.num_attention_heads return { @@ -304,16 +315,26 @@ def _process_outputs(self, outputs, use_torch): return logits, past_key_values - def generate_past_key_values(self, constructor: Any, params: Dict[str, int], dtype: Any): + def process_past_key_values(self, past_key_values: Tuple[torch.Tensor]): + return past_key_values + + def prepare_past_key_values( + self, + batch_size: int, + sequence_length: int, + use_torch: bool, + ): # GPT BigCode uses muti-query attention, and has the specificity of putting both key and value in the same cache tensor. - shape_key_and_value = (params["batch_size"], 0, params["embed_size_per_head"] * 2) - key_and_value = constructor.zeros(shape_key_and_value, dtype=dtype) + params = self.get_shape_params_from_normalized_config() + key_or_value_shape = (batch_size, 0, params["embed_size_per_head"] * 2) - past_key_values = tuple(key_and_value for _ in range(len(self.key_value_input_names))) + constructor, dtype = self.get_constructor(use_torch) + key_or_value = constructor.zeros(key_or_value_shape, dtype=dtype) + past_key_values = tuple(key_or_value for _ in range(len(self.key_value_input_names))) for _, value in zip(self.key_value_output_names, past_key_values): shape = [*value.shape] - shape[1] += params["sequence_length"] + shape[1] += sequence_length return past_key_values diff --git a/tests/ryzenai/vaip_config.json b/tests/ryzenai/vaip_config.json index a2debc82..fbca88f0 100644 --- a/tests/ryzenai/vaip_config.json +++ b/tests/ryzenai/vaip_config.json @@ -13,6 +13,24 @@ "methodName": "rules" } }, + { + "name": "fuse_softmax", + "plugin": "vaip-pass_py_ext", + "disabled": false, + "pyExt": { + "moduleName": "voe.passes.fuse_softmax", + "methodName": "rules" + } + }, + { + "name": "fuse_topk", + "plugin": "vaip-pass_py_ext", + "disabled": false, + "pyExt": { + "moduleName": "voe.passes.fuse_topk", + "methodName": "rules" + } + }, { "name": "fuse_decode_filter_boxes", "plugin": "vaip-pass_py_ext", @@ -266,7 +284,7 @@ "boolValue" : false }, "profile" : { - "boolValue" : false + "intValue" : 0 }, "prefetch" : { "boolValue" : false @@ -280,7 +298,8 @@ "concat_skip_code_gen" : { "boolValue" : false } - } + }, + "minimum_num_of_conv": 2 } } ] From efb2082a17342bb1d45685a08bde159519d59acc Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Mon, 18 Mar 2024 11:59:38 +0530 Subject: [PATCH 08/28] fixed errors --- optimum/amd/ryzenai/modeling.py | 45 ++++++++++++++-- optimum/amd/ryzenai/modeling_decoder.py | 70 +++++++++++++++---------- 2 files changed, 81 insertions(+), 34 deletions(-) diff --git a/optimum/amd/ryzenai/modeling.py b/optimum/amd/ryzenai/modeling.py index 49ae6af0..249d7a5d 100644 --- a/optimum/amd/ryzenai/modeling.py +++ b/optimum/amd/ryzenai/modeling.py @@ -224,7 +224,7 @@ def _generate_regular_names_for_filename(filename: str): return [filename, f"{name}_quantized.{extension}", f"{name}_optimized.{extension}"] @classmethod - def _from_pretrained( + def _load_model_and_processors( cls, model_id: Union[str, Path], config: PretrainedConfig, @@ -240,7 +240,6 @@ def _from_pretrained( session_options: Optional[ort.SessionOptions] = None, provider_options: Optional[Dict[str, Any]] = None, model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, - init_cls: Optional["RyzenAIModel"] = None, **kwargs, ) -> "RyzenAIModel": model_path = Path(model_id) @@ -348,10 +347,46 @@ def _from_pretrained( if model_save_dir is None: model_save_dir = new_model_save_dir - if init_cls is None: - init_cls = cls + return model, vaip_config, model_save_dir, preprocessors + + @classmethod + def _from_pretrained( + cls, + model_id: Union[str, Path], + config: PretrainedConfig, + vaip_config: Optional[str] = None, + use_auth_token: Optional[Union[bool, str]] = None, + revision: Optional[str] = None, + force_download: bool = False, + cache_dir: Optional[str] = None, + file_name: Optional[str] = None, + subfolder: str = "", + local_files_only: bool = False, + provider: str = "VitisAIExecutionProvider", + session_options: Optional[ort.SessionOptions] = None, + provider_options: Optional[Dict[str, Any]] = None, + model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, + **kwargs, + ) -> "RyzenAIModel": + model, vaip_config, model_save_dir, preprocessors = cls._load_model_and_processors( + model_id, + config, + vaip_config, + use_auth_token, + revision, + force_download, + cache_dir, + file_name, + subfolder, + local_files_only, + provider, + session_options, + provider_options, + model_save_dir, + **kwargs, + ) - return init_cls( + return cls( model=model, config=config, vaip_config=vaip_config, diff --git a/optimum/amd/ryzenai/modeling_decoder.py b/optimum/amd/ryzenai/modeling_decoder.py index 9d8cfbe2..02c6c1ff 100644 --- a/optimum/amd/ryzenai/modeling_decoder.py +++ b/optimum/amd/ryzenai/modeling_decoder.py @@ -1,6 +1,6 @@ # Copyright 2023 The HuggingFace Team. All rights reserved. # Licensed under the MIT License. -"""RyzenAIModelForXXX classes, allowing to run ONNX Models with ONNX Runtime VITIS-AI EP using the same API as Transformers.""" +"""RyzenAIModelForCausalLM classes, allowing to run ONNX Models with ONNX Runtime VITIS-AI EP using the same API as Transformers.""" from pathlib import Path from tempfile import TemporaryDirectory @@ -52,18 +52,16 @@ def __init__( self._initialize_params(use_cache, generation_config) - self.device = torch.device("cpu") - - use_cache = True - self.use_fp16 = False - - if use_cache ^ self.use_cache: - raise ValueError( - f"`use_cache` was set to `{use_cache}` but the loaded model only supports `use_cache={self.use_cache}`. " - f"Please load your current model with `use_cache={self.use_cache}` or export the original model " - f"once again with `use_cache={use_cache}` when calling the `from_pretrained` method. " - ) + for inp in model.get_inputs(): + if ( + inp.name == "past_key_values" or inp.name in self.key_value_input_names + ) and inp.type == "tensor(float16)": + self.use_fp16 = True + break + + # need for generate + self.device = torch.device("cpu") def _get_key_value_names(self): key_names = [key for key in self.inputs_names if (".key" in key) or (".value" in key)] @@ -79,6 +77,13 @@ def _initialize_params(self, use_cache, generation_config): self.use_cache = len(self.key_value_input_names) > 0 self.generation_config = generation_config or GenerationConfig.from_model_config(self.config) + if use_cache ^ self.use_cache: + raise ValueError( + f"`use_cache` was set to `{use_cache}` but the loaded model only supports `use_cache={self.use_cache}`. " + f"Please load your current model with `use_cache={self.use_cache}` or export the original model " + f"once again with `use_cache={use_cache}` when calling the `from_pretrained` method. " + ) + def forward(self, input_ids, attention_mask=None, position_ids=None, past_key_values=None, **kwargs): use_torch = isinstance(input_ids, torch.Tensor) @@ -107,14 +112,23 @@ def _prepare_inputs(self, input_ids, attention_mask, position_ids, past_key_valu batch_size, sequence_length = input_ids.shape past_key_values = self.prepare_past_key_values(batch_size, sequence_length, use_torch) else: - past_key_values = self.process_past_key_values(past_key_values) + past_key_values = self.process_input_past_key_values(past_key_values) for input_name, past_key_value in zip(self.key_value_input_names, past_key_values): inputs[input_name] = self._convert_to_numpy(past_key_value, use_torch) return inputs + def process_input_past_key_values(self, past_key_values: Tuple[torch.Tensor]): + # Override this method in subclasses to adapt to the specific model's configuration + past_key_values = tuple( + past_key_value for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer + ) + + return past_key_values + def _process_outputs(self, outputs, use_torch): + # Override this method in subclasses to adapt to the specific model's configuration logits = self._convert_to_tensor(outputs[self.output_names["logits"]], use_torch) past_key_values = None @@ -130,13 +144,6 @@ def _process_outputs(self, outputs, use_torch): return logits, past_key_values - def process_past_key_values(self, past_key_values: Tuple[torch.Tensor]): - past_key_values = tuple( - past_key_value for pkv_per_layer in past_key_values for past_key_value in pkv_per_layer - ) - - return past_key_values - def get_shape_params_from_normalized_config(self): # Override this method in subclasses to adapt to the specific model's configuration num_attention_heads = self.normalized_config.num_attention_heads @@ -189,13 +196,12 @@ def _from_pretrained( session_options: Optional[ort.SessionOptions] = None, provider_options: Optional[Dict[str, Any]] = None, model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, - init_cls: Optional["RyzenAIModel"] = None, + use_cache: bool = True, **kwargs, ) -> RyzenAIModel: - if init_cls is None: - init_cls = model_type_to_class.get(config.model_type, RyzenAIModelForCausalLM) + init_cls = model_type_to_class.get(config.model_type, RyzenAIModelForCausalLM) - model = super()._from_pretrained( + model, vaip_config, model_save_dir, preprocessors = cls._load_model_and_processors( model_id, config, vaip_config, @@ -210,11 +216,17 @@ def _from_pretrained( session_options, provider_options, model_save_dir, - init_cls, **kwargs, ) - return model + return init_cls( + model, + config=config, + vaip_config=vaip_config, + model_save_dir=model_save_dir, + preprocessors=preprocessors, + use_cache=use_cache, + ) # Adapted from transformers.models.gpt2.modeling_gpt2.GPT2LMHeadModel.prepare_inputs_for_generation def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwargs): @@ -303,6 +315,9 @@ def prepare_inputs_for_generation(self, input_ids, past_key_values=None, **kwarg class RyzenAIGPTBigCodeForCausalLM(RyzenAIModelForCausalLM): + def process_input_past_key_values(self, past_key_values: Tuple[torch.Tensor]): + return past_key_values + def _process_outputs(self, outputs, use_torch): logits = self._convert_to_tensor(outputs[self.output_names["logits"]], use_torch) @@ -315,9 +330,6 @@ def _process_outputs(self, outputs, use_torch): return logits, past_key_values - def process_past_key_values(self, past_key_values: Tuple[torch.Tensor]): - return past_key_values - def prepare_past_key_values( self, batch_size: int, From 268fbcc1fac8715a3b9e4979f5d6ceb53298764b Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Mon, 18 Mar 2024 17:19:18 +0530 Subject: [PATCH 09/28] set env varibales --- optimum/amd/ryzenai/modeling_decoder.py | 5 ++ optimum/amd/ryzenai/utils.py | 71 +++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/optimum/amd/ryzenai/modeling_decoder.py b/optimum/amd/ryzenai/modeling_decoder.py index 02c6c1ff..62724129 100644 --- a/optimum/amd/ryzenai/modeling_decoder.py +++ b/optimum/amd/ryzenai/modeling_decoder.py @@ -18,6 +18,7 @@ from transformers.modeling_outputs import CausalLMOutputWithPast from .modeling import RyzenAIModel +from .utils import set_builtins, set_environment_variables if TYPE_CHECKING: @@ -199,6 +200,10 @@ def _from_pretrained( use_cache: bool = True, **kwargs, ) -> RyzenAIModel: + # set environment variables + set_environment_variables() + set_builtins() + init_cls = model_type_to_class.get(config.model_type, RyzenAIModelForCausalLM) model, vaip_config, model_save_dir, preprocessors = cls._load_model_and_processors( diff --git a/optimum/amd/ryzenai/utils.py b/optimum/amd/ryzenai/utils.py index 395a0302..75f658ed 100644 --- a/optimum/amd/ryzenai/utils.py +++ b/optimum/amd/ryzenai/utils.py @@ -2,12 +2,27 @@ # Licensed under the MIT License. +import builtins +import logging +import os +import subprocess + import onnxruntime as ort +logger = logging.getLogger(__name__) + ONNX_WEIGHTS_NAME = "model.onnx" ONNX_WEIGHTS_NAME_STATIC = "model_static.onnx" +DEFAULT_TVM_GEMM_M = "1,8," +DEFAULT_TVM_DLL_NUM = "2" +DEFAULT_DEVICE = "phx" +DEFAULT_DLL_FILES = ["qlinear\\libGemmQnnAie_1x2048_2048x2048.dll", "qlinear\\libGemmQnnAie_8x2048_2048x2048.dll"] + +DEFAULT_BUILTIN_IMPL = "v0" +DEFAULT_BUILTIN_QUANT_MODE = "w8a8" + def validate_provider_availability(provider: str): """ @@ -21,3 +36,59 @@ def validate_provider_availability(provider: str): raise ValueError( f"Asked to use {provider} as an ONNX Runtime execution provider, but the available execution providers are {available_providers}." ) + + +def set_builtins(): + """Set the builtins.impl and builtins.quant_mode environment variables.""" + builtins.impl = os.getenv("BUILTINS_IMPL", DEFAULT_BUILTIN_IMPL) + builtins.quant_mode = os.getenv("BUILTINS_IMPL", DEFAULT_BUILTIN_QUANT_MODE) + + +def clone_repository(repo_url, repo_path): + if repo_path not in os.listdir(): + subprocess.run(["git", "clone", "--depth", "1", "--branch", "main", repo_url]) + + +def set_env_var(key, value): + if key not in os.environ: + os.environ[key] = value + + +def normalize_path(path): + return os.path.normpath(path) + + +def set_environment_variables(): + ryzenai_sw_path = os.environ.get("RYZENAI_SW_PATH") + if not ryzenai_sw_path: + logger.warning( + "RYZENAI_SW_PATH environment variable is not set. " + "Please set it to the path of the RyzenAI-SW repository. " + "Attempting to clone RyzenAI-SW repository now..." + ) + clone_repository("https://github.com/amd/RyzenAI-SW/", "RyzenAI-SW") + ryzenai_sw_path = normalize_path(os.path.join(os.getcwd(), "RyzenAI-SW")) + + # Set other environment variables + ryzenai_transformers_path = normalize_path(os.path.join(ryzenai_sw_path, "example/transformers")) + third_party = normalize_path(os.path.join(ryzenai_transformers_path, "third_party")) + + set_env_var("THIRD_PARTY", third_party) + set_env_var( + "TVM_LIBRARY_PATH", + f"{normalize_path(os.path.join(third_party, 'lib'))};{normalize_path(os.path.join(third_party, 'bin'))}", + ) + set_env_var("DEVICE", DEFAULT_DEVICE) + set_env_var( + "XLNX_VART_FIRMWARE", + normalize_path(os.path.join(ryzenai_transformers_path, "xclbin", os.environ.get("DEVICE", DEFAULT_DEVICE))), + ) + + dll_path = os.path.join(ryzenai_transformers_path, "dll", os.environ.get("DEVICE", DEFAULT_DEVICE)) + tvm_module_paths = [] + for dll_file in DEFAULT_DLL_FILES: + tvm_module_paths.append(normalize_path(os.path.join(dll_path, dll_file))) + + set_env_var("TVM_MODULE_PATH", ",".join(tvm_module_paths) + ",") + set_env_var("TVM_GEMM_M", DEFAULT_TVM_GEMM_M) + set_env_var("TVM_DLL_NUM", DEFAULT_TVM_DLL_NUM) From 7782a3528ed7dd366aba5a38257495f0ea32fddf Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Tue, 19 Mar 2024 14:05:38 +0530 Subject: [PATCH 10/28] add tests --- .gitignore | 2 + .../quantization/brevitas/quantize_llm.py | 9 +- optimum/amd/ryzenai/modeling_decoder.py | 2 +- optimum/amd/ryzenai/utils.py | 4 +- tests/ryzenai/operators_baseline.json | 15 + tests/ryzenai/rewriter.py | 242 +++++++++++ tests/ryzenai/test_modeling.py | 119 +++++- tests/ryzenai/test_quantization.py | 37 +- tests/ryzenai/testing_models.py | 366 ++++++++++++++++ tests/ryzenai/testing_utils.py | 391 ++---------------- tests/ryzenai/vaip_config_transformers.json | 35 ++ utils/ryzenai/generate_operators_baseline.py | 2 +- utils/ryzenai/notification_service.py | 2 +- 13 files changed, 825 insertions(+), 401 deletions(-) create mode 100644 tests/ryzenai/rewriter.py create mode 100644 tests/ryzenai/testing_models.py create mode 100644 tests/ryzenai/vaip_config_transformers.json diff --git a/.gitignore b/.gitignore index a1a258fb..7e34cbf6 100644 --- a/.gitignore +++ b/.gitignore @@ -166,3 +166,5 @@ cython_debug/ runs sweeps experiments +ryzen_cache +RyzenAI-SW \ No newline at end of file diff --git a/examples/quantization/brevitas/quantize_llm.py b/examples/quantization/brevitas/quantize_llm.py index 9e441306..7f4ebcf1 100644 --- a/examples/quantization/brevitas/quantize_llm.py +++ b/examples/quantization/brevitas/quantize_llm.py @@ -1,10 +1,8 @@ -import os from argparse import ArgumentParser import torch from brevitas.export.onnx.standard.qcdq.manager import StdQCDQONNXManager from brevitas_examples.llm.llm_quant.export import brevitas_proxy_export_mode -from rewriter import find_and_insert_matmulinteger from optimum.amd import BrevitasQuantizationConfig, BrevitasQuantizer from optimum.amd.brevitas.accelerate_utils import calc_cpu_device_map, calc_gpu_device_map, offload_model, remove_hooks @@ -83,16 +81,15 @@ def main(args): # Export to ONNX through optimum.exporters. export_manager = StdQCDQONNXManager - # export_manager.change_weight_export(export_weight_q_node=True) + export_manager.change_weight_export(export_weight_q_node=True) with torch.no_grad(), brevitas_proxy_export_mode(quantized_model, export_manager=export_manager): onnx_export_from_model( quantized_model, args.onnx_output_path, task="text-generation-with-past", do_validation=False, - no_post_process=True) - - find_and_insert_matmulinteger(os.path.join(args.onnx_output_path, 'model.onnx')) + no_post_process=True, + ) return return_val diff --git a/optimum/amd/ryzenai/modeling_decoder.py b/optimum/amd/ryzenai/modeling_decoder.py index 62724129..76599f5a 100644 --- a/optimum/amd/ryzenai/modeling_decoder.py +++ b/optimum/amd/ryzenai/modeling_decoder.py @@ -105,7 +105,7 @@ def _prepare_inputs(self, input_ids, attention_mask, position_ids, past_key_valu if "position_ids" in self.inputs_names: if position_ids is None: raise ValueError("`position_ids` was not passed but is a required input for this ONNX model.") - inputs["position_ids"] = self._convert_to_numpy(attention_mask, position_ids) + inputs["position_ids"] = self._convert_to_numpy(position_ids, use_torch) if self.use_cache: if past_key_values is None: diff --git a/optimum/amd/ryzenai/utils.py b/optimum/amd/ryzenai/utils.py index 75f658ed..71738f44 100644 --- a/optimum/amd/ryzenai/utils.py +++ b/optimum/amd/ryzenai/utils.py @@ -46,7 +46,7 @@ def set_builtins(): def clone_repository(repo_url, repo_path): if repo_path not in os.listdir(): - subprocess.run(["git", "clone", "--depth", "1", "--branch", "main", repo_url]) + subprocess.run(["git", "clone", "--depth", "1", "--branch", "main", repo_url, repo_path]) def set_env_var(key, value): @@ -66,8 +66,8 @@ def set_environment_variables(): "Please set it to the path of the RyzenAI-SW repository. " "Attempting to clone RyzenAI-SW repository now..." ) - clone_repository("https://github.com/amd/RyzenAI-SW/", "RyzenAI-SW") ryzenai_sw_path = normalize_path(os.path.join(os.getcwd(), "RyzenAI-SW")) + clone_repository("https://github.com/amd/RyzenAI-SW/", ryzenai_sw_path) # Set other environment variables ryzenai_transformers_path = normalize_path(os.path.join(ryzenai_sw_path, "example/transformers")) diff --git a/tests/ryzenai/operators_baseline.json b/tests/ryzenai/operators_baseline.json index ee166aa3..a337abea 100644 --- a/tests/ryzenai/operators_baseline.json +++ b/tests/ryzenai/operators_baseline.json @@ -89,6 +89,21 @@ "dpu": 546, "cpu": 13 }, + "echarlaix_tiny-random-mistral": { + "all": 394, + "cpu": 379, + "matmulinteger": 15 + }, + "fxmarty_tiny-llama-fast-tokenizer": { + "all": 342, + "cpu": 327, + "matmulinteger": 15 + }, + "hf-internal-testing_tiny-random-optforcausallm": { + "all": 658, + "cpu": 627, + "matmulinteger": 31 + }, "timm_botnet26t_256.c1_in1k": { "all": 635, "dpu": 0, diff --git a/tests/ryzenai/rewriter.py b/tests/ryzenai/rewriter.py new file mode 100644 index 00000000..2c32f69c --- /dev/null +++ b/tests/ryzenai/rewriter.py @@ -0,0 +1,242 @@ +import pathlib + +import onnx +from onnx_tool import Model +from onnx_tool.fusion import FusionPattern +from onnx_tool.node import create_node +from onnx_tool.tensor import Tensor + +from optimum.onnx.graph_transformations import check_and_save_model + + +## Pattern to find and replace with MatMulInteger +MatMul = [ + { + "name": "deq_linear_0", + "op": "DequantizeLinear", + "attrs": [], + "inport": [], + "outport": [[0, "transpose_0", 0]], + }, + { + "name": "transpose_0", + "op": "Transpose", + "attrs": [], + "inport": [[0, "deq_linear_0", 0]], + "outport": [[0, "matmul_0", 1]], + }, + { + "name": "quant_linear_1", + "op": "DynamicQuantizeLinear", + "attrs": [], + "inport": [], + "outport": [[0, "deq_linear_1", 0], [1, "deq_linear_1", 1], [2, "deq_linear_1", 2]], + }, + { + "name": "deq_linear_1", + "op": "DequantizeLinear", + "attrs": [], + "inport": [ + [0, "quant_linear_1", 0], + [1, "quant_linear_1", 1], + [2, "quant_linear_1", 2], + ], + "outport": [[0, "matmul_0", 0]], + }, + { + "name": "matmul_0", + "op": "MatMul", + "attrs": [], + "inport": [ + [0, "deq_linear_1", 0], + [1, "transpose_0", 0], + ], + "outport": [], + }, +] + +GEMM = [ + { + "name": "deq_linear_0", + "op": "DequantizeLinear", + "attrs": [], + "inport": [], + "outport": [[0, "gemm_0", 1]], + }, + { + "name": "quant_linear_1", + "op": "DynamicQuantizeLinear", + "attrs": [], + "inport": [], + "outport": [[0, "deq_linear_1", 0], [1, "deq_linear_1", 1], [2, "deq_linear_1", 2]], + }, + { + "name": "deq_linear_1", + "op": "DequantizeLinear", + "attrs": [], + "inport": [ + [0, "quant_linear_1", 0], + [1, "quant_linear_1", 1], + [2, "quant_linear_1", 2], + ], + "outport": [[0, "gemm_0", 0]], + }, + { + "name": "gemm_0", + "op": "Gemm", + "attrs": [], + "inport": [ + [0, "deq_linear_1", 0], + [1, "deq_linear_0", 0], + ], + "outport": [], + }, +] + + +def create_nodes(graph, op, name, inputs, outputs, intermediate=None, **kwargs): + if intermediate is None: + intermediate = [] + newnode = onnx.helper.make_node(op, inputs + intermediate, outputs, name=name, **kwargs) + newnode = create_node(newnode) + newnode.input = inputs + intermediate + newnode.output = outputs + for i in inputs: + if i in graph.consumedby: + graph.consumedby[i].append(name) + if i in graph.producedby.keys(): + newnode.prevnodes.append(graph.producedby[i]) + for o in outputs: + graph.producedby[o] = [name] + if o in graph.consumedby.keys(): + newnode.nextnodes.append(graph.consumedby[o]) + graph.nodemap[name] = newnode + graph.tensormap[name] = Tensor(name) + + return graph + + +def replace_matmul_to_matmulinteger(compute_graph, found_nodes): + for i, found_pattern in enumerate(found_nodes): + deq_linear = compute_graph.nodemap[found_pattern[0]] + dyn_q = compute_graph.nodemap[found_pattern[2]] + dq_weight = deq_linear.prevnodes[0] + compute_graph.add_initial(f"dq_weights_0_{i}", dq_weight.value.transpose()) + compute_graph.add_initial(f"dq_weights_1_{i}", deq_linear.prevnodes[1].value) + compute_graph.add_initial(f"dq_weights_2_{i}", deq_linear.prevnodes[2].value) + + matmul = compute_graph.nodemap[found_pattern[-1]] + for name in found_pattern: + if "DynamicQuantizeLinear" in name: + continue + compute_graph.remove_node(name) + + compute_graph.remove_node(deq_linear.prevnodes[0].name) + compute_graph.remove_node(deq_linear.prevnodes[1].name) + if deq_linear.prevnodes[2].name in compute_graph.nodemap: + compute_graph.remove_node(deq_linear.prevnodes[2].name) + + compute_graph = create_nodes( + compute_graph, + "MatMulInteger", + f"matmul_integer_{i}", + [dyn_q.output[0], f"dq_weights_0_{i}", dyn_q.output[2], f"dq_weights_2_{i}"], + [f"matmul_integer_{i}"], + ) + compute_graph = create_nodes( + compute_graph, "Cast", f"cast_{i}", [f"matmul_integer_{i}"], [f"cast_{i}"], to=int(1) + ) + compute_graph = create_nodes( + compute_graph, "Mul", f"mulscales_{i}", [dyn_q.output[1], f"dq_weights_1_{i}"], [f"mulscales_{i}"] + ) + compute_graph = create_nodes( + compute_graph, "Mul", f"mulvalues_{i}", [f"mulscales_{i}", f"cast_{i}"], [matmul.output[0]] + ) + return compute_graph + + +def replace_gemm_to_matmulinteger(compute_graph, found_nodes): + k = 100 + for i, found_pattern in enumerate(found_nodes): + k = i + 100 + gemm = compute_graph.nodemap[found_pattern[-1]] + bias = gemm.input[-1] + deq_linear = compute_graph.nodemap[found_pattern[0]] + dyn_q = compute_graph.nodemap[found_pattern[1]] + dq_weight = deq_linear.prevnodes[0] + compute_graph.add_initial(f"dq_weights_0_{k}", dq_weight.value.transpose()) + compute_graph.add_initial(f"dq_weights_1_{k}", deq_linear.prevnodes[1].value) + compute_graph.add_initial(f"dq_weights_2_{k}", deq_linear.prevnodes[2].value) + + matmul = compute_graph.nodemap[found_pattern[-1]] + for name in found_pattern: + if "DynamicQuantizeLinear" in name: + continue + compute_graph.remove_node(name) + compute_graph.remove_node(deq_linear.prevnodes[0].name) + compute_graph.remove_node(deq_linear.prevnodes[1].name) + if deq_linear.prevnodes[2].name in compute_graph.nodemap: + compute_graph.remove_node(deq_linear.prevnodes[2].name) + + compute_graph = create_nodes( + compute_graph, + "MatMulInteger", + f"matmul_integer_{k}", + [dyn_q.output[0], f"dq_weights_0_{k}", dyn_q.output[2], f"dq_weights_2_{k}"], + [f"matmul_integer_{k}"], + ) + compute_graph = create_nodes( + compute_graph, "Cast", f"cast_{k}", [f"matmul_integer_{k}"], [f"cast_{k}"], to=int(1) + ) + compute_graph = create_nodes( + compute_graph, "Mul", f"mulscales_{k}", [dyn_q.output[1], f"dq_weights_1_{k}"], [f"mulscales_{k}"] + ) + compute_graph = create_nodes( + compute_graph, "Mul", f"mulvalues_{k}", [f"mulscales_{k}", f"cast_{k}"], [f"mulvalues_{k}"] + ) + compute_graph = create_nodes( + compute_graph, "Add", f"addbias_{k}", [bias, f"mulvalues_{k}"], [matmul.output[0]] + ) + return compute_graph + + +def find_and_insert_matmulinteger(model_path): + print("Rewriting ONNX Graph with MatMulInteger ") + + cfg = {"constant_folding": False, "node_rename": False, "if_fixed_branch": None, "fixed_topk": 0, "verbose": True} + original_output = onnx.load(model_path).graph.output + model = Model(model_path, cfg) + graph = model.graph + + print("Replacing MatMul with MatMulInteger") + pattern = FusionPattern(MatMul) + found_nodes = pattern.search_pattern(graph) + graph = replace_matmul_to_matmulinteger(graph, found_nodes) + + print("Replacing GEMM with MatMulInteger + Add") + pattern = FusionPattern(GEMM) + found_nodes = pattern.search_pattern(graph) + graph = replace_gemm_to_matmulinteger(graph, found_nodes) + + graph.graph_reorder_nodes() + + print("Saving the new ONNX model") + full_path = pathlib.Path(model_path) + + graph = graph.make_graph_onnx( + graph.nodemap.keys(), "graph", graph.input, graph.output, with_initializer=True, with_shape_info=False + ) + + attr = {"producer_name": "onnx_tool"} + model_to_save = onnx.helper.make_model(graph, **attr) + # onnx_tools might remove the output nodes from the ONNX graph, so we need to restore it. + for out in original_output: + if out not in model_to_save.graph.output: + model_to_save.graph.output.append(out) + + model_to_save.ir_version = model.mproto.ir_version + model_to_save.opset_import.pop() + for opset in model.mproto.opset_import: + model_to_save.opset_import.append(opset) + + check_and_save_model(model_to_save, full_path) diff --git a/tests/ryzenai/test_modeling.py b/tests/ryzenai/test_modeling.py index c086c17c..a0e28e0e 100644 --- a/tests/ryzenai/test_modeling.py +++ b/tests/ryzenai/test_modeling.py @@ -12,21 +12,33 @@ import onnxruntime import pytest import requests +import torch +from brevitas.export.onnx.standard.qcdq.manager import StdQCDQONNXManager +from brevitas_examples.llm.llm_quant.export import brevitas_proxy_export_mode from parameterized import parameterized from PIL import Image -from testing_utils import ( - DEFAULT_CACHE_DIR, - DEFAULT_VAIP_CONFIG, +from rewriter import find_and_insert_matmulinteger +from testing_models import ( + PYTORCH_MODELS, RYZEN_PREQUANTIZED_MODEL_CUSTOM_TASKS, RYZEN_PREQUANTIZED_MODEL_IMAGE_CLASSIFICATION, RYZEN_PREQUANTIZED_MODEL_IMAGE_SEGMENTATION, RYZEN_PREQUANTIZED_MODEL_IMAGE_TO_IMAGE, RYZEN_PREQUANTIZED_MODEL_OBJECT_DETECTION, +) +from testing_utils import ( + DEFAULT_CACHE_DIR, + DEFAULT_VAIP_CONFIG, + DEFAULT_VAIP_CONFIG_TRANSFORMERS, RyzenAITestCaseMixin, + get_models_to_test, ) +from optimum.amd import BrevitasQuantizationConfig, BrevitasQuantizer +from optimum.amd.brevitas.data_utils import get_dataset_for_model from optimum.amd.ryzenai import ( RyzenAIModel, + RyzenAIModelForCausalLM, RyzenAIModelForCustomTasks, RyzenAIModelForImageClassification, RyzenAIModelForImageSegmentation, @@ -34,10 +46,12 @@ RyzenAIModelForObjectDetection, pipeline, ) +from optimum.exporters.onnx import onnx_export_from_model from optimum.utils import ( DummyInputGenerator, logging, ) +from transformers import AutoTokenizer from transformers.testing_utils import slow @@ -287,3 +301,102 @@ def test_model(self, model_id): self.assertEqual(baseline_ops["dpu"], current_ops["dpu"], f"DPU operators do not match! {current_ops}") gc.collect() + + +class RyzenAIModelForCausalLMIntegrationTest(unittest.TestCase, RyzenAITestCaseMixin): + SUPPORTED_ARCHITECTURES = { + "opt", + "llama", + "mistral", + } + + @parameterized.expand( + get_models_to_test( + PYTORCH_MODELS, + library_name="transformers", + supported_archs=SUPPORTED_ARCHITECTURES, + tasks="text-generation-with-past", + ) + ) + @pytest.mark.brevitas_ryzen_test + def test_model(self, test_name: str, model_type: str, model_id: str, task: str): + dataset_name = "wikitext2" + num_calib_samples = 10 + + quantization_dir = tempfile.TemporaryDirectory() + + tokenizer = AutoTokenizer.from_pretrained(model_id) + + # TODO: Replace with the new method in brevitas + quantization_config = BrevitasQuantizationConfig( + apply_gptq=False, + apply_weight_equalization=False, + activations_equalization="layerwise", + is_static=False, + weights_symmetric=True, + activations_symmetric=False, + ) + + # Load the data for calibration and evaluation. + calibration_dataset = get_dataset_for_model( + model_id, + qconfig=quantization_config, + dataset_name=dataset_name, + tokenizer=tokenizer, + nsamples=num_calib_samples, + seqlen=64, + split="train", + device=None, + fuse_sequences=False, + ) + + # quantize model + quantizer = BrevitasQuantizer.from_pretrained(model_id, device_map="cpu") + + quantized_model = quantizer.quantize(quantization_config, calibration_dataset) + + # export model + # TODO: Replace with the new method in brevitas + export_manager = StdQCDQONNXManager + with torch.no_grad(), brevitas_proxy_export_mode(quantized_model, export_manager=export_manager): + onnx_export_from_model( + quantized_model, + quantization_dir.name, + task="text-generation-with-past", + do_validation=False, + no_post_process=True, + ) + + find_and_insert_matmulinteger(os.path.join(quantization_dir.name, "model.onnx")) + + # inference + cache_dir = DEFAULT_CACHE_DIR + cache_key = model_id.replace("/", "_").lower() + vaip_config = DEFAULT_VAIP_CONFIG_TRANSFORMERS + + prompt = "Hey, are you conscious? Can you talk to me?" + inputs = tokenizer(prompt, return_tensors="np") + ort_inputs = {key: np.array(inputs[key], dtype=np.int64) for key in inputs.keys()} + + if model_type in {"llama", "mistral"}: + attention_mask = torch.tensor(inputs["attention_mask"]) + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + ort_inputs["position_ids"] = position_ids.numpy() + + outputs_ipu, outputs_cpu = self.prepare_outputs( + quantization_dir.name, RyzenAIModelForCausalLM, ort_inputs, vaip_config, cache_dir, cache_key + ) + + for output_ipu, output_cpu in zip(outputs_ipu.values(), outputs_cpu.values()): + self.assertTrue(np.allclose(output_ipu, output_cpu, atol=1e-4)) + + current_ops = self.get_ops(cache_dir, cache_key) + baseline_ops = self.get_baseline_ops(cache_key) + + self.assertEqual(baseline_ops["all"], current_ops["all"], f"Total operators do not match! {current_ops}") + self.assertEqual( + baseline_ops["matmulinteger"], current_ops["matmulinteger"], f"MATMULINTEGERs do not match! {current_ops}" + ) + + quantization_dir.cleanup() diff --git a/tests/ryzenai/test_quantization.py b/tests/ryzenai/test_quantization.py index 3e97f25a..a1bf3507 100644 --- a/tests/ryzenai/test_quantization.py +++ b/tests/ryzenai/test_quantization.py @@ -4,19 +4,18 @@ import tempfile import unittest from functools import partial -from typing import Dict import pytest import timm import torch from datasets import load_dataset from parameterized import parameterized +from testing_models import PYTORCH_TIMM_MODEL_SUBSET, PYTORCH_TIMM_MODELS from testing_utils import ( DEFAULT_CACHE_DIR, DEFAULT_VAIP_CONFIG, - PYTORCH_TIMM_MODEL, - PYTORCH_TIMM_MODEL_SUBSET, RyzenAITestCaseMixin, + get_models_to_test, ) from optimum.amd.ryzenai import ( @@ -25,38 +24,10 @@ RyzenAIOnnxQuantizer, ) from optimum.exporters.onnx import main_export -from optimum.exporters.tasks import TasksManager from transformers import PretrainedConfig from transformers.testing_utils import slow -def _get_models_to_test(export_models_dict: Dict, library_name: str = "timm"): - models_to_test = [] - for model_type, model_names_tasks in export_models_dict.items(): - task_config_mapping = TasksManager.get_supported_tasks_for_model_type( - model_type, "onnx", library_name=library_name - ) - - if isinstance(model_names_tasks, str): # test export of all tasks on the same model - tasks = list(task_config_mapping.keys()) - model_tasks = {model_names_tasks: tasks} - else: - model_tasks = model_names_tasks # possibly, test different tasks on different models - - for model_name, tasks in model_tasks.items(): - for task in tasks: - models_to_test.append( - ( - f"{model_type}_{task}_{model_name}", - model_type, - model_name, - task, - ) - ) - - return sorted(models_to_test) - - class TestTimmQuantization(unittest.TestCase, RyzenAITestCaseMixin): def _quantize( self, @@ -133,7 +104,7 @@ def preprocess_fn(ex, transforms): export_dir.cleanup() quantization_dir.cleanup() - @parameterized.expand(_get_models_to_test(PYTORCH_TIMM_MODEL_SUBSET, library_name="timm")) + @parameterized.expand(get_models_to_test(PYTORCH_TIMM_MODEL_SUBSET, library_name="timm")) def test_timm_quantization_subset( self, test_name: str, @@ -143,7 +114,7 @@ def test_timm_quantization_subset( ): self._quantize(model_name=model_name) - @parameterized.expand(_get_models_to_test(PYTORCH_TIMM_MODEL, library_name="timm")) + @parameterized.expand(get_models_to_test(PYTORCH_TIMM_MODELS, library_name="timm")) @pytest.mark.quant_test @slow def test_timm_quantization( diff --git a/tests/ryzenai/testing_models.py b/tests/ryzenai/testing_models.py new file mode 100644 index 00000000..cee8b8af --- /dev/null +++ b/tests/ryzenai/testing_models.py @@ -0,0 +1,366 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# Licensed under the MIT License. + +RYZEN_PREQUANTIZED_MODEL_IMAGE_CLASSIFICATION = [ + "amd/efficientnet-es", + "amd/ese_vovnet39b", + "amd/inception_v4", + "amd/mnasnet_b1", + "amd/mobilenet_v2_1.0_224", + "amd/resnet50", + "amd/squeezenet", +] + +RYZEN_PREQUANTIZED_MODEL_OBJECT_DETECTION = { + "retinaface": "amd/retinaface", + "yolov3": "amd/yolov3", + "yolov5": "amd/yolov5s", + "yolov8": "amd/yolov8m", + "yolox": "amd/yolox-s", +} + +RYZEN_PREQUANTIZED_MODEL_IMAGE_SEGMENTATION = [ + "amd/HRNet", + "amd/SemanticFPN", +] + +RYZEN_PREQUANTIZED_MODEL_IMAGE_TO_IMAGE = ["amd/PAN", "amd/rcan", "amd/sesr"] + +RYZEN_PREQUANTIZED_MODEL_CUSTOM_TASKS = ["amd/movenet"] + +PYTORCH_TIMM_MODEL_SUBSET = { + "default-timm-config": { + "timm/densenet121.ra_in1k": ["image-classification"], + "timm/ese_vovnet19b_dw.ra_in1k": ["image-classification"], + "timm/ghostnet_100.in1k": ["image-classification"], + "timm/inception_v4.tf_in1k": ["image-classification"], + "timm/repvgg_b0.rvgg_in1k": ["image-classification"], + "timm/resnet10t.c3_in1k": ["image-classification"], + "timm/vgg19.tv_in1k": ["image-classification"], + } +} + +PYTORCH_TIMM_MODELS = { + "default-timm-config": { + "timm/botnet26t_256.c1_in1k": ["image-classification"], + "timm/cs3darknet_focus_l.c2ns_in1k": ["image-classification"], + "timm/cs3darknet_focus_m.c2ns_in1k": ["image-classification"], + "timm/cs3darknet_l.c2ns_in1k": ["image-classification"], + "timm/cs3darknet_m.c2ns_in1k": ["image-classification"], + "timm/cs3darknet_x.c2ns_in1k": ["image-classification"], + "timm/cs3edgenet_x.c2_in1k": ["image-classification"], + "timm/cs3se_edgenet_x.c2ns_in1k": ["image-classification"], + "timm/cs3sedarknet_l.c2ns_in1k": ["image-classification"], + "timm/cs3sedarknet_x.c2ns_in1k": ["image-classification"], + "timm/cspdarknet53.ra_in1k": ["image-classification"], + "timm/cspresnet50.ra_in1k": ["image-classification"], + "timm/cspresnext50.ra_in1k": ["image-classification"], + "timm/densenet121.ra_in1k": ["image-classification"], + "timm/densenet169.tv_in1k": ["image-classification"], + "timm/densenetblur121d.ra_in1k": ["image-classification"], + "timm/dla102.in1k": ["image-classification"], + "timm/dla102x.in1k": ["image-classification"], + "timm/dla102x2.in1k": ["image-classification"], + "timm/dla169.in1k": ["image-classification"], + "timm/dla34.in1k": ["image-classification"], + "timm/dla46_c.in1k": ["image-classification"], + "timm/dla46x_c.in1k": ["image-classification"], + "timm/dla60.in1k": ["image-classification"], + "timm/dla60_res2net.in1k": ["image-classification"], + "timm/dla60_res2next.in1k": ["image-classification"], + "timm/dla60x.in1k": ["image-classification"], + "timm/dla60x_c.in1k": ["image-classification"], + "timm/dpn68.mx_in1k": ["image-classification"], + "timm/dpn68b.ra_in1k": ["image-classification"], + "timm/dpn92.mx_in1k": ["image-classification"], + "timm/dpn98.mx_in1k": ["image-classification"], + "timm/eca_botnext26ts_256.c1_in1k": ["image-classification"], + "timm/eca_nfnet_l0.ra2_in1k": ["image-classification"], + "timm/eca_resnet33ts.ra2_in1k": ["image-classification"], + "timm/eca_resnext26ts.ch_in1k": ["image-classification"], + "timm/ecaresnet101d.miil_in1k": ["image-classification"], + "timm/ecaresnet101d_pruned.miil_in1k": ["image-classification"], + "timm/ecaresnet26t.ra2_in1k": ["image-classification"], + "timm/ecaresnet50d.miil_in1k": ["image-classification"], + "timm/ecaresnet50d_pruned.miil_in1k": ["image-classification"], + "timm/ecaresnet50t.ra2_in1k": ["image-classification"], + "timm/ecaresnetlight.miil_in1k": ["image-classification"], + "timm/edgenext_base.usi_in1k": ["image-classification"], + "timm/edgenext_small.usi_in1k": ["image-classification"], + "timm/edgenext_small_rw.sw_in1k": ["image-classification"], + "timm/edgenext_x_small.in1k": ["image-classification"], + "timm/edgenext_xx_small.in1k": ["image-classification"], + "timm/efficientnet_b0.ra_in1k": ["image-classification"], + "timm/efficientnet_b1.ft_in1k": ["image-classification"], + "timm/efficientnet_b2.ra_in1k": ["image-classification"], + "timm/efficientnet_b3.ra2_in1k": ["image-classification"], + "timm/efficientnet_el.ra_in1k": ["image-classification"], + "timm/efficientnet_el_pruned.in1k": ["image-classification"], + "timm/efficientnet_em.ra2_in1k": ["image-classification"], + "timm/efficientnet_es.ra_in1k": ["image-classification"], + "timm/efficientnet_es_pruned.in1k": ["image-classification"], + "timm/efficientnet_lite0.ra_in1k": ["image-classification"], + "timm/efficientnetv2_rw_s.ra2_in1k": ["image-classification"], + "timm/efficientnetv2_rw_t.ra2_in1k": ["image-classification"], + "timm/ese_vovnet19b_dw.ra_in1k": ["image-classification"], + "timm/ese_vovnet39b.ra_in1k": ["image-classification"], + "timm/fbnetc_100.rmsp_in1k": ["image-classification"], + "timm/fbnetv3_b.ra2_in1k": ["image-classification"], + "timm/fbnetv3_d.ra2_in1k": ["image-classification"], + "timm/fbnetv3_g.ra2_in1k": ["image-classification"], + "timm/gcresnet33ts.ra2_in1k": ["image-classification"], + "timm/gcresnet50t.ra2_in1k": ["image-classification"], + "timm/gcresnext26ts.ch_in1k": ["image-classification"], + "timm/gcresnext50ts.ch_in1k": ["image-classification"], + "timm/gernet_l.idstcv_in1k": ["image-classification"], + "timm/gernet_m.idstcv_in1k": ["image-classification"], + "timm/gernet_s.idstcv_in1k": ["image-classification"], + "timm/ghostnet_100.in1k": ["image-classification"], + "timm/hardcorenas_a.miil_green_in1k": ["image-classification"], + "timm/hardcorenas_b.miil_green_in1k": ["image-classification"], + "timm/hardcorenas_c.miil_green_in1k": ["image-classification"], + "timm/hardcorenas_d.miil_green_in1k": ["image-classification"], + "timm/hardcorenas_e.miil_green_in1k": ["image-classification"], + "timm/hardcorenas_f.miil_green_in1k": ["image-classification"], + "timm/hrnet_w18_small.gluon_in1k": ["image-classification"], + "timm/hrnet_w18_small_v2.gluon_in1k": ["image-classification"], + "timm/inception_v3.gluon_in1k": ["image-classification"], + "timm/inception_v3.tf_adv_in1k": ["image-classification"], + "timm/inception_v3.tf_in1k": ["image-classification"], + "timm/inception_v3.tv_in1k": ["image-classification"], + "timm/inception_v4.tf_in1k": ["image-classification"], + "timm/lambda_resnet26rpt_256.c1_in1k": ["image-classification"], + "timm/lambda_resnet26t.c1_in1k": ["image-classification"], + "timm/lambda_resnet50ts.a1h_in1k": ["image-classification"], + "timm/lcnet_050.ra2_in1k": ["image-classification"], + "timm/lcnet_075.ra2_in1k": ["image-classification"], + "timm/lcnet_100.ra2_in1k": ["image-classification"], + "timm/mixer_b16_224.goog_in21k_ft_in1k": ["image-classification"], + "timm/mixer_b16_224.miil_in21k_ft_in1k": ["image-classification"], + "timm/mixnet_l.ft_in1k": ["image-classification"], + "timm/mixnet_m.ft_in1k": ["image-classification"], + "timm/mixnet_s.ft_in1k": ["image-classification"], + "timm/mnasnet_100.rmsp_in1k": ["image-classification"], + "timm/mnasnet_small.lamb_in1k": ["image-classification"], + "timm/mobilenetv2_050.lamb_in1k": ["image-classification"], + "timm/mobilenetv2_100.ra_in1k": ["image-classification"], + "timm/mobilenetv2_110d.ra_in1k": ["image-classification"], + "timm/mobilenetv2_120d.ra_in1k": ["image-classification"], + "timm/mobilenetv2_140.ra_in1k": ["image-classification"], + "timm/mobilenetv3_large_100.miil_in21k_ft_in1k": ["image-classification"], + "timm/mobilenetv3_large_100.ra_in1k": ["image-classification"], + "timm/mobilenetv3_rw.rmsp_in1k": ["image-classification"], + "timm/mobilenetv3_small_050.lamb_in1k": ["image-classification"], + "timm/mobilenetv3_small_075.lamb_in1k": ["image-classification"], + "timm/mobilenetv3_small_100.lamb_in1k": ["image-classification"], + "timm/nest_tiny_jx.goog_in1k": ["image-classification"], + "timm/nf_resnet50.ra2_in1k": ["image-classification"], + "timm/nfnet_l0.ra2_in1k": ["image-classification"], + "timm/regnetx_002.pycls_in1k": ["image-classification"], + "timm/regnetx_004.pycls_in1k": ["image-classification"], + "timm/regnetx_004_tv.tv2_in1k": ["image-classification"], + "timm/regnetx_006.pycls_in1k": ["image-classification"], + "timm/regnetx_008.pycls_in1k": ["image-classification"], + "timm/regnetx_008.tv2_in1k": ["image-classification"], + "timm/regnetx_016.pycls_in1k": ["image-classification"], + "timm/regnetx_016.tv2_in1k": ["image-classification"], + "timm/regnetx_032.pycls_in1k": ["image-classification"], + "timm/regnetx_032.tv2_in1k": ["image-classification"], + "timm/regnetx_040.pycls_in1k": ["image-classification"], + "timm/regnetx_064.pycls_in1k": ["image-classification"], + "timm/regnetx_080.pycls_in1k": ["image-classification"], + "timm/regnetx_080.tv2_in1k": ["image-classification"], + "timm/regnetx_120.pycls_in1k": ["image-classification"], + "timm/regnetx_160.pycls_in1k": ["image-classification"], + "timm/regnetx_160.tv2_in1k": ["image-classification"], + "timm/regnety_002.pycls_in1k": ["image-classification"], + "timm/regnety_004.pycls_in1k": ["image-classification"], + "timm/regnety_004.tv2_in1k": ["image-classification"], + "timm/regnety_006.pycls_in1k": ["image-classification"], + "timm/regnety_008.pycls_in1k": ["image-classification"], + "timm/regnety_008_tv.tv2_in1k": ["image-classification"], + "timm/regnety_016.pycls_in1k": ["image-classification"], + "timm/regnety_016.tv2_in1k": ["image-classification"], + "timm/regnety_032.pycls_in1k": ["image-classification"], + "timm/regnety_032.ra_in1k": ["image-classification"], + "timm/regnety_032.tv2_in1k": ["image-classification"], + "timm/regnety_040.pycls_in1k": ["image-classification"], + "timm/regnety_040.ra3_in1k": ["image-classification"], + "timm/regnety_064.pycls_in1k": ["image-classification"], + "timm/regnety_064.ra3_in1k": ["image-classification"], + "timm/regnety_080.pycls_in1k": ["image-classification"], + "timm/regnety_080.ra3_in1k": ["image-classification"], + "timm/regnety_080_tv.tv2_in1k": ["image-classification"], + "timm/regnety_120.pycls_in1k": ["image-classification"], + "timm/regnety_120.sw_in12k_ft_in1k": ["image-classification"], + "timm/regnety_160.lion_in12k_ft_in1k": ["image-classification"], + "timm/regnety_160.pycls_in1k": ["image-classification"], + "timm/regnety_160.sw_in12k_ft_in1k": ["image-classification"], + "timm/regnety_160.swag_ft_in1k": ["image-classification"], + "timm/regnety_160.swag_lc_in1k": ["image-classification"], + "timm/regnety_160.tv2_in1k": ["image-classification"], + "timm/regnety_320.seer_ft_in1k": ["image-classification"], + "timm/regnety_320.swag_ft_in1k": ["image-classification"], + "timm/regnetz_040.ra3_in1k": ["image-classification"], + "timm/regnetz_040_h.ra3_in1k": ["image-classification"], + "timm/regnetz_b16.ra3_in1k": ["image-classification"], + "timm/regnetz_c16.ra3_in1k": ["image-classification"], + "timm/regnetz_d32.ra3_in1k": ["image-classification"], + "timm/regnetz_d8.ra3_in1k": ["image-classification"], + "timm/repvgg_a2.rvgg_in1k": ["image-classification"], + "timm/repvgg_b0.rvgg_in1k": ["image-classification"], + "timm/repvgg_b1.rvgg_in1k": ["image-classification"], + "timm/repvgg_b1g4.rvgg_in1k": ["image-classification"], + "timm/repvgg_b2.rvgg_in1k": ["image-classification"], + "timm/repvgg_b2g4.rvgg_in1k": ["image-classification"], + "timm/repvgg_b3.rvgg_in1k": ["image-classification"], + "timm/repvgg_b3g4.rvgg_in1k": ["image-classification"], + "timm/res2net50_14w_8s.in1k": ["image-classification"], + "timm/res2net50_26w_4s.in1k": ["image-classification"], + "timm/res2net50_26w_6s.in1k": ["image-classification"], + "timm/res2net50_26w_8s.in1k": ["image-classification"], + "timm/res2net50_48w_2s.in1k": ["image-classification"], + "timm/res2next50.in1k": ["image-classification"], + "timm/resmlp_12_224.fb_distilled_in1k": ["image-classification"], + "timm/resmlp_12_224.fb_in1k": ["image-classification"], + "timm/resmlp_24_224.fb_distilled_in1k": ["image-classification"], + "timm/resmlp_24_224.fb_in1k": ["image-classification"], + "timm/resmlp_big_24_224.fb_distilled_in1k": ["image-classification"], + "timm/resnest14d.gluon_in1k": ["image-classification"], + "timm/resnest26d.gluon_in1k": ["image-classification"], + "timm/resnest50d.in1k": ["image-classification"], + "timm/resnest50d_1s4x24d.in1k": ["image-classification"], + "timm/resnest50d_4s2x40d.in1k": ["image-classification"], + "timm/resnet101.a1h_in1k": ["image-classification"], + "timm/resnet101.gluon_in1k": ["image-classification"], + "timm/resnet101.tv_in1k": ["image-classification"], + "timm/resnet101c.gluon_in1k": ["image-classification"], + "timm/resnet101d.gluon_in1k": ["image-classification"], + "timm/resnet101d.ra2_in1k": ["image-classification"], + "timm/resnet101s.gluon_in1k": ["image-classification"], + "timm/resnet10t.c3_in1k": ["image-classification"], + "timm/resnet14t.c3_in1k": ["image-classification"], + "timm/resnet152.a1h_in1k": ["image-classification"], + "timm/resnet152.gluon_in1k": ["image-classification"], + "timm/resnet152.tv_in1k": ["image-classification"], + "timm/resnet152c.gluon_in1k": ["image-classification"], + "timm/resnet152d.gluon_in1k": ["image-classification"], + "timm/resnet152d.ra2_in1k": ["image-classification"], + "timm/resnet152s.gluon_in1k": ["image-classification"], + "timm/resnet18.a1_in1k": ["image-classification"], + "timm/resnet18.fb_ssl_yfcc100m_ft_in1k": ["image-classification"], + "timm/resnet18.fb_swsl_ig1b_ft_in1k": ["image-classification"], + "timm/resnet18.gluon_in1k": ["image-classification"], + "timm/resnet18d.ra2_in1k": ["image-classification"], + "timm/resnet200d.ra2_in1k": ["image-classification"], + "timm/resnet26.bt_in1k": ["image-classification"], + "timm/resnet26d.bt_in1k": ["image-classification"], + "timm/resnet26t.ra2_in1k": ["image-classification"], + "timm/resnet32ts.ra2_in1k": ["image-classification"], + "timm/resnet33ts.ra2_in1k": ["image-classification"], + "timm/resnet34.a1_in1k": ["image-classification"], + "timm/resnet34.gluon_in1k": ["image-classification"], + "timm/resnet34.tv_in1k": ["image-classification"], + "timm/resnet34d.ra2_in1k": ["image-classification"], + "timm/resnet50.a1_in1k": ["image-classification"], + "timm/resnet50.fb_ssl_yfcc100m_ft_in1k": ["image-classification"], + "timm/resnet50.fb_swsl_ig1b_ft_in1k": ["image-classification"], + "timm/resnet50.gluon_in1k": ["image-classification"], + "timm/resnet50.tv_in1k": ["image-classification"], + "timm/resnet50_gn.a1h_in1k": ["image-classification"], + "timm/resnet50c.gluon_in1k": ["image-classification"], + "timm/resnet50d.gluon_in1k": ["image-classification"], + "timm/resnet50d.ra2_in1k": ["image-classification"], + "timm/resnet50s.gluon_in1k": ["image-classification"], + "timm/resnet51q.ra2_in1k": ["image-classification"], + "timm/resnet61q.ra2_in1k": ["image-classification"], + "timm/resnetaa50.a1h_in1k": ["image-classification"], + "timm/resnetblur50.bt_in1k": ["image-classification"], + "timm/resnetrs101.tf_in1k": ["image-classification"], + "timm/resnetrs50.tf_in1k": ["image-classification"], + "timm/resnetv2_101.a1h_in1k": ["image-classification"], + "timm/resnetv2_50.a1h_in1k": ["image-classification"], + "timm/resnetv2_50d_gn.ah_in1k": ["image-classification"], + "timm/resnetv2_50x1_bit.goog_distilled_in1k": ["image-classification"], + "timm/resnetv2_50x1_bit.goog_in21k_ft_in1k": ["image-classification"], + "timm/resnetv2_50x3_bit.goog_in21k_ft_in1k": ["image-classification"], + "timm/resnext101_32x4d.fb_ssl_yfcc100m_ft_in1k": ["image-classification"], + "timm/resnext101_32x4d.fb_swsl_ig1b_ft_in1k": ["image-classification"], + "timm/resnext101_32x4d.gluon_in1k": ["image-classification"], + "timm/resnext101_32x8d.fb_ssl_yfcc100m_ft_in1k": ["image-classification"], + "timm/resnext101_32x8d.fb_swsl_ig1b_ft_in1k": ["image-classification"], + "timm/resnext101_32x8d.fb_wsl_ig1b_ft_in1k": ["image-classification"], + "timm/resnext101_64x4d.c1_in1k": ["image-classification"], + "timm/resnext101_64x4d.gluon_in1k": ["image-classification"], + "timm/resnext26ts.ra2_in1k": ["image-classification"], + "timm/resnext50_32x4d.a1h_in1k": ["image-classification"], + "timm/resnext50_32x4d.fb_ssl_yfcc100m_ft_in1k": ["image-classification"], + "timm/resnext50_32x4d.fb_swsl_ig1b_ft_in1k": ["image-classification"], + "timm/resnext50_32x4d.gluon_in1k": ["image-classification"], + "timm/resnext50_32x4d.tv_in1k": ["image-classification"], + "timm/resnext50d_32x4d.bt_in1k": ["image-classification"], + "timm/rexnet_100.nav_in1k": ["image-classification"], + "timm/rexnet_130.nav_in1k": ["image-classification"], + "timm/rexnet_150.nav_in1k": ["image-classification"], + "timm/rexnet_200.nav_in1k": ["image-classification"], + "timm/rexnet_300.nav_in1k": ["image-classification"], + "timm/rexnetr_200.sw_in12k_ft_in1k": ["image-classification"], + "timm/rexnetr_300.sw_in12k_ft_in1k": ["image-classification"], + "timm/sebotnet33ts_256.a1h_in1k": ["image-classification"], + "timm/semnasnet_075.rmsp_in1k": ["image-classification"], + "timm/semnasnet_100.rmsp_in1k": ["image-classification"], + "timm/seresnet33ts.ra2_in1k": ["image-classification"], + "timm/seresnet50.a1_in1k": ["image-classification"], + "timm/seresnext101_32x4d.gluon_in1k": ["image-classification"], + "timm/seresnext26d_32x4d.bt_in1k": ["image-classification"], + "timm/seresnext26t_32x4d.bt_in1k": ["image-classification"], + "timm/seresnext26ts.ch_in1k": ["image-classification"], + "timm/seresnext50_32x4d.gluon_in1k": ["image-classification"], + "timm/seresnext50_32x4d.racm_in1k": ["image-classification"], + "timm/skresnet18.ra_in1k": ["image-classification"], + "timm/skresnet34.ra_in1k": ["image-classification"], + "timm/skresnext50_32x4d.ra_in1k": ["image-classification"], + "timm/spnasnet_100.rmsp_in1k": ["image-classification"], + "timm/tf_efficientnet_el.in1k": ["image-classification"], + "timm/tf_efficientnet_em.in1k": ["image-classification"], + "timm/tf_efficientnet_es.in1k": ["image-classification"], + "timm/tf_efficientnet_lite0.in1k": ["image-classification"], + "timm/tf_efficientnet_lite1.in1k": ["image-classification"], + "timm/tf_efficientnet_lite2.in1k": ["image-classification"], + "timm/tf_efficientnet_lite3.in1k": ["image-classification"], + "timm/tf_efficientnet_lite4.in1k": ["image-classification"], + "timm/tf_efficientnetv2_b0.in1k": ["image-classification"], + "timm/tf_efficientnetv2_b1.in1k": ["image-classification"], + "timm/tf_efficientnetv2_b2.in1k": ["image-classification"], + "timm/tf_efficientnetv2_b3.in1k": ["image-classification"], + "timm/tf_efficientnetv2_b3.in21k_ft_in1k": ["image-classification"], + "timm/tf_mobilenetv3_large_minimal_100.in1k": ["image-classification"], + "timm/tf_mobilenetv3_small_075.in1k": ["image-classification"], + "timm/tf_mobilenetv3_small_100.in1k": ["image-classification"], + "timm/tf_mobilenetv3_small_minimal_100.in1k": ["image-classification"], + "timm/tinynet_a.in1k": ["image-classification"], + "timm/tinynet_d.in1k": ["image-classification"], + "timm/vgg11.tv_in1k": ["image-classification"], + "timm/vgg11_bn.tv_in1k": ["image-classification"], + "timm/vgg13.tv_in1k": ["image-classification"], + "timm/vgg13_bn.tv_in1k": ["image-classification"], + "timm/vgg16.tv_in1k": ["image-classification"], + "timm/vgg16_bn.tv_in1k": ["image-classification"], + "timm/vgg19.tv_in1k": ["image-classification"], + "timm/vgg19_bn.tv_in1k": ["image-classification"], + "timm/wide_resnet101_2.tv_in1k": ["image-classification"], + "timm/wide_resnet50_2.racm_in1k": ["image-classification"], + "timm/xception41.tf_in1k": ["image-classification"], + "timm/xception41p.ra3_in1k": ["image-classification"], + "timm/xception65.ra3_in1k": ["image-classification"], + "timm/xception65p.ra3_in1k": ["image-classification"], + "timm/xception71.tf_in1k": ["image-classification"], + } +} + + +PYTORCH_MODELS = { + "llama": "fxmarty/tiny-llama-fast-tokenizer", + "mistral": "echarlaix/tiny-random-mistral", + "opt": "hf-internal-testing/tiny-random-OPTForCausalLM", +} diff --git a/tests/ryzenai/testing_utils.py b/tests/ryzenai/testing_utils.py index b99e8f1d..6ed3760c 100644 --- a/tests/ryzenai/testing_utils.py +++ b/tests/ryzenai/testing_utils.py @@ -3,7 +3,9 @@ import json import os +from typing import Dict +from optimum.exporters import TasksManager from transformers import set_seed @@ -11,6 +13,7 @@ BASELINE_JSON = os.path.normpath("./tests/ryzenai/operators_baseline.json") # For RyzenSDK 1.0.1 DEFAULT_VAIP_CONFIG = os.path.normpath("./tests/ryzenai/vaip_config.json") +DEFAULT_VAIP_CONFIG_TRANSFORMERS = os.path.normpath("./tests/ryzenai/vaip_config_transformers.json") DEFAULT_CACHE_DIR = "ryzen_cache" @@ -18,7 +21,7 @@ def parse_json(json_path): with open(json_path, "r") as json_file: data = json.load(json_file) - result = {"all": 0, "dpu": 0, "cpu": 0} + result = {"all": 0, "dpu": 0, "cpu": 0, "matmulinteger": 0} for entry in data["deviceStat"]: result[entry["name"].lower()] = entry["nodeNum"] return result @@ -97,359 +100,39 @@ def get_baseline_ops(self, key): return data[key] -RYZEN_PREQUANTIZED_MODEL_IMAGE_CLASSIFICATION = [ - "amd/efficientnet-es", - "amd/ese_vovnet39b", - "amd/inception_v4", - "amd/mnasnet_b1", - "amd/mobilenet_v2_1.0_224", - "amd/resnet50", - "amd/squeezenet", -] +def get_models_to_test( + export_models_dict: Dict, library_name: str = "timm", supported_archs: list = None, tasks: list = None +): + models_to_test = [] + for model_type, model_names_tasks in export_models_dict.items(): + if model_type not in supported_archs: + continue -RYZEN_PREQUANTIZED_MODEL_OBJECT_DETECTION = { - "retinaface": "amd/retinaface", - "yolov3": "amd/yolov3", - "yolov5": "amd/yolov5s", - "yolov8": "amd/yolov8m", - "yolox": "amd/yolox-s", -} + if not isinstance(tasks, list): + tasks = [tasks] -RYZEN_PREQUANTIZED_MODEL_IMAGE_SEGMENTATION = [ - "amd/HRNet", - "amd/SemanticFPN", -] + if tasks is None: + task_config_mapping = TasksManager.get_supported_tasks_for_model_type( + model_type, "onnx", library_name=library_name + ) -RYZEN_PREQUANTIZED_MODEL_IMAGE_TO_IMAGE = ["amd/PAN", "amd/rcan", "amd/sesr"] - -RYZEN_PREQUANTIZED_MODEL_CUSTOM_TASKS = ["amd/movenet"] - -PYTORCH_TIMM_MODEL_SUBSET = { - "default-timm-config": { - "timm/densenet121.ra_in1k": ["image-classification"], - "timm/ese_vovnet19b_dw.ra_in1k": ["image-classification"], - "timm/ghostnet_100.in1k": ["image-classification"], - "timm/inception_v4.tf_in1k": ["image-classification"], - "timm/repvgg_b0.rvgg_in1k": ["image-classification"], - "timm/resnet10t.c3_in1k": ["image-classification"], - "timm/vgg19.tv_in1k": ["image-classification"], - } -} - -PYTORCH_TIMM_MODEL = { - "default-timm-config": { - "timm/botnet26t_256.c1_in1k": ["image-classification"], - "timm/cs3darknet_focus_l.c2ns_in1k": ["image-classification"], - "timm/cs3darknet_focus_m.c2ns_in1k": ["image-classification"], - "timm/cs3darknet_l.c2ns_in1k": ["image-classification"], - "timm/cs3darknet_m.c2ns_in1k": ["image-classification"], - "timm/cs3darknet_x.c2ns_in1k": ["image-classification"], - "timm/cs3edgenet_x.c2_in1k": ["image-classification"], - "timm/cs3se_edgenet_x.c2ns_in1k": ["image-classification"], - "timm/cs3sedarknet_l.c2ns_in1k": ["image-classification"], - "timm/cs3sedarknet_x.c2ns_in1k": ["image-classification"], - "timm/cspdarknet53.ra_in1k": ["image-classification"], - "timm/cspresnet50.ra_in1k": ["image-classification"], - "timm/cspresnext50.ra_in1k": ["image-classification"], - "timm/densenet121.ra_in1k": ["image-classification"], - "timm/densenet169.tv_in1k": ["image-classification"], - "timm/densenetblur121d.ra_in1k": ["image-classification"], - "timm/dla102.in1k": ["image-classification"], - "timm/dla102x.in1k": ["image-classification"], - "timm/dla102x2.in1k": ["image-classification"], - "timm/dla169.in1k": ["image-classification"], - "timm/dla34.in1k": ["image-classification"], - "timm/dla46_c.in1k": ["image-classification"], - "timm/dla46x_c.in1k": ["image-classification"], - "timm/dla60.in1k": ["image-classification"], - "timm/dla60_res2net.in1k": ["image-classification"], - "timm/dla60_res2next.in1k": ["image-classification"], - "timm/dla60x.in1k": ["image-classification"], - "timm/dla60x_c.in1k": ["image-classification"], - "timm/dpn68.mx_in1k": ["image-classification"], - "timm/dpn68b.ra_in1k": ["image-classification"], - "timm/dpn92.mx_in1k": ["image-classification"], - "timm/dpn98.mx_in1k": ["image-classification"], - "timm/eca_botnext26ts_256.c1_in1k": ["image-classification"], - "timm/eca_nfnet_l0.ra2_in1k": ["image-classification"], - "timm/eca_resnet33ts.ra2_in1k": ["image-classification"], - "timm/eca_resnext26ts.ch_in1k": ["image-classification"], - "timm/ecaresnet101d.miil_in1k": ["image-classification"], - "timm/ecaresnet101d_pruned.miil_in1k": ["image-classification"], - "timm/ecaresnet26t.ra2_in1k": ["image-classification"], - "timm/ecaresnet50d.miil_in1k": ["image-classification"], - "timm/ecaresnet50d_pruned.miil_in1k": ["image-classification"], - "timm/ecaresnet50t.ra2_in1k": ["image-classification"], - "timm/ecaresnetlight.miil_in1k": ["image-classification"], - "timm/edgenext_base.usi_in1k": ["image-classification"], - "timm/edgenext_small.usi_in1k": ["image-classification"], - "timm/edgenext_small_rw.sw_in1k": ["image-classification"], - "timm/edgenext_x_small.in1k": ["image-classification"], - "timm/edgenext_xx_small.in1k": ["image-classification"], - "timm/efficientnet_b0.ra_in1k": ["image-classification"], - "timm/efficientnet_b1.ft_in1k": ["image-classification"], - "timm/efficientnet_b2.ra_in1k": ["image-classification"], - "timm/efficientnet_b3.ra2_in1k": ["image-classification"], - "timm/efficientnet_el.ra_in1k": ["image-classification"], - "timm/efficientnet_el_pruned.in1k": ["image-classification"], - "timm/efficientnet_em.ra2_in1k": ["image-classification"], - "timm/efficientnet_es.ra_in1k": ["image-classification"], - "timm/efficientnet_es_pruned.in1k": ["image-classification"], - "timm/efficientnet_lite0.ra_in1k": ["image-classification"], - "timm/efficientnetv2_rw_s.ra2_in1k": ["image-classification"], - "timm/efficientnetv2_rw_t.ra2_in1k": ["image-classification"], - "timm/ese_vovnet19b_dw.ra_in1k": ["image-classification"], - "timm/ese_vovnet39b.ra_in1k": ["image-classification"], - "timm/fbnetc_100.rmsp_in1k": ["image-classification"], - "timm/fbnetv3_b.ra2_in1k": ["image-classification"], - "timm/fbnetv3_d.ra2_in1k": ["image-classification"], - "timm/fbnetv3_g.ra2_in1k": ["image-classification"], - "timm/gcresnet33ts.ra2_in1k": ["image-classification"], - "timm/gcresnet50t.ra2_in1k": ["image-classification"], - "timm/gcresnext26ts.ch_in1k": ["image-classification"], - "timm/gcresnext50ts.ch_in1k": ["image-classification"], - "timm/gernet_l.idstcv_in1k": ["image-classification"], - "timm/gernet_m.idstcv_in1k": ["image-classification"], - "timm/gernet_s.idstcv_in1k": ["image-classification"], - "timm/ghostnet_100.in1k": ["image-classification"], - "timm/hardcorenas_a.miil_green_in1k": ["image-classification"], - "timm/hardcorenas_b.miil_green_in1k": ["image-classification"], - "timm/hardcorenas_c.miil_green_in1k": ["image-classification"], - "timm/hardcorenas_d.miil_green_in1k": ["image-classification"], - "timm/hardcorenas_e.miil_green_in1k": ["image-classification"], - "timm/hardcorenas_f.miil_green_in1k": ["image-classification"], - "timm/hrnet_w18_small.gluon_in1k": ["image-classification"], - "timm/hrnet_w18_small_v2.gluon_in1k": ["image-classification"], - "timm/inception_v3.gluon_in1k": ["image-classification"], - "timm/inception_v3.tf_adv_in1k": ["image-classification"], - "timm/inception_v3.tf_in1k": ["image-classification"], - "timm/inception_v3.tv_in1k": ["image-classification"], - "timm/inception_v4.tf_in1k": ["image-classification"], - "timm/lambda_resnet26rpt_256.c1_in1k": ["image-classification"], - "timm/lambda_resnet26t.c1_in1k": ["image-classification"], - "timm/lambda_resnet50ts.a1h_in1k": ["image-classification"], - "timm/lcnet_050.ra2_in1k": ["image-classification"], - "timm/lcnet_075.ra2_in1k": ["image-classification"], - "timm/lcnet_100.ra2_in1k": ["image-classification"], - "timm/mixer_b16_224.goog_in21k_ft_in1k": ["image-classification"], - "timm/mixer_b16_224.miil_in21k_ft_in1k": ["image-classification"], - "timm/mixnet_l.ft_in1k": ["image-classification"], - "timm/mixnet_m.ft_in1k": ["image-classification"], - "timm/mixnet_s.ft_in1k": ["image-classification"], - "timm/mnasnet_100.rmsp_in1k": ["image-classification"], - "timm/mnasnet_small.lamb_in1k": ["image-classification"], - "timm/mobilenetv2_050.lamb_in1k": ["image-classification"], - "timm/mobilenetv2_100.ra_in1k": ["image-classification"], - "timm/mobilenetv2_110d.ra_in1k": ["image-classification"], - "timm/mobilenetv2_120d.ra_in1k": ["image-classification"], - "timm/mobilenetv2_140.ra_in1k": ["image-classification"], - "timm/mobilenetv3_large_100.miil_in21k_ft_in1k": ["image-classification"], - "timm/mobilenetv3_large_100.ra_in1k": ["image-classification"], - "timm/mobilenetv3_rw.rmsp_in1k": ["image-classification"], - "timm/mobilenetv3_small_050.lamb_in1k": ["image-classification"], - "timm/mobilenetv3_small_075.lamb_in1k": ["image-classification"], - "timm/mobilenetv3_small_100.lamb_in1k": ["image-classification"], - "timm/nest_tiny_jx.goog_in1k": ["image-classification"], - "timm/nf_resnet50.ra2_in1k": ["image-classification"], - "timm/nfnet_l0.ra2_in1k": ["image-classification"], - "timm/regnetx_002.pycls_in1k": ["image-classification"], - "timm/regnetx_004.pycls_in1k": ["image-classification"], - "timm/regnetx_004_tv.tv2_in1k": ["image-classification"], - "timm/regnetx_006.pycls_in1k": ["image-classification"], - "timm/regnetx_008.pycls_in1k": ["image-classification"], - "timm/regnetx_008.tv2_in1k": ["image-classification"], - "timm/regnetx_016.pycls_in1k": ["image-classification"], - "timm/regnetx_016.tv2_in1k": ["image-classification"], - "timm/regnetx_032.pycls_in1k": ["image-classification"], - "timm/regnetx_032.tv2_in1k": ["image-classification"], - "timm/regnetx_040.pycls_in1k": ["image-classification"], - "timm/regnetx_064.pycls_in1k": ["image-classification"], - "timm/regnetx_080.pycls_in1k": ["image-classification"], - "timm/regnetx_080.tv2_in1k": ["image-classification"], - "timm/regnetx_120.pycls_in1k": ["image-classification"], - "timm/regnetx_160.pycls_in1k": ["image-classification"], - "timm/regnetx_160.tv2_in1k": ["image-classification"], - "timm/regnety_002.pycls_in1k": ["image-classification"], - "timm/regnety_004.pycls_in1k": ["image-classification"], - "timm/regnety_004.tv2_in1k": ["image-classification"], - "timm/regnety_006.pycls_in1k": ["image-classification"], - "timm/regnety_008.pycls_in1k": ["image-classification"], - "timm/regnety_008_tv.tv2_in1k": ["image-classification"], - "timm/regnety_016.pycls_in1k": ["image-classification"], - "timm/regnety_016.tv2_in1k": ["image-classification"], - "timm/regnety_032.pycls_in1k": ["image-classification"], - "timm/regnety_032.ra_in1k": ["image-classification"], - "timm/regnety_032.tv2_in1k": ["image-classification"], - "timm/regnety_040.pycls_in1k": ["image-classification"], - "timm/regnety_040.ra3_in1k": ["image-classification"], - "timm/regnety_064.pycls_in1k": ["image-classification"], - "timm/regnety_064.ra3_in1k": ["image-classification"], - "timm/regnety_080.pycls_in1k": ["image-classification"], - "timm/regnety_080.ra3_in1k": ["image-classification"], - "timm/regnety_080_tv.tv2_in1k": ["image-classification"], - "timm/regnety_120.pycls_in1k": ["image-classification"], - "timm/regnety_120.sw_in12k_ft_in1k": ["image-classification"], - "timm/regnety_160.lion_in12k_ft_in1k": ["image-classification"], - "timm/regnety_160.pycls_in1k": ["image-classification"], - "timm/regnety_160.sw_in12k_ft_in1k": ["image-classification"], - "timm/regnety_160.swag_ft_in1k": ["image-classification"], - "timm/regnety_160.swag_lc_in1k": ["image-classification"], - "timm/regnety_160.tv2_in1k": ["image-classification"], - "timm/regnety_320.seer_ft_in1k": ["image-classification"], - "timm/regnety_320.swag_ft_in1k": ["image-classification"], - "timm/regnetz_040.ra3_in1k": ["image-classification"], - "timm/regnetz_040_h.ra3_in1k": ["image-classification"], - "timm/regnetz_b16.ra3_in1k": ["image-classification"], - "timm/regnetz_c16.ra3_in1k": ["image-classification"], - "timm/regnetz_d32.ra3_in1k": ["image-classification"], - "timm/regnetz_d8.ra3_in1k": ["image-classification"], - "timm/repvgg_a2.rvgg_in1k": ["image-classification"], - "timm/repvgg_b0.rvgg_in1k": ["image-classification"], - "timm/repvgg_b1.rvgg_in1k": ["image-classification"], - "timm/repvgg_b1g4.rvgg_in1k": ["image-classification"], - "timm/repvgg_b2.rvgg_in1k": ["image-classification"], - "timm/repvgg_b2g4.rvgg_in1k": ["image-classification"], - "timm/repvgg_b3.rvgg_in1k": ["image-classification"], - "timm/repvgg_b3g4.rvgg_in1k": ["image-classification"], - "timm/res2net50_14w_8s.in1k": ["image-classification"], - "timm/res2net50_26w_4s.in1k": ["image-classification"], - "timm/res2net50_26w_6s.in1k": ["image-classification"], - "timm/res2net50_26w_8s.in1k": ["image-classification"], - "timm/res2net50_48w_2s.in1k": ["image-classification"], - "timm/res2next50.in1k": ["image-classification"], - "timm/resmlp_12_224.fb_distilled_in1k": ["image-classification"], - "timm/resmlp_12_224.fb_in1k": ["image-classification"], - "timm/resmlp_24_224.fb_distilled_in1k": ["image-classification"], - "timm/resmlp_24_224.fb_in1k": ["image-classification"], - "timm/resmlp_big_24_224.fb_distilled_in1k": ["image-classification"], - "timm/resnest14d.gluon_in1k": ["image-classification"], - "timm/resnest26d.gluon_in1k": ["image-classification"], - "timm/resnest50d.in1k": ["image-classification"], - "timm/resnest50d_1s4x24d.in1k": ["image-classification"], - "timm/resnest50d_4s2x40d.in1k": ["image-classification"], - "timm/resnet101.a1h_in1k": ["image-classification"], - "timm/resnet101.gluon_in1k": ["image-classification"], - "timm/resnet101.tv_in1k": ["image-classification"], - "timm/resnet101c.gluon_in1k": ["image-classification"], - "timm/resnet101d.gluon_in1k": ["image-classification"], - "timm/resnet101d.ra2_in1k": ["image-classification"], - "timm/resnet101s.gluon_in1k": ["image-classification"], - "timm/resnet10t.c3_in1k": ["image-classification"], - "timm/resnet14t.c3_in1k": ["image-classification"], - "timm/resnet152.a1h_in1k": ["image-classification"], - "timm/resnet152.gluon_in1k": ["image-classification"], - "timm/resnet152.tv_in1k": ["image-classification"], - "timm/resnet152c.gluon_in1k": ["image-classification"], - "timm/resnet152d.gluon_in1k": ["image-classification"], - "timm/resnet152d.ra2_in1k": ["image-classification"], - "timm/resnet152s.gluon_in1k": ["image-classification"], - "timm/resnet18.a1_in1k": ["image-classification"], - "timm/resnet18.fb_ssl_yfcc100m_ft_in1k": ["image-classification"], - "timm/resnet18.fb_swsl_ig1b_ft_in1k": ["image-classification"], - "timm/resnet18.gluon_in1k": ["image-classification"], - "timm/resnet18d.ra2_in1k": ["image-classification"], - "timm/resnet200d.ra2_in1k": ["image-classification"], - "timm/resnet26.bt_in1k": ["image-classification"], - "timm/resnet26d.bt_in1k": ["image-classification"], - "timm/resnet26t.ra2_in1k": ["image-classification"], - "timm/resnet32ts.ra2_in1k": ["image-classification"], - "timm/resnet33ts.ra2_in1k": ["image-classification"], - "timm/resnet34.a1_in1k": ["image-classification"], - "timm/resnet34.gluon_in1k": ["image-classification"], - "timm/resnet34.tv_in1k": ["image-classification"], - "timm/resnet34d.ra2_in1k": ["image-classification"], - "timm/resnet50.a1_in1k": ["image-classification"], - "timm/resnet50.fb_ssl_yfcc100m_ft_in1k": ["image-classification"], - "timm/resnet50.fb_swsl_ig1b_ft_in1k": ["image-classification"], - "timm/resnet50.gluon_in1k": ["image-classification"], - "timm/resnet50.tv_in1k": ["image-classification"], - "timm/resnet50_gn.a1h_in1k": ["image-classification"], - "timm/resnet50c.gluon_in1k": ["image-classification"], - "timm/resnet50d.gluon_in1k": ["image-classification"], - "timm/resnet50d.ra2_in1k": ["image-classification"], - "timm/resnet50s.gluon_in1k": ["image-classification"], - "timm/resnet51q.ra2_in1k": ["image-classification"], - "timm/resnet61q.ra2_in1k": ["image-classification"], - "timm/resnetaa50.a1h_in1k": ["image-classification"], - "timm/resnetblur50.bt_in1k": ["image-classification"], - "timm/resnetrs101.tf_in1k": ["image-classification"], - "timm/resnetrs50.tf_in1k": ["image-classification"], - "timm/resnetv2_101.a1h_in1k": ["image-classification"], - "timm/resnetv2_50.a1h_in1k": ["image-classification"], - "timm/resnetv2_50d_gn.ah_in1k": ["image-classification"], - "timm/resnetv2_50x1_bit.goog_distilled_in1k": ["image-classification"], - "timm/resnetv2_50x1_bit.goog_in21k_ft_in1k": ["image-classification"], - "timm/resnetv2_50x3_bit.goog_in21k_ft_in1k": ["image-classification"], - "timm/resnext101_32x4d.fb_ssl_yfcc100m_ft_in1k": ["image-classification"], - "timm/resnext101_32x4d.fb_swsl_ig1b_ft_in1k": ["image-classification"], - "timm/resnext101_32x4d.gluon_in1k": ["image-classification"], - "timm/resnext101_32x8d.fb_ssl_yfcc100m_ft_in1k": ["image-classification"], - "timm/resnext101_32x8d.fb_swsl_ig1b_ft_in1k": ["image-classification"], - "timm/resnext101_32x8d.fb_wsl_ig1b_ft_in1k": ["image-classification"], - "timm/resnext101_64x4d.c1_in1k": ["image-classification"], - "timm/resnext101_64x4d.gluon_in1k": ["image-classification"], - "timm/resnext26ts.ra2_in1k": ["image-classification"], - "timm/resnext50_32x4d.a1h_in1k": ["image-classification"], - "timm/resnext50_32x4d.fb_ssl_yfcc100m_ft_in1k": ["image-classification"], - "timm/resnext50_32x4d.fb_swsl_ig1b_ft_in1k": ["image-classification"], - "timm/resnext50_32x4d.gluon_in1k": ["image-classification"], - "timm/resnext50_32x4d.tv_in1k": ["image-classification"], - "timm/resnext50d_32x4d.bt_in1k": ["image-classification"], - "timm/rexnet_100.nav_in1k": ["image-classification"], - "timm/rexnet_130.nav_in1k": ["image-classification"], - "timm/rexnet_150.nav_in1k": ["image-classification"], - "timm/rexnet_200.nav_in1k": ["image-classification"], - "timm/rexnet_300.nav_in1k": ["image-classification"], - "timm/rexnetr_200.sw_in12k_ft_in1k": ["image-classification"], - "timm/rexnetr_300.sw_in12k_ft_in1k": ["image-classification"], - "timm/sebotnet33ts_256.a1h_in1k": ["image-classification"], - "timm/semnasnet_075.rmsp_in1k": ["image-classification"], - "timm/semnasnet_100.rmsp_in1k": ["image-classification"], - "timm/seresnet33ts.ra2_in1k": ["image-classification"], - "timm/seresnet50.a1_in1k": ["image-classification"], - "timm/seresnext101_32x4d.gluon_in1k": ["image-classification"], - "timm/seresnext26d_32x4d.bt_in1k": ["image-classification"], - "timm/seresnext26t_32x4d.bt_in1k": ["image-classification"], - "timm/seresnext26ts.ch_in1k": ["image-classification"], - "timm/seresnext50_32x4d.gluon_in1k": ["image-classification"], - "timm/seresnext50_32x4d.racm_in1k": ["image-classification"], - "timm/skresnet18.ra_in1k": ["image-classification"], - "timm/skresnet34.ra_in1k": ["image-classification"], - "timm/skresnext50_32x4d.ra_in1k": ["image-classification"], - "timm/spnasnet_100.rmsp_in1k": ["image-classification"], - "timm/tf_efficientnet_el.in1k": ["image-classification"], - "timm/tf_efficientnet_em.in1k": ["image-classification"], - "timm/tf_efficientnet_es.in1k": ["image-classification"], - "timm/tf_efficientnet_lite0.in1k": ["image-classification"], - "timm/tf_efficientnet_lite1.in1k": ["image-classification"], - "timm/tf_efficientnet_lite2.in1k": ["image-classification"], - "timm/tf_efficientnet_lite3.in1k": ["image-classification"], - "timm/tf_efficientnet_lite4.in1k": ["image-classification"], - "timm/tf_efficientnetv2_b0.in1k": ["image-classification"], - "timm/tf_efficientnetv2_b1.in1k": ["image-classification"], - "timm/tf_efficientnetv2_b2.in1k": ["image-classification"], - "timm/tf_efficientnetv2_b3.in1k": ["image-classification"], - "timm/tf_efficientnetv2_b3.in21k_ft_in1k": ["image-classification"], - "timm/tf_mobilenetv3_large_minimal_100.in1k": ["image-classification"], - "timm/tf_mobilenetv3_small_075.in1k": ["image-classification"], - "timm/tf_mobilenetv3_small_100.in1k": ["image-classification"], - "timm/tf_mobilenetv3_small_minimal_100.in1k": ["image-classification"], - "timm/tinynet_a.in1k": ["image-classification"], - "timm/tinynet_d.in1k": ["image-classification"], - "timm/vgg11.tv_in1k": ["image-classification"], - "timm/vgg11_bn.tv_in1k": ["image-classification"], - "timm/vgg13.tv_in1k": ["image-classification"], - "timm/vgg13_bn.tv_in1k": ["image-classification"], - "timm/vgg16.tv_in1k": ["image-classification"], - "timm/vgg16_bn.tv_in1k": ["image-classification"], - "timm/vgg19.tv_in1k": ["image-classification"], - "timm/vgg19_bn.tv_in1k": ["image-classification"], - "timm/wide_resnet101_2.tv_in1k": ["image-classification"], - "timm/wide_resnet50_2.racm_in1k": ["image-classification"], - "timm/xception41.tf_in1k": ["image-classification"], - "timm/xception41p.ra3_in1k": ["image-classification"], - "timm/xception65.ra3_in1k": ["image-classification"], - "timm/xception65p.ra3_in1k": ["image-classification"], - "timm/xception71.tf_in1k": ["image-classification"], - } -} + if isinstance(model_names_tasks, str): # test export of all tasks on the same model + tasks = list(task_config_mapping.keys()) + model_tasks = {model_names_tasks: tasks} + else: + model_tasks = model_names_tasks # possibly, test different tasks on different models + else: + model_tasks = {model_names_tasks: tasks} + + for model_name, tasks in model_tasks.items(): + for task in tasks: + models_to_test.append( + ( + f"{model_type}_{task}_{model_name}", + model_type, + model_name, + task, + ) + ) + + return sorted(models_to_test) diff --git a/tests/ryzenai/vaip_config_transformers.json b/tests/ryzenai/vaip_config_transformers.json new file mode 100644 index 00000000..187dcfc1 --- /dev/null +++ b/tests/ryzenai/vaip_config_transformers.json @@ -0,0 +1,35 @@ +{ + "passes": [ + { + "name": "init", + "plugin": "vaip-pass_init" + }, + { + "name": "fuse_GEMM", + "plugin": "vaip-pass_py_ext", + "disabled": false, + "pyExt": { + "moduleName": "voe.passes.fuse_GEMM", + "methodName": "rules" + } + }, + { + "name": "fuse_MATMUL", + "plugin": "vaip-pass_py_ext", + "disabled": false, + "pyExt": { + "moduleName": "voe.passes.fuse_MATMUL", + "methodName": "rules" + } + }, + { + "name": "fuse_MATMULINTEGER", + "plugin": "vaip-pass_py_ext", + "disabled": false, + "pyExt": { + "moduleName": "voe.passes.fuse_MATMULINTEGER", + "methodName": "rules" + } + } + ] + } \ No newline at end of file diff --git a/utils/ryzenai/generate_operators_baseline.py b/utils/ryzenai/generate_operators_baseline.py index 33153b9e..9883e1c9 100644 --- a/utils/ryzenai/generate_operators_baseline.py +++ b/utils/ryzenai/generate_operators_baseline.py @@ -9,7 +9,7 @@ def parse_json(json_path): with open(json_path, "r") as json_file: data = json.load(json_file) - result = {"all": 0, "dpu": 0, "cpu": 0} + result = {"all": 0, "dpu": 0, "cpu": 0, "matmulinteger": 0} for entry in data["deviceStat"]: result[entry["name"].lower()] = entry["nodeNum"] return result diff --git a/utils/ryzenai/notification_service.py b/utils/ryzenai/notification_service.py index f269ee53..b579ee1b 100644 --- a/utils/ryzenai/notification_service.py +++ b/utils/ryzenai/notification_service.py @@ -294,7 +294,7 @@ def extract_model_id(self, line): def extract_operator_values(self, trace, all_baseline_value, dpu_baseline_value, cpu_baseline_value): # Extract values from trace and compare with baseline if "DPU operators do not match!" in trace or "Total operators do not match!" in trace: - match = re.search(r"\{'all': (\d+), 'dpu': (\d+), 'cpu': (\d+)\}", trace) + match = re.search(r"\{'all': (\d+), 'dpu': (\d+), 'cpu': (\d+), 'matmulinteger': (\d+)\}", trace) all_value = int(match.group(1)) dpu_value = int(match.group(2)) cpu_value = int(match.group(3)) From a8afdfe80bae9dea49a77697ef2d6e14f57fbe5c Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Tue, 19 Mar 2024 14:10:12 +0530 Subject: [PATCH 11/28] set env varibales --- tests/ryzenai/test_quantization.py | 4 ++-- tests/ryzenai/testing_models.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ryzenai/test_quantization.py b/tests/ryzenai/test_quantization.py index a1bf3507..7fc897bd 100644 --- a/tests/ryzenai/test_quantization.py +++ b/tests/ryzenai/test_quantization.py @@ -10,7 +10,7 @@ import torch from datasets import load_dataset from parameterized import parameterized -from testing_models import PYTORCH_TIMM_MODEL_SUBSET, PYTORCH_TIMM_MODELS +from testing_models import PYTORCH_TIMM_MODEL_SUBSET, PYTORCH_TIMM_MODEL from testing_utils import ( DEFAULT_CACHE_DIR, DEFAULT_VAIP_CONFIG, @@ -114,7 +114,7 @@ def test_timm_quantization_subset( ): self._quantize(model_name=model_name) - @parameterized.expand(get_models_to_test(PYTORCH_TIMM_MODELS, library_name="timm")) + @parameterized.expand(get_models_to_test(PYTORCH_TIMM_MODEL, library_name="timm")) @pytest.mark.quant_test @slow def test_timm_quantization( diff --git a/tests/ryzenai/testing_models.py b/tests/ryzenai/testing_models.py index cee8b8af..3181aefe 100644 --- a/tests/ryzenai/testing_models.py +++ b/tests/ryzenai/testing_models.py @@ -40,7 +40,7 @@ } } -PYTORCH_TIMM_MODELS = { +PYTORCH_TIMM_MODEL = { "default-timm-config": { "timm/botnet26t_256.c1_in1k": ["image-classification"], "timm/cs3darknet_focus_l.c2ns_in1k": ["image-classification"], From 2e3e0623d9913d0aef3fb35f1472cebc3760e0d4 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Tue, 19 Mar 2024 14:10:34 +0530 Subject: [PATCH 12/28] rename macro --- tests/ryzenai/test_quantization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ryzenai/test_quantization.py b/tests/ryzenai/test_quantization.py index 7fc897bd..6c5fa41e 100644 --- a/tests/ryzenai/test_quantization.py +++ b/tests/ryzenai/test_quantization.py @@ -10,7 +10,7 @@ import torch from datasets import load_dataset from parameterized import parameterized -from testing_models import PYTORCH_TIMM_MODEL_SUBSET, PYTORCH_TIMM_MODEL +from testing_models import PYTORCH_TIMM_MODEL, PYTORCH_TIMM_MODEL_SUBSET from testing_utils import ( DEFAULT_CACHE_DIR, DEFAULT_VAIP_CONFIG, From da34127ec53453e6c66468aab42590cc50911b35 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Tue, 19 Mar 2024 14:27:31 +0530 Subject: [PATCH 13/28] fix tests --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5f87421a..01fb7493 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,11 @@ "torch==2.2.1", "torchvision==0.17.1", "opencv-python", + "brevitas", + "datasets>=2.17", + "onnx", + "accelerate", + "onnx-tool", ] QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241,<=0.0.259"] @@ -34,7 +39,7 @@ EXTRAS_REQUIRE = { "quality": QUALITY_REQUIRE, "tests": TESTS_REQUIRE, - "brevitas": ["brevitas", "datasets>=2.17", "onnx", "onnxruntime", "accelerate"], + "brevitas": ["brevitas", "datasets>=2.17", "onnx", "onnxruntime", "accelerate", "onnx-tool"], } setup( From 1408e7789b4b5bb77963ed05e7ea7788c763cc9c Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Tue, 19 Mar 2024 14:51:16 +0530 Subject: [PATCH 14/28] fix tests --- optimum/amd/ryzenai/utils.py | 3 +- tests/ryzenai/testing_utils.py | 76 +++++++++++++++++----------------- 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/optimum/amd/ryzenai/utils.py b/optimum/amd/ryzenai/utils.py index 71738f44..b954f2b9 100644 --- a/optimum/amd/ryzenai/utils.py +++ b/optimum/amd/ryzenai/utils.py @@ -63,8 +63,7 @@ def set_environment_variables(): if not ryzenai_sw_path: logger.warning( "RYZENAI_SW_PATH environment variable is not set. " - "Please set it to the path of the RyzenAI-SW repository. " - "Attempting to clone RyzenAI-SW repository now..." + "Attempting to clone RyzenAI-SW repository now...\n" ) ryzenai_sw_path = normalize_path(os.path.join(os.getcwd(), "RyzenAI-SW")) clone_repository("https://github.com/amd/RyzenAI-SW/", ryzenai_sw_path) diff --git a/tests/ryzenai/testing_utils.py b/tests/ryzenai/testing_utils.py index 6ed3760c..09976795 100644 --- a/tests/ryzenai/testing_utils.py +++ b/tests/ryzenai/testing_utils.py @@ -18,6 +18,44 @@ DEFAULT_CACHE_DIR = "ryzen_cache" +def get_models_to_test( + export_models_dict: Dict, library_name: str = "timm", supported_archs: list = None, tasks: list = None +): + models_to_test = [] + for model_type, model_names_tasks in export_models_dict.items(): + if supported_archs is not None and model_type not in supported_archs: + continue + + if not isinstance(tasks, list): + tasks = [tasks] + + if tasks is None: + task_config_mapping = TasksManager.get_supported_tasks_for_model_type( + model_type, "onnx", library_name=library_name + ) + + if isinstance(model_names_tasks, str): # test export of all tasks on the same model + tasks = list(task_config_mapping.keys()) + model_tasks = {model_names_tasks: tasks} + else: + model_tasks = model_names_tasks # possibly, test different tasks on different models + else: + model_tasks = {model_names_tasks: tasks} + + for model_name, tasks in model_tasks.items(): + for task in tasks: + models_to_test.append( + ( + f"{model_type}_{task}_{model_name}", + model_type, + model_name, + task, + ) + ) + + return sorted(models_to_test) + + def parse_json(json_path): with open(json_path, "r") as json_file: data = json.load(json_file) @@ -98,41 +136,3 @@ def get_baseline_ops(self, key): with open(BASELINE_JSON, "r") as json_file: data = json.load(json_file) return data[key] - - -def get_models_to_test( - export_models_dict: Dict, library_name: str = "timm", supported_archs: list = None, tasks: list = None -): - models_to_test = [] - for model_type, model_names_tasks in export_models_dict.items(): - if model_type not in supported_archs: - continue - - if not isinstance(tasks, list): - tasks = [tasks] - - if tasks is None: - task_config_mapping = TasksManager.get_supported_tasks_for_model_type( - model_type, "onnx", library_name=library_name - ) - - if isinstance(model_names_tasks, str): # test export of all tasks on the same model - tasks = list(task_config_mapping.keys()) - model_tasks = {model_names_tasks: tasks} - else: - model_tasks = model_names_tasks # possibly, test different tasks on different models - else: - model_tasks = {model_names_tasks: tasks} - - for model_name, tasks in model_tasks.items(): - for task in tasks: - models_to_test.append( - ( - f"{model_type}_{task}_{model_name}", - model_type, - model_name, - task, - ) - ) - - return sorted(models_to_test) From fee83e333088adc0c85dadfbbe91591b30553068 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Tue, 19 Mar 2024 16:23:01 +0530 Subject: [PATCH 15/28] add marker --- .github/workflows/build_pr_documentation.yml | 2 +- optimum/amd/ryzenai/modeling.py | 3 ++- optimum/amd/ryzenai/utils.py | 5 ++--- pyproject.toml | 1 + setup.py | 15 +++++++++++---- tests/ryzenai/test_modeling.py | 3 ++- tests/ryzenai/testing_utils.py | 2 +- 7 files changed, 20 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_pr_documentation.yml b/.github/workflows/build_pr_documentation.yml index f05c59bf..c26c9739 100644 --- a/.github/workflows/build_pr_documentation.yml +++ b/.github/workflows/build_pr_documentation.yml @@ -51,7 +51,7 @@ jobs: git clone --depth 1 --branch v3.5 https://github.com/Xilinx/Vitis-AI.git && cd Vitis-AI/src/vai_quantizer/vai_q_onnx && sh build.sh && pip install pkgs/*.whl cd ../../../../optimum-amd - pip install .[brevitas,tests] + pip install .[brevitas] pip install onnxruntime==1.14.0 cd .. diff --git a/optimum/amd/ryzenai/modeling.py b/optimum/amd/ryzenai/modeling.py index 249d7a5d..e31f1b54 100644 --- a/optimum/amd/ryzenai/modeling.py +++ b/optimum/amd/ryzenai/modeling.py @@ -127,7 +127,8 @@ def __init__( **kwargs, ) - self.model_type = config.model_type + if config: + self.model_type = config.model_type self.inputs_names = {input_key.name: idx for idx, input_key in enumerate(model.get_inputs())} self.output_names = {output_key.name: idx for idx, output_key in enumerate(model.get_outputs())} diff --git a/optimum/amd/ryzenai/utils.py b/optimum/amd/ryzenai/utils.py index b954f2b9..3833630c 100644 --- a/optimum/amd/ryzenai/utils.py +++ b/optimum/amd/ryzenai/utils.py @@ -45,7 +45,7 @@ def set_builtins(): def clone_repository(repo_url, repo_path): - if repo_path not in os.listdir(): + if not os.path.exists(repo_path): subprocess.run(["git", "clone", "--depth", "1", "--branch", "main", repo_url, repo_path]) @@ -62,8 +62,7 @@ def set_environment_variables(): ryzenai_sw_path = os.environ.get("RYZENAI_SW_PATH") if not ryzenai_sw_path: logger.warning( - "RYZENAI_SW_PATH environment variable is not set. " - "Attempting to clone RyzenAI-SW repository now...\n" + "RYZENAI_SW_PATH environment variable is not set. Attempting to clone RyzenAI-SW repository now...\n" ) ryzenai_sw_path = normalize_path(os.path.join(os.getcwd(), "RyzenAI-SW")) clone_repository("https://github.com/amd/RyzenAI-SW/", ryzenai_sw_path) diff --git a/pyproject.toml b/pyproject.toml index a52f4f9a..4a9d2fc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ known-first-party = ["transformers"] [tool.pytest.ini_options] markers = [ + "brevitas_transformers_test", "prequantized_model_test", "quant_test", ] \ No newline at end of file diff --git a/setup.py b/setup.py index 01fb7493..b619c073 100644 --- a/setup.py +++ b/setup.py @@ -14,24 +14,31 @@ assert False, "Error: Could not open '%s' due %s\n" % (filepath, error) # ORT 1.16 is not compatible: https://github.com/Xilinx/Vitis-AI/issues/1343 -INSTALL_REQUIRE = ["optimum", "transformers>=4.38", "onnx", "onnxruntime-extensions"] +INSTALL_REQUIRE = [ + "optimum", + "transformers>=4.38", + "onnx", + "onnxruntime-extensions", + "opencv-python", + "timm", + "torch", + "torchvision", + "onnx-tool", +] # TODO: unpin pytest once https://github.com/huggingface/transformers/pull/29154 is merged & released TESTS_REQUIRE = [ "pytest<=7.4.4", "parameterized", "evaluate", - "timm", "scikit-learn", "onnxruntime", "torch==2.2.1", "torchvision==0.17.1", - "opencv-python", "brevitas", "datasets>=2.17", "onnx", "accelerate", - "onnx-tool", ] QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241,<=0.0.259"] diff --git a/tests/ryzenai/test_modeling.py b/tests/ryzenai/test_modeling.py index a0e28e0e..648538ac 100644 --- a/tests/ryzenai/test_modeling.py +++ b/tests/ryzenai/test_modeling.py @@ -318,7 +318,8 @@ class RyzenAIModelForCausalLMIntegrationTest(unittest.TestCase, RyzenAITestCaseM tasks="text-generation-with-past", ) ) - @pytest.mark.brevitas_ryzen_test + @slow + @pytest.mark.brevitas_transformers_test def test_model(self, test_name: str, model_type: str, model_id: str, task: str): dataset_name = "wikitext2" num_calib_samples = 10 diff --git a/tests/ryzenai/testing_utils.py b/tests/ryzenai/testing_utils.py index 09976795..9217ec6f 100644 --- a/tests/ryzenai/testing_utils.py +++ b/tests/ryzenai/testing_utils.py @@ -26,7 +26,7 @@ def get_models_to_test( if supported_archs is not None and model_type not in supported_archs: continue - if not isinstance(tasks, list): + if tasks is not None and not isinstance(tasks, list): tasks = [tasks] if tasks is None: From 0ca3e8fe934d65e8bc9ced46c8ee1b405d81f002 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Tue, 19 Mar 2024 16:25:20 +0530 Subject: [PATCH 16/28] update setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b619c073..fbc922d7 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,6 @@ "timm", "torch", "torchvision", - "onnx-tool", ] # TODO: unpin pytest once https://github.com/huggingface/transformers/pull/29154 is merged & released @@ -39,6 +38,7 @@ "datasets>=2.17", "onnx", "accelerate", + "onnx-tool", ] QUALITY_REQUIRE = ["black~=23.1", "ruff>=0.0.241,<=0.0.259"] From 95cccdae03f76a62606827c5c5439af074d6e9ea Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Tue, 19 Mar 2024 18:03:52 +0530 Subject: [PATCH 17/28] update notifications --- .github/workflows/test_ryzenai_nightly.yaml | 8 +++ pyproject.toml | 2 +- tests/ryzenai/test_modeling.py | 3 +- utils/ryzenai/notification_service.py | 64 ++++++++++++++------- 4 files changed, 54 insertions(+), 23 deletions(-) diff --git a/.github/workflows/test_ryzenai_nightly.yaml b/.github/workflows/test_ryzenai_nightly.yaml index f573d1a1..8645a801 100644 --- a/.github/workflows/test_ryzenai_nightly.yaml +++ b/.github/workflows/test_ryzenai_nightly.yaml @@ -27,6 +27,13 @@ jobs: timeout_minutes: 1200 secrets: hf_hub_read_token: ${{ secrets.HF_READ_TOKEN }} + run_tests_brevitas_quantized_decoder_llms: + uses: huggingface/hf-workflows/.github/workflows/ryzenai_ci.yaml@main + with: + pytest_marker: "brevitas_quantized_decoder_llms_test" + test_file: "tests/ryzenai/test_modeling.py" + report_name: "tests_brevitas_quantized_decoder_llms" + slow_test: true send_results: name: Send results to webhook runs-on: ubuntu-22.04 @@ -34,6 +41,7 @@ jobs: needs: [ run_tests_prequantized_models, run_tests_quantization, + run_tests_brevitas_quantized_decoder_llms, ] steps: - uses: actions/checkout@v3 diff --git a/pyproject.toml b/pyproject.toml index 4a9d2fc9..63e02fcd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ known-first-party = ["transformers"] [tool.pytest.ini_options] markers = [ - "brevitas_transformers_test", + "brevitas_quantized_decoder_llms_test", "prequantized_model_test", "quant_test", ] \ No newline at end of file diff --git a/tests/ryzenai/test_modeling.py b/tests/ryzenai/test_modeling.py index 648538ac..94d87f80 100644 --- a/tests/ryzenai/test_modeling.py +++ b/tests/ryzenai/test_modeling.py @@ -318,8 +318,7 @@ class RyzenAIModelForCausalLMIntegrationTest(unittest.TestCase, RyzenAITestCaseM tasks="text-generation-with-past", ) ) - @slow - @pytest.mark.brevitas_transformers_test + @pytest.mark.brevitas_quantized_decoder_llms_test def test_model(self, test_name: str, model_type: str, model_id: str, task: str): dataset_name = "wikitext2" num_calib_samples = 10 diff --git a/utils/ryzenai/notification_service.py b/utils/ryzenai/notification_service.py index b579ee1b..9407fafb 100644 --- a/utils/ryzenai/notification_service.py +++ b/utils/ryzenai/notification_service.py @@ -27,7 +27,7 @@ sys.path.append(os.path.join(os.getcwd())) -import tests.ryzenai.testing_utils as tu # noqa +import tests.ryzenai.testing_models as tu # noqa client = WebClient(token=os.environ["CI_SLACK_BOT_TOKEN"]) @@ -36,18 +36,15 @@ def infer_model_id(model): model_name_replacement = model.replace(".", "_").replace("-", "_") - if "timm" in model: - all_model_names = list(tu.PYTORCH_TIMM_MODEL["default-timm-config"].keys()) - elif "amd" in model: - all_model_names = ( - tu.RYZEN_PREQUANTIZED_MODEL_IMAGE_CLASSIFICATION - + list(tu.RYZEN_PREQUANTIZED_MODEL_OBJECT_DETECTION.values()) - + tu.RYZEN_PREQUANTIZED_MODEL_IMAGE_SEGMENTATION - + tu.RYZEN_PREQUANTIZED_MODEL_IMAGE_TO_IMAGE - + tu.RYZEN_PREQUANTIZED_MODEL_CUSTOM_TASKS - ) - else: - return model + all_model_names = ( + list(tu.PYTORCH_TIMM_MODEL["default-timm-config"].keys()) + + tu.RYZEN_PREQUANTIZED_MODEL_IMAGE_CLASSIFICATION + + list(tu.RYZEN_PREQUANTIZED_MODEL_OBJECT_DETECTION.values()) + + tu.RYZEN_PREQUANTIZED_MODEL_IMAGE_SEGMENTATION + + tu.RYZEN_PREQUANTIZED_MODEL_IMAGE_TO_IMAGE + + tu.RYZEN_PREQUANTIZED_MODEL_CUSTOM_TASKS + + list(tu.PYTORCH_MODELS.values()) + ) for model_name in all_model_names: if model_name.replace(".", "_").replace("-", "_") == model_name_replacement: @@ -259,15 +256,22 @@ def model_failures(self): cpu_baseline_value = baseline_ops.get("cpu", 0) dpu_baseline_value = baseline_ops.get("dpu", 0) all_baseline_value = baseline_ops.get("all", 0) + matmulinteger_baseline_value = baseline_ops.get("matmulinteger", 0) # Extract and compare values from the failure trace - all_value_str, dpu_value_str, cpu_value_str, regressed = self.extract_operator_values( - trace, all_baseline_value, dpu_baseline_value, cpu_baseline_value + ( + all_value_str, + dpu_value_str, + cpu_value_str, + matmulinteger_value_str, + regressed, + ) = self.extract_operator_values( + trace, all_baseline_value, dpu_baseline_value, cpu_baseline_value, matmulinteger_baseline_value ) # Append information about the failure failures_info.append( - f"{all_value_str.rjust(9)} | {dpu_value_str.rjust(9)} | {cpu_value_str.rjust(9)} | {regressed.rjust(4)} | {model_id[:40]}" + f"{all_value_str.rjust(9)} | {dpu_value_str.rjust(9)} | {cpu_value_str.rjust(9)} | {matmulinteger_value_str.rjust(13)} | {regressed.rjust(4)} | {model_id[:40]}" ) if len(failures_info): @@ -288,16 +292,27 @@ def extract_model_id(self, line): match = re.search(r"default_timm_config_image_classification_timm_(\w+)", line) if match: return "timm/" + match.group(1) + else: + match = re.search(r"text_generation_with_past_(\w+)", line) + if match: + return match.group(1) raise ValueError("Model id could not be determined!") - def extract_operator_values(self, trace, all_baseline_value, dpu_baseline_value, cpu_baseline_value): + def extract_operator_values( + self, trace, all_baseline_value, dpu_baseline_value, cpu_baseline_value, matmulinteger_baseline_value + ): # Extract values from trace and compare with baseline - if "DPU operators do not match!" in trace or "Total operators do not match!" in trace: + if ( + "DPU operators do not match!" in trace + or "Total operators do not match!" in trace + or "MATMULINTEGERs do not match!" in trace + ): match = re.search(r"\{'all': (\d+), 'dpu': (\d+), 'cpu': (\d+), 'matmulinteger': (\d+)\}", trace) all_value = int(match.group(1)) dpu_value = int(match.group(2)) cpu_value = int(match.group(3)) + matmulinteger_value = int(match.group(4)) # Process values and compare with baseline all_value_str = f"{all_value}({all_baseline_value})" if "Total" in trace else str(all_value) @@ -307,15 +322,23 @@ def extract_operator_values(self, trace, all_baseline_value, dpu_baseline_value, cpu_value_str = ( f"{cpu_value}({cpu_baseline_value})" if cpu_value != cpu_baseline_value else str(cpu_baseline_value) ) + matmulinteger_value_str = ( + f"{matmulinteger_value}({matmulinteger_baseline_value})" + if matmulinteger_value != matmulinteger_baseline_value + else str(matmulinteger_baseline_value) + ) + if matmulinteger_baseline_value: + dpu_value_str = "-" regressed = "Y" else: # No regression, do not print values cpu_value_str = "-" dpu_value_str = "-" all_value_str = "-" + matmulinteger_value_str = "-" regressed = "N" - return all_value_str, dpu_value_str, cpu_value_str, regressed + return all_value_str, dpu_value_str, cpu_value_str, matmulinteger_value_str, regressed def prepare_model_failure_sections(self, idx, key, job_link, failures_info): # Prepare sections for model failures @@ -338,7 +361,7 @@ def prepare_model_failure_sections(self, idx, key, job_link, failures_info): ) # Section for detailed failure reports - model_header = "Total Ops | DPU Ops | CPU Ops | Reg. | Model\n" + model_header = "Total Ops | DPU Ops | CPU Ops | MatMulInt Ops | Reg. | Model\n" model_failures_report = prepare_reports(title="", header=model_header, reports=failures_info) model_failure_sections.append( @@ -661,6 +684,7 @@ def prepare_reports(title, header, reports, to_truncate=True): test_categories = { "Pre-Quantized Model": "run_tests_prequantized_models", "Timm Quantization": "run_tests_quantization", + "Brevitas Quantized Decoder LLMs": "run_tests_brevitas_quantized_decoder_llms", } results = { From 34be37a7c938be650adc7086325e57ae04c4f5dd Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Tue, 19 Mar 2024 19:31:42 +0530 Subject: [PATCH 18/28] update cloning --- optimum/amd/ryzenai/utils.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/optimum/amd/ryzenai/utils.py b/optimum/amd/ryzenai/utils.py index 3833630c..4283cbbc 100644 --- a/optimum/amd/ryzenai/utils.py +++ b/optimum/amd/ryzenai/utils.py @@ -5,6 +5,7 @@ import builtins import logging import os +import shutil import subprocess import onnxruntime as ort @@ -23,6 +24,8 @@ DEFAULT_BUILTIN_IMPL = "v0" DEFAULT_BUILTIN_QUANT_MODE = "w8a8" +RYZEN_SW_COMMIT_HASH = "82c524a06693a18e167f032dbf5574a98dd24452" + def validate_provider_availability(provider: str): """ @@ -44,9 +47,15 @@ def set_builtins(): builtins.quant_mode = os.getenv("BUILTINS_IMPL", DEFAULT_BUILTIN_QUANT_MODE) -def clone_repository(repo_url, repo_path): - if not os.path.exists(repo_path): - subprocess.run(["git", "clone", "--depth", "1", "--branch", "main", repo_url, repo_path]) +def clone_repository(repo_url: str, repo_path: str): + try: + if not os.path.exists(repo_path): + subprocess.run(["git", "clone", "--depth", "1", "--branch", "main", repo_url, repo_path], check=True) + subprocess.run(["git", "-C", repo_path, "checkout", RYZEN_SW_COMMIT_HASH], check=True) + except subprocess.CalledProcessError as e: + print(f"Error: {e}") + if os.path.exists(repo_path): + shutil.rmtree(repo_path) def set_env_var(key, value): From 4df250eaf456ee1e4511e8659ad1d8b0be287ca2 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Tue, 19 Mar 2024 19:32:34 +0530 Subject: [PATCH 19/28] add comment for has --- optimum/amd/ryzenai/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/optimum/amd/ryzenai/utils.py b/optimum/amd/ryzenai/utils.py index 4283cbbc..b0486137 100644 --- a/optimum/amd/ryzenai/utils.py +++ b/optimum/amd/ryzenai/utils.py @@ -24,6 +24,7 @@ DEFAULT_BUILTIN_IMPL = "v0" DEFAULT_BUILTIN_QUANT_MODE = "w8a8" +# The commit hash of the RyzenAI-SW (https://github.com/amd/RyzenAI-SW/) repository to use RYZEN_SW_COMMIT_HASH = "82c524a06693a18e167f032dbf5574a98dd24452" From e46b1bccb115c9f9f2aed9404e318c7a58b21d7f Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Tue, 19 Mar 2024 20:21:31 +0530 Subject: [PATCH 20/28] add error checks --- optimum/amd/ryzenai/utils.py | 43 +++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/optimum/amd/ryzenai/utils.py b/optimum/amd/ryzenai/utils.py index b0486137..a3d31ab3 100644 --- a/optimum/amd/ryzenai/utils.py +++ b/optimum/amd/ryzenai/utils.py @@ -46,6 +46,7 @@ def set_builtins(): """Set the builtins.impl and builtins.quant_mode environment variables.""" builtins.impl = os.getenv("BUILTINS_IMPL", DEFAULT_BUILTIN_IMPL) builtins.quant_mode = os.getenv("BUILTINS_IMPL", DEFAULT_BUILTIN_QUANT_MODE) + print(f"Builtins: impl={builtins.impl}, quant_mode={builtins.quant_mode}") def clone_repository(repo_url: str, repo_path: str): @@ -68,6 +69,22 @@ def normalize_path(path): return os.path.normpath(path) +def check_env_path_exists(env_var_name): + paths = os.environ.get(env_var_name) + if ";" in paths: + paths = paths.strip(";").split(";") + elif "," in paths: + paths = paths.strip(",").split(",") + else: + paths = [paths] + + for path in paths: + if not os.path.exists(path): + raise OSError( + f"The path '{path}' does not exist. Please ensure that the `{env_var_name}` environment variable is set correctly!" + ) + + def set_environment_variables(): ryzenai_sw_path = os.environ.get("RYZENAI_SW_PATH") if not ryzenai_sw_path: @@ -76,27 +93,37 @@ def set_environment_variables(): ) ryzenai_sw_path = normalize_path(os.path.join(os.getcwd(), "RyzenAI-SW")) clone_repository("https://github.com/amd/RyzenAI-SW/", ryzenai_sw_path) + else: + if not os.path.exists(ryzenai_sw_path): + raise OSError( + f"The path '{ryzenai_sw_path}' does not exist. Please ensure that the `RYZENAI_SW_PATH` environment variable " + "is set correctly!" + ) - # Set other environment variables ryzenai_transformers_path = normalize_path(os.path.join(ryzenai_sw_path, "example/transformers")) third_party = normalize_path(os.path.join(ryzenai_transformers_path, "third_party")) + device = os.environ.get("DEVICE", DEFAULT_DEVICE) set_env_var("THIRD_PARTY", third_party) + check_env_path_exists("THIRD_PARTY") + set_env_var( "TVM_LIBRARY_PATH", - f"{normalize_path(os.path.join(third_party, 'lib'))};{normalize_path(os.path.join(third_party, 'bin'))}", - ) - set_env_var("DEVICE", DEFAULT_DEVICE) - set_env_var( - "XLNX_VART_FIRMWARE", - normalize_path(os.path.join(ryzenai_transformers_path, "xclbin", os.environ.get("DEVICE", DEFAULT_DEVICE))), + normalize_path(os.path.join(third_party, "lib")) + ";" + normalize_path(os.path.join(third_party, "bin")), ) + check_env_path_exists("TVM_LIBRARY_PATH") + + set_env_var("XLNX_VART_FIRMWARE", normalize_path(os.path.join(ryzenai_transformers_path, "xclbin", device))) + check_env_path_exists("XLNX_VART_FIRMWARE") - dll_path = os.path.join(ryzenai_transformers_path, "dll", os.environ.get("DEVICE", DEFAULT_DEVICE)) + dll_path = normalize_path(os.path.join(ryzenai_transformers_path, "dll", device)) tvm_module_paths = [] for dll_file in DEFAULT_DLL_FILES: tvm_module_paths.append(normalize_path(os.path.join(dll_path, dll_file))) set_env_var("TVM_MODULE_PATH", ",".join(tvm_module_paths) + ",") + check_env_path_exists("TVM_MODULE_PATH") + + set_env_var("DEVICE", DEFAULT_DEVICE) set_env_var("TVM_GEMM_M", DEFAULT_TVM_GEMM_M) set_env_var("TVM_DLL_NUM", DEFAULT_TVM_DLL_NUM) From fd86dbd64f2e5ce78fb2b3a1728165d62e191bec Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Fri, 22 Mar 2024 19:44:42 +0530 Subject: [PATCH 21/28] update eapis --- tests/ryzenai/rewriter.py | 242 --------------------------------- tests/ryzenai/test_modeling.py | 19 +-- 2 files changed, 2 insertions(+), 259 deletions(-) delete mode 100644 tests/ryzenai/rewriter.py diff --git a/tests/ryzenai/rewriter.py b/tests/ryzenai/rewriter.py deleted file mode 100644 index 2c32f69c..00000000 --- a/tests/ryzenai/rewriter.py +++ /dev/null @@ -1,242 +0,0 @@ -import pathlib - -import onnx -from onnx_tool import Model -from onnx_tool.fusion import FusionPattern -from onnx_tool.node import create_node -from onnx_tool.tensor import Tensor - -from optimum.onnx.graph_transformations import check_and_save_model - - -## Pattern to find and replace with MatMulInteger -MatMul = [ - { - "name": "deq_linear_0", - "op": "DequantizeLinear", - "attrs": [], - "inport": [], - "outport": [[0, "transpose_0", 0]], - }, - { - "name": "transpose_0", - "op": "Transpose", - "attrs": [], - "inport": [[0, "deq_linear_0", 0]], - "outport": [[0, "matmul_0", 1]], - }, - { - "name": "quant_linear_1", - "op": "DynamicQuantizeLinear", - "attrs": [], - "inport": [], - "outport": [[0, "deq_linear_1", 0], [1, "deq_linear_1", 1], [2, "deq_linear_1", 2]], - }, - { - "name": "deq_linear_1", - "op": "DequantizeLinear", - "attrs": [], - "inport": [ - [0, "quant_linear_1", 0], - [1, "quant_linear_1", 1], - [2, "quant_linear_1", 2], - ], - "outport": [[0, "matmul_0", 0]], - }, - { - "name": "matmul_0", - "op": "MatMul", - "attrs": [], - "inport": [ - [0, "deq_linear_1", 0], - [1, "transpose_0", 0], - ], - "outport": [], - }, -] - -GEMM = [ - { - "name": "deq_linear_0", - "op": "DequantizeLinear", - "attrs": [], - "inport": [], - "outport": [[0, "gemm_0", 1]], - }, - { - "name": "quant_linear_1", - "op": "DynamicQuantizeLinear", - "attrs": [], - "inport": [], - "outport": [[0, "deq_linear_1", 0], [1, "deq_linear_1", 1], [2, "deq_linear_1", 2]], - }, - { - "name": "deq_linear_1", - "op": "DequantizeLinear", - "attrs": [], - "inport": [ - [0, "quant_linear_1", 0], - [1, "quant_linear_1", 1], - [2, "quant_linear_1", 2], - ], - "outport": [[0, "gemm_0", 0]], - }, - { - "name": "gemm_0", - "op": "Gemm", - "attrs": [], - "inport": [ - [0, "deq_linear_1", 0], - [1, "deq_linear_0", 0], - ], - "outport": [], - }, -] - - -def create_nodes(graph, op, name, inputs, outputs, intermediate=None, **kwargs): - if intermediate is None: - intermediate = [] - newnode = onnx.helper.make_node(op, inputs + intermediate, outputs, name=name, **kwargs) - newnode = create_node(newnode) - newnode.input = inputs + intermediate - newnode.output = outputs - for i in inputs: - if i in graph.consumedby: - graph.consumedby[i].append(name) - if i in graph.producedby.keys(): - newnode.prevnodes.append(graph.producedby[i]) - for o in outputs: - graph.producedby[o] = [name] - if o in graph.consumedby.keys(): - newnode.nextnodes.append(graph.consumedby[o]) - graph.nodemap[name] = newnode - graph.tensormap[name] = Tensor(name) - - return graph - - -def replace_matmul_to_matmulinteger(compute_graph, found_nodes): - for i, found_pattern in enumerate(found_nodes): - deq_linear = compute_graph.nodemap[found_pattern[0]] - dyn_q = compute_graph.nodemap[found_pattern[2]] - dq_weight = deq_linear.prevnodes[0] - compute_graph.add_initial(f"dq_weights_0_{i}", dq_weight.value.transpose()) - compute_graph.add_initial(f"dq_weights_1_{i}", deq_linear.prevnodes[1].value) - compute_graph.add_initial(f"dq_weights_2_{i}", deq_linear.prevnodes[2].value) - - matmul = compute_graph.nodemap[found_pattern[-1]] - for name in found_pattern: - if "DynamicQuantizeLinear" in name: - continue - compute_graph.remove_node(name) - - compute_graph.remove_node(deq_linear.prevnodes[0].name) - compute_graph.remove_node(deq_linear.prevnodes[1].name) - if deq_linear.prevnodes[2].name in compute_graph.nodemap: - compute_graph.remove_node(deq_linear.prevnodes[2].name) - - compute_graph = create_nodes( - compute_graph, - "MatMulInteger", - f"matmul_integer_{i}", - [dyn_q.output[0], f"dq_weights_0_{i}", dyn_q.output[2], f"dq_weights_2_{i}"], - [f"matmul_integer_{i}"], - ) - compute_graph = create_nodes( - compute_graph, "Cast", f"cast_{i}", [f"matmul_integer_{i}"], [f"cast_{i}"], to=int(1) - ) - compute_graph = create_nodes( - compute_graph, "Mul", f"mulscales_{i}", [dyn_q.output[1], f"dq_weights_1_{i}"], [f"mulscales_{i}"] - ) - compute_graph = create_nodes( - compute_graph, "Mul", f"mulvalues_{i}", [f"mulscales_{i}", f"cast_{i}"], [matmul.output[0]] - ) - return compute_graph - - -def replace_gemm_to_matmulinteger(compute_graph, found_nodes): - k = 100 - for i, found_pattern in enumerate(found_nodes): - k = i + 100 - gemm = compute_graph.nodemap[found_pattern[-1]] - bias = gemm.input[-1] - deq_linear = compute_graph.nodemap[found_pattern[0]] - dyn_q = compute_graph.nodemap[found_pattern[1]] - dq_weight = deq_linear.prevnodes[0] - compute_graph.add_initial(f"dq_weights_0_{k}", dq_weight.value.transpose()) - compute_graph.add_initial(f"dq_weights_1_{k}", deq_linear.prevnodes[1].value) - compute_graph.add_initial(f"dq_weights_2_{k}", deq_linear.prevnodes[2].value) - - matmul = compute_graph.nodemap[found_pattern[-1]] - for name in found_pattern: - if "DynamicQuantizeLinear" in name: - continue - compute_graph.remove_node(name) - compute_graph.remove_node(deq_linear.prevnodes[0].name) - compute_graph.remove_node(deq_linear.prevnodes[1].name) - if deq_linear.prevnodes[2].name in compute_graph.nodemap: - compute_graph.remove_node(deq_linear.prevnodes[2].name) - - compute_graph = create_nodes( - compute_graph, - "MatMulInteger", - f"matmul_integer_{k}", - [dyn_q.output[0], f"dq_weights_0_{k}", dyn_q.output[2], f"dq_weights_2_{k}"], - [f"matmul_integer_{k}"], - ) - compute_graph = create_nodes( - compute_graph, "Cast", f"cast_{k}", [f"matmul_integer_{k}"], [f"cast_{k}"], to=int(1) - ) - compute_graph = create_nodes( - compute_graph, "Mul", f"mulscales_{k}", [dyn_q.output[1], f"dq_weights_1_{k}"], [f"mulscales_{k}"] - ) - compute_graph = create_nodes( - compute_graph, "Mul", f"mulvalues_{k}", [f"mulscales_{k}", f"cast_{k}"], [f"mulvalues_{k}"] - ) - compute_graph = create_nodes( - compute_graph, "Add", f"addbias_{k}", [bias, f"mulvalues_{k}"], [matmul.output[0]] - ) - return compute_graph - - -def find_and_insert_matmulinteger(model_path): - print("Rewriting ONNX Graph with MatMulInteger ") - - cfg = {"constant_folding": False, "node_rename": False, "if_fixed_branch": None, "fixed_topk": 0, "verbose": True} - original_output = onnx.load(model_path).graph.output - model = Model(model_path, cfg) - graph = model.graph - - print("Replacing MatMul with MatMulInteger") - pattern = FusionPattern(MatMul) - found_nodes = pattern.search_pattern(graph) - graph = replace_matmul_to_matmulinteger(graph, found_nodes) - - print("Replacing GEMM with MatMulInteger + Add") - pattern = FusionPattern(GEMM) - found_nodes = pattern.search_pattern(graph) - graph = replace_gemm_to_matmulinteger(graph, found_nodes) - - graph.graph_reorder_nodes() - - print("Saving the new ONNX model") - full_path = pathlib.Path(model_path) - - graph = graph.make_graph_onnx( - graph.nodemap.keys(), "graph", graph.input, graph.output, with_initializer=True, with_shape_info=False - ) - - attr = {"producer_name": "onnx_tool"} - model_to_save = onnx.helper.make_model(graph, **attr) - # onnx_tools might remove the output nodes from the ONNX graph, so we need to restore it. - for out in original_output: - if out not in model_to_save.graph.output: - model_to_save.graph.output.append(out) - - model_to_save.ir_version = model.mproto.ir_version - model_to_save.opset_import.pop() - for opset in model.mproto.opset_import: - model_to_save.opset_import.append(opset) - - check_and_save_model(model_to_save, full_path) diff --git a/tests/ryzenai/test_modeling.py b/tests/ryzenai/test_modeling.py index 24af5797..d2a81716 100644 --- a/tests/ryzenai/test_modeling.py +++ b/tests/ryzenai/test_modeling.py @@ -13,11 +13,8 @@ import pytest import requests import torch -from brevitas.export.onnx.standard.qcdq.manager import StdQCDQONNXManager -from brevitas_examples.llm.llm_quant.export import brevitas_proxy_export_mode from parameterized import parameterized from PIL import Image -from rewriter import find_and_insert_matmulinteger from testing_models import ( PYTORCH_MODELS, RYZEN_PREQUANTIZED_MODEL_CUSTOM_TASKS, @@ -36,6 +33,7 @@ from optimum.amd import BrevitasQuantizationConfig, BrevitasQuantizer from optimum.amd.brevitas.data_utils import get_dataset_for_model +from optimum.amd.brevitas.export import onnx_export_from_quantized_model from optimum.amd.ryzenai import ( RyzenAIModel, RyzenAIModelForCausalLM, @@ -46,7 +44,6 @@ RyzenAIModelForSemanticSegmentation, pipeline, ) -from optimum.exporters.onnx import onnx_export_from_model from optimum.utils import ( DummyInputGenerator, logging, @@ -327,7 +324,6 @@ def test_model(self, test_name: str, model_type: str, model_id: str, task: str): tokenizer = AutoTokenizer.from_pretrained(model_id) - # TODO: Replace with the new method in brevitas quantization_config = BrevitasQuantizationConfig( apply_gptq=False, apply_weight_equalization=False, @@ -356,18 +352,7 @@ def test_model(self, test_name: str, model_type: str, model_id: str, task: str): quantized_model = quantizer.quantize(quantization_config, calibration_dataset) # export model - # TODO: Replace with the new method in brevitas - export_manager = StdQCDQONNXManager - with torch.no_grad(), brevitas_proxy_export_mode(quantized_model, export_manager=export_manager): - onnx_export_from_model( - quantized_model, - quantization_dir.name, - task="text-generation-with-past", - do_validation=False, - no_post_process=True, - ) - - find_and_insert_matmulinteger(os.path.join(quantization_dir.name, "model.onnx")) + onnx_export_from_quantized_model(quantized_model, quantization_dir.name) # inference cache_dir = DEFAULT_CACHE_DIR From af34ab541bdda5c673614a1c255f7fe4ca7b317a Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Tue, 26 Mar 2024 17:57:09 +0530 Subject: [PATCH 22/28] fix error msgs --- optimum/amd/ryzenai/modeling_decoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/amd/ryzenai/modeling_decoder.py b/optimum/amd/ryzenai/modeling_decoder.py index 76599f5a..e5c1b4c1 100644 --- a/optimum/amd/ryzenai/modeling_decoder.py +++ b/optimum/amd/ryzenai/modeling_decoder.py @@ -82,7 +82,7 @@ def _initialize_params(self, use_cache, generation_config): raise ValueError( f"`use_cache` was set to `{use_cache}` but the loaded model only supports `use_cache={self.use_cache}`. " f"Please load your current model with `use_cache={self.use_cache}` or export the original model " - f"once again with `use_cache={use_cache}` when calling the `from_pretrained` method. " + f"once again with past-key-values." ) def forward(self, input_ids, attention_mask=None, position_ids=None, past_key_values=None, **kwargs): From ec3e4635fbdac5460ae58a1e38434f00832239e7 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Wed, 27 Mar 2024 13:18:04 +0530 Subject: [PATCH 23/28] review comments --- optimum/amd/ryzenai/modeling.py | 77 +++++++++++++++++-------- optimum/amd/ryzenai/modeling_decoder.py | 22 ++----- optimum/amd/ryzenai/utils.py | 4 +- tests/ryzenai/testing_models.py | 1 + tests/ryzenai/testing_utils.py | 4 +- utils/ryzenai/notification_service.py | 27 ++++++--- 6 files changed, 81 insertions(+), 54 deletions(-) diff --git a/optimum/amd/ryzenai/modeling.py b/optimum/amd/ryzenai/modeling.py index 4cb5b410..1b21955c 100644 --- a/optimum/amd/ryzenai/modeling.py +++ b/optimum/amd/ryzenai/modeling.py @@ -109,7 +109,7 @@ def shared_attributes_init( def __init__( self, model: ort.InferenceSession, - config: PretrainedConfig, + config: Optional[PretrainedConfig] = None, vaip_config: Union[str, Path] = None, model_save_dir: Optional[Union[str, Path, TemporaryDirectory]] = None, preprocessors: Optional[List] = None, @@ -128,9 +128,6 @@ def __init__( **kwargs, ) - if config: - self.model_type = config.model_type - self.inputs_names = {input_key.name: idx for idx, input_key in enumerate(model.get_inputs())} self.output_names = {output_key.name: idx for idx, output_key in enumerate(model.get_outputs())} @@ -176,12 +173,6 @@ def load_model( else: providers_options = None - # is_dynamic = RyzenAIModel._check_uses_static_shape(path) - # if is_dynamic and provider == "VitisAIExecutionProvider": - # raise ValueError( - # "The model provided has dynamic axes in input/output. Please provide model with static shapes for inference with RyzenAI." - # ) - return ort.InferenceSession( path, providers=providers, @@ -299,6 +290,8 @@ def _load_model_and_processors( preprocessors = None if model_path.is_dir(): + cls.validate_static_shape_compatibility(model_path / file_name, provider) + model = RyzenAIModel.load_model( model_path / file_name, provider=provider, @@ -335,6 +328,8 @@ def _load_model_and_processors( # model doesn't use external data pass + cls.validate_static_shape_compatibility(model_cache_path, provider) + model = RyzenAIModel.load_model( model_cache_path, provider=provider, @@ -440,21 +435,41 @@ def from_pretrained( **kwargs, ): """ - provider (`str`, defaults to `"VitisAIExecutionProvider"`): - ONNX Runtime provider to use for loading the model. See https://onnxruntime.ai/docs/execution-providers/ for - possible providers. - session_options (`Optional[onnxruntime.SessionOptions]`, defaults to `None`),: - ONNX Runtime session options to use for loading the model. - provider_options (`Optional[Dict[str, Any]]`, defaults to `None`): - Provider option dictionaries corresponding to the provider used. See available options - for each provider: https://onnxruntime.ai/docs/api/c/group___global.html . - kwargs (`Dict[str, Any]`): - Will be passed to the underlying model loading methods. - - > Parameters for decoder models (RyzenAIForSpeechSeq2Seq) - - use_cache (`Optional[bool]`, defaults to `True`): - Whether or not past key/values cache should be used. Defaults to `True`. + Instantiate a RyzenAIModel model from a model identifier. + + Args: + model_id (`Union[str, Path]`): + The model identifier to instantiate the model from. + vaip_config (`str`, defaults to `None`): + The path to the Vitis AI config file. + export (`bool`, defaults to `False`): + Whether to export the model to ONNX before loading it. + force_download (`bool`, defaults to `False`): + Whether to force the download of the model files. + use_auth_token (`Optional[str]`, defaults to `None`): + The authorization token to use for downloading the model. + cache_dir (`Optional[str]`, defaults to `None`): + The directory to cache the model files. + subfolder (`str`, defaults to `""`): + The subfolder to look for the model files. + config (`Optional[PretrainedConfig]`, defaults to `None`): + The configuration to use for the model. + local_files_only (`bool`, defaults to `False`): + Whether to only look for the model files locally. + provider (`str`, defaults to `"VitisAIExecutionProvider"`): + ONNX Runtime provider to use for loading the model. + session_options (`Optional[onnxruntime.SessionOptions]`, defaults to `None`): + ONNX Runtime session options to use for loading the model. + provider_options (`Optional[Dict[str, Any]]`, defaults to `None`): + Provider option dictionaries corresponding to the provider used. + trust_remote_code (`bool`, defaults to `False`): + Whether to trust the remote code when exporting the model. + revision (`Optional[str]`, defaults to `None`): + The revision of the model to load. + library_name (`Optional[Dict[str, Any]]`, defaults to `None`): + The library name to use for the model. + kwargs (`Dict[str, Any]`): + Will be passed to the underlying model loading methods. Returns: `RyzenAIModel`: The loaded RyzenAIModel model. @@ -604,6 +619,10 @@ def reshape( return model_path + @staticmethod + def validate_static_shape_compatibility(path: Union[str, Path], provider: str): + raise True + def _convert_to_numpy(self, value, use_torch): return value.cpu().detach().numpy() if use_torch else value @@ -718,6 +737,14 @@ def _export( **kwargs, ) + @staticmethod + def validate_static_shape_compatibility(path: Union[str, Path], provider: str): + is_dynamic = RyzenAIModel._check_uses_static_shape(path) + if is_dynamic and provider == "VitisAIExecutionProvider": + raise ValueError( + "The model provided has dynamic axes in input/output. Please provide model with static shapes for inference with RyzenAI." + ) + class RyzenAIModelForObjectDetection(RyzenAIModelForCustomTasks): def forward(self, pixel_values): diff --git a/optimum/amd/ryzenai/modeling_decoder.py b/optimum/amd/ryzenai/modeling_decoder.py index e5c1b4c1..2b2a2288 100644 --- a/optimum/amd/ryzenai/modeling_decoder.py +++ b/optimum/amd/ryzenai/modeling_decoder.py @@ -10,11 +10,12 @@ import onnxruntime as ort import torch -from optimum.utils import NormalizedConfigManager, check_if_transformers_greater +from optimum.utils import NormalizedConfigManager from transformers import ( AutoModelForCausalLM, GenerationConfig, ) +from transformers.generation import GenerationMixin from transformers.modeling_outputs import CausalLMOutputWithPast from .modeling import RyzenAIModel @@ -24,11 +25,6 @@ if TYPE_CHECKING: from transformers import PretrainedConfig -if check_if_transformers_greater("4.25.0"): - from transformers.generation import GenerationMixin -else: - from transformers.generation_utils import GenerationMixin - class RyzenAIModelForCausalLM(RyzenAIModel, GenerationMixin): """ @@ -53,14 +49,6 @@ def __init__( self._initialize_params(use_cache, generation_config) - self.use_fp16 = False - for inp in model.get_inputs(): - if ( - inp.name == "past_key_values" or inp.name in self.key_value_input_names - ) and inp.type == "tensor(float16)": - self.use_fp16 = True - break - # need for generate self.device = torch.device("cpu") @@ -70,6 +58,9 @@ def _get_key_value_names(self): return key_names, value_names def _initialize_params(self, use_cache, generation_config): + if self.config is None: + raise ValueError("The model config must be provided to instantiate the model.") + self.num_pkv = 2 self.normalized_config = NormalizedConfigManager.get_normalized_config_class(self.config.model_type)( self.config @@ -177,8 +168,7 @@ def prepare_past_key_values( def get_constructor(self, use_torch): constructor = torch if use_torch else np - dtype = constructor.float16 if self.use_fp16 else constructor.float32 - return constructor, dtype + return constructor, constructor.float32 @classmethod def _from_pretrained( diff --git a/optimum/amd/ryzenai/utils.py b/optimum/amd/ryzenai/utils.py index a3d31ab3..11ef56e3 100644 --- a/optimum/amd/ryzenai/utils.py +++ b/optimum/amd/ryzenai/utils.py @@ -46,7 +46,7 @@ def set_builtins(): """Set the builtins.impl and builtins.quant_mode environment variables.""" builtins.impl = os.getenv("BUILTINS_IMPL", DEFAULT_BUILTIN_IMPL) builtins.quant_mode = os.getenv("BUILTINS_IMPL", DEFAULT_BUILTIN_QUANT_MODE) - print(f"Builtins: impl={builtins.impl}, quant_mode={builtins.quant_mode}") + logger.info(f"Builtins: impl={builtins.impl}, quant_mode={builtins.quant_mode}") def clone_repository(repo_url: str, repo_path: str): @@ -55,7 +55,7 @@ def clone_repository(repo_url: str, repo_path: str): subprocess.run(["git", "clone", "--depth", "1", "--branch", "main", repo_url, repo_path], check=True) subprocess.run(["git", "-C", repo_path, "checkout", RYZEN_SW_COMMIT_HASH], check=True) except subprocess.CalledProcessError as e: - print(f"Error: {e}") + logger.error(f"Error: {e}") if os.path.exists(repo_path): shutil.rmtree(repo_path) diff --git a/tests/ryzenai/testing_models.py b/tests/ryzenai/testing_models.py index 3181aefe..23998c05 100644 --- a/tests/ryzenai/testing_models.py +++ b/tests/ryzenai/testing_models.py @@ -360,6 +360,7 @@ PYTORCH_MODELS = { + "gpt_bigcode": "hf-internal-testing/tiny-random-GPTBigCodeModel", "llama": "fxmarty/tiny-llama-fast-tokenizer", "mistral": "echarlaix/tiny-random-mistral", "opt": "hf-internal-testing/tiny-random-OPTForCausalLM", diff --git a/tests/ryzenai/testing_utils.py b/tests/ryzenai/testing_utils.py index 5a12b396..6f7e1242 100644 --- a/tests/ryzenai/testing_utils.py +++ b/tests/ryzenai/testing_utils.py @@ -11,7 +11,7 @@ SEED = 42 -BASELINE_JSON = os.path.normpath("./tests/ryzenai/operators_baseline.json") # For RyzenSDK 1.1 +BASELINE_OPERATORS_JSON = os.path.normpath("./tests/ryzenai/operators_baseline.json") # For RyzenSDK 1.1 DEFAULT_VAIP_CONFIG = os.path.normpath("./tests/ryzenai/vaip_config.json") DEFAULT_VAIP_CONFIG_TRANSFORMERS = os.path.normpath("./tests/ryzenai/vaip_config_transformers.json") @@ -133,6 +133,6 @@ def get_ops(self, cache_dir, cache_key): return result def get_baseline_ops(self, key): - with open(BASELINE_JSON, "r") as json_file: + with open(BASELINE_OPERATORS_JSON, "r") as json_file: data = json.load(json_file) return data[key] diff --git a/utils/ryzenai/notification_service.py b/utils/ryzenai/notification_service.py index 9407fafb..7fec9744 100644 --- a/utils/ryzenai/notification_service.py +++ b/utils/ryzenai/notification_service.py @@ -27,7 +27,16 @@ sys.path.append(os.path.join(os.getcwd())) -import tests.ryzenai.testing_models as tu # noqa +from tests.ryzenai.testing_models import ( # noqa + PYTORCH_MODELS, + PYTORCH_TIMM_MODEL, + RYZEN_PREQUANTIZED_MODEL_CUSTOM_TASKS, + RYZEN_PREQUANTIZED_MODEL_IMAGE_CLASSIFICATION, + RYZEN_PREQUANTIZED_MODEL_IMAGE_SEGMENTATION, + RYZEN_PREQUANTIZED_MODEL_IMAGE_TO_IMAGE, + RYZEN_PREQUANTIZED_MODEL_OBJECT_DETECTION, +) +from tests.ryzenai.testing_utils import BASELINE_OPERATORS_JSON # noqa client = WebClient(token=os.environ["CI_SLACK_BOT_TOKEN"]) @@ -37,13 +46,13 @@ def infer_model_id(model): model_name_replacement = model.replace(".", "_").replace("-", "_") all_model_names = ( - list(tu.PYTORCH_TIMM_MODEL["default-timm-config"].keys()) - + tu.RYZEN_PREQUANTIZED_MODEL_IMAGE_CLASSIFICATION - + list(tu.RYZEN_PREQUANTIZED_MODEL_OBJECT_DETECTION.values()) - + tu.RYZEN_PREQUANTIZED_MODEL_IMAGE_SEGMENTATION - + tu.RYZEN_PREQUANTIZED_MODEL_IMAGE_TO_IMAGE - + tu.RYZEN_PREQUANTIZED_MODEL_CUSTOM_TASKS - + list(tu.PYTORCH_MODELS.values()) + list(PYTORCH_TIMM_MODEL["default-timm-config"].keys()) + + RYZEN_PREQUANTIZED_MODEL_IMAGE_CLASSIFICATION + + list(RYZEN_PREQUANTIZED_MODEL_OBJECT_DETECTION.values()) + + RYZEN_PREQUANTIZED_MODEL_IMAGE_SEGMENTATION + + RYZEN_PREQUANTIZED_MODEL_IMAGE_TO_IMAGE + + RYZEN_PREQUANTIZED_MODEL_CUSTOM_TASKS + + list(PYTORCH_MODELS.values()) ) for model_name in all_model_names: @@ -217,7 +226,7 @@ def category_failures(self) -> Dict: @property def model_failures(self): # Load baseline data from a JSON file - with open(tu.BASELINE_JSON, "r") as json_file: + with open(BASELINE_OPERATORS_JSON, "r") as json_file: baseline_data = json.load(json_file) model_failure_sections = [] From d8ac6280aea7a9d24719b8de11837d487735018f Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Wed, 27 Mar 2024 13:37:16 +0530 Subject: [PATCH 24/28] review comments --- optimum/amd/ryzenai/modeling.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/optimum/amd/ryzenai/modeling.py b/optimum/amd/ryzenai/modeling.py index 1b21955c..4230d414 100644 --- a/optimum/amd/ryzenai/modeling.py +++ b/optimum/amd/ryzenai/modeling.py @@ -346,6 +346,10 @@ def _load_model_and_processors( return model, vaip_config, model_save_dir, preprocessors + @staticmethod + def validate_static_shape_compatibility(path: Union[str, Path], provider: str): + return True + @classmethod def _from_pretrained( cls, @@ -619,10 +623,6 @@ def reshape( return model_path - @staticmethod - def validate_static_shape_compatibility(path: Union[str, Path], provider: str): - raise True - def _convert_to_numpy(self, value, use_torch): return value.cpu().detach().numpy() if use_torch else value From 9ebfe574d46eb9a236a67fbf60a9b0d3aa15b779 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Thu, 28 Mar 2024 17:30:00 +0530 Subject: [PATCH 25/28] fixed errors --- tests/ryzenai/operators_baseline.json | 5 + tests/ryzenai/test_modeling.py | 7 +- tests/ryzenai/vaip_config.json | 306 -------------------- tests/ryzenai/vaip_config_transformers.json | 35 --- 4 files changed, 8 insertions(+), 345 deletions(-) delete mode 100644 tests/ryzenai/vaip_config.json delete mode 100644 tests/ryzenai/vaip_config_transformers.json diff --git a/tests/ryzenai/operators_baseline.json b/tests/ryzenai/operators_baseline.json index 200c7338..467a0c4b 100644 --- a/tests/ryzenai/operators_baseline.json +++ b/tests/ryzenai/operators_baseline.json @@ -99,6 +99,11 @@ "cpu": 327, "matmulinteger": 15 }, + "hf-internal-testing_tiny-random-gptbigcodemodel": { + "all": 449, + "cpu": 428, + "matmulinteger": 21 + }, "hf-internal-testing_tiny-random-optforcausallm": { "all": 658, "cpu": 627, diff --git a/tests/ryzenai/test_modeling.py b/tests/ryzenai/test_modeling.py index 95126c88..a63e60e8 100644 --- a/tests/ryzenai/test_modeling.py +++ b/tests/ryzenai/test_modeling.py @@ -25,7 +25,6 @@ ) from testing_utils import ( DEFAULT_CACHE_DIR, - DEFAULT_VAIP_CONFIG_TRANSFORMERS, RyzenAITestCaseMixin, get_models_to_test, ) @@ -292,6 +291,7 @@ def test_model(self, model_id): class RyzenAIModelForCausalLMIntegrationTest(unittest.TestCase, RyzenAITestCaseMixin): SUPPORTED_ARCHITECTURES = { + "gpt_bigcode", "opt", "llama", "mistral", @@ -347,20 +347,19 @@ def test_model(self, test_name: str, model_type: str, model_id: str, task: str): # inference cache_dir = DEFAULT_CACHE_DIR cache_key = model_id.replace("/", "_").lower() - vaip_config = DEFAULT_VAIP_CONFIG_TRANSFORMERS prompt = "Hey, are you conscious? Can you talk to me?" inputs = tokenizer(prompt, return_tensors="np") ort_inputs = {key: np.array(inputs[key], dtype=np.int64) for key in inputs.keys()} - if model_type in {"llama", "mistral"}: + if model_type in {"llama", "mistral", "gpt_bigcode"}: attention_mask = torch.tensor(inputs["attention_mask"]) position_ids = attention_mask.long().cumsum(-1) - 1 position_ids.masked_fill_(attention_mask == 0, 1) ort_inputs["position_ids"] = position_ids.numpy() outputs_ipu, outputs_cpu = self.prepare_outputs( - quantization_dir.name, RyzenAIModelForCausalLM, ort_inputs, vaip_config, cache_dir, cache_key + quantization_dir.name, RyzenAIModelForCausalLM, ort_inputs, cache_dir, cache_key ) for output_ipu, output_cpu in zip(outputs_ipu.values(), outputs_cpu.values()): diff --git a/tests/ryzenai/vaip_config.json b/tests/ryzenai/vaip_config.json deleted file mode 100644 index fbca88f0..00000000 --- a/tests/ryzenai/vaip_config.json +++ /dev/null @@ -1,306 +0,0 @@ -{ - "passes": [ - { - "name": "init", - "plugin": "vaip-pass_init" - }, - { - "name": "fuse_resize_norm", - "plugin": "vaip-pass_py_ext", - "disabled": false, - "pyExt": { - "moduleName": "voe.passes.fuse_resize_norm", - "methodName": "rules" - } - }, - { - "name": "fuse_softmax", - "plugin": "vaip-pass_py_ext", - "disabled": false, - "pyExt": { - "moduleName": "voe.passes.fuse_softmax", - "methodName": "rules" - } - }, - { - "name": "fuse_topk", - "plugin": "vaip-pass_py_ext", - "disabled": false, - "pyExt": { - "moduleName": "voe.passes.fuse_topk", - "methodName": "rules" - } - }, - { - "name": "fuse_decode_filter_boxes", - "plugin": "vaip-pass_py_ext", - "disabled": false, - "pyExt": { - "moduleName": "voe.passes.fuse_decode_filter_boxes", - "methodName": "rules" - } - }, - { - "name": "fuse_NMS", - "plugin": "vaip-pass_py_ext", - "disabled": true, - "pyExt": { - "moduleName": "voe.passes.fuse_NMS", - "methodName": "rules" - } - }, - { - "name": "fuse_DPU", - "plugin": "vaip-pass_level1_dpu", - "passDpuParam": { - "subPass": [ - { - "_comment" : "# issue 1048", - "name": "convert_ending_blacklist_ops_to_unknown_op", - "plugin": "vaip-pass_convert_ending_blacklist_ops_to_unknown_op", - "disabled": false - }, - { - "_comment" : "test case : yolov5s6", - "name": "manual_partition", - "plugin": "vaip-pass_manual_partition", - "disabled": true, - "manualPartition": { - "fromOps": [ - "1745/duplicated_token_14", - "1764/duplicated_token_10", - "1783/duplicated_token_6", - "1802/duplicated_token_2" - ], - "toOps": [ - "2895" - ] - } - }, - { - "name": "dynamic_input_batch", - "plugin": "vaip-pass_dynamic_input_batch" - }, - { - "_comment" : "test case q_operator_resnet50", - "name": "convert_qlinear_to_qdq", - "plugin": "vaip-pass_py_ext", - "disabled": true, - "enableGc": true, - "pyExt": { - "moduleName": "voe.passes.convert_qlinear_to_qdq", - "methodName": "rules" - } - }, - { - "name": "create_const_op", - "plugin": "vaip-pass_create_const_op" - }, - { - "name": "convert_to_xir_op", - "plugin": "vaip-pass_py_ext", - "disabled" : false, - "pyExt": { - "moduleName": "voe.passes.convert_to_xir_op", - "methodName": "rules" - } - }, - { - "name": "to_xir", - "plugin": "vaip-pass_to_xir_ops" - }, - { - "name": "remove_extra_q_dq", - "plugin": "vaip-pass_remove_extra_q_dq" - }, - { - "name": "merge_add_into_conv_bias", - "plugin": "vaip-pass_merge_add_into_conv_bias" - }, - { - "name": "merge_fix", - "plugin": "vaip-pass_py_ext", - "enableGc": true, - "pyExt": { - "moduleName": "voe.passes.merge_fix", - "methodName": "rules" - } - }, - { - "name": "layoutransform", - "plugin": "vaip-pass_layout_transform_via_adding_transpose" - }, - { - "name": "gc_after_layout_transform", - "plugin": "vaip-pass_remove_isolated_node" - }, - { - "name": "fuse_transpose", - "plugin": "vaip-pass_fuse_transpose", - "enableGc": true - }, - { - "name": "gc_after_fuse_transpose", - "plugin": "vaip-pass_remove_isolated_node" - }, - { - "name": "remove_identity", - "plugin": "vaip-pass_remove_identity", - "logVerbosity": 1 - }, - { - "name": "add_fix_after_const", - "plugin": "vaip-pass_const_add_fix" - }, - { - "_comment" : "test case 41 see issue #611 #626 for more detail", - "name": "merge_duplicated_fix", - "plugin": "vaip-pass_merge_duplicated_fix", - "disabled": true, - "enableGc": true - }, - { - "_comment": "test case 112", - "name": "remove_reshape_fix", - "plugin": "vaip-pass_py_ext", - "pyExt": { - "moduleName": "voe.passes.remove_reshape_fix", - "methodName": "rules" - } - }, - { - "_comment" : "test case 5", - "name": "const_fold_batchnorm_to_scale", - "plugin": "vaip-pass_py_ext", - "pyExt": { - "moduleName": "voe.passes.const_fold_batchnorm_to_scale", - "methodName": "rules" - } - }, - { - "name": "const_fold_transpose", - "plugin": "vaip-pass_const_fold_transpose" - }, - { - "name": "merge_pad", - "plugin": "vaip-pass_merge_pad" - }, - { - "name": "merge_hard_sigmoid", - "plugin": "vaip-pass_merge_hard_sigmoid" - }, - { - "_comment" : "test case 112", - "name": "merge_mul", - "plugin": "vaip-pass_py_ext", - "pyExt": { - "moduleName": "voe.passes.merge_mul", - "methodName": "rules" - } - }, - { - "name": "merge_consecutive_fix", - "plugin": "vaip-pass_merge_consecutive_fix", - "disabled": true, - "enableLog": true, - "logVerbosity": 1 - }, - { - "name": "graph_output_add_node", - "plugin": "vaip-pass_graph_output_add_node", - "disabled": true - }, - { - "_comment" : "test case 20", - "name": "convert_transpose_add_fix_input_fix_input", - "plugin": "vaip-pass_py_ext", - "disabled": true, - "pyExt": { - "moduleName": "voe.passes.convert_transpose_add_fix_input_fix_input", - "methodName": "process" - } - }, - { - "_comment" : "test case 100", - "name": "convert_transpose_fix_pad_fix_input", - "plugin": "vaip-pass_py_ext", - "disabled": true, - "pyExt": { - "moduleName": "voe.passes.convert_transpose_fix_pad_fix_input", - "methodName": "process" - } - }, - { - "_comment" : "test case 100", - "name": "convert_transpose_fix_input", - "plugin": "vaip-pass_py_ext", - "enableGc": true, - "disabled": true, - "pyExt": { - "moduleName": "voe.passes.convert_transpose_fix_input", - "methodName": "process" - } - }, - { - "_comment": "test case 110", - "name": "convert_softmax_to_hard_softmax", - "plugin": "vaip-pass_py_ext", - "disabled" : true, - "pyExt": { - "moduleName": "voe.passes.convert_softmax_to_hard_softmax", - "methodName": "rules" - } - }, - { - "_comment": "test case 43", - "name": "remove_top_transpose", - "plugin": "vaip-pass_merge_input_transpose", - "disabled": true, - "enableGc": true - }, - { - "_comment": "test case 110", - "name": "remove_bottom_transpose", - "plugin": "vaip-pass_remove_bottom_transpose", - "disabled": true, - "enableGc": true - }, - { - "name": "final_gc", - "plugin": "vaip-pass_remove_isolated_node" - } - ], - "xcompilerAttrs": { - "debug_mode" : { - "stringValue" : "performance" - }, - "dpu_subgraph_num" : { - "intValue" : 32 - }, - "opt_level" : { - "intValue" : 0 - }, - "dump_subgraph_ops" : { - "boolValue" : false - }, - "profile" : { - "intValue" : 0 - }, - "prefetch" : { - "boolValue" : false - }, - "preassign" : { - "boolValue" : false - }, - "disable_std_quant" : { - "boolValue" : false - }, - "concat_skip_code_gen" : { - "boolValue" : false - } - }, - "minimum_num_of_conv": 2 - } - } - ] -} diff --git a/tests/ryzenai/vaip_config_transformers.json b/tests/ryzenai/vaip_config_transformers.json deleted file mode 100644 index 187dcfc1..00000000 --- a/tests/ryzenai/vaip_config_transformers.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "passes": [ - { - "name": "init", - "plugin": "vaip-pass_init" - }, - { - "name": "fuse_GEMM", - "plugin": "vaip-pass_py_ext", - "disabled": false, - "pyExt": { - "moduleName": "voe.passes.fuse_GEMM", - "methodName": "rules" - } - }, - { - "name": "fuse_MATMUL", - "plugin": "vaip-pass_py_ext", - "disabled": false, - "pyExt": { - "moduleName": "voe.passes.fuse_MATMUL", - "methodName": "rules" - } - }, - { - "name": "fuse_MATMULINTEGER", - "plugin": "vaip-pass_py_ext", - "disabled": false, - "pyExt": { - "moduleName": "voe.passes.fuse_MATMULINTEGER", - "methodName": "rules" - } - } - ] - } \ No newline at end of file From b175108602f46fc2a45728dbcb00d69bc2c279f5 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Thu, 28 Mar 2024 18:02:38 +0530 Subject: [PATCH 26/28] add pipeline --- optimum/amd/ryzenai/models/__init__.py | 2 +- optimum/amd/ryzenai/pipelines/__init__.py | 119 +++++++++++++++++----- tests/brevitas/test_onnx_export.py | 2 +- tests/brevitas/test_quantization.py | 5 +- 4 files changed, 98 insertions(+), 30 deletions(-) diff --git a/optimum/amd/ryzenai/models/__init__.py b/optimum/amd/ryzenai/models/__init__.py index e341da49..9e7263c3 100644 --- a/optimum/amd/ryzenai/models/__init__.py +++ b/optimum/amd/ryzenai/models/__init__.py @@ -22,7 +22,7 @@ from .semanticfpn import SemanticFPNImageProcessor from .yolov3 import YoloV3ImageProcessor from .yolov5 import YoloV5ImageProcessor - from .yolov8 import YoloV3ImageProcessor + from .yolov8 import YoloV8ImageProcessor from .yolox import YoloXImageProcessor else: import sys diff --git a/optimum/amd/ryzenai/pipelines/__init__.py b/optimum/amd/ryzenai/pipelines/__init__.py index 7b9c0051..ca602f0f 100644 --- a/optimum/amd/ryzenai/pipelines/__init__.py +++ b/optimum/amd/ryzenai/pipelines/__init__.py @@ -2,10 +2,17 @@ # Licensed under the MIT License. -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Tuple, Union from optimum.exporters import TasksManager -from transformers import ImageClassificationPipeline, Pipeline, PretrainedConfig +from transformers import ( + ImageClassificationPipeline, + Pipeline, + PretrainedConfig, + PreTrainedTokenizer, + PreTrainedTokenizerFast, + TextGenerationPipeline, +) from transformers import pipeline as transformers_pipeline from transformers.image_processing_utils import BaseImageProcessor from transformers.onnx.utils import get_preprocessor @@ -16,6 +23,7 @@ RyzenAIModelForObjectDetection, RyzenAIModelForSemanticSegmentation, ) +from ..modeling_decoder import RyzenAIModelForCausalLM from ..models import ( HRNetImageProcessor, SemanticFPNImageProcessor, @@ -48,13 +56,6 @@ "default": "amd/resnet50", "type": "image", }, - "object-detection": { - "impl": YoloObjectDetectionPipeline, - "class": (RyzenAIModelForObjectDetection,), - "default": "amd/yolox-s", - "type": "image", - "model_type": "yolox", - }, "image-segmentation": { "impl": ImageSegmentationPipeline, "class": (RyzenAIModelForSemanticSegmentation,), @@ -62,6 +63,18 @@ "type": "image", "model_type": "semantic_fpn", }, + "text-generation": { + "impl": TextGenerationPipeline, + "class": (RyzenAIModelForCausalLM,), + "type": "text", + }, + "object-detection": { + "impl": YoloObjectDetectionPipeline, + "class": (RyzenAIModelForObjectDetection,), + "default": "amd/yolox-s", + "type": "image", + "model_type": "yolox", + }, } @@ -101,11 +114,70 @@ def load_model( return model, model_id, model_type +def get_processor_from_model(model: RyzenAIModel, type: Tuple[Any], name: str) -> Any: + for preprocessor in model.preprocessors: + if isinstance(preprocessor, type): + return preprocessor + if preprocessor is None: + raise ValueError( + f"Could not automatically find a {name} for the model, you must specify the argument `{name}` explicitly." + ) + return None + + +def get_processor( + task: str, + model: RyzenAIModel, + model_id: Optional[str] = None, + tokenizer: Optional[Union[str, PreTrainedTokenizer, PreTrainedTokenizerFast]] = None, + image_processor: Optional[Union[str, BaseImageProcessor]] = None, + feature_extractor: Optional[Union[str, "PreTrainedFeatureExtractor"]] = None, +) -> Tuple[Union[PreTrainedTokenizer, PreTrainedTokenizerFast], BaseImageProcessor]: + supported_tasks = RYZENAI_SUPPORTED_TASKS + + no_image_processor_tasks, no_tokenizer_tasks = get_task_processor_map(supported_tasks) + + load_tokenizer = False if task in no_tokenizer_tasks else True + load_image_processor = False if task in no_image_processor_tasks else True + + if tokenizer is None and load_tokenizer: + tokenizer = ( + get_preprocessor(model_id) + if model_id + else get_processor_from_model(model, (PreTrainedTokenizer, PreTrainedTokenizerFast), "tokenizer") + ) + + if image_processor is None and feature_extractor is None and load_image_processor: + library_name = TasksManager._infer_library_from_model(model) + if library_name != "timm": + image_processor = ( + get_preprocessor(model_id) + if model_id + else get_processor_from_model(model, BaseImageProcessor, "image_processor") + ) + + return tokenizer, image_processor + + +def get_task_processor_map(supported_tasks): + no_image_processor_tasks = set() + no_tokenizer_tasks = set() + for _task, values in supported_tasks.items(): + if values["type"] == "text": + no_image_processor_tasks.add(_task) + elif values["type"] == "image": + no_tokenizer_tasks.add(_task) + else: + raise ValueError(f"SUPPORTED_TASK {_task} contains invalid type {values['type']}") + return no_image_processor_tasks, no_tokenizer_tasks + + def pipeline( task, model: Optional[Any] = None, vaip_config: Optional[str] = None, model_type: Optional[str] = None, + tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, feature_extractor: Optional[Union[str, "PreTrainedFeatureExtractor"]] = None, image_processor: Optional[Union[str, BaseImageProcessor]] = None, use_fast: bool = True, @@ -123,6 +195,8 @@ def pipeline( task (`str`): The task defining which pipeline will be returned. Available tasks include: - "image-classification" + - "image-segmentation" + - "text-generation" - "object-detection" model (`Optional[Any]`, defaults to `None`): The model that will be used by the pipeline to make predictions. This can be a model identifier or an @@ -132,6 +206,9 @@ def pipeline( extracted during installation under the name `vaip_config.json`. model_type (`Optional[str]`, defaults to `None`): Model type for the model + tokenizer (`Optional[Union[str, PreTrainedTokenizer]]`, defaults to `None`): + The tokenizer that will be used by the pipeline to encode data for the model. This can be a model identifier + or an actual pretrained tokenizer. feature_extractor (`Union[str, "PreTrainedFeatureExtractor"]`, defaults to `None`): The feature extractor that will be used by the pipeline to encode data for the model. This can be a model identifier or an actual pretrained feature extractor. @@ -161,6 +238,7 @@ def pipeline( revision=revision, ) + ryzen_pipeline = transformers_pipeline if model.config is None: if model_type is None: raise ValueError( @@ -177,30 +255,19 @@ def pipeline( model.config = PretrainedConfig.from_dict({}) else: - library_name = TasksManager._infer_library_from_model(model) + library_name = TasksManager._infer_library_from_model(model) if task == "image-classification" else None - if library_name != "timm" and image_processor is None: - if model_id: - if feature_extractor is None and image_processor is None: - image_processor = get_preprocessor(model_id) - else: - for preprocessor in model.preprocessors: - if isinstance(preprocessor, BaseImageProcessor): - image_processor = preprocessor - break - if image_processor is None: - raise ValueError( - "Could not automatically find an image processor for the model, you must specifiy the argument `image_processor` explicitly." - ) - - if task == "image-classification" and library_name == "timm": + if library_name == "timm": ryzen_pipeline = TimmImageClassificationPipeline else: - ryzen_pipeline = transformers_pipeline + tokenizer, image_processor = get_processor( + task, model, model_id, tokenizer, image_processor, feature_extractor + ) return ryzen_pipeline( task=task, model=model, + tokenizer=tokenizer, feature_extractor=feature_extractor, image_processor=image_processor, use_fast=use_fast, diff --git a/tests/brevitas/test_onnx_export.py b/tests/brevitas/test_onnx_export.py index 836d21b8..f92bdb32 100644 --- a/tests/brevitas/test_onnx_export.py +++ b/tests/brevitas/test_onnx_export.py @@ -9,11 +9,11 @@ import onnx import torch -from brevitas.export.onnx.standard.qcdq.manager import StdQCDQONNXManager from brevitas_examples.llm.llm_quant.export import brevitas_proxy_export_mode from parameterized import parameterized from testing_utils import SUPPORTED_MODELS_TINY, VALIDATE_EXPORT_ON_SHAPES, get_quantized_model +from brevitas.export.onnx.standard.qcdq.manager import StdQCDQONNXManager from optimum.amd.brevitas.export import find_and_insert_matmulinteger from optimum.exporters import TasksManager from optimum.exporters.onnx import ( diff --git a/tests/brevitas/test_quantization.py b/tests/brevitas/test_quantization.py index e37992f1..91a81ec0 100644 --- a/tests/brevitas/test_quantization.py +++ b/tests/brevitas/test_quantization.py @@ -4,11 +4,12 @@ import unittest import torch -from brevitas.nn.quant_linear import QuantLinear -from brevitas.proxy.runtime_quant import ActQuantProxyFromInjector, DynamicActQuantProxyFromInjector from parameterized import parameterized from testing_utils import SUPPORTED_MODELS_TINY, get_quantized_model +from brevitas.nn.quant_linear import QuantLinear +from brevitas.proxy.runtime_quant import ActQuantProxyFromInjector, DynamicActQuantProxyFromInjector + def _get_all_model_ids(model_type: str): if isinstance(SUPPORTED_MODELS_TINY[model_type], str): From e0cb89f63287724efe4d31d096e95d7910b211a2 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Thu, 28 Mar 2024 18:12:00 +0530 Subject: [PATCH 27/28] add pipeline --- optimum/amd/ryzenai/pipelines/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optimum/amd/ryzenai/pipelines/__init__.py b/optimum/amd/ryzenai/pipelines/__init__.py index ca602f0f..7fa64894 100644 --- a/optimum/amd/ryzenai/pipelines/__init__.py +++ b/optimum/amd/ryzenai/pipelines/__init__.py @@ -88,7 +88,7 @@ def load_model( revision: str = "main", ): if model is None: - if task != "object-detection": + if task in {"image-classification", "text-generation"}: raise ValueError("Creating pipeline without model for the task is not supported!") model_id = SUPPORTED_TASKS[task]["default"] From bb53a9b163fdf57fed41af177a41c387399c95a1 Mon Sep 17 00:00:00 2001 From: Mohit Sharma Date: Mon, 8 Apr 2024 15:15:06 +0530 Subject: [PATCH 28/28] add group --- optimum/amd/ryzenai/modeling.py | 7 ++++ optimum/amd/ryzenai/modeling_decoder.py | 6 +++- optimum/amd/ryzenai/utils.py | 46 +++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/optimum/amd/ryzenai/modeling.py b/optimum/amd/ryzenai/modeling.py index 266d398a..46a829a1 100644 --- a/optimum/amd/ryzenai/modeling.py +++ b/optimum/amd/ryzenai/modeling.py @@ -175,6 +175,13 @@ def load_model( else: providers_options = None + from .utils import matmul_group_onnx + + # from pdb import set_trace; set_trace() + + path = matmul_group_onnx(path) + # from pdb import set_trace; set_trace() + return ort.InferenceSession( path, providers=providers, diff --git a/optimum/amd/ryzenai/modeling_decoder.py b/optimum/amd/ryzenai/modeling_decoder.py index 6dd509f7..3363e4d3 100644 --- a/optimum/amd/ryzenai/modeling_decoder.py +++ b/optimum/amd/ryzenai/modeling_decoder.py @@ -19,7 +19,11 @@ from transformers.modeling_outputs import CausalLMOutputWithPast from .modeling import RyzenAIModel -from .utils import DEFAULT_VAIP_CONFIG_TRANSFORMERS_EAGER_MODE, set_builtins, set_environment_variables +from .utils import ( + DEFAULT_VAIP_CONFIG_TRANSFORMERS_EAGER_MODE, + set_builtins, + set_environment_variables, +) if TYPE_CHECKING: diff --git a/optimum/amd/ryzenai/utils.py b/optimum/amd/ryzenai/utils.py index 3440a223..8b82e047 100644 --- a/optimum/amd/ryzenai/utils.py +++ b/optimum/amd/ryzenai/utils.py @@ -7,6 +7,8 @@ import os import shutil import subprocess +import sys +from pathlib import Path import onnxruntime as ort @@ -55,6 +57,48 @@ def set_builtins(): logger.info(f"Builtins: impl={builtins.impl}, quant_mode={builtins.quant_mode}") +def set_paths(paths): + for path in paths: + sys.path.append(path) + + +def restore_paths(paths): + for path in paths: + sys.path.remove(path) + + +def matmul_group_onnx(model_path: str): + model_path = normalize_path(model_path) + + check_env_path_exists("RYZENAI_SW_PATH") + ryzenai_sw_path = os.environ.get("RYZENAI_SW_PATH") + + onnx_graph_path = normalize_path(os.path.join(ryzenai_sw_path, "example", "transformers", "onnx-ops", "python")) + onnx_group_matmul_path = normalize_path(os.path.join(onnx_graph_path, "group", "matmulint")) + sys_paths = [onnx_graph_path, onnx_group_matmul_path] + + model_path = Path(model_path) + group_model = normalize_path(os.path.join(model_path.parent, "grouped_" + model_path.name)) + + set_paths(sys_paths) + from onnx_group import GroupMatMulInteger + + g = GroupMatMulInteger(model_path) + status = g.group() + + if status is False: + logger.warning("Unable to perform MatMulInteger group operation....") + restore_paths(sys_paths) + return model_path.as_posix() + else: + logger.info("MatMulInteger Grouping successfull....") + g.save(group_model) + + restore_paths(sys_paths) + + return group_model + + def clone_repository(repo_url: str, repo_path: str): try: if not os.path.exists(repo_path): @@ -110,6 +154,8 @@ def set_environment_variables(): third_party = normalize_path(os.path.join(ryzenai_transformers_path, "third_party")) device = os.environ.get("DEVICE", DEFAULT_DEVICE) + set_env_var("RYZENAI_SW_PATH", ryzenai_sw_path) + set_env_var("THIRD_PARTY", third_party) check_env_path_exists("THIRD_PARTY")