Skip to content

Add quark quantizer #149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mht-sharma wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Add quark quantizer #149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
01739e6
add quark quantizer
mht-sharma Jul 17, 2024
0a677ee
fix
mht-sharma Jul 17, 2024
99ba7ad
remove pdb
mht-sharma Jul 17, 2024
a8925a7
fix save_pretrained
mht-sharma Jul 17, 2024
876efed
fix save_pretrained
mht-sharma Jul 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions docker/quark-quantization/amd/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# Copyright 2024 The HuggingFace Team. All rights reserved.
# Licensed under the MIT License.

FROM rocm/dev-ubuntu-22.04:6.1

LABEL maintainer="Hugging Face"

ARG DEBIAN_FRONTEND=noninteractive

RUN apt-get update && apt-get install -y --no-install-recommends \
sudo \
python3.10 \
python3.10-dev \
python3-pip \
git \
wget \
unzip \
libsndfile1-dev \
tesseract-ocr \
espeak-ng \
rocthrust-dev \
hipsparse-dev \
hipblas-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
update-alternatives --install /usr/bin/python python /usr/bin/python3.10 1 && \
python -m pip install -U pip

RUN pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm6.0 --no-cache-dir

WORKDIR /quark
RUN wget -O quark.zip https://www.xilinx.com/bin/public/openDownload?filename=quark-0.1.0+a9827f5.zip && \
unzip quark.zip -d quark && \
pip install quark/quark-0.1.0+a9827f5/quark-0.1.0+a9827f5-py39-none-any.whl && \
rm -rf quark.zip quark

RUN python -c "import quark.torch.kernel"

ENTRYPOINT ["bash"]
8 changes: 8 additions & 0 deletions optimum/amd/quantization/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Copyright 2023 The HuggingFace Team. All rights reserved.
# Licensed under the MIT License.

from .quark import (
AutoQuantizationConfig,
QuarkQuantizationConfig,
QuarkQuantizer,
)
195 changes: 195 additions & 0 deletions optimum/amd/quantization/quark/algo_config_constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
# Copyright 2023 The HuggingFace Team. All rights reserved.
# Licensed under the MIT License.


ALGO_CONFIG_PARAMS = {
"llama": {
"scaling_layers": [
{
"prev_op": "input_layernorm",
"layers": ["self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj"],
"inp": "self_attn.q_proj",
"module2inspect": "self_attn",
"has_kwargs": True,
"help": "attention input",
},
{
"prev_op": "self_attn.v_proj",
"layers": ["self_attn.o_proj"],
"inp": "self_attn.o_proj",
"module2inspect": None,
"has_kwargs": False,
"help": "attention out, Please refer to https://github.com/mit-han-lab/llm-awq/pull/67#issue-1850622696, if module.self_attn.v_proj.weight.shape == module.self_attn.o_proj.weight.shape",
"condition": "module.self_attn.v_proj.weight.shape == module.self_attn.o_proj.weight.shape",
},
{
"prev_op": "post_attention_layernorm",
"layers": ["mlp.gate_proj", "mlp.up_proj"],
"inp": "mlp.gate_proj",
"module2inspect": "mlp",
"has_kwargs": False,
"help": "linear 1",
},
{
"prev_op": "mlp.up_proj",
"layers": ["mlp.down_proj"],
"inp": "mlp.down_proj",
"module2inspect": None,
"has_kwargs": False,
"help": "linear 2",
},
],
"inside_layer_modules": [
"self_attn.k_proj",
"self_attn.v_proj",
"self_attn.q_proj",
"self_attn.o_proj",
"mlp.up_proj",
"mlp.gate_proj",
"mlp.down_proj",
],
"model_decoder_layers": "model.layers",
"embedding_layers": ["model.embed_tokens"],
},
"mistral": {
"scaling_layers": [
{
"prev_op": "input_layernorm",
"layers": ["self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj"],
"inp": "self_attn.q_proj",
"module2inspect": "self_attn",
"has_kwargs": True,
"help": "attention input",
},
{
"prev_op": "self_attn.v_proj",
"layers": ["self_attn.o_proj"],
"inp": "self_attn.o_proj",
"module2inspect": None,
"has_kwargs": False,
"help": "attention out, Please refer to https://github.com/mit-han-lab/llm-awq/pull/67#issue-1850622696, if module.self_attn.v_proj.weight.shape == module.self_attn.o_proj.weight.shape",
"condition": "module.self_attn.v_proj.weight.shape == module.self_attn.o_proj.weight.shape",
},
{
"prev_op": "post_attention_layernorm",
"layers": ["mlp.gate_proj", "mlp.up_proj"],
"inp": "mlp.gate_proj",
"module2inspect": "mlp",
"has_kwargs": False,
"help": "linear 1",
},
{
"prev_op": "mlp.up_proj",
"layers": ["mlp.down_proj"],
"inp": "mlp.down_proj",
"module2inspect": None,
"has_kwargs": False,
"help": "linear 2",
},
],
"inside_layer_modules": [
"self_attn.k_proj",
"self_attn.v_proj",
"self_attn.q_proj",
"self_attn.o_proj",
"mlp.up_proj",
"mlp.gate_proj",
"mlp.down_proj",
],
"model_decoder_layers": "model.layers",
"embedding_layers": ["model.embed_tokens"],
},
"opt": {
"scaling_layers": [
{
"prev_op": "self_attn_layer_norm",
"layers": ["self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj"],
"inp": "self_attn.q_proj",
"module2inspect": "self_attn",
"has_kwargs": True,
"help": "attention input",
},
{
"prev_op": "self_attn.v_proj",
"layers": ["self_attn.out_proj"],
"inp": "self_attn.out_proj",
"module2inspect": None,
"has_kwargs": False,
"help": "attention out",
},
{
"prev_op": "final_layer_norm",
"layers": ["fc1"],
"inp": "fc1",
"module2inspect": None,
"has_kwargs": False,
"help": "linear 1",
},
{
"prev_op": "fc1",
"layers": ["fc2"],
"inp": "fc2",
"module2inspect": None,
"has_kwargs": False,
"help": "linear 2",
},
],
"inside_layer_modules": [
"self_attn.k_proj",
"self_attn.v_proj",
"self_attn.q_proj",
"self_attn.out_proj",
"fc1",
"fc2",
],
"model_decoder_layers": "model.decoder.layers",
"embedding_layers": ["model.decoder.embed_tokens", "model.decoder.embed_positions"],
},
"qwen2": {
"scaling_layers": [
{
"prev_op": "input_layernorm",
"layers": ["self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj"],
"inp": "self_attn.q_proj",
"module2inspect": "self_attn",
"has_kwargs": True,
"help": "attention input",
},
{
"prev_op": "self_attn.v_proj",
"layers": ["self_attn.o_proj"],
"inp": "self_attn.o_proj",
"module2inspect": None,
"has_kwargs": False,
"help": "attention out, Please refer to https://github.com/mit-han-lab/llm-awq/pull/67#issue-1850622696, if module.self_attn.v_proj.weight.shape == module.self_attn.o_proj.weight.shape",
},
{
"prev_op": "post_attention_layernorm",
"layers": ["mlp.gate_proj", "mlp.up_proj"],
"inp": "mlp.gate_proj",
"module2inspect": "mlp",
"has_kwargs": False,
"help": "linear 1",
},
{
"prev_op": "mlp.up_proj",
"layers": ["mlp.down_proj"],
"inp": "mlp.down_proj",
"module2inspect": None,
"has_kwargs": False,
"help": "linear 2",
},
],
"inside_layer_modules": [
"self_attn.k_proj",
"self_attn.v_proj",
"self_attn.q_proj",
"self_attn.o_proj",
"mlp.up_proj",
"mlp.gate_proj",
"mlp.down_proj",
],
"model_decoder_layers": "model.layers",
"embedding_layers": ["model.embed_tokens"],
},
}
Loading