diff --git a/AWQ/main.py b/AWQ/main.py index bb0af1a..36273fd 100644 --- a/AWQ/main.py +++ b/AWQ/main.py @@ -7,6 +7,8 @@ import transformers import torch +from llmcompressor.utils import dispatch_for_generation # important for runtime hooks + # Select model and load it. MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct" @@ -77,10 +79,6 @@ def tokenize(sample): model.save_pretrained(SAVE_DIR, save_compressed=True) tokenizer.save_pretrained(SAVE_DIR) - -from transformers import AutoModelForCausalLM, AutoTokenizer -from llmcompressor.utils import dispatch_for_generation # important for runtime hooks - SAVE_DIR = "Meta-Llama-3-8B-Instruct-awq-asym" # your folder name tokenizer = AutoTokenizer.from_pretrained(SAVE_DIR)