Implement model quantization format support

README.md

@@ -1,12 +1,13 @@
 # MSQuant
 
-Model Quantization Tool with NiceGUI interface for AWQ and NVFP4 quantization methods.
+Model Quantization Tool with NiceGUI interface for AWQ, NVFP4, and GGUF quantization methods.
 
 ## Features
 
 - **Quantization Methods**
   - AWQ (Activation-aware Weight Quantization): 4-bit integer quantization
   - NVFP4 (NVIDIA FP4): 4-bit floating-point quantization
+  - GGUF (GGML Universal File): Multiple quantization levels (Q4_K_M, Q5_K_M, Q6_K, Q8_0, etc.) using llama.cpp
 
 - **Web Interface**
   - Real-time GPU monitoring with visual charts (Highcharts)
@@ -118,11 +119,21 @@ pixi install
 
 Navigate to the **Configure** page and set:
 - Model ID (e.g., `meta-llama/Llama-3.1-8B`)
-- Quantization method (AWQ or NVFP4)
-- Calibration dataset settings
-- Method-specific parameters
-
-**Note:** Output format is determined by the quantization backend. The saved checkpoint format follows the backend's default conventions.
+- Quantization method (AWQ, NVFP4, or GGUF)
+- Calibration dataset settings (required for AWQ and NVFP4)
+- Method-specific parameters:
+  - **AWQ**: Weight bits, group size, zero point
+  - **NVFP4**: Activation/weight schemes
+  - **GGUF**: Quantization type (Q4_K_M recommended, Q5_K_M for best quality), intermediate format (f16 default)
+
+**Note:**
+- AWQ and NVFP4 output formats follow llmcompressor conventions (binary or safetensors)
+- GGUF produces `.gguf` files compatible with llama.cpp, Ollama, and other GGUF-compatible inference engines
+- GGUF quantization types:
+  - **Q4_K_M**: Recommended for balanced quality and size
+  - **Q5_K_M**: Best quality while maintaining reasonable size
+  - **Q6_K, Q8_0**: Higher precision options
+  - **Q2_K, Q3_K**: Smaller sizes with reduced quality
 
 ### 2. Monitor Progress
 
@@ -176,6 +187,7 @@ Replace `OWNER` with your GitHub username/organization.
 
 Built with:
 - [NiceGUI](https://nicegui.io/) - Web interface
-- [llmcompressor](https://github.com/vllm-project/llm-compressor) - Quantization engine
+- [llmcompressor](https://github.com/vllm-project/llm-compressor) - Quantization engine for AWQ/NVFP4
+- [llama.cpp](https://github.com/ggerganov/llama.cpp) - GGUF quantization and inference
 - [vLLM](https://github.com/vllm-project/vllm) - LLM inference
 - [Pixi](https://pixi.sh/) - Package management

docker/Dockerfile.gpu

@@ -32,6 +32,25 @@ RUN pip install "transformers>=4.52.0"
 
 RUN pip install "llmcompressor"
 
+# Install llama.cpp for GGUF quantization support
+ARG LLAMA_CPP_VERSION=b6945
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    wget \
+    unzip \
+    && apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+
+# Download and install pre-compiled llama.cpp binary
+RUN wget -q https://github.com/ggml-org/llama.cpp/releases/download/${LLAMA_CPP_VERSION}/llama-${LLAMA_CPP_VERSION}-bin-ubuntu-x64.zip -O /tmp/llama.zip && \
+    unzip -q /tmp/llama.zip -d /opt && \
+    mv /opt/llama-${LLAMA_CPP_VERSION}-bin-ubuntu-x64 /opt/llama.cpp && \
+    rm /tmp/llama.zip && \
+    chmod +x /opt/llama.cpp/llama-* && \
+    pip install gguf
+
+# Add llama.cpp to PATH
+ENV PATH="/opt/llama.cpp:${PATH}"
+
 WORKDIR /workspace
 
 # Copy source

pixi.toml

@@ -40,6 +40,8 @@ huggingface_hub = "*"
 hf_transfer = "*"
 tqdm = "*"
 pandas = ">=2.2.0"
+# GGUF support
+gguf = "*"
 # Optional dev extras
 types-tqdm = "*"
 

src/msquant/app/pages/configure.py

@@ -20,6 +20,8 @@ def create_configure_page(job_service: JobService, storage_service: StorageServi
         'w_bit': 4,
         'group_size': 128,
         'zero_point': True,
+        'gguf_quant_type': 'Q4_K_M',
+        'gguf_intermediate_format': 'f16',
     }
 
     def start_quantization():
@@ -37,6 +39,8 @@ def start_quantization():
                 w_bit=form_data['w_bit'],
                 group_size=form_data['group_size'],
                 zero_point=form_data['zero_point'],
+                gguf_quant_type=form_data['gguf_quant_type'],
+                gguf_intermediate_format=form_data['gguf_intermediate_format'],
             )
 
             if job_service.start_job(config):
@@ -61,7 +65,7 @@ def start_quantization():
                 # Quantization method
                 ui.label('Quantization Method').classes('text-2xl font-bold')
                 with ui.row().classes('w-full gap-4'):
-                    ui.radio(['awq', 'nvfp4'], value='awq').bind_value(form_data, 'quant_method').props('inline')
+                    ui.radio(['awq', 'nvfp4', 'gguf'], value='awq').bind_value(form_data, 'quant_method').props('inline')
 
                 ui.separator()
 
@@ -99,9 +103,32 @@ def start_quantization():
                         label='Group Size'
                     ).classes('flex-1').bind_value(form_data, 'group_size').props('use-input new-value-mode=add-unique')
                 ui.checkbox('Zero Point', value=True).bind_value(form_data, 'zero_point')
-                
+
                 ui.separator()
-
+
+                # GGUF-specific settings
+                ui.label('GGUF Settings (only for GGUF method)').classes('text-2xl font-bold')
+                with ui.row().classes('w-full gap-4'):
+                    ui.select(
+                        ['Q2_K', 'Q3_K_S', 'Q3_K_M', 'Q3_K_L', 'Q4_0', 'Q4_1', 'Q4_K_S', 'Q4_K_M',
+                         'Q5_0', 'Q5_1', 'Q5_K_S', 'Q5_K_M', 'Q6_K', 'Q8_0', 'F16', 'F32'],
+                        value='Q4_K_M',
+                        label='Quantization Type'
+                    ).classes('flex-1').bind_value(form_data, 'gguf_quant_type')
+                    ui.select(
+                        ['f16', 'f32', 'q8_0'],
+                        value='f16',
+                        label='Intermediate Format'
+                    ).classes('flex-1').bind_value(form_data, 'gguf_intermediate_format')
+                ui.html('''
+                    <p class="text-sm text-gray-600">
+                        <strong>Recommended:</strong> Q4_K_M (balanced), Q5_K_M (best quality)<br>
+                        <strong>Intermediate:</strong> f16 (default), f32 (higher precision), q8_0 (smaller)
+                    </p>
+                ''', sanitize=False)
+
+                ui.separator()
+
                 # Actions
                 with ui.row().classes('w-full gap-4 justify-end'):
                     ui.button('Start Quantization', on_click=start_quantization).props('color=primary size=lg')

src/msquant/core/quantizer/config.py

@@ -24,6 +24,9 @@ def __init__(
         w_scheme: str = "fp4",
         non_uniform: bool = False,
         mix_fp8: bool = False,
+        # GGUF params
+        gguf_quant_type: str = "Q4_K_M",
+        gguf_intermediate_format: str = "f16",
         # Paths
         hf_home: str = "/workspace/hf",
         hf_datasets_cache: str = "/workspace/hf/datasets",
@@ -71,7 +74,16 @@ def __init__(
         self.w_scheme = w_scheme
         self.non_uniform = non_uniform
         self.mix_fp8 = mix_fp8
-
+
+        # GGUF - validate and convert types
+        if not isinstance(gguf_quant_type, str):
+            raise TypeError(f"gguf_quant_type must be a string, got {type(gguf_quant_type).__name__}")
+        self.gguf_quant_type = gguf_quant_type.upper()
+
+        if not isinstance(gguf_intermediate_format, str):
+            raise TypeError(f"gguf_intermediate_format must be a string, got {type(gguf_intermediate_format).__name__}")
+        self.gguf_intermediate_format = gguf_intermediate_format.lower()
+
         # Paths
         self.hf_home = hf_home
         self.hf_datasets_cache = hf_datasets_cache
@@ -89,15 +101,36 @@ def validate(self):
         """Validate configuration."""
         if not self.model_id:
             raise ValueError("model_id is required")
-        
-        if self.quant_method not in ["awq", "nvfp4"]:
+
+        if self.quant_method not in ["awq", "nvfp4", "gguf"]:
             raise ValueError(f"Invalid quant_method: {self.quant_method}")
-        
+
         if self.output_format not in ["binary", "safetensors"]:
             raise ValueError(f"Invalid output_format: {self.output_format}")
-        
+
         if self.quant_method == "awq":
             if self.w_bit not in [2, 3, 4, 5, 8]:
                 raise ValueError(f"Invalid w_bit for AWQ: {self.w_bit}")
             if self.group_size <= 0:
                 raise ValueError(f"Invalid group_size: {self.group_size}")
+
+        if self.quant_method == "gguf":
+            # Validate GGUF quantization type
+            valid_gguf_types = [
+                "Q2_K", "Q3_K_S", "Q3_K_M", "Q3_K_L",
+                "Q4_0", "Q4_1", "Q4_K_S", "Q4_K_M",
+                "Q5_0", "Q5_1", "Q5_K_S", "Q5_K_M",
+                "Q6_K", "Q8_0", "F16", "F32"
+            ]
+            if self.gguf_quant_type not in valid_gguf_types:
+                raise ValueError(
+                    f"Invalid gguf_quant_type: {self.gguf_quant_type}. "
+                    f"Must be one of: {', '.join(valid_gguf_types)}"
+                )
+
+            # Validate intermediate format
+            if self.gguf_intermediate_format not in ["f16", "f32", "q8_0"]:
+                raise ValueError(
+                    f"Invalid gguf_intermediate_format: {self.gguf_intermediate_format}. "
+                    f"Must be one of: f16, f32, q8_0"
+                )