From 8addbec2d117207bfbb184559ce7117d6e66f43a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Nov 2025 07:32:59 +0000 Subject: [PATCH 1/3] Add GGUF quantization format support Implemented comprehensive GGUF quantization support using llama.cpp, enabling conversion of HuggingFace models to GGUF format with multiple quantization levels (Q4_K_M, Q5_K_M, Q6_K, Q8_0, etc.). Key changes: - Added llama.cpp installation to Docker container for GGUF conversion - Implemented GGUFQuantizer class with two-step quantization process: 1. Convert HF model to intermediate GGUF format (f16/f32/q8_0) 2. Quantize to target precision using llama-quantize - Extended QuantizationConfig with GGUF-specific parameters: - gguf_quant_type: Target quantization level (Q4_K_M default) - gguf_intermediate_format: Intermediate format (f16 default) - Added comprehensive input validation for GGUF quantization types - Updated UI with GGUF selection and parameter controls - Enhanced documentation with GGUF usage guidelines and recommendations - Added error handling for llama.cpp availability checks - Included subprocess-based streaming output for conversion progress Quantization types supported: - Q2_K, Q3_K_S, Q3_K_M, Q3_K_L (smaller sizes) - Q4_0, Q4_1, Q4_K_S, Q4_K_M (balanced - Q4_K_M recommended) - Q5_0, Q5_1, Q5_K_S, Q5_K_M (best quality) - Q6_K, Q8_0 (high precision) - F16, F32 (full precision) GGUF outputs are compatible with llama.cpp, Ollama, and other GGUF-compatible inference engines. --- README.md | 26 +++- docker/Dockerfile.gpu | 19 +++ pixi.toml | 2 + src/msquant/app/pages/configure.py | 33 +++- src/msquant/core/quantizer/config.py | 38 ++++- src/msquant/core/quantizer/engine.py | 218 +++++++++++++++++++++++++++ 6 files changed, 321 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index b1b3d01..aa33951 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ # MSQuant -Model Quantization Tool with NiceGUI interface for AWQ and NVFP4 quantization methods. +Model Quantization Tool with NiceGUI interface for AWQ, NVFP4, and GGUF quantization methods. ## Features - **Quantization Methods** - AWQ (Activation-aware Weight Quantization): 4-bit integer quantization - NVFP4 (NVIDIA FP4): 4-bit floating-point quantization + - GGUF (GGML Universal File): Multiple quantization levels (Q4_K_M, Q5_K_M, Q6_K, Q8_0, etc.) using llama.cpp - **Web Interface** - Real-time GPU monitoring with visual charts (Highcharts) @@ -118,11 +119,21 @@ pixi install Navigate to the **Configure** page and set: - Model ID (e.g., `meta-llama/Llama-3.1-8B`) -- Quantization method (AWQ or NVFP4) -- Calibration dataset settings -- Method-specific parameters - -**Note:** Output format is determined by the quantization backend. The saved checkpoint format follows the backend's default conventions. +- Quantization method (AWQ, NVFP4, or GGUF) +- Calibration dataset settings (required for AWQ and NVFP4) +- Method-specific parameters: + - **AWQ**: Weight bits, group size, zero point + - **NVFP4**: Activation/weight schemes + - **GGUF**: Quantization type (Q4_K_M recommended, Q5_K_M for best quality), intermediate format (f16 default) + +**Note:** +- AWQ and NVFP4 output formats follow llmcompressor conventions (binary or safetensors) +- GGUF produces `.gguf` files compatible with llama.cpp, Ollama, and other GGUF-compatible inference engines +- GGUF quantization types: + - **Q4_K_M**: Recommended for balanced quality and size + - **Q5_K_M**: Best quality while maintaining reasonable size + - **Q6_K, Q8_0**: Higher precision options + - **Q2_K, Q3_K**: Smaller sizes with reduced quality ### 2. Monitor Progress @@ -176,6 +187,7 @@ Replace `OWNER` with your GitHub username/organization. Built with: - [NiceGUI](https://nicegui.io/) - Web interface -- [llmcompressor](https://github.com/vllm-project/llm-compressor) - Quantization engine +- [llmcompressor](https://github.com/vllm-project/llm-compressor) - Quantization engine for AWQ/NVFP4 +- [llama.cpp](https://github.com/ggerganov/llama.cpp) - GGUF quantization and inference - [vLLM](https://github.com/vllm-project/vllm) - LLM inference - [Pixi](https://pixi.sh/) - Package management diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index 7fd7dc9..24f465a 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -32,6 +32,25 @@ RUN pip install "transformers>=4.52.0" RUN pip install "llmcompressor" +# Install llama.cpp for GGUF quantization support +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + build-essential \ + cmake \ + && apt-get clean && \ + rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* + +# Clone and build llama.cpp +RUN git clone https://github.com/ggerganov/llama.cpp /opt/llama.cpp && \ + cd /opt/llama.cpp && \ + git checkout master && \ + make -j$(nproc) && \ + pip install -r requirements.txt && \ + pip install gguf + +# Add llama.cpp to PATH +ENV PATH="/opt/llama.cpp:${PATH}" + WORKDIR /workspace # Copy source diff --git a/pixi.toml b/pixi.toml index efd3edb..5e1e34b 100644 --- a/pixi.toml +++ b/pixi.toml @@ -40,6 +40,8 @@ huggingface_hub = "*" hf_transfer = "*" tqdm = "*" pandas = ">=2.2.0" +# GGUF support +gguf = "*" # Optional dev extras types-tqdm = "*" diff --git a/src/msquant/app/pages/configure.py b/src/msquant/app/pages/configure.py index 113e39b..5307293 100644 --- a/src/msquant/app/pages/configure.py +++ b/src/msquant/app/pages/configure.py @@ -20,6 +20,8 @@ def create_configure_page(job_service: JobService, storage_service: StorageServi 'w_bit': 4, 'group_size': 128, 'zero_point': True, + 'gguf_quant_type': 'Q4_K_M', + 'gguf_intermediate_format': 'f16', } def start_quantization(): @@ -37,6 +39,8 @@ def start_quantization(): w_bit=form_data['w_bit'], group_size=form_data['group_size'], zero_point=form_data['zero_point'], + gguf_quant_type=form_data['gguf_quant_type'], + gguf_intermediate_format=form_data['gguf_intermediate_format'], ) if job_service.start_job(config): @@ -61,7 +65,7 @@ def start_quantization(): # Quantization method ui.label('Quantization Method').classes('text-2xl font-bold') with ui.row().classes('w-full gap-4'): - ui.radio(['awq', 'nvfp4'], value='awq').bind_value(form_data, 'quant_method').props('inline') + ui.radio(['awq', 'nvfp4', 'gguf'], value='awq').bind_value(form_data, 'quant_method').props('inline') ui.separator() @@ -99,9 +103,32 @@ def start_quantization(): label='Group Size' ).classes('flex-1').bind_value(form_data, 'group_size').props('use-input new-value-mode=add-unique') ui.checkbox('Zero Point', value=True).bind_value(form_data, 'zero_point') - + ui.separator() - + + # GGUF-specific settings + ui.label('GGUF Settings (only for GGUF method)').classes('text-2xl font-bold') + with ui.row().classes('w-full gap-4'): + ui.select( + ['Q2_K', 'Q3_K_S', 'Q3_K_M', 'Q3_K_L', 'Q4_0', 'Q4_1', 'Q4_K_S', 'Q4_K_M', + 'Q5_0', 'Q5_1', 'Q5_K_S', 'Q5_K_M', 'Q6_K', 'Q8_0', 'F16', 'F32'], + value='Q4_K_M', + label='Quantization Type' + ).classes('flex-1').bind_value(form_data, 'gguf_quant_type') + ui.select( + ['f16', 'f32', 'q8_0'], + value='f16', + label='Intermediate Format' + ).classes('flex-1').bind_value(form_data, 'gguf_intermediate_format') + ui.html(''' +

+ Recommended: Q4_K_M (balanced), Q5_K_M (best quality)
+ Intermediate: f16 (default), f32 (higher precision), q8_0 (smaller) +

+ ''') + + ui.separator() + # Actions with ui.row().classes('w-full gap-4 justify-end'): ui.button('Start Quantization', on_click=start_quantization).props('color=primary size=lg') diff --git a/src/msquant/core/quantizer/config.py b/src/msquant/core/quantizer/config.py index 0a18c1a..9030ab9 100644 --- a/src/msquant/core/quantizer/config.py +++ b/src/msquant/core/quantizer/config.py @@ -24,6 +24,9 @@ def __init__( w_scheme: str = "fp4", non_uniform: bool = False, mix_fp8: bool = False, + # GGUF params + gguf_quant_type: str = "Q4_K_M", + gguf_intermediate_format: str = "f16", # Paths hf_home: str = "/workspace/hf", hf_datasets_cache: str = "/workspace/hf/datasets", @@ -71,7 +74,11 @@ def __init__( self.w_scheme = w_scheme self.non_uniform = non_uniform self.mix_fp8 = mix_fp8 - + + # GGUF + self.gguf_quant_type = gguf_quant_type.upper() if isinstance(gguf_quant_type, str) else gguf_quant_type + self.gguf_intermediate_format = gguf_intermediate_format.lower() if isinstance(gguf_intermediate_format, str) else gguf_intermediate_format + # Paths self.hf_home = hf_home self.hf_datasets_cache = hf_datasets_cache @@ -89,15 +96,36 @@ def validate(self): """Validate configuration.""" if not self.model_id: raise ValueError("model_id is required") - - if self.quant_method not in ["awq", "nvfp4"]: + + if self.quant_method not in ["awq", "nvfp4", "gguf"]: raise ValueError(f"Invalid quant_method: {self.quant_method}") - + if self.output_format not in ["binary", "safetensors"]: raise ValueError(f"Invalid output_format: {self.output_format}") - + if self.quant_method == "awq": if self.w_bit not in [2, 3, 4, 5, 8]: raise ValueError(f"Invalid w_bit for AWQ: {self.w_bit}") if self.group_size <= 0: raise ValueError(f"Invalid group_size: {self.group_size}") + + if self.quant_method == "gguf": + # Validate GGUF quantization type + valid_gguf_types = [ + "Q2_K", "Q3_K_S", "Q3_K_M", "Q3_K_L", + "Q4_0", "Q4_1", "Q4_K_S", "Q4_K_M", + "Q5_0", "Q5_1", "Q5_K_S", "Q5_K_M", + "Q6_K", "Q8_0", "F16", "F32" + ] + if self.gguf_quant_type not in valid_gguf_types: + raise ValueError( + f"Invalid gguf_quant_type: {self.gguf_quant_type}. " + f"Must be one of: {', '.join(valid_gguf_types)}" + ) + + # Validate intermediate format + if self.gguf_intermediate_format not in ["f16", "f32", "q8_0"]: + raise ValueError( + f"Invalid gguf_intermediate_format: {self.gguf_intermediate_format}. " + f"Must be one of: f16, f32, q8_0" + ) diff --git a/src/msquant/core/quantizer/engine.py b/src/msquant/core/quantizer/engine.py index a21d486..a07396e 100644 --- a/src/msquant/core/quantizer/engine.py +++ b/src/msquant/core/quantizer/engine.py @@ -240,6 +240,222 @@ def run(config: QuantizationConfig, logger: QuantizationLogger): return config.out_dir +class GGUFQuantizer: + """GGUF quantization handler using llama.cpp.""" + + @staticmethod + def _check_llama_cpp_available(): + """Check if llama.cpp tools are available.""" + try: + # Check if convert-hf-to-gguf.py is available + result = subprocess.run( + ["bash", "-lc", "which python"], + capture_output=True, + text=True, + check=False + ) + if result.returncode != 0: + raise RuntimeError("Python not found in PATH") + + # Check if llama-quantize is available + result = subprocess.run( + ["bash", "-lc", "which llama-quantize"], + capture_output=True, + text=True, + check=False + ) + if result.returncode != 0: + raise RuntimeError( + "llama-quantize not found in PATH. " + "Please ensure llama.cpp is properly installed." + ) + return True + except Exception as e: + raise RuntimeError(f"llama.cpp availability check failed: {e}") from e + + @staticmethod + def _download_model(model_id: str, cache_dir: str, logger: QuantizationLogger) -> str: + """Download HuggingFace model to local cache.""" + from huggingface_hub import snapshot_download + + logger.info(f"Downloading model {model_id} to cache...") + try: + local_path = snapshot_download( + repo_id=model_id, + cache_dir=cache_dir, + local_dir_use_symlinks=False + ) + logger.info(f"Model downloaded to {local_path}") + return local_path + except Exception as e: + raise RuntimeError(f"Failed to download model: {e}") from e + + @staticmethod + def _convert_to_gguf_intermediate( + model_path: str, + output_file: str, + intermediate_format: str, + logger: QuantizationLogger + ): + """Convert HuggingFace model to intermediate GGUF format.""" + logger.info(f"Converting model to GGUF {intermediate_format} format...") + + # Find the convert-hf-to-gguf.py script + convert_script = "/opt/llama.cpp/convert_hf_to_gguf.py" + + # Build the conversion command + cmd = [ + "python", + convert_script, + model_path, + "--outfile", output_file, + "--outtype", intermediate_format + ] + + logger.info(f"Running command: {' '.join(cmd)}") + + try: + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1 + ) + + # Stream output + if process.stdout: + for line in process.stdout: + line = line.rstrip() + if line: + logger.info(f"[convert] {line}") + + process.wait() + + if process.returncode != 0: + raise RuntimeError(f"Conversion failed with exit code {process.returncode}") + + logger.info(f"Conversion completed. Intermediate GGUF file: {output_file}") + + except Exception as e: + raise RuntimeError(f"GGUF conversion failed: {e}") from e + + @staticmethod + def _quantize_gguf( + input_file: str, + output_file: str, + quant_type: str, + logger: QuantizationLogger + ): + """Quantize GGUF file to target precision.""" + logger.info(f"Quantizing GGUF to {quant_type}...") + + # Skip quantization if target format is already F16 or F32 + if quant_type in ["F16", "F32"]: + logger.info(f"Target format {quant_type} matches intermediate format, skipping quantization") + # Copy the file instead + import shutil + shutil.copy2(input_file, output_file) + return + + # Build the quantization command + cmd = [ + "llama-quantize", + input_file, + output_file, + quant_type + ] + + logger.info(f"Running command: {' '.join(cmd)}") + + try: + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1 + ) + + # Stream output + if process.stdout: + for line in process.stdout: + line = line.rstrip() + if line: + logger.info(f"[quantize] {line}") + + process.wait() + + if process.returncode != 0: + raise RuntimeError(f"Quantization failed with exit code {process.returncode}") + + logger.info(f"Quantization completed. Output file: {output_file}") + + except Exception as e: + raise RuntimeError(f"GGUF quantization failed: {e}") from e + + @staticmethod + def run(config: QuantizationConfig, logger: QuantizationLogger): + """Run GGUF quantization.""" + import os + + # Check if llama.cpp is available + GGUFQuantizer._check_llama_cpp_available() + + logger.info("GGUF Quantization config:", { + "quant_type": config.gguf_quant_type, + "intermediate_format": config.gguf_intermediate_format, + "model_id": config.model_id, + }) + + t0 = time.time() + + # Step 1: Download model from HuggingFace + model_path = GGUFQuantizer._download_model( + config.model_id, + config.hf_home, + logger + ) + + # Step 2: Convert to intermediate GGUF format + os.makedirs(config.out_dir, exist_ok=True) + safe_name = config.model_id.split("/")[-1].replace(":", "-") + intermediate_file = os.path.join( + config.out_dir, + f"{safe_name}-{config.gguf_intermediate_format}.gguf" + ) + + GGUFQuantizer._convert_to_gguf_intermediate( + model_path, + intermediate_file, + config.gguf_intermediate_format, + logger + ) + + # Step 3: Quantize to target precision + final_file = os.path.join( + config.out_dir, + f"{safe_name}-{config.gguf_quant_type}.gguf" + ) + + GGUFQuantizer._quantize_gguf( + intermediate_file, + final_file, + config.gguf_quant_type, + logger + ) + + # Clean up intermediate file if different from final + if intermediate_file != final_file and os.path.exists(intermediate_file): + logger.info(f"Cleaning up intermediate file: {intermediate_file}") + os.remove(intermediate_file) + + dt = time.time() - t0 + logger.info(f"Completed. Saved GGUF quantized model to {final_file} in {dt:.1f}s") + + return config.out_dir + + def quantize(config: QuantizationConfig, log_callback: Optional[Callable[[str], None]] = None): """ Main quantization entry point. @@ -285,6 +501,8 @@ def quantize(config: QuantizationConfig, log_callback: Optional[Callable[[str], output_dir = AWQQuantizer.run(config, logger) elif config.quant_method == "nvfp4": output_dir = NVFP4Quantizer.run(config, logger) + elif config.quant_method == "gguf": + output_dir = GGUFQuantizer.run(config, logger) else: raise ValueError(f"Unsupported quantization method: {config.quant_method}") From 3beaa349e1365fe36823c5c40c43fec6e443bf91 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Nov 2025 16:06:28 +0000 Subject: [PATCH 2/3] Fix typecheck error: add sanitize parameter to ui.html() NiceGUI 3.0 introduced a mandatory sanitize parameter for ui.html() to address CVE-2025-53354 (XSS vulnerability). Since the HTML content is static and safe, passing sanitize=False to indicate no sanitization is needed. --- src/msquant/app/pages/configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/msquant/app/pages/configure.py b/src/msquant/app/pages/configure.py index 5307293..5bed9fb 100644 --- a/src/msquant/app/pages/configure.py +++ b/src/msquant/app/pages/configure.py @@ -125,7 +125,7 @@ def start_quantization(): Recommended: Q4_K_M (balanced), Q5_K_M (best quality)
Intermediate: f16 (default), f32 (higher precision), q8_0 (smaller)

- ''') + ''', sanitize=False) ui.separator() From 2615f8e794f3432f62ea2d9351fcf69c2fb96a52 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Nov 2025 16:27:28 +0000 Subject: [PATCH 3/3] Fix GGUF implementation issues for production deployment Three critical fixes based on code review: 1. Dockerfile: Use pre-compiled llama.cpp binaries instead of building from source - Changed from git clone + make to downloading pre-compiled release - Added ARG LLAMA_CPP_VERSION=b6945 for version parameterization - Removed build-essential, cmake, git dependencies (only need wget, unzip) - Significantly faster Docker builds and smaller image size 2. config.py: Fix type validation for GGUF parameters - Changed conditional uppercase/lowercase conversion to explicit type checking - Added TypeError with descriptive messages if non-string types are passed - Ensures validation at line 120 works correctly with exact string matches - Prevents unexpected validation failures from non-string inputs 3. engine.py: Fix convert script name (hyphens vs underscores) - Changed convert_hf_to_gguf.py to convert-hf-to-gguf.py - The actual llama.cpp script uses hyphens, not underscores - This was causing runtime failures when attempting conversion All changes verified with syntax checks and validation tests. --- docker/Dockerfile.gpu | 18 +++++++++--------- src/msquant/core/quantizer/config.py | 11 ++++++++--- src/msquant/core/quantizer/engine.py | 2 +- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/docker/Dockerfile.gpu b/docker/Dockerfile.gpu index 24f465a..474a6f5 100644 --- a/docker/Dockerfile.gpu +++ b/docker/Dockerfile.gpu @@ -33,19 +33,19 @@ RUN pip install "transformers>=4.52.0" RUN pip install "llmcompressor" # Install llama.cpp for GGUF quantization support +ARG LLAMA_CPP_VERSION=b6945 RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - build-essential \ - cmake \ + wget \ + unzip \ && apt-get clean && \ rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* -# Clone and build llama.cpp -RUN git clone https://github.com/ggerganov/llama.cpp /opt/llama.cpp && \ - cd /opt/llama.cpp && \ - git checkout master && \ - make -j$(nproc) && \ - pip install -r requirements.txt && \ +# Download and install pre-compiled llama.cpp binary +RUN wget -q https://github.com/ggml-org/llama.cpp/releases/download/${LLAMA_CPP_VERSION}/llama-${LLAMA_CPP_VERSION}-bin-ubuntu-x64.zip -O /tmp/llama.zip && \ + unzip -q /tmp/llama.zip -d /opt && \ + mv /opt/llama-${LLAMA_CPP_VERSION}-bin-ubuntu-x64 /opt/llama.cpp && \ + rm /tmp/llama.zip && \ + chmod +x /opt/llama.cpp/llama-* && \ pip install gguf # Add llama.cpp to PATH diff --git a/src/msquant/core/quantizer/config.py b/src/msquant/core/quantizer/config.py index 9030ab9..145a04e 100644 --- a/src/msquant/core/quantizer/config.py +++ b/src/msquant/core/quantizer/config.py @@ -75,9 +75,14 @@ def __init__( self.non_uniform = non_uniform self.mix_fp8 = mix_fp8 - # GGUF - self.gguf_quant_type = gguf_quant_type.upper() if isinstance(gguf_quant_type, str) else gguf_quant_type - self.gguf_intermediate_format = gguf_intermediate_format.lower() if isinstance(gguf_intermediate_format, str) else gguf_intermediate_format + # GGUF - validate and convert types + if not isinstance(gguf_quant_type, str): + raise TypeError(f"gguf_quant_type must be a string, got {type(gguf_quant_type).__name__}") + self.gguf_quant_type = gguf_quant_type.upper() + + if not isinstance(gguf_intermediate_format, str): + raise TypeError(f"gguf_intermediate_format must be a string, got {type(gguf_intermediate_format).__name__}") + self.gguf_intermediate_format = gguf_intermediate_format.lower() # Paths self.hf_home = hf_home diff --git a/src/msquant/core/quantizer/engine.py b/src/msquant/core/quantizer/engine.py index a07396e..79b97dc 100644 --- a/src/msquant/core/quantizer/engine.py +++ b/src/msquant/core/quantizer/engine.py @@ -301,7 +301,7 @@ def _convert_to_gguf_intermediate( logger.info(f"Converting model to GGUF {intermediate_format} format...") # Find the convert-hf-to-gguf.py script - convert_script = "/opt/llama.cpp/convert_hf_to_gguf.py" + convert_script = "/opt/llama.cpp/convert-hf-to-gguf.py" # Build the conversion command cmd = [