From d37495f5cf857e3fb55fbebe1d9d472c7789906d Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 1 May 2026 16:05:00 +0200 Subject: [PATCH 01/19] The great quant laboratory squash --- ggml/include/ggml-cpu.h | 5 +- ggml/include/ggml.h | 26 +- ggml/src/CMakeLists.txt | 7 + ggml/src/ggml-backend-meta.cpp | 2 +- ggml/src/ggml-blas/ggml-blas.cpp | 17 +- ggml/src/ggml-common.h | 116 + ggml/src/ggml-cpu/arch-fallback.h | 13 + ggml/src/ggml-cpu/arch/arm/quants.c | 208 +- ggml/src/ggml-cpu/arch/loongarch/quants.c | 22 +- ggml/src/ggml-cpu/arch/powerpc/quants.c | 28 +- ggml/src/ggml-cpu/arch/riscv/quants.c | 24 +- ggml/src/ggml-cpu/arch/s390/quants.c | 28 +- ggml/src/ggml-cpu/arch/wasm/quants.c | 23 +- ggml/src/ggml-cpu/arch/x86/quants.c | 434 +- ggml/src/ggml-cpu/ggml-cpu.c | 83 +- ggml/src/ggml-cpu/llamafile/sgemm.cpp | 44 +- ggml/src/ggml-cpu/llamafile/sgemm.h | 2 +- ggml/src/ggml-cpu/ops.cpp | 88 +- ggml/src/ggml-cpu/quants.c | 497 +- ggml/src/ggml-cpu/quants.h | 123 +- ggml/src/ggml-cpu/vec.cpp | 9 +- ggml/src/ggml-cpu/vec.h | 8 +- ggml/src/ggml-cuda/common.cuh | 53 + ggml/src/ggml-cuda/convert.cu | 195 + ggml/src/ggml-cuda/convert.cuh | 16 + ggml/src/ggml-cuda/ggml-cuda.cu | 29 +- ggml/src/ggml-cuda/mmq.cu | 32 + ggml/src/ggml-cuda/mmq.cuh | 88 + ggml/src/ggml-cuda/mmvq.cu | 75 + ggml/src/ggml-cuda/vecdotq.cuh | 188 + ggml/src/ggml-quants.c | 4447 ++++++++++++++++- ggml/src/ggml-quants.h | 262 +- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 4 +- ggml/src/ggml.c | 97 +- ggml/src/gguf.cpp | 82 +- gguf-py/gguf/constants.py | 1175 ++--- include/llama.h | 8 + pocs/vdot/q8dot.cpp | 4 +- pocs/vdot/vdot.cpp | 4 +- scripts/analyze-ffn-down.py | 604 +++ scripts/compute-imatrix.py | 105 + scripts/extract-activations.py | 210 + scripts/extract-tensor-data.py | 74 + src/llama-graph.h | 1 + src/llama-model-loader.cpp | 15 + src/llama-model.cpp | 169 + src/llama-model.h | 18 + src/llama-quant.cpp | 824 ++- tests/CMakeLists.txt | 3 + tests/test-backend-ops.cpp | 2 +- tests/test-quant-laboratory.cpp | 355 ++ tests/test-quantize-fns.cpp | 8 +- tests/test-quantize-perf.cpp | 4 +- tests/test-quantize-stats.cpp | 2 +- tools/CMakeLists.txt | 1 + tools/capture-layer-data/CMakeLists.txt | 9 + .../capture-layer-data/capture-layer-data.cpp | 251 + tools/export-lora/export-lora.cpp | 2 +- tools/quantize/quantize.cpp | 12 + 59 files changed, 10158 insertions(+), 1077 deletions(-) create mode 100644 scripts/analyze-ffn-down.py create mode 100644 scripts/compute-imatrix.py create mode 100644 scripts/extract-activations.py create mode 100644 scripts/extract-tensor-data.py create mode 100644 tests/test-quant-laboratory.cpp create mode 100644 tools/capture-layer-data/CMakeLists.txt create mode 100644 tools/capture-layer-data/capture-layer-data.cpp diff --git a/ggml/include/ggml-cpu.h b/ggml/include/ggml-cpu.h index e3e067c916f1..9c3a6da5dcf7 100644 --- a/ggml/include/ggml-cpu.h +++ b/ggml/include/ggml-cpu.h @@ -111,13 +111,14 @@ extern "C" { // Internal types and functions exposed for tests and benchmarks typedef void (*ggml_vec_dot_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx, - const void * GGML_RESTRICT y, size_t by, int nrc); + const void * GGML_RESTRICT y, size_t by, int nrc, const void * levels); struct ggml_type_traits_cpu { ggml_from_float_t from_float; ggml_vec_dot_t vec_dot; enum ggml_type vec_dot_type; - int64_t nrows; // number of rows to process simultaneously + int64_t nrows; // number of rows to process simultaneously + size_t levels_row_stride; // bytes to add per row to get next row's quant_levels (0 = per-tensor) }; GGML_BACKEND_API const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type); diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index d6807b6dd47a..84593c5c9b65 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -429,7 +429,15 @@ extern "C" { GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, - GGML_TYPE_COUNT = 42, + GGML_TYPE_Q3_PT = 42, // 3.875 bpw per-tensor Lloyd-Max, 16-elem affine sub-blocks + GGML_TYPE_Q3_KPT = 43, // Q3_K with learned per-tensor levels (3.4375 bpw) + GGML_TYPE_Q4_DPT = 44, // IQ4_NL with learned per-tensor int8 levels (4.125 bpw) + GGML_TYPE_Q2_DPT = 45, // 2-bit with learned per-tensor int8 levels (2.5 bpw) + GGML_TYPE_Q2_KPT = 46, // Q2_K with learned per-tensor float levels (2.625 bpw) + GGML_TYPE_IQ2_TQ = 47, // Trellis quantized with RNG codebook (2.0625 bpw) + GGML_TYPE_IQ3_TQ = 48, // 3-bit with per-tensor trained grid table (3.5625 bpw) + GGML_TYPE_IQ1_BN = 49, // 8D vector quantized with per-tensor trained codebook (1.5625 bpw) + GGML_TYPE_COUNT = 50, }; // precision @@ -463,6 +471,7 @@ extern "C" { GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors + GGML_FTYPE_MOSTLY_Q3_PT = 26, // except 1d tensors GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors @@ -471,8 +480,11 @@ extern "C" { GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors - GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors - GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors + GGML_FTYPE_MOSTLY_Q3_KPT = 27, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_DPT = 28, // except 1d tensors + GGML_FTYPE_MOSTLY_Q2_KPT = 29, // except 1d tensors + GGML_FTYPE_MOSTLY_NVFP4 = 30, // except 1d tensors + GGML_FTYPE_MOSTLY_Q1_0 = 31, // except 1d tensors }; // available tensor operations: @@ -693,9 +705,8 @@ extern "C" { char name[GGML_MAX_NAME]; - void * extra; // extra things e.g. for ggml-cuda.cu - - char padding[8]; + void * extra; // extra things e.g. for ggml-cuda.cu + void * quant_levels; // per-tensor quantization levels (replaces char padding[8]; same size on 64-bit) }; static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor); @@ -2811,7 +2822,7 @@ extern "C" { # define GGML_RESTRICT restrict # endif #endif - typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); struct ggml_type_traits { @@ -2822,6 +2833,7 @@ extern "C" { bool is_quantized; ggml_to_float_t to_float; ggml_from_float_t from_float_ref; + size_t levels_row_stride; // bytes to advance quant_levels per row (0 = per-tensor) }; GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type); diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index 89e5180d931f..b3be09cd8170 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -208,6 +208,13 @@ add_library(ggml-base ggml-quants.h gguf.cpp) +# Enable native SIMD for ggml-quants.c (needed for K-means training in quantization) +include(CheckCCompilerFlag) +check_c_compiler_flag("-march=native" GGML_COMPILER_SUPPORTS_MARCH_NATIVE) +if (GGML_COMPILER_SUPPORTS_MARCH_NATIVE) + set_source_files_properties(ggml-quants.c PROPERTIES COMPILE_FLAGS "-march=native") +endif() + set_target_properties(ggml-base PROPERTIES VERSION ${GGML_VERSION} SOVERSION ${GGML_VERSION_MAJOR} diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 1f29ec86712d..6d8c795f5dd6 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -424,7 +424,7 @@ struct ggml_backend_meta_buffer_context { // FIXME // The size of the split state cache is unbounded and can theoretically grow infinitely large. // However, it is also expensive to build and clearing it on every rebuild in ggml_backend_meta_graph_compute is too expensive. - static constexpr size_t nbtc = GGML_TENSOR_SIZE - sizeof(ggml_tensor::padding); + static constexpr size_t nbtc = GGML_TENSOR_SIZE - sizeof(ggml_tensor::quant_levels); std::map, std::pair> split_state_cache; int debug; diff --git a/ggml/src/ggml-blas/ggml-blas.cpp b/ggml/src/ggml-blas/ggml-blas.cpp index b4c735267e04..a979bb7a73ad 100644 --- a/ggml/src/ggml-blas/ggml-blas.cpp +++ b/ggml/src/ggml-blas/ggml-blas.cpp @@ -1,5 +1,15 @@ #include "ggml-impl.h" #include "ggml-blas.h" + +// Helper: compute quant_levels stride for a given row. +// For Q2_KPT (per-block levels), stride depends on tensor width. +static inline size_t ggml_quant_levels_stride(ggml_type type, size_t constant_stride, int64_t ne0) { + if (type == GGML_TYPE_Q2_KPT) { + return (size_t)(ne0 / 256) * 4 * sizeof(float); + } + return constant_stride; +} + #include "ggml-backend-impl.h" #include @@ -77,10 +87,11 @@ static void ggml_backend_blas_mul_mat(ggml_backend_blas_context * ctx, struct gg const int min_rows_per_thread = std::max((int)(min_cols_per_thread/ne00), 1); const int n_threads = std::max(std::min(ctx->n_threads, (int)(ne01/min_rows_per_thread)), 1); + const size_t lrs = ggml_quant_levels_stride(src0->type, ggml_get_type_traits(src0->type)->levels_row_stride, src0->ne[0]); #ifdef GGML_USE_OPENMP #pragma omp parallel for num_threads(n_threads) for (int64_t i01 = 0; i01 < ne01; i01++) { - to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00); + to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00, (const char*)src0->quant_levels + i01*lrs); } #else for (int i = 1; i < n_threads; i++) { @@ -89,7 +100,7 @@ static void ggml_backend_blas_mul_mat(ggml_backend_blas_context * ctx, struct gg if (start < end) { ctx->tasks.push_back(std::async(std::launch::async, [=]() { for (int64_t i01 = start; i01 < end; i01++) { - to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00); + to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00, (const char*)src0->quant_levels + i01*lrs); } })); } @@ -99,7 +110,7 @@ static void ggml_backend_blas_mul_mat(ggml_backend_blas_context * ctx, struct gg const int64_t start = 0; const int64_t end = ne01/n_threads; for (int64_t i01 = start; i01 < end; i01++) { - to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00); + to_float((const char *) x + i01*nb01, wplane + i01*ne00, ne00, (const char*)src0->quant_levels + i01*lrs); } } #endif diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h index a51b37dcdb59..a0d99948fcc8 100644 --- a/ggml/src/ggml-common.h +++ b/ggml/src/ggml-common.h @@ -298,6 +298,7 @@ typedef struct { } block_q2_K; static_assert(sizeof(block_q2_K) == 2*sizeof(ggml_half) + QK_K/16 + QK_K/4, "wrong q2_K block size/padding"); + // 3-bit quantization // weight is represented as x = a * q // 16 blocks of 16 elements each @@ -327,6 +328,12 @@ typedef struct { } block_q4_K; static_assert(sizeof(block_q4_K) == 2*sizeof(ggml_half) + K_SCALE_SIZE + QK_K/2, "wrong q4_K block size/padding"); +// Q3_KPT: Q3_K with learned per-tensor levels +// Reuses block_q3_K structure but maps 3-bit indices through learned level table +typedef block_q3_K block_q3_kpt; +#define Q3KPT_N_LEVELS 8 + + // 5-bit quantization // 8 blocks of 32 elements each // weight is represented as x = a * q + b @@ -449,6 +456,115 @@ typedef struct { } block_iq4_xs; static_assert(sizeof(block_iq4_xs) == sizeof(ggml_half) + sizeof(uint16_t) + QK_K/64 + QK_K/2, "wrong iq4_xs block size/padding"); +// 3.875 bpw - per-tensor Lloyd-Max scalar quantization +// 256 elements = 16 sub-blocks of 16, 8-entry level table trained per tensor +// Layout: 2 (d) + 2 (dmin) + 24 (scales: 32x6-bit) + 96 (qs: 256x3-bit) = 124 bytes +typedef struct { + ggml_half d; // 2 bytes: global scale for 16-elem sub-block ranges + ggml_half dmin; // 2 bytes: global scale for sub-block neg_mins + uint8_t scales[3*QK_K/32]; // 24 bytes: 32 x 6-bit (indices 0..15 = ranges, 16..31 = neg_mins) + uint8_t qs[3*QK_K/8]; // 96 bytes: 256 x 3-bit Lloyd-Max level index, sequential +} block_q3_pt; +static_assert(sizeof(block_q3_pt) == 124, "wrong q3_pt block size"); + +#define Q3PT_N_LEVELS 8 + +// Q4_DPT: IQ4_NL with learned per-tensor int8 levels (4.125 bpw) +// Block format: identical to block_iq4_nl (2 + 16 = 18 bytes per 32 elements) +typedef block_iq4_nl block_q4_dpt; +#define Q4DPT_N_LEVELS 16 + +// Q2_DPT: 2-bit per-tensor Lloyd-Max scalar quantization (2.5 bpw) +// Block format: 2 bytes (FP16 scale) + 8 bytes (2-bit indices for 32 elements) = 10 bytes per block +// 4 learned int8 levels per tensor, optimized via Lloyd-Max k-means +typedef struct { + ggml_half d; // 2 bytes: FP16 scale (delta) + uint8_t qs[8]; // 8 bytes: 2-bit indices (4 values per byte, 32 elements total) +} block_q2_dpt; +static_assert(sizeof(block_q2_dpt) == sizeof(ggml_half) + 8, "wrong q2_dpt block size/padding"); + +#define QK2_DPT 32 +#define Q2DPT_N_LEVELS 4 + +// Q2_KPT: Q2_K with learned per-tensor float levels (2.625 bpw) +// Reuses block_q2_K structure but maps 2-bit indices through learned level table +typedef block_q2_K block_q2_kpt; +#define Q2KPT_N_LEVELS 4 + +// IQ2_TQ: Trellis Quantized with RNG codebook (2.0625 bpw) +// +// Reconstruction: y[i] = d * hash(seed, block_idx, position, trellis_state, qs_idx) +// where hash is a deterministic function mapping to [-1, 1] +// and trellis_state evolves as: next = (state + idx + 1) & 7 +// +// Block layout (66 bytes per 256 elements): +// IQ2_TQ: 2-bit scalar quantization with per-tensor trained asymmetric grid table +// 32 groups of 8 elements per 256-element super-block +// - ggml_half d (2 bytes): super-block scale +// - uint8_t scales[16] (16 bytes): 32 × 4-bit grid entry index per group +// - uint8_t qs[64] (64 bytes): 256 × 2-bit element index within grid entry +// recon[j] = d * IQ2TQ_GRID_SCALE * grid[group_idx][elem_idx] +typedef struct { + ggml_half d; // Super-block scale (2 bytes) + uint8_t scales[QK_K/16]; // 32 × 4-bit grid entry index per group (16 bytes) + uint8_t qs[QK_K/4]; // 256 × 2-bit element index (64 bytes) +} block_iq2_tq; +static_assert(sizeof(block_iq2_tq) == 82, "wrong iq2_tq block size"); +// 2 + 16 + 64 = 82 bytes per 256 weights = 2.5625 bpw + +#define IQ2TQ_GROUP_SIZE 8 // Elements per group +#define IQ2TQ_N_GROUPS (QK_K / IQ2TQ_GROUP_SIZE) // 32 groups per super-block +#define IQ2TQ_GRID_SCALE 0.125f // Grid value multiplier: recon = d * GRID_SCALE * grid_int8 + +// IQ3_TQ: 3-bit scalar quantization with per-tensor trained asymmetric grid table (3.5625 bpw) +// 32 groups of 8 elements per 256-element super-block +// Each grid entry has 8 int8 levels (3 bits → 8 values per element) +// Grid table: 16 entries × 8 int8 = 128 bytes per tensor +// Block layout: +// - ggml_half d (2 bytes): super-block scale +// - uint8_t scales[16] (16 bytes): 32 × 4-bit grid entry index per group +// - uint8_t qs[96] (96 bytes): 256 × 3-bit element index within grid entry +// recon[j] = d * IQ3TQ_GRID_SCALE * grid[group_idx][elem_idx] +typedef struct { + ggml_half d; // Super-block scale (2 bytes) + uint8_t scales[QK_K/16]; // 32 × 4-bit grid entry index per group (16 bytes) + uint8_t qs[3*QK_K/8]; // 256 × 3-bit element index (96 bytes) +} block_iq3_tq; +static_assert(sizeof(block_iq3_tq) == 114, "wrong iq3_tq block size"); +// 2 + 16 + 96 = 114 bytes per 256 weights = 3.5625 bpw + +#define IQ3TQ_GROUP_SIZE 8 // Elements per group +#define IQ3TQ_N_GROUPS (QK_K / IQ3TQ_GROUP_SIZE) // 32 groups per super-block +#define IQ3TQ_N_LEVELS 8 // 3-bit → 8 levels per grid entry +#define IQ3TQ_GRID_SCALE 0.125f // Grid value multiplier +#define IQ3TQ_GRID_SIZE 128 // 16 entries × 8 int8 = 128 bytes per tensor + +// IQ1_BN: 8D vector quantized with per-tensor trained 4096-entry codebook (1.5625 bpw) +// 32 groups of 8 elements per 256-element super-block +// Each group selects one of 4096 trained 8D vectors via 12-bit codebook index +// Codebook: 4096 entries × 8 int8 = 32768 bytes per tensor +// Block layout: +// - ggml_half d (2 bytes): super-block scale +// - uint8_t qs[48] (48 bytes): 32 × 12-bit codebook indices packed in pairs +// 12-bit pair packing (groups 2k, 2k+1 → 3 bytes at qs[3k]): +// idx_even = qs[3k] | ((qs[3k+1] & 0x0F) << 8) +// idx_odd = (qs[3k+1] >> 4) | (qs[3k+2] << 4) +// recon[g*8+k] = d * IQ1BN_GRID_SCALE * codebook[ci][k] +typedef struct { + ggml_half d; // Super-block scale (2 bytes) + uint8_t qs[3*QK_K/16]; // 32 × 12-bit codebook indices packed in pairs (48 bytes) +} block_iq1_bn; +static_assert(sizeof(block_iq1_bn) == 50, "wrong iq1_bn block size"); +// 2 + 48 = 50 bytes per 256 weights = 1.5625 bpw + +#define IQ1BN_GROUP_SIZE 8 +#define IQ1BN_N_GROUPS (QK_K / IQ1BN_GROUP_SIZE) // 32 +#define IQ1BN_CODEBOOK_K 4096 // number of codebook entries +#define IQ1BN_CODEBOOK_DIM 8 // vector dimension (= group size) +#define IQ1BN_GRID_SCALE 0.125f // Grid value multiplier +#define IQ1BN_CODEBOOK_SIZE (IQ1BN_CODEBOOK_K * IQ1BN_CODEBOOK_DIM) // 32768 bytes +#define IQ1BN_AUX_SIZE IQ1BN_CODEBOOK_SIZE // 32768 bytes + #endif // GGML_COMMON_DECL #endif // GGML_COMMON_DECL diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index da2ac98a48f1..90adb7eebdc1 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -33,6 +33,8 @@ #define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K #define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0 #define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K +#define ggml_vec_dot_q3_pt_q8_K_generic ggml_vec_dot_q3_pt_q8_K +#define ggml_vec_dot_q4_dpt_q8_0_generic ggml_vec_dot_q4_dpt_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 @@ -201,6 +203,15 @@ #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__riscv) // quants.c +#define quantize_row_q8_K_generic quantize_row_q8_K +#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K +#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K +#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K +#define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0 +#define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K +#define ggml_vec_dot_q3_pt_q8_K_generic ggml_vec_dot_q3_pt_q8_K +#define ggml_vec_dot_q4_dpt_q8_0_generic ggml_vec_dot_q4_dpt_q8_0 +#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1 @@ -303,6 +314,8 @@ #define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K #define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0 #define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K +#define ggml_vec_dot_q3_pt_q8_K_generic ggml_vec_dot_q3_pt_q8_K +#define ggml_vec_dot_q4_dpt_q8_0_generic ggml_vec_dot_q4_dpt_q8_0 #define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c index fe6213329708..f45355ecfc1a 100644 --- a/ggml/src/ggml-cpu/arch/arm/quants.c +++ b/ggml/src/ggml-cpu/arch/arm/quants.c @@ -137,7 +137,111 @@ void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in //===================================== Dot products ================================= -void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + const int qk = QK1_0; // 128 + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + GGML_UNUSED(levels); + + const block_q1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + float sumf = 0.0f; + +#if defined(__ARM_NEON) + float32x4_t sumv = vdupq_n_f32(0.0f); + + for (int i = 0; i < nb; i++) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); + + // Process 4 Q8_0 blocks (each has 32 elements) + for (int k = 0; k < 4; k++) { + const block_q8_0 * GGML_RESTRICT yb = &y[i * 4 + k]; + const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); + + // Get the 4 bytes of bits for this Q8_0 block (32 bits = 4 bytes) + // Bits are at offset k*4 bytes in x[i].qs + const uint8_t * bits = &x[i].qs[k * 4]; + + // Load 32 int8 values from y + const int8x16_t y0 = vld1q_s8(yb->qs); + const int8x16_t y1 = vld1q_s8(yb->qs + 16); + + // Byte 0-1: bits for y0[0..15] + const uint64_t expand0 = table_b2b_0[bits[0]]; + const uint64_t expand1 = table_b2b_0[bits[1]]; + // Byte 2-3: bits for y1[0..15] + const uint64_t expand2 = table_b2b_0[bits[2]]; + const uint64_t expand3 = table_b2b_0[bits[3]]; + + // Build the sign vectors by reinterpreting the table values + uint8x8_t e0 = vcreate_u8(expand0); + uint8x8_t e1 = vcreate_u8(expand1); + uint8x8_t e2 = vcreate_u8(expand2); + uint8x8_t e3 = vcreate_u8(expand3); + + // Shift right by 4 to get 0 or 1 + int8x8_t s0 = vreinterpret_s8_u8(vshr_n_u8(e0, 4)); + int8x8_t s1 = vreinterpret_s8_u8(vshr_n_u8(e1, 4)); + int8x8_t s2 = vreinterpret_s8_u8(vshr_n_u8(e2, 4)); + int8x8_t s3 = vreinterpret_s8_u8(vshr_n_u8(e3, 4)); + + // Convert 0/1 to -1/+1: sign = 2*val - 1 + int8x8_t one = vdup_n_s8(1); + s0 = vsub_s8(vadd_s8(s0, s0), one); // 2*s0 - 1 + s1 = vsub_s8(vadd_s8(s1, s1), one); + s2 = vsub_s8(vadd_s8(s2, s2), one); + s3 = vsub_s8(vadd_s8(s3, s3), one); + + // Combine into 16-element vectors + int8x16_t signs0 = vcombine_s8(s0, s1); + int8x16_t signs1 = vcombine_s8(s2, s3); + + // Multiply signs with y values and accumulate + // dot(signs, y) where signs are +1/-1 + int32x4_t p0 = ggml_vdotq_s32(vdupq_n_s32(0), signs0, y0); + int32x4_t p1 = ggml_vdotq_s32(p0, signs1, y1); + + // Scale by d1 and accumulate + sumv = vmlaq_n_f32(sumv, vcvtq_f32_s32(p1), d0 * d1); + } + } + + sumf = vaddvq_f32(sumv); +#else + // Scalar fallback + for (int i = 0; i < nb; i++) { + const float d0 = GGML_FP16_TO_FP32(x[i].d); + + // Process 4 Q8_0 blocks + for (int k = 0; k < 4; k++) { + const float d1 = GGML_FP16_TO_FP32(y[i*4 + k].d); + + int sumi = 0; + for (int j = 0; j < QK8_0; j++) { + const int bit_index = k * QK8_0 + j; + const int byte_index = bit_index / 8; + const int bit_offset = bit_index % 8; + + const int xi = ((x[i].qs[byte_index] >> bit_offset) & 1) ? 1 : -1; + sumi += xi * y[i*4 + k].qs[j]; + } + sumf += d0 * d1 * sumi; + } + } +#endif + + *s = sumf; +} + + +void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK1_0; // 128 const int nb = n / qk; @@ -220,7 +324,7 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi } -void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -513,7 +617,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; } -void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_1; const int nb = n / qk; @@ -733,12 +837,13 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo *s = sumf; } -void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); assert(n % QK_NVFP4 == 0); const block_nvfp4 * GGML_RESTRICT x = vx; @@ -843,7 +948,92 @@ void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo *s = sumf; } -void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + GGML_UNUSED(levels); + assert(n % QK_NVFP4 == 0); + + const block_nvfp4 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + // Each NVFP4 super-block (64 elements) spans 2 q8_0 blocks + const int nb = n / QK_NVFP4; + + float sumf = 0; + +#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA) + const int8x16_t values = vld1q_s8(kvalues_mxfp4); + const uint8x16_t m4b = vdupq_n_u8(0x0f); + float32x4_t acc = vdupq_n_f32(0.0f); + + for (int ib = 0; ib < nb; ++ib) { + const uint8x16_t q4bits_0 = vld1q_u8(x[ib].qs); + const uint8x16_t q4bits_1 = vld1q_u8(x[ib].qs + 16); + + const int8x16_t q4_lo_0 = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits_0, m4b)); + const int8x16_t q4_hi_0 = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits_0, 4)); + const int8x16_t q4_lo_1 = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits_1, m4b)); + const int8x16_t q4_hi_1 = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits_1, 4)); + + const int8x16_t q8_0a = vld1q_s8(y[2*ib].qs); + const int8x16_t q8_0b = vld1q_s8(y[2*ib].qs + 16); + const int8x16_t q8_lo_0 = vcombine_s8(vget_low_s8(q8_0a), vget_low_s8(q8_0b)); + const int8x16_t q8_hi_0 = vcombine_s8(vget_high_s8(q8_0a), vget_high_s8(q8_0b)); + + const int8x16_t q8_1a = vld1q_s8(y[2*ib+1].qs); + const int8x16_t q8_1b = vld1q_s8(y[2*ib+1].qs + 16); + const int8x16_t q8_lo_1 = vcombine_s8(vget_low_s8(q8_1a), vget_low_s8(q8_1b)); + const int8x16_t q8_hi_1 = vcombine_s8(vget_high_s8(q8_1a), vget_high_s8(q8_1b)); + + const int32x4_t p0 = vaddq_s32( + ggml_vdotq_s32(vdupq_n_s32(0), q4_lo_0, q8_lo_0), + ggml_vdotq_s32(vdupq_n_s32(0), q4_hi_0, q8_hi_0)); + const int32x4_t p1 = vaddq_s32( + ggml_vdotq_s32(vdupq_n_s32(0), q4_lo_1, q8_lo_1), + ggml_vdotq_s32(vdupq_n_s32(0), q4_hi_1, q8_hi_1)); + + const int32x4_t sums = vpaddq_s32(p0, p1); + + // Decode 4 UE4M3 scales to f32 and multiply with q8 scales + const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d); + const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d); + const float32x4_t nvsc = { + ggml_ue4m3_to_fp32(x[ib].d[0]), + ggml_ue4m3_to_fp32(x[ib].d[1]), + ggml_ue4m3_to_fp32(x[ib].d[2]), + ggml_ue4m3_to_fp32(x[ib].d[3]) + }; + const float32x4_t scales = vmulq_f32(nvsc, (float32x4_t){dy0, dy0, dy1, dy1}); + + acc = vfmaq_f32(acc, vcvtq_f32_s32(sums), scales); + } + sumf = vaddvq_f32(acc); +#else + for (int ib = 0; ib < nb; ++ib) { + for (int si = 0; si < 4; ++si) { + const float d = ggml_ue4m3_to_fp32(x[ib].d[si]); + const int q8b = si / 2; + const int q8o = (si % 2) * QK_NVFP4_SUB; + const float dy = GGML_CPU_FP16_TO_FP32(y[2*ib + q8b].d); + + int sumi_lo = 0, sumi_hi = 0; + for (int j = 0; j < QK_NVFP4_SUB/2; ++j) { + const uint8_t qv = x[ib].qs[si*(QK_NVFP4_SUB/2) + j]; + sumi_lo += y[2*ib + q8b].qs[q8o + j + 0] * kvalues_mxfp4[qv & 0xf]; + sumi_hi += y[2*ib + q8b].qs[q8o + j + QK_NVFP4_SUB/2] * kvalues_mxfp4[qv >> 4]; + } + sumf += dy * d * (sumi_lo + sumi_hi); + } + } +#endif + *s = sumf; +} + +void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -955,7 +1145,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; } -void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_1; const int nb = n / qk; @@ -1073,7 +1263,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; } -void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -3959,6 +4149,10 @@ void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo #endif } +void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); +} + void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); diff --git a/ggml/src/ggml-cpu/arch/loongarch/quants.c b/ggml/src/ggml-cpu/arch/loongarch/quants.c index 9c43da6cf89a..e25b9c291781 100644 --- a/ggml/src/ggml-cpu/arch/loongarch/quants.c +++ b/ggml/src/ggml-cpu/arch/loongarch/quants.c @@ -644,7 +644,7 @@ static inline __m128i get_scale_shuffle(int i) { } #endif -void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -772,7 +772,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; } -void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_1; const int nb = n / qk; @@ -827,11 +827,11 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(y); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -880,11 +880,11 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(sumf); UNUSED(x); UNUSED(y); - ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_1; const int nb = n / qk; @@ -936,11 +936,11 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(sumf); UNUSED(x); UNUSED(y); - ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -1012,7 +1012,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(sumf); UNUSED(x); UNUSED(y); - ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -2078,6 +2078,10 @@ void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo #endif } +void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); +} + #if defined(__loongarch_asx) static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) { const __m256i a = __lasx_xvmulwev_h_b(x, y); diff --git a/ggml/src/ggml-cpu/arch/powerpc/quants.c b/ggml/src/ggml-cpu/arch/powerpc/quants.c index 644c380c7381..c8fc0a3766ce 100644 --- a/ggml/src/ggml-cpu/arch/powerpc/quants.c +++ b/ggml/src/ggml-cpu/arch/powerpc/quants.c @@ -141,7 +141,7 @@ void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, i //===================================== Dot products ================================= -void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -207,11 +207,11 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(y); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_q4_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_1; const int nb = n / qk; @@ -274,7 +274,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(y); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -340,11 +340,11 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(y); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -412,11 +412,11 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(sumf); UNUSED(x); UNUSED(y); - ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_1; const int nb = n / qk; @@ -488,11 +488,11 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(sumf); UNUSED(x); UNUSED(y); - ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -557,7 +557,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(y); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -2000,6 +2000,10 @@ void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo #endif } +void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); +} + void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); @@ -2190,7 +2194,7 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v UNUSED(nb); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_iq4_nl_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_nl_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } diff --git a/ggml/src/ggml-cpu/arch/riscv/quants.c b/ggml/src/ggml-cpu/arch/riscv/quants.c index 47e9180bf9bb..69e3fba49261 100644 --- a/ggml/src/ggml-cpu/arch/riscv/quants.c +++ b/ggml/src/ggml-cpu/arch/riscv/quants.c @@ -219,7 +219,7 @@ void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in //===================================== Dot products ================================= -void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined(__riscv_v) const int qk = QK8_0; const int nb = n / qk; @@ -270,11 +270,11 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; #else - ggml_vec_dot_q4_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined(__riscv_v) const int qk = QK8_1; const int nb = n / qk; @@ -321,11 +321,11 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; #else - ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined(__riscv_v) const int qk = QK8_0; const int nb = n / qk; @@ -375,11 +375,11 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; #else - ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined(__riscv_v) const int qk = QK8_1; const int nb = n / qk; @@ -428,11 +428,11 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; #else - ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -476,7 +476,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -5095,6 +5095,10 @@ void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo #endif } +void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); +} + #if defined __riscv_v static NOINLINE void ggml_vec_dot_iq3_xxs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(n % QK_K == 0); diff --git a/ggml/src/ggml-cpu/arch/s390/quants.c b/ggml/src/ggml-cpu/arch/s390/quants.c index 500857579a70..c1186901926e 100644 --- a/ggml/src/ggml-cpu/arch/s390/quants.c +++ b/ggml/src/ggml-cpu/arch/s390/quants.c @@ -146,7 +146,7 @@ void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, i //===================================== Dot products ================================= -void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -201,11 +201,11 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(y); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_q4_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_1; const int nb = n / qk; @@ -258,7 +258,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(y); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -353,11 +353,11 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(y); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -495,11 +495,11 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(y); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_1; const int nb = n / qk; @@ -648,11 +648,11 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(y); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -698,7 +698,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(y); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -1388,7 +1388,7 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v UNUSED(nb); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_iq4_nl_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_nl_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -1463,3 +1463,7 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); #endif } + +void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); +} diff --git a/ggml/src/ggml-cpu/arch/wasm/quants.c b/ggml/src/ggml-cpu/arch/wasm/quants.c index 0a7119b4e1fb..d2df68fddc4f 100644 --- a/ggml/src/ggml-cpu/arch/wasm/quants.c +++ b/ggml/src/ggml-cpu/arch/wasm/quants.c @@ -229,7 +229,7 @@ void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in //===================================== Dot products ================================= -void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -355,7 +355,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; } -void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_1; const int nb = n / qk; @@ -365,6 +365,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q4_1 * GGML_RESTRICT x = vx; const block_q8_1 * GGML_RESTRICT y = vy; @@ -423,11 +424,11 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(sumf); ggml_vec_dot_q4_1_q8_1_generic( - n, s, bs, vx, bx, vy, by, nrc); + n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -514,11 +515,11 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(sumf); UNUSED(x); UNUSED(y); - ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_1; const int nb = n / qk; @@ -609,11 +610,11 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(sumf); UNUSED(x); UNUSED(y); - ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; @@ -677,7 +678,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(y); UNUSED(ib); UNUSED(sumf); - ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q8_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -1290,3 +1291,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); #endif } + +void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); +} diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c index ea54cfe44ce4..2740abfed479 100644 --- a/ggml/src/ggml-cpu/arch/x86/quants.c +++ b/ggml/src/ggml-cpu/arch/x86/quants.c @@ -698,7 +698,155 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi #endif } -void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); + const int qk = QK1_0; + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + +#if defined(__AVX2__) + const __m256i ones_8 = _mm256_set1_epi8(1); + const __m256i ones_16 = _mm256_set1_epi16(1); + const __m256i byte_shuf = _mm256_setr_epi8( + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3); + const __m256i bit_masks = _mm256_setr_epi8( + 1, 2, 4, 8, 16, 32, 64, (char) -128, 1, 2, 4, 8, 16, 32, 64, (char) -128, + 1, 2, 4, 8, 16, 32, 64, (char) -128, 1, 2, 4, 8, 16, 32, 64, (char) -128); + const __m256i zero = _mm256_setzero_si256(); + __m256 acc = _mm256_setzero_ps(); + + for (int ib = 0; ib < nb; ++ib) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d); + const uint32_t * GGML_RESTRICT qs32 = (const uint32_t *) x[ib].qs; + const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4]; + + __m256 acc_block; + { + const __m256i qy = _mm256_loadu_si256((const __m256i *) y_ptr[0].qs); + const __m256i sm = _mm256_cmpeq_epi8( + _mm256_and_si256(_mm256_shuffle_epi8(_mm256_set1_epi32((int) qs32[0]), byte_shuf), bit_masks), zero); + const __m256i sy = _mm256_sub_epi8(_mm256_xor_si256(qy, sm), sm); + const __m256i s32 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy), ones_16); + acc_block = _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[0].d)), _mm256_cvtepi32_ps(s32)); + } + for (int K = 1; K < 4; ++K) { + const __m256i qy = _mm256_loadu_si256((const __m256i *) y_ptr[K].qs); + const __m256i sm = _mm256_cmpeq_epi8( + _mm256_and_si256(_mm256_shuffle_epi8(_mm256_set1_epi32((int) qs32[K]), byte_shuf), bit_masks), zero); + const __m256i sy = _mm256_sub_epi8(_mm256_xor_si256(qy, sm), sm); + const __m256i s32 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy), ones_16); + acc_block = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[K].d)), _mm256_cvtepi32_ps(s32), acc_block); + } + acc = _mm256_fmadd_ps(_mm256_set1_ps(d0), acc_block, acc); + } + + *s = hsum_float_8(acc); +#elif defined(__AVX__) + const __m128i ones_8 = _mm_set1_epi8(1); + const __m128i ones_16 = _mm_set1_epi16(1); + const __m128i zero = _mm_setzero_si128(); + __m256 acc = _mm256_setzero_ps(); + + for (int ib = 0; ib < nb; ++ib) { + const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d); + const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4]; + __m256 acc_block; + { + const __m256i bit_mask = bytes_from_bits_32(&x[ib].qs[0]); + const __m128i bit_mask_0 = _mm256_castsi256_si128(bit_mask); + const __m128i bit_mask_1 = _mm256_extractf128_si256(bit_mask, 1); + const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[0].qs[0]); + const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[0].qs[16]); + const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero); + const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero); + const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0); + const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1); + const __m128i sum16_0 = _mm_maddubs_epi16(ones_8, sy_0); + const __m128i sum16_1 = _mm_maddubs_epi16(ones_8, sy_1); + const __m128i sum32_0 = _mm_madd_epi16(sum16_0, ones_16); + const __m128i sum32_1 = _mm_madd_epi16(sum16_1, ones_16); + const __m256 q = _mm256_cvtepi32_ps(MM256_SET_M128I(sum32_1, sum32_0)); + acc_block = _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[0].d)), q); + } + for(int K = 1; K < 4; ++K) { + const __m256i bit_mask = bytes_from_bits_32(&x[ib].qs[(K) * 4]); + const __m128i bit_mask_0 = _mm256_castsi256_si128(bit_mask); + const __m128i bit_mask_1 = _mm256_extractf128_si256(bit_mask, 1); + const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[(K)].qs[0]); + const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[(K)].qs[16]); + const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero); + const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero); + const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0); + const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1); + const __m128i sum16_0 = _mm_maddubs_epi16(ones_8, sy_0); + const __m128i sum16_1 = _mm_maddubs_epi16(ones_8, sy_1); + const __m128i sum32_0 = _mm_madd_epi16(sum16_0, ones_16); + const __m128i sum32_1 = _mm_madd_epi16(sum16_1, ones_16); + const __m256 q = _mm256_cvtepi32_ps(MM256_SET_M128I(sum32_1, sum32_0)); + acc_block = _mm256_add_ps(acc_block, _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[(K)].d)), q)); + } +#undef Q1_AVX_BLOCK + + acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_set1_ps(d0), acc_block)); + } + + *s = hsum_float_8(acc); +#elif defined(__SSSE3__) + const __m128i ones_8 = _mm_set1_epi8(1); + const __m128i ones_16 = _mm_set1_epi16(1); + const __m128i zero = _mm_setzero_si128(); + __m128 acc_0 = _mm_setzero_ps(); + __m128 acc_1 = _mm_setzero_ps(); + __m128 acc_2 = _mm_setzero_ps(); + __m128 acc_3 = _mm_setzero_ps(); + + for (int ib = 0; ib < nb; ++ib) { + const __m128 d0 = _mm_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d)); + const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4]; + +#define Q1_SSSE3_BLOCK(QS_OFF, Y_IDX, ACC) \ + { \ + const __m128i bit_mask_0 = bytes_from_bits_16(&x[ib].qs[(QS_OFF) + 0]); \ + const __m128i bit_mask_1 = bytes_from_bits_16(&x[ib].qs[(QS_OFF) + 2]); \ + const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[(Y_IDX)].qs[0]); \ + const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[(Y_IDX)].qs[16]); \ + const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero); \ + const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero); \ + const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0); \ + const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1); \ + const __m128i sum_0 = _mm_madd_epi16(_mm_maddubs_epi16(ones_8, sy_0), ones_16); \ + const __m128i sum_1 = _mm_madd_epi16(_mm_maddubs_epi16(ones_8, sy_1), ones_16); \ + const __m128 q = _mm_cvtepi32_ps(_mm_add_epi32(sum_0, sum_1)); \ + (ACC) = _mm_add_ps((ACC), _mm_mul_ps(_mm_mul_ps(d0, _mm_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[(Y_IDX)].d))), q)); \ + } + Q1_SSSE3_BLOCK(0, 0, acc_0) + Q1_SSSE3_BLOCK(4, 1, acc_1) + Q1_SSSE3_BLOCK(8, 2, acc_2) + Q1_SSSE3_BLOCK(12, 3, acc_3) +#undef Q1_SSSE3_BLOCK + } + + *s = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3); +#else + UNUSED(nb); + UNUSED(x); + UNUSED(y); + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); +#endif +} + +void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); const int qk = QK8_0; const int nb = n / qk; @@ -856,7 +1004,8 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; } -void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); const int qk = QK8_1; const int nb = n / qk; @@ -911,11 +1060,12 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(ib); - ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -1001,7 +1151,8 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo *s = sumf; } -void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -1139,7 +1290,8 @@ void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo *s = sumf; } -void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); const int qk = QK8_0; const int nb = n / qk; @@ -1215,11 +1367,12 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(ib); UNUSED(x); UNUSED(y); - ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); const int qk = QK8_1; const int nb = n / qk; @@ -1301,11 +1454,12 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(ib); UNUSED(x); UNUSED(y); - ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_1_q8_1_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); const int qk = QK8_0; const int nb = n / qk; @@ -1373,7 +1527,8 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; } -void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -1501,11 +1656,12 @@ void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -1567,11 +1723,12 @@ void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -1759,11 +1916,12 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -2031,11 +2189,12 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -2209,11 +2368,12 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(kmask2); UNUSED(kmask3); UNUSED(utmp); - ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -2419,11 +2579,12 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(kmask2); UNUSED(kmask3); UNUSED(utmp); - ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -2616,7 +2777,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -2657,7 +2818,8 @@ static const int8_t keven_signs_q2xs[1024] = { }; #endif -void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -2771,11 +2933,12 @@ void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -3068,11 +3231,12 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -3253,11 +3417,12 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -3377,11 +3542,12 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -3587,11 +3753,17 @@ void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); + ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); +} + +void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -3706,11 +3878,12 @@ void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -3913,11 +4086,12 @@ void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(y); UNUSED(nb); UNUSED(scale); - ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -4001,7 +4175,185 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v *s = sumf; } -void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_dpt_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + assert(n % QK4_NL == 0); + static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same"); + + const block_q4_dpt * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + const int nb = n / QK4_NL; + + const int8_t * values = (const int8_t *)levels; + GGML_ASSERT(values != NULL && "Q4_DPT levels not set for tensor"); + + int ib = 0; + float sumf = 0; + +#if defined __AVX2__ + + const __m128i values128 = _mm_loadu_si128((const __m128i*)values); + const __m128i m4b = _mm_set1_epi8(0x0f); + const __m256i mone = _mm256_set1_epi16(1); + + __m256 accum1 = _mm256_setzero_ps(); + __m256 accum2 = _mm256_setzero_ps(); + for (; ib + 1 < nb; ib += 2) { + const __m128i q4bits_1 = _mm_loadu_si128((const __m128i*)x[ib + 0].qs); + const __m128i q4bits_2 = _mm_loadu_si128((const __m128i*)x[ib + 1].qs); + const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)y[ib + 0].qs); + const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)y[ib + 1].qs); + const __m256i q4b_1 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)), + _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b))); + const __m256i q4b_2 = MM256_SET_M128I(_mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)), + _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b))); + const __m256i p16_1 = mul_add_epi8(q4b_1, q8b_1); + const __m256i p16_2 = mul_add_epi8(q4b_2, q8b_2); + const __m256i p_1 = _mm256_madd_epi16(p16_1, mone); + const __m256i p_2 = _mm256_madd_epi16(p16_2, mone); + accum1 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 0].d)*GGML_CPU_FP16_TO_FP32(x[ib + 0].d)), + _mm256_cvtepi32_ps(p_1), accum1); + accum2 = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 1].d)*GGML_CPU_FP16_TO_FP32(x[ib + 1].d)), + _mm256_cvtepi32_ps(p_2), accum2); + } + + sumf = hsum_float_8(_mm256_add_ps(accum1, accum2)); + +#elif defined __AVX__ + const __m128i values128 = _mm_loadu_si128((const __m128i*)values); + const __m128i m4b = _mm_set1_epi8(0x0f); + + __m256 accum = _mm256_setzero_ps(); + for (; ib + 1 < nb; ib += 2) { + const __m128i q4bits_1 = _mm_loadu_si128((const __m128i *)x[ib + 0].qs); + const __m128i q4bits_2 = _mm_loadu_si128((const __m128i *)x[ib + 1].qs); + const __m128i q8b_1_0 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs); + const __m128i q8b_1_1 = _mm_loadu_si128((const __m128i *)y[ib + 0].qs + 1); + const __m128i q8b_2_0 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs); + const __m128i q8b_2_1 = _mm_loadu_si128((const __m128i *)y[ib + 1].qs + 1); + + const __m128i q4b_1_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_1, m4b)); + const __m128i q4b_1_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_1, 4), m4b)); + const __m128i q4b_2_0 = _mm_shuffle_epi8(values128, _mm_and_si128(q4bits_2, m4b)); + const __m128i q4b_2_1 = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q4bits_2, 4), m4b)); + + const __m256 p = mul_sum_i8_quad_float(q4b_1_0, q4b_1_1, q4b_2_0, q4b_2_1, q8b_1_0, q8b_1_1, q8b_2_0, q8b_2_1); + const __m256 deltas = quad_fp16_delta_float(x[ib].d, y[ib].d, x[ib + 1].d, y[ib + 1].d); + accum = _mm256_add_ps(_mm256_mul_ps(deltas, p), accum); + } + + sumf = hsum_float_8(accum); + +#endif + for (; ib < nb; ++ib) { + const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_FP16_TO_FP32(x[ib].d); + int sumi1 = 0, sumi2 = 0; + for (int j = 0; j < QK4_NL/2; ++j) { + sumi1 += y[ib].qs[j+ 0] * values[x[ib].qs[j] & 0xf]; + sumi2 += y[ib].qs[j+QK4_NL/2] * values[x[ib].qs[j] >> 4]; + } + sumf += d * (sumi1 + sumi2); + } + *s = sumf; +} + +void ggml_vec_dot_q2_dpt_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + assert(n % QK2_DPT == 0); + static_assert(QK2_DPT == QK8_0, "QK2_DPT and QK8_0 must be the same"); + + const block_q2_dpt * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + const int nb = n / QK2_DPT; + + const int8_t * values = (const int8_t *)levels; + GGML_ASSERT(values != NULL && "Q2_DPT levels not set for tensor"); + + int ib = 0; + float sumf = 0; + +#if defined __AVX2__ + + const __m128i values128 = _mm_loadu_si128((const __m128i*)values); + const __m128i m3 = _mm_set1_epi8(0x03); + + __m256 accum = _mm256_setzero_ps(); + for (; ib + 1 < nb; ib += 2) { + const __m128i q2bits_1 = _mm_loadu_si128((const __m128i*)x[ib + 0].qs); + const __m128i q2bits_2 = _mm_loadu_si128((const __m128i*)x[ib + 1].qs); + const __m256i q8b_1 = _mm256_loadu_si256((const __m256i *)y[ib + 0].qs); + const __m256i q8b_2 = _mm256_loadu_si256((const __m256i *)y[ib + 1].qs); + + // Extract 2-bit indices and lookup values - process 8 elements at a time + // For each byte of q2bits, we have 4 x 2-bit indices + const __m128i q2_01_l = _mm_shuffle_epi8(values128, _mm_and_si128(q2bits_1, m3)); + const __m128i q2_01_h = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q2bits_1, 2), m3)); + const __m128i q2_02_l = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q2bits_1, 4), m3)); + const __m128i q2_02_h = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q2bits_1, 6), m3)); + const __m128i q2_11_l = _mm_shuffle_epi8(values128, _mm_and_si128(q2bits_2, m3)); + const __m128i q2_11_h = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q2bits_2, 2), m3)); + const __m128i q2_12_l = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q2bits_2, 4), m3)); + const __m128i q2_12_h = _mm_shuffle_epi8(values128, _mm_and_si128(_mm_srli_epi16(q2bits_2, 6), m3)); + + // Combine pairs into __m256i + const __m256i q4b_1a = MM256_SET_M128I(q2_01_h, q2_01_l); + const __m256i q4b_1b = MM256_SET_M128I(q2_02_h, q2_02_l); + const __m256i q4b_2a = MM256_SET_M128I(q2_11_h, q2_11_l); + const __m256i q4b_2b = MM256_SET_M128I(q2_12_h, q2_12_l); + + // Split q8 into pairs and compute dot products + const __m256i q8b_1a = _mm256_and_si256(q8b_1, _mm256_set1_epi16(0x00ff)); + const __m256i q8b_1b = _mm256_srli_epi16(q8b_1, 8); + const __m256i q8b_2a = _mm256_and_si256(q8b_2, _mm256_set1_epi16(0x00ff)); + const __m256i q8b_2b = _mm256_srli_epi16(q8b_2, 8); + + const __m256i p16_1a = mul_add_epi8(q4b_1a, q8b_1a); + const __m256i p16_1b = mul_add_epi8(q4b_1b, q8b_1b); + const __m256i p16_2a = mul_add_epi8(q4b_2a, q8b_2a); + const __m256i p16_2b = mul_add_epi8(q4b_2b, q8b_2b); + + const __m256i mone = _mm256_set1_epi16(1); + const __m256i p_1 = _mm256_add_epi32(_mm256_madd_epi16(p16_1a, mone), _mm256_madd_epi16(p16_1b, mone)); + const __m256i p_2 = _mm256_add_epi32(_mm256_madd_epi16(p16_2a, mone), _mm256_madd_epi16(p16_2b, mone)); + + accum = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 0].d)*GGML_CPU_FP16_TO_FP32(x[ib + 0].d)), + _mm256_cvtepi32_ps(p_1), accum); + accum = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y[ib + 1].d)*GGML_CPU_FP16_TO_FP32(x[ib + 1].d)), + _mm256_cvtepi32_ps(p_2), accum); + } + + sumf = hsum_float_8(accum); + +#endif + for (; ib < nb; ++ib) { + const float d = GGML_CPU_FP16_TO_FP32(y[ib].d)*GGML_CPU_FP16_TO_FP32(x[ib].d); + int sumi = 0; + for (int j = 0; j < QK2_DPT/4; ++j) { + uint8_t q = x[ib].qs[j]; + sumi += y[ib].qs[j*4 + 0] * values[(q >> 0) & 3]; + sumi += y[ib].qs[j*4 + 1] * values[(q >> 2) & 3]; + sumi += y[ib].qs[j*4 + 2] * values[(q >> 4) & 3]; + sumi += y[ib].qs[j*4 + 3] * values[(q >> 6) & 3]; + } + sumf += d * sumi; + } + *s = sumf; +} + +void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -4103,6 +4455,6 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 8a0e5c255fa2..b8c459dd410c 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -3,6 +3,7 @@ #include "ggml-backend-impl.h" #include "ggml-backend.h" +#include "ggml-quants.h" #include "traits.h" #include "ggml-cpu-impl.h" #include "ggml-impl.h" @@ -403,6 +404,52 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_K, .nrows = 1, }, + [GGML_TYPE_Q3_PT] = { + // from_float not set — requires codebook initialization via q3pt_set_codebook() + .vec_dot = ggml_vec_dot_q3_pt_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, + }, + [GGML_TYPE_Q3_KPT] = { + // from_float not set — requires level initialization via q3kpt_set_levels() + .vec_dot = ggml_vec_dot_q3_kpt_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, + }, + [GGML_TYPE_Q4_DPT] = { + // from_float not set — requires level initialization via q4dpt_set_levels() + .vec_dot = ggml_vec_dot_q4_dpt_q8_0, + .vec_dot_type = GGML_TYPE_Q8_0, + .nrows = 1, + }, + [GGML_TYPE_Q2_DPT] = { + // from_float not set — requires level initialization via q2dpt_set_levels() + .vec_dot = ggml_vec_dot_q2_dpt_q8_0, + .vec_dot_type = GGML_TYPE_Q8_0, + .nrows = 1, + }, + [GGML_TYPE_Q2_KPT] = { + // from_float not set — requires level initialization via q2kpt_set_levels() + .vec_dot = ggml_vec_dot_q2_kpt_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, + .levels_row_stride = 0, // computed dynamically: (ne0/QK_K)*Q2KPT_N_LEVELS*sizeof(float) + }, + [GGML_TYPE_IQ2_TQ] = { + .vec_dot = ggml_vec_dot_iq2_tq_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, + }, + [GGML_TYPE_IQ3_TQ] = { + .vec_dot = ggml_vec_dot_iq3_tq_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, + }, + [GGML_TYPE_IQ1_BN] = { + .vec_dot = ggml_vec_dot_iq1_bn_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, + }, [GGML_TYPE_I32] = { .from_float = (ggml_from_float_t) ggml_cpu_fp32_to_i32, }, @@ -1172,8 +1219,15 @@ static void ggml_compute_forward_mul_mat_one_chunk( const bool src1_cont = ggml_is_contiguous(src1); - ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot; - enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; + ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot; + enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; + // For Q2_KPT, levels are per-block: stride = (ne00 / QK_K) * Q2KPT_N_LEVELS * sizeof(float) + // ne00 is the number of elements per row in src0 (input dimension), NOT ne0 (= ne01 = output rows). + // For non-square matrices (e.g. ffn_up: [hidden, intermediate]) ne00 != ne01, so ne00 is correct. + // For other types, use the static stride from type_traits_cpu + const size_t levels_row_stride = (type == GGML_TYPE_Q2_KPT) + ? (ne00 / QK_K) * Q2KPT_N_LEVELS * sizeof(float) + : type_traits_cpu[type].levels_row_stride; // broadcast factors const int64_t r2 = ne12 / ne02; @@ -1234,7 +1288,11 @@ static void ggml_compute_forward_mul_mat_one_chunk( //} for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ir0 += num_rows_per_vec_dot) { - vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot); + // For Q2_KPT, levels are stored per-expert: [expert0_rows, expert1_rows, ...] + // So for 3D tensors we need to index by (i03 * ne01 + ir0) + const size_t levels_row_idx = (type == GGML_TYPE_Q2_KPT && ne03 > 1) ? (i03 * ne01 + ir0) : ir0; + const void * row_levels = (const char*)src0->quant_levels + levels_row_idx * levels_row_stride; + vec_dot(ne00, &tmp[ir0 - iir0], (num_rows_per_vec_dot > 1 ? 16 : 0), src0_row + ir0 * nb01, (num_rows_per_vec_dot > 1 ? nb01 : 0), src1_col, (num_rows_per_vec_dot > 1 ? src1_col_stride : 0), num_rows_per_vec_dot, row_levels); } for (int cn = 0; cn < num_rows_per_vec_dot; ++cn) { @@ -1306,7 +1364,8 @@ void ggml_compute_forward_mul_mat( nb1/ggml_type_size(dst->type), src0->type, src1->type, - dst->type)) + dst->type, + src0->quant_levels)) goto UseGgmlGemm1; return; } @@ -1374,7 +1433,8 @@ UseGgmlGemm1:; nb1/ggml_type_size(dst->type), src0->type, vec_dot_type, - dst->type)) + dst->type, + src0->quant_levels)) goto UseGgmlGemm2; return; } @@ -1474,8 +1534,14 @@ static void ggml_compute_forward_mul_mat_id_one_chunk( const enum ggml_type type = src0->type; - ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot; - enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; + ggml_vec_dot_t const vec_dot = type_traits_cpu[type].vec_dot; + enum ggml_type const vec_dot_type = type_traits_cpu[type].vec_dot_type; + // For Q2_KPT, levels are per-block: stride = (ne00 / QK_K) * Q2KPT_N_LEVELS * sizeof(float) + // ne00 is the input dimension (elements per row in src0), NOT ne0 (= ne01 = output rows). + // For other types, use the static stride from type_traits_cpu + const size_t levels_row_stride = (type == GGML_TYPE_Q2_KPT) + ? (ne00 / QK_K) * Q2KPT_N_LEVELS * sizeof(float) + : type_traits_cpu[type].levels_row_stride; const int64_t blck_0 = 16; const int64_t blck_1 = 16; @@ -1508,7 +1574,8 @@ static void ggml_compute_forward_mul_mat_id_one_chunk( float * dst_col = (float *) ((char *) dst->data + (i1*nb1 + i2*nb2)); for (int64_t ir0 = iir0; ir0 < iir0 + blck_0 && ir0 < ir0_end; ++ir0) { - vec_dot(ne00, &tmp[ir0 - iir0], 0, src0_cur + ir0*nb01, 0, src1_col, 0, 1); + const void * row_levels = (const char*)src0->quant_levels + (cur_a * ne01 + ir0) * levels_row_stride; + vec_dot(ne00, &tmp[ir0 - iir0], 0, src0_cur + ir0*nb01, 0, src1_col, 0, 1, row_levels); } memcpy(&dst_col[iir0], tmp, (MIN(iir0 + blck_0, ir0_end) - iir0)*sizeof(float)); diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index 0b8323e60ce6..aa86707bfeb6 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -1356,16 +1356,20 @@ class tinyBLAS_Q0_AVX { const TA *A, int64_t lda, const TB *B, int64_t ldb, TC *C, int64_t ldc, - int ith, int nth) + int ith, int nth, + const int8_t * custom_table = nullptr) : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) { - const int8_t kvalues_iq4nl[16] = { - -127, -104, -83, -65, - -49, -35, -22, -10, - 1, 13, 25, 38, - 53, 69, 89, 113 - }; - - iq4nlt = _mm_loadu_si128((const __m128i *)kvalues_iq4nl); + if (custom_table) { + iq4nlt = _mm_loadu_si128((const __m128i *)custom_table); + } else { + const int8_t kvalues_iq4nl[16] = { + -127, -104, -83, -65, + -49, -35, -22, -10, + 1, 13, 25, 38, + 53, 69, 89, 113 + }; + iq4nlt = _mm_loadu_si128((const __m128i *)kvalues_iq4nl); + } } void matmul(int64_t m, int64_t n) { @@ -3692,7 +3696,7 @@ class tinyBLAS_PPC { */ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64_t n, int64_t k, const void *A, int64_t lda, const void *B, int64_t ldb, void *C, - int64_t ldc, int Atype, int Btype, int Ctype) { + int64_t ldc, int Atype, int Btype, int Ctype, const void * quant_levels) { assert(m >= 0); assert(n >= 0); @@ -4032,6 +4036,26 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 #endif } + case GGML_TYPE_Q4_DPT: { + if (Btype != GGML_TYPE_Q8_0) + return false; +#if defined(__AVX2__) || defined(__AVX512F__) || defined(__AVX__) + // Q4_DPT has identical block layout to IQ4_NL (block_q4_dpt = block_iq4_nl) + // but uses a per-tensor lookup table instead of the fixed IQ4_NL values. + const int8_t * levels = (const int8_t *)quant_levels; + if (!levels) return false; + tinyBLAS_Q0_AVX tb{ + k, (const block_iq4_nl *)A, lda, + (const block_q8_0 *)B, ldb, + (float *)C, ldc, + params->ith, params->nth, levels}; + tb.matmul(m, n); + return true; +#else + return false; +#endif + } + default: return false; } diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.h b/ggml/src/ggml-cpu/llamafile/sgemm.h index 867b0c04aee8..117a36560e7e 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.h +++ b/ggml/src/ggml-cpu/llamafile/sgemm.h @@ -18,7 +18,7 @@ extern "C" { bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t, int64_t, int64_t, const void *, int64_t, const void *, int64_t, void *, int64_t, - int, int, int); + int, int, int, const void * quant_levels); #ifdef __cplusplus } diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index c555831ce72d..58111d58c871 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -8,6 +8,19 @@ #include "unary-ops.h" #include "vec.h" +// Helper: compute quant_levels stride for a given row. +// For most types this is the constant levels_row_stride from type_traits. +// For Q2_KPT (per-block levels), stride depends on tensor width (ne[0]). +static inline size_t ggml_quant_levels_stride(ggml_type type, size_t constant_stride, int64_t ne0) { + if (type == GGML_TYPE_Q2_KPT) { + // Q2_KPT has Q2KPT_N_LEVELS floats per 256-element block + // Stride = (ne0 / 256) * Q2KPT_N_LEVELS * sizeof(float) + return (size_t)(ne0 / 256) * 4 * sizeof(float); + } + return constant_stride; +} + + #include #include #include @@ -517,9 +530,11 @@ static void ggml_compute_forward_dup_from_q( const int64_t i10 = i - i13*ne10*ne11*ne12 - i12*ne10*ne11 - i11*ne10; const int64_t dst_offset = i10*nb10 + i11*nb11 + i12*nb12 + i13*nb13; + const size_t q_lrs0 = ggml_quant_levels_stride(src0->type, ggml_get_type_traits_cpu(src0->type)->levels_row_stride, src0->ne[0]); dequantize_row_q( (const void *) ((char *) src0->data + x_offset), - (float *) ((char *) dst->data + dst_offset), qk); + (float *) ((char *) dst->data + dst_offset), qk, + (const char*)src0->quant_levels + i01 * q_lrs0); } } @@ -639,7 +654,8 @@ static void ggml_compute_forward_add_q_f32( assert(ne00 % 32 == 0); // unquantize row from src0 to temp buffer - dequantize_row_q(src0_row, wdata, ne00); + const size_t q_lrs_add = ggml_quant_levels_stride(src0->type, ggml_get_type_traits_cpu(src0->type)->levels_row_stride, src0->ne[0]); + dequantize_row_q(src0_row, wdata, ne00, (const char*)src0->quant_levels + i1 * q_lrs_add); // add src1 ggml_vec_acc_f32(ne00, wdata, src1_row); // quantize row to dst @@ -688,6 +704,9 @@ void ggml_compute_forward_add( case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ2_TQ: + case GGML_TYPE_IQ3_TQ: + case GGML_TYPE_IQ1_BN: { ggml_compute_forward_add_q_f32(params, dst); } break; @@ -974,7 +993,8 @@ static void ggml_compute_forward_add1_q_f32( assert(ne0 % 32 == 0); // unquantize row from src0 to temp buffer - dequantize_row_q(src0_row, wdata, ne0); + const size_t q_lrs_add = ggml_quant_levels_stride(src0->type, ggml_get_type_traits_cpu(src0->type)->levels_row_stride, src0->ne[0]); + dequantize_row_q(src0_row, wdata, ne00, (const char*)src0->quant_levels + i1 * q_lrs_add); // add src1 ggml_vec_acc1_f32(ne0, wdata, v); // quantize row to dst @@ -1139,6 +1159,9 @@ void ggml_compute_forward_add1( case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ2_TQ: + case GGML_TYPE_IQ3_TQ: + case GGML_TYPE_IQ1_BN: { ggml_compute_forward_add1_q_f32(params, dst); } break; @@ -1269,6 +1292,9 @@ void ggml_compute_forward_acc( case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ2_TQ: + case GGML_TYPE_IQ3_TQ: + case GGML_TYPE_IQ1_BN: default: { GGML_ABORT("fatal error"); @@ -4440,7 +4466,8 @@ static void ggml_compute_forward_out_prod_q_f32( float * s1 = (float *) ((char *) src1->data + (i1*nb10 + i11*nb11 + i12*nb12 + i13*nb13)); float * d = (float *) ((char *) dst->data + ( i1*nb1 + i2*nb2 + i3*nb3)); - dequantize_row_q(s0, wdata, ne0); + const size_t q_lrs_op = ggml_quant_levels_stride(src0->type, ggml_get_type_traits_cpu(src0->type)->levels_row_stride, src0->ne[0]); + dequantize_row_q(s0, wdata, ne0, (const char*)src0->quant_levels + i01 * q_lrs_op); ggml_vec_mad_f32(ne0, d, wdata, *s1); } } @@ -4477,6 +4504,9 @@ void ggml_compute_forward_out_prod( case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ2_TQ: + case GGML_TYPE_IQ3_TQ: + case GGML_TYPE_IQ1_BN: { ggml_compute_forward_out_prod_q_f32(params, dst); } break; @@ -4754,6 +4784,9 @@ void ggml_compute_forward_set( case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ2_TQ: + case GGML_TYPE_IQ3_TQ: + case GGML_TYPE_IQ1_BN: default: { GGML_ABORT("fatal error"); @@ -4817,9 +4850,21 @@ static void ggml_compute_forward_get_rows_q( GGML_ASSERT(i01 >= 0 && i01 < ne01); + const size_t q_lrs_gr = ggml_quant_levels_stride(src0->type, ggml_get_type_traits_cpu(src0->type)->levels_row_stride, src0->ne[0]); + // For Q2_KPT with 3D tensors, levels are indexed by [i12 * ne02 * ne01 + i11 * ne01 + i01] + // For 2D tensors, levels are indexed by [i11 * ne01 + i01] (or just [i01] if ne02 == 1) + size_t levels_row_idx; + if (type == GGML_TYPE_Q2_KPT && ne03 > 1) { + levels_row_idx = (i12 * ne02 + i11) * ne01 + i01; + } else if (type == GGML_TYPE_Q2_KPT) { + levels_row_idx = i11 * ne01 + i01; + } else { + levels_row_idx = i01; + } dequantize_row_q( (const void *) ((char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03), - (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc); + (float *) ((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3), nc, + (const char*)src0->quant_levels + levels_row_idx * q_lrs_gr); } } @@ -4978,6 +5023,9 @@ void ggml_compute_forward_get_rows( case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: + case GGML_TYPE_IQ2_TQ: + case GGML_TYPE_IQ3_TQ: + case GGML_TYPE_IQ1_BN: { ggml_compute_forward_get_rows_q(params, dst); } break; @@ -5555,7 +5603,7 @@ static void ggml_compute_forward_soft_max_ext_back_f32( // linear runtime, no additional memory float dot_y_dy = 0; - ggml_vec_dot_f32 (nc, &dot_y_dy, 0, y, 0, dy, 0, 1); + ggml_vec_dot_f32 (nc, &dot_y_dy, 0, y, 0, dy, 0, 1, nullptr); ggml_vec_cpy_f32 (nc, dx, dy); ggml_vec_acc1_f32 (nc, dx, -dot_y_dy); ggml_vec_mul_f32 (nc, dx, dx, y); @@ -5690,6 +5738,8 @@ void ggml_compute_forward_clamp( case GGML_TYPE_NVFP4: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: + case GGML_TYPE_Q3_KPT: + case GGML_TYPE_Q4_DPT: case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: @@ -5702,6 +5752,12 @@ void ggml_compute_forward_clamp( case GGML_TYPE_IQ1_M: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS: + case GGML_TYPE_Q3_PT: + case GGML_TYPE_Q2_KPT: + case GGML_TYPE_Q2_DPT: + case GGML_TYPE_IQ2_TQ: + case GGML_TYPE_IQ3_TQ: + case GGML_TYPE_IQ1_BN: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ2_S: case GGML_TYPE_Q8_K: @@ -6126,7 +6182,7 @@ static void ggml_compute_forward_conv_transpose_1d_f16_f32( float v = 0; ggml_vec_dot_f16(ne02, &v, 0, (ggml_fp16_t *) wdata_src + i1n, 0, - (ggml_fp16_t *) wdata_kernel + i00*ne02, 0, 1); + (ggml_fp16_t *) wdata_kernel + i00*ne02, 0, 1, nullptr); dst_data[i10*s0 + i00] += v; } } @@ -6214,7 +6270,7 @@ static void ggml_compute_forward_conv_transpose_1d_f32( float v = 0; ggml_vec_dot_f32(ne02, &v, 0, wdata_src + i1n, 0, - wdata_kernel + i00*ne02, 0, 1); + wdata_kernel + i00*ne02, 0, 1, nullptr); dst_data[i10*s0 + i00] += v; } } @@ -7212,11 +7268,11 @@ static void ggml_compute_forward_conv_transpose_2d_impl( if constexpr (std::is_same_v) { ggml_vec_dot_f16(ne03, &v, 0, wdata_src + i1n, 0, - wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1); + wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1, nullptr); } else { ggml_vec_dot_f32(ne03, &v, 0, wdata_src + i1n, 0, - wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1); + wdata_kernel + i01*ne00*ne03 + i00*ne03, 0, 1, nullptr); } dst_data[(i11*stride + i01)*ne0 + i10*stride + i00] += v; } @@ -8489,7 +8545,7 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk( float s; // KQ value const char * k_data = (const char *) k->data + ( ic*nbk1 + ik2*nbk2 + ik3*nbk3); - kq_vec_dot(DK, &s, 0, k_data, 0, Q_q, 0, 1); + kq_vec_dot(DK, &s, 0, k_data, 0, Q_q, 0, 1, k->quant_levels); s = s*scale; // scale KQ value @@ -8536,7 +8592,7 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk( // V += v*expf(s - M) if (v_to_float) { - v_to_float(v_data, V32, DV); + v_to_float(v_data, V32, DV, v->quant_levels); ggml_vec_mad_f32(DV, VKQ32, V32, vs); } else { // V is F32 @@ -9254,7 +9310,7 @@ static void ggml_compute_forward_flash_attn_back_f32( ggml_vec_dot_f32(neq0, S + i1, 0, (float *) ((char *) k->data + (ik1*nbk1 + ik2*nbk2 + ik3*nbk3)), 0, - (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1); + (float *) ((char *) q->data + (iq1*nbq1 + iq2*nbq2 + iq3*nbq3)), 0, 1, nullptr); } // scale @@ -9368,7 +9424,7 @@ static void ggml_compute_forward_flash_attn_back_f32( // S = SM * (S - dot(SM, S)) float dot_SM_gradSM = 0; - ggml_vec_dot_f32 (masked_begin, &dot_SM_gradSM, 0, SM, 0, S, 0, 1); + ggml_vec_dot_f32 (masked_begin, &dot_SM_gradSM, 0, SM, 0, S, 0, 1, nullptr); ggml_vec_acc1_f32(M, S, -dot_SM_gradSM); ggml_vec_mul_f32 (masked_begin, S, S, SM); @@ -10746,7 +10802,7 @@ static void ggml_compute_forward_gated_delta_net_one_chunk( // delta[j] = sum_i S[i][j] * k[i] = dot(row j of M, k) for (int64_t j = 0; j < S_v; ++j) { float sum = 0.0f; - ggml_vec_dot_f32(S_v, &sum, 0, &s_out[j * S_v], 0, k_d, 0, 1); + ggml_vec_dot_f32(S_v, &sum, 0, &s_out[j * S_v], 0, k_d, 0, 1, nullptr); delta[j] = (v_d[j] - sum) * beta_val; } @@ -10758,7 +10814,7 @@ static void ggml_compute_forward_gated_delta_net_one_chunk( // attn_out[j] = sum_i S[i][j] * q[i] = dot(row j of M, q) for (int64_t j = 0; j < S_v; ++j) { float sum = 0.0f; - ggml_vec_dot_f32(S_v, &sum, 0, &s_out[j * S_v], 0, q_d, 0, 1); + ggml_vec_dot_f32(S_v, &sum, 0, &s_out[j * S_v], 0, q_d, 0, 1, nullptr); attn_data[j] = sum * scale; } diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index e5f9a4083f9c..91908ec22c02 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -120,7 +120,8 @@ void quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRI //===================================== Dot products ================================= -void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); const int qk = QK1_0; const int nb = n / qk; @@ -171,7 +172,8 @@ void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, c } -void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); const int qk = QK8_0; const int nb = n / qk; @@ -208,7 +210,8 @@ void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, c } // TODO: add WASM SIMD -void ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); const int qk = QK8_1; const int nb = n / qk; @@ -244,7 +247,8 @@ void ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, c *s = sumf; } -void ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -276,7 +280,8 @@ void ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, } // NVFP4: super-block of 64 elements = 4 sub-blocks of 16 = 2 q8_0 blocks -void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -311,7 +316,8 @@ void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = sumf; } -void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); const int qk = QK8_0; const int nb = n / qk; @@ -354,7 +360,8 @@ void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, c *s = sumf; } -void ggml_vec_dot_q5_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); const int qk = QK8_1; const int nb = n / qk; @@ -397,7 +404,8 @@ void ggml_vec_dot_q5_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, c *s = sumf; } -void ggml_vec_dot_q8_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q8_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); const int qk = QK8_0; const int nb = n / qk; @@ -427,7 +435,8 @@ void ggml_vec_dot_q8_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, c *s = sumf; } -void ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -479,7 +488,8 @@ void ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = sumf; } -void ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -511,7 +521,8 @@ void ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = sumf; } -void ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -563,7 +574,8 @@ void ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, c *s = sumf; } -void ggml_vec_dot_q3_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q3_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -642,7 +654,8 @@ void ggml_vec_dot_q3_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, c *s = sumf; } -void ggml_vec_dot_q4_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -716,8 +729,7 @@ void ggml_vec_dot_q4_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, c for (int l = 0; l < 8; ++l) sumf += sums[l]; *s = sumf; } - -void ggml_vec_dot_q5_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -747,6 +759,7 @@ void ggml_vec_dot_q5_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, c float sumf = 0; for (int i = 0; i < nb; ++i) { + GGML_UNUSED(levels); const uint8_t * GGML_RESTRICT q4 = x[i].qs; const uint8_t * GGML_RESTRICT hm = x[i].qh; const int8_t * GGML_RESTRICT q8 = y[i].qs; @@ -797,7 +810,8 @@ void ggml_vec_dot_q5_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, c *s = sumf; } -void ggml_vec_dot_q6_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q6_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -852,7 +866,8 @@ void ggml_vec_dot_q6_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, c *s = sumf; } -void ggml_vec_dot_iq2_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -894,7 +909,8 @@ void ggml_vec_dot_iq2_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs *s = 0.125f * sumf; } -void ggml_vec_dot_iq2_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -944,7 +960,8 @@ void ggml_vec_dot_iq2_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = 0.125f * sumf; } -void ggml_vec_dot_iq2_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -996,7 +1013,8 @@ void ggml_vec_dot_iq2_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = 0.125f * sumf; } -void ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -1040,7 +1058,8 @@ void ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs *s = 0.25f * sumf; } -void ggml_vec_dot_iq3_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq3_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -1096,7 +1115,65 @@ void ggml_vec_dot_iq3_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = sumf; } -void ggml_vec_dot_iq1_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q3_pt_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q3_pt * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + const float * lv = (const float *)levels; + GGML_ASSERT(lv != NULL && "Q3_PT levels not set for tensor"); + + float sumf = 0.f; + for (int i = 0; i < nb; ++i) { + const float xd = GGML_CPU_FP16_TO_FP32(x[i].d); + const float xdmin = GGML_CPU_FP16_TO_FP32(x[i].dmin); + const float yd = y[i].d; + const uint8_t * sc = x[i].scales; + const uint8_t * qs = x[i].qs; + const int8_t * q8 = y[i].qs; + + float block_sum = 0.f; + for (int ib = 0; ib < QK_K/16; ++ib) { + // Inline 6-bit unpack for range scale (index ib) and neg_min scale (index ib + QK_K/16) + const int sbit0 = ib * 6, sbyte0 = sbit0 / 8, soff0 = sbit0 % 8; + const int sbit1 = (ib + QK_K/16) * 6, sbyte1 = sbit1 / 8, soff1 = sbit1 % 8; + uint8_t qrange = (sc[sbyte0] >> soff0) & 0x3F; + if (soff0 > 2) { qrange |= (uint8_t)((sc[sbyte0+1] << (8 - soff0)) & 0x3F); } + uint8_t qnmin = (sc[sbyte1] >> soff1) & 0x3F; + if (soff1 > 2) { qnmin |= (uint8_t)((sc[sbyte1+1] << (8 - soff1)) & 0x3F); } + const float range = xd * (float)qrange; + const float sub_min = -xdmin * (float)qnmin; + + float sum_lq = 0.f; + for (int j = 0; j < 16; ++j) { + // Inline 3-bit unpack + const int qk = ib * 16 + j; + const int qbit = qk * 3; + const int qbyte = qbit / 8; + const int qoff = qbit % 8; + int q = (qs[qbyte] >> qoff) & 0x7; + if (qoff > 5) { q |= (int)((qs[qbyte+1] << (8 - qoff)) & 0x7); } + sum_lq += lv[q] * (float)q8[qk]; + } + // min contribution uses precomputed 16-element sum from block_q8_K.bsums + block_sum += sum_lq * range + sub_min * (float)y[i].bsums[ib]; + } + sumf += block_sum * yd; + } + *s = sumf; +} + +void ggml_vec_dot_iq1_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -1139,7 +1216,375 @@ void ggml_vec_dot_iq1_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = sumf; } -void ggml_vec_dot_iq1_m_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +// Q3_KPT vec_dot - similar to Q3_K but with learned levels +void ggml_vec_dot_q3_kpt_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q3_kpt * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + const float * lv = (const float *)levels; + GGML_ASSERT(lv != NULL && "Q3_KPT levels not set for tensor"); + + const uint32_t kmask1 = 0x03030303; + const uint32_t kmask2 = 0x0f0f0f0f; + + float sumf = 0.f; + for (int i = 0; i < nb; ++i) { + const float d_all = GGML_CPU_FP16_TO_FP32(x[i].d); + const float yd = y[i].d; + const uint8_t * q = x[i].qs; + const uint8_t * hm = x[i].hmask; + const int8_t * q8 = y[i].qs; + uint8_t m = 1; + + uint32_t aux32[4]; + memcpy(aux32, x[i].scales, 12); + uint32_t tmp = aux32[2]; + aux32[2] = ((aux32[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4); + aux32[3] = ((aux32[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4); + aux32[0] = (aux32[0] & kmask2) | (((tmp >> 0) & kmask1) << 4); + aux32[1] = (aux32[1] & kmask2) | (((tmp >> 2) & kmask1) << 4); + const uint8_t * aux = (const uint8_t *)aux32; + + int is = 0; + float block_sum = 0.f; + for (int blk = 0; blk < QK_K; blk += 128) { + int shift = 0; + for (int j = 0; j < 4; ++j) { + int sc1 = (int)aux[is] - 32; + int sc2 = (int)aux[is+1] - 32; + is += 2; + float dl1 = d_all * sc1; + float dl2 = d_all * sc2; + + float sum1 = 0.f, sum2 = 0.f; + for (int l = 0; l < 16; ++l) { + int k_idx = ((q[l+0] >> shift) & 3) + ((hm[l+0] & m) ? 4 : 0); + sum1 += (lv[k_idx] * 7.0f - 4.0f) * (float)q8[l+0]; + } + for (int l = 0; l < 16; ++l) { + int k_idx = ((q[l+16] >> shift) & 3) + ((hm[l+16] & m) ? 4 : 0); + sum2 += (lv[k_idx] * 7.0f - 4.0f) * (float)q8[l+16]; + } + block_sum += dl1 * sum1 + dl2 * sum2; + + shift += 2; + m <<= 1; + q8 += 32; + } + q += 32; + } + sumf += block_sum * yd; + } + *s = sumf; +} + +void ggml_vec_dot_q3_kpt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + ggml_vec_dot_q3_kpt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); +} + +// Q2_KPT vec_dot - similar to Q2_K but with learned levels +void ggml_vec_dot_q2_kpt_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_q2_kpt * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + const float * lv = (const float *)levels; + GGML_ASSERT(lv != NULL && "Q2_KPT levels not set for tensor"); + + float sumf = 0; + + for (int i = 0; i < nb; ++i) { + // Per-block levels: block i uses lv[i*4 + 0..3] + const float * block_lv = lv + i * Q2KPT_N_LEVELS; + + // Precompute mapped levels for this block: ml[k] = levels[k] * 3.0 + float ml[Q2KPT_N_LEVELS]; + for (int k = 0; k < Q2KPT_N_LEVELS; ++k) { + ml[k] = block_lv[k] * 3.0f; + } + + const uint8_t * q2 = x[i].qs; + const int8_t * q8 = y[i].qs; + const uint8_t * sc = x[i].scales; + + // Min term: accumulate integer bsums * min_scale (same as Q2_K) + int summs = 0; + for (int j = 0; j < 16; ++j) { + summs += y[i].bsums[j] * (sc[j] >> 4); + } + + const float dall = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].d); + const float dmin = y[i].d * GGML_CPU_FP16_TO_FP32(x[i].dmin); + + // Scale term: need floating-point because levels are non-uniform + int is = 0; + float fsum = 0; + for (int k = 0; k < QK_K/128; ++k) { + int shift = 0; + for (int j = 0; j < 4; ++j) { + int d_sc = sc[is++] & 0xF; + float suml = 0; + for (int l = 0; l < 16; ++l) { + int idx = (q2[l] >> shift) & 3; + suml += ml[idx] * (float)q8[l]; + } + fsum += d_sc * suml; + + d_sc = sc[is++] & 0xF; + suml = 0; + for (int l = 16; l < 32; ++l) { + int idx = (q2[l] >> shift) & 3; + suml += ml[idx] * (float)q8[l]; + } + fsum += d_sc * suml; + + shift += 2; + q8 += 32; + } + q2 += 32; + } + sumf += dall * fsum - dmin * summs; + } + *s = sumf; +} + +void ggml_vec_dot_q2_kpt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + ggml_vec_dot_q2_kpt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); +} + +// IQ2_TQ: 2-bit with asymmetric 4-tuple grid per group +// Default grid table — only used when no per-tensor grid is available +static const int8_t iq2tq_grid_cpu[16][4] = { + {-20, -8, -2, 6}, {-14, -8, -2, 4}, {-16,-10, 0, 12}, {-14, -4, 2, 8}, + {-20, -4, 4, 12}, {-8, -4, 0, 4}, {-8, -4, 0, 8}, {-12, -6, 2, 12}, + {-4, -2, 2, 4}, {-10, -2, 4, 8}, {-16, -6, 4, 20}, {-12, -2, 6, 14}, + {-8, -2, 4, 14}, {-4, 0, 4, 8}, {-8, -2, 6, 22}, {-4, 2, 8, 14}, +}; + +void ggml_vec_dot_iq2_tq_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + + const int8_t (*grid)[4] = levels ? (const int8_t (*)[4])levels : (const int8_t (*)[4])iq2tq_grid_cpu; + const block_iq2_tq * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + const int nb = n / QK_K; + + float sumf = 0; + + for (int i = 0; i < nb; ++i) { + const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * IQ2TQ_GRID_SCALE; + const float yd = y[i].d; + const int8_t * q8 = y[i].qs; + + int32_t fsum = 0; + + for (int g = 0; g < IQ2TQ_N_GROUPS; ++g) { + int si = (x[i].scales[g / 2] >> (4 * (g % 2))) & 0xF; + const int8_t * ge = grid[si]; + const int8_t * q8g = q8 + g * 8; + + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + int qi = (x[i].qs[j / 4] >> ((j % 4) * 2)) & 3; + fsum += (int32_t)ge[qi] * (int32_t)q8g[k]; + } + } + + sumf += d * yd * (float)fsum; + } + + *s = sumf; +} + +// IQ3_TQ default grid (must match ggml-quants.c) +static const int8_t iq3tq_grid_cpu[16][8] = { + {-24,-18,-12, -6, 0, 6, 12, 18}, + {-20,-15,-10, -5, 0, 5, 10, 15}, + {-16,-12, -8, -4, 0, 4, 8, 12}, + {-12, -8, -4, -2, 0, 2, 4, 8}, + {-24,-16, -8, -2, 2, 6, 10, 14}, + {-14,-10, -6, -2, 2, 8, 16, 24}, + {-20,-14, -8, -4, 0, 4, 10, 18}, + {-18,-10, -4, 0, 4, 8, 14, 20}, + { -8, -6, -4, -2, 0, 2, 4, 6}, + {-10, -6, -4, -2, 2, 4, 6, 10}, + {-22,-14, -6, -2, 2, 6, 14, 22}, + {-16, -8, -4, -2, 0, 4, 8, 16}, + {-24,-20,-16,-12, -8, -4, 0, 4}, + { -4, 0, 4, 8, 12, 16, 20, 24}, + {-20,-16,-10, -4, 4, 10, 16, 20}, + {-12, -8, -6, -2, 2, 6, 8, 12}, +}; + +void ggml_vec_dot_iq3_tq_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + + const int8_t (*grid)[8] = levels ? (const int8_t (*)[8])levels : (const int8_t (*)[8])iq3tq_grid_cpu; + const block_iq3_tq * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + const int nb = n / QK_K; + + float sumf = 0; + + for (int i = 0; i < nb; ++i) { + const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * IQ3TQ_GRID_SCALE; + const float yd = y[i].d; + const int8_t * q8 = y[i].qs; + + int32_t fsum = 0; + + for (int g = 0; g < IQ3TQ_N_GROUPS; ++g) { + int si = (x[i].scales[g / 2] >> (4 * (g % 2))) & 0xF; + const int8_t * ge = grid[si]; + const int8_t * q8g = q8 + g * 8; + + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + // 3-bit unpack + int bit_pos = j * 3; + int byte_idx = bit_pos >> 3; + int bit_off = bit_pos & 7; + uint16_t val = x[i].qs[byte_idx]; + if (bit_off > 5) val |= ((uint16_t)x[i].qs[byte_idx + 1] << 8); + int qi = (val >> bit_off) & 7; + fsum += (int32_t)ge[qi] * (int32_t)q8g[k]; + } + } + + sumf += d * yd * (float)fsum; + } + + *s = sumf; +} + +// IQ1_BN: 8D vector quantized — codebook[256][8] + scale_table[16] +void ggml_vec_dot_iq1_bn_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + + GGML_ASSERT(levels && "IQ1_BN requires per-tensor codebook in quant_levels"); + const int8_t * codebook = (const int8_t *)levels; + const block_iq1_bn * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + const int nb = n / QK_K; + + float sumf = 0; + + for (int i = 0; i < nb; ++i) { + const float d = GGML_CPU_FP16_TO_FP32(x[i].d) * IQ1BN_GRID_SCALE; + const float yd = y[i].d; + const int8_t * q8 = y[i].qs; + + int32_t block_sum = 0; + + for (int g = 0; g < IQ1BN_N_GROUPS; ++g) { + int ci = (g & 1) + ? ((x[i].qs[3*(g/2)+1] >> 4) | ((int)x[i].qs[3*(g/2)+2] << 4)) + : (x[i].qs[3*(g/2)] | (((int)x[i].qs[3*(g/2)+1] & 0x0F) << 8)); + const int8_t * cb = codebook + ci * IQ1BN_CODEBOOK_DIM; + const int8_t * q8g = q8 + g * IQ1BN_GROUP_SIZE; + + for (int k = 0; k < IQ1BN_CODEBOOK_DIM; ++k) { + block_sum += (int32_t)cb[k] * (int32_t)q8g[k]; + } + } + + sumf += d * yd * (float)block_sum; + } + + *s = sumf; +} + +void ggml_vec_dot_q4_dpt_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + assert(n % QK4_NL == 0); + static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same"); + + const block_q4_dpt * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + const int nb = n / QK4_NL; + + const int8_t * values = (const int8_t *)levels; + GGML_ASSERT(values != NULL && "Q4_DPT levels not set for tensor"); + + float sumf = 0; + for (int ib = 0; ib < nb; ++ib) { + const float d = GGML_CPU_FP16_TO_FP32(y[ib].d) * GGML_CPU_FP16_TO_FP32(x[ib].d); + int32_t blk = 0; + for (int j = 0; j < QK4_NL/2; ++j) { + blk += (int32_t)y[ib].qs[j+ 0] * (int32_t)values[x[ib].qs[j] & 0xf]; + blk += (int32_t)y[ib].qs[j+QK4_NL/2] * (int32_t)values[x[ib].qs[j] >> 4]; + } + sumf += d * (float)blk; + } + *s = sumf; +} + +void ggml_vec_dot_q2_dpt_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + assert(n % QK2_DPT == 0); + static_assert(QK2_DPT == QK8_0, "QK2_DPT and QK8_0 must be the same"); + + const block_q2_dpt * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + const int nb = n / QK2_DPT; + + const int8_t * values = (const int8_t *)levels; + GGML_ASSERT(values != NULL && "Q2_DPT levels not set for tensor"); + + float sumf = 0; + for (int ib = 0; ib < nb; ++ib) { + const float d = GGML_CPU_FP16_TO_FP32(y[ib].d) * GGML_CPU_FP16_TO_FP32(x[ib].d); + int32_t blk = 0; + for (int j = 0; j < QK2_DPT/4; ++j) { + uint8_t q = x[ib].qs[j]; + blk += (int32_t)y[ib].qs[j*4 + 0] * (int32_t)values[(q >> 0) & 3]; + blk += (int32_t)y[ib].qs[j*4 + 1] * (int32_t)values[(q >> 2) & 3]; + blk += (int32_t)y[ib].qs[j*4 + 2] * (int32_t)values[(q >> 4) & 3]; + blk += (int32_t)y[ib].qs[j*4 + 3] * (int32_t)values[(q >> 6) & 3]; + } + sumf += d * (float)blk; + } + *s = sumf; +} + +void ggml_vec_dot_iq1_m_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); @@ -1200,7 +1645,8 @@ void ggml_vec_dot_iq1_m_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = sumf; } -void ggml_vec_dot_iq4_nl_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq4_nl_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); @@ -1229,7 +1675,8 @@ void ggml_vec_dot_iq4_nl_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = sumf; } -void ggml_vec_dot_iq4_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq4_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h index d4bc87a1c052..731e19d757f3 100644 --- a/ggml/src/ggml-cpu/quants.h +++ b/ggml/src/ggml-cpu/quants.h @@ -37,66 +37,79 @@ void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); // Dot product -void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); - -void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); - -void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); - -void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); - -void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq1_m_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); + +void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); + +void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); + +void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); + +void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq1_m_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q3_pt_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q3_kpt_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q3_kpt_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q4_dpt_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q4_dpt_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q2_dpt_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q2_dpt_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q2_kpt_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q2_kpt_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq2_tq_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq3_tq_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq1_bn_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); // Generic implementation void quantize_row_q8_0_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void quantize_row_q8_1_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k); void quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); -void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q5_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q8_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); - -void ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); - -void ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); - -void ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q3_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q4_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q5_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_q6_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq2_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq2_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq2_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq3_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq1_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq1_m_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq4_nl_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); -void ggml_vec_dot_iq4_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_q1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q4_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q4_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q5_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q5_1_q8_1_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q8_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); + +void ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); + +void ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); + +void ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q3_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q4_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q5_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q6_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq2_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq2_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq2_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq3_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq1_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq1_m_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq4_nl_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_iq4_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); +void ggml_vec_dot_q3_pt_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels); #ifdef __cplusplus } diff --git a/ggml/src/ggml-cpu/vec.cpp b/ggml/src/ggml-cpu/vec.cpp index ff2b636df86c..9aec0684d761 100644 --- a/ggml/src/ggml-cpu/vec.cpp +++ b/ggml/src/ggml-cpu/vec.cpp @@ -8,7 +8,8 @@ ggml_fp16_t ggml_table_gelu_f16[1 << 16]; // precomputed quick gelu table for f16 (128 KB) ggml_fp16_t ggml_table_gelu_quick_f16[1 << 16]; -void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc) { +void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); GGML_UNUSED(nrc); GGML_UNUSED(bx); @@ -136,7 +137,8 @@ void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * G *s = sumf; } -void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc) { +void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); GGML_UNUSED(nrc); GGML_UNUSED(bx); @@ -261,7 +263,8 @@ void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * *s = sumf; } -void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc) { +void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); assert(nrc == 1); GGML_UNUSED(nrc); GGML_UNUSED(bx); diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h index 5de9cb5b7e09..6d35277b4274 100644 --- a/ggml/src/ggml-cpu/vec.h +++ b/ggml/src/ggml-cpu/vec.h @@ -68,9 +68,9 @@ extern ggml_fp16_t ggml_table_gelu_quick_f16[1 << 16]; // fundamental operations // -void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc); -void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc); -void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc); +void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * GGML_RESTRICT x, size_t bx, const float * GGML_RESTRICT y, size_t by, int nrc, const void * levels); +void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * GGML_RESTRICT x, size_t bx, ggml_bf16_t * GGML_RESTRICT y, size_t by, int nrc, const void * levels); +void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * GGML_RESTRICT x, size_t bx, ggml_fp16_t * GGML_RESTRICT y, size_t by, int nrc, const void * levels); void ggml_vec_silu_f32(const int n, float * y, const float * x); ggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const float mean); //it will also center y ( y = y - mean ) @@ -855,7 +855,7 @@ inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float } } -inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1); *s = sqrtf(*s); } +inline static void ggml_vec_norm_f32 (const int n, float * s, const float * x) { ggml_vec_dot_f32(n, s, 0, x, 0, x, 0, 1, NULL); *s = sqrtf(*s); } inline static void ggml_vec_sqr_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]*x[i]; } inline static void ggml_vec_sqr_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) { for (int i = 0; i < n; ++i) { diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index e6e50e041195..7e7cfc328f8b 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1098,6 +1098,27 @@ struct ggml_cuda_type_traits { static constexpr int qi = QI4_NL; }; +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK4_NL; + static constexpr int qr = QR4_NL; + static constexpr int qi = QI4_NL; +}; + +// Per-tensor lookup table for Q4_DPT (device global memory). +// Each TU gets its own copy; initialized via cudaGetSymbolAddress + cudaMemcpyAsync before use. +__device__ int8_t q4dpt_levels_cuda[16]; + +// Per-tensor lookup table for Q2_DPT (4 int8 levels). +__device__ int8_t q2dpt_levels_cuda[4]; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK2_DPT; + static constexpr int qr = 4; // 4 elements per "quantum" (2-bit) + static constexpr int qi = 1; // 1 uint32 per block +}; + template<> struct ggml_cuda_type_traits { static constexpr int qk = QK_K; @@ -1105,6 +1126,38 @@ struct ggml_cuda_type_traits { static constexpr int qi = QI4_XS; }; +// Per-tensor grid for IQ2_TQ (16 × 4 int8 = 64 bytes). +__device__ int8_t iq2tq_grid_cuda[64]; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = 4; + static constexpr int qi = QK_K / (4*4); // 16 +}; + +// Per-tensor grid for IQ3_TQ (16 × 8 int8 = 128 bytes). +__device__ int8_t iq3tq_grid_cuda[128]; + + +// Per-tensor codebook for IQ1_BN (4096 × 8 int8 = 32768 bytes). +__device__ int8_t iq1bn_codebook_cuda[IQ1BN_CODEBOOK_SIZE]; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = 4; + static constexpr int qi = QK_K / (4*4); // 16 +}; + + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = 4; + static constexpr int qi = QK_K / (4*4); // 16 +}; + template<> struct ggml_cuda_type_traits { static constexpr int qk = QK_K; diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index 61630a35a29b..d30b51c2830c 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -593,12 +593,187 @@ static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int64_t dequantize_block_iq1_s<<>>(vx, y); } +void ggml_cuda_set_q4dpt_levels(const int8_t * levels, cudaStream_t stream) { + int8_t * d_q4dpt_levels; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_q4dpt_levels, q4dpt_levels_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_q4dpt_levels, levels, 16, cudaMemcpyDeviceToDevice, stream)); +} + +void ggml_cuda_set_q2dpt_levels(const int8_t * levels, cudaStream_t stream) { + int8_t * d_q2dpt_levels; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_q2dpt_levels, q2dpt_levels_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_q2dpt_levels, levels, 4, cudaMemcpyDeviceToDevice, stream)); +} + +void ggml_cuda_set_iq2tq_grid(const void * grid, cudaStream_t stream) { + int8_t * d_grid; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_grid, iq2tq_grid_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_grid, grid, 64, cudaMemcpyHostToDevice, stream)); +} + +void ggml_cuda_set_iq3tq_grid(const void * grid, cudaStream_t stream) { + int8_t * d_grid; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_grid, iq3tq_grid_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_grid, grid, 128, cudaMemcpyHostToDevice, stream)); +} + + +void ggml_cuda_set_iq1bn_aux(const void * aux, cudaStream_t stream) { + int8_t * d_cb; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_cb, iq1bn_codebook_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_cb, aux, IQ1BN_CODEBOOK_SIZE, cudaMemcpyHostToDevice, stream)); +} + template static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { const int nb = (k + QK_K - 1) / QK_K; dequantize_block_iq4_nl<<>>(vx, y); } +template +static __global__ void dequantize_block_q4_dpt(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; + const block_q4_dpt * x = (const block_q4_dpt *) vx + i*(QK_K/QK4_NL); + + const int64_t tid = threadIdx.x; + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 4*il; + const uint8_t * q4 = x[ib].qs + 4*il; + const float d = (float)x[ib].d; + for (int j = 0; j < 4; ++j) { + y[j+ 0] = d * q4dpt_levels_cuda[q4[j] & 0xf]; + y[j+16] = d * q4dpt_levels_cuda[q4[j] >> 4]; + } +} + +template +static void dequantize_row_q4_dpt_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nb = (k + QK_K - 1) / QK_K; + dequantize_block_q4_dpt<<>>(vx, y); +} + +template +static __global__ void dequantize_block_q2_dpt(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; + const block_q2_dpt * x = (const block_q2_dpt *) vx + i*(QK_K/QK2_DPT); + + const int64_t tid = threadIdx.x; + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 4*il; + const uint8_t * q2 = x[ib].qs + il; + const float d = (float)x[ib].d; + uint8_t q = q2[0]; + y[ 0] = d * q2dpt_levels_cuda[(q >> 0) & 3]; + y[ 1] = d * q2dpt_levels_cuda[(q >> 2) & 3]; + y[ 2] = d * q2dpt_levels_cuda[(q >> 4) & 3]; + y[ 3] = d * q2dpt_levels_cuda[(q >> 6) & 3]; +} + +template +static void dequantize_row_q2_dpt_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nb = (k + QK_K - 1) / QK_K; + dequantize_block_q2_dpt<<>>(vx, y); +} + +template +static __global__ void dequantize_block_iq2_tq(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; + const block_iq2_tq * bq = (const block_iq2_tq *) vx + i; + const int g = threadIdx.x; // group index 0..31 + + const float dq = __half2float(bq->d) * IQ2TQ_GRID_SCALE; + + const int si = (bq->scales[g / 2] >> (4 * (g & 1))) & 0xF; + const int8_t * ge = iq2tq_grid_cuda + si * 4; + + dst_t * y = yy + i * QK_K + g * 8; + const uint8_t * qs = bq->qs + g * 2; + + y[0] = dq * ge[(qs[0] >> 0) & 3]; + y[1] = dq * ge[(qs[0] >> 2) & 3]; + y[2] = dq * ge[(qs[0] >> 4) & 3]; + y[3] = dq * ge[(qs[0] >> 6) & 3]; + y[4] = dq * ge[(qs[1] >> 0) & 3]; + y[5] = dq * ge[(qs[1] >> 2) & 3]; + y[6] = dq * ge[(qs[1] >> 4) & 3]; + y[7] = dq * ge[(qs[1] >> 6) & 3]; +} + +template +static void dequantize_row_iq2_tq_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq2_tq<<>>(vx, y); +} + +template +static __global__ void dequantize_block_iq3_tq(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; + const block_iq3_tq * bq = (const block_iq3_tq *) vx + i; + const int g = threadIdx.x; // group index 0..31 + + const float dq = __half2float(bq->d) * IQ3TQ_GRID_SCALE; + + const int si = (bq->scales[g / 2] >> (4 * (g & 1))) & 0xF; + const int8_t * ge = iq3tq_grid_cuda + si * 8; + + dst_t * y = yy + i * QK_K + g * 8; + const uint8_t * qs = bq->qs + g * 3; + const uint32_t bits = qs[0] | ((uint32_t)qs[1] << 8) | ((uint32_t)qs[2] << 16); + + y[0] = dq * ge[(bits >> 0) & 7]; + y[1] = dq * ge[(bits >> 3) & 7]; + y[2] = dq * ge[(bits >> 6) & 7]; + y[3] = dq * ge[(bits >> 9) & 7]; + y[4] = dq * ge[(bits >> 12) & 7]; + y[5] = dq * ge[(bits >> 15) & 7]; + y[6] = dq * ge[(bits >> 18) & 7]; + y[7] = dq * ge[(bits >> 21) & 7]; +} + +template +static void dequantize_row_iq3_tq_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq3_tq<<>>(vx, y); +} + + +template +static __global__ void dequantize_block_iq1_bn(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; + const block_iq1_bn * bq = (const block_iq1_bn *) vx + i; + const int g = threadIdx.x; // group index 0..31 + + const float dq = __half2float(bq->d) * IQ1BN_GRID_SCALE; + + // Extract 12-bit codebook index + const int pair = g / 2; + int ci; + if (g & 1) { + ci = (bq->qs[3*pair+1] >> 4) | ((int)bq->qs[3*pair+2] << 4); + } else { + ci = bq->qs[3*pair] | (((int)bq->qs[3*pair+1] & 0x0F) << 8); + } + const int8_t * cb = iq1bn_codebook_cuda + ci * IQ1BN_CODEBOOK_DIM; + + dst_t * y = yy + i * QK_K + g * IQ1BN_GROUP_SIZE; + y[0] = dq * cb[0]; + y[1] = dq * cb[1]; + y[2] = dq * cb[2]; + y[3] = dq * cb[3]; + y[4] = dq * cb[4]; + y[5] = dq * cb[5]; + y[6] = dq * cb[6]; + y[7] = dq * cb[7]; +} + +template +static void dequantize_row_iq1_bn_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq1_bn<<>>(vx, y); +} + template static void dequantize_row_iq1_m_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { const int nb = k / QK_K; @@ -750,6 +925,16 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { return dequantize_row_iq1_m_cuda; case GGML_TYPE_IQ4_NL: return dequantize_row_iq4_nl_cuda; + case GGML_TYPE_Q4_DPT: + return dequantize_row_q4_dpt_cuda; + case GGML_TYPE_Q2_DPT: + return dequantize_row_q2_dpt_cuda; + case GGML_TYPE_IQ2_TQ: + return dequantize_row_iq2_tq_cuda; + case GGML_TYPE_IQ3_TQ: + return dequantize_row_iq3_tq_cuda; + case GGML_TYPE_IQ1_BN: + return dequantize_row_iq1_bn_cuda; case GGML_TYPE_IQ4_XS: return dequantize_row_iq4_xs_cuda; case GGML_TYPE_IQ3_S: @@ -805,6 +990,16 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { return dequantize_row_iq1_m_cuda; case GGML_TYPE_IQ4_NL: return dequantize_row_iq4_nl_cuda; + case GGML_TYPE_Q4_DPT: + return dequantize_row_q4_dpt_cuda; + case GGML_TYPE_Q2_DPT: + return dequantize_row_q2_dpt_cuda; + case GGML_TYPE_IQ2_TQ: + return dequantize_row_iq2_tq_cuda; + case GGML_TYPE_IQ3_TQ: + return dequantize_row_iq3_tq_cuda; + case GGML_TYPE_IQ1_BN: + return dequantize_row_iq1_bn_cuda; case GGML_TYPE_IQ4_XS: return dequantize_row_iq4_xs_cuda; case GGML_TYPE_IQ3_S: diff --git a/ggml/src/ggml-cuda/convert.cuh b/ggml/src/ggml-cuda/convert.cuh index f5d37c7b9987..3b96662dd35f 100644 --- a/ggml/src/ggml-cuda/convert.cuh +++ b/ggml/src/ggml-cuda/convert.cuh @@ -31,6 +31,22 @@ to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type); to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type); to_bf16_nc_cuda_t ggml_get_to_bf16_nc_cuda(ggml_type type); +// Set the Q4_DPT lookup table in device constant memory. +void ggml_cuda_set_q4dpt_levels(const int8_t * levels, cudaStream_t stream); + +// Set the Q2_DPT lookup table in device constant memory. +void ggml_cuda_set_q2dpt_levels(const int8_t * levels, cudaStream_t stream); + +// Set the IQ2_TQ per-tensor grid (64 bytes: 16 entries × 4 int8 levels). +void ggml_cuda_set_iq2tq_grid(const void * grid, cudaStream_t stream); + +// Set the IQ3_TQ per-tensor grid (128 bytes: 16 entries × 8 int8 levels). +void ggml_cuda_set_iq3tq_grid(const void * grid, cudaStream_t stream); + + +// Set the IQ1_BN per-tensor codebook+scale (2064 bytes). +void ggml_cuda_set_iq1bn_aux(const void * aux, cudaStream_t stream); + template __host__ __device__ inline dst_t ggml_cuda_cast(src_t x) { if constexpr (std::is_same_v) { diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index cda31bbfbb23..8b75e95f3480 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4,6 +4,7 @@ #include "ggml-cuda/allreduce.cuh" #include "ggml-cuda/common.cuh" +#include "ggml-quants.h" #include "ggml-cuda/acc.cuh" #include "ggml-cuda/add-id.cuh" #include "ggml-cuda/arange.cuh" @@ -1690,6 +1691,24 @@ static void ggml_cuda_op_mul_mat_cublas( row_diff == src0->ne[1] && dst->op_params[0] == GGML_PREC_DEFAULT; + // Upload per-tensor grids/levels before any dequantize path (fp16, fp32, or bf16) + if (src0->type == GGML_TYPE_Q4_DPT) { + GGML_ASSERT(src0->quant_levels && "Q4_DPT MUL_MAT requires levels (set tensor->quant_levels)"); + ggml_cuda_set_q4dpt_levels((const int8_t *)src0->quant_levels, stream); + } + if (src0->type == GGML_TYPE_IQ2_TQ) { + GGML_ASSERT(src0->quant_levels && "IQ2_TQ MUL_MAT requires grid (set tensor->quant_levels)"); + ggml_cuda_set_iq2tq_grid(src0->quant_levels, stream); + } + if (src0->type == GGML_TYPE_IQ3_TQ) { + GGML_ASSERT(src0->quant_levels && "IQ3_TQ MUL_MAT requires grid (set tensor->quant_levels)"); + ggml_cuda_set_iq3tq_grid(src0->quant_levels, stream); + } + if (src0->type == GGML_TYPE_IQ1_BN) { + GGML_ASSERT(src0->quant_levels && "IQ1_BN MUL_MAT requires codebook (set tensor->quant_levels)"); + ggml_cuda_set_iq1bn_aux(src0->quant_levels, stream); + } + if (supports_bf16 && src0->type == GGML_TYPE_BF16 && ggml_is_contiguous(src0) && row_diff == src0->ne[1]) { ggml_cuda_pool_alloc src1_as_bf16(ctx.pool(id)); if (src1->type != GGML_TYPE_BF16) { @@ -5291,6 +5310,10 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_Q4_DPT: + case GGML_TYPE_IQ2_TQ: + case GGML_TYPE_IQ3_TQ: + case GGML_TYPE_IQ1_BN: case GGML_TYPE_IQ4_XS: case GGML_TYPE_BF16: return true; @@ -5326,7 +5349,8 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g { return (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 || op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 || - op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) && + op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL || + op->type == GGML_TYPE_Q4_DPT) && op->src[0]->type == GGML_TYPE_F32 && (op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32); } break; @@ -5379,6 +5403,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_IQ4_NL) { return true; } + if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_Q4_DPT) { + return true; + } if (src0_type == GGML_TYPE_F32 && src1_type == GGML_TYPE_I32) { return true; } diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 6b3b0d064a55..347f08d19166 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -2,6 +2,8 @@ #include "mmq.cuh" #include "quantize.cuh" #include "mmid.cuh" +#include "convert.cuh" +#include "ggml-quants.h" static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, const mmq_args & args, cudaStream_t stream) { switch (args.type_x) { @@ -68,6 +70,12 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con case GGML_TYPE_IQ4_NL: mul_mat_q_case(ctx, args, stream); break; + case GGML_TYPE_Q4_DPT: + mul_mat_q_case(ctx, args, stream); + break; + case GGML_TYPE_Q2_DPT: + mul_mat_q_case(ctx, args, stream); + break; default: GGML_ABORT("fatal error"); break; @@ -85,6 +93,22 @@ void ggml_cuda_mul_mat_q( cudaStream_t stream = ctx.stream(); const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + // Set Q4_DPT lookup table from tensor's quant_levels + if (src0->type == GGML_TYPE_Q4_DPT) { + GGML_ASSERT(src0->quant_levels && "Q4_DPT MUL_MAT requires levels (set tensor->quant_levels)"); + int8_t * d_q4dpt_levels; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_q4dpt_levels, q4dpt_levels_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_q4dpt_levels, src0->quant_levels, Q4DPT_N_LEVELS * sizeof(int8_t), cudaMemcpyHostToDevice, stream)); + } + + // Set Q2_DPT lookup table from tensor's quant_levels + if (src0->type == GGML_TYPE_Q2_DPT) { + GGML_ASSERT(src0->quant_levels && "Q2_DPT MUL_MAT requires levels (set tensor->quant_levels)"); + int8_t * d_q2dpt_levels; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_q2dpt_levels, q2dpt_levels_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_q2dpt_levels, src0->quant_levels, Q2DPT_N_LEVELS * sizeof(int8_t), cudaMemcpyHostToDevice, stream)); + } + const size_t ts_src0 = ggml_type_size(src0->type); const size_t ts_src1 = ggml_type_size(src1->type); const size_t ts_dst = ggml_type_size(dst->type); @@ -293,6 +317,8 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_Q4_DPT: + case GGML_TYPE_Q2_DPT: mmq_supported = true; break; default: @@ -377,3 +403,9 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t return (!GGML_CUDA_CC_IS_CDNA(cc)) || ne11 < MMQ_DP4A_MAX_BATCH_SIZE; } + +// Q4_DPT must be instantiated in this TU (not a separate template-instance file) +// because it accesses the TU-local __device__ variable q4dpt_levels_cuda, +// which is initialized by the code above. +DECL_MMQ_CASE(GGML_TYPE_Q4_DPT); +DECL_MMQ_CASE(GGML_TYPE_Q2_DPT); diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index edf546d8f1e2..cc22d61cb364 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -1,6 +1,7 @@ #pragma once #include "common.cuh" +#include "ggml.h" #include "vecdotq.cuh" #include "mma.cuh" @@ -93,6 +94,8 @@ static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) { return MMQ_Q8_1_DS_LAYOUT_DS4; case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_Q4_DPT: + case GGML_TYPE_Q2_DPT: return MMQ_Q8_1_DS_LAYOUT_D4; default: GGML_ABORT("fatal error"); @@ -212,6 +215,8 @@ static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0; case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0; case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_Q4_DPT: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_Q2_DPT: return MMQ_DP4A_TXS_Q8_0_16; default: return tile_x_sizes{0, 0, 0}; } } @@ -262,6 +267,8 @@ static constexpr __host__ __device__ int mmq_get_mma_tile_x_k(ggml_type type) { case GGML_TYPE_IQ1_S: return MMQ_MMA_TILE_X_K_Q8_0; case GGML_TYPE_IQ4_XS: return MMQ_MMA_TILE_X_K_Q8_0; case GGML_TYPE_IQ4_NL: return MMQ_MMA_TILE_X_K_Q8_0; + case GGML_TYPE_Q4_DPT: return MMQ_MMA_TILE_X_K_Q8_0; + case GGML_TYPE_Q2_DPT: return MMQ_MMA_TILE_X_K_Q8_0; default: return 0; } } @@ -2738,6 +2745,71 @@ template static __device__ __forceinline__ void loa } } +template static __device__ __forceinline__ void load_tiles_q4_dpt( + const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + constexpr int nwarps = mmq_get_nwarps_device(); + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_Q4_DPT, mmq_y); + int * x_qs = (int *) x_tile; + float * x_df = (float *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR4_NL); + constexpr int nrows = warp_size / threads_per_row; + const int txi = warp_size > threads_per_row ? threadIdx.x % threads_per_row : threadIdx.x; + const int kbx = txi / QI4_NL; + const int kqsx = txi % QI4_NL; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nrows*nwarps) { + int i = i0 + (nrows == 1 ? threadIdx.y : threadIdx.y*nrows + threadIdx.x/threads_per_row); + + if (need_check) { + i = min(i, i_max); + } + + const block_q4_dpt * bxi = (const block_q4_dpt *) x + kbx0 + i*stride + kbx; + + const int aux_q4 = get_int_b2(bxi->qs, kqsx); + const int2 v = get_int_from_table_16(aux_q4, q4dpt_levels_cuda); + const int k0 = kbx * (2 * QI4_NL) + kqsx; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + 0] = v.x; + x_qs[i*MMQ_MMA_TILE_X_K_Q8_0 + k0 + QI4_NL] = v.y; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + 0] = v.x; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + k0 + QI4_NL] = v.y; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + constexpr int blocks_per_tile_x_row = MMQ_TILE_NE_K / QI4_NL; + constexpr int rows_per_warp = warp_size / blocks_per_tile_x_row; + const int kbxd = threadIdx.x % blocks_per_tile_x_row; + +#pragma unroll + for (int i0 = 0; i0 < mmq_y; i0 += nwarps * rows_per_warp) { + int i = i0 + threadIdx.y * rows_per_warp + threadIdx.x / blocks_per_tile_x_row; + + if (need_check) { + i = min(i, i_max); + } + + const block_q4_dpt * bxi = (const block_q4_dpt *) x + kbx0 + i*stride + kbxd; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_df[i*MMQ_MMA_TILE_X_K_Q8_0 + kbxd] = __half2float(bxi->d); +#else + x_df[i*(MMQ_TILE_NE_K/QI4_NL) + i/QI4_NL + kbxd] = __half2float(bxi->d); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + template static __device__ __forceinline__ void load_tiles_iq2_xxs( const char * __restrict__ x, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { constexpr int nwarps = mmq_get_nwarps_device(); @@ -3435,6 +3507,22 @@ struct mmq_type_traits { static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; }; +template +struct mmq_type_traits { + static constexpr int vdr = VDR_Q4_DPT_Q8_1_MMQ; + static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_dpt; + static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma; + static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; +}; + +template +struct mmq_type_traits { + static constexpr int vdr = VDR_Q2_DPT_Q8_1_MMQ; + static constexpr load_tiles_mmq_t load_tiles = load_tiles_q4_dpt; // Reuse Q4_DPT loader (same layout) + static constexpr vec_dot_mmq_t vec_dot_mma = vec_dot_q8_0_q8_1_mma; + static constexpr vec_dot_mmq_t vec_dot_dp4a = vec_dot_q8_0_q8_1_dp4a; +}; + template struct mmq_type_traits { static constexpr int vdr = VDR_IQ4_XS_Q8_1_MMQ; diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index fe44a58da918..2cc7d76a0e54 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -2,6 +2,8 @@ #include "quantize.cuh" #include "unary.cuh" #include "vecdotq.cuh" +#include "convert.cuh" +#include "ggml-quants.h" #include @@ -29,6 +31,11 @@ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) case GGML_TYPE_IQ1_S: return vec_dot_iq1_s_q8_1; case GGML_TYPE_IQ1_M: return vec_dot_iq1_m_q8_1; case GGML_TYPE_IQ4_NL: return vec_dot_iq4_nl_q8_1; + case GGML_TYPE_Q4_DPT: return vec_dot_q4_dpt_q8_1; + case GGML_TYPE_Q2_DPT: return vec_dot_q2_dpt_q8_1; + case GGML_TYPE_IQ2_TQ: return vec_dot_iq2_tq_q8_1; + case GGML_TYPE_IQ3_TQ: return vec_dot_iq3_tq_q8_1; + case GGML_TYPE_IQ1_BN: return vec_dot_iq1_bn_q8_1; case GGML_TYPE_IQ4_XS: return vec_dot_iq4_xs_q8_1; case GGML_TYPE_IQ3_S: return vec_dot_iq3_s_q8_1; default: return nullptr; @@ -56,6 +63,11 @@ static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) { case GGML_TYPE_IQ3_XXS: return VDR_IQ3_XXS_Q8_1_MMVQ; case GGML_TYPE_IQ3_S: return VDR_IQ3_S_Q8_1_MMVQ; case GGML_TYPE_IQ4_NL: return VDR_IQ4_NL_Q8_1_MMVQ; + case GGML_TYPE_Q4_DPT: return VDR_Q4_DPT_Q8_1_MMVQ; + case GGML_TYPE_Q2_DPT: return VDR_Q2_DPT_Q8_1_MMVQ; + case GGML_TYPE_IQ2_TQ: return VDR_IQ2_TQ_Q8_1_MMVQ; + case GGML_TYPE_IQ3_TQ: return VDR_IQ3_TQ_Q8_1_MMVQ; + case GGML_TYPE_IQ1_BN: return VDR_IQ1_BN_Q8_1_MMVQ; case GGML_TYPE_IQ4_XS: return VDR_IQ4_XS_Q8_1_MMVQ; default: return 1; } @@ -1102,6 +1114,30 @@ static void mul_mat_vec_q_switch_type( nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); break; + case GGML_TYPE_Q4_DPT: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; + case GGML_TYPE_IQ2_TQ: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; + case GGML_TYPE_IQ3_TQ: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; + case GGML_TYPE_IQ1_BN: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; case GGML_TYPE_IQ4_XS: mul_mat_vec_q_switch_ncols_dst (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, @@ -1131,6 +1167,45 @@ void ggml_cuda_mul_mat_vec_q( cudaStream_t stream = ctx.stream(); + // Set Q4_DPT lookup table from tensor's quant_levels + if (src0->type == GGML_TYPE_Q4_DPT) { + GGML_ASSERT(src0->quant_levels && "Q4_DPT MUL_MAT requires levels (set tensor->quant_levels)"); + int8_t * d_q4dpt_levels; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_q4dpt_levels, q4dpt_levels_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_q4dpt_levels, src0->quant_levels, Q4DPT_N_LEVELS * sizeof(int8_t), cudaMemcpyHostToDevice, stream)); + } + + // Set Q2_DPT lookup table from tensor's quant_levels + if (src0->type == GGML_TYPE_Q2_DPT) { + GGML_ASSERT(src0->quant_levels && "Q2_DPT MUL_MAT requires levels (set tensor->quant_levels)"); + int8_t * d_q2dpt_levels; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_q2dpt_levels, q2dpt_levels_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_q2dpt_levels, src0->quant_levels, Q2DPT_N_LEVELS * sizeof(int8_t), cudaMemcpyHostToDevice, stream)); + } + + // Set IQ2_TQ per-tensor grid + if (src0->type == GGML_TYPE_IQ2_TQ) { + GGML_ASSERT(src0->quant_levels && "IQ2_TQ MUL_MAT requires grid (set tensor->quant_levels)"); + int8_t * d_grid; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_grid, iq2tq_grid_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_grid, src0->quant_levels, 64, cudaMemcpyHostToDevice, stream)); + } + + // Set IQ3_TQ per-tensor grid + if (src0->type == GGML_TYPE_IQ3_TQ) { + GGML_ASSERT(src0->quant_levels && "IQ3_TQ MUL_MAT requires grid (set tensor->quant_levels)"); + int8_t * d_grid; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_grid, iq3tq_grid_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_grid, src0->quant_levels, 128, cudaMemcpyHostToDevice, stream)); + } + + + // Set IQ1_BN per-tensor codebook+scale + if (src0->type == GGML_TYPE_IQ1_BN) { + GGML_ASSERT(src0->quant_levels && "IQ1_BN MUL_MAT requires codebook (set tensor->quant_levels)"); + ggml_cuda_set_iq1bn_aux(src0->quant_levels, stream); + } + const size_t ts_src0 = ggml_type_size(src0->type); const size_t ts_src1 = ggml_type_size(src1->type); const size_t ts_dst = ggml_type_size(dst->type); diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuh index d1741cc8d7ba..fddc969dc1d6 100644 --- a/ggml/src/ggml-cuda/vecdotq.cuh +++ b/ggml/src/ggml-cuda/vecdotq.cuh @@ -1288,6 +1288,194 @@ static __device__ __forceinline__ float vec_dot_iq4_nl_q8_1( return d * sumi; } +#define VDR_Q4_DPT_Q8_1_MMVQ 2 +#define VDR_Q4_DPT_Q8_1_MMQ 4 + +static __device__ __forceinline__ float vec_dot_q4_dpt_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q4_dpt * bq4 = (const block_q4_dpt *) vbq + kbx; + + const int * q8 = (const int *) bq8_1->qs + iqs; + + int sumi = 0; +#pragma unroll + for (int l = 0; l < VDR_Q4_DPT_Q8_1_MMVQ; ++l) { + const int aux_q4 = get_int_b2(bq4->qs, iqs + l); + const int2 v = get_int_from_table_16(aux_q4, q4dpt_levels_cuda); + + sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi); + sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi); + } + + const float d = __half2float(bq4->d) * __low2float(bq8_1->ds); + return d * sumi; +} + +// Q2_DPT: 2-bit quantization with 4 learned levels +// Helper: lookup 4 int8 levels using 2-bit indices packed in a 32-bit int +static __device__ __forceinline__ int4 get_int_from_table_4(const int & q2, const int8_t * table) { + int4 result; + result.x = table[(q2 >> 0) & 3]; + result.y = table[(q2 >> 8) & 3]; + result.z = table[(q2 >> 16) & 3]; + result.w = table[(q2 >> 24) & 3]; + return result; +} + +#define VDR_Q2_DPT_Q8_1_MMVQ 4 +#define VDR_Q2_DPT_Q8_1_MMQ 8 + +static __device__ __forceinline__ float vec_dot_q2_dpt_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q2_dpt * bq2 = (const block_q2_dpt *) vbq + kbx; + + const int * q8 = (const int *) bq8_1->qs + iqs; + + int sumi = 0; +#pragma unroll + for (int l = 0; l < VDR_Q2_DPT_Q8_1_MMVQ; ++l) { + const int aux_q2 = get_int_b4(bq2->qs, l); + const int4 v = get_int_from_table_4(aux_q2, q2dpt_levels_cuda); + + sumi = ggml_cuda_dp4a(v.x, q8[l + 0], sumi); + sumi = ggml_cuda_dp4a(v.y, q8[l + 4], sumi); + sumi = ggml_cuda_dp4a(v.z, q8[l + 8], sumi); + sumi = ggml_cuda_dp4a(v.w, q8[l + 12], sumi); + } + + const float d = __half2float(bq2->d) * __low2float(bq8_1->ds); + return d * sumi; +} + +// IQ2_TQ: 2-bit with per-tensor trained 16×4 grid table +// Grid lookup helper: 4 × 2-bit indices packed in a byte → 4 grid values packed as int32 +static __device__ __forceinline__ int iq2tq_grid_lookup4(uint8_t qbyte, const int8_t * grid_entry) { + uint32_t r = (uint32_t)(uint8_t)grid_entry[(qbyte >> 0) & 3]; + r |= (uint32_t)(uint8_t)grid_entry[(qbyte >> 2) & 3] << 8; + r |= (uint32_t)(uint8_t)grid_entry[(qbyte >> 4) & 3] << 16; + r |= (uint32_t)(uint8_t)grid_entry[(qbyte >> 6) & 3] << 24; + return (int)r; +} + +#define VDR_IQ2_TQ_Q8_1_MMVQ 1 +#define VDR_IQ2_TQ_Q8_1_MMQ 1 + +static __device__ __forceinline__ float vec_dot_iq2_tq_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_iq2_tq * bq = (const block_iq2_tq *) vbq + kbx; + + // iqs selects which 16-element portion (0..15): 2 groups of 8 elements + const int q8b = iqs / 2; // Q8_1 block index (0..7) + const int q8off = (iqs & 1) * 4; // int32 offset within Q8_1 block (0 or 4) + + // Grid indices for groups iqs*2 and iqs*2+1 + const uint8_t sc = bq->scales[iqs]; + const int8_t * ge0 = iq2tq_grid_cuda + (sc & 0xF) * 4; + const int8_t * ge1 = iq2tq_grid_cuda + (sc >> 4) * 4; + + const uint8_t * qs = bq->qs + iqs * 4; + const int * q8 = (const int *)bq8_1[q8b].qs + q8off; + + int sumi = 0; + + // Group 0: 8 elements = 2 bytes qs, 2 int32 Q8_1 + sumi = ggml_cuda_dp4a(iq2tq_grid_lookup4(qs[0], ge0), q8[0], sumi); + sumi = ggml_cuda_dp4a(iq2tq_grid_lookup4(qs[1], ge0), q8[1], sumi); + + // Group 1: next 8 elements + sumi = ggml_cuda_dp4a(iq2tq_grid_lookup4(qs[2], ge1), q8[2], sumi); + sumi = ggml_cuda_dp4a(iq2tq_grid_lookup4(qs[3], ge1), q8[3], sumi); + + return __half2float(bq->d) * IQ2TQ_GRID_SCALE * __low2float(bq8_1[q8b].ds) * sumi; +} + +// IQ3_TQ: 3-bit with per-tensor trained 16×8 grid table +#define VDR_IQ3_TQ_Q8_1_MMVQ 1 +#define VDR_IQ3_TQ_Q8_1_MMQ 1 + +static __device__ __forceinline__ float vec_dot_iq3_tq_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_iq3_tq * bq = (const block_iq3_tq *) vbq + kbx; + + const int q8b = iqs / 2; + const int q8off = (iqs & 1) * 4; + + const uint8_t sc = bq->scales[iqs]; + const int8_t * ge0 = iq3tq_grid_cuda + (sc & 0xF) * 8; + const int8_t * ge1 = iq3tq_grid_cuda + (sc >> 4) * 8; + + const int * q8 = (const int *)bq8_1[q8b].qs + q8off; + + int sumi = 0; + + // Group 0: 8 elements, 3 bytes of qs + { + const uint8_t * qs = bq->qs + (iqs * 2) * 3; + const uint32_t bits = qs[0] | ((uint32_t)qs[1] << 8) | ((uint32_t)qs[2] << 16); + + int v0 = (uint8_t)ge0[(bits >> 0) & 7] | ((uint32_t)(uint8_t)ge0[(bits >> 3) & 7] << 8) + | ((uint32_t)(uint8_t)ge0[(bits >> 6) & 7] << 16) | ((uint32_t)(uint8_t)ge0[(bits >> 9) & 7] << 24); + sumi = ggml_cuda_dp4a(v0, q8[0], sumi); + + int v1 = (uint8_t)ge0[(bits >> 12) & 7] | ((uint32_t)(uint8_t)ge0[(bits >> 15) & 7] << 8) + | ((uint32_t)(uint8_t)ge0[(bits >> 18) & 7] << 16) | ((uint32_t)(uint8_t)ge0[(bits >> 21) & 7] << 24); + sumi = ggml_cuda_dp4a(v1, q8[1], sumi); + } + + // Group 1: next 8 elements, next 3 bytes of qs + { + const uint8_t * qs = bq->qs + (iqs * 2 + 1) * 3; + const uint32_t bits = qs[0] | ((uint32_t)qs[1] << 8) | ((uint32_t)qs[2] << 16); + + int v0 = (uint8_t)ge1[(bits >> 0) & 7] | ((uint32_t)(uint8_t)ge1[(bits >> 3) & 7] << 8) + | ((uint32_t)(uint8_t)ge1[(bits >> 6) & 7] << 16) | ((uint32_t)(uint8_t)ge1[(bits >> 9) & 7] << 24); + sumi = ggml_cuda_dp4a(v0, q8[2], sumi); + + int v1 = (uint8_t)ge1[(bits >> 12) & 7] | ((uint32_t)(uint8_t)ge1[(bits >> 15) & 7] << 8) + | ((uint32_t)(uint8_t)ge1[(bits >> 18) & 7] << 16) | ((uint32_t)(uint8_t)ge1[(bits >> 21) & 7] << 24); + sumi = ggml_cuda_dp4a(v1, q8[3], sumi); + } + + return __half2float(bq->d) * IQ3TQ_GRID_SCALE * __low2float(bq8_1[q8b].ds) * sumi; +} + + +// IQ1_BN: 8D vector quantized with per-tensor trained 4096-entry codebook +#define VDR_IQ1_BN_Q8_1_MMVQ 1 +#define VDR_IQ1_BN_Q8_1_MMQ 1 + +static __device__ __forceinline__ float vec_dot_iq1_bn_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_iq1_bn * bq = (const block_iq1_bn *) vbq + kbx; + + // iqs = 0..15, each thread handles 2 groups (16 elements) + const int q8b = iqs / 2; + const int q8off = (iqs & 1) * 4; + + // Extract two 12-bit codebook indices from qs[3*iqs .. 3*iqs+2] + const uint8_t * qs = bq->qs + 3 * iqs; + const int ci0 = qs[0] | (((int)qs[1] & 0x0F) << 8); + const int ci1 = (qs[1] >> 4) | ((int)qs[2] << 4); + + const int * cb0 = (const int *)(iq1bn_codebook_cuda + ci0 * IQ1BN_CODEBOOK_DIM); + const int * cb1 = (const int *)(iq1bn_codebook_cuda + ci1 * IQ1BN_CODEBOOK_DIM); + + const int * q8 = (const int *)bq8_1[q8b].qs + q8off; + + int sumi = 0; + sumi = ggml_cuda_dp4a(cb0[0], q8[0], sumi); + sumi = ggml_cuda_dp4a(cb0[1], q8[1], sumi); + sumi = ggml_cuda_dp4a(cb1[0], q8[2], sumi); + sumi = ggml_cuda_dp4a(cb1[1], q8[3], sumi); + + return __half2float(bq->d) * IQ1BN_GRID_SCALE * __low2float(bq8_1[q8b].ds) * (float)sumi; +} + #define VDR_IQ4_XS_Q8_1_MMVQ 4 #define VDR_IQ4_XS_Q8_1_MMQ 4 diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 15d231f70c0d..a730d29f240a 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -378,7 +378,8 @@ void quantize_row_nvfp4_ref(const float * GGML_RESTRICT x, block_nvfp4 * GGML_RE } } -void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + GGML_UNUSED(levels); static const int qk = QK1_0; assert(k % qk == 0); @@ -398,7 +399,7 @@ void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRI } } -void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { static const int qk = QK4_0; assert(k % qk == 0); @@ -418,7 +419,7 @@ void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRI } } -void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { static const int qk = QK4_1; assert(k % qk == 0); @@ -439,7 +440,7 @@ void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRI } } -void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { static const int qk = QK5_0; assert(k % qk == 0); @@ -465,7 +466,7 @@ void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRI } } -void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { static const int qk = QK5_1; assert(k % qk == 0); @@ -492,7 +493,7 @@ void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRI } } -void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { static const int qk = QK8_0; assert(k % qk == 0); @@ -508,7 +509,7 @@ void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRI } } -void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { static const int qk = QK_MXFP4; assert(k % qk == 0); @@ -528,7 +529,8 @@ void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_REST } } -void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + GGML_UNUSED(levels); static const int qk = QK_NVFP4; static const int qk_sub = QK_NVFP4_SUB; static const int n_sub = QK_NVFP4 / QK_NVFP4_SUB; @@ -828,6 +830,15 @@ static inline void get_scale_min_k4(int j, const uint8_t * GGML_RESTRICT q, uint } } +// Extract only the scale (not min) from Q4_K-style packed scales +static inline void get_scale_k4_only(int j, const uint8_t * GGML_RESTRICT q, uint8_t * GGML_RESTRICT d) { + if (j < 4) { + *d = q[j] & 63; + } else { + *d = (q[j+4] & 0xF) | ((q[j-4] >> 6) << 4); + } +} + //========================- 2-bit (de)-quantization void quantize_row_q2_K_ref(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k) { @@ -900,7 +911,7 @@ void quantize_row_q2_K_ref(const float * GGML_RESTRICT x, block_q2_K * GGML_REST } } -void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int nb = k / QK_K; @@ -1244,7 +1255,7 @@ void quantize_row_q3_K_ref(const float * GGML_RESTRICT x, block_q3_K * GGML_REST } } -void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int nb = k / QK_K; @@ -1468,7 +1479,7 @@ void quantize_row_q4_K_ref(const float * GGML_RESTRICT x, block_q4_K * GGML_REST } } -void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int nb = k / QK_K; @@ -1670,7 +1681,7 @@ void quantize_row_q5_K_ref(const float * GGML_RESTRICT x, block_q5_K * GGML_REST } } -void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -1878,7 +1889,7 @@ void quantize_row_q6_K_ref(const float * GGML_RESTRICT x, block_q6_K * GGML_REST } } -void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2353,7 +2364,7 @@ size_t quantize_tq2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, return nrow * row_size; } -void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2392,7 +2403,7 @@ void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_REST } } -void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2413,7 +2424,7 @@ void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_REST // ====================== "True" 2-bit (de)-quantization -void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2441,7 +2452,7 @@ void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_ // ====================== 2.3125 bpw (de)-quantization -void dequantize_row_iq2_xs(const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_iq2_xs(const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2468,7 +2479,7 @@ void dequantize_row_iq2_xs(const block_iq2_xs * GGML_RESTRICT x, float * GGML_RE // ====================== 2.5625 bpw (de)-quantization -void dequantize_row_iq2_s(const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_iq2_s(const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2500,7 +2511,7 @@ void dequantize_row_iq2_s(const block_iq2_s * GGML_RESTRICT x, float * GGML_REST // ====================== 3.0625 bpw (de)-quantization -void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2532,7 +2543,7 @@ void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_ // ====================== 3.3125 bpw (de)-quantization -void dequantize_row_iq3_s(const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_iq3_s(const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2575,7 +2586,7 @@ void dequantize_row_iq3_s(const block_iq3_s * GGML_RESTRICT x, float * GGML_REST // ====================== 1.5625 bpw (de)-quantization -void dequantize_row_iq1_s(const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_iq1_s(const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2600,7 +2611,7 @@ void dequantize_row_iq1_s(const block_iq1_s * GGML_RESTRICT x, float * GGML_REST } } -void dequantize_row_iq1_m(const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_iq1_m(const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2650,7 +2661,7 @@ void dequantize_row_iq1_m(const block_iq1_m * GGML_RESTRICT x, float * GGML_REST } } -void dequantize_row_iq4_nl(const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_iq4_nl(const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK4_NL == 0); const int64_t nb = k / QK4_NL; @@ -2668,7 +2679,7 @@ void dequantize_row_iq4_nl(const block_iq4_nl * GGML_RESTRICT x, float * GGML_RE } } -void dequantize_row_iq4_xs(const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_iq4_xs(const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2732,7 +2743,7 @@ void quantize_row_q8_K_ref(const float * GGML_RESTRICT x, block_q8_K * GGML_REST } } -void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { +void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -4305,194 +4316,4233 @@ void quantize_row_iq3_s_ref(const float * GGML_RESTRICT x, block_iq3_s * GGML_RE quantize_iq3_s(x, y, 1, k, NULL); } +// ====================== Q3_KPT: Q3_K with learned per-tensor levels ====================== +// +// Block format: Identical to block_q3_K (110 bytes per QK_K=256 elements) +// hmask[QK_K/8] : high bit for 3-bit indices +// qs[QK_K/4] : low 2 bits for 3-bit indices +// scales[12] : 6-bit quantized scales +// d : super-block scale +// +// The difference from Q3_K: instead of q ∈ {-4,-3,-2,-1,0,1,2,3}, +// we use learned levels L[0..7] and compute: x = d * sc * (L[k] - 4) +// where k is the 3-bit index. +// +// Per-tensor: 8 float32 "levels" in [0,1] from Lloyd-Max training. +// Stored in GGUF as "q3_kpt.levels" (float32 array). -// =================================== 1.5 bpw =================================================== +static float q3kpt_levels[Q3KPT_N_LEVELS]; +static bool q3kpt_levels_set = false; -static int iq1_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid, - const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float * scale, int8_t * GGML_RESTRICT L, int ngrid) { - int num_neighbors = neighbours[0]; - GGML_ASSERT(num_neighbors > 0); - float best_score = -FLT_MAX; - int grid_index = -1; - for (int j = 1; j <= num_neighbors; ++j) { - const int8_t * pg = (const int8_t *)(grid + neighbours[j]); - float sumqx = 0, sumq2 = 0; - for (int i = 0; i < 8; ++i) { - float q = (pg[i] - 3)/2; - float w = weight[i]; - sumqx += w*q*xval[i]; - sumq2 += w*q*q; - } - if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { - *scale = sumqx/sumq2; best_score = *scale * sumqx; - grid_index = neighbours[j]; - } - } - if (grid_index < 0) { - for (int i = 0; i < ngrid; ++i) { - const int8_t * grid_i = (const int8_t *)(grid + i); - float sumqx = 0, sumq2 = 0; - for (int j = 0; j < 8; ++j) { - float w = weight[j]; - float q = (grid_i[j] - 3)/2; - sumqx += w*q*xval[j]; - sumq2 += w*q*q; - } - if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { - *scale = sumqx/sumq2; best_score = *scale*sumqx; - grid_index = i; - } - } - } - if (grid_index < 0) { - printf("Oops, did not find grid point\n"); - printf("Have %d neighbours\n", num_neighbors); - for (int j = 1; j <= num_neighbors; ++j) { - const int8_t * pg = (const int8_t *)(grid + neighbours[j]); - float sumqx = 0, sumq2 = 0; - for (int i = 0; i < 8; ++i) { - float q = (pg[i] - 3)/2; - float w = weight[i]; - sumqx += w*q*xval[i]; - sumq2 += w*q*q; +GGML_API void q3kpt_set_levels(const float * levels) { + memcpy(q3kpt_levels, levels, Q3KPT_N_LEVELS * sizeof(float)); + q3kpt_levels_set = true; +} + +GGML_API const float * q3kpt_get_levels(void) { + return q3kpt_levels_set ? q3kpt_levels : NULL; +} + +GGML_API void q3kpt_free_levels(void) { + q3kpt_levels_set = false; +} + + +// Train levels in the symmetric quantization space +GGML_API void q3kpt_train_levels(const float * data, + int64_t nrow, + int64_t n_per_row, + const float * imatrix, + float levels_out[Q3KPT_N_LEVELS]) { + // Binning parameters + const int N_BINS = 8192; + const float bin_width = 1.0f / N_BINS; + float * bin_sum_w = (float *) calloc(N_BINS, sizeof(float)); + float * bin_sum_wt = (float *) calloc(N_BINS, sizeof(float)); + GGML_ASSERT(bin_sum_w && bin_sum_wt); + + const int nb = (int) (n_per_row / QK_K); + + // Single pass: use simple max_abs/4 scale estimation per sub-block, then bin + for (int64_t row = 0; row < nrow; ++row) { + const float * xrow = data + row * n_per_row; + + for (int i = 0; i < nb; i++) { + const float * x = xrow + i * QK_K; + + for (int j = 0; j < QK_K / 16; ++j) { + // Simple symmetric scale: max_abs / 4 + float amax = 0; + for (int l = 0; l < 16; ++l) { + float ax = fabsf(x[16 * j + l]); + if (ax > amax) { + amax = ax; + } + } + if (amax < 1e-10f) { + continue; + } + + float d = amax / 4.0f; + float inv_d = 1.0f / d; + + for (int l = 0; l < 16; ++l) { + float val = x[16 * j + l] * inv_d; + // Map from [-4, 3] symmetric space to [0, 1] + float t = (val + 4.0f) / 7.0f; + + if (t < 0.0f) { + t = 0.0f; + } + if (t > 1.0f) { + t = 1.0f; + } + + int bin_idx = (int) (t * N_BINS); + if (bin_idx >= N_BINS) { + bin_idx = N_BINS - 1; + } + + int elem = i * QK_K + 16 * j + l; + float w = imatrix ? imatrix[elem] : 1.0f; + if (w < 1e-10f) { + w = 1e-10f; + } + w *= d * d; + + bin_sum_w[bin_idx] += w; + bin_sum_wt[bin_idx] += w * t; + } } - printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2); } } - GGML_ASSERT(grid_index >= 0); - //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - *scale *= 1.05f; // This is a fudge factor. Don't ask me why it improves the result. - //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - const int8_t * pg = (const int8_t *)(grid + grid_index); - for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2; - return grid_index; -} -static int iq1_find_best_neighbour2(const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid, - const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, const float * GGML_RESTRICT xg, int8_t * GGML_RESTRICT L, int ngrid) { - int num_neighbors = neighbours[0]; - GGML_ASSERT(num_neighbors > 0); - float best_score = FLT_MAX; - int grid_index = -1; - for (int j = 1; j <= num_neighbors; ++j) { - const int8_t * pg = (const int8_t *)(grid + neighbours[j]); - float d2 = 0; - for (int i = 0; i < 8; ++i) { - float q = xg[(pg[i] - 1)/2]; - float w = weight[i]; - float diff = scale*q - xval[i]; - d2 += w*diff*diff; - } - if (d2 < best_score) { - best_score = d2; - grid_index = neighbours[j]; - } + // Initialize 8 levels uniformly in [0, 1] + float levels[Q3KPT_N_LEVELS]; + for (int k = 0; k < Q3KPT_N_LEVELS; ++k) { + levels[k] = (float) k / (Q3KPT_N_LEVELS - 1); } - if (grid_index < 0) { - for (int i = 0; i < ngrid; ++i) { - const int8_t * grid_i = (const int8_t *)(grid + i); - float d2 = 0; - for (int j = 0; j < 8; ++j) { - float w = weight[j]; - float q = xg[(grid_i[j] - 1)/2]; - float diff = scale*q - xval[i]; - d2 += w*diff*diff; + + // Lloyd-Max iterations on bins + for (int iter = 0; iter < 100; ++iter) { + float sum_w[Q3KPT_N_LEVELS] = { 0 }; + float sum_wt[Q3KPT_N_LEVELS] = { 0 }; + + for (int b = 0; b < N_BINS; ++b) { + if (bin_sum_w[b] < 1e-12f) { + continue; } - if (d2 < best_score) { - best_score = d2; - grid_index = i; + const float t = (b + 0.5f) * bin_width; + int best = 0; + float best_d2 = (t - levels[0]) * (t - levels[0]); + for (int k = 1; k < Q3KPT_N_LEVELS; ++k) { + float d2 = (t - levels[k]) * (t - levels[k]); + if (d2 < best_d2) { + best_d2 = d2; + best = k; + } } + sum_w[best] += bin_sum_w[b]; + sum_wt[best] += bin_sum_wt[b]; } - } - if (grid_index < 0) { - printf("Oops, did not find grid point\n"); - printf("Have %d neighbours\n", num_neighbors); - for (int j = 1; j <= num_neighbors; ++j) { - const int8_t * pg = (const int8_t *)(grid + neighbours[j]); - float sumqx = 0, sumq2 = 0; - for (int i = 0; i < 8; ++i) { - float q = xg[(pg[i] - 1)/2]; - float w = weight[i]; - sumqx += w*q*xval[i]; - sumq2 += w*q*q; + + float max_delta = 0.0f; + for (int k = 0; k < Q3KPT_N_LEVELS; ++k) { + if (sum_w[k] > 1e-12f) { + float new_level = sum_wt[k] / sum_w[k]; + max_delta = fmaxf(max_delta, fabsf(new_level - levels[k])); + levels[k] = new_level; } - printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2); + } + if (max_delta < 1e-10f) { + break; + } + + for (int k = 1; k < Q3KPT_N_LEVELS; ++k) { + float v = levels[k]; + int m = k - 1; + while (m >= 0 && levels[m] > v) { + levels[m + 1] = levels[m]; + m--; + } + levels[m + 1] = v; } } - GGML_ASSERT(grid_index >= 0); - const int8_t * pg = (const int8_t *)(grid + grid_index); - for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2; - return grid_index; -} -static int iq1_sort_helper(const void * left, const void * right) { - const float * l = left; - const float * r = right; - return *l < *r ? -1 : *l > *r ? 1 : 0; + memcpy(levels_out, levels, Q3KPT_N_LEVELS * sizeof(float)); + q3kpt_set_levels(levels); + free(bin_sum_w); + free(bin_sum_wt); } -#define IQ1S_BLOCK_SIZE 32 -#define IQ1M_BLOCK_SIZE 16 -static void quantize_row_iq1_s_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights, - float * scales, - float * weight, - float * sumx, - float * sumw, - float * pairs, - int8_t * L, - uint16_t * index, - int8_t * shifts) { +void dequantize_row_q3_kpt(const block_q3_kpt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + assert(k % QK_K == 0); + const int64_t nb = k / QK_K; + const float * lv = (const float *)levels; + GGML_ASSERT(lv != NULL && "Q3_KPT levels not set for tensor"); - const int gindex = iq2_data_index(GGML_TYPE_IQ1_S); + // levels are in [0,1], map to approximate [-4, 3] range for Q3_K compatibility + // The dequant formula: y = d * sc * (L[k] * 8 - 4) = d * sc * (L[k] - 0.5) * 8 + // But simpler: store shifted levels and use: y = d * sc * L_shifted[k] + // where L_shifted[k] = (L[k] - 0.5) * 8 or just use (L[k] - 4) if L is in [0,7] - const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; - const int * kmap_q2xs = iq2_data[gindex].map; - const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours; + // Actually, let's use: reconstructed = d * sc * (L[k] - 4) + // where L[k] is in [0, 7] (shifted from [0,1]) - GGML_ASSERT(quant_weights && "missing quantization weights"); - GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?"); - GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?"); - GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?"); - GGML_ASSERT(n%QK_K == 0); + const uint32_t kmask1 = 0x03030303; + const uint32_t kmask2 = 0x0f0f0f0f; - block_iq1_s * y = vy; + for (int i = 0; i < nb; i++) { + const float d_all = GGML_FP16_TO_FP32(x[i].d); + const uint8_t * q = x[i].qs; + const uint8_t * hm = x[i].hmask; + uint8_t m = 1; + + uint32_t aux32[4]; + memcpy(aux32, x[i].scales, 12); + uint32_t tmp = aux32[2]; + aux32[2] = ((aux32[0] >> 4) & kmask2) | (((tmp >> 4) & kmask1) << 4); + aux32[3] = ((aux32[1] >> 4) & kmask2) | (((tmp >> 6) & kmask1) << 4); + aux32[0] = (aux32[0] & kmask2) | (((tmp >> 0) & kmask1) << 4); + aux32[1] = (aux32[1] & kmask2) | (((tmp >> 2) & kmask1) << 4); + const uint8_t * aux = (const uint8_t *) aux32; - const int64_t nbl = n/QK_K; + int is = 0; + for (int n = 0; n < QK_K; n += 128) { + int shift = 0; + for (int j = 0; j < 4; ++j) { + int sc1 = (int) aux[is] - 32; + int sc2 = (int) aux[is + 1] - 32; + is += 2; + float dl1 = d_all * sc1; + float dl2 = d_all * sc2; - const int block_size = IQ1S_BLOCK_SIZE; + for (int l = 0; l < 16; ++l) { + int k_idx = ((q[l + 0] >> shift) & 3) + ((hm[l + 0] & m) ? 4 : 0); + y[l + 0] = dl1 * (lv[k_idx] * 7.0f - 4.0f); + } + for (int l = 0; l < 16; ++l) { + int k_idx = ((q[l + 16] >> shift) & 3) + ((hm[l + 16] & m) ? 4 : 0); + y[l + 16] = dl2 * (lv[k_idx] * 7.0f - 4.0f); + } + y += 32; + shift += 2; + m <<= 1; + } + q += 32; + } + } +} - const float x_p[3] = {-1 + IQ1S_DELTA, IQ1S_DELTA, 1 + IQ1S_DELTA}; - const float x_m[3] = {-1 - IQ1S_DELTA, -IQ1S_DELTA, 1 - IQ1S_DELTA}; +// Helper: find optimal symmetric scale for non-uniform mapped levels. +// Closely mirrors make_qx_quants but uses nearest-mapped-level assignment +// instead of rounding to nearest integer. +// mapped_levels[k] = levels[k]*7 - 4, k=0..7. +// Returns the per-sub-block scale d such that x[i] ≈ d * ml[L[i]]. +// L[i] gets the best level index [0..7]. +static float make_q3kpt_quants(int n, + const float * GGML_RESTRICT x, + int8_t * GGML_RESTRICT L, + const float * GGML_RESTRICT weight, + const float * mapped_levels) { + // Find the most negative and most positive mapped levels + float ml_neg = mapped_levels[0], ml_pos = mapped_levels[Q3KPT_N_LEVELS - 1]; + + // Precompute boundaries for branchless nearest-level search + float bounds[Q3KPT_N_LEVELS - 1]; + for (int k = 0; k < Q3KPT_N_LEVELS - 1; ++k) { + bounds[k] = 0.5f * (mapped_levels[k] + mapped_levels[k + 1]); + } + + // Find max absolute value in data (and its sign) + float max = 0, amax = 0; + for (int i = 0; i < n; ++i) { + float ax = fabsf(x[i]); + if (ax > amax) { + amax = ax; + max = x[i]; + } + } + if (amax < GROUP_MAX_EPS) { + // Find level closest to 0 + int zero_k = 0; + float zero_d = fabsf(mapped_levels[0]); + for (int k = 1; k < Q3KPT_N_LEVELS; ++k) { + if (fabsf(mapped_levels[k]) < zero_d) { + zero_d = fabsf(mapped_levels[k]); + zero_k = k; + } + } + for (int i = 0; i < n; ++i) { + L[i] = zero_k; + } + return 0.f; + } + float best_scale = 0; + float best_obj = 0; + bool first = true; - int * idx = (int *)(pairs + 1); + for (int is = -15; is <= 15; ++is) { + float iscales[2] = { + -(fabsf(ml_neg) + 0.1f * is) / max, // map max to ml_neg (Q3_K style) + (fabsf(ml_pos) + 0.1f * is) / max // map max to ml_pos + }; - for (int ibl = 0; ibl < nbl; ++ibl) { + for (int opt = 0; opt < 2; ++opt) { + float iscale = iscales[opt]; - y[ibl].d = GGML_FP32_TO_FP16(0.f); - memset(y[ibl].qs, 0, QK_K/8); - memset(y[ibl].qh, 0, QK_K/16); + float sumlx = 0, suml2 = 0; + for (int i = 0; i < n; ++i) { + float scaled = x[i] * iscale; + // Branchless nearest level assignment + int best_k = (scaled > bounds[0]) + (scaled > bounds[1]) + (scaled > bounds[2]) + + (scaled > bounds[3]) + (scaled > bounds[4]) + (scaled > bounds[5]) + (scaled > bounds[6]); + float w = weight ? weight[i] : x[i] * x[i]; + sumlx += w * x[i] * mapped_levels[best_k]; + suml2 += w * mapped_levels[best_k] * mapped_levels[best_k]; + } + + if (suml2 > 0 && (first || sumlx * sumlx > best_obj * suml2)) { + float scale = sumlx / suml2; + best_obj = scale * sumlx; + best_scale = scale; + first = false; + // Re-assign L with this iscale + for (int i = 0; i < n; ++i) { + float scaled = x[i] * iscale; + int best_k = (scaled > bounds[0]) + (scaled > bounds[1]) + (scaled > bounds[2]) + + (scaled > bounds[3]) + (scaled > bounds[4]) + (scaled > bounds[5]) + (scaled > bounds[6]); + L[i] = best_k; + } + } + } + } + return best_scale; +} - float max_scale = 0; +static void quantize_row_q3_kpt_impl(const float * GGML_RESTRICT x, + block_q3_kpt * GGML_RESTRICT y, + int64_t n_per_row, + const float * GGML_RESTRICT quant_weights) { + assert(n_per_row % QK_K == 0); + const int nb = n_per_row / QK_K; + const float * levels = q3kpt_get_levels(); + GGML_ASSERT(levels != NULL && "Q3_KPT levels not set - call q3kpt_set_levels() first"); - const float * xbl = x + QK_K*ibl; + // Precompute mapped levels: ml[k] = levels[k] * 7 - 4 + float mapped_levels[Q3KPT_N_LEVELS]; + for (int k = 0; k < Q3KPT_N_LEVELS; ++k) { + mapped_levels[k] = levels[k] * 7.0f - 4.0f; + } + + // Precompute boundaries for branchless nearest-level search + float bounds[Q3KPT_N_LEVELS - 1]; + for (int k = 0; k < Q3KPT_N_LEVELS - 1; ++k) { + bounds[k] = 0.5f * (mapped_levels[k] + mapped_levels[k + 1]); + } + + int8_t L[QK_K]; + float scales[QK_K / 16]; + float weight[16]; + float sw[QK_K / 16]; + int8_t Ls[QK_K / 16]; + + for (int i = 0; i < nb; i++) { float sumx2 = 0; - for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i]; - float sigma2 = 2*sumx2/QK_K; + for (int j = 0; j < QK_K; ++j) { + sumx2 += x[j] * x[j]; + } + float sigma2 = 2 * sumx2 / QK_K; - for (int ib = 0; ib < QK_K/block_size; ++ib) { - const float * xb = xbl + block_size*ib; - const float * qw = quant_weights + QK_K*ibl + block_size*ib; - for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); - float max = fabsf(xb[0]); - for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i])); - if (max < GROUP_MAX_EPS_IQ1_S) { - scales[ib] = 0; - shifts[ib] = 1; - memset(L, 1, block_size); - continue; + // First pass: find per-sub-block scales optimized for mapped levels + for (int j = 0; j < QK_K / 16; ++j) { + if (quant_weights) { + const float * qw = quant_weights + QK_K * i + 16 * j; + for (int l = 0; l < 16; ++l) { + weight[l] = qw[l] * sqrtf(sigma2 + x[16 * j + l] * x[16 * j + l]); + } + } else { + for (int l = 0; l < 16; ++l) { + weight[l] = x[16 * j + l] * x[16 * j + l]; + } } - // Here we solve exactly the sum of squared difference (SSD) weighted minimization problem. + float sumw = 0; + for (int l = 0; l < 16; ++l) { + sumw += weight[l]; + } + sw[j] = sumw; + + scales[j] = make_q3kpt_quants(16, x + 16 * j, L + 16 * j, weight, mapped_levels); + } + + // Two-tier scale quantization (identical to Q3_K) + memset(y[i].scales, 0, 12); + float d_block = make_qx_quants(QK_K / 16, 32, scales, Ls, 1, sw); + for (int j = 0; j < QK_K / 16; ++j) { + int l = Ls[j]; + if (j < 8) { + y[i].scales[j] = l & 0xF; + } else { + y[i].scales[j - 8] |= ((l & 0xF) << 4); + } + l >>= 4; + y[i].scales[j % 4 + 8] |= (l << (2 * (j / 4))); + } + y[i].d = GGML_FP32_TO_FP16(d_block); + + // Second pass: level assignment using the quantized scales but + // assigning nearest LEARNED LEVEL instead of nearest integer + int8_t sc; + for (int j = 0; j < QK_K / 16; ++j) { + sc = j < 8 ? y[i].scales[j] & 0xF : y[i].scales[j - 8] >> 4; + sc = (sc | (((y[i].scales[8 + j % 4] >> (2 * (j / 4))) & 3) << 4)) - 32; + float d = GGML_FP16_TO_FP32(y[i].d) * sc; + if (!d) { + // Find level closest to 0 for zero-scale sub-blocks + int zero_k = 0; + float zero_dist = fabsf(mapped_levels[0]); + for (int k = 1; k < Q3KPT_N_LEVELS; ++k) { + if (fabsf(mapped_levels[k]) < zero_dist) { + zero_dist = fabsf(mapped_levels[k]); + zero_k = k; + } + } + for (int ii = 0; ii < 16; ++ii) { + L[16 * j + ii] = zero_k; + } + continue; + } + for (int ii = 0; ii < 16; ++ii) { + float scaled = x[16 * j + ii] / d; + // Branchless nearest level assignment + int best_k = (scaled > bounds[0]) + (scaled > bounds[1]) + (scaled > bounds[2]) + + (scaled > bounds[3]) + (scaled > bounds[4]) + (scaled > bounds[5]) + (scaled > bounds[6]); + L[16 * j + ii] = best_k; + } + } + + // Pack level indices (same bit layout as Q3_K) + memset(y[i].hmask, 0, QK_K / 8); + int m = 0; + uint8_t hm = 1; + for (int j = 0; j < QK_K; ++j) { + if (L[j] > 3) { + y[i].hmask[m] |= hm; + L[j] -= 4; + } + if (++m == QK_K / 8) { + m = 0; + hm <<= 1; + } + } + for (int j = 0; j < QK_K; j += 128) { + for (int l = 0; l < 32; ++l) { + y[i].qs[j / 4 + l] = L[j + l] | (L[j + l + 32] << 2) | (L[j + l + 64] << 4) | (L[j + l + 96] << 6); + } + } + x += QK_K; + } +} + +size_t quantize_q3_kpt(const float * GGML_RESTRICT src, + void * GGML_RESTRICT dst, + int64_t nrow, + int64_t n_per_row, + const float * imatrix) { + size_t row_size = ggml_row_size(GGML_TYPE_Q3_KPT, n_per_row); + char * qrow = (char *) dst; + for (int64_t row = 0; row < nrow; ++row) { + quantize_row_q3_kpt_impl(src, (block_q3_kpt *) qrow, n_per_row, imatrix); + src += n_per_row; + qrow += row_size; + } + return nrow * row_size; +} + +void quantize_row_q3_kpt_ref(const float * GGML_RESTRICT x, block_q3_kpt * GGML_RESTRICT y, int64_t k) { + assert(k % QK_K == 0); + quantize_q3_kpt(x, y, 1, k, NULL); +} + +// Forward declaration needed since quantize_row_iq4_nl_impl is defined later in this file. +static void quantize_row_iq4_nl_impl(const int super_block_size, const int block_size, + const float * GGML_RESTRICT x, + ggml_fp16_t * dh, uint8_t * q4, uint16_t * scales_h, uint8_t * scales_l, + float * scales, float * weight, uint8_t * L, + const int8_t * values, + const float * quant_weights, + const int ntry); + +// ====================== Q4_DPT: IQ4_NL with learned per-tensor int8 levels ====================== +// +// Block format: identical to block_iq4_nl (18 bytes per QK4_NL=32 elements) +// d : ggml_half — per-block scale +// qs : QK4_NL/2 bytes — 4-bit indices into the 16-entry level table +// +// The difference from IQ4_NL: instead of the fixed kvalues_iq4nl int8 table, +// we use 16 int8 levels learned per-tensor via weighted Lloyd-Max k-means. +// Normalization: symmetric (x/amax), bin domain [-1, 1]. +// Levels stored in GGUF as "q4_dpt.levels" (int8 array, 16 values per tensor). + +static int8_t q4dpt_levels[Q4DPT_N_LEVELS]; +static bool q4dpt_levels_set = false; + +void q4dpt_set_levels(const int8_t * levels) { + memcpy(q4dpt_levels, levels, Q4DPT_N_LEVELS * sizeof(int8_t)); + q4dpt_levels_set = true; +} + +const int8_t * q4dpt_get_levels(void) { + return q4dpt_levels_set ? q4dpt_levels : NULL; +} + +void q4dpt_free_levels(void) { + q4dpt_levels_set = false; +} + + +// Run Lloyd-Max iterations on a pre-built histogram. +// levels[] (n_levels entries) is updated in-place (and kept sorted). +static void q4dpt_run_lloyd_max(const float * bin_sum_w, const float * bin_sum_wt, + float * levels, int n_levels, int n_bins, float bin_width, int max_iter) { + // sw/swt sized for the max possible n_levels (Q4DPT_N_LEVELS) + float sw[Q4DPT_N_LEVELS] = { 0 }; + float swt[Q4DPT_N_LEVELS] = { 0 }; + for (int iter = 0; iter < max_iter; ++iter) { + for (int k = 0; k < n_levels; ++k) { sw[k] = 0; swt[k] = 0; } + for (int b = 0; b < n_bins; ++b) { + if (bin_sum_w[b] < 1e-12f) { continue; } + float t = -1.0f + (b + 0.5f) * bin_width; + int best = 0; + float bd = (t - levels[0]) * (t - levels[0]); + for (int k = 1; k < n_levels; ++k) { + float d = (t - levels[k]) * (t - levels[k]); + if (d < bd) { bd = d; best = k; } + } + sw[best] += bin_sum_w[b]; + swt[best] += bin_sum_wt[b]; + } + float max_delta = 0.0f; + for (int k = 0; k < n_levels; ++k) { + if (sw[k] > 1e-12f) { + float nl = swt[k] / sw[k]; + max_delta = fmaxf(max_delta, fabsf(nl - levels[k])); + levels[k] = nl; + } + } + if (max_delta < 1e-10f) { break; } + for (int k = 1; k < n_levels; ++k) { + float v = levels[k]; + int m = k - 1; + while (m >= 0 && levels[m] > v) { levels[m+1] = levels[m]; m--; } + levels[m+1] = v; + } + } +} + +// Train 16 Lloyd-Max int8 levels. +// Bins x/amax values from 32-element IQ4_NL-style blocks into [-1,1], +// runs weighted k-means (seeded from IQ4_NL values), then rounds float +// centroids to sorted int8[16] with post-rounding local search. +void q4dpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, int8_t levels_out[Q4DPT_N_LEVELS]) { + const int N_BINS = 8192; + const float bin_width = 2.0f / N_BINS; + float * bin_sum_w = (float *) calloc(N_BINS, sizeof(float)); + float * bin_sum_wt = (float *) calloc(N_BINS, sizeof(float)); + GGML_ASSERT(bin_sum_w && bin_sum_wt); + + const int64_t n_blocks = n_per_row / QK4_NL; + + // Build weighted histogram: normalize each block by amax, bin into [-1, 1] + for (int64_t row = 0; row < nrow; ++row) { + const float * xrow = data + row * n_per_row; + for (int64_t ib = 0; ib < n_blocks; ++ib) { + const float * xb = xrow + ib * QK4_NL; + float amax = 0.0f; + for (int j = 0; j < QK4_NL; ++j) { + float ax = fabsf(xb[j]); + if (ax > amax) { amax = ax; } + } + if (amax < 1e-10f) { continue; } + const float inv_amax = 1.0f / amax; + for (int j = 0; j < QK4_NL; ++j) { + float w = 1.0f; + if (imatrix) { + w = imatrix[ib * QK4_NL + j]; + if (w < 1e-10f) { w = 1e-10f; } + } + w *= amax * amax; + float t = xb[j] * inv_amax; + int bin_idx = (int)((t + 1.0f) * 0.5f * N_BINS); + if (bin_idx < 0) { bin_idx = 0; } + if (bin_idx >= N_BINS) { bin_idx = N_BINS - 1; } + bin_sum_w[bin_idx] += w; + bin_sum_wt[bin_idx] += w * t; + } + } + } + + // Initialize from IQ4_NL values normalized to [-1, 1], then run Lloyd-Max + float best_levels[Q4DPT_N_LEVELS]; + for (int k = 0; k < Q4DPT_N_LEVELS; ++k) { + best_levels[k] = (float)kvalues_iq4nl[k] / 127.0f; + } + q4dpt_run_lloyd_max(bin_sum_w, bin_sum_wt, best_levels, Q4DPT_N_LEVELS, N_BINS, bin_width, 500); + + // Round float centroids to int8, preserve sort order + int8_t levels_i8[Q4DPT_N_LEVELS]; + for (int k = 0; k < Q4DPT_N_LEVELS; ++k) { + int v = (int)roundf(best_levels[k] * 127.0f); + if (v < -128) { v = -128; } + if (v > 127) { v = 127; } + levels_i8[k] = (int8_t)v; + } + + // Post-rounding local search: try ±1 adjustments to each level greedily. + // The int8 rounding can introduce sub-optimal level placement; this + // hill-climbing on discrete int8 values often recovers a better solution. + for (int pass = 0; pass < 10; ++pass) { + int improved = 0; + for (int k = 0; k < Q4DPT_N_LEVELS; ++k) { + // Evaluate current histogram MSE with int8 levels + float cur_levels[Q4DPT_N_LEVELS]; + for (int i = 0; i < Q4DPT_N_LEVELS; ++i) { + cur_levels[i] = (float)levels_i8[i] / 127.0f; + } + float cur_mse = 0.0f; + for (int b = 0; b < N_BINS; ++b) { + if (bin_sum_w[b] < 1e-12f) { continue; } + float t = -1.0f + (b + 0.5f) * bin_width; + float bd = (t - cur_levels[0]) * (t - cur_levels[0]); + for (int i = 1; i < Q4DPT_N_LEVELS; ++i) { + float d = (t - cur_levels[i]) * (t - cur_levels[i]); + if (d < bd) { bd = d; } + } + cur_mse += bin_sum_w[b] * bd; + } + + int8_t best_val = levels_i8[k]; + int8_t lo = (k > 0) ? (int8_t)(levels_i8[k-1] + 1) : -128; + int8_t hi = (k < Q4DPT_N_LEVELS - 1) ? (int8_t)(levels_i8[k+1] - 1) : 127; + for (int delta = -1; delta <= 1; delta += 2) { + int8_t nv = (int8_t)(levels_i8[k] + delta); + if (nv < lo || nv > hi) { continue; } + cur_levels[k] = (float)nv / 127.0f; + float test_mse = 0.0f; + for (int b = 0; b < N_BINS; ++b) { + if (bin_sum_w[b] < 1e-12f) { continue; } + float t = -1.0f + (b + 0.5f) * bin_width; + float bd = (t - cur_levels[0]) * (t - cur_levels[0]); + for (int i = 1; i < Q4DPT_N_LEVELS; ++i) { + float d = (t - cur_levels[i]) * (t - cur_levels[i]); + if (d < bd) { bd = d; } + } + test_mse += bin_sum_w[b] * bd; + } + if (test_mse < cur_mse) { + best_val = nv; + cur_mse = test_mse; + improved = 1; + } + cur_levels[k] = (float)levels_i8[k] / 127.0f; // restore + } + levels_i8[k] = best_val; + } + if (!improved) { break; } + } + + memcpy(levels_out, levels_i8, Q4DPT_N_LEVELS * sizeof(int8_t)); + + q4dpt_set_levels(levels_out); + + free(bin_sum_w); + free(bin_sum_wt); +} + +void dequantize_row_q4_dpt(const block_q4_dpt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + assert(k % QK4_NL == 0); + const int64_t nb = k / QK4_NL; + const int8_t * values = (const int8_t *)levels; + GGML_ASSERT(values != NULL && "Q4_DPT levels not set for tensor"); + + for (int i = 0; i < nb; i++) { + const uint8_t * qs = x[i].qs; + const float d = GGML_FP16_TO_FP32(x[i].d); + for (int j = 0; j < QK4_NL/2; ++j) { + y[j] = d * (float)values[qs[j] & 0xf]; + y[j + QK4_NL/2] = d * (float)values[qs[j] >> 4]; + } + y += QK4_NL; + } +} + +// Quantize one 32-element block using int8 levels and optimal per-block scale. +// IQ4_NL-style scale perturbation with negative-scale support and final re-assignment. +static void quantize_block_q4_dpt(const float * GGML_RESTRICT xb, block_q4_dpt * GGML_RESTRICT out, + const int8_t * values, const float * qw, int ntry) { + float amax = 0.0f, max_val = 0.0f; + for (int j = 0; j < QK4_NL; ++j) { + float ax = fabsf(xb[j]); + if (ax > amax) { amax = ax; max_val = xb[j]; } + } + if (amax < 1e-10f) { + out->d = 0; + memset(out->qs, 0, QK4_NL/2); + return; + } + + // Initial scale: d = -max/values[0] (allows negative d for asymmetric levels) + float d = ntry > 0 ? -max_val / (float)values[0] : max_val / (float)values[0]; + float id = (fabsf(d) > 1e-20f) ? 1.0f / d : 0.0f; + + // Initial assignment + optimal scale via least-squares + uint8_t L[QK4_NL]; + float sumqx = 0.0f, sumq2 = 0.0f; + for (int j = 0; j < QK4_NL; ++j) { + float al = id * xb[j]; + int bk = 0; + float bd = fabsf(al - (float)values[0]); + for (int k = 1; k < Q4DPT_N_LEVELS; ++k) { + float dist = fabsf(al - (float)values[k]); + if (dist < bd) { bd = dist; bk = k; } + } + L[j] = (uint8_t)bk; + float q = (float)values[bk]; + float w = qw ? qw[j] : 1.0f; + sumqx += w * q * xb[j]; + sumq2 += w * q * q; + } + d = (sumq2 > 1e-20f) ? sumqx / sumq2 : d; + float best = d * sumqx; + uint8_t best_L[QK4_NL]; + memcpy(best_L, L, QK4_NL); + float best_d = d; + + // Scale perturbation: id = (itry + values[0]) / max_val (IQ4_NL-style) + for (int itry = -ntry; itry <= ntry; ++itry) { + id = ((float)itry + (float)values[0]) / max_val; + sumqx = sumq2 = 0.0f; + for (int j = 0; j < QK4_NL; ++j) { + float al = id * xb[j]; + int bk = 0; + float bd = fabsf(al - (float)values[0]); + for (int k = 1; k < Q4DPT_N_LEVELS; ++k) { + float dist = fabsf(al - (float)values[k]); + if (dist < bd) { bd = dist; bk = k; } + } + L[j] = (uint8_t)bk; + float q = (float)values[bk]; + float w = qw ? qw[j] : 1.0f; + sumqx += w * q * xb[j]; + sumq2 += w * q * q; + } + if (sumq2 > 0.0f && sumqx * sumqx > best * sumq2) { + d = sumqx / sumq2; + best = d * sumqx; + best_d = d; + memcpy(best_L, L, QK4_NL); + } + } + + // Final re-assignment using the best scale + id = (fabsf(best_d) > 1e-20f) ? 1.0f / best_d : 0.0f; + for (int j = 0; j < QK4_NL; ++j) { + float al = id * xb[j]; + int bk = 0; + float bd = fabsf(al - (float)values[0]); + for (int k = 1; k < Q4DPT_N_LEVELS; ++k) { + float dist = fabsf(al - (float)values[k]); + if (dist < bd) { bd = dist; bk = k; } + } + best_L[j] = (uint8_t)bk; + } + + out->d = GGML_FP32_TO_FP16(best_d); + for (int j = 0; j < QK4_NL/2; ++j) { + out->qs[j] = best_L[j] | (best_L[j + QK4_NL/2] << 4); + } +} + +size_t quantize_q4_dpt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, + int64_t nrow, int64_t n_per_row, const float * quant_weights) { + GGML_ASSERT(n_per_row % QK4_NL == 0); + const int8_t * values = q4dpt_get_levels(); + GGML_ASSERT(values != NULL && "Q4_DPT levels not set - call q4dpt_set_levels() first"); + + const int64_t nblock = n_per_row / QK4_NL; + char * qrow = (char *) dst; + + for (int64_t row = 0; row < nrow; ++row) { + block_q4_dpt * q4 = (block_q4_dpt *) qrow; + for (int64_t ibl = 0; ibl < nblock; ++ibl) { + const float * qw = quant_weights ? quant_weights + QK4_NL * ibl : NULL; + quantize_block_q4_dpt(src + QK4_NL * ibl, &q4[ibl], values, qw, 15); + } + src += n_per_row; + qrow += nblock * sizeof(block_q4_dpt); + } + return (size_t) nrow * nblock * sizeof(block_q4_dpt); +} + +void quantize_row_q4_dpt_ref(const float * GGML_RESTRICT x, block_q4_dpt * GGML_RESTRICT y, int64_t k) { + assert(k % QK4_NL == 0); + quantize_q4_dpt(x, y, 1, k, NULL); +} + +//////////////////////////////////////////////////////////////////////////////// +// Q2_DPT - 2-bit per-tensor Lloyd-Max quantization (2.5 bpw) +//////////////////////////////////////////////////////////////////////////////// + +// Global levels (used during quantization for the current tensor) +static int8_t q2dpt_levels[Q2DPT_N_LEVELS]; +static bool q2dpt_levels_set = false; + +void q2dpt_set_levels(const int8_t * levels) { + memcpy(q2dpt_levels, levels, Q2DPT_N_LEVELS * sizeof(int8_t)); + q2dpt_levels_set = true; +} + +const int8_t * q2dpt_get_levels(void) { + return q2dpt_levels_set ? q2dpt_levels : NULL; +} + +void q2dpt_free_levels(void) { + q2dpt_levels_set = false; +} + +// Lloyd-Max k-means for Q2_DPT: train 4 int8 levels from weight data. +// Bins normalized values (x/amax) in [-1,1], runs weighted k-means, rounds to sorted int8[4]. +// Also sets the global levels via q2dpt_set_levels(). +void q2dpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, int8_t levels_out[Q2DPT_N_LEVELS]) { + GGML_ASSERT(nrow * n_per_row > 0); + GGML_ASSERT(n_per_row % QK2_DPT == 0); + + const int N_BINS = 8192; + const float bin_width = 2.0f / N_BINS; + + // Allocate and clear histogram buffers + float * bin_sum_w = (float *) calloc(N_BINS, sizeof(float)); + float * bin_sum_wt = (float *) calloc(N_BINS, sizeof(float)); + GGML_ASSERT(bin_sum_w && bin_sum_wt); + + // Build histogram: bin normalized values (x/amax), weight by amax^2 + for (int64_t row = 0; row < nrow; ++row) { + const float * row_data = data + row * n_per_row; + const float * row_w = imatrix ? imatrix + row * n_per_row : NULL; + + for (int64_t ibl = 0; ibl < n_per_row / QK2_DPT; ++ibl) { + const float * block = row_data + ibl * QK2_DPT; + const float * w = row_w ? row_w + ibl * QK2_DPT : NULL; + + // Find max abs in block + float amax = 0.0f; + for (int j = 0; j < QK2_DPT; ++j) { + float ax = fabsf(block[j]); + if (ax > amax) amax = ax; + } + if (amax < 1e-10f) continue; + + // Bin normalized values + for (int j = 0; j < QK2_DPT; ++j) { + float x = block[j] / amax; + float wt = amax * amax * (w ? w[j] : 1.0f); + int bin = (int)((x + 1.0f) / bin_width); + bin = (bin < 0) ? 0 : (bin >= N_BINS) ? N_BINS - 1 : bin; + bin_sum_w[bin] += wt; + bin_sum_wt[bin] += x * wt; + } + } + } + + // Initialize from Q4_DPT levels (subsample to 4 levels) + const int8_t * q4dpt_init = q4dpt_get_levels(); + float best_levels[Q2DPT_N_LEVELS]; + if (q4dpt_init) { + // Subsample Q4_DPT's 16 levels to 4 levels + best_levels[0] = (float)q4dpt_init[0] / 127.0f; + best_levels[1] = (float)q4dpt_init[5] / 127.0f; + best_levels[2] = (float)q4dpt_init[10] / 127.0f; + best_levels[3] = (float)q4dpt_init[15] / 127.0f; + } else { + // Fallback: uniform asymmetric initialization + best_levels[0] = -1.0f; + best_levels[1] = -0.33f; + best_levels[2] = 0.33f; + best_levels[3] = 1.0f; + } + + // Run Lloyd-Max iterations + q4dpt_run_lloyd_max(bin_sum_w, bin_sum_wt, best_levels, Q2DPT_N_LEVELS, N_BINS, bin_width, 500); + + // Round to int8 and enforce sorted order + int8_t levels_i8[Q2DPT_N_LEVELS]; + for (int k = 0; k < Q2DPT_N_LEVELS; ++k) { + int v = (int)roundf(best_levels[k] * 127.0f); + if (v < -128) v = -128; + if (v > 127) v = 127; + levels_i8[k] = (int8_t)v; + } + + // Greedy local search: try +/-1 adjustments + float base_score = 0.0f; + for (int bin = 0; bin < N_BINS; ++bin) { + if (bin_sum_w[bin] > 0) { + float x = bin_sum_wt[bin] / bin_sum_w[bin]; + float best_dist = fabsf(x - best_levels[0]); + for (int k = 1; k < Q2DPT_N_LEVELS; ++k) { + float dist = fabsf(x - best_levels[k]); + if (dist < best_dist) best_dist = dist; + } + base_score += best_dist * bin_sum_w[bin]; + } + } + + for (int pass = 0; pass < 10; ++pass) { + bool improved = false; + for (int k = 0; k < Q2DPT_N_LEVELS; ++k) { + int8_t orig = levels_i8[k]; + for (int delta = -1; delta <= 1; delta += 2) { + int8_t trial = (int8_t)(orig + delta); + if (k > 0 && trial <= levels_i8[k-1]) continue; + if (k < Q2DPT_N_LEVELS - 1 && trial >= levels_i8[k+1]) continue; + + levels_i8[k] = trial; + float cur_levels[Q2DPT_N_LEVELS]; + for (int i = 0; i < Q2DPT_N_LEVELS; ++i) + cur_levels[i] = (float)levels_i8[i] / 127.0f; + + float cur_score = 0.0f; + for (int bin = 0; bin < N_BINS; ++bin) { + if (bin_sum_w[bin] > 0) { + float x = bin_sum_wt[bin] / bin_sum_w[bin]; + float best_dist = fabsf(x - cur_levels[0]); + for (int i = 1; i < Q2DPT_N_LEVELS; ++i) { + float dist = fabsf(x - cur_levels[i]); + if (dist < best_dist) best_dist = dist; + } + cur_score += best_dist * bin_sum_w[bin]; + } + } + + if (cur_score < base_score) { + base_score = cur_score; + improved = true; + } else { + levels_i8[k] = orig; + } + } + } + if (!improved) break; + } + + memcpy(levels_out, levels_i8, Q2DPT_N_LEVELS * sizeof(int8_t)); + q2dpt_set_levels(levels_i8); + + free(bin_sum_w); + free(bin_sum_wt); +} + +void dequantize_row_q2_dpt(const block_q2_dpt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + assert(k % QK2_DPT == 0); + const int64_t nb = k / QK2_DPT; + const int8_t * values = (const int8_t *)levels; + GGML_ASSERT(values != NULL && "Q2_DPT levels not set for tensor"); + + for (int i = 0; i < nb; i++) { + const uint8_t * qs = x[i].qs; + const float d = GGML_FP16_TO_FP32(x[i].d); + for (int j = 0; j < QK2_DPT/4; ++j) { + uint8_t q = qs[j]; + y[j*4 + 0] = d * (float)values[(q >> 0) & 3]; + y[j*4 + 1] = d * (float)values[(q >> 2) & 3]; + y[j*4 + 2] = d * (float)values[(q >> 4) & 3]; + y[j*4 + 3] = d * (float)values[(q >> 6) & 3]; + } + y += QK2_DPT; + } +} + +// Strategy bitmask for quantize_block_q2_dpt (for A/B testing). +// Bit 0: level-anchor CD (approach A) +// Bit 1: boundary sweep+CD (approach B) +// Bit 2: dual-extreme CD (approach C: max_val AND min_val anchors) +// Bit 3: element-anchor CD (approach D: every xb[j]/values[k] as anchor) +// Bit 4: brute-force monotone partition (approach E: exhaustive search, O(n^3) per block) +static int q2dpt_quant_strategy = 0x3; // default: A+B + +void q2dpt_set_quant_strategy(int s) { q2dpt_quant_strategy = s; } + +// Refine d via iterated CD until convergence. Returns best d. +static float q2dpt_cd_refine(const float * GGML_RESTRICT xb, const float * qw, + const int8_t * values, float d) { + for (int iter = 0; iter < 8; ++iter) { + float id = (fabsf(d) > 1e-20f) ? 1.0f / d : 0.0f; + float sumqx = 0.0f, sumq2 = 0.0f; + for (int j = 0; j < QK2_DPT; ++j) { + float al = id * xb[j]; + int bk = 0; + float bd = fabsf(al - (float)values[0]); + for (int m = 1; m < Q2DPT_N_LEVELS; ++m) { + float dist = fabsf(al - (float)values[m]); + if (dist < bd) { bd = dist; bk = m; } + } + float q = (float)values[bk]; + float w = qw ? qw[j] : 1.0f; + sumqx += w * q * xb[j]; + sumq2 += w * q * q; + } + if (sumq2 < 1e-20f) break; + float d_new = sumqx / sumq2; + if (fabsf(d_new - d) < 1e-8f * fabsf(d)) break; + d = d_new; + } + return d; +} + +// Evaluate a candidate d: returns objective, fills L[]. +static float q2dpt_eval(const float * GGML_RESTRICT xb, const float * qw, + const int8_t * values, float d, uint8_t * L) { + float id = (fabsf(d) > 1e-20f) ? 1.0f / d : 0.0f; + float sumqx = 0.0f, sumq2 = 0.0f; + for (int j = 0; j < QK2_DPT; ++j) { + float al = id * xb[j]; + int bk = 0; + float bd = fabsf(al - (float)values[0]); + for (int m = 1; m < Q2DPT_N_LEVELS; ++m) { + float dist = fabsf(al - (float)values[m]); + if (dist < bd) { bd = dist; bk = m; } + } + L[j] = (uint8_t)bk; + float q = (float)values[bk]; + float w = qw ? qw[j] : 1.0f; + sumqx += w * q * xb[j]; + sumq2 += w * q * q; + } + if (sumq2 < 1e-20f) return -1e30f; + return (sumqx / sumq2) * sumqx; +} + +// Helper: try a single starting d, refine via CD, update best if improved. +static inline void q2dpt_try_d(const float * GGML_RESTRICT xb, const float * qw, + const int8_t * values, float d_init, + float * best, float * best_d, uint8_t * best_L) { + uint8_t L[QK2_DPT]; + float d = q2dpt_cd_refine(xb, qw, values, d_init); + float score = q2dpt_eval(xb, qw, values, d, L); + if (score > *best) { + *best = score; *best_d = d; + memcpy(best_L, L, QK2_DPT); + } +} + +// Quantize one 32-element block using 4 int8 levels and optimal per-block scale. +// The q2dpt_quant_strategy bitmask selects which search approaches are used: +// Bit 0 (A): level anchors + CD (d = max_val / values[k] for each k) +// Bit 1 (B): boundary sweep + CD (d = xb[j] / boundary for each j and boundary) +// Bit 2 (C): dual-extreme anchors + CD (A using both max_val AND min_val) +// Bit 3 (D): element-anchor scan + CD (d = xb[j] / values[k] for each j, k) +// Bit 4 (E): brute-force monotone partition (exhaustive over all C(35,3) partitions) +static void quantize_block_q2_dpt(const float * GGML_RESTRICT xb, block_q2_dpt * GGML_RESTRICT out, + const int8_t * values, const float * qw, int ntry) { + (void)ntry; + const int strat = q2dpt_quant_strategy; + + float amax = 0.0f, max_val = 0.0f, min_val = 0.0f; + for (int j = 0; j < QK2_DPT; ++j) { + float ax = fabsf(xb[j]); + if (ax > amax) { amax = ax; max_val = xb[j]; } + if (xb[j] < min_val) min_val = xb[j]; + if (xb[j] > max_val) max_val = xb[j]; + } + if (amax < 1e-10f) { + out->d = 0; + memset(out->qs, 0, QK2_DPT/4); + return; + } + + uint8_t best_L[QK2_DPT]; + float best = -1e30f; + float best_d = 0.0f; + + // --- A: level-anchor CD (4 starting points) --- + if (strat & 0x1) { + for (int k = 0; k < Q2DPT_N_LEVELS; ++k) { + if (values[k] == 0) continue; + q2dpt_try_d(xb, qw, values, max_val / (float)values[k], + &best, &best_d, best_L); + } + } + + // --- B: boundary-crossing sweep + CD --- + if (strat & 0x2) { + for (int b = 0; b < Q2DPT_N_LEVELS - 1; ++b) { + float bnd = ((float)values[b] + (float)values[b + 1]) * 0.5f; + if (fabsf(bnd) < 0.5f) continue; + for (int j = 0; j < QK2_DPT; ++j) { + if (fabsf(xb[j]) < 1e-12f) continue; + q2dpt_try_d(xb, qw, values, xb[j] / bnd, + &best, &best_d, best_L); + } + } + } + + // --- C: dual-extreme anchors + CD (8 starting points) --- + if (strat & 0x4) { + float extremes[2] = { max_val, min_val }; + for (int e = 0; e < 2; ++e) { + for (int k = 0; k < Q2DPT_N_LEVELS; ++k) { + if (values[k] == 0) continue; + q2dpt_try_d(xb, qw, values, extremes[e] / (float)values[k], + &best, &best_d, best_L); + } + } + } + + // --- D: element-anchor scan + CD (32 x 4 starting points) --- + if (strat & 0x8) { + for (int j = 0; j < QK2_DPT; ++j) { + if (fabsf(xb[j]) < 1e-12f) continue; + for (int k = 0; k < Q2DPT_N_LEVELS; ++k) { + if (values[k] == 0) continue; + q2dpt_try_d(xb, qw, values, xb[j] / (float)values[k], + &best, &best_d, best_L); + } + } + } + + // --- E: brute-force monotone partition enumeration --- + // For a single scale d, the optimal assignment must be monotone on sorted x: + // if x_i < x_j then L[i] <= L[j] (for d>0) or L[i] >= L[j] (for d<0). + // We enumerate all C(32+3, 3) = C(35,3) = 6545 ways to partition 32 sorted + // elements into 4 groups, score each in O(1) using prefix sums, then pick best. + if (strat & 0x10) { + // Sort elements by value, keeping original indices + int idx[QK2_DPT]; + for (int j = 0; j < QK2_DPT; ++j) idx[j] = j; + // Simple insertion sort (only 32 elements) + for (int i = 1; i < QK2_DPT; ++i) { + int t = idx[i]; + float tv = xb[t]; + int j = i - 1; + while (j >= 0 && xb[idx[j]] > tv) { + idx[j + 1] = idx[j]; + --j; + } + idx[j + 1] = t; + } + + // Build weighted prefix sums: swx[i] = sum_{j0 maps sorted ascending to values ascending, + // d<0 maps sorted ascending to values descending. + for (int flip = 0; flip < 2; ++flip) { + float v[Q2DPT_N_LEVELS]; + if (flip == 0) { + for (int k = 0; k < Q2DPT_N_LEVELS; ++k) v[k] = (float)values[k]; + } else { + for (int k = 0; k < Q2DPT_N_LEVELS; ++k) v[k] = (float)values[Q2DPT_N_LEVELS - 1 - k]; + } + + // Precompute per-level pair products for scoring + float vv[Q2DPT_N_LEVELS]; + for (int k = 0; k < Q2DPT_N_LEVELS; ++k) vv[k] = v[k] * v[k]; + + float bf_best = -1e30f; + int bf_b1 = 0, bf_b2 = 0, bf_b3 = 0; + + // Enumerate partition boundaries b1 <= b2 <= b3 where group k = [b_k, b_{k+1}) + // b0=0, b4=32. 0 <= b1 <= b2 <= b3 <= 32. + for (int b1 = 0; b1 <= QK2_DPT; ++b1) { + // Segment 0: indices [0, b1) assigned to v[0] + float s0_wx = swx[b1] - swx[0]; + float s0_w = sw[b1] - sw[0]; + for (int b2 = b1; b2 <= QK2_DPT; ++b2) { + // Segment 1: indices [b1, b2) assigned to v[1] + float s01_wx = s0_wx + (swx[b2] - swx[b1]); + float s01_w = s0_w + (sw[b2] - sw[b1]); + // sumqx and sumq2 for segments 0+1 can be expressed but we need all 4. + // For efficiency, compute incrementally for b3. + float partial_sumqx = v[0] * s0_wx + v[1] * (swx[b2] - swx[b1]); + float partial_sumq2 = vv[0] * s0_w + vv[1] * (sw[b2] - sw[b1]); + (void)s01_wx; (void)s01_w; + for (int b3 = b2; b3 <= QK2_DPT; ++b3) { + // Segment 2: [b2, b3) -> v[2], Segment 3: [b3, 32) -> v[3] + float seg2_wx = swx[b3] - swx[b2]; + float seg2_w = sw[b3] - sw[b2]; + float seg3_wx = swx[QK2_DPT] - swx[b3]; + float seg3_w = sw[QK2_DPT] - sw[b3]; + + float sumqx = partial_sumqx + v[2] * seg2_wx + v[3] * seg3_wx; + float sumq2 = partial_sumq2 + vv[2] * seg2_w + vv[3] * seg3_w; + + if (sumq2 < 1e-20f) continue; + // score = d * sumqx = sumqx^2 / sumq2 (only valid when sumqx > 0) + float score = sumqx * sumqx / sumq2; + // d = sumqx / sumq2; for validity we need d > 0 when flip==0, d < 0 when flip==1 + // i.e. sumqx > 0 for flip==0, sumqx < 0 for flip==1 + if (flip == 0 && sumqx <= 0.0f) continue; + if (flip == 1 && sumqx >= 0.0f) continue; + if (score > bf_best) { + bf_best = score; + bf_b1 = b1; bf_b2 = b2; bf_b3 = b3; + } + } + } + } + + if (bf_best > -1e29f) { + // Reconstruct the assignment for the best partition + uint8_t L_bf[QK2_DPT]; + for (int j = 0; j < bf_b1; ++j) { + int orig = idx[j]; + L_bf[orig] = flip == 0 ? 0 : (Q2DPT_N_LEVELS - 1); + } + for (int j = bf_b1; j < bf_b2; ++j) { + int orig = idx[j]; + L_bf[orig] = flip == 0 ? 1 : (Q2DPT_N_LEVELS - 2); + } + for (int j = bf_b2; j < bf_b3; ++j) { + int orig = idx[j]; + L_bf[orig] = flip == 0 ? 2 : (Q2DPT_N_LEVELS - 1 - 2); + } + for (int j = bf_b3; j < QK2_DPT; ++j) { + int orig = idx[j]; + L_bf[orig] = flip == 0 ? 3 : 0; + } + // Compute d from this assignment + float sumqx = 0.0f, sumq2 = 0.0f; + for (int j = 0; j < QK2_DPT; ++j) { + float q = (float)values[L_bf[j]]; + float w = qw ? qw[j] : 1.0f; + sumqx += w * q * xb[j]; + sumq2 += w * q * q; + } + if (sumq2 > 1e-20f) { + float d_bf = sumqx / sumq2; + float score = (sumqx / sumq2) * sumqx; + if (score > best) { + best = score; + best_d = d_bf; + memcpy(best_L, L_bf, QK2_DPT); + } + } + } + } + } + + // Final re-assignment with best scale + float id = (fabsf(best_d) > 1e-20f) ? 1.0f / best_d : 0.0f; + for (int j = 0; j < QK2_DPT; ++j) { + float al = id * xb[j]; + int bk = 0; + float bd = fabsf(al - (float)values[0]); + for (int k = 1; k < Q2DPT_N_LEVELS; ++k) { + float dist = fabsf(al - (float)values[k]); + if (dist < bd) { bd = dist; bk = k; } + } + best_L[j] = (uint8_t)bk; + } + + // Pack 2-bit indices: 4 values per byte + out->d = GGML_FP32_TO_FP16(best_d); + for (int j = 0; j < QK2_DPT/4; ++j) { + out->qs[j] = best_L[j*4] | (best_L[j*4+1] << 2) | (best_L[j*4+2] << 4) | (best_L[j*4+3] << 6); + } +} + +size_t quantize_q2_dpt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, + int64_t nrow, int64_t n_per_row, const float * quant_weights) { + GGML_ASSERT(n_per_row % QK2_DPT == 0); + const int8_t * values = q2dpt_get_levels(); + GGML_ASSERT(values != NULL && "Q2_DPT levels not set - call q2dpt_set_levels() first"); + + const int64_t nblock = n_per_row / QK2_DPT; + char * qrow = (char *) dst; + + for (int64_t row = 0; row < nrow; ++row) { + block_q2_dpt * q2 = (block_q2_dpt *) qrow; + for (int64_t ibl = 0; ibl < nblock; ++ibl) { + const float * qw = quant_weights ? quant_weights + QK2_DPT * ibl : NULL; + quantize_block_q2_dpt(src + QK2_DPT * ibl, &q2[ibl], values, qw, 7); + } + src += n_per_row; + qrow += nblock * sizeof(block_q2_dpt); + } + return (size_t) nrow * nblock * sizeof(block_q2_dpt); +} + +void quantize_row_q2_dpt_ref(const float * GGML_RESTRICT x, block_q2_dpt * GGML_RESTRICT y, int64_t k) { + assert(k % QK2_DPT == 0); + quantize_q2_dpt(x, y, 1, k, NULL); +} + +// ====================== Q2_KPT: Q2_K with learned per-tensor float levels ====================== + +static float q2kpt_levels[Q2KPT_N_LEVELS]; +static bool q2kpt_levels_set = false; + +// Global level storage for Q2_KPT (per-block levels for last quantized tensor) +static float *q2kpt_block_levels = NULL; +static size_t q2kpt_max_levels = 0; +static size_t q2kpt_cur_levels = 0; + +// Prepare the levels buffer for a tensor with given dimensions. +// This should be called before parallel quantization to pre-allocate storage. +void q2kpt_prepare_levels(int64_t nrows, int64_t n_per_row) { + const int nb = (int)(n_per_row / QK_K); + const size_t total_levels = (size_t)nrows * nb * Q2KPT_N_LEVELS; + if (total_levels > q2kpt_max_levels) { + q2kpt_block_levels = (float *) realloc(q2kpt_block_levels, total_levels * sizeof(float)); + q2kpt_max_levels = total_levels; + } + q2kpt_cur_levels = total_levels; +} + +void q2kpt_set_levels(const float * levels) { + memcpy(q2kpt_levels, levels, Q2KPT_N_LEVELS * sizeof(float)); + q2kpt_levels_set = true; +} + +const float * q2kpt_get_levels(void) { + // Return per-block levels if available, otherwise global levels + if (q2kpt_block_levels && q2kpt_cur_levels > 0) { + return q2kpt_block_levels; + } + return q2kpt_levels_set ? q2kpt_levels : NULL; +} + +void q2kpt_free_levels(void) { + q2kpt_levels_set = false; + if (q2kpt_block_levels) { + free(q2kpt_block_levels); + q2kpt_block_levels = NULL; + q2kpt_max_levels = 0; + q2kpt_cur_levels = 0; + } +} + +// Train 4 Lloyd-Max float levels for a single 256-element block. +// Normalizes sub-block values to [0,1] using Q2_K-style scale+min estimation, then runs k-means. +// Forward declaration (defined later in this file) +static float make_q2kpt_quants(int n, const float * GGML_RESTRICT x, + uint8_t * GGML_RESTRICT L, float * GGML_RESTRICT the_min, + const float * mapped_levels, const float * weight); + +// ---- q2kpt_quantize_block_given_levels ---------------------------------------- +// Quantize one QK_K-element block using caller-specified levels (no training). +// block_x: QK_K floats of original data +// y: output block_q2_kpt (filled in place) +// quant_weights: QK_K importance weights (or NULL → use x[i]²) +// sigma2: mean(x²) for the block (for weight formula) +// levels: Q2KPT_N_LEVELS values in [0,1], must be sorted ascending +// ------------------------------------------------------------------------------- +static void q2kpt_quantize_block_given_levels( + const float * GGML_RESTRICT block_x, + block_q2_kpt * GGML_RESTRICT y, + const float * GGML_RESTRICT quant_weights, + float sigma2, + const float levels[Q2KPT_N_LEVELS]) { + + float mapped_levels[Q2KPT_N_LEVELS]; + float bounds[Q2KPT_N_LEVELS - 1]; + for (int k = 0; k < Q2KPT_N_LEVELS; ++k) mapped_levels[k] = levels[k] * 3.0f; + for (int k = 0; k < Q2KPT_N_LEVELS - 1; ++k) + bounds[k] = 0.5f * (mapped_levels[k] + mapped_levels[k + 1]); + + uint8_t L[QK_K]; + float mins[QK_K / 16], scales[QK_K / 16], sw[QK_K / 16]; + float weight[16]; + uint8_t Ls[QK_K / 16], Lm[QK_K / 16]; + + memset(sw, 0, sizeof(sw)); + float sumx2 = sigma2 * QK_K; // reconstitute (or recompute below) + + for (int j = 0; j < QK_K / 16; ++j) { + const float * bx = block_x + 16 * j; + if (quant_weights) { + const float * qw = quant_weights + 16 * j; + for (int l = 0; l < 16; ++l) + weight[l] = qw[l] * sqrtf(sigma2 + bx[l] * bx[l]); + } else { + for (int l = 0; l < 16; ++l) + weight[l] = bx[l] * bx[l]; + } + for (int l = 0; l < 16; ++l) sw[j] += weight[l]; + scales[j] = make_q2kpt_quants(16, bx, L + 16 * j, &mins[j], mapped_levels, weight); + } + + float dm = make_qp_quants(QK_K / 16, 15, scales, Ls, sw); + float mm = make_qp_quants(QK_K / 16, 15, mins, Lm, sw); + + y->d = GGML_FP32_TO_FP16(dm); + y->dmin = GGML_FP32_TO_FP16(mm); + dm = GGML_FP16_TO_FP32(y->d); + mm = GGML_FP16_TO_FP32(y->dmin); + + for (int j = 0; j < QK_K / 16; ++j) y->scales[j] = Ls[j] | (Lm[j] << 4); + + // Second pass: reassign with quantized scales + for (int j = 0; j < QK_K / 16; ++j) { + const float d = dm * (y->scales[j] & 0xF); + if (!d) { + int zero_k = 0; + float zero_d = fabsf(mapped_levels[0]); + for (int k = 1; k < Q2KPT_N_LEVELS; ++k) { + if (fabsf(mapped_levels[k]) < zero_d) { zero_d = fabsf(mapped_levels[k]); zero_k = k; } + } + for (int ii = 0; ii < 16; ++ii) L[16 * j + ii] = zero_k; + continue; + } + const float m = mm * (y->scales[j] >> 4); + for (int ii = 0; ii < 16; ++ii) { + float scaled = (block_x[16 * j + ii] + m) / d; + L[16 * j + ii] = (scaled > bounds[0]) + (scaled > bounds[1]) + (scaled > bounds[2]); + } + } + + // Pack 2-bit indices (Q2_K layout) + for (int j = 0; j < QK_K; j += 128) { + for (int l = 0; l < 32; ++l) { + y->qs[j / 4 + l] = L[j + l] | (L[j + l + 32] << 2) + | (L[j + l + 64] << 4) | (L[j + l + 96] << 6); + } + } + + (void)sumx2; +} + +// ---- Histogram Lloyd-Max helper ---------------------------------------------- +// Runs weighted Lloyd-Max iterations on a pre-built histogram. +// bin_centers[b]: representative value for bin b (weighted centroid) +// bin_w[b]: total weight of data in bin b +// levels[]: centroids, in/out (must be sorted ascending on entry) +// ------------------------------------------------------------------------------- +static void q2kpt_histogram_lloyd_max( + int n_bins, const float * bin_centers, const float * bin_w, + float * levels, int n_levels, int n_iter) { + + for (int iter = 0; iter < n_iter; ++iter) { + float sum_w[Q2KPT_N_LEVELS] = { 0.0f }; + float sum_wt[Q2KPT_N_LEVELS] = { 0.0f }; + + for (int b = 0; b < n_bins; ++b) { + float w = bin_w[b]; + if (w < 1e-30f) continue; + float t = bin_centers[b]; + int best = 0; + float bd2 = (t - levels[0]) * (t - levels[0]); + for (int k = 1; k < n_levels; ++k) { + float d2 = (t - levels[k]) * (t - levels[k]); + if (d2 < bd2) { bd2 = d2; best = k; } + } + sum_w[best] += w; + sum_wt[best] += w * t; + } + + float new_levels[Q2KPT_N_LEVELS]; + float max_delta = 0.0f; + for (int k = 0; k < n_levels; ++k) { + new_levels[k] = (sum_w[k] > 1e-30f) ? sum_wt[k] / sum_w[k] : levels[k]; + } + // Sort ascending (insertion sort, n_levels=4) + for (int k = 1; k < n_levels; ++k) { + float v = new_levels[k]; int m = k - 1; + while (m >= 0 && new_levels[m] > v) { new_levels[m + 1] = new_levels[m]; --m; } + new_levels[m + 1] = v; + } + for (int k = 0; k < n_levels; ++k) + max_delta = fmaxf(max_delta, fabsf(new_levels[k] - levels[k])); + memcpy(levels, new_levels, n_levels * sizeof(float)); + if (max_delta < 1e-7f) break; + } +} + +// ---- q2kpt_optimize_block_levels ---------------------------------------------- +// Full closed-loop EM training for one QK_K block using histogram binning: +// 1. Init: histogram Lloyd-Max on normalized [0,1] sub-block values +// 2. EM cycles: full E-step → build effective-T histogram → cheap Lloyd-Max +// 3. Final quantize with best levels seen +// +// block_x: QK_K floats of original data +// block_y: workspace / output (holds the best quantization on return) +// quant_weights: QK_K per-element importance weights (or NULL → use x[i]²) +// sigma2: mean(x²) for the block +// levels_out: Q2KPT_N_LEVELS trained levels in [0,1], ascending +// ------------------------------------------------------------------------------- +#define Q2KPT_N_BINS 128 // histogram bins +#define Q2KPT_INIT_LLOYD 10 // Lloyd-Max iters on init histogram +#define Q2KPT_N_EM_CYCLES 4 // number of full E-step calls +#define Q2KPT_LLOYD_PER_CYCLE 10 // cheap histogram Lloyd-Max iters per cycle + +static void q2kpt_optimize_block_levels( + const float * GGML_RESTRICT block_x, + block_q2_kpt * GGML_RESTRICT block_y, + const float * GGML_RESTRICT quant_weights, + float sigma2, + float levels_out[Q2KPT_N_LEVELS]) { + + const float inv_bins = 1.0f / Q2KPT_N_BINS; + + // ---- Build per-element weights and sub-block-normalised values ----------- + float weights[QK_K]; + float norm_vals[QK_K]; + + for (int sb = 0; sb < QK_K / 16; ++sb) { + const float * xsb = block_x + sb * 16; + float xmin = xsb[0], xmax = xsb[0]; + for (int l = 1; l < 16; ++l) { + if (xsb[l] < xmin) xmin = xsb[l]; + if (xsb[l] > xmax) xmax = xsb[l]; + } + if (xmin > 0.0f) xmin = 0.0f; + float range = xmax - xmin; + float inv_range = (range > 1e-10f) ? 1.0f / range : 0.0f; + + for (int l = 0; l < 16; ++l) { + int el = sb * 16 + l; + float t = (range > 1e-10f) ? + fmaxf(0.0f, fminf(1.0f, (xsb[l] - xmin) * inv_range)) : 0.0f; + norm_vals[el] = t; + + float w; + if (quant_weights) { + w = quant_weights[el] * sqrtf(sigma2 + xsb[l] * xsb[l]); + } else { + w = xsb[l] * xsb[l]; + } + // Scale by range² so normalised-space errors match actual-space + weights[el] = fmaxf(w * range * range, 1e-30f); + } + } + + // ---- Phase 1: Init levels via histogram Lloyd-Max on norm_vals ---------- + float bin_w[Q2KPT_N_BINS]; + float bin_wt[Q2KPT_N_BINS]; + float bin_centers[Q2KPT_N_BINS]; + + memset(bin_w, 0, sizeof(bin_w)); + memset(bin_wt, 0, sizeof(bin_wt)); + + for (int i = 0; i < QK_K; ++i) { + float t = norm_vals[i]; + int b = (int)(t * Q2KPT_N_BINS); + if (b >= Q2KPT_N_BINS) b = Q2KPT_N_BINS - 1; + bin_w[b] += weights[i]; + bin_wt[b] += weights[i] * t; + } + for (int b = 0; b < Q2KPT_N_BINS; ++b) + bin_centers[b] = (bin_w[b] > 1e-30f) ? bin_wt[b] / bin_w[b] : (b + 0.5f) * inv_bins; + + float levels[Q2KPT_N_LEVELS]; + for (int k = 0; k < Q2KPT_N_LEVELS; ++k) + levels[k] = (float)k / (Q2KPT_N_LEVELS - 1); + + q2kpt_histogram_lloyd_max(Q2KPT_N_BINS, bin_centers, bin_w, + levels, Q2KPT_N_LEVELS, Q2KPT_INIT_LLOYD); + + // ---- Phase 2: EM cycles ------------------------------------------------- + // Each cycle: full E-step → build effective-T histogram → cheap Lloyd-Max. + // Effective-T: T_i = (x_i + B_i) / A_i, W_i = w_i * A_i² + // The M-step optimal level for class k is the W-weighted mean of T_i in k. + float best_levels[Q2KPT_N_LEVELS]; + memcpy(best_levels, levels, sizeof(levels)); + float best_err = 1e38f; + + float eff_bin_w[Q2KPT_N_BINS]; + float eff_bin_wt[Q2KPT_N_BINS]; + float eff_bin_centers[Q2KPT_N_BINS]; + + for (int cycle = 0; cycle < Q2KPT_N_EM_CYCLES; ++cycle) { + + // Full E-step + q2kpt_quantize_block_given_levels(block_x, block_y, quant_weights, sigma2, levels); + + float d_all_q = GGML_FP16_TO_FP32(block_y->d); + float dmin_q = GGML_FP16_TO_FP32(block_y->dmin); + + memset(eff_bin_w, 0, sizeof(eff_bin_w)); + memset(eff_bin_wt, 0, sizeof(eff_bin_wt)); + float err = 0.0f; + + for (int el = 0; el < QK_K; ++el) { + int sb = el / 16; + int k_j = block_y->scales[sb] & 0xF; + int m_j = block_y->scales[sb] >> 4; + + float A = d_all_q * (float)k_j * 3.0f; + float mn = dmin_q * (float)m_j; + + int qs_byte = (el / 128) * 32 + el % 32; + int shift = ((el % 128) / 32) * 2; + int idx = (block_y->qs[qs_byte] >> shift) & 3; + + float w = quant_weights ? + quant_weights[el] * sqrtf(sigma2 + block_x[el] * block_x[el]) : + block_x[el] * block_x[el]; + w = fmaxf(w, 1e-30f); + + float y_approx = A * levels[idx] - mn; + float diff = y_approx - block_x[el]; + err += w * diff * diff; + + // Build effective-T histogram for histogram Lloyd-Max M-step + if (A > 1e-10f) { + float T = fmaxf(0.0f, fminf(1.0f, (block_x[el] + mn) / A)); + float W = w * A * A; + int b = (int)(T * Q2KPT_N_BINS); + if (b >= Q2KPT_N_BINS) b = Q2KPT_N_BINS - 1; + eff_bin_w[b] += W; + eff_bin_wt[b] += W * T; + } + } + + if (err < best_err) { + best_err = err; + memcpy(best_levels, levels, sizeof(levels)); + } + + for (int b = 0; b < Q2KPT_N_BINS; ++b) + eff_bin_centers[b] = (eff_bin_w[b] > 1e-30f) + ? eff_bin_wt[b] / eff_bin_w[b] + : (b + 0.5f) * inv_bins; + + q2kpt_histogram_lloyd_max(Q2KPT_N_BINS, eff_bin_centers, eff_bin_w, + levels, Q2KPT_N_LEVELS, Q2KPT_LLOYD_PER_CYCLE); + } + + // Final quantize with the best levels found across all cycles + memcpy(levels_out, best_levels, sizeof(best_levels)); + q2kpt_quantize_block_given_levels(block_x, block_y, quant_weights, sigma2, best_levels); +} + +// Train 4 Lloyd-Max float levels from tensor data for Q2_KPT. +// Uses Q2_K-style scale+min estimation to normalize sub-block values to [0,1]. +GGML_API void q2kpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float levels_out[Q2KPT_N_LEVELS]) { + const int N_BINS = 8192; + const float bin_width = 1.0f / N_BINS; + float * bin_sum_w = (float *) calloc(N_BINS, sizeof(float)); + float * bin_sum_wt = (float *) calloc(N_BINS, sizeof(float)); + GGML_ASSERT(bin_sum_w && bin_sum_wt); + + const int nb = (int)(n_per_row / QK_K); + + // Single pass: for each 16-element sub-block, estimate scale+min, normalize to [0,1], bin + for (int64_t row = 0; row < nrow; ++row) { + const float * xrow = data + row * n_per_row; + + for (int i = 0; i < nb; i++) { + const float * x = xrow + i * QK_K; + + for (int j = 0; j < QK_K / 16; ++j) { + // Find min and max of sub-block + float xmin = x[16 * j], xmax = x[16 * j]; + for (int l = 1; l < 16; ++l) { + if (x[16 * j + l] < xmin) xmin = x[16 * j + l]; + if (x[16 * j + l] > xmax) xmax = x[16 * j + l]; + } + // Q2_K clamps min to <= 0 + if (xmin > 0) xmin = 0; + float range = xmax - xmin; + if (range < 1e-10f) continue; + + float inv_range = 1.0f / range; + + for (int l = 0; l < 16; ++l) { + // Normalize to [0, 1]: t = (x - min) / range + float t = (x[16 * j + l] - xmin) * inv_range; + if (t < 0.0f) t = 0.0f; + if (t > 1.0f) t = 1.0f; + + int bin_idx = (int)(t * N_BINS); + if (bin_idx >= N_BINS) bin_idx = N_BINS - 1; + + int elem = i * QK_K + 16 * j + l; + float w = imatrix ? imatrix[elem] : 1.0f; + if (w < 1e-10f) w = 1e-10f; + // Weight by range² (like Q3_KPT weights by d²) + w *= range * range; + + bin_sum_w[bin_idx] += w; + bin_sum_wt[bin_idx] += w * t; + } + } + } + } + + // Initialize 4 levels uniformly in [0, 1] + float levels[Q2KPT_N_LEVELS]; + for (int k = 0; k < Q2KPT_N_LEVELS; ++k) { + levels[k] = (float)k / (Q2KPT_N_LEVELS - 1); + } + + // Lloyd-Max iterations on bins + for (int iter = 0; iter < 100; ++iter) { + float sum_w[Q2KPT_N_LEVELS] = { 0 }; + float sum_wt[Q2KPT_N_LEVELS] = { 0 }; + + for (int b = 0; b < N_BINS; ++b) { + if (bin_sum_w[b] < 1e-12f) continue; + const float t = (b + 0.5f) * bin_width; + int best = 0; + float bd2 = (t - levels[0]) * (t - levels[0]); + for (int k = 1; k < Q2KPT_N_LEVELS; ++k) { + float d2 = (t - levels[k]) * (t - levels[k]); + if (d2 < bd2) { bd2 = d2; best = k; } + } + sum_w[best] += bin_sum_w[b]; + sum_wt[best] += bin_sum_wt[b]; + } + + float max_delta = 0.0f; + for (int k = 0; k < Q2KPT_N_LEVELS; ++k) { + if (sum_w[k] > 1e-12f) { + float new_level = sum_wt[k] / sum_w[k]; + max_delta = fmaxf(max_delta, fabsf(new_level - levels[k])); + levels[k] = new_level; + } + } + if (max_delta < 1e-10f) break; + + // Sort levels + for (int k = 1; k < Q2KPT_N_LEVELS; ++k) { + float v = levels[k]; + int m = k - 1; + while (m >= 0 && levels[m] > v) { + levels[m + 1] = levels[m]; + m--; + } + levels[m + 1] = v; + } + } + + memcpy(levels_out, levels, Q2KPT_N_LEVELS * sizeof(float)); + q2kpt_set_levels(levels); + free(bin_sum_w); + free(bin_sum_wt); +} + +void dequantize_row_q2_kpt(const block_q2_kpt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + assert(k % QK_K == 0); + const int64_t nb = k / QK_K; + const float * lv = (const float *)levels; + GGML_ASSERT(lv != NULL && "Q2_KPT levels not set for tensor"); + + for (int i = 0; i < nb; i++) { + // Per-block levels: block i uses lv[i*4 + 0..3] + const float * block_lv = lv + i * Q2KPT_N_LEVELS; + + // Precompute mapped levels: ml[k] = levels[k] * 3.0 + float ml[Q2KPT_N_LEVELS]; + for (int j = 0; j < Q2KPT_N_LEVELS; ++j) { + ml[j] = block_lv[j] * 3.0f; + } + + const float d_all = GGML_FP16_TO_FP32(x[i].d); + const float m_all = GGML_FP16_TO_FP32(x[i].dmin); + const uint8_t * q = x[i].qs; + + int is = 0; + for (int n = 0; n < QK_K; n += 128) { + int shift = 0; + for (int j = 0; j < 4; ++j) { + uint8_t sc = x[i].scales[is++]; + float dl = d_all * (sc & 0xF); + float mn = m_all * (sc >> 4); + for (int l = 0; l < 16; ++l) { + int idx = (q[l] >> shift) & 3; + *y++ = dl * ml[idx] - mn; + } + + sc = x[i].scales[is++]; + dl = d_all * (sc & 0xF); + mn = m_all * (sc >> 4); + for (int l = 0; l < 16; ++l) { + int idx = (q[l + 16] >> shift) & 3; + *y++ = dl * ml[idx] - mn; + } + + shift += 2; + } + q += 32; + } + } +} + +// Helper: find optimal (scale, min) for non-uniform mapped levels with offset. +// mapped_levels[k] = levels[k]*3, k=0..3. +// Model: x[i] ≈ scale * ml[L[i]] - min_offset, with min_offset >= 0. +// Returns the per-sub-block scale; *the_min receives the min offset. +// L[i] gets the best level index [0..3]. +static float make_q2kpt_quants(int n, + const float * GGML_RESTRICT x, + uint8_t * GGML_RESTRICT L, + float * GGML_RESTRICT the_min, + const float * mapped_levels, + const float * weight) { + // Precompute boundaries for nearest-level assignment + float bounds[Q2KPT_N_LEVELS - 1]; + for (int k = 0; k < Q2KPT_N_LEVELS - 1; ++k) { + bounds[k] = 0.5f * (mapped_levels[k] + mapped_levels[k + 1]); + } + + float xmin = x[0], xmax = x[0]; + for (int i = 1; i < n; ++i) { + if (x[i] < xmin) xmin = x[i]; + if (x[i] > xmax) xmax = x[i]; + } + if (xmin > 0) xmin = 0; + if (xmax <= xmin) { + for (int i = 0; i < n; ++i) L[i] = 0; + *the_min = -xmin; + return 0.f; + } + + float ml_max = mapped_levels[Q2KPT_N_LEVELS - 1]; + + float best_scale = 0, best_min = 0; + float best_obj = 0; + bool first = true; + + // Grid search: try multiple trial scales + for (int is = -9; is <= 36; ++is) { + float iscale = (ml_max + 0.1f * is) / (xmax - xmin); + float trial_min = -xmin; + + float sum_l = 0, sum_l2 = 0, sum_lx = 0; + float sum_x = 0, sum_w = 0; + for (int i = 0; i < n; ++i) { + float scaled = iscale * (x[i] + trial_min); + int best_k = (scaled > bounds[0]) + (scaled > bounds[1]) + (scaled > bounds[2]); + float ml_val = mapped_levels[best_k]; + float w = weight ? weight[i] : x[i] * x[i]; + sum_l += w * ml_val; + sum_l2 += w * ml_val * ml_val; + sum_lx += w * ml_val * x[i]; + sum_x += w * x[i]; + sum_w += w; + } + + float D = sum_w * sum_l2 - sum_l * sum_l; + if (D > 0) { + float this_scale = (sum_w * sum_lx - sum_x * sum_l) / D; + float this_min = (sum_l2 * sum_x - sum_l * sum_lx) / D; + + if (this_min > 0) { + this_min = 0; + this_scale = sum_lx / sum_l2; + } + + float cur_error = 0; + for (int i = 0; i < n; ++i) { + float scaled = (x[i] - this_min) / (this_scale > 1e-15f ? this_scale : 1e-15f); + int best_k = (scaled > bounds[0]) + (scaled > bounds[1]) + (scaled > bounds[2]); + float diff = this_scale * mapped_levels[best_k] + this_min - x[i]; + float w = weight ? weight[i] : x[i] * x[i]; + cur_error += w * diff * diff; + } + + if (first || cur_error < best_obj) { + best_obj = cur_error; + best_scale = this_scale; + best_min = this_min; + first = false; + } + } + } + + // Inner EM refinement from best found by grid search: iterate + // assign→refit→assign until convergence (fixes Problem 3) + for (int refine = 0; refine < 8; ++refine) { + float sum_l = 0, sum_l2 = 0, sum_lx = 0, sum_x = 0, sum_w = 0; + for (int i = 0; i < n; ++i) { + float scaled = (x[i] - best_min) / (best_scale > 1e-15f ? best_scale : 1e-15f); + int best_k = (scaled > bounds[0]) + (scaled > bounds[1]) + (scaled > bounds[2]); + float ml_val = mapped_levels[best_k]; + float w = weight ? weight[i] : x[i] * x[i]; + sum_l += w * ml_val; + sum_l2 += w * ml_val * ml_val; + sum_lx += w * ml_val * x[i]; + sum_x += w * x[i]; + sum_w += w; + } + float D = sum_w * sum_l2 - sum_l * sum_l; + if (D <= 0) break; + float new_scale = (sum_w * sum_lx - sum_x * sum_l) / D; + float new_min = (sum_l2 * sum_x - sum_l * sum_lx) / D; + if (new_min > 0) { + new_min = 0; + new_scale = sum_l2 > 1e-30f ? sum_lx / sum_l2 : 0.f; + } + float cur_error = 0; + for (int i = 0; i < n; ++i) { + float scaled = (x[i] - new_min) / (new_scale > 1e-15f ? new_scale : 1e-15f); + int best_k = (scaled > bounds[0]) + (scaled > bounds[1]) + (scaled > bounds[2]); + float diff = new_scale * mapped_levels[best_k] + new_min - x[i]; + float w = weight ? weight[i] : x[i] * x[i]; + cur_error += w * diff * diff; + } + if (cur_error >= best_obj - 1e-12f * best_obj) { break; } + best_obj = cur_error; + best_scale = new_scale; + best_min = new_min; + } + + // Final assignment with best (scale, min) + for (int i = 0; i < n; ++i) { + float scaled = (x[i] - best_min) / (best_scale > 1e-15f ? best_scale : 1e-15f); + L[i] = (scaled > bounds[0]) + (scaled > bounds[1]) + (scaled > bounds[2]); + } + + *the_min = -best_min; + return best_scale; +} + +static void quantize_row_q2_kpt_impl(const float * GGML_RESTRICT x, + block_q2_kpt * GGML_RESTRICT y, + int64_t n_per_row, + const float * GGML_RESTRICT quant_weights, + const float * GGML_RESTRICT imatrix, + float * GGML_RESTRICT block_levels) { + assert(n_per_row % QK_K == 0); + const int nb = n_per_row / QK_K; + GGML_ASSERT(block_levels != NULL && "block_levels buffer must be provided"); + + for (int i = 0; i < nb; ++i) { + const float * block_x = x + i * QK_K; + + // Per-block quant_weights and imatrix slices (fixes the imatrix offset bug: + // previously the full-row imatrix was passed and indexed from 0 for every block) + const float * block_qw = quant_weights ? quant_weights + i * QK_K : NULL; + const float * block_im = imatrix ? imatrix + i * QK_K : NULL; + + float sumx2 = 0.0f; + for (int j = 0; j < QK_K; ++j) sumx2 += block_x[j] * block_x[j]; + float sigma2 = sumx2 / QK_K; + + float block_lv[Q2KPT_N_LEVELS]; + // Runs k-means init + EM loop; fills block_lv AND writes the best + // quantized block into y[i] as a side-effect. + q2kpt_optimize_block_levels(block_x, &y[i], block_qw, sigma2, block_lv); + + memcpy(block_levels + i * Q2KPT_N_LEVELS, block_lv, Q2KPT_N_LEVELS * sizeof(float)); + + (void)block_im; // imatrix is folded into block_qw; retained for future use + } +} + +size_t quantize_q2_kpt(const float * GGML_RESTRICT src, + void * GGML_RESTRICT dst, + int64_t start_row, + int64_t nrow, + int64_t n_per_row, + const float * imatrix) { + size_t row_size = ggml_row_size(GGML_TYPE_Q2_KPT, n_per_row); + char * qrow = (char *) dst; + const int nb = (int)(n_per_row / QK_K); + const size_t total_levels = (size_t)nrow * nb * Q2KPT_N_LEVELS; + const size_t levels_needed = (size_t)(start_row + nrow) * nb * Q2KPT_N_LEVELS; + + // Ensure buffer is large enough (should have been pre-allocated via q2kpt_prepare_levels) + if (levels_needed > q2kpt_max_levels) { + q2kpt_block_levels = (float *) realloc(q2kpt_block_levels, levels_needed * sizeof(float)); + q2kpt_max_levels = levels_needed; + } + q2kpt_cur_levels = levels_needed; + + // Temporary buffer for one row's block levels + float * row_block_levels = (float *) malloc(nb * Q2KPT_N_LEVELS * sizeof(float)); + + for (int64_t row = 0; row < nrow; ++row) { + // Quantize row with per-block trained levels + quantize_row_q2_kpt_impl(src, (block_q2_kpt *) qrow, n_per_row, imatrix, imatrix, row_block_levels); + // Copy this row's block levels to the global buffer at the correct offset + memcpy(q2kpt_block_levels + (start_row + row) * nb * Q2KPT_N_LEVELS, row_block_levels, + nb * Q2KPT_N_LEVELS * sizeof(float)); + src += n_per_row; + qrow += row_size; + } + free(row_block_levels); + return nrow * row_size; +} + +// Train per-row levels for all rows of a tensor and write to out_levels. +// out_levels must be pre-allocated to nrow * Q2KPT_N_LEVELS floats. +void q2kpt_train_all_row_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float * out_levels) { + for (int64_t r = 0; r < nrow; ++r) { + q2kpt_train_levels(data + r * n_per_row, 1, n_per_row, imatrix, + out_levels + r * Q2KPT_N_LEVELS); + } +} + +void quantize_row_q2_kpt_ref(const float * GGML_RESTRICT x, block_q2_kpt * GGML_RESTRICT y, int64_t k) { + assert(k % QK_K == 0); + quantize_q2_kpt(x, y, 0, 1, k, NULL); +} + + +// ============================================================================ +// ============================================================================ +// IQ2_TQ: E8 Lattice Vector Quantization (2.1875 bpw) +// ============================================================================ +// +// --- Per-tensor grid state --- +static int8_t iq2tq_active_grid[16][4]; +static bool iq2tq_grid_set = false; + +void iq2tq_set_grid(const int8_t grid[64]) { + memcpy(iq2tq_active_grid, grid, 64); + iq2tq_grid_set = true; +} + +const int8_t * iq2tq_get_grid(void) { + return iq2tq_grid_set ? (const int8_t *)iq2tq_active_grid : NULL; +} + +// --- Default grid: trained via K-means on optimal per-group level-sets --- +// recon[j] = d * IQ2TQ_GRID_SCALE * grid[si][qi] +// Only used when no per-tensor grid is available (fallback) +static const int8_t iq2tq_grid_default[16][4] = { + {-20, -8, -2, 6}, // 0: {-2.50, -1.00, -0.25, 0.75} + {-14, -8, -2, 4}, // 1: {-1.75, -1.00, -0.25, 0.50} + {-16, -10, 0, 12}, // 2: {-2.00, -1.25, 0.00, 1.50} + {-14, -4, 2, 8}, // 3: {-1.75, -0.50, 0.25, 1.00} + {-20, -4, 4, 12}, // 4: {-2.50, -0.50, 0.50, 1.50} + {-8, -4, 0, 4}, // 5: {-1.00, -0.50, 0.00, 0.50} + {-8, -4, 0, 8}, // 6: {-1.00, -0.50, 0.00, 1.00} + {-12, -6, 2, 12}, // 7: {-1.50, -0.75, 0.25, 1.50} + {-4, -2, 2, 4}, // 8: {-0.50, -0.25, 0.25, 0.50} + {-10, -2, 4, 8}, // 9: {-1.25, -0.25, 0.50, 1.00} + {-16, -6, 4, 20}, // 10: {-2.00, -0.75, 0.50, 2.50} + {-12, -2, 6, 14}, // 11: {-1.50, -0.25, 0.75, 1.75} + {-8, -2, 4, 14}, // 12: {-1.00, -0.25, 0.50, 1.75} + {-4, 0, 4, 8}, // 13: {-0.50, 0.00, 0.50, 1.00} + {-8, -2, 6, 22}, // 14: {-1.00, -0.25, 0.75, 2.75} + {-4, 2, 8, 14}, // 15: {-0.50, 0.25, 1.00, 1.75} +}; + +// Get current grid: per-tensor if set, else default +static inline const int8_t (*iq2tq_cur_grid(void))[4] { + return iq2tq_grid_set ? (const int8_t (*)[4])iq2tq_active_grid + : (const int8_t (*)[4])iq2tq_grid_default; +} + +// Unpack 2-bit index from qs array: element j is in bits (j%4)*2..(j%4)*2+1 of byte j/4 +static inline int iq2tq_get_qi(const uint8_t * qs, int j) { + return (qs[j / 4] >> ((j % 4) * 2)) & 3; +} + +// Pack 2-bit index into qs array +static inline void iq2tq_set_qi(uint8_t * qs, int j, int val) { + int byte_idx = j / 4; + int bit_off = (j % 4) * 2; + qs[byte_idx] = (qs[byte_idx] & ~(3 << bit_off)) | ((val & 3) << bit_off); +} + +// Find nearest qi in sorted grid entry via midpoint boundaries +static inline int iq2tq_nearest_qi(float xn, const int8_t * g) { + if (xn <= 0.5f * (g[0] + g[1])) return 0; + if (xn <= 0.5f * (g[1] + g[2])) return 1; + if (xn <= 0.5f * (g[2] + g[3])) return 2; + return 3; +} + +// Dequantization — 2-bit with asymmetric grid per group +void dequantize_row_iq2_tq(const block_iq2_tq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + const int8_t (*grid)[4] = levels ? (const int8_t (*)[4])levels + : (const int8_t (*)[4])iq2tq_grid_default; + const int nb = k / QK_K; + + for (int i = 0; i < nb; ++i) { + const block_iq2_tq * b = x + i; + const float dq = GGML_FP16_TO_FP32(b->d) * IQ2TQ_GRID_SCALE; + + for (int g = 0; g < IQ2TQ_N_GROUPS; ++g) { + int si = (b->scales[g / 2] >> (4 * (g % 2))) & 0xF; + const int8_t * ge = grid[si]; + + for (int k2 = 0; k2 < 8; ++k2) { + int j = g * 8 + k2; + int qi = iq2tq_get_qi(b->qs, j); + y[i * QK_K + j] = dq * (float)ge[qi]; + } + } + } +} + +// Reference quantization +void quantize_row_iq2_tq_ref(const float * GGML_RESTRICT x, block_iq2_tq * GGML_RESTRICT y, int64_t k) { + quantize_iq2_tq(x, y, 1, k, NULL); +} + +// Quantize one row — 2-bit with asymmetric grid + OLS d +static void quantize_row_iq2_tq_impl( + const float * GGML_RESTRICT x, + block_iq2_tq * GGML_RESTRICT y, + int64_t n_per_row, + const float * GGML_RESTRICT quant_weights +) { + assert(n_per_row % QK_K == 0); + const int8_t (*grid)[4] = iq2tq_cur_grid(); + const int nb = n_per_row / QK_K; + + for (int bi = 0; bi < nb; ++bi) { + const float * xb = x + bi * QK_K; + block_iq2_tq * yb = y + bi; + memset(yb, 0, sizeof(block_iq2_tq)); + + // Compute sigma2 for importance weighting (higher = more uniform, lower = more magnitude-biased) + float sigma2 = 0; + for (int j = 0; j < QK_K; ++j) sigma2 += xb[j] * xb[j]; + sigma2 = 8.0f * sigma2 / QK_K; + + float amax = 0; + for (int j = 0; j < QK_K; ++j) { + float av = fabsf(xb[j]); + if (av > amax) amax = av; + } + if (amax < 1e-15f) continue; + + // Initial d: max grid value ~24 in int8, recon = d * 0.125 * 24 = 3d → d = amax/3 + float d = amax / 3.0f; + + uint8_t qs[64] = {0}; + uint8_t scales_out[16] = {0}; + int grid_idx[IQ2TQ_N_GROUPS]; // chosen grid entry per group + + // Phase 1: For each group, try all 16 grid entries, pick best + float dq = d * IQ2TQ_GRID_SCALE; + float inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; + for (int g = 0; g < IQ2TQ_N_GROUPS; ++g) { + float best_err = 1e30f; + int best_si = 3; // default: centered medium + + for (int si = 0; si < 16; ++si) { + const int8_t * ge = grid[si]; + float g_err = 0; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + float xn = xb[j] * inv_dq; + int qi = iq2tq_nearest_qi(xn, ge); + float recon = dq * (float)ge[qi]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float err = xb[j] - recon; + g_err += wk * err * err; + } + if (g_err < best_err) { best_err = g_err; best_si = si; } + } + grid_idx[g] = best_si; + + // Quantize elements with chosen grid + const int8_t * ge = grid[best_si]; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + iq2tq_set_qi(qs, j, iq2tq_nearest_qi(xb[j] * inv_dq, ge)); + } + } + + // Iterative refinement + for (int iter = 0; iter < 12; ++iter) { + // Re-fit d via weighted OLS: d = sum(w*x*g) / (GRID_SCALE * sum(w*g*g)) + double sumxg = 0, sumgg = 0; + for (int j = 0; j < QK_K; ++j) { + int g = j / 8; + float gval = (float)grid[grid_idx[g]][iq2tq_get_qi(qs, j)]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + sumxg += wk * xb[j] * gval; + sumgg += wk * gval * gval; + } + d = (sumgg > 0) ? (float)(sumxg / (IQ2TQ_GRID_SCALE * sumgg)) : d; + + // Re-optimize grids and re-quantize + dq = d * IQ2TQ_GRID_SCALE; + inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; + memset(scales_out, 0, 16); + for (int g = 0; g < IQ2TQ_N_GROUPS; ++g) { + float best_err = 1e30f; + int best_si = 3; + + for (int si = 0; si < 16; ++si) { + const int8_t * ge = grid[si]; + float g_err = 0; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + float xn = xb[j] * inv_dq; + int qi = iq2tq_nearest_qi(xn, ge); + float recon = dq * (float)ge[qi]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float err = xb[j] - recon; + g_err += wk * err * err; + } + if (g_err < best_err) { best_err = g_err; best_si = si; } + } + scales_out[g / 2] |= (best_si << (4 * (g % 2))); + grid_idx[g] = best_si; + + const int8_t * ge = grid[best_si]; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + iq2tq_set_qi(qs, j, iq2tq_nearest_qi(xb[j] * inv_dq, ge)); + } + } + } + + // Final OLS d + { + double sumxg = 0, sumgg = 0; + for (int j = 0; j < QK_K; ++j) { + int g = j / 8; + float gval = (float)grid[grid_idx[g]][iq2tq_get_qi(qs, j)]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + sumxg += wk * xb[j] * gval; + sumgg += wk * gval * gval; + } + d = (sumgg > 0) ? (float)(sumxg / (IQ2TQ_GRID_SCALE * sumgg)) : d; + } + + // Multi-d search: try nearby d values and re-optimize grids + { + float best_d = d; + float best_total_err = 1e30f; + uint8_t best_scales[16], best_qs[64]; + int best_grid_idx[IQ2TQ_N_GROUPS]; + static const float d_factors[] = {0.8f, 0.85f, 0.9f, 0.925f, 0.95f, 0.975f, 1.0f, 1.025f, 1.05f, 1.075f, 1.1f, 1.15f, 1.2f}; + static const int n_d_factors = sizeof(d_factors) / sizeof(d_factors[0]); + + for (int df = 0; df < n_d_factors; ++df) { + float td = d * d_factors[df]; + float tdq = td * IQ2TQ_GRID_SCALE; + float tinv = (fabsf(tdq) > 1e-15f) ? 1.0f / tdq : 0.0f; + uint8_t tscales[16] = {0}; + uint8_t tqs[64] = {0}; + int tgrid[IQ2TQ_N_GROUPS]; + float total_err = 0; + + for (int g = 0; g < IQ2TQ_N_GROUPS; ++g) { + float best_err = 1e30f; + int best_si = 3; + for (int si = 0; si < 16; ++si) { + const int8_t * ge = grid[si]; + float g_err = 0; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + float xn = xb[j] * tinv; + int qi = iq2tq_nearest_qi(xn, ge); + float recon = tdq * (float)ge[qi]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float err = xb[j] - recon; + g_err += wk * err * err; + } + if (g_err < best_err) { best_err = g_err; best_si = si; } + } + tscales[g / 2] |= (best_si << (4 * (g % 2))); + tgrid[g] = best_si; + const int8_t * ge = grid[best_si]; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + iq2tq_set_qi(tqs, j, iq2tq_nearest_qi(xb[j] * tinv, ge)); + } + total_err += best_err; + } + + if (total_err < best_total_err) { + best_total_err = total_err; + best_d = td; + memcpy(best_scales, tscales, 16); + memcpy(best_qs, tqs, 64); + memcpy(best_grid_idx, tgrid, sizeof(tgrid)); + } + } + d = best_d; + memcpy(scales_out, best_scales, 16); + memcpy(qs, best_qs, 64); + memcpy(grid_idx, best_grid_idx, sizeof(grid_idx)); + } + + // Post multi-d refinement: 2 more OLS+grid iterations from the best d + for (int iter = 0; iter < 2; ++iter) { + double sumxg = 0, sumgg = 0; + for (int j = 0; j < QK_K; ++j) { + int g = j / 8; + float gval = (float)grid[grid_idx[g]][iq2tq_get_qi(qs, j)]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + sumxg += wk * xb[j] * gval; + sumgg += wk * gval * gval; + } + d = (sumgg > 0) ? (float)(sumxg / (IQ2TQ_GRID_SCALE * sumgg)) : d; + + dq = d * IQ2TQ_GRID_SCALE; + inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; + memset(scales_out, 0, 16); + for (int g = 0; g < IQ2TQ_N_GROUPS; ++g) { + float best_err = 1e30f; + int best_si = 3; + for (int si = 0; si < 16; ++si) { + const int8_t * ge = grid[si]; + float g_err = 0; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + float xn = xb[j] * inv_dq; + int qi = iq2tq_nearest_qi(xn, ge); + float recon = dq * (float)ge[qi]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float err = xb[j] - recon; + g_err += wk * err * err; + } + if (g_err < best_err) { best_err = g_err; best_si = si; } + } + scales_out[g / 2] |= (best_si << (4 * (g % 2))); + grid_idx[g] = best_si; + const int8_t * ge = grid[best_si]; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + iq2tq_set_qi(qs, j, iq2tq_nearest_qi(xb[j] * inv_dq, ge)); + } + } + } + + // Write result + yb->d = GGML_FP32_TO_FP16(d); + memcpy(yb->scales, scales_out, 16); + memcpy(yb->qs, qs, 64); + } +} + +// Public quantize function +size_t quantize_iq2_tq(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, + int64_t nrows, int64_t n_per_row, const float * imatrix) { + size_t row_size = ggml_row_size(GGML_TYPE_IQ2_TQ, n_per_row); + char * qrow = (char *)dst; + for (int64_t row = 0; row < nrows; ++row) { + quantize_row_iq2_tq_impl(src, (block_iq2_tq *)qrow, n_per_row, imatrix); + src += n_per_row; + qrow += row_size; + } + return nrows * row_size; +} + +// Train per-tensor grid via K-means on optimal per-group level-sets +void iq2tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, int8_t grid_out[64]) { + assert(n_per_row % QK_K == 0); + const int nb_per_row = n_per_row / QK_K; + const int max_groups = 200000; + + // Collect optimal per-group level-sets with importance weights + float (*sets)[4] = (float (*)[4])malloc(max_groups * 4 * sizeof(float)); + float * set_weights = (float *)malloc(max_groups * sizeof(float)); + int n_sets = 0; + + for (int64_t row = 0; row < nrow && n_sets < max_groups - IQ2TQ_N_GROUPS; row++) { + const float * xrow = data + row * n_per_row; + for (int bi = 0; bi < nb_per_row && n_sets < max_groups; bi++) { + const float * xb = xrow + bi * QK_K; + + // Compute d = amax / 3 + float amax = 0; + for (int j = 0; j < QK_K; j++) { + float av = fabsf(xb[j]); + if (av > amax) amax = av; + } + if (amax < 1e-15f) continue; + float inv_dq = 1.0f / (amax / 3.0f * IQ2TQ_GRID_SCALE); + + for (int g = 0; g < IQ2TQ_N_GROUPS && n_sets < max_groups; g++) { + // Compute importance weight for this group + float gw = 1.0f; + if (imatrix) { + gw = 0; + for (int k = 0; k < 8; k++) gw += imatrix[g * 8 + k]; + gw /= 8.0f; + if (gw < 1e-10f) gw = 1e-10f; + } + + // Normalize group values by d*GRID_SCALE + float xn[8]; + for (int k = 0; k < 8; k++) { + xn[k] = xb[g * 8 + k] * inv_dq; + } + + // Sort + for (int i = 0; i < 7; i++) + for (int j2 = i + 1; j2 < 8; j2++) + if (xn[j2] < xn[i]) { float t = xn[i]; xn[i] = xn[j2]; xn[j2] = t; } + + // Optimal 4-level: split sorted 8 values into 4 pairs, compute centroids + // Use Lloyd-Max: iterate assignment + centroid recomputation + float L[4] = { xn[0], xn[2], xn[5], xn[7] }; + for (int iter = 0; iter < 15; iter++) { + float b01 = 0.5f * (L[0] + L[1]); + float b12 = 0.5f * (L[1] + L[2]); + float b23 = 0.5f * (L[2] + L[3]); + float sum[4] = {0}; int cnt[4] = {0}; + for (int k = 0; k < 8; k++) { + int qi = (xn[k] <= b01) ? 0 : (xn[k] <= b12) ? 1 : (xn[k] <= b23) ? 2 : 3; + sum[qi] += xn[k]; cnt[qi]++; + } + int converged = 1; + for (int q = 0; q < 4; q++) { + float newL = (cnt[q] > 0) ? sum[q] / cnt[q] : L[q]; + if (fabsf(newL - L[q]) > 0.01f) converged = 0; + L[q] = newL; + } + if (converged) break; + } + + memcpy(sets[n_sets], L, 4 * sizeof(float)); + set_weights[n_sets] = gw; + n_sets++; + } + } + } + + // K-means clustering of level-sets into 16 clusters + float centroids[16][4]; + // Initialize: spread evenly + for (int i = 0; i < 16; i++) { + int idx = (int)((int64_t)i * n_sets / 16); + memcpy(centroids[i], sets[idx], 4 * sizeof(float)); + } + + int * assign = (int *)calloc(n_sets, sizeof(int)); + for (int iter = 0; iter < 100; iter++) { + int changed = 0; + for (int i = 0; i < n_sets; i++) { + float best_dist = 1e30f; + int best_c = 0; + for (int c = 0; c < 16; c++) { + float dist = 0; + for (int d = 0; d < 4; d++) { + float diff = sets[i][d] - centroids[c][d]; + dist += diff * diff; + } + if (dist < best_dist) { best_dist = dist; best_c = c; } + } + if (assign[i] != best_c) { assign[i] = best_c; changed++; } + } + + double sum[16][4] = {{0}}; + double wcnt[16] = {0}; + for (int i = 0; i < n_sets; i++) { + int c = assign[i]; + float w = set_weights[i]; + for (int d = 0; d < 4; d++) sum[c][d] += w * sets[i][d]; + wcnt[c] += w; + } + for (int c = 0; c < 16; c++) { + if (wcnt[c] > 0) { + for (int d = 0; d < 4; d++) centroids[c][d] = (float)(sum[c][d] / wcnt[c]); + } + } + if (changed == 0) break; + } + + // Sort centroids by sum of levels for consistent ordering + for (int i = 0; i < 15; i++) { + for (int j2 = i + 1; j2 < 16; j2++) { + float si2 = centroids[i][0] + centroids[i][1] + centroids[i][2] + centroids[i][3]; + float sj = centroids[j2][0] + centroids[j2][1] + centroids[j2][2] + centroids[j2][3]; + if (sj < si2) { + float tmp[4]; + memcpy(tmp, centroids[i], 16); + memcpy(centroids[i], centroids[j2], 16); + memcpy(centroids[j2], tmp, 16); + } + } + } + + // Round to int8 and output + for (int i = 0; i < 16; i++) { + for (int d = 0; d < 4; d++) { + float v = centroids[i][d]; + int8_t iv = (int8_t)(v > 0 ? v + 0.5f : v - 0.5f); + grid_out[i * 4 + d] = iv; + } + } + + free(sets); + free(set_weights); + free(assign); +} + +// ============================================================================ +// IQ3_TQ: 3-bit scalar quantization with per-tensor trained grid table +// ============================================================================ + +// --- Per-tensor grid state (16 entries × 8 levels = 128 bytes) --- +static int8_t iq3tq_active_grid[16][IQ3TQ_N_LEVELS]; +static bool iq3tq_grid_set = false; + +void iq3tq_set_grid(const int8_t grid[IQ3TQ_GRID_SIZE]) { + memcpy(iq3tq_active_grid, grid, IQ3TQ_GRID_SIZE); + iq3tq_grid_set = true; +} + +const int8_t * iq3tq_get_grid(void) { + return iq3tq_grid_set ? (const int8_t *)iq3tq_active_grid : NULL; +} + +// Default grid: 16 entries × 8 int8 levels (fallback only, per-tensor training is used in practice) +static const int8_t iq3tq_grid_default[16][IQ3TQ_N_LEVELS] = { + {-24,-18,-12, -6, 0, 6, 12, 18}, + {-20,-15,-10, -5, 0, 5, 10, 15}, + {-16,-12, -8, -4, 0, 4, 8, 12}, + {-12, -8, -4, -2, 0, 2, 4, 8}, + {-24,-16, -8, -2, 2, 6, 10, 14}, + {-14,-10, -6, -2, 2, 8, 16, 24}, + {-20,-14, -8, -4, 0, 4, 10, 18}, + {-18,-10, -4, 0, 4, 8, 14, 20}, + { -8, -6, -4, -2, 0, 2, 4, 6}, + {-10, -6, -4, -2, 2, 4, 6, 10}, + {-22,-14, -6, -2, 2, 6, 14, 22}, + {-16, -8, -4, -2, 0, 4, 8, 16}, + {-24,-20,-16,-12, -8, -4, 0, 4}, + { -4, 0, 4, 8, 12, 16, 20, 24}, + {-20,-16,-10, -4, 4, 10, 16, 20}, + {-12, -8, -6, -2, 2, 6, 8, 12}, +}; + +static inline const int8_t (*iq3tq_cur_grid(void))[IQ3TQ_N_LEVELS] { + return iq3tq_grid_set ? (const int8_t (*)[IQ3TQ_N_LEVELS])iq3tq_active_grid + : (const int8_t (*)[IQ3TQ_N_LEVELS])iq3tq_grid_default; +} + +// 3-bit pack/unpack +static inline int iq3tq_get_qi(const uint8_t * qs, int j) { + int bit_pos = j * 3; + int byte_idx = bit_pos >> 3; + int bit_off = bit_pos & 7; + uint16_t val = qs[byte_idx]; + if (bit_off > 5) val |= ((uint16_t)qs[byte_idx + 1] << 8); + return (val >> bit_off) & 7; +} + +static inline void iq3tq_set_qi(uint8_t * qs, int j, int val) { + int bit_pos = j * 3; + int byte_idx = bit_pos >> 3; + int bit_off = bit_pos & 7; + qs[byte_idx] &= ~((7 << bit_off) & 0xFF); + qs[byte_idx] |= ((val & 7) << bit_off) & 0xFF; + if (bit_off > 5) { + qs[byte_idx + 1] &= ~((7 >> (8 - bit_off)) & 0xFF); + qs[byte_idx + 1] |= ((val & 7) >> (8 - bit_off)) & 0xFF; + } +} + +// Find nearest qi in sorted 8-level grid entry +static inline int iq3tq_nearest_qi(float xn, const int8_t * g) { + for (int i = 0; i < 7; ++i) { + if (xn <= 0.5f * (g[i] + g[i+1])) return i; + } + return 7; +} + +// Dequantization +void dequantize_row_iq3_tq(const block_iq3_tq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + const int8_t (*grid)[IQ3TQ_N_LEVELS] = levels ? (const int8_t (*)[IQ3TQ_N_LEVELS])levels + : (const int8_t (*)[IQ3TQ_N_LEVELS])iq3tq_grid_default; + const int nb = k / QK_K; + + for (int i = 0; i < nb; ++i) { + const block_iq3_tq * b = x + i; + const float dq = GGML_FP16_TO_FP32(b->d) * IQ3TQ_GRID_SCALE; + + for (int g = 0; g < IQ3TQ_N_GROUPS; ++g) { + int si = (b->scales[g / 2] >> (4 * (g % 2))) & 0xF; + const int8_t * ge = grid[si]; + + for (int k2 = 0; k2 < 8; ++k2) { + int j = g * 8 + k2; + int qi = iq3tq_get_qi(b->qs, j); + y[i * QK_K + j] = dq * (float)ge[qi]; + } + } + } +} + +void quantize_row_iq3_tq_ref(const float * GGML_RESTRICT x, block_iq3_tq * GGML_RESTRICT y, int64_t k) { + quantize_iq3_tq(x, y, 1, k, NULL); +} + +// Quantize one row — 3-bit with asymmetric grid + OLS d +static void quantize_row_iq3_tq_impl( + const float * GGML_RESTRICT x, + block_iq3_tq * GGML_RESTRICT y, + int64_t n_per_row, + const float * GGML_RESTRICT quant_weights +) { + assert(n_per_row % QK_K == 0); + const int8_t (*grid)[IQ3TQ_N_LEVELS] = iq3tq_cur_grid(); + const int nb = n_per_row / QK_K; + + for (int bi = 0; bi < nb; ++bi) { + const float * xb = x + bi * QK_K; + block_iq3_tq * yb = y + bi; + memset(yb, 0, sizeof(block_iq3_tq)); + + float sigma2 = 0; + for (int j = 0; j < QK_K; ++j) sigma2 += xb[j] * xb[j]; + sigma2 = 8.0f * sigma2 / QK_K; + + float amax = 0; + for (int j = 0; j < QK_K; ++j) { + float av = fabsf(xb[j]); + if (av > amax) amax = av; + } + if (amax < 1e-15f) continue; + + float d = amax / 3.0f; + + uint8_t qs[96] = {0}; + uint8_t scales_out[16] = {0}; + int grid_idx[IQ3TQ_N_GROUPS]; + + // Phase 1: For each group, try all 16 grid entries, pick best + float dq = d * IQ3TQ_GRID_SCALE; + float inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; + for (int g = 0; g < IQ3TQ_N_GROUPS; ++g) { + float best_err = 1e30f; + int best_si = 0; + + for (int si = 0; si < 16; ++si) { + const int8_t * ge = grid[si]; + float g_err = 0; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + float xn = xb[j] * inv_dq; + int qi = iq3tq_nearest_qi(xn, ge); + float recon = dq * (float)ge[qi]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float err = xb[j] - recon; + g_err += wk * err * err; + } + if (g_err < best_err) { best_err = g_err; best_si = si; } + } + grid_idx[g] = best_si; + const int8_t * ge = grid[best_si]; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + iq3tq_set_qi(qs, j, iq3tq_nearest_qi(xb[j] * inv_dq, ge)); + } + } + + // Iterative refinement + for (int iter = 0; iter < 12; ++iter) { + double sumxg = 0, sumgg = 0; + for (int j = 0; j < QK_K; ++j) { + int g = j / 8; + float gval = (float)grid[grid_idx[g]][iq3tq_get_qi(qs, j)]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + sumxg += wk * xb[j] * gval; + sumgg += wk * gval * gval; + } + d = (sumgg > 0) ? (float)(sumxg / (IQ3TQ_GRID_SCALE * sumgg)) : d; + + dq = d * IQ3TQ_GRID_SCALE; + inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; + memset(scales_out, 0, 16); + for (int g = 0; g < IQ3TQ_N_GROUPS; ++g) { + float best_err = 1e30f; + int best_si = 0; + for (int si = 0; si < 16; ++si) { + const int8_t * ge = grid[si]; + float g_err = 0; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + float xn = xb[j] * inv_dq; + int qi = iq3tq_nearest_qi(xn, ge); + float recon = dq * (float)ge[qi]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float err = xb[j] - recon; + g_err += wk * err * err; + } + if (g_err < best_err) { best_err = g_err; best_si = si; } + } + scales_out[g / 2] |= (best_si << (4 * (g % 2))); + grid_idx[g] = best_si; + const int8_t * ge = grid[best_si]; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + iq3tq_set_qi(qs, j, iq3tq_nearest_qi(xb[j] * inv_dq, ge)); + } + } + } + + // Final OLS d + { + double sumxg = 0, sumgg = 0; + for (int j = 0; j < QK_K; ++j) { + int g = j / 8; + float gval = (float)grid[grid_idx[g]][iq3tq_get_qi(qs, j)]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + sumxg += wk * xb[j] * gval; + sumgg += wk * gval * gval; + } + d = (sumgg > 0) ? (float)(sumxg / (IQ3TQ_GRID_SCALE * sumgg)) : d; + } + + // Multi-d search + { + float best_d = d; + float best_total_err = 1e30f; + uint8_t best_scales[16], best_qs[96]; + int best_grid_idx[IQ3TQ_N_GROUPS]; + static const float d_factors[] = {0.8f, 0.85f, 0.9f, 0.925f, 0.95f, 0.975f, 1.0f, 1.025f, 1.05f, 1.075f, 1.1f, 1.15f, 1.2f}; + static const int n_d_factors = sizeof(d_factors) / sizeof(d_factors[0]); + + for (int df = 0; df < n_d_factors; ++df) { + float td = d * d_factors[df]; + float tdq = td * IQ3TQ_GRID_SCALE; + float tinv = (fabsf(tdq) > 1e-15f) ? 1.0f / tdq : 0.0f; + uint8_t tscales[16] = {0}; + uint8_t tqs[96] = {0}; + int tgrid[IQ3TQ_N_GROUPS]; + float total_err = 0; + + for (int g = 0; g < IQ3TQ_N_GROUPS; ++g) { + float best_err = 1e30f; + int best_si = 0; + for (int si = 0; si < 16; ++si) { + const int8_t * ge = grid[si]; + float g_err = 0; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + float xn = xb[j] * tinv; + int qi = iq3tq_nearest_qi(xn, ge); + float recon = tdq * (float)ge[qi]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float err = xb[j] - recon; + g_err += wk * err * err; + } + if (g_err < best_err) { best_err = g_err; best_si = si; } + } + tscales[g / 2] |= (best_si << (4 * (g % 2))); + tgrid[g] = best_si; + const int8_t * ge = grid[best_si]; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + iq3tq_set_qi(tqs, j, iq3tq_nearest_qi(xb[j] * tinv, ge)); + } + total_err += best_err; + } + + if (total_err < best_total_err) { + best_total_err = total_err; + best_d = td; + memcpy(best_scales, tscales, 16); + memcpy(best_qs, tqs, 96); + memcpy(best_grid_idx, tgrid, sizeof(tgrid)); + } + } + d = best_d; + memcpy(scales_out, best_scales, 16); + memcpy(qs, best_qs, 96); + memcpy(grid_idx, best_grid_idx, sizeof(grid_idx)); + } + + // Post multi-d refinement + for (int iter = 0; iter < 2; ++iter) { + double sumxg = 0, sumgg = 0; + for (int j = 0; j < QK_K; ++j) { + int g = j / 8; + float gval = (float)grid[grid_idx[g]][iq3tq_get_qi(qs, j)]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + sumxg += wk * xb[j] * gval; + sumgg += wk * gval * gval; + } + d = (sumgg > 0) ? (float)(sumxg / (IQ3TQ_GRID_SCALE * sumgg)) : d; + + dq = d * IQ3TQ_GRID_SCALE; + inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; + memset(scales_out, 0, 16); + for (int g = 0; g < IQ3TQ_N_GROUPS; ++g) { + float best_err = 1e30f; + int best_si = 0; + for (int si = 0; si < 16; ++si) { + const int8_t * ge = grid[si]; + float g_err = 0; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + float xn = xb[j] * inv_dq; + int qi = iq3tq_nearest_qi(xn, ge); + float recon = dq * (float)ge[qi]; + float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float err = xb[j] - recon; + g_err += wk * err * err; + } + if (g_err < best_err) { best_err = g_err; best_si = si; } + } + scales_out[g / 2] |= (best_si << (4 * (g % 2))); + grid_idx[g] = best_si; + const int8_t * ge = grid[best_si]; + for (int k = 0; k < 8; ++k) { + int j = g * 8 + k; + iq3tq_set_qi(qs, j, iq3tq_nearest_qi(xb[j] * inv_dq, ge)); + } + } + } + + yb->d = GGML_FP32_TO_FP16(d); + memcpy(yb->scales, scales_out, 16); + memcpy(yb->qs, qs, 96); + } +} + +size_t quantize_iq3_tq(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, + int64_t nrows, int64_t n_per_row, const float * imatrix) { + size_t row_size = ggml_row_size(GGML_TYPE_IQ3_TQ, n_per_row); + char * qrow = (char *)dst; + for (int64_t row = 0; row < nrows; ++row) { + quantize_row_iq3_tq_impl(src, (block_iq3_tq *)qrow, n_per_row, imatrix); + src += n_per_row; + qrow += row_size; + } + return nrows * row_size; +} + +// Train per-tensor grid: 16 entries × 8 levels via K-means on optimal per-group level-sets +void iq3tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, int8_t grid_out[IQ3TQ_GRID_SIZE]) { + assert(n_per_row % QK_K == 0); + const int nb_per_row = n_per_row / QK_K; + const int max_groups = 200000; + const int NL = IQ3TQ_N_LEVELS; // 8 + + float (*sets)[IQ3TQ_N_LEVELS] = (float (*)[IQ3TQ_N_LEVELS])malloc(max_groups * NL * sizeof(float)); + float * set_weights = (float *)malloc(max_groups * sizeof(float)); + int n_sets = 0; + + for (int64_t row = 0; row < nrow && n_sets < max_groups - IQ3TQ_N_GROUPS; row++) { + const float * xrow = data + row * n_per_row; + for (int bi = 0; bi < nb_per_row && n_sets < max_groups; bi++) { + const float * xb = xrow + bi * QK_K; + + float amax = 0; + for (int j = 0; j < QK_K; j++) { + float av = fabsf(xb[j]); + if (av > amax) amax = av; + } + if (amax < 1e-15f) continue; + float inv_dq = 1.0f / (amax / 3.0f * IQ3TQ_GRID_SCALE); + + for (int g = 0; g < IQ3TQ_N_GROUPS && n_sets < max_groups; g++) { + float gw = 1.0f; + if (imatrix) { + gw = 0; + for (int k = 0; k < 8; k++) gw += imatrix[g * 8 + k]; + gw /= 8.0f; + if (gw < 1e-10f) gw = 1e-10f; + } + + // Normalize and sort group values + float xn[8]; + for (int k = 0; k < 8; k++) xn[k] = xb[g * 8 + k] * inv_dq; + for (int a = 0; a < 7; a++) + for (int b2 = a + 1; b2 < 8; b2++) + if (xn[b2] < xn[a]) { float t = xn[a]; xn[a] = xn[b2]; xn[b2] = t; } + + // With 8 elements and 8 levels, optimal assignment is 1:1 + // But we still run Lloyd-Max to handle cases with duplicates + float L[IQ3TQ_N_LEVELS]; + for (int q = 0; q < NL; q++) L[q] = xn[q]; + for (int iter = 0; iter < 10; iter++) { + float bounds[7]; + for (int q = 0; q < NL - 1; q++) bounds[q] = 0.5f * (L[q] + L[q+1]); + float sum[IQ3TQ_N_LEVELS] = {0}; int cnt[IQ3TQ_N_LEVELS] = {0}; + for (int k = 0; k < 8; k++) { + int qi = NL - 1; + for (int q = 0; q < NL - 1; q++) { + if (xn[k] <= bounds[q]) { qi = q; break; } + } + sum[qi] += xn[k]; cnt[qi]++; + } + int converged = 1; + for (int q = 0; q < NL; q++) { + float newL = (cnt[q] > 0) ? sum[q] / cnt[q] : L[q]; + if (fabsf(newL - L[q]) > 0.01f) converged = 0; + L[q] = newL; + } + if (converged) break; + } + + memcpy(sets[n_sets], L, NL * sizeof(float)); + set_weights[n_sets] = gw; + n_sets++; + } + } + } + + // K-means clustering into 16 clusters + float centroids[16][IQ3TQ_N_LEVELS]; + for (int i = 0; i < 16; i++) { + int idx = (int)((int64_t)i * n_sets / 16); + memcpy(centroids[i], sets[idx], NL * sizeof(float)); + } + + int * assign = (int *)calloc(n_sets, sizeof(int)); + for (int iter = 0; iter < 100; iter++) { + int changed = 0; + for (int i = 0; i < n_sets; i++) { + float best_dist = 1e30f; + int best_c = 0; + for (int c = 0; c < 16; c++) { + float dist = 0; + for (int dd = 0; dd < NL; dd++) { + float diff = sets[i][dd] - centroids[c][dd]; + dist += diff * diff; + } + if (dist < best_dist) { best_dist = dist; best_c = c; } + } + if (assign[i] != best_c) { assign[i] = best_c; changed++; } + } + + double sum[16][IQ3TQ_N_LEVELS]; + double wcnt[16]; + memset(sum, 0, sizeof(sum)); + memset(wcnt, 0, sizeof(wcnt)); + for (int i = 0; i < n_sets; i++) { + int c = assign[i]; + float w = set_weights[i]; + for (int dd = 0; dd < NL; dd++) sum[c][dd] += w * sets[i][dd]; + wcnt[c] += w; + } + for (int c = 0; c < 16; c++) { + if (wcnt[c] > 0) { + for (int dd = 0; dd < NL; dd++) centroids[c][dd] = (float)(sum[c][dd] / wcnt[c]); + } + } + if (changed == 0) break; + } + + // Sort centroids by sum of levels + for (int i = 0; i < 15; i++) { + for (int j2 = i + 1; j2 < 16; j2++) { + float si2 = 0, sj = 0; + for (int dd = 0; dd < NL; dd++) { si2 += centroids[i][dd]; sj += centroids[j2][dd]; } + if (sj < si2) { + float tmp[IQ3TQ_N_LEVELS]; + memcpy(tmp, centroids[i], NL * sizeof(float)); + memcpy(centroids[i], centroids[j2], NL * sizeof(float)); + memcpy(centroids[j2], tmp, NL * sizeof(float)); + } + } + } + + // Round to int8 + for (int i = 0; i < 16; i++) { + for (int dd = 0; dd < NL; dd++) { + float v = centroids[i][dd]; + grid_out[i * NL + dd] = (int8_t)(v > 0 ? v + 0.5f : v - 0.5f); + } + } + + free(sets); + free(set_weights); + free(assign); +} + +// ===================================================================================== +// IQ1_BN: 8D Vector Quantized with per-tensor trained 4096-entry codebook (1.5625 bpw) +// ===================================================================================== + +// Global state for current tensor's codebook (32768 bytes) +static int8_t iq1bn_active_aux[IQ1BN_AUX_SIZE]; +static bool iq1bn_aux_set = false; + +static const int8_t iq1bn_default_aux[IQ1BN_AUX_SIZE] = {0}; + +void iq1bn_set_aux(const int8_t aux[IQ1BN_AUX_SIZE]) { + memcpy(iq1bn_active_aux, aux, IQ1BN_AUX_SIZE); + iq1bn_aux_set = true; +} + +const int8_t * iq1bn_get_aux(void) { + return iq1bn_aux_set ? iq1bn_active_aux : iq1bn_default_aux; +} + +static inline const int8_t * iq1bn_cur_codebook(void) { + return iq1bn_aux_set ? iq1bn_active_aux : iq1bn_default_aux; +} + +// 12-bit index extraction helpers +static inline int iq1bn_get_idx(const uint8_t * qs, int g) { + int pair = g / 2; + if (g & 1) { + return (qs[3*pair+1] >> 4) | ((int)qs[3*pair+2] << 4); + } else { + return qs[3*pair] | (((int)qs[3*pair+1] & 0x0F) << 8); + } +} + +static inline void iq1bn_set_idx(uint8_t * qs, int pair, int idx0, int idx1) { + qs[3*pair] = idx0 & 0xFF; + qs[3*pair+1] = ((idx0 >> 8) & 0x0F) | ((idx1 & 0x0F) << 4); + qs[3*pair+2] = (idx1 >> 4) & 0xFF; +} + +// Dequantization +void dequantize_row_iq1_bn(const block_iq1_bn * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + const int8_t * codebook = levels ? (const int8_t *)levels : iq1bn_cur_codebook(); + const int nb = k / QK_K; + + for (int i = 0; i < nb; ++i) { + const block_iq1_bn * b = x + i; + const float dq = GGML_FP16_TO_FP32(b->d) * IQ1BN_GRID_SCALE; + + for (int g = 0; g < IQ1BN_N_GROUPS; ++g) { + int ci = iq1bn_get_idx(b->qs, g); + const int8_t * cb = codebook + ci * IQ1BN_CODEBOOK_DIM; + + for (int j = 0; j < IQ1BN_CODEBOOK_DIM; ++j) { + y[i * QK_K + g * IQ1BN_GROUP_SIZE + j] = dq * (float)cb[j]; + } + } + } +} + +// Reference quantization +void quantize_row_iq1_bn_ref(const float * GGML_RESTRICT x, block_iq1_bn * GGML_RESTRICT y, int64_t k) { + assert(k % QK_K == 0); + const int nb = k / QK_K; + const int8_t * codebook = iq1bn_cur_codebook(); + + for (int i = 0; i < nb; ++i) { + float amax = 0; + for (int j = 0; j < QK_K; ++j) { + float ax = fabsf(x[i * QK_K + j]); + if (ax > amax) amax = ax; + } + if (amax < 1e-15f) { + memset(y + i, 0, sizeof(block_iq1_bn)); + continue; + } + float d = amax / 3.0f; + float inv_dq = 1.0f / (d * IQ1BN_GRID_SCALE); + + uint16_t ci_out[IQ1BN_N_GROUPS]; + for (int g = 0; g < IQ1BN_N_GROUPS; ++g) { + float xn[IQ1BN_CODEBOOK_DIM]; + for (int j = 0; j < IQ1BN_CODEBOOK_DIM; ++j) { + xn[j] = x[i * QK_K + g * IQ1BN_GROUP_SIZE + j] * inv_dq; + } + + float best_err = 1e30f; + int best_ci = 0; + for (int c = 0; c < IQ1BN_CODEBOOK_K; ++c) { + const int8_t * cb = codebook + c * IQ1BN_CODEBOOK_DIM; + float err = 0; + for (int j = 0; j < IQ1BN_CODEBOOK_DIM; ++j) { + float diff = xn[j] - (float)cb[j]; + err += diff * diff; + } + if (err < best_err) { best_err = err; best_ci = c; } + } + ci_out[g] = (uint16_t)best_ci; + } + + y[i].d = GGML_FP32_TO_FP16(d); + for (int g = 0; g < IQ1BN_N_GROUPS; g += 2) { + iq1bn_set_idx(y[i].qs, g/2, ci_out[g], ci_out[g+1]); + } + } +} + +// L2 distance helper for 8D float vectors +#if defined(__AVX2__) +static inline float iq1bn_l2_dist_8d(__m256 va, __m256 vb) { + __m256 vdiff = _mm256_sub_ps(va, vb); + __m256 vd2 = _mm256_mul_ps(vdiff, vdiff); + __m128 hi = _mm256_extractf128_ps(vd2, 1); + __m128 lo = _mm256_castps256_ps128(vd2); + __m128 sum4 = _mm_add_ps(lo, hi); + __m128 sum2 = _mm_add_ps(sum4, _mm_movehl_ps(sum4, sum4)); + __m128 sum1 = _mm_add_ss(sum2, _mm_movehdup_ps(sum2)); + return _mm_cvtss_f32(sum1); +} +#endif + +// Find best codebook entry for a normalized group (weighted) — float codebook version +static inline int iq1bn_find_best(const float * xn, const float * wg, + const float * cb_float, const float * cb_norm2, + int K, float * out_err) { + float best_err = 1e30f; + int best_c = 0; +#if defined(__AVX2__) + __m256 vxn = _mm256_loadu_ps(xn); + __m256 vwg = _mm256_loadu_ps(wg); + for (int c = 0; c < K; ++c) { + if (cb_norm2[c] < 1e-10f) continue; + __m256 vcb = _mm256_loadu_ps(cb_float + c * 8); + __m256 vdiff = _mm256_sub_ps(vxn, vcb); + __m256 vd2 = _mm256_mul_ps(vdiff, vdiff); + __m256 vwd2 = _mm256_mul_ps(vwg, vd2); + // Horizontal sum + __m128 hi = _mm256_extractf128_ps(vwd2, 1); + __m128 lo = _mm256_castps256_ps128(vwd2); + __m128 sum4 = _mm_add_ps(lo, hi); + __m128 sum2 = _mm_add_ps(sum4, _mm_movehl_ps(sum4, sum4)); + __m128 sum1 = _mm_add_ss(sum2, _mm_movehdup_ps(sum2)); + float err = _mm_cvtss_f32(sum1); + if (err < best_err) { best_err = err; best_c = c; } + } +#else + for (int c = 0; c < K; ++c) { + if (cb_norm2[c] < 1e-10f) continue; + const float * cb = cb_float + c * IQ1BN_CODEBOOK_DIM; + float err = 0; + for (int j = 0; j < IQ1BN_CODEBOOK_DIM; ++j) { + float diff = xn[j] - cb[j]; + err += wg[j] * diff * diff; + } + if (err < best_err) { best_err = err; best_c = c; } + } +#endif + if (out_err) *out_err = best_err; + return best_c; +} + +// Full quantizer with OLS refinement + multi-d search +static size_t quantize_row_iq1_bn_impl(const float * GGML_RESTRICT x, block_iq1_bn * GGML_RESTRICT y, + int64_t n_per_row, const float * quant_weights) { + const int nb = n_per_row / QK_K; + const int8_t * codebook = iq1bn_cur_codebook(); + + // Precompute float codebook and norms (avoids repeated int8→float conversion) + float * cb_float = (float *)malloc(IQ1BN_CODEBOOK_K * IQ1BN_CODEBOOK_DIM * sizeof(float)); + float * cb_norm2 = (float *)malloc(IQ1BN_CODEBOOK_K * sizeof(float)); + for (int c = 0; c < IQ1BN_CODEBOOK_K; ++c) { + float nn = 0; + const int8_t * cb = codebook + c * IQ1BN_CODEBOOK_DIM; + for (int j = 0; j < IQ1BN_CODEBOOK_DIM; ++j) { + float v = (float)cb[j]; + cb_float[c * IQ1BN_CODEBOOK_DIM + j] = v; + nn += v * v; + } + cb_norm2[c] = nn; + } + + for (int i = 0; i < nb; ++i) { + const float * xb = x + i * QK_K; + const float * wt = quant_weights ? quant_weights + i * QK_K : NULL; + + float amax = 0; + for (int j = 0; j < QK_K; ++j) { + float ax = fabsf(xb[j]); + if (ax > amax) amax = ax; + } + if (amax < 1e-15f) { + memset(y + i, 0, sizeof(block_iq1_bn)); + continue; + } + + float d = amax / 3.0f; + uint16_t ci_out[IQ1BN_N_GROUPS]; + + // OLS iterations + for (int iter = 0; iter < 5; ++iter) { + float inv_dq = 1.0f / (d * IQ1BN_GRID_SCALE); + + for (int g = 0; g < IQ1BN_N_GROUPS; ++g) { + float xn[IQ1BN_CODEBOOK_DIM]; + float wg[IQ1BN_CODEBOOK_DIM]; + for (int j = 0; j < IQ1BN_CODEBOOK_DIM; ++j) { + xn[j] = xb[g * IQ1BN_GROUP_SIZE + j] * inv_dq; + wg[j] = wt ? wt[g * IQ1BN_GROUP_SIZE + j] : 1.0f; + } + ci_out[g] = (uint16_t)iq1bn_find_best(xn, wg, cb_float, cb_norm2, IQ1BN_CODEBOOK_K, NULL); + } + + // OLS: recompute d from current assignments + float sumxg = 0, sumgg = 0; + for (int g = 0; g < IQ1BN_N_GROUPS; ++g) { + const int8_t * cb = codebook + ci_out[g] * IQ1BN_CODEBOOK_DIM; + for (int j = 0; j < IQ1BN_CODEBOOK_DIM; ++j) { + float gval = (float)cb[j]; + float w = wt ? wt[g * IQ1BN_GROUP_SIZE + j] : 1.0f; + sumxg += w * xb[g * IQ1BN_GROUP_SIZE + j] * gval; + sumgg += w * gval * gval; + } + } + if (sumgg > 0) { + d = sumxg / (IQ1BN_GRID_SCALE * sumgg); + } + } + + // Multi-d search + static const float d_factors[] = {0.8f, 0.85f, 0.9f, 0.95f, 1.0f, 1.05f, 1.1f, 1.15f, 1.2f}; + float best_d = d; + float best_total_err = 1e30f; + uint16_t best_ci_all[IQ1BN_N_GROUPS]; + + for (int df = 0; df < 9; ++df) { + float td = d * d_factors[df]; + float inv_dq = 1.0f / (td * IQ1BN_GRID_SCALE); + + uint16_t tci[IQ1BN_N_GROUPS]; + float total_err = 0; + + for (int g = 0; g < IQ1BN_N_GROUPS; ++g) { + float xn[IQ1BN_CODEBOOK_DIM]; + float wg[IQ1BN_CODEBOOK_DIM]; + for (int j = 0; j < IQ1BN_CODEBOOK_DIM; ++j) { + xn[j] = xb[g * IQ1BN_GROUP_SIZE + j] * inv_dq; + wg[j] = wt ? wt[g * IQ1BN_GROUP_SIZE + j] : 1.0f; + } + float gerr; + tci[g] = (uint16_t)iq1bn_find_best(xn, wg, cb_float, cb_norm2, IQ1BN_CODEBOOK_K, &gerr); + total_err += gerr; + } + + if (total_err < best_total_err) { + best_total_err = total_err; + best_d = td; + memcpy(best_ci_all, tci, IQ1BN_N_GROUPS * sizeof(uint16_t)); + } + } + + // Post multi-d OLS refinement (2 iterations) + d = best_d; + memcpy(ci_out, best_ci_all, IQ1BN_N_GROUPS * sizeof(uint16_t)); + + for (int post = 0; post < 2; ++post) { + float sumxg = 0, sumgg = 0; + for (int g = 0; g < IQ1BN_N_GROUPS; ++g) { + const int8_t * cb = codebook + ci_out[g] * IQ1BN_CODEBOOK_DIM; + for (int j = 0; j < IQ1BN_CODEBOOK_DIM; ++j) { + float gval = (float)cb[j]; + float w = wt ? wt[g * IQ1BN_GROUP_SIZE + j] : 1.0f; + sumxg += w * xb[g * IQ1BN_GROUP_SIZE + j] * gval; + sumgg += w * gval * gval; + } + } + if (sumgg > 0) { + d = sumxg / (IQ1BN_GRID_SCALE * sumgg); + } + + float inv_dq = 1.0f / (d * IQ1BN_GRID_SCALE); + for (int g = 0; g < IQ1BN_N_GROUPS; ++g) { + float xn[IQ1BN_CODEBOOK_DIM]; + float wg[IQ1BN_CODEBOOK_DIM]; + for (int j = 0; j < IQ1BN_CODEBOOK_DIM; ++j) { + xn[j] = xb[g * IQ1BN_GROUP_SIZE + j] * inv_dq; + wg[j] = wt ? wt[g * IQ1BN_GROUP_SIZE + j] : 1.0f; + } + ci_out[g] = (uint16_t)iq1bn_find_best(xn, wg, cb_float, cb_norm2, IQ1BN_CODEBOOK_K, NULL); + } + } + + // Write block — pack 12-bit indices + y[i].d = GGML_FP32_TO_FP16(d); + for (int g = 0; g < IQ1BN_N_GROUPS; g += 2) { + iq1bn_set_idx(y[i].qs, g/2, ci_out[g], ci_out[g+1]); + } + } + + free(cb_float); + free(cb_norm2); + return nb * sizeof(block_iq1_bn); +} + +// Public quantization entry point +size_t quantize_iq1_bn(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, + int64_t nrows, int64_t n_per_row, const float * imatrix) { + GGML_ASSERT(n_per_row % QK_K == 0); + float * quant_weights = (float *)malloc(n_per_row * sizeof(float)); + size_t total = 0; + + for (int64_t row = 0; row < nrows; ++row) { + const float * row_data = src + row * n_per_row; + + if (imatrix) { + const float * im_row = imatrix; + float sumw = 0; + for (int64_t j = 0; j < n_per_row; ++j) { + quant_weights[j] = im_row[j]; + sumw += im_row[j]; + } + float scale = sumw > 0 ? n_per_row / sumw : 1.0f; + for (int64_t j = 0; j < n_per_row; ++j) { + quant_weights[j] *= scale; + } + } else { + float sigma2 = 0; + for (int64_t j = 0; j < n_per_row; ++j) { + sigma2 += row_data[j] * row_data[j]; + } + sigma2 = 8.0f * sigma2 / n_per_row; + if (sigma2 < 1e-15f) sigma2 = 1.0f; + for (int64_t j = 0; j < n_per_row; ++j) { + float w = row_data[j] * row_data[j] + sigma2; + quant_weights[j] = w; + } + } + + total += quantize_row_iq1_bn_impl(row_data, (block_iq1_bn *)((char *)dst + total), + n_per_row, quant_weights); + } + + free(quant_weights); + return total; +} + +// Thread work structs for parallel K-means +#include + +// Worker for K-means++ min_dist update (parallel over samples) +typedef struct { + const float * samples; + float * min_dist; + const float * centroid; // single centroid to compute distance to + int i_start; + int i_end; + int dim; +} iq1bn_kmpp_work_t; + +static void * iq1bn_kmpp_worker(void * arg) { + iq1bn_kmpp_work_t * w = (iq1bn_kmpp_work_t *)arg; + const float * cc = w->centroid; + const int dim = w->dim; + // Use scalar distance to match original K-means++ pick sequence + // (SIMD horizontal sum rounding differences cascade in the pick chain) + for (int i = w->i_start; i < w->i_end; ++i) { + const float * s = w->samples + i * dim; + float dist = 0; + for (int j = 0; j < dim; ++j) { + float diff = s[j] - cc[j]; + dist += diff * diff; + } + if (dist < w->min_dist[i]) w->min_dist[i] = dist; + } + return NULL; +} + +// Worker for K-means iteration (parallel over samples) +typedef struct { + const float * samples; + const float * weights; + const float * centroids; + int * assign; + float * csum; // per-thread accumulator [K * dim] + float * cwt; // per-thread accumulator [K] + int i_start; + int i_end; + int K; + int dim; + int changed; +} iq1bn_kmeans_work_t; + +static void * iq1bn_kmeans_worker(void * arg) { + iq1bn_kmeans_work_t * w = (iq1bn_kmeans_work_t *)arg; + const int K = w->K; + const int dim = w->dim; + int changed = 0; + + memset(w->csum, 0, K * dim * sizeof(float)); + memset(w->cwt, 0, K * sizeof(float)); + + for (int i = w->i_start; i < w->i_end; ++i) { + const float * s = w->samples + i * dim; + float best_dist = 1e30f; + int best_c = 0; +#if defined(__AVX2__) + __m256 vs = _mm256_loadu_ps(s); + for (int c = 0; c < K; ++c) { + float dist = iq1bn_l2_dist_8d(vs, _mm256_loadu_ps(w->centroids + c * dim)); + if (dist < best_dist) { best_dist = dist; best_c = c; } + } +#else + for (int c = 0; c < K; ++c) { + const float * cc = w->centroids + c * dim; + float dist = 0; + for (int j = 0; j < dim; ++j) { + float diff = s[j] - cc[j]; + dist += diff * diff; + } + if (dist < best_dist) { best_dist = dist; best_c = c; } + } +#endif + if (w->assign[i] != best_c) { changed++; w->assign[i] = best_c; } + float wt = w->weights[i]; + for (int j = 0; j < dim; ++j) { + w->csum[best_c * dim + j] += wt * s[j]; + } + w->cwt[best_c] += wt; + } + w->changed = changed; + return NULL; +} + +// K-means codebook training (K=4096, random init) +void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, int8_t aux_out[IQ1BN_AUX_SIZE], int nthread) { + const int max_samples = 200000; + const int dim = IQ1BN_CODEBOOK_DIM; + const int K = IQ1BN_CODEBOOK_K; // 4096 + + float * samples = (float *)malloc(max_samples * dim * sizeof(float)); + float * weights = (float *)malloc(max_samples * sizeof(float)); + int n_samples = 0; + + // Phase 1: Collect normalized 8D sub-vectors + for (int64_t row = 0; row < nrow && n_samples < max_samples; ++row) { + const float * row_data = data + row * n_per_row; + const int n_blocks = n_per_row / QK_K; + + for (int bi = 0; bi < n_blocks && n_samples < max_samples; ++bi) { + const float * block = row_data + bi * QK_K; + + float amax = 0; + for (int j = 0; j < QK_K; ++j) { + float ax = fabsf(block[j]); + if (ax > amax) amax = ax; + } + if (amax < 1e-15f) continue; + float dd = amax / 3.0f; + float inv_dq = 1.0f / (dd * IQ1BN_GRID_SCALE); + + for (int g = 0; g < IQ1BN_N_GROUPS && n_samples < max_samples; ++g) { + float gw = 1.0f; + if (imatrix) { + gw = 0; + for (int j = 0; j < dim; ++j) { + int idx = bi * QK_K + g * dim + j; + if (idx < n_per_row) gw += imatrix[idx]; + } + gw /= dim; + if (gw < 1e-10f) gw = 1e-10f; + } + + float * s = samples + n_samples * dim; + for (int j = 0; j < dim; ++j) { + s[j] = block[g * dim + j] * inv_dq; + } + weights[n_samples] = gw; + n_samples++; + } + } + } + + if (n_samples == 0) { + memset(aux_out, 0, IQ1BN_AUX_SIZE); + free(samples); free(weights); + return; + } + + fprintf(stderr, "IQ1BN: training K=%d codebook from %d samples\n", K, n_samples); + + // Phase 2: K-means++ initialization + float * centroids = (float *)calloc(K * dim, sizeof(float)); + int * assign = (int *)calloc(n_samples, sizeof(int)); + float * min_dist = (float *)malloc(n_samples * sizeof(float)); + + // Pick first centroid: weighted random + { + float total_w = 0; + for (int i = 0; i < n_samples; ++i) total_w += weights[i]; + float r = ((float)rand() / RAND_MAX) * total_w; + float cumw = 0; + int pick = 0; + for (int i = 0; i < n_samples; ++i) { + cumw += weights[i]; + if (cumw >= r) { pick = i; break; } + } + memcpy(centroids, samples + pick * dim, dim * sizeof(float)); + for (int i = 0; i < n_samples; ++i) min_dist[i] = 1e30f; + } + + // Full K-means++ initialization with parallel min_dist update + if (nthread < 1) nthread = 1; + if (nthread > n_samples) nthread = n_samples; + + // Set up thread pool for K-means++ min_dist update + pthread_t * threads = (pthread_t *)calloc(nthread, sizeof(pthread_t)); + iq1bn_kmpp_work_t * kmpp_workers = (iq1bn_kmpp_work_t *)calloc(nthread, sizeof(iq1bn_kmpp_work_t)); + for (int t = 0; t < nthread; ++t) { + kmpp_workers[t].samples = samples; + kmpp_workers[t].min_dist = min_dist; + kmpp_workers[t].dim = dim; + int chunk = n_samples / nthread; + kmpp_workers[t].i_start = t * chunk; + kmpp_workers[t].i_end = (t == nthread - 1) ? n_samples : (t + 1) * chunk; + } + + const int n_kmpp = 256; // full K-means++ for first n_kmpp, stale-dist random for rest + for (int c = 0; c < K; ++c) { + if (c < n_kmpp) { + // Update min distances using SCALAR code (parallel over samples) + // Scalar avoids SIMD rounding differences that cascade in K-means++ pick chain + const float * cc = centroids + c * dim; + for (int t = 0; t < nthread; ++t) { + kmpp_workers[t].centroid = cc; + } + if (nthread > 1) { + for (int t = 0; t < nthread; ++t) { + pthread_create(&threads[t], NULL, iq1bn_kmpp_worker, &kmpp_workers[t]); + } + for (int t = 0; t < nthread; ++t) { + pthread_join(threads[t], NULL); + } + } else { + iq1bn_kmpp_worker(&kmpp_workers[0]); + } + } + + if (c + 1 < K) { + // Pick next proportional to weighted distance + float total = 0; + for (int i = 0; i < n_samples; ++i) total += weights[i] * min_dist[i]; + float r = ((float)rand() / RAND_MAX) * total; + float cumw = 0; + int pick = 0; + for (int i = 0; i < n_samples; ++i) { + cumw += weights[i] * min_dist[i]; + if (cumw >= r) { pick = i; break; } + } + memcpy(centroids + (c + 1) * dim, samples + pick * dim, dim * sizeof(float)); + } + } + free(min_dist); + free(kmpp_workers); + + // Phase 3: Full-batch K-means with SIMD and pthread parallelism + + // Allocate per-thread accumulators (reuse threads array from K-means++ init) + iq1bn_kmeans_work_t * workers = (iq1bn_kmeans_work_t *)calloc(nthread, sizeof(iq1bn_kmeans_work_t)); + for (int t = 0; t < nthread; ++t) { + workers[t].samples = samples; + workers[t].weights = weights; + workers[t].centroids = centroids; + workers[t].assign = assign; + workers[t].csum = (float *)malloc(K * dim * sizeof(float)); + workers[t].cwt = (float *)malloc(K * sizeof(float)); + workers[t].K = K; + workers[t].dim = dim; + int chunk = n_samples / nthread; + workers[t].i_start = t * chunk; + workers[t].i_end = (t == nthread - 1) ? n_samples : (t + 1) * chunk; + } + + const int n_iters = 30; + for (int iter = 0; iter < n_iters; ++iter) { + // Launch threads for sample assignment + for (int t = 0; t < nthread; ++t) { + if (nthread > 1) { + pthread_create(&threads[t], NULL, iq1bn_kmeans_worker, &workers[t]); + } else { + iq1bn_kmeans_worker(&workers[0]); + } + } + if (nthread > 1) { + for (int t = 0; t < nthread; ++t) { + pthread_join(threads[t], NULL); + } + } + + // Merge per-thread accumulators + int changed = 0; + for (int t = 0; t < nthread; ++t) { + changed += workers[t].changed; + } + + // Sum csum and cwt across threads + // Use workers[0]'s arrays as the merge target + for (int t = 1; t < nthread; ++t) { + for (int c = 0; c < K; ++c) { + for (int j = 0; j < dim; ++j) { + workers[0].csum[c * dim + j] += workers[t].csum[c * dim + j]; + } + workers[0].cwt[c] += workers[t].cwt[c]; + } + } + + // Update centroids and find most populous cluster + int empty = 0; + int max_pop_c = 0; + float max_pop_w = 0; + for (int c = 0; c < K; ++c) { + if (workers[0].cwt[c] > 0) { + for (int j = 0; j < dim; ++j) { + centroids[c * dim + j] = workers[0].csum[c * dim + j] / workers[0].cwt[c]; + } + if (workers[0].cwt[c] > max_pop_w) { + max_pop_w = workers[0].cwt[c]; + max_pop_c = c; + } + } else { + empty++; + } + } + + // Split most populous cluster to fill empty clusters + if (empty > 0 && max_pop_w > 0) { + for (int c = 0; c < K; ++c) { + if (workers[0].cwt[c] <= 0) { + for (int j = 0; j < dim; ++j) { + float perturb = 0.01f * ((float)(rand() % 201 - 100) / 100.0f); + centroids[c * dim + j] = centroids[max_pop_c * dim + j] + perturb; + } + } + } + } + + if (iter % 5 == 0 || iter == n_iters - 1) { + fprintf(stderr, " iter %2d: changed=%d/%d empty=%d\n", iter, changed, n_samples, empty); + } + if (iter > 5 && changed < n_samples / 1000) break; + } + + // Free per-thread accumulators + for (int t = 0; t < nthread; ++t) { + free(workers[t].csum); + free(workers[t].cwt); + } + free(workers); + free(threads); + + // Round centroids to int8 codebook + int8_t * codebook_out = aux_out; + for (int c = 0; c < K; ++c) { + for (int j = 0; j < dim; ++j) { + float v = centroids[c * dim + j]; + int iv = (int)roundf(v); + if (iv < -127) iv = -127; + if (iv > 127) iv = 127; + codebook_out[c * dim + j] = (int8_t)iv; + } + } + + free(samples); free(weights); + free(centroids); free(assign); +} + +// Also add to validate_row_data switch (done via ops.cpp) + +// Global levels (used during quantization for the current tensor) +static float q3pt_levels[Q3PT_N_LEVELS]; +static bool q3pt_levels_set = false; + +void q3pt_set_levels(const float * levels) { + memcpy(q3pt_levels, levels, Q3PT_N_LEVELS * sizeof(float)); + q3pt_levels_set = true; +} + +const float * q3pt_get_levels(void) { + return q3pt_levels_set ? q3pt_levels : NULL; +} + +void q3pt_free_levels(void) { + q3pt_levels_set = false; +} + + +void q3pt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float levels_out[Q3PT_N_LEVELS]) { + + const int64_t n_sub = n_per_row / 16; // 16-element sub-blocks per row + + // Binning parameters + const int N_BINS = 8192; + const float bin_width = 1.0f / N_BINS; + float * bin_sum_w = (float *)calloc(N_BINS, sizeof(float)); + float * bin_sum_wt = (float *)calloc(N_BINS, sizeof(float)); + GGML_ASSERT(bin_sum_w && bin_sum_wt); + + // First pass: bin the affine-normalized values with their weights + for (int64_t row = 0; row < nrow; ++row) { + const float * xrow = data + row * n_per_row; + for (int64_t ib = 0; ib < n_sub; ++ib) { + const float * xb = xrow + ib * 16; + const int col_base = (int)(ib * 16); + float sb_min = xb[0], sb_max = xb[0]; + for (int j = 1; j < 16; ++j) { + if (xb[j] < sb_min) sb_min = xb[j]; + if (xb[j] > sb_max) sb_max = xb[j]; + } + const float sb_range = sb_max - sb_min; + for (int j = 0; j < 16; ++j) { + float w = 1.0f; + if (imatrix) { + w = imatrix[col_base + j]; + if (w < 1e-10f) w = 1e-10f; + } + if (sb_range > 1e-6f) { + w *= sb_range; + float t = (xb[j] - sb_min) / sb_range; + int bin_idx = (int)(t * N_BINS); + if (bin_idx >= N_BINS) bin_idx = N_BINS - 1; + bin_sum_w[bin_idx] += w; + bin_sum_wt[bin_idx] += w * t; + } + } + } + } + + // Initialize 8 levels uniformly in [0, 1] + float levels[Q3PT_N_LEVELS]; + for (int k = 0; k < Q3PT_N_LEVELS; ++k) { + levels[k] = (float)k / (Q3PT_N_LEVELS - 1); + } + + // Lloyd-Max (weighted k-means) iterations with early convergence + for (int iter = 0; iter < 300; ++iter) { + float sum_w [Q3PT_N_LEVELS] = {0}; + float sum_wt[Q3PT_N_LEVELS] = {0}; + + // Process bins instead of individual values + for (int b = 0; b < N_BINS; ++b) { + if (bin_sum_w[b] < 1e-12f) continue; + const float t = (b + 0.5f) * bin_width; // representative value at bin center + int best = 0; + float best_d2 = (t - levels[0]) * (t - levels[0]); + for (int k = 1; k < Q3PT_N_LEVELS; ++k) { + float d2 = (t - levels[k]) * (t - levels[k]); + if (d2 < best_d2) { best_d2 = d2; best = k; } + } + sum_w [best] += bin_sum_w[b]; + sum_wt[best] += bin_sum_wt[b]; + } + + // Check for early convergence + float max_delta = 0.0f; + for (int k = 0; k < Q3PT_N_LEVELS; ++k) { + if (sum_w[k] > 1e-12f) { + float new_level = sum_wt[k] / sum_w[k]; + max_delta = fmaxf(max_delta, fabsf(new_level - levels[k])); + levels[k] = new_level; + } + } + if (max_delta < 1e-10f) break; + + // Keep levels sorted (insertion sort — 8 elements) + for (int k = 1; k < Q3PT_N_LEVELS; ++k) { + float v = levels[k]; int m = k - 1; + while (m >= 0 && levels[m] > v) { levels[m+1] = levels[m]; m--; } + levels[m+1] = v; + } + } + + memcpy(levels_out, levels, Q3PT_N_LEVELS * sizeof(float)); + q3pt_set_levels(levels); + free(bin_sum_w); + free(bin_sum_wt); +} + +// --- Q3_PT bit-packing helpers --- + +// 6-bit sequential packing: 32 values in 24 bytes (4 values per 3 bytes). +// Indices 0..15 = sub-block ranges, 16..31 = sub-block neg_mins. +static inline uint8_t q3pt_sc_get(const uint8_t * GGML_RESTRICT sc, int i) { + const int bit = i * 6; + const int byte = bit / 8; + const int off = bit % 8; + uint8_t val = (sc[byte] >> off) & 0x3F; + if (off > 2) { val |= (uint8_t)((sc[byte+1] << (8 - off)) & 0x3F); } + return val; +} + +static inline void q3pt_sc_set(uint8_t * GGML_RESTRICT sc, int i, uint8_t v) { + const int bit = i * 6; + const int byte = bit / 8; + const int off = bit % 8; + sc[byte] |= (uint8_t)((v & 0x3F) << off); + if (off > 2) { sc[byte+1] |= (uint8_t)(v >> (8 - off)); } +} + +// 3-bit sequential packing: 256 values in 96 bytes (8 values per 3 bytes). +static inline int q3pt_unpack3(const uint8_t * GGML_RESTRICT qs, int k) { + const int bit = k * 3; + const int byte = bit / 8; + const int off = bit % 8; + int val = (qs[byte] >> off) & 0x7; + if (off > 5) { val |= (int)((qs[byte+1] << (8 - off)) & 0x7); } + return val; +} + +static inline void q3pt_pack3(uint8_t * GGML_RESTRICT qs, int k, int v) { + const int bit = k * 3; + const int byte = bit / 8; + const int off = bit % 8; + qs[byte] |= (uint8_t)((v & 0x7) << off); + if (off > 5) { qs[byte+1] |= (uint8_t)((v & 0x7) >> (8 - off)); } +} + +void dequantize_row_q3_pt(const block_q3_pt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + assert(k % QK_K == 0); + const int nb = k / QK_K; + const float * L = (const float *)levels; + GGML_ASSERT(L != NULL && "Q3_PT levels not set for tensor"); + + for (int i = 0; i < nb; i++) { + const float d = GGML_FP16_TO_FP32(x[i].d); + const float dmin = GGML_FP16_TO_FP32(x[i].dmin); + const uint8_t * sc = x[i].scales; + const uint8_t * qs = x[i].qs; + + for (int ib = 0; ib < QK_K/16; ++ib) { + const float range = d * (float)q3pt_sc_get(sc, ib); + const float sub_min = -dmin * (float)q3pt_sc_get(sc, ib + QK_K/16); + for (int j = 0; j < 16; ++j) { + const int q = q3pt_unpack3(qs, ib*16 + j); + y[ib*16 + j] = L[q] * range + sub_min; + } + } + y += QK_K; + } +} + +#define Q3PT_REFINE_ITERS 5 + +// Find the optimal global d-scale for 6-bit (nmax=63) sub-block range quantization, +// minimizing Σ_i weights[i] * (vals[i] - d * clamp(round(vals[i]/d), 0, nmax))^2. +// Tries d = vals[i] / nmax as "anchor" for each sub-block i (O(n^2), n=QK_K/16=16). +// Without imatrix all weights are equal and the winner is always max/nmax, so this is a no-op. +// With imatrix it can redirect scale resolution to important sub-blocks at the cost of +// less important ones that would otherwise dominate via raw max(). +static float q3pt_find_optimal_d(const float * GGML_RESTRICT vals, + const float * GGML_RESTRICT weights, + int n, int nmax) { + float max_val = 0.f; + for (int i = 0; i < n; ++i) { if (vals[i] > max_val) max_val = vals[i]; } + if (max_val < 1e-6f) return 0.f; + float best_d = max_val / (float)nmax, best_err = FLT_MAX; + for (int i = 0; i < n; ++i) { + if (vals[i] < 1e-6f) continue; + const float d_cand = vals[i] / (float)nmax; + float err = 0.f; + for (int j = 0; j < n; ++j) { + int q = (int)(vals[j] / d_cand + 0.5f); + if (q > nmax) q = nmax; + const float delta = vals[j] - d_cand * (float)q; + err += weights[j] * delta * delta; + } + if (err < best_err) { best_err = err; best_d = d_cand; } + } + return best_d; +} + +static void quantize_row_q3_pt_impl(const float * GGML_RESTRICT x, + void * GGML_RESTRICT vy, + int64_t n, + const float * GGML_RESTRICT quant_weights) { + GGML_ASSERT(q3pt_levels_set && "Q3_PT levels not set - call q3pt_set_levels() first"); + GGML_ASSERT(n % QK_K == 0); + + const int64_t nbl = n / QK_K; + block_q3_pt * y = (block_q3_pt *) vy; + const float * L = q3pt_levels; + + for (int ibl = 0; ibl < nbl; ++ibl) { + const float * xbl = x + QK_K * ibl; + block_q3_pt * blk = &y[ibl]; + + float sigma2 = 0; + if (quant_weights) { + for (int i = 0; i < QK_K; ++i) { + sigma2 += xbl[i] * xbl[i]; + } + sigma2 = 2.f * sigma2 / QK_K; + } + + // Per-sub-block importance weights: sum of AWQ weights over 16 elements. + // Used by q3pt_find_optimal_d() to direct scale resolution toward important sub-blocks. + float w_ib[QK_K / 16]; + for (int ib = 0; ib < QK_K / 16; ++ib) { + float wsum = 0.f; + if (quant_weights) { + for (int j = 0; j < 16; ++j) { + const int elem = ib * 16 + j; + wsum += quant_weights[QK_K * ibl + elem] * sqrtf(sigma2 + xbl[elem] * xbl[elem]); + } + } else { + wsum = 16.f; // uniform — find_optimal_d is a no-op (max/63 always wins) + } + w_ib[ib] = wsum; + } + + // Compute per-sub-block ranges and neg_mins from raw min/max + float sub_ranges[QK_K / 16]; + float neg_mins[QK_K / 16]; + for (int ib = 0; ib < QK_K / 16; ++ib) { + const float * xb = xbl + ib * 16; + float sb_min = xb[0], sb_max = xb[0]; + for (int j = 1; j < 16; ++j) { + if (xb[j] < sb_min) { + sb_min = xb[j]; + } + if (xb[j] > sb_max) { + sb_max = xb[j]; + } + } + sub_ranges[ib] = sb_max - sb_min; + neg_mins[ib] = MAX(-sb_min, 0.f); + } + + // Pre-refinement: one weighted-LS pass with continuous (float) ranges before 6-bit + // quantization. Finds better initial (range, neg_min) from the raw min/max assignments, + // avoiding scale quantization noise in the very first set of level assignments. + for (int ib = 0; ib < QK_K / 16; ++ib) { + const float * xb = xbl + ib * 16; + if (sub_ranges[ib] < 1e-6f) { + continue; + } + const float inv_range0 = 1.f / sub_ranges[ib]; + const float sub_min0 = -neg_mins[ib]; + double sA = 0, sB = 0, sC = 0, sD = 0, sE = 0; + for (int j = 0; j < 16; ++j) { + const int elem = ib * 16 + j; + const float xj = xb[j]; + const float w = quant_weights ? quant_weights[QK_K * ibl + elem] * sqrtf(sigma2 + xj * xj) : 1.0f; + const float t = (xj - sub_min0) * inv_range0; + int best = 0; + float best_d2 = (t - L[0]) * (t - L[0]); + for (int k = 1; k < Q3PT_N_LEVELS; ++k) { + const float d2 = (t - L[k]) * (t - L[k]); + if (d2 < best_d2) { + best_d2 = d2; + best = k; + } + } + const float lq = L[best]; + sA += (double) w * (double) lq * (double) lq; + sB += (double) w * (double) lq; + sC += (double) w; + sD += (double) w * (double) xj * (double) lq; + sE += (double) w * (double) xj; + } + const double det = sA * sC - sB * sB; + if (det > 1e-20) { + const float nr = (float) ((sD * sC - sE * sB) / det); + const float nm = (float) (-(sE * sA - sD * sB) / det); + if (nr > 0.f) { + sub_ranges[ib] = nr; + } + if (nm > 0.f) { + neg_mins[ib] = nm; + } + } + } + + // Importance-weighted d/dmin search (replaces plain max/63) + float d_val = q3pt_find_optimal_d(sub_ranges, w_ib, QK_K / 16, 63); + float dmin_val = q3pt_find_optimal_d(neg_mins, w_ib, QK_K / 16, 63); + + // Quantize ranges and neg_mins to 6-bit + memset(blk->scales, 0, sizeof(blk->scales)); + memset(blk->qs, 0, sizeof(blk->qs)); + const float inv_d = d_val > 0 ? 1.f / d_val : 0.f; + const float inv_dmin = dmin_val > 0 ? 1.f / dmin_val : 0.f; + for (int ib = 0; ib < QK_K / 16; ++ib) { + uint8_t sc = MIN(63, nearest_int(inv_d * sub_ranges[ib])); + uint8_t sm = MIN(63, nearest_int(inv_dmin * neg_mins[ib])); + q3pt_sc_set(blk->scales, ib, sc); + q3pt_sc_set(blk->scales, ib + QK_K / 16, sm); + } + blk->d = GGML_FP32_TO_FP16(d_val); + blk->dmin = GGML_FP32_TO_FP16(dmin_val); + + // Initial level assignment + for (int ib = 0; ib < QK_K / 16; ++ib) { + const float range = d_val * (float) q3pt_sc_get(blk->scales, ib); + const float sub_min = -dmin_val * (float) q3pt_sc_get(blk->scales, ib + QK_K / 16); + const float inv_range = range > 1e-6f ? 1.f / range : 0.f; + for (int j = 0; j < 16; ++j) { + const int elem = ib * 16 + j; + const float t = (xbl[elem] - sub_min) * inv_range; + int best = 0; + float best_d2 = (t - L[0]) * (t - L[0]); + for (int k = 1; k < Q3PT_N_LEVELS; ++k) { + const float d2 = (t - L[k]) * (t - L[k]); + if (d2 < best_d2) { + best_d2 = d2; + best = k; + } + } + q3pt_pack3(blk->qs, elem, best); + } + } + + // Iterative refinement: weighted LS for (range, neg_min) + importance-weighted d/dmin. + for (int iter = 0; iter < Q3PT_REFINE_ITERS; ++iter) { + for (int ib = 0; ib < QK_K / 16; ++ib) { + double sA = 0, sB = 0, sC = 0, sD = 0, sE = 0; + for (int j = 0; j < 16; ++j) { + const int elem = ib * 16 + j; + const float xj = xbl[elem]; + const float w = quant_weights ? quant_weights[QK_K * ibl + elem] * sqrtf(sigma2 + xj * xj) : 1.0f; + const float lq = L[q3pt_unpack3(blk->qs, elem)]; + sA += (double) w * (double) lq * (double) lq; + sB += (double) w * (double) lq; + sC += (double) w; + sD += (double) w * (double) xj * (double) lq; + sE += (double) w * (double) xj; + } + const double det = sA * sC - sB * sB; + if (det < 1e-20) { + continue; + } + const float new_range = (float) ((sD * sC - sE * sB) / det); + const float new_negmin = (float) (-(sE * sA - sD * sB) / det); + sub_ranges[ib] = new_range > 0.f ? new_range : 0.f; + neg_mins[ib] = new_negmin > 0.f ? new_negmin : 0.f; + } + + // Importance-weighted d/dmin search on updated sub_ranges/neg_mins + d_val = q3pt_find_optimal_d(sub_ranges, w_ib, QK_K / 16, 63); + dmin_val = q3pt_find_optimal_d(neg_mins, w_ib, QK_K / 16, 63); + + // Re-pack scales + memset(blk->scales, 0, sizeof(blk->scales)); + const float inv_d2 = d_val > 0 ? 1.f / d_val : 0.f; + const float inv_dmin2 = dmin_val > 0 ? 1.f / dmin_val : 0.f; + for (int ib = 0; ib < QK_K / 16; ++ib) { + uint8_t sc = MIN(63, nearest_int(inv_d2 * sub_ranges[ib])); + uint8_t sm = MIN(63, nearest_int(inv_dmin2 * neg_mins[ib])); + q3pt_sc_set(blk->scales, ib, sc); + q3pt_sc_set(blk->scales, ib + QK_K / 16, sm); + } + blk->d = GGML_FP32_TO_FP16(d_val); + blk->dmin = GGML_FP32_TO_FP16(dmin_val); + + // Re-assign levels + memset(blk->qs, 0, sizeof(blk->qs)); + for (int ib = 0; ib < QK_K / 16; ++ib) { + const float range = d_val * (float) q3pt_sc_get(blk->scales, ib); + const float sub_min = -dmin_val * (float) q3pt_sc_get(blk->scales, ib + QK_K / 16); + const float inv_range = range > 1e-6f ? 1.f / range : 0.f; + for (int j = 0; j < 16; ++j) { + const int elem = ib * 16 + j; + const float t = (xbl[elem] - sub_min) * inv_range; + int best = 0; + float best_d2 = (t - L[0]) * (t - L[0]); + for (int k = 1; k < Q3PT_N_LEVELS; ++k) { + const float d2 = (t - L[k]) * (t - L[k]); + if (d2 < best_d2) { + best_d2 = d2; + best = k; + } + } + q3pt_pack3(blk->qs, elem, best); + } + } + } + } +} + +size_t quantize_q3_pt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + GGML_ASSERT(n_per_row % QK_K == 0); + int64_t nblock = n_per_row / QK_K; + char * qrow = (char *)dst; + for (int64_t row = 0; row < nrow; ++row) { + quantize_row_q3_pt_impl(src, qrow, n_per_row, quant_weights); + src += n_per_row; + qrow += nblock * sizeof(block_q3_pt); + } + return nrow * nblock * sizeof(block_q3_pt); +} + +void quantize_row_q3_pt_ref(const float * GGML_RESTRICT x, block_q3_pt * GGML_RESTRICT y, int64_t k) { + assert(k % QK_K == 0); + quantize_q3_pt(x, y, 1, k, NULL); +} + +// =================================== 1.5 bpw =================================================== + +static int iq1_find_best_neighbour(const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid, + const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float * scale, int8_t * GGML_RESTRICT L, int ngrid) { + int num_neighbors = neighbours[0]; + GGML_ASSERT(num_neighbors > 0); + float best_score = -FLT_MAX; + int grid_index = -1; + for (int j = 1; j <= num_neighbors; ++j) { + const int8_t * pg = (const int8_t *)(grid + neighbours[j]); + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < 8; ++i) { + float q = (pg[i] - 3)/2; + float w = weight[i]; + sumqx += w*q*xval[i]; + sumq2 += w*q*q; + } + if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { + *scale = sumqx/sumq2; best_score = *scale * sumqx; + grid_index = neighbours[j]; + } + } + if (grid_index < 0) { + for (int i = 0; i < ngrid; ++i) { + const int8_t * grid_i = (const int8_t *)(grid + i); + float sumqx = 0, sumq2 = 0; + for (int j = 0; j < 8; ++j) { + float w = weight[j]; + float q = (grid_i[j] - 3)/2; + sumqx += w*q*xval[j]; + sumq2 += w*q*q; + } + if (sumqx > 0 && sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { + *scale = sumqx/sumq2; best_score = *scale*sumqx; + grid_index = i; + } + } + } + if (grid_index < 0) { + printf("Oops, did not find grid point\n"); + printf("Have %d neighbours\n", num_neighbors); + for (int j = 1; j <= num_neighbors; ++j) { + const int8_t * pg = (const int8_t *)(grid + neighbours[j]); + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < 8; ++i) { + float q = (pg[i] - 3)/2; + float w = weight[i]; + sumqx += w*q*xval[i]; + sumq2 += w*q*q; + } + printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2); + } + } + GGML_ASSERT(grid_index >= 0); + //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + *scale *= 1.05f; // This is a fudge factor. Don't ask me why it improves the result. + //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + const int8_t * pg = (const int8_t *)(grid + grid_index); + for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2; + return grid_index; +} + +static int iq1_find_best_neighbour2(const uint16_t * GGML_RESTRICT neighbours, const uint64_t * GGML_RESTRICT grid, + const float * GGML_RESTRICT xval, const float * GGML_RESTRICT weight, float scale, const float * GGML_RESTRICT xg, int8_t * GGML_RESTRICT L, int ngrid) { + int num_neighbors = neighbours[0]; + GGML_ASSERT(num_neighbors > 0); + float best_score = FLT_MAX; + int grid_index = -1; + for (int j = 1; j <= num_neighbors; ++j) { + const int8_t * pg = (const int8_t *)(grid + neighbours[j]); + float d2 = 0; + for (int i = 0; i < 8; ++i) { + float q = xg[(pg[i] - 1)/2]; + float w = weight[i]; + float diff = scale*q - xval[i]; + d2 += w*diff*diff; + } + if (d2 < best_score) { + best_score = d2; + grid_index = neighbours[j]; + } + } + if (grid_index < 0) { + for (int i = 0; i < ngrid; ++i) { + const int8_t * grid_i = (const int8_t *)(grid + i); + float d2 = 0; + for (int j = 0; j < 8; ++j) { + float w = weight[j]; + float q = xg[(grid_i[j] - 1)/2]; + float diff = scale*q - xval[i]; + d2 += w*diff*diff; + } + if (d2 < best_score) { + best_score = d2; + grid_index = i; + } + } + } + if (grid_index < 0) { + printf("Oops, did not find grid point\n"); + printf("Have %d neighbours\n", num_neighbors); + for (int j = 1; j <= num_neighbors; ++j) { + const int8_t * pg = (const int8_t *)(grid + neighbours[j]); + float sumqx = 0, sumq2 = 0; + for (int i = 0; i < 8; ++i) { + float q = xg[(pg[i] - 1)/2]; + float w = weight[i]; + sumqx += w*q*xval[i]; + sumq2 += w*q*q; + } + printf(" neighbour %d: sumqx = %g sumq2 = %g\n", j, (double)sumqx, (double)sumq2); + } + } + GGML_ASSERT(grid_index >= 0); + const int8_t * pg = (const int8_t *)(grid + grid_index); + for (int i = 0; i < 8; ++i) L[i] = (pg[i] - 1)/2; + return grid_index; +} + +static int iq1_sort_helper(const void * left, const void * right) { + const float * l = left; + const float * r = right; + return *l < *r ? -1 : *l > *r ? 1 : 0; +} + +#define IQ1S_BLOCK_SIZE 32 +#define IQ1M_BLOCK_SIZE 16 +static void quantize_row_iq1_s_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights, + float * scales, + float * weight, + float * sumx, + float * sumw, + float * pairs, + int8_t * L, + uint16_t * index, + int8_t * shifts) { + + const int gindex = iq2_data_index(GGML_TYPE_IQ1_S); + + const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; + const int * kmap_q2xs = iq2_data[gindex].map; + const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours; + + GGML_ASSERT(quant_weights && "missing quantization weights"); + GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(n%QK_K == 0); + + block_iq1_s * y = vy; + + const int64_t nbl = n/QK_K; + + const int block_size = IQ1S_BLOCK_SIZE; + + const float x_p[3] = {-1 + IQ1S_DELTA, IQ1S_DELTA, 1 + IQ1S_DELTA}; + const float x_m[3] = {-1 - IQ1S_DELTA, -IQ1S_DELTA, 1 - IQ1S_DELTA}; + + + int * idx = (int *)(pairs + 1); + + for (int ibl = 0; ibl < nbl; ++ibl) { + + y[ibl].d = GGML_FP32_TO_FP16(0.f); + memset(y[ibl].qs, 0, QK_K/8); + memset(y[ibl].qh, 0, QK_K/16); + + float max_scale = 0; + + const float * xbl = x + QK_K*ibl; + float sumx2 = 0; + for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i]; + float sigma2 = 2*sumx2/QK_K; + + for (int ib = 0; ib < QK_K/block_size; ++ib) { + const float * xb = xbl + block_size*ib; + const float * qw = quant_weights + QK_K*ibl + block_size*ib; + for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); + float max = fabsf(xb[0]); + for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i])); + if (max < GROUP_MAX_EPS_IQ1_S) { + scales[ib] = 0; + shifts[ib] = 1; + memset(L, 1, block_size); + continue; + } + // Here we solve exactly the sum of squared difference (SSD) weighted minimization problem. // With just 3 allowed quant values (-1, 0, 1), we can search exhaustively for the two // boundaries that split the weights xb[i] into 3 groups. To do so, we sort the weights // in ascending order, compute Si = sum[weight[j] xb[j], j = 0...i] and @@ -5573,6 +9623,39 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte { VALIDATE_ROW_DATA_D_F16_IMPL(block_iq4_nl, data, nb); } break; + case GGML_TYPE_Q3_PT: + { + VALIDATE_ROW_DATA_D_F16_IMPL(block_q3_pt, data, nb); + } break; + case GGML_TYPE_Q3_KPT: + { + VALIDATE_ROW_DATA_D_F16_IMPL(block_q3_kpt, data, nb); + } break; + case GGML_TYPE_Q4_DPT: + { + VALIDATE_ROW_DATA_D_F16_IMPL(block_q4_dpt, data, nb); + } break; + case GGML_TYPE_Q2_DPT: + { + VALIDATE_ROW_DATA_D_F16_IMPL(block_q2_dpt, data, nb); + } break; + case GGML_TYPE_Q2_KPT: + { + VALIDATE_ROW_DATA_DM_F16_IMPL(block_q2_kpt, data, nb, d, dmin); + } break; + case GGML_TYPE_IQ2_TQ: + { + VALIDATE_ROW_DATA_D_F16_IMPL(block_iq2_tq, data, nb); + } break; + + case GGML_TYPE_IQ3_TQ: + { + VALIDATE_ROW_DATA_D_F16_IMPL(block_iq3_tq, data, nb); + } break; + case GGML_TYPE_IQ1_BN: + { + VALIDATE_ROW_DATA_D_F16_IMPL(block_iq1_bn, data, nb); + } break; case GGML_TYPE_I8: case GGML_TYPE_I16: @@ -5589,3 +9672,5 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte return true; } + + diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h index d56c86da8909..a248e98cebf5 100644 --- a/ggml/src/ggml-quants.h +++ b/ggml/src/ggml-quants.h @@ -27,6 +27,7 @@ GGML_API void quantize_row_nvfp4_ref(const float * GGML_RESTRICT x, block_nvfp4 GGML_API void quantize_row_q2_K_ref(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q3_K_ref(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k); +GGML_API void quantize_row_q3_kpt_ref(const float * GGML_RESTRICT x, block_q3_kpt * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q4_K_ref(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q5_K_ref(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_q6_K_ref(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k); @@ -42,36 +43,37 @@ GGML_API void quantize_row_iq3_s_ref (const float * GGML_RESTRICT x, block_iq3_ GGML_API void quantize_row_iq2_s_ref (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k); // Dequantization -GGML_API void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -//GGML_API void dequantize_row_q8_1(const block_q8_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); - -GGML_API void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); - -GGML_API void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); - -GGML_API void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); - -GGML_API void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_iq1_m (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_iq4_xs (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); -GGML_API void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +//GGML_API void dequantize_row_q8_1(const block_q8_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); + +GGML_API void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); + +GGML_API void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_q3_kpt(const block_q3_kpt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); + +GGML_API void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); + +GGML_API void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_iq1_m (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_iq4_xs (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); // Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization") GGML_API size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); @@ -82,6 +84,14 @@ GGML_API size_t quantize_iq1_s (const float * GGML_RESTRICT src, void * GGML_RE GGML_API size_t quantize_iq1_m (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_iq4_nl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_q3_kpt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + +// Q3_KPT level management +GGML_API void q3kpt_set_levels(const float * levels); +GGML_API const float * q3kpt_get_levels(void); +GGML_API void q3kpt_free_levels(void); +GGML_API void q3kpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float levels_out[Q3KPT_N_LEVELS]); GGML_API size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_tq1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); @@ -102,6 +112,198 @@ GGML_API size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTR GGML_API size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API void quantize_row_q3_pt_ref(const float * GGML_RESTRICT x, block_q3_pt * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_q3_pt(const block_q3_pt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API size_t quantize_q3_pt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + +// Q3_PT levels management (per-tensor Lloyd-Max levels in [0,1]) +GGML_API void q3pt_set_levels(const float * levels); // set global levels (quantization) +GGML_API const float * q3pt_get_levels(void); +GGML_API void q3pt_free_levels(void); + +// Per-tensor levels registry (inference — range-based lookup by data address) + +// Train 8 Lloyd-Max levels from tensor data via weighted k-means on affine-normalized +// 16-element sub-block values. Also sets the global levels via q3pt_set_levels(). +// data: float array [nrow * n_per_row], imatrix: importance weights [n_per_row] or NULL. +GGML_API void q3pt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float levels_out[8]); + +// Q4_DPT: IQ4_NL with learned per-tensor int8 levels +GGML_API void dequantize_row_q4_dpt(const block_q4_dpt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void quantize_row_q4_dpt_ref(const float * GGML_RESTRICT x, block_q4_dpt * GGML_RESTRICT y, int64_t k); +GGML_API size_t quantize_q4_dpt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + +// Q4_DPT levels management (per-tensor Lloyd-Max int8 levels) +GGML_API void q4dpt_set_levels(const int8_t * levels); +GGML_API const int8_t * q4dpt_get_levels(void); +GGML_API void q4dpt_free_levels(void); + +// Q2_DPT: 2-bit with learned per-tensor int8 levels +GGML_API void dequantize_row_q2_dpt(const block_q2_dpt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void quantize_row_q2_dpt_ref(const float * GGML_RESTRICT x, block_q2_dpt * GGML_RESTRICT y, int64_t k); +GGML_API size_t quantize_q2_dpt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + +// Q2_DPT levels management (per-tensor Lloyd-Max int8 levels) +GGML_API void q2dpt_set_levels(const int8_t * levels); +GGML_API const int8_t * q2dpt_get_levels(void); +GGML_API void q2dpt_free_levels(void); +GGML_API void q2dpt_set_quant_strategy(int s); + +// Train 4 Lloyd-Max int8 levels from tensor data for Q2_DPT. +// Bins normalized values (x/amax) in [-1,1], runs weighted k-means, rounds to sorted int8[4]. +// Also sets the global levels via q2dpt_set_levels(). +GGML_API void q2dpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, int8_t levels_out[Q2DPT_N_LEVELS]); + +// Q2_KPT: Q2_K with learned per-tensor float levels +GGML_API void dequantize_row_q2_kpt(const block_q2_kpt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void quantize_row_q2_kpt_ref(const float * GGML_RESTRICT x, block_q2_kpt * GGML_RESTRICT y, int64_t k); +GGML_API size_t quantize_q2_kpt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t start_row, int64_t nrows, int64_t n_per_row, const float * imatrix); + +// Q2_KPT levels management (per-tensor float levels in [0,1]) +GGML_API void q2kpt_set_levels(const float * levels); +GGML_API const float * q2kpt_get_levels(void); +GGML_API void q2kpt_free_levels(void); +// Prepare levels buffer for a tensor with given dimensions (call before parallel quantization) +GGML_API void q2kpt_prepare_levels(int64_t nrows, int64_t n_per_row); + +// Train 4 Lloyd-Max float levels from tensor data for Q2_KPT. +// Bins normalized sub-block values in [0,1], runs weighted k-means for 4 centroids. +// Also sets the global levels via q2kpt_set_levels(). +GGML_API void q2kpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float levels_out[Q2KPT_N_LEVELS]); + +// Train per-row levels for all rows: writes nrow * Q2KPT_N_LEVELS floats to out_levels. +GGML_API void q2kpt_train_all_row_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float * out_levels); + +// IQ2_TQ: 2-bit scalar with per-group asymmetric grid (2.5625 bpw) +GGML_API void dequantize_row_iq2_tq(const block_iq2_tq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void quantize_row_iq2_tq_ref(const float * GGML_RESTRICT x, block_iq2_tq * GGML_RESTRICT y, int64_t k); +GGML_API size_t quantize_iq2_tq(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API void iq2tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t grid_out[64]); +GGML_API void iq2tq_set_grid(const int8_t grid[64]); +GGML_API const int8_t * iq2tq_get_grid(void); + +// IQ3_TQ: 3-bit scalar with per-group asymmetric grid (3.5625 bpw) +GGML_API void dequantize_row_iq3_tq(const block_iq3_tq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void quantize_row_iq3_tq_ref(const float * GGML_RESTRICT x, block_iq3_tq * GGML_RESTRICT y, int64_t k); +GGML_API size_t quantize_iq3_tq(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API void iq3tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t grid_out[IQ3TQ_GRID_SIZE]); +GGML_API void iq3tq_set_grid(const int8_t grid[IQ3TQ_GRID_SIZE]); +GGML_API const int8_t * iq3tq_get_grid(void); + +// IQ1_BN: 8D vector quantized with per-tensor trained codebook (1.5625 bpw) +GGML_API void dequantize_row_iq1_bn(const block_iq1_bn * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void quantize_row_iq1_bn_ref(const float * GGML_RESTRICT x, block_iq1_bn * GGML_RESTRICT y, int64_t k); +GGML_API size_t quantize_iq1_bn(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t aux_out[IQ1BN_AUX_SIZE], int nthread); +GGML_API void iq1bn_set_aux(const int8_t aux[IQ1BN_AUX_SIZE]); +GGML_API const int8_t * iq1bn_get_aux(void); + +// Train 16 Lloyd-Max int8 levels from tensor data. +// Bins normalized values (x/amax) in [-1,1], runs weighted k-means, rounds to sorted int8[16]. +// Also sets the global levels via q4dpt_set_levels(). +GGML_API void q4dpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, int8_t levels_out[Q4DPT_N_LEVELS]); + +GGML_API void quantize_row_q3_pt_ref(const float * GGML_RESTRICT x, block_q3_pt * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_q3_pt(const block_q3_pt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API size_t quantize_q3_pt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + +// Q3_PT levels management (per-tensor Lloyd-Max levels in [0,1]) +GGML_API void q3pt_set_levels(const float * levels); // set global levels (quantization) +GGML_API const float * q3pt_get_levels(void); +GGML_API void q3pt_free_levels(void); + +// Per-tensor levels registry (inference — range-based lookup by data address) + +// Train 8 Lloyd-Max levels from tensor data via weighted k-means on affine-normalized +// 16-element sub-block values. Also sets the global levels via q3pt_set_levels(). +// data: float array [nrow * n_per_row], imatrix: importance weights [n_per_row] or NULL. +GGML_API void q3pt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float levels_out[8]); + +// Q4_DPT: IQ4_NL with learned per-tensor int8 levels +GGML_API void dequantize_row_q4_dpt(const block_q4_dpt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void quantize_row_q4_dpt_ref(const float * GGML_RESTRICT x, block_q4_dpt * GGML_RESTRICT y, int64_t k); +GGML_API size_t quantize_q4_dpt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + +// Q4_DPT levels management (per-tensor Lloyd-Max int8 levels) +GGML_API void q4dpt_set_levels(const int8_t * levels); +GGML_API const int8_t * q4dpt_get_levels(void); +GGML_API void q4dpt_free_levels(void); + +// Q2_DPT: 2-bit with learned per-tensor int8 levels +GGML_API void dequantize_row_q2_dpt(const block_q2_dpt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void quantize_row_q2_dpt_ref(const float * GGML_RESTRICT x, block_q2_dpt * GGML_RESTRICT y, int64_t k); +GGML_API size_t quantize_q2_dpt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + +// Q2_DPT levels management (per-tensor Lloyd-Max int8 levels) +GGML_API void q2dpt_set_levels(const int8_t * levels); +GGML_API const int8_t * q2dpt_get_levels(void); +GGML_API void q2dpt_free_levels(void); +GGML_API void q2dpt_set_quant_strategy(int s); + +// Train 4 Lloyd-Max int8 levels from tensor data for Q2_DPT. +// Bins normalized values (x/amax) in [-1,1], runs weighted k-means, rounds to sorted int8[4]. +// Also sets the global levels via q2dpt_set_levels(). +GGML_API void q2dpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, int8_t levels_out[Q2DPT_N_LEVELS]); + +// Q2_KPT: Q2_K with learned per-tensor float levels +GGML_API void dequantize_row_q2_kpt(const block_q2_kpt * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void quantize_row_q2_kpt_ref(const float * GGML_RESTRICT x, block_q2_kpt * GGML_RESTRICT y, int64_t k); +GGML_API size_t quantize_q2_kpt(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t start_row, int64_t nrows, int64_t n_per_row, const float * imatrix); + +// Q2_KPT levels management (per-tensor float levels in [0,1]) +GGML_API void q2kpt_set_levels(const float * levels); +GGML_API const float * q2kpt_get_levels(void); +GGML_API void q2kpt_free_levels(void); +// Prepare levels buffer for a tensor with given dimensions (call before parallel quantization) +GGML_API void q2kpt_prepare_levels(int64_t nrows, int64_t n_per_row); + +// Train 4 Lloyd-Max float levels from tensor data for Q2_KPT. +// Bins normalized sub-block values in [0,1], runs weighted k-means for 4 centroids. +// Also sets the global levels via q2kpt_set_levels(). +GGML_API void q2kpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float levels_out[Q2KPT_N_LEVELS]); + +// Train per-row levels for all rows: writes nrow * Q2KPT_N_LEVELS floats to out_levels. +GGML_API void q2kpt_train_all_row_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float * out_levels); + +// IQ2_TQ: 2-bit scalar with per-group asymmetric grid (2.5625 bpw) +GGML_API void dequantize_row_iq2_tq(const block_iq2_tq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void quantize_row_iq2_tq_ref(const float * GGML_RESTRICT x, block_iq2_tq * GGML_RESTRICT y, int64_t k); +GGML_API size_t quantize_iq2_tq(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API void iq2tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t grid_out[64]); +GGML_API void iq2tq_set_grid(const int8_t grid[64]); +GGML_API const int8_t * iq2tq_get_grid(void); + +// IQ3_TQ: 3-bit scalar with per-group asymmetric grid (3.5625 bpw) +GGML_API void dequantize_row_iq3_tq(const block_iq3_tq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void quantize_row_iq3_tq_ref(const float * GGML_RESTRICT x, block_iq3_tq * GGML_RESTRICT y, int64_t k); +GGML_API size_t quantize_iq3_tq(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API void iq3tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t grid_out[IQ3TQ_GRID_SIZE]); +GGML_API void iq3tq_set_grid(const int8_t grid[IQ3TQ_GRID_SIZE]); +GGML_API const int8_t * iq3tq_get_grid(void); + +// IQ1_BN: 8D vector quantized with per-tensor trained codebook (1.5625 bpw) +GGML_API void dequantize_row_iq1_bn(const block_iq1_bn * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels); +GGML_API void quantize_row_iq1_bn_ref(const float * GGML_RESTRICT x, block_iq1_bn * GGML_RESTRICT y, int64_t k); +GGML_API size_t quantize_iq1_bn(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t aux_out[IQ1BN_AUX_SIZE], int nthread); +GGML_API void iq1bn_set_aux(const int8_t aux[IQ1BN_AUX_SIZE]); +GGML_API const int8_t * iq1bn_get_aux(void); + +// Train 16 Lloyd-Max int8 levels from tensor data. +// Bins normalized values (x/amax) in [-1,1], runs weighted k-means, rounds to sorted int8[16]. +// Also sets the global levels via q4dpt_set_levels(). +GGML_API void q4dpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, int8_t levels_out[Q4DPT_N_LEVELS]); + GGML_API void iq2xs_init_impl(enum ggml_type type); GGML_API void iq2xs_free_impl(enum ggml_type type); GGML_API void iq3xs_init_impl(int grid_size); diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index c0ab9f1c6584..f43975deaa92 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -13906,7 +13906,7 @@ static void ggml_vk_quantize_data(const float * from, void * to, size_t ne, ggml ggml_quantize_chunk(quant, from, to, 0, 1, ne, nullptr); } -static void ggml_vk_dequantize_data(const void * from, float * to, size_t ne, ggml_type quant) { +static void ggml_vk_dequantize_data(const void * from, float * to, size_t ne, ggml_type quant, const void * levels = nullptr) { if (quant == GGML_TYPE_F32) { memcpy(to, from, sizeof(float) * ne); return; @@ -13916,7 +13916,7 @@ static void ggml_vk_dequantize_data(const void * from, float * to, size_t ne, gg ggml_to_float_t dequant_fn = tt->to_float; - dequant_fn(from, to, ne); + dequant_fn(from, to, ne, levels); } static void ggml_vk_test_dequant(ggml_backend_vk_context * ctx, size_t ne, ggml_type quant) { diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 0f682fd1856c..a480e93d58b6 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -471,6 +471,11 @@ void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) { } } +static void ggml_fp16_to_fp32_row_leveled(const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t n, const void * levels) { + GGML_UNUSED(levels); + ggml_fp16_to_fp32_row((const ggml_fp16_t *)x, y, n); +} + void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) { int i = 0; for (; i < n; ++i) { @@ -485,6 +490,11 @@ void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) { } } +static void ggml_bf16_to_fp32_row_leveled(const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t n, const void * levels) { + GGML_UNUSED(levels); + ggml_bf16_to_fp32_row((const ggml_bf16_t *)x, y, n); +} + void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n) { for (int i = 0; i < n; i++) { y[i] = ggml_compute_fp32_to_bf16(x[i]); @@ -660,7 +670,7 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .blck_size = 1, .type_size = sizeof(ggml_fp16_t), .is_quantized = false, - .to_float = (ggml_to_float_t) ggml_fp16_to_fp32_row, + .to_float = ggml_fp16_to_fp32_row_leveled, .from_float_ref = (ggml_from_float_t) ggml_fp32_to_fp16_row, }, [GGML_TYPE_Q1_0] = { @@ -869,7 +879,7 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .blck_size = 1, .type_size = sizeof(ggml_bf16_t), .is_quantized = false, - .to_float = (ggml_to_float_t) ggml_bf16_to_fp32_row, + .to_float = ggml_bf16_to_fp32_row_leveled, .from_float_ref = (ggml_from_float_t) ggml_fp32_to_bf16_row_ref, }, [31] = { // GGML_TYPE_Q4_0_4_4 @@ -924,6 +934,71 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .type_size = 0, .is_quantized = false, }, + [GGML_TYPE_Q3_PT] = { + .type_name = "q3_pt", + .blck_size = QK_K, + .type_size = sizeof(block_q3_pt), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_q3_pt, + .from_float_ref = (ggml_from_float_t) quantize_row_q3_pt_ref, + }, + [GGML_TYPE_Q3_KPT] = { + .type_name = "q3_kpt", + .blck_size = QK_K, + .type_size = sizeof(block_q3_kpt), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_q3_kpt, + .from_float_ref = (ggml_from_float_t) quantize_row_q3_kpt_ref, + }, + [GGML_TYPE_Q4_DPT] = { + .type_name = "q4_dpt", + .blck_size = QK4_NL, + .type_size = sizeof(block_q4_dpt), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_q4_dpt, + .from_float_ref = (ggml_from_float_t) quantize_row_q4_dpt_ref, + }, + [GGML_TYPE_Q2_DPT] = { + .type_name = "q2_dpt", + .blck_size = QK2_DPT, + .type_size = sizeof(block_q2_dpt), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_q2_dpt, + .from_float_ref = (ggml_from_float_t) quantize_row_q2_dpt_ref, + }, + [GGML_TYPE_Q2_KPT] = { + .type_name = "q2_kpt", + .blck_size = QK_K, + .type_size = sizeof(block_q2_kpt), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_q2_kpt, + .from_float_ref = (ggml_from_float_t) quantize_row_q2_kpt_ref, + .levels_row_stride = 0, // computed dynamically: (ne[0]/256)*4*sizeof(float) + }, + [GGML_TYPE_IQ2_TQ] = { + .type_name = "iq2_tq", + .blck_size = QK_K, + .type_size = sizeof(block_iq2_tq), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_iq2_tq, + .from_float_ref = (ggml_from_float_t) quantize_row_iq2_tq_ref, + }, + [GGML_TYPE_IQ3_TQ] = { + .type_name = "iq3_tq", + .blck_size = QK_K, + .type_size = sizeof(block_iq3_tq), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_iq3_tq, + .from_float_ref = (ggml_from_float_t) quantize_row_iq3_tq_ref, + }, + [GGML_TYPE_IQ1_BN] = { + .type_name = "iq1_bn", + .blck_size = QK_K, + .type_size = sizeof(block_iq1_bn), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_iq1_bn, + .from_float_ref = (ggml_from_float_t) quantize_row_iq1_bn_ref, + }, }; const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type) { @@ -1426,6 +1501,10 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_IQ4_XS: wtype = GGML_TYPE_IQ4_XS; break; case GGML_FTYPE_MOSTLY_IQ3_S: wtype = GGML_TYPE_IQ3_S; break; case GGML_FTYPE_MOSTLY_IQ2_S: wtype = GGML_TYPE_IQ2_S; break; + case GGML_FTYPE_MOSTLY_Q3_PT: wtype = GGML_TYPE_Q3_PT; break; + case GGML_FTYPE_MOSTLY_Q3_KPT: wtype = GGML_TYPE_Q3_KPT; break; + case GGML_FTYPE_MOSTLY_Q4_DPT: wtype = GGML_TYPE_Q4_DPT; break; + case GGML_FTYPE_MOSTLY_Q2_KPT: wtype = GGML_TYPE_Q2_KPT; break; case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break; case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: wtype = GGML_TYPE_COUNT; break; } @@ -7674,6 +7753,13 @@ void ggml_quantize_init(enum ggml_type type) { case GGML_TYPE_IQ1_M: iq2xs_init_impl(type); break; case GGML_TYPE_IQ3_XXS: iq3xs_init_impl(256); break; case GGML_TYPE_IQ3_S: iq3xs_init_impl(512); break; + case GGML_TYPE_IQ2_TQ: break; // per-tensor grid stored in tensor->quant_levels + case GGML_TYPE_IQ3_TQ: break; // per-tensor grid stored in tensor->quant_levels + case GGML_TYPE_IQ1_BN: break; // per-tensor codebook stored in tensor->quant_levels + case GGML_TYPE_Q3_PT: break; // levels stored in tensor->quant_levels + case GGML_TYPE_Q3_KPT: break; // levels stored in tensor->quant_levels + case GGML_TYPE_Q4_DPT: break; // levels stored in tensor->quant_levels + case GGML_TYPE_Q2_KPT: break; // levels stored in tensor->quant_levels default: // nothing break; } @@ -7752,6 +7838,13 @@ size_t ggml_quantize_chunk( case GGML_TYPE_IQ1_M: result = quantize_iq1_m (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_IQ4_NL: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_IQ4_XS: result = quantize_iq4_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q3_PT: result = quantize_q3_pt (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q3_KPT: result = quantize_q3_kpt (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q4_DPT: result = quantize_q4_dpt (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_Q2_KPT: result = quantize_q2_kpt (src + start, (char *) dst + start_row * row_size, start_row, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ2_TQ: result = quantize_iq2_tq (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ3_TQ: result = quantize_iq3_tq (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ1_BN: result = quantize_iq1_bn (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_F16: { size_t elemsize = sizeof(ggml_fp16_t); diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index 5e1986182515..ba1eb5bca056 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -1463,37 +1463,63 @@ struct gguf_writer_base { if (kv.is_array) { write(GGUF_TYPE_ARRAY); - write(kv.get_type()); + const enum gguf_type elem_type = kv.get_type(); + write(elem_type); write(ne); + // Write array element data based on element type + switch (elem_type) { + case GGUF_TYPE_UINT8: + case GGUF_TYPE_INT8: + case GGUF_TYPE_UINT16: + case GGUF_TYPE_INT16: + case GGUF_TYPE_UINT32: + case GGUF_TYPE_INT32: + case GGUF_TYPE_FLOAT32: + case GGUF_TYPE_UINT64: + case GGUF_TYPE_INT64: + case GGUF_TYPE_FLOAT64: { + // Write raw bytes inline for array data + for (size_t i = 0; i < kv.data.size(); ++i) { + write(kv.data[i]); + } + } break; + case GGUF_TYPE_BOOL: { + for (size_t i = 0; i < ne; ++i) { + write(kv.get_val(i)); + } + } break; + case GGUF_TYPE_STRING: { + for (size_t i = 0; i < ne; ++i) { + write(kv.get_val(i)); + } + } break; + case GGUF_TYPE_ARRAY: + default: GGML_ABORT("invalid array element type"); + } } else { write(kv.get_type()); - } - - switch (kv.get_type()) { - case GGUF_TYPE_UINT8: - case GGUF_TYPE_INT8: - case GGUF_TYPE_UINT16: - case GGUF_TYPE_INT16: - case GGUF_TYPE_UINT32: - case GGUF_TYPE_INT32: - case GGUF_TYPE_FLOAT32: - case GGUF_TYPE_UINT64: - case GGUF_TYPE_INT64: - case GGUF_TYPE_FLOAT64: { - write(kv.data); - } break; - case GGUF_TYPE_BOOL: { - for (size_t i = 0; i < ne; ++i) { - write(kv.get_val(i)); - } - } break; - case GGUF_TYPE_STRING: { - for (size_t i = 0; i < ne; ++i) { - write(kv.get_val(i)); - } - } break; - case GGUF_TYPE_ARRAY: - default: GGML_ABORT("invalid type"); + switch (kv.get_type()) { + case GGUF_TYPE_UINT8: + case GGUF_TYPE_INT8: + case GGUF_TYPE_UINT16: + case GGUF_TYPE_INT16: + case GGUF_TYPE_UINT32: + case GGUF_TYPE_INT32: + case GGUF_TYPE_FLOAT32: + case GGUF_TYPE_UINT64: + case GGUF_TYPE_INT64: + case GGUF_TYPE_FLOAT64: { + write(kv.data); + } break; + case GGUF_TYPE_BOOL: { + write(kv.get_val(0)); + } break; + case GGUF_TYPE_STRING: { + write(kv.get_val(0)); + } break; + case GGUF_TYPE_ARRAY: + default: GGML_ABORT("invalid type"); + } } } diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index cd4cdef8991f..6015694083a6 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -7,10 +7,10 @@ # constants # -GGUF_MAGIC = 0x46554747 # "GGUF" -GGUF_VERSION = 3 +GGUF_MAGIC = 0x46554747 # "GGUF" +GGUF_VERSION = 3 GGUF_DEFAULT_ALIGNMENT = 32 -GGML_QUANT_VERSION = 2 # GGML_QNT_VERSION from ggml.h +GGML_QUANT_VERSION = 2 # GGML_QNT_VERSION from ggml.h # # metadata keys @@ -19,97 +19,101 @@ class Keys: class General: - TYPE = "general.type" - ARCHITECTURE = "general.architecture" - QUANTIZATION_VERSION = "general.quantization_version" - ALIGNMENT = "general.alignment" - FILE_TYPE = "general.file_type" + TYPE = "general.type" + ARCHITECTURE = "general.architecture" + QUANTIZATION_VERSION = "general.quantization_version" + ALIGNMENT = "general.alignment" + FILE_TYPE = "general.file_type" # Recommended Sampler Parameters - SAMPLING_SEQUENCE = "general.sampling.sequence" - SAMPLING_TOP_K = "general.sampling.top_k" - SAMPLING_TOP_P = "general.sampling.top_p" - SAMPLING_MIN_P = "general.sampling.min_p" - SAMPLING_XTC_PROBABILITY = "general.sampling.xtc_probability" - SAMPLING_XTC_THRESHOLD = "general.sampling.xtc_threshold" - SAMPLING_TEMP = "general.sampling.temp" - SAMPLING_PENALTY_LAST_N = "general.sampling.penalty_last_n" - SAMPLING_PENALTY_REPEAT = "general.sampling.penalty_repeat" - SAMPLING_MIROSTAT = "general.sampling.mirostat" - SAMPLING_MIROSTAT_TAU = "general.sampling.mirostat_tau" - SAMPLING_MIROSTAT_ETA = "general.sampling.mirostat_eta" + SAMPLING_SEQUENCE = "general.sampling.sequence" + SAMPLING_TOP_K = "general.sampling.top_k" + SAMPLING_TOP_P = "general.sampling.top_p" + SAMPLING_MIN_P = "general.sampling.min_p" + SAMPLING_XTC_PROBABILITY = "general.sampling.xtc_probability" + SAMPLING_XTC_THRESHOLD = "general.sampling.xtc_threshold" + SAMPLING_TEMP = "general.sampling.temp" + SAMPLING_PENALTY_LAST_N = "general.sampling.penalty_last_n" + SAMPLING_PENALTY_REPEAT = "general.sampling.penalty_repeat" + SAMPLING_MIROSTAT = "general.sampling.mirostat" + SAMPLING_MIROSTAT_TAU = "general.sampling.mirostat_tau" + SAMPLING_MIROSTAT_ETA = "general.sampling.mirostat_eta" # Authorship Metadata - NAME = "general.name" - AUTHOR = "general.author" - VERSION = "general.version" - ORGANIZATION = "general.organization" + NAME = "general.name" + AUTHOR = "general.author" + VERSION = "general.version" + ORGANIZATION = "general.organization" - FINETUNE = "general.finetune" - BASENAME = "general.basename" + FINETUNE = "general.finetune" + BASENAME = "general.basename" - DESCRIPTION = "general.description" - QUANTIZED_BY = "general.quantized_by" + DESCRIPTION = "general.description" + QUANTIZED_BY = "general.quantized_by" - SIZE_LABEL = "general.size_label" + SIZE_LABEL = "general.size_label" # Licensing details - LICENSE = "general.license" - LICENSE_NAME = "general.license.name" - LICENSE_LINK = "general.license.link" + LICENSE = "general.license" + LICENSE_NAME = "general.license.name" + LICENSE_LINK = "general.license.link" # Typically represents the converted GGUF repo (Unless native) - URL = "general.url" # Model Website/Paper - DOI = "general.doi" - UUID = "general.uuid" - REPO_URL = "general.repo_url" # Model Source Repository (git/svn/etc...) + URL = "general.url" # Model Website/Paper + DOI = "general.doi" + UUID = "general.uuid" + REPO_URL = "general.repo_url" # Model Source Repository (git/svn/etc...) # Model Source during conversion - SOURCE_URL = "general.source.url" # Model Website/Paper - SOURCE_DOI = "general.source.doi" - SOURCE_UUID = "general.source.uuid" - SOURCE_REPO_URL = "general.source.repo_url" # Model Source Repository (git/svn/etc...) + SOURCE_URL = "general.source.url" # Model Website/Paper + SOURCE_DOI = "general.source.doi" + SOURCE_UUID = "general.source.uuid" + SOURCE_REPO_URL = ( + "general.source.repo_url" # Model Source Repository (git/svn/etc...) + ) # Base Model Source. There can be more than one source if it's a merged # model like with 'Mistral-7B-Merge-14-v0.1'. This will assist in # tracing linage of models as it is finetuned or merged over time. - BASE_MODEL_COUNT = "general.base_model.count" - BASE_MODEL_NAME = "general.base_model.{id}.name" - BASE_MODEL_AUTHOR = "general.base_model.{id}.author" - BASE_MODEL_VERSION = "general.base_model.{id}.version" - BASE_MODEL_ORGANIZATION = "general.base_model.{id}.organization" - BASE_MODEL_DESCRIPTION = "general.base_model.{id}.description" - BASE_MODEL_URL = "general.base_model.{id}.url" # Model Website/Paper - BASE_MODEL_DOI = "general.base_model.{id}.doi" - BASE_MODEL_UUID = "general.base_model.{id}.uuid" - BASE_MODEL_REPO_URL = "general.base_model.{id}.repo_url" # Model Source Repository (git/svn/etc...) + BASE_MODEL_COUNT = "general.base_model.count" + BASE_MODEL_NAME = "general.base_model.{id}.name" + BASE_MODEL_AUTHOR = "general.base_model.{id}.author" + BASE_MODEL_VERSION = "general.base_model.{id}.version" + BASE_MODEL_ORGANIZATION = "general.base_model.{id}.organization" + BASE_MODEL_DESCRIPTION = "general.base_model.{id}.description" + BASE_MODEL_URL = "general.base_model.{id}.url" # Model Website/Paper + BASE_MODEL_DOI = "general.base_model.{id}.doi" + BASE_MODEL_UUID = "general.base_model.{id}.uuid" + BASE_MODEL_REPO_URL = "general.base_model.{id}.repo_url" # Model Source Repository (git/svn/etc...) # Dataset Source - DATASET_COUNT = "general.dataset.count" - DATASET_NAME = "general.dataset.{id}.name" - DATASET_AUTHOR = "general.dataset.{id}.author" - DATASET_VERSION = "general.dataset.{id}.version" - DATASET_ORGANIZATION = "general.dataset.{id}.organization" - DATASET_DESCRIPTION = "general.dataset.{id}.description" - DATASET_URL = "general.dataset.{id}.url" # Model Website/Paper - DATASET_DOI = "general.dataset.{id}.doi" - DATASET_UUID = "general.dataset.{id}.uuid" - DATASET_REPO_URL = "general.dataset.{id}.repo_url" # Model Source Repository (git/svn/etc...) + DATASET_COUNT = "general.dataset.count" + DATASET_NAME = "general.dataset.{id}.name" + DATASET_AUTHOR = "general.dataset.{id}.author" + DATASET_VERSION = "general.dataset.{id}.version" + DATASET_ORGANIZATION = "general.dataset.{id}.organization" + DATASET_DESCRIPTION = "general.dataset.{id}.description" + DATASET_URL = "general.dataset.{id}.url" # Model Website/Paper + DATASET_DOI = "general.dataset.{id}.doi" + DATASET_UUID = "general.dataset.{id}.uuid" + DATASET_REPO_URL = ( + "general.dataset.{id}.repo_url" # Model Source Repository (git/svn/etc...) + ) # Array based KV stores - TAGS = "general.tags" - LANGUAGES = "general.languages" + TAGS = "general.tags" + LANGUAGES = "general.languages" class LLM: - VOCAB_SIZE = "{arch}.vocab_size" - CONTEXT_LENGTH = "{arch}.context_length" - EMBEDDING_LENGTH = "{arch}.embedding_length" - EMBEDDING_LENGTH_OUT = "{arch}.embedding_length_out" - FEATURES_LENGTH = "{arch}.features_length" - BLOCK_COUNT = "{arch}.block_count" - LEADING_DENSE_BLOCK_COUNT = "{arch}.leading_dense_block_count" - FEED_FORWARD_LENGTH = "{arch}.feed_forward_length" - EXPERT_FEED_FORWARD_LENGTH = "{arch}.expert_feed_forward_length" + VOCAB_SIZE = "{arch}.vocab_size" + CONTEXT_LENGTH = "{arch}.context_length" + EMBEDDING_LENGTH = "{arch}.embedding_length" + EMBEDDING_LENGTH_OUT = "{arch}.embedding_length_out" + FEATURES_LENGTH = "{arch}.features_length" + BLOCK_COUNT = "{arch}.block_count" + LEADING_DENSE_BLOCK_COUNT = "{arch}.leading_dense_block_count" + FEED_FORWARD_LENGTH = "{arch}.feed_forward_length" + EXPERT_FEED_FORWARD_LENGTH = "{arch}.expert_feed_forward_length" EXPERT_SHARED_FEED_FORWARD_LENGTH = "{arch}.expert_shared_feed_forward_length" EXPERT_CHUNK_FEED_FORWARD_LENGTH = "{arch}.expert_chunk_feed_forward_length" USE_PARALLEL_RESIDUAL = "{arch}.use_parallel_residual" @@ -161,21 +165,21 @@ class LLM: NORM_BEFORE_RESIDUAL = "{arch}.norm_before_residual" class Attention: - HEAD_COUNT = "{arch}.attention.head_count" - HEAD_COUNT_KV = "{arch}.attention.head_count_kv" - MAX_ALIBI_BIAS = "{arch}.attention.max_alibi_bias" - CLAMP_KQV = "{arch}.attention.clamp_kqv" - KEY_LENGTH = "{arch}.attention.key_length" - VALUE_LENGTH = "{arch}.attention.value_length" - LAYERNORM_EPS = "{arch}.attention.layer_norm_epsilon" - LAYERNORM_RMS_EPS = "{arch}.attention.layer_norm_rms_epsilon" - GROUPNORM_EPS = "{arch}.attention.group_norm_epsilon" - GROUPNORM_GROUPS = "{arch}.attention.group_norm_groups" - CAUSAL = "{arch}.attention.causal" - Q_LORA_RANK = "{arch}.attention.q_lora_rank" - KV_LORA_RANK = "{arch}.attention.kv_lora_rank" - DECAY_LORA_RANK = "{arch}.attention.decay_lora_rank" - ICLR_LORA_RANK = "{arch}.attention.iclr_lora_rank" + HEAD_COUNT = "{arch}.attention.head_count" + HEAD_COUNT_KV = "{arch}.attention.head_count_kv" + MAX_ALIBI_BIAS = "{arch}.attention.max_alibi_bias" + CLAMP_KQV = "{arch}.attention.clamp_kqv" + KEY_LENGTH = "{arch}.attention.key_length" + VALUE_LENGTH = "{arch}.attention.value_length" + LAYERNORM_EPS = "{arch}.attention.layer_norm_epsilon" + LAYERNORM_RMS_EPS = "{arch}.attention.layer_norm_rms_epsilon" + GROUPNORM_EPS = "{arch}.attention.group_norm_epsilon" + GROUPNORM_GROUPS = "{arch}.attention.group_norm_groups" + CAUSAL = "{arch}.attention.causal" + Q_LORA_RANK = "{arch}.attention.q_lora_rank" + KV_LORA_RANK = "{arch}.attention.kv_lora_rank" + DECAY_LORA_RANK = "{arch}.attention.decay_lora_rank" + ICLR_LORA_RANK = "{arch}.attention.iclr_lora_rank" VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank" GATE_LORA_RANK = "{arch}.attention.gate_lora_rank" REL_BUCKETS_COUNT = "{arch}.attention.relative_buckets_count" @@ -199,7 +203,7 @@ class Attention: class Indexer: HEAD_COUNT = "{arch}.attention.indexer.head_count" KEY_LENGTH = "{arch}.attention.indexer.key_length" - TOP_K = "{arch}.attention.indexer.top_k" + TOP_K = "{arch}.attention.indexer.top_k" class HyperConnection: COUNT = "{arch}.hyper_connection.count" @@ -207,35 +211,35 @@ class HyperConnection: EPSILON = "{arch}.hyper_connection.epsilon" class Rope: - DIMENSION_COUNT = "{arch}.rope.dimension_count" - DIMENSION_COUNT_SWA = "{arch}.rope.dimension_count_swa" - DIMENSION_SECTIONS = "{arch}.rope.dimension_sections" - FREQ_BASE = "{arch}.rope.freq_base" - FREQ_BASE_SWA = "{arch}.rope.freq_base_swa" - SCALING_TYPE = "{arch}.rope.scaling.type" - SCALING_FACTOR = "{arch}.rope.scaling.factor" + DIMENSION_COUNT = "{arch}.rope.dimension_count" + DIMENSION_COUNT_SWA = "{arch}.rope.dimension_count_swa" + DIMENSION_SECTIONS = "{arch}.rope.dimension_sections" + FREQ_BASE = "{arch}.rope.freq_base" + FREQ_BASE_SWA = "{arch}.rope.freq_base_swa" + SCALING_TYPE = "{arch}.rope.scaling.type" + SCALING_FACTOR = "{arch}.rope.scaling.factor" SCALING_ALPHA = "{arch}.rope.scaling.alpha" - SCALING_ATTN_FACTOR = "{arch}.rope.scaling.attn_factor" - SCALING_ORIG_CTX_LEN = "{arch}.rope.scaling.original_context_length" - SCALING_FINETUNED = "{arch}.rope.scaling.finetuned" - SCALING_YARN_LOG_MUL = "{arch}.rope.scaling.yarn_log_multiplier" - SCALING_YARN_EXT_FACTOR = "{arch}.rope.scaling.yarn_ext_factor" - SCALING_YARN_ATTN_FACTOR = "{arch}.rope.scaling.yarn_attn_factor" - SCALING_YARN_BETA_FAST = "{arch}.rope.scaling.yarn_beta_fast" - SCALING_YARN_BETA_SLOW = "{arch}.rope.scaling.yarn_beta_slow" + SCALING_ATTN_FACTOR = "{arch}.rope.scaling.attn_factor" + SCALING_ORIG_CTX_LEN = "{arch}.rope.scaling.original_context_length" + SCALING_FINETUNED = "{arch}.rope.scaling.finetuned" + SCALING_YARN_LOG_MUL = "{arch}.rope.scaling.yarn_log_multiplier" + SCALING_YARN_EXT_FACTOR = "{arch}.rope.scaling.yarn_ext_factor" + SCALING_YARN_ATTN_FACTOR = "{arch}.rope.scaling.yarn_attn_factor" + SCALING_YARN_BETA_FAST = "{arch}.rope.scaling.yarn_beta_fast" + SCALING_YARN_BETA_SLOW = "{arch}.rope.scaling.yarn_beta_slow" class Split: - LLM_KV_SPLIT_NO = "split.no" - LLM_KV_SPLIT_COUNT = "split.count" + LLM_KV_SPLIT_NO = "split.no" + LLM_KV_SPLIT_COUNT = "split.count" LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count" class SSM: - CONV_KERNEL = "{arch}.ssm.conv_kernel" - INNER_SIZE = "{arch}.ssm.inner_size" - STATE_SIZE = "{arch}.ssm.state_size" + CONV_KERNEL = "{arch}.ssm.conv_kernel" + INNER_SIZE = "{arch}.ssm.inner_size" + STATE_SIZE = "{arch}.ssm.state_size" TIME_STEP_RANK = "{arch}.ssm.time_step_rank" - GROUP_COUNT = "{arch}.ssm.group_count" - DT_B_C_RMS = "{arch}.ssm.dt_b_c_rms" + GROUP_COUNT = "{arch}.ssm.group_count" + DT_B_C_RMS = "{arch}.ssm.dt_b_c_rms" class KDA: HEAD_DIM = "{arch}.kda.head_dim" @@ -245,11 +249,11 @@ class WKV: class PosNet: EMBEDDING_LENGTH = "{arch}.posnet.embedding_length" - BLOCK_COUNT = "{arch}.posnet.block_count" + BLOCK_COUNT = "{arch}.posnet.block_count" class ConvNext: EMBEDDING_LENGTH = "{arch}.convnext.embedding_length" - BLOCK_COUNT = "{arch}.convnext.block_count" + BLOCK_COUNT = "{arch}.convnext.block_count" class Classifier: OUTPUT_LABELS = "{arch}.classifier.output_labels" @@ -258,26 +262,28 @@ class ShortConv: L_CACHE = "{arch}.shortconv.l_cache" class Tokenizer: - MODEL = "tokenizer.ggml.model" - PRE = "tokenizer.ggml.pre" - LIST = "tokenizer.ggml.tokens" - TOKEN_TYPE = "tokenizer.ggml.token_type" - TOKEN_TYPE_COUNT = "tokenizer.ggml.token_type_count" # for BERT-style token types - SCORES = "tokenizer.ggml.scores" - MERGES = "tokenizer.ggml.merges" - BOS_ID = "tokenizer.ggml.bos_token_id" - EOS_ID = "tokenizer.ggml.eos_token_id" - EOT_ID = "tokenizer.ggml.eot_token_id" - EOM_ID = "tokenizer.ggml.eom_token_id" - UNK_ID = "tokenizer.ggml.unknown_token_id" - SEP_ID = "tokenizer.ggml.seperator_token_id" - PAD_ID = "tokenizer.ggml.padding_token_id" - MASK_ID = "tokenizer.ggml.mask_token_id" - ADD_BOS = "tokenizer.ggml.add_bos_token" - ADD_EOS = "tokenizer.ggml.add_eos_token" - ADD_SEP = "tokenizer.ggml.add_sep_token" - ADD_PREFIX = "tokenizer.ggml.add_space_prefix" - REMOVE_EXTRA_WS = "tokenizer.ggml.remove_extra_whitespaces" + MODEL = "tokenizer.ggml.model" + PRE = "tokenizer.ggml.pre" + LIST = "tokenizer.ggml.tokens" + TOKEN_TYPE = "tokenizer.ggml.token_type" + TOKEN_TYPE_COUNT = ( + "tokenizer.ggml.token_type_count" # for BERT-style token types + ) + SCORES = "tokenizer.ggml.scores" + MERGES = "tokenizer.ggml.merges" + BOS_ID = "tokenizer.ggml.bos_token_id" + EOS_ID = "tokenizer.ggml.eos_token_id" + EOT_ID = "tokenizer.ggml.eot_token_id" + EOM_ID = "tokenizer.ggml.eom_token_id" + UNK_ID = "tokenizer.ggml.unknown_token_id" + SEP_ID = "tokenizer.ggml.seperator_token_id" + PAD_ID = "tokenizer.ggml.padding_token_id" + MASK_ID = "tokenizer.ggml.mask_token_id" + ADD_BOS = "tokenizer.ggml.add_bos_token" + ADD_EOS = "tokenizer.ggml.add_eos_token" + ADD_SEP = "tokenizer.ggml.add_sep_token" + ADD_PREFIX = "tokenizer.ggml.add_space_prefix" + REMOVE_EXTRA_WS = "tokenizer.ggml.remove_extra_whitespaces" PRECOMPILED_CHARSMAP = "tokenizer.ggml.precompiled_charsmap" SUPPRESS_TOKENS = "tokenizer.ggml.suppress_tokens" HF_JSON = "tokenizer.huggingface.json" @@ -289,55 +295,55 @@ class Tokenizer: NORMALIZER_LOWERCASE = "tokenizer.ggml.normalizer.lowercase" NORMALIZER_STRIP_ACCENTS = "tokenizer.ggml.normalizer.strip_accents" # FIM/Infill special tokens constants - FIM_PRE_ID = "tokenizer.ggml.fim_pre_token_id" - FIM_SUF_ID = "tokenizer.ggml.fim_suf_token_id" - FIM_MID_ID = "tokenizer.ggml.fim_mid_token_id" - FIM_PAD_ID = "tokenizer.ggml.fim_pad_token_id" - FIM_REP_ID = "tokenizer.ggml.fim_rep_token_id" - FIM_SEP_ID = "tokenizer.ggml.fim_sep_token_id" + FIM_PRE_ID = "tokenizer.ggml.fim_pre_token_id" + FIM_SUF_ID = "tokenizer.ggml.fim_suf_token_id" + FIM_MID_ID = "tokenizer.ggml.fim_mid_token_id" + FIM_PAD_ID = "tokenizer.ggml.fim_pad_token_id" + FIM_REP_ID = "tokenizer.ggml.fim_rep_token_id" + FIM_SEP_ID = "tokenizer.ggml.fim_sep_token_id" # deprecated: - PREFIX_ID = "tokenizer.ggml.prefix_token_id" - SUFFIX_ID = "tokenizer.ggml.suffix_token_id" - MIDDLE_ID = "tokenizer.ggml.middle_token_id" + PREFIX_ID = "tokenizer.ggml.prefix_token_id" + SUFFIX_ID = "tokenizer.ggml.suffix_token_id" + MIDDLE_ID = "tokenizer.ggml.middle_token_id" class Adapter: - TYPE = "adapter.type" - LORA_ALPHA = "adapter.lora.alpha" - LORA_TASK_NAME = "adapter.lora.task_name" - LORA_PROMPT_PREFIX = "adapter.lora.prompt_prefix" + TYPE = "adapter.type" + LORA_ALPHA = "adapter.lora.alpha" + LORA_TASK_NAME = "adapter.lora.task_name" + LORA_PROMPT_PREFIX = "adapter.lora.prompt_prefix" ALORA_INVOCATION_TOKENS = "adapter.alora.invocation_tokens" class IMatrix: CHUNK_COUNT = "imatrix.chunk_count" - CHUNK_SIZE = "imatrix.chunk_size" - DATASETS = "imatrix.datasets" + CHUNK_SIZE = "imatrix.chunk_size" + DATASETS = "imatrix.datasets" class Clip: - PROJECTOR_TYPE = "clip.projector_type" - HAS_VISION_ENCODER = "clip.has_vision_encoder" - HAS_AUDIO_ENCODER = "clip.has_audio_encoder" - HAS_LLAVA_PROJECTOR = "clip.has_llava_projector" + PROJECTOR_TYPE = "clip.projector_type" + HAS_VISION_ENCODER = "clip.has_vision_encoder" + HAS_AUDIO_ENCODER = "clip.has_audio_encoder" + HAS_LLAVA_PROJECTOR = "clip.has_llava_projector" class ClipVision: - PROJECTOR_TYPE = "clip.vision.projector_type" # for mixed modality models - IMAGE_SIZE = "clip.vision.image_size" - IMAGE_MIN_PIXELS = "clip.vision.image_min_pixels" - IMAGE_MAX_PIXELS = "clip.vision.image_max_pixels" - PREPROC_MIN_TILES = "clip.vision.preproc_min_tiles" - PREPROC_MAX_TILES = "clip.vision.preproc_max_tiles" - PREPROC_IMAGE_SIZE = "clip.vision.preproc_image_size" - PATCH_SIZE = "clip.vision.patch_size" - EMBEDDING_LENGTH = "clip.vision.embedding_length" + PROJECTOR_TYPE = "clip.vision.projector_type" # for mixed modality models + IMAGE_SIZE = "clip.vision.image_size" + IMAGE_MIN_PIXELS = "clip.vision.image_min_pixels" + IMAGE_MAX_PIXELS = "clip.vision.image_max_pixels" + PREPROC_MIN_TILES = "clip.vision.preproc_min_tiles" + PREPROC_MAX_TILES = "clip.vision.preproc_max_tiles" + PREPROC_IMAGE_SIZE = "clip.vision.preproc_image_size" + PATCH_SIZE = "clip.vision.patch_size" + EMBEDDING_LENGTH = "clip.vision.embedding_length" FEED_FORWARD_LENGTH = "clip.vision.feed_forward_length" - PROJECTION_DIM = "clip.vision.projection_dim" - BLOCK_COUNT = "clip.vision.block_count" - IMAGE_MEAN = "clip.vision.image_mean" - IMAGE_STD = "clip.vision.image_std" - SPATIAL_MERGE_SIZE = "clip.vision.spatial_merge_size" - USE_GELU = "clip.use_gelu" - USE_SILU = "clip.use_silu" - N_WA_PATTERN = "clip.vision.n_wa_pattern" # used by qwen2.5vl - WA_LAYER_INDEXES = "clip.vision.wa_layer_indexes" # used by youtuvl + PROJECTION_DIM = "clip.vision.projection_dim" + BLOCK_COUNT = "clip.vision.block_count" + IMAGE_MEAN = "clip.vision.image_mean" + IMAGE_STD = "clip.vision.image_std" + SPATIAL_MERGE_SIZE = "clip.vision.spatial_merge_size" + USE_GELU = "clip.use_gelu" + USE_SILU = "clip.use_silu" + N_WA_PATTERN = "clip.vision.n_wa_pattern" # used by qwen2.5vl + WA_LAYER_INDEXES = "clip.vision.wa_layer_indexes" # used by youtuvl WA_PATTERN_MODE = "clip.vision.wa_pattern_mode" # used by mimovl, per-layer -1/0/1 IS_DEEPSTACK_LAYERS = "clip.vision.is_deepstack_layers" WINDOW_SIZE = "clip.vision.window_size" @@ -345,9 +351,9 @@ class ClipVision: IMAGE_GRID_PINPOINTS = "clip.vision.image_grid_pinpoints" # Granite4 Vision class Attention: - HEAD_COUNT = "clip.vision.attention.head_count" + HEAD_COUNT = "clip.vision.attention.head_count" HEAD_COUNT_KV = "clip.vision.attention.head_count_kv" # used by mimovl (GQA) - LAYERNORM_EPS = "clip.vision.attention.layer_norm_epsilon" + LAYERNORM_EPS = "clip.vision.attention.layer_norm_epsilon" class Projector: SCALE_FACTOR = "clip.vision.projector.scale_factor" @@ -356,40 +362,40 @@ class Projector: SPATIAL_OFFSETS = "clip.vision.projector.spatial_offsets" class SAM: - BLOCK_COUNT = "clip.vision.sam.block_count" - EMBEDDING_LENGTH = "clip.vision.sam.embedding_length" - HEAD_COUNT = "clip.vision.sam.head_count" + BLOCK_COUNT = "clip.vision.sam.block_count" + EMBEDDING_LENGTH = "clip.vision.sam.embedding_length" + HEAD_COUNT = "clip.vision.sam.head_count" class ClipAudio: - PROJECTOR_TYPE = "clip.audio.projector_type" # for mixed modality models - NUM_MEL_BINS = "clip.audio.num_mel_bins" - EMBEDDING_LENGTH = "clip.audio.embedding_length" + PROJECTOR_TYPE = "clip.audio.projector_type" # for mixed modality models + NUM_MEL_BINS = "clip.audio.num_mel_bins" + EMBEDDING_LENGTH = "clip.audio.embedding_length" FEED_FORWARD_LENGTH = "clip.audio.feed_forward_length" - PROJECTION_DIM = "clip.audio.projection_dim" - BLOCK_COUNT = "clip.audio.block_count" + PROJECTION_DIM = "clip.audio.projection_dim" + BLOCK_COUNT = "clip.audio.block_count" CHUNK_SIZE = "clip.audio.chunk_size" CONV_KERNEL_SIZE = "clip.audio.conv_kernel_size" MAX_POS_EMB = "clip.audio.max_pos_emb" FEATURE_LAYERS = "clip.audio.feature_layer" # Granite Speech Plus class Attention: - HEAD_COUNT = "clip.audio.attention.head_count" - LAYERNORM_EPS = "clip.audio.attention.layer_norm_epsilon" + HEAD_COUNT = "clip.audio.attention.head_count" + LAYERNORM_EPS = "clip.audio.attention.layer_norm_epsilon" class Projector: - STACK_FACTOR = "clip.audio.projector.stack_factor" + STACK_FACTOR = "clip.audio.projector.stack_factor" WINDOW_SIZE = "clip.audio.projector.window_size" DOWNSAMPLE_RATE = "clip.audio.projector.downsample_rate" HEAD_COUNT = "clip.audio.projector.head_count" class Diffusion: - SHIFT_LOGITS = "diffusion.shift_logits" + SHIFT_LOGITS = "diffusion.shift_logits" class xIELU: - ALPHA_P = "xielu.alpha_p" - ALPHA_N = "xielu.alpha_n" - BETA = "xielu.beta" - EPS = "xielu.eps" + ALPHA_P = "xielu.alpha_p" + ALPHA_N = "xielu.alpha_n" + BETA = "xielu.beta" + EPS = "xielu.eps" # @@ -398,10 +404,10 @@ class xIELU: class GGUFType: - MODEL = "model" + MODEL = "model" ADAPTER = "adapter" IMATRIX = "imatrix" - MMPROJ = "mmproj" # dummy, unused for now + MMPROJ = "mmproj" # dummy, unused for now class MODEL_ARCH(IntEnum): @@ -542,17 +548,17 @@ class MODEL_ARCH(IntEnum): class VISION_PROJECTOR_TYPE(IntEnum): - MLP = auto() - LDP = auto() - LDPV2 = auto() + MLP = auto() + LDP = auto() + LDPV2 = auto() RESAMPLER = auto() - GLM_EDGE = auto() - MERGER = auto() - GEMMA3N = auto() - GEMMA3 = auto() - QWEN3VL = auto() - STEP3VL = auto() - COGVLM = auto() + GLM_EDGE = auto() + MERGER = auto() + GEMMA3N = auto() + GEMMA3 = auto() + QWEN3VL = auto() + STEP3VL = auto() + COGVLM = auto() class MODEL_TENSOR(IntEnum): @@ -898,49 +904,49 @@ class MODEL_TENSOR(IntEnum): V_MULTI_PROJ_POST_NORM = auto() # audio (mtmd) - A_ENC_EMBD_POS = auto() - A_ENC_EMBD_NORM = auto() - A_ENC_EMBD_TO_LOGITS = auto() # lfm2 - A_ENC_INP_PROJ = auto() # gemma4 - A_ENC_CONV1D = auto() - A_ENC_CONV1D_NORM = auto() # gemma3n - A_ENC_CONV2D = auto() - A_ENC_CONV_OUT = auto() - A_PRE_NORM = auto() - A_POST_NORM = auto() - A_ENC_LAYER_PRE_NORM = auto() # gemma3n - A_ENC_ATTN_Q = auto() - A_ENC_ATTN_K = auto() - A_ENC_ATTN_V = auto() - A_ENC_ATTN_POST_NORM = auto() - A_ENC_ATTN_PRE_NORM = auto() - A_ENC_ATTN_K_REL = auto() # gemma4 - A_ENC_PER_DIM_SCALE = auto() # gemma3n - A_ENC_INPUT_NORM = auto() - A_ENC_OUTPUT = auto() # TODO @ngxson: rename to ATTN_OUT - A_ENC_OUTPUT_NORM = auto() # TODO @ngxson: rename to ATTN_OUT - A_ENC_FFN_UP = auto() - A_ENC_FFN_NORM = auto() - A_ENC_FFN_POST_NORM = auto() # gemma3n - A_ENC_FFN_SCALE = auto() # gemma3n - A_ENC_FFN_GATE = auto() - A_ENC_FFN_DOWN = auto() - A_ENC_FFN_UP_1 = auto() # lfm2, gemma3n - A_ENC_FFN_NORM_1 = auto() # lfm2, gemma3n (pre-norm) - A_ENC_FFN_POST_NORM_1 = auto() # gemma3n - A_ENC_FFN_SCALE_1 = auto() # gemma3n - A_ENC_FFN_GATE_1 = auto() # lfm2, gemma3n - A_ENC_FFN_DOWN_1 = auto() # lfm2, gemma3n - A_MMPROJ = auto() - A_MMPROJ_FC = auto() - A_MM_NORM_PRE = auto() - A_MM_NORM_MID = auto() - A_MM_EMBEDDING = auto() # gemma3n - A_MM_HARD_EMB_NORM = auto() # gemma3n - A_MM_SOFT_EMB_NORM = auto() # gemma3n - A_MM_INP_PROJ = auto() # gemma3n - A_PER_DIM_K_SCALE = auto() # gemma4 - A_PER_DIM_SCALE = auto() # gemma4 + A_ENC_EMBD_POS = auto() + A_ENC_EMBD_NORM = auto() + A_ENC_EMBD_TO_LOGITS = auto() # lfm2 + A_ENC_INP_PROJ = auto() # gemma4 + A_ENC_CONV1D = auto() + A_ENC_CONV1D_NORM = auto() # gemma3n + A_ENC_CONV2D = auto() + A_ENC_CONV_OUT = auto() + A_PRE_NORM = auto() + A_POST_NORM = auto() + A_ENC_LAYER_PRE_NORM = auto() # gemma3n + A_ENC_ATTN_Q = auto() + A_ENC_ATTN_K = auto() + A_ENC_ATTN_V = auto() + A_ENC_ATTN_POST_NORM = auto() + A_ENC_ATTN_PRE_NORM = auto() + A_ENC_ATTN_K_REL = auto() # gemma4 + A_ENC_PER_DIM_SCALE = auto() # gemma3n + A_ENC_INPUT_NORM = auto() + A_ENC_OUTPUT = auto() # TODO @ngxson: rename to ATTN_OUT + A_ENC_OUTPUT_NORM = auto() # TODO @ngxson: rename to ATTN_OUT + A_ENC_FFN_UP = auto() + A_ENC_FFN_NORM = auto() + A_ENC_FFN_POST_NORM = auto() # gemma3n + A_ENC_FFN_SCALE = auto() # gemma3n + A_ENC_FFN_GATE = auto() + A_ENC_FFN_DOWN = auto() + A_ENC_FFN_UP_1 = auto() # lfm2, gemma3n + A_ENC_FFN_NORM_1 = auto() # lfm2, gemma3n (pre-norm) + A_ENC_FFN_POST_NORM_1 = auto() # gemma3n + A_ENC_FFN_SCALE_1 = auto() # gemma3n + A_ENC_FFN_GATE_1 = auto() # lfm2, gemma3n + A_ENC_FFN_DOWN_1 = auto() # lfm2, gemma3n + A_MMPROJ = auto() + A_MMPROJ_FC = auto() + A_MM_NORM_PRE = auto() + A_MM_NORM_MID = auto() + A_MM_EMBEDDING = auto() # gemma3n + A_MM_HARD_EMB_NORM = auto() # gemma3n + A_MM_SOFT_EMB_NORM = auto() # gemma3n + A_MM_INP_PROJ = auto() # gemma3n + A_PER_DIM_K_SCALE = auto() # gemma4 + A_PER_DIM_SCALE = auto() # gemma4 # nextn/mtp NEXTN_PROJ_PRE = auto() NEXTN_PROJ_POST = auto() @@ -954,15 +960,15 @@ class MODEL_TENSOR(IntEnum): FC = auto() # feature fusion layer D2T = auto() # draft to target vocabulary mapping # lfm2 audio - A_ENC_NORM_CONV = auto() - A_ENC_LINEAR_POS = auto() - A_ENC_POS_BIAS_U = auto() - A_ENC_POS_BIAS_V = auto() - A_ENC_OUT = auto() - A_ENC_CONV_DW = auto() # SSM conv - A_ENC_CONV_NORM = auto() # SSM conv - A_ENC_CONV_PW1 = auto() - A_ENC_CONV_PW2 = auto() + A_ENC_NORM_CONV = auto() + A_ENC_LINEAR_POS = auto() + A_ENC_POS_BIAS_U = auto() + A_ENC_POS_BIAS_V = auto() + A_ENC_OUT = auto() + A_ENC_CONV_DW = auto() # SSM conv + A_ENC_CONV_NORM = auto() # SSM conv + A_ENC_CONV_PW1 = auto() + A_ENC_CONV_PW2 = auto() A_CTC_OUT = auto() A_CTC_OUT_MID = auto() A_ENC_ATTN_REL_POS_EMB = auto() @@ -1081,17 +1087,17 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.GRANITE_HYBRID: "granitehybrid", MODEL_ARCH.CHAMELEON: "chameleon", MODEL_ARCH.WAVTOKENIZER_DEC: "wavtokenizer-dec", - MODEL_ARCH.PLM: "plm", - MODEL_ARCH.BAILINGMOE: "bailingmoe", - MODEL_ARCH.BAILINGMOE2: "bailingmoe2", - MODEL_ARCH.DOTS1: "dots1", - MODEL_ARCH.ARCEE: "arcee", - MODEL_ARCH.AFMOE: "afmoe", - MODEL_ARCH.ERNIE4_5: "ernie4_5", - MODEL_ARCH.ERNIE4_5_MOE: "ernie4_5-moe", - MODEL_ARCH.FALCON_H1: "falcon-h1", - MODEL_ARCH.HUNYUAN_MOE: "hunyuan-moe", - MODEL_ARCH.HUNYUAN_DENSE: "hunyuan-dense", + MODEL_ARCH.PLM: "plm", + MODEL_ARCH.BAILINGMOE: "bailingmoe", + MODEL_ARCH.BAILINGMOE2: "bailingmoe2", + MODEL_ARCH.DOTS1: "dots1", + MODEL_ARCH.ARCEE: "arcee", + MODEL_ARCH.AFMOE: "afmoe", + MODEL_ARCH.ERNIE4_5: "ernie4_5", + MODEL_ARCH.ERNIE4_5_MOE: "ernie4_5-moe", + MODEL_ARCH.FALCON_H1: "falcon-h1", + MODEL_ARCH.HUNYUAN_MOE: "hunyuan-moe", + MODEL_ARCH.HUNYUAN_DENSE: "hunyuan-dense", MODEL_ARCH.HUNYUAN_VL: "hunyuan_vl", MODEL_ARCH.SMOLLM3: "smollm3", MODEL_ARCH.GPT_OSS: "gpt-oss", @@ -1123,15 +1129,15 @@ class MODEL_TENSOR(IntEnum): } VISION_PROJECTOR_TYPE_NAMES: dict[VISION_PROJECTOR_TYPE, str] = { - VISION_PROJECTOR_TYPE.MLP: "mlp", - VISION_PROJECTOR_TYPE.LDP: "ldp", - VISION_PROJECTOR_TYPE.LDPV2: "ldpv2", + VISION_PROJECTOR_TYPE.MLP: "mlp", + VISION_PROJECTOR_TYPE.LDP: "ldp", + VISION_PROJECTOR_TYPE.LDPV2: "ldpv2", VISION_PROJECTOR_TYPE.RESAMPLER: "resampler", - VISION_PROJECTOR_TYPE.GLM_EDGE: "adapter", - VISION_PROJECTOR_TYPE.MERGER: "qwen2vl_merger", - VISION_PROJECTOR_TYPE.GEMMA3: "gemma3", - VISION_PROJECTOR_TYPE.QWEN3VL: "qwen3vl_merger", - VISION_PROJECTOR_TYPE.STEP3VL: "step3vl", + VISION_PROJECTOR_TYPE.GLM_EDGE: "adapter", + VISION_PROJECTOR_TYPE.MERGER: "qwen2vl_merger", + VISION_PROJECTOR_TYPE.GEMMA3: "gemma3", + VISION_PROJECTOR_TYPE.QWEN3VL: "qwen3vl_merger", + VISION_PROJECTOR_TYPE.STEP3VL: "step3vl", } TENSOR_NAMES: dict[MODEL_TENSOR, str] = { @@ -1374,46 +1380,46 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_ENC_ATTN_O: "v.blk.{bid}.attn_out", MODEL_TENSOR.V_ENC_ATTN_O_NORM: "v.blk.{bid}.attn_out_norm", MODEL_TENSOR.V_ENC_ATTN_SINKS: "v.blk.{bid}.attn_sinks", - MODEL_TENSOR.V_ENC_POST_ATTN_NORM: "v.blk.{bid}.ln2", - MODEL_TENSOR.V_ENC_FFN_UP: "v.blk.{bid}.ffn_up", - MODEL_TENSOR.V_ENC_FFN_GATE: "v.blk.{bid}.ffn_gate", - MODEL_TENSOR.V_ENC_FFN_DOWN: "v.blk.{bid}.ffn_down", - MODEL_TENSOR.V_ENC_ATTN_POST_NORM: "v.blk.{bid}.attn_post_norm", - MODEL_TENSOR.V_ENC_FFN_POST_NORM: "v.blk.{bid}.ffn_post_norm", - MODEL_TENSOR.V_LAYER_SCALE_1: "v.blk.{bid}.ls1", - MODEL_TENSOR.V_LAYER_SCALE_2: "v.blk.{bid}.ls2", - MODEL_TENSOR.V_LAYER_OUT_SCALE: "v.blk.{bid}.out_scale", - MODEL_TENSOR.V_PRE_NORM: "v.pre_ln", - MODEL_TENSOR.V_POST_NORM: "v.post_ln", - MODEL_TENSOR.V_MM_POST_NORM: "mm.post_norm", - MODEL_TENSOR.V_MM_INP_PROJ: "mm.input_projection", - MODEL_TENSOR.V_MM_INP_NORM: "mm.input_norm", - MODEL_TENSOR.V_MM_SOFT_EMB_NORM: "mm.soft_emb_norm", # gemma3n - MODEL_TENSOR.V_MM_EMBEDDING: "mm.embedding", # gemma3n - MODEL_TENSOR.V_MM_HARD_EMB_NORM: "mm.hard_emb_norm", # gemma3n - MODEL_TENSOR.V_ENC_CONV_STEM: "v.conv_stem.conv", # gemma3n - MODEL_TENSOR.V_ENC_CONV_STEM_NORM: "v.conv_stem.bn", # gemma3n - MODEL_TENSOR.V_ENC_MSFA_EXP: "v.msfa.ffn.pw_exp.conv", # gemma3n - MODEL_TENSOR.V_ENC_MSFA_EXP_NORM: "v.msfa.ffn.pw_exp.bn", # gemma3n - MODEL_TENSOR.V_ENC_MSFA_PROJ: "v.msfa.ffn.pw_proj.conv", # gemma3n - MODEL_TENSOR.V_ENC_MSFA_PROJ_NORM: "v.msfa.ffn.pw_proj.bn", # gemma3n - MODEL_TENSOR.V_ENC_MSFA_NORM: "v.msfa.norm", # gemma3n - MODEL_TENSOR.V_RESMPL_POS_EMBD_K: "resampler.pos_embd_k", - MODEL_TENSOR.V_RESMPL_ATTN_Q: "resampler.attn.q", - MODEL_TENSOR.V_RESMPL_ATTN_K: "resampler.attn.k", - MODEL_TENSOR.V_RESMPL_ATTN_V: "resampler.attn.v", - MODEL_TENSOR.V_RESMPL_ATTN_OUT: "resampler.attn.out", - MODEL_TENSOR.V_RESMPL_KV: "resampler.kv", - MODEL_TENSOR.V_RESMPL_KV_NORM: "resampler.ln_kv", - MODEL_TENSOR.V_RESMPL_POST_NORM: "resampler.ln_post", - MODEL_TENSOR.V_RESMPL_Q_NORM: "resampler.ln_q", - MODEL_TENSOR.V_RESMPL_PROJ: "resampler.proj", - MODEL_TENSOR.V_RESMPL_QUERY: "resampler.query", - MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK: "v.token_embd.img_break", # pixtral - MODEL_TENSOR.V_MM_PATCH_MERGER: "mm.patch_merger", # mistral small 3.1 - MODEL_TENSOR.V_DS_NORM: "v.deepstack.{bid}.norm", - MODEL_TENSOR.V_DS_FC1: "v.deepstack.{bid}.fc1", - MODEL_TENSOR.V_DS_FC2: "v.deepstack.{bid}.fc2", + MODEL_TENSOR.V_ENC_POST_ATTN_NORM: "v.blk.{bid}.ln2", + MODEL_TENSOR.V_ENC_FFN_UP: "v.blk.{bid}.ffn_up", + MODEL_TENSOR.V_ENC_FFN_GATE: "v.blk.{bid}.ffn_gate", + MODEL_TENSOR.V_ENC_FFN_DOWN: "v.blk.{bid}.ffn_down", + MODEL_TENSOR.V_ENC_ATTN_POST_NORM: "v.blk.{bid}.attn_post_norm", + MODEL_TENSOR.V_ENC_FFN_POST_NORM: "v.blk.{bid}.ffn_post_norm", + MODEL_TENSOR.V_LAYER_SCALE_1: "v.blk.{bid}.ls1", + MODEL_TENSOR.V_LAYER_SCALE_2: "v.blk.{bid}.ls2", + MODEL_TENSOR.V_LAYER_OUT_SCALE: "v.blk.{bid}.out_scale", + MODEL_TENSOR.V_PRE_NORM: "v.pre_ln", + MODEL_TENSOR.V_POST_NORM: "v.post_ln", + MODEL_TENSOR.V_MM_POST_NORM: "mm.post_norm", + MODEL_TENSOR.V_MM_INP_PROJ: "mm.input_projection", + MODEL_TENSOR.V_MM_INP_NORM: "mm.input_norm", + MODEL_TENSOR.V_MM_SOFT_EMB_NORM: "mm.soft_emb_norm", # gemma3n + MODEL_TENSOR.V_MM_EMBEDDING: "mm.embedding", # gemma3n + MODEL_TENSOR.V_MM_HARD_EMB_NORM: "mm.hard_emb_norm", # gemma3n + MODEL_TENSOR.V_ENC_CONV_STEM: "v.conv_stem.conv", # gemma3n + MODEL_TENSOR.V_ENC_CONV_STEM_NORM: "v.conv_stem.bn", # gemma3n + MODEL_TENSOR.V_ENC_MSFA_EXP: "v.msfa.ffn.pw_exp.conv", # gemma3n + MODEL_TENSOR.V_ENC_MSFA_EXP_NORM: "v.msfa.ffn.pw_exp.bn", # gemma3n + MODEL_TENSOR.V_ENC_MSFA_PROJ: "v.msfa.ffn.pw_proj.conv", # gemma3n + MODEL_TENSOR.V_ENC_MSFA_PROJ_NORM: "v.msfa.ffn.pw_proj.bn", # gemma3n + MODEL_TENSOR.V_ENC_MSFA_NORM: "v.msfa.norm", # gemma3n + MODEL_TENSOR.V_RESMPL_POS_EMBD_K: "resampler.pos_embd_k", + MODEL_TENSOR.V_RESMPL_ATTN_Q: "resampler.attn.q", + MODEL_TENSOR.V_RESMPL_ATTN_K: "resampler.attn.k", + MODEL_TENSOR.V_RESMPL_ATTN_V: "resampler.attn.v", + MODEL_TENSOR.V_RESMPL_ATTN_OUT: "resampler.attn.out", + MODEL_TENSOR.V_RESMPL_KV: "resampler.kv", + MODEL_TENSOR.V_RESMPL_KV_NORM: "resampler.ln_kv", + MODEL_TENSOR.V_RESMPL_POST_NORM: "resampler.ln_post", + MODEL_TENSOR.V_RESMPL_Q_NORM: "resampler.ln_q", + MODEL_TENSOR.V_RESMPL_PROJ: "resampler.proj", + MODEL_TENSOR.V_RESMPL_QUERY: "resampler.query", + MODEL_TENSOR.V_TOK_EMBD_IMG_BREAK: "v.token_embd.img_break", # pixtral + MODEL_TENSOR.V_MM_PATCH_MERGER: "mm.patch_merger", # mistral small 3.1 + MODEL_TENSOR.V_DS_NORM: "v.deepstack.{bid}.norm", + MODEL_TENSOR.V_DS_FC1: "v.deepstack.{bid}.fc1", + MODEL_TENSOR.V_DS_FC2: "v.deepstack.{bid}.fc2", MODEL_TENSOR.V_MERGER_LN1: "v.vit_merger.ln1", MODEL_TENSOR.V_MERGER_ATTN_Q: "v.vit_merger.attn_q", MODEL_TENSOR.V_MERGER_ATTN_K: "v.vit_merger.attn_k", @@ -1422,17 +1428,17 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_MERGER_DS_LN: "v.vit_merger.ds_ln", MODEL_TENSOR.V_MERGER_DS_UP: "v.vit_merger.ds_ffn_up", MODEL_TENSOR.V_MERGER_DS_DOWN: "v.vit_merger.ds_ffn_down", - MODEL_TENSOR.V_MM_POST_FC_NORM: "mm.post_fc_norm", # cogvlm - MODEL_TENSOR.V_MM_UP: "mm.up", - MODEL_TENSOR.V_MM_DOWN: "mm.down", - MODEL_TENSOR.V_MM_GATE: "mm.gate", - MODEL_TENSOR.V_TOK_BOI: "v.boi", - MODEL_TENSOR.V_TOK_EOI: "v.eoi", - MODEL_TENSOR.V_MM_PRE_NORM: "mm.pre_norm", - MODEL_TENSOR.V_TOK_IMG_BEGIN: "mm.image_begin", - MODEL_TENSOR.V_TOK_IMG_END: "mm.image_end", - MODEL_TENSOR.V_STD_BIAS: "v.std_bias", # gemma4 - MODEL_TENSOR.V_STD_SCALE: "v.std_scale", # gemma4 + MODEL_TENSOR.V_MM_POST_FC_NORM: "mm.post_fc_norm", # cogvlm + MODEL_TENSOR.V_MM_UP: "mm.up", + MODEL_TENSOR.V_MM_DOWN: "mm.down", + MODEL_TENSOR.V_MM_GATE: "mm.gate", + MODEL_TENSOR.V_TOK_BOI: "v.boi", + MODEL_TENSOR.V_TOK_EOI: "v.eoi", + MODEL_TENSOR.V_MM_PRE_NORM: "mm.pre_norm", + MODEL_TENSOR.V_TOK_IMG_BEGIN: "mm.image_begin", + MODEL_TENSOR.V_TOK_IMG_END: "mm.image_end", + MODEL_TENSOR.V_STD_BIAS: "v.std_bias", # gemma4 + MODEL_TENSOR.V_STD_SCALE: "v.std_scale", # gemma4 # DeepSeek-OCR SAM MODEL_TENSOR.V_SAM_POS_EMBD: "v.sam.pos_embd", MODEL_TENSOR.V_SAM_PATCH_EMBD: "v.sam.patch_embd", @@ -1474,61 +1480,62 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.V_MULTI_PROJ_LINEAR: "v.proj_blk.{bid}.linear", MODEL_TENSOR.V_MULTI_PROJ_POST_NORM: "v.proj_blk.{bid}.post_norm", + # audio (mtmd) # note: all audio tensor names must use prefix "a." or "mm.a." - MODEL_TENSOR.A_ENC_EMBD_POS: "a.position_embd", - MODEL_TENSOR.A_ENC_EMBD_NORM: "a.position_embd_norm", - MODEL_TENSOR.A_ENC_EMBD_TO_LOGITS: "a.embd_to_logits", - MODEL_TENSOR.A_ENC_INP_PROJ: "a.input_projection", - MODEL_TENSOR.A_ENC_CONV1D: "a.conv1d.{bid}", - MODEL_TENSOR.A_ENC_CONV2D: "a.conv2d.{bid}", - MODEL_TENSOR.A_ENC_CONV_OUT: "a.conv_out", - MODEL_TENSOR.A_ENC_CONV1D_NORM: "a.conv1d.{bid}.norm", - MODEL_TENSOR.A_PRE_NORM: "a.pre_ln", - MODEL_TENSOR.A_POST_NORM: "a.post_ln", - MODEL_TENSOR.A_ENC_LAYER_PRE_NORM: "a.blk.{bid}.layer_pre_norm", - MODEL_TENSOR.A_ENC_ATTN_Q: "a.blk.{bid}.attn_q", - MODEL_TENSOR.A_ENC_ATTN_K: "a.blk.{bid}.attn_k", - MODEL_TENSOR.A_ENC_ATTN_V: "a.blk.{bid}.attn_v", - MODEL_TENSOR.A_ENC_ATTN_POST_NORM: "a.blk.{bid}.attn_post_norm", - MODEL_TENSOR.A_ENC_ATTN_PRE_NORM: "a.blk.{bid}.attn_pre_norm", - MODEL_TENSOR.A_ENC_ATTN_K_REL: "a.blk.{bid}.attn_k_rel", - MODEL_TENSOR.A_ENC_PER_DIM_SCALE: "a.blk.{bid}.per_dim_scale", - MODEL_TENSOR.A_ENC_INPUT_NORM: "a.blk.{bid}.ln1", - MODEL_TENSOR.A_ENC_OUTPUT: "a.blk.{bid}.attn_out", - MODEL_TENSOR.A_ENC_OUTPUT_NORM: "a.blk.{bid}.ln2", - MODEL_TENSOR.A_ENC_FFN_NORM: "a.blk.{bid}.ffn_norm", - MODEL_TENSOR.A_ENC_FFN_POST_NORM: "a.blk.{bid}.ffn_post_norm", - MODEL_TENSOR.A_ENC_FFN_SCALE: "a.blk.{bid}.ffn_scale", - MODEL_TENSOR.A_ENC_FFN_UP: "a.blk.{bid}.ffn_up", - MODEL_TENSOR.A_ENC_FFN_GATE: "a.blk.{bid}.ffn_gate", - MODEL_TENSOR.A_ENC_FFN_DOWN: "a.blk.{bid}.ffn_down", - MODEL_TENSOR.A_ENC_FFN_NORM_1: "a.blk.{bid}.ffn_norm_1", - MODEL_TENSOR.A_ENC_FFN_POST_NORM_1: "a.blk.{bid}.ffn_post_norm_1", - MODEL_TENSOR.A_ENC_FFN_SCALE_1: "a.blk.{bid}.ffn_scale_1", - MODEL_TENSOR.A_ENC_FFN_UP_1: "a.blk.{bid}.ffn_up_1", - MODEL_TENSOR.A_ENC_FFN_GATE_1: "a.blk.{bid}.ffn_gate_1", - MODEL_TENSOR.A_ENC_FFN_DOWN_1: "a.blk.{bid}.ffn_down_1", - MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}", - MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc", - MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre", - MODEL_TENSOR.A_MM_NORM_MID: "mm.a.norm_mid", - MODEL_TENSOR.A_MM_INP_PROJ: "mm.a.input_projection", # gemma3n - MODEL_TENSOR.A_MM_SOFT_EMB_NORM: "mm.a.soft_emb_norm", # gemma3n - MODEL_TENSOR.A_MM_EMBEDDING: "mm.a.embedding", # gemma3n - MODEL_TENSOR.A_MM_HARD_EMB_NORM: "mm.a.hard_emb_norm", # gemma3n - MODEL_TENSOR.A_PER_DIM_K_SCALE: "a.blk.{bid}.per_dim_k_scale", # gemma4 - MODEL_TENSOR.A_PER_DIM_SCALE: "a.blk.{bid}.per_dim_scale", # gemma4 + MODEL_TENSOR.A_ENC_EMBD_POS: "a.position_embd", + MODEL_TENSOR.A_ENC_EMBD_NORM: "a.position_embd_norm", + MODEL_TENSOR.A_ENC_EMBD_TO_LOGITS: "a.embd_to_logits", + MODEL_TENSOR.A_ENC_INP_PROJ: "a.input_projection", + MODEL_TENSOR.A_ENC_CONV1D: "a.conv1d.{bid}", + MODEL_TENSOR.A_ENC_CONV2D: "a.conv2d.{bid}", + MODEL_TENSOR.A_ENC_CONV_OUT: "a.conv_out", + MODEL_TENSOR.A_ENC_CONV1D_NORM: "a.conv1d.{bid}.norm", + MODEL_TENSOR.A_PRE_NORM: "a.pre_ln", + MODEL_TENSOR.A_POST_NORM: "a.post_ln", + MODEL_TENSOR.A_ENC_LAYER_PRE_NORM: "a.blk.{bid}.layer_pre_norm", + MODEL_TENSOR.A_ENC_ATTN_Q: "a.blk.{bid}.attn_q", + MODEL_TENSOR.A_ENC_ATTN_K: "a.blk.{bid}.attn_k", + MODEL_TENSOR.A_ENC_ATTN_V: "a.blk.{bid}.attn_v", + MODEL_TENSOR.A_ENC_ATTN_POST_NORM: "a.blk.{bid}.attn_post_norm", + MODEL_TENSOR.A_ENC_ATTN_PRE_NORM: "a.blk.{bid}.attn_pre_norm", + MODEL_TENSOR.A_ENC_ATTN_K_REL: "a.blk.{bid}.attn_k_rel", + MODEL_TENSOR.A_ENC_PER_DIM_SCALE: "a.blk.{bid}.per_dim_scale", + MODEL_TENSOR.A_ENC_INPUT_NORM: "a.blk.{bid}.ln1", + MODEL_TENSOR.A_ENC_OUTPUT: "a.blk.{bid}.attn_out", + MODEL_TENSOR.A_ENC_OUTPUT_NORM: "a.blk.{bid}.ln2", + MODEL_TENSOR.A_ENC_FFN_NORM: "a.blk.{bid}.ffn_norm", + MODEL_TENSOR.A_ENC_FFN_POST_NORM: "a.blk.{bid}.ffn_post_norm", + MODEL_TENSOR.A_ENC_FFN_SCALE: "a.blk.{bid}.ffn_scale", + MODEL_TENSOR.A_ENC_FFN_UP: "a.blk.{bid}.ffn_up", + MODEL_TENSOR.A_ENC_FFN_GATE: "a.blk.{bid}.ffn_gate", + MODEL_TENSOR.A_ENC_FFN_DOWN: "a.blk.{bid}.ffn_down", + MODEL_TENSOR.A_ENC_FFN_NORM_1: "a.blk.{bid}.ffn_norm_1", + MODEL_TENSOR.A_ENC_FFN_POST_NORM_1: "a.blk.{bid}.ffn_post_norm_1", + MODEL_TENSOR.A_ENC_FFN_SCALE_1: "a.blk.{bid}.ffn_scale_1", + MODEL_TENSOR.A_ENC_FFN_UP_1: "a.blk.{bid}.ffn_up_1", + MODEL_TENSOR.A_ENC_FFN_GATE_1: "a.blk.{bid}.ffn_gate_1", + MODEL_TENSOR.A_ENC_FFN_DOWN_1: "a.blk.{bid}.ffn_down_1", + MODEL_TENSOR.A_MMPROJ: "mm.a.mlp.{bid}", + MODEL_TENSOR.A_MMPROJ_FC: "mm.a.fc", + MODEL_TENSOR.A_MM_NORM_PRE: "mm.a.norm_pre", + MODEL_TENSOR.A_MM_NORM_MID: "mm.a.norm_mid", + MODEL_TENSOR.A_MM_INP_PROJ: "mm.a.input_projection", # gemma3n + MODEL_TENSOR.A_MM_SOFT_EMB_NORM: "mm.a.soft_emb_norm", # gemma3n + MODEL_TENSOR.A_MM_EMBEDDING: "mm.a.embedding", # gemma3n + MODEL_TENSOR.A_MM_HARD_EMB_NORM: "mm.a.hard_emb_norm", # gemma3n + MODEL_TENSOR.A_PER_DIM_K_SCALE: "a.blk.{bid}.per_dim_k_scale", # gemma4 + MODEL_TENSOR.A_PER_DIM_SCALE: "a.blk.{bid}.per_dim_scale", # gemma4 # lfm2 audio - MODEL_TENSOR.A_ENC_NORM_CONV: "a.blk.{bid}.norm_conv", - MODEL_TENSOR.A_ENC_LINEAR_POS: "a.blk.{bid}.linear_pos", - MODEL_TENSOR.A_ENC_POS_BIAS_U: "a.blk.{bid}.pos_bias_u", - MODEL_TENSOR.A_ENC_POS_BIAS_V: "a.blk.{bid}.pos_bias_v", - MODEL_TENSOR.A_ENC_OUT: "a.pre_encode.out", - MODEL_TENSOR.A_ENC_CONV_DW: "a.blk.{bid}.conv_dw", - MODEL_TENSOR.A_ENC_CONV_NORM: "a.blk.{bid}.conv_norm", - MODEL_TENSOR.A_ENC_CONV_PW1: "a.blk.{bid}.conv_pw1", - MODEL_TENSOR.A_ENC_CONV_PW2: "a.blk.{bid}.conv_pw2", + MODEL_TENSOR.A_ENC_NORM_CONV: "a.blk.{bid}.norm_conv", + MODEL_TENSOR.A_ENC_LINEAR_POS: "a.blk.{bid}.linear_pos", + MODEL_TENSOR.A_ENC_POS_BIAS_U: "a.blk.{bid}.pos_bias_u", + MODEL_TENSOR.A_ENC_POS_BIAS_V: "a.blk.{bid}.pos_bias_v", + MODEL_TENSOR.A_ENC_OUT: "a.pre_encode.out", + MODEL_TENSOR.A_ENC_CONV_DW: "a.blk.{bid}.conv_dw", + MODEL_TENSOR.A_ENC_CONV_NORM: "a.blk.{bid}.conv_norm", + MODEL_TENSOR.A_ENC_CONV_PW1: "a.blk.{bid}.conv_pw1", + MODEL_TENSOR.A_ENC_CONV_PW2: "a.blk.{bid}.conv_pw2", MODEL_TENSOR.A_CTC_OUT: "a.enc_ctc_out", MODEL_TENSOR.A_CTC_OUT_MID: "a.enc_ctc_out_mid", MODEL_TENSOR.A_ENC_ATTN_REL_POS_EMB: "a.blk.{bid}.attn_rel_pos_emb", @@ -2234,7 +2241,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.SSM_NORM, MODEL_TENSOR.SSM_IN, MODEL_TENSOR.SSM_BETA_ALPHA, - MODEL_TENSOR.SSM_OUT + MODEL_TENSOR.SSM_OUT, ], MODEL_ARCH.QWEN3VL: [ MODEL_TENSOR.TOKEN_EMBD, @@ -3273,7 +3280,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FFN_DOWN, ], - MODEL_ARCH.CHATGLM : [ + MODEL_ARCH.CHATGLM: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.OUTPUT_NORM, @@ -3288,7 +3295,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], - MODEL_ARCH.GLM4 : [ + MODEL_ARCH.GLM4: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.OUTPUT_NORM, @@ -3852,36 +3859,30 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.FALCON_H1: [ # Token embedding MODEL_TENSOR.TOKEN_EMBD, - # Input layernorm MODEL_TENSOR.ATTN_NORM, - # Attention components - MODEL_TENSOR.ATTN_Q, # Query projection - MODEL_TENSOR.ATTN_K, # Key projection - MODEL_TENSOR.ATTN_V, # Value projection - MODEL_TENSOR.ATTN_OUT, # Output projection - + MODEL_TENSOR.ATTN_Q, # Query projection + MODEL_TENSOR.ATTN_K, # Key projection + MODEL_TENSOR.ATTN_V, # Value projection + MODEL_TENSOR.ATTN_OUT, # Output projection # SSM components (Mamba2 specific) - MODEL_TENSOR.SSM_IN, # Input projection for SSM - MODEL_TENSOR.SSM_CONV1D, # Convolution layer - MODEL_TENSOR.SSM_DT, # Delta time projection - MODEL_TENSOR.SSM_A, # A parameter (log form) - MODEL_TENSOR.SSM_D, # D parameter - MODEL_TENSOR.SSM_NORM, # Normalization in SSM - MODEL_TENSOR.SSM_OUT, # Output projection - + MODEL_TENSOR.SSM_IN, # Input projection for SSM + MODEL_TENSOR.SSM_CONV1D, # Convolution layer + MODEL_TENSOR.SSM_DT, # Delta time projection + MODEL_TENSOR.SSM_A, # A parameter (log form) + MODEL_TENSOR.SSM_D, # D parameter + MODEL_TENSOR.SSM_NORM, # Normalization in SSM + MODEL_TENSOR.SSM_OUT, # Output projection # Pre-feedforward layernorm MODEL_TENSOR.FFN_PRE_NORM, - # Feed-forward network components - MODEL_TENSOR.FFN_GATE, # Gate projection (SwiGLU) - MODEL_TENSOR.FFN_DOWN, # Down projection - MODEL_TENSOR.FFN_UP, # Up projection - + MODEL_TENSOR.FFN_GATE, # Gate projection (SwiGLU) + MODEL_TENSOR.FFN_DOWN, # Down projection + MODEL_TENSOR.FFN_UP, # Up projection # Post-feedforward layernorm - MODEL_TENSOR.OUTPUT_NORM, # Final layer norm - MODEL_TENSOR.OUTPUT, # Output projection (lm_head) + MODEL_TENSOR.OUTPUT_NORM, # Final layer norm + MODEL_TENSOR.OUTPUT, # Output projection (lm_head) ], MODEL_ARCH.HUNYUAN_MOE: [ MODEL_TENSOR.TOKEN_EMBD, @@ -3978,7 +3979,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FFN_NORM, - MODEL_TENSOR.ATTN_NORM, # operator_norm + MODEL_TENSOR.ATTN_NORM, # operator_norm MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K_NORM, MODEL_TENSOR.ATTN_Q, @@ -3986,7 +3987,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.ATTN_V, MODEL_TENSOR.ATTN_OUT, MODEL_TENSOR.OUTPUT, - MODEL_TENSOR.DENSE_2_OUT, # LFM2-ColBert-350M + MODEL_TENSOR.DENSE_2_OUT, # LFM2-ColBert-350M ], MODEL_ARCH.LFM2MOE: [ MODEL_TENSOR.TOKEN_EMBD, @@ -3998,7 +3999,7 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, MODEL_TENSOR.FFN_NORM, - MODEL_TENSOR.ATTN_NORM, # operator_norm + MODEL_TENSOR.ATTN_NORM, # operator_norm MODEL_TENSOR.ATTN_Q_NORM, MODEL_TENSOR.ATTN_K_NORM, MODEL_TENSOR.ATTN_Q, @@ -4475,64 +4476,72 @@ class MODEL_TENSOR(IntEnum): class TokenType(IntEnum): - NORMAL = 1 - UNKNOWN = 2 - CONTROL = 3 + NORMAL = 1 + UNKNOWN = 2 + CONTROL = 3 USER_DEFINED = 4 - UNUSED = 5 - BYTE = 6 + UNUSED = 5 + BYTE = 6 class RopeScalingType(Enum): - NONE = 'none' - LINEAR = 'linear' - YARN = 'yarn' - LONGROPE = 'longrope' + NONE = "none" + LINEAR = "linear" + YARN = "yarn" + LONGROPE = "longrope" class PoolingType(IntEnum): NONE = 0 MEAN = 1 - CLS = 2 + CLS = 2 LAST = 3 RANK = 4 class GGMLQuantizationType(IntEnum): - F32 = 0 - F16 = 1 - Q4_0 = 2 - Q4_1 = 3 - Q5_0 = 6 - Q5_1 = 7 - Q8_0 = 8 - Q8_1 = 9 - Q2_K = 10 - Q3_K = 11 - Q4_K = 12 - Q5_K = 13 - Q6_K = 14 - Q8_K = 15 + F32 = 0 + F16 = 1 + Q4_0 = 2 + Q4_1 = 3 + Q5_0 = 6 + Q5_1 = 7 + Q8_0 = 8 + Q8_1 = 9 + Q2_K = 10 + Q3_K = 11 + Q4_K = 12 + Q5_K = 13 + Q6_K = 14 + Q8_K = 15 IQ2_XXS = 16 - IQ2_XS = 17 + IQ2_XS = 17 IQ3_XXS = 18 - IQ1_S = 19 - IQ4_NL = 20 - IQ3_S = 21 - IQ2_S = 22 - IQ4_XS = 23 - I8 = 24 - I16 = 25 - I32 = 26 - I64 = 27 - F64 = 28 - IQ1_M = 29 - BF16 = 30 - TQ1_0 = 34 - TQ2_0 = 35 - MXFP4 = 39 - NVFP4 = 40 - Q1_0 = 41 + IQ1_S = 19 + IQ4_NL = 20 + IQ3_S = 21 + IQ2_S = 22 + IQ4_XS = 23 + I8 = 24 + I16 = 25 + I32 = 26 + I64 = 27 + F64 = 28 + IQ1_M = 29 + BF16 = 30 + TQ1_0 = 34 + TQ2_0 = 35 + MXFP4 = 39 + NVFP4 = 40 + Q1_0 = 41 + Q3_PT = 42 + Q3_KPT = 43 + Q4_DPT = 44 + Q2_DPT = 45 + Q2_KPT = 46 + IQ2_TQ = 47 + IQ3_TQ = 48 + IQ1_BN = 49 class ExpertGatingFuncType(IntEnum): @@ -4547,49 +4556,56 @@ class ExpertGatingFuncType(IntEnum): # from llama_ftype in llama.h # ALL VALUES SHOULD BE THE SAME HERE AS THEY ARE OVER THERE. class LlamaFileType(IntEnum): - ALL_F32 = 0 - MOSTLY_F16 = 1 # except 1d tensors - MOSTLY_Q4_0 = 2 # except 1d tensors - MOSTLY_Q4_1 = 3 # except 1d tensors + ALL_F32 = 0 + MOSTLY_F16 = 1 # except 1d tensors + MOSTLY_Q4_0 = 2 # except 1d tensors + MOSTLY_Q4_1 = 3 # except 1d tensors # MOSTLY_Q4_1_SOME_F16 = 4 # tok_embeddings.weight and output.weight are F16 # MOSTLY_Q4_2 = 5 # support has been removed # MOSTLY_Q4_3 = 6 # support has been removed - MOSTLY_Q8_0 = 7 # except 1d tensors - MOSTLY_Q5_0 = 8 # except 1d tensors - MOSTLY_Q5_1 = 9 # except 1d tensors - MOSTLY_Q2_K = 10 # except 1d tensors - MOSTLY_Q3_K_S = 11 # except 1d tensors - MOSTLY_Q3_K_M = 12 # except 1d tensors - MOSTLY_Q3_K_L = 13 # except 1d tensors - MOSTLY_Q4_K_S = 14 # except 1d tensors - MOSTLY_Q4_K_M = 15 # except 1d tensors - MOSTLY_Q5_K_S = 16 # except 1d tensors - MOSTLY_Q5_K_M = 17 # except 1d tensors - MOSTLY_Q6_K = 18 # except 1d tensors - MOSTLY_IQ2_XXS = 19 # except 1d tensors - MOSTLY_IQ2_XS = 20 # except 1d tensors - MOSTLY_Q2_K_S = 21 # except 1d tensors - MOSTLY_IQ3_XS = 22 # except 1d tensors - MOSTLY_IQ3_XXS = 23 # except 1d tensors - MOSTLY_IQ1_S = 24 # except 1d tensors - MOSTLY_IQ4_NL = 25 # except 1d tensors - MOSTLY_IQ3_S = 26 # except 1d tensors - MOSTLY_IQ3_M = 27 # except 1d tensors - MOSTLY_IQ2_S = 28 # except 1d tensors - MOSTLY_IQ2_M = 29 # except 1d tensors - MOSTLY_IQ4_XS = 30 # except 1d tensors - MOSTLY_IQ1_M = 31 # except 1d tensors - MOSTLY_BF16 = 32 # except 1d tensors + MOSTLY_Q8_0 = 7 # except 1d tensors + MOSTLY_Q5_0 = 8 # except 1d tensors + MOSTLY_Q5_1 = 9 # except 1d tensors + MOSTLY_Q2_K = 10 # except 1d tensors + MOSTLY_Q3_K_S = 11 # except 1d tensors + MOSTLY_Q3_K_M = 12 # except 1d tensors + MOSTLY_Q3_K_L = 13 # except 1d tensors + MOSTLY_Q4_K_S = 14 # except 1d tensors + MOSTLY_Q4_K_M = 15 # except 1d tensors + MOSTLY_Q5_K_S = 16 # except 1d tensors + MOSTLY_Q5_K_M = 17 # except 1d tensors + MOSTLY_Q6_K = 18 # except 1d tensors + MOSTLY_IQ2_XXS = 19 # except 1d tensors + MOSTLY_IQ2_XS = 20 # except 1d tensors + MOSTLY_Q2_K_S = 21 # except 1d tensors + MOSTLY_IQ3_XS = 22 # except 1d tensors + MOSTLY_IQ3_XXS = 23 # except 1d tensors + MOSTLY_IQ1_S = 24 # except 1d tensors + MOSTLY_IQ4_NL = 25 # except 1d tensors + MOSTLY_IQ3_S = 26 # except 1d tensors + MOSTLY_IQ3_M = 27 # except 1d tensors + MOSTLY_IQ2_S = 28 # except 1d tensors + MOSTLY_IQ2_M = 29 # except 1d tensors + MOSTLY_IQ4_XS = 30 # except 1d tensors + MOSTLY_IQ1_M = 31 # except 1d tensors + MOSTLY_BF16 = 32 # except 1d tensors # MOSTLY_Q4_0_4_4 = 33 # removed from gguf files, use Q4_0 and runtime repack # MOSTLY_Q4_0_4_8 = 34 # removed from gguf files, use Q4_0 and runtime repack # MOSTLY_Q4_0_8_8 = 35 # removed from gguf files, use Q4_0 and runtime repack - MOSTLY_TQ1_0 = 36 # except 1d tensors - MOSTLY_TQ2_0 = 37 # except 1d tensors - MOSTLY_MXFP4_MOE = 38 # except 1d tensors - MOSTLY_NVFP4 = 39 # except 1d tensors - MOSTLY_Q1_0 = 40 # except 1d tensors + MOSTLY_TQ1_0 = 36 # except 1d tensors + MOSTLY_TQ2_0 = 37 # except 1d tensors + MOSTLY_MXFP4_MOE = 38 # except 1d tensors + MOSTLY_NVFP4 = 39 # except 1d tensors + MOSTLY_Q1_0 = 40 # except 1d tensors + MOSTLY_Q3_PT = 41 # except 1d tensors + MOSTLY_Q3_KPT = 42 # except 1d tensors + MOSTLY_Q4_DPT = 43 # except 1d tensors + MOSTLY_Q2_KPT = 44 # except 1d tensors + MOSTLY_IQ2_TQ = 45 # except 1d tensors, trellis quantized with RNG codebook + MOSTLY_IQ3_TQ = 46 # except 1d tensors, 3-bit with per-tensor trained grid + MOSTLY_IQ1_BN = 47 # except 1d tensors, 8D vector quantized with trained codebook - GUESSED = 1024 # not specified in the model file + GUESSED = 1024 # not specified in the model file class GGUFEndian(IntEnum): @@ -4598,18 +4614,18 @@ class GGUFEndian(IntEnum): class GGUFValueType(IntEnum): - UINT8 = 0 - INT8 = 1 - UINT16 = 2 - INT16 = 3 - UINT32 = 4 - INT32 = 5 + UINT8 = 0 + INT8 = 1 + UINT16 = 2 + INT16 = 3 + UINT32 = 4 + INT32 = 5 FLOAT32 = 6 - BOOL = 7 - STRING = 8 - ARRAY = 9 - UINT64 = 10 - INT64 = 11 + BOOL = 7 + STRING = 8 + ARRAY = 9 + UINT64 = 10 + INT64 = 11 FLOAT64 = 12 @staticmethod @@ -4648,10 +4664,10 @@ class VisionProjectorType: STEP3VL = "step3vl" ULTRAVOX = "ultravox" INTERNVL = "internvl" - QWEN2A = "qwen2a" # audio - QWEN3A = "qwen3a" # audio - GLMA = "glma" # audio - QWEN25O = "qwen2.5o" # omni + QWEN2A = "qwen2a" # audio + QWEN3A = "qwen3a" # audio + GLMA = "glma" # audio + QWEN25O = "qwen2.5o" # omni VOXTRAL = "voxtral" MERALION = "meralion" # audio: Whisper + gated MLP adaptor LFM2 = "lfm2" @@ -4669,6 +4685,7 @@ class VisionProjectorType: GLM4V = "glm4v" YOUTUVL = "youtuvl" NEMOTRON_V2_VL = "nemotron_v2_vl" + HUNYUANOCR = "hunyuanocr" HUNYUANVL = "hunyuanvl" MINICPMV4_6 = "minicpmv4_6" GRANITE_SPEECH = "granite_speech" # audio @@ -4679,110 +4696,118 @@ class VisionProjectorType: # Items here are (block size, type size) QK_K = 256 GGML_QUANT_SIZES: dict[GGMLQuantizationType, tuple[int, int]] = { - GGMLQuantizationType.F32: (1, 4), - GGMLQuantizationType.F16: (1, 2), - GGMLQuantizationType.Q4_0: (32, 2 + 16), - GGMLQuantizationType.Q4_1: (32, 2 + 2 + 16), - GGMLQuantizationType.Q5_0: (32, 2 + 4 + 16), - GGMLQuantizationType.Q5_1: (32, 2 + 2 + 4 + 16), - GGMLQuantizationType.Q8_0: (32, 2 + 32), - GGMLQuantizationType.Q8_1: (32, 4 + 4 + 32), - GGMLQuantizationType.Q2_K: (256, 2 + 2 + QK_K // 16 + QK_K // 4), - GGMLQuantizationType.Q3_K: (256, 2 + QK_K // 4 + QK_K // 8 + 12), - GGMLQuantizationType.Q4_K: (256, 2 + 2 + QK_K // 2 + 12), - GGMLQuantizationType.Q5_K: (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12), - GGMLQuantizationType.Q6_K: (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16), - GGMLQuantizationType.Q8_K: (256, 4 + QK_K + QK_K // 8), + GGMLQuantizationType.F32: (1, 4), + GGMLQuantizationType.F16: (1, 2), + GGMLQuantizationType.Q4_0: (32, 2 + 16), + GGMLQuantizationType.Q4_1: (32, 2 + 2 + 16), + GGMLQuantizationType.Q5_0: (32, 2 + 4 + 16), + GGMLQuantizationType.Q5_1: (32, 2 + 2 + 4 + 16), + GGMLQuantizationType.Q8_0: (32, 2 + 32), + GGMLQuantizationType.Q8_1: (32, 4 + 4 + 32), + GGMLQuantizationType.Q2_K: (256, 2 + 2 + QK_K // 16 + QK_K // 4), + GGMLQuantizationType.Q3_K: (256, 2 + QK_K // 4 + QK_K // 8 + 12), + GGMLQuantizationType.Q4_K: (256, 2 + 2 + QK_K // 2 + 12), + GGMLQuantizationType.Q5_K: (256, 2 + 2 + QK_K // 2 + QK_K // 8 + 12), + GGMLQuantizationType.Q6_K: (256, 2 + QK_K // 2 + QK_K // 4 + QK_K // 16), + GGMLQuantizationType.Q8_K: (256, 4 + QK_K + QK_K // 8), GGMLQuantizationType.IQ2_XXS: (256, 2 + QK_K // 4), - GGMLQuantizationType.IQ2_XS: (256, 2 + QK_K // 4 + QK_K // 32), + GGMLQuantizationType.IQ2_XS: (256, 2 + QK_K // 4 + QK_K // 32), GGMLQuantizationType.IQ3_XXS: (256, 2 + QK_K // 4 + QK_K // 8), - GGMLQuantizationType.IQ1_S: (256, 2 + QK_K // 8 + QK_K // 16), - GGMLQuantizationType.IQ4_NL: (32, 2 + 16), - GGMLQuantizationType.IQ3_S: (256, 2 + QK_K // 4 + QK_K // 8 + QK_K // 32 + 4), - GGMLQuantizationType.IQ2_S: (256, 2 + QK_K // 4 + QK_K // 16), - GGMLQuantizationType.IQ4_XS: (256, 2 + 2 + QK_K // 2 + QK_K // 64), - GGMLQuantizationType.I8: (1, 1), - GGMLQuantizationType.I16: (1, 2), - GGMLQuantizationType.I32: (1, 4), - GGMLQuantizationType.I64: (1, 8), - GGMLQuantizationType.F64: (1, 8), - GGMLQuantizationType.IQ1_M: (256, QK_K // 8 + QK_K // 16 + QK_K // 32), - GGMLQuantizationType.BF16: (1, 2), - GGMLQuantizationType.TQ1_0: (256, 2 + 4 * 13), - GGMLQuantizationType.TQ2_0: (256, 2 + 64), - GGMLQuantizationType.MXFP4: (32, 1 + 16), - GGMLQuantizationType.NVFP4: (64, 4 + 32), - GGMLQuantizationType.Q1_0: (128, 2 + 16), + GGMLQuantizationType.IQ1_S: (256, 2 + QK_K // 8 + QK_K // 16), + GGMLQuantizationType.IQ4_NL: (32, 2 + 16), + GGMLQuantizationType.IQ3_S: (256, 2 + QK_K // 4 + QK_K // 8 + QK_K // 32 + 4), + GGMLQuantizationType.IQ2_S: (256, 2 + QK_K // 4 + QK_K // 16), + GGMLQuantizationType.IQ4_XS: (256, 2 + 2 + QK_K // 2 + QK_K // 64), + GGMLQuantizationType.I8: (1, 1), + GGMLQuantizationType.I16: (1, 2), + GGMLQuantizationType.I32: (1, 4), + GGMLQuantizationType.I64: (1, 8), + GGMLQuantizationType.F64: (1, 8), + GGMLQuantizationType.IQ1_M: (256, QK_K // 8 + QK_K // 16 + QK_K // 32), + GGMLQuantizationType.BF16: (1, 2), + GGMLQuantizationType.TQ1_0: (256, 2 + 4 * 13), + GGMLQuantizationType.TQ2_0: (256, 2 + 64), + GGMLQuantizationType.MXFP4: (32, 1 + 16), + GGMLQuantizationType.NVFP4: (64, 4 + 32), + GGMLQuantizationType.Q1_0: (128, 2 + 16), + GGMLQuantizationType.Q3_PT: (256, 124), + GGMLQuantizationType.Q3_KPT: (256, 110), + GGMLQuantizationType.Q4_DPT: (32, 18), + GGMLQuantizationType.Q2_DPT: (32, 10), + GGMLQuantizationType.Q2_KPT: (256, 84), + GGMLQuantizationType.IQ2_TQ: (256, 82), + GGMLQuantizationType.IQ3_TQ: (256, 114), + GGMLQuantizationType.IQ1_BN: (256, 50), } # Aliases for backward compatibility. # general -KEY_GENERAL_ARCHITECTURE = Keys.General.ARCHITECTURE +KEY_GENERAL_ARCHITECTURE = Keys.General.ARCHITECTURE KEY_GENERAL_QUANTIZATION_VERSION = Keys.General.QUANTIZATION_VERSION -KEY_GENERAL_ALIGNMENT = Keys.General.ALIGNMENT -KEY_GENERAL_NAME = Keys.General.NAME -KEY_GENERAL_AUTHOR = Keys.General.AUTHOR -KEY_GENERAL_URL = Keys.General.URL -KEY_GENERAL_DESCRIPTION = Keys.General.DESCRIPTION -KEY_GENERAL_LICENSE = Keys.General.LICENSE -KEY_GENERAL_SOURCE_URL = Keys.General.SOURCE_URL -KEY_GENERAL_FILE_TYPE = Keys.General.FILE_TYPE +KEY_GENERAL_ALIGNMENT = Keys.General.ALIGNMENT +KEY_GENERAL_NAME = Keys.General.NAME +KEY_GENERAL_AUTHOR = Keys.General.AUTHOR +KEY_GENERAL_URL = Keys.General.URL +KEY_GENERAL_DESCRIPTION = Keys.General.DESCRIPTION +KEY_GENERAL_LICENSE = Keys.General.LICENSE +KEY_GENERAL_SOURCE_URL = Keys.General.SOURCE_URL +KEY_GENERAL_FILE_TYPE = Keys.General.FILE_TYPE # LLM -KEY_VOCAB_SIZE = Keys.LLM.VOCAB_SIZE -KEY_CONTEXT_LENGTH = Keys.LLM.CONTEXT_LENGTH -KEY_EMBEDDING_LENGTH = Keys.LLM.EMBEDDING_LENGTH -KEY_BLOCK_COUNT = Keys.LLM.BLOCK_COUNT -KEY_FEED_FORWARD_LENGTH = Keys.LLM.FEED_FORWARD_LENGTH +KEY_VOCAB_SIZE = Keys.LLM.VOCAB_SIZE +KEY_CONTEXT_LENGTH = Keys.LLM.CONTEXT_LENGTH +KEY_EMBEDDING_LENGTH = Keys.LLM.EMBEDDING_LENGTH +KEY_BLOCK_COUNT = Keys.LLM.BLOCK_COUNT +KEY_FEED_FORWARD_LENGTH = Keys.LLM.FEED_FORWARD_LENGTH KEY_USE_PARALLEL_RESIDUAL = Keys.LLM.USE_PARALLEL_RESIDUAL -KEY_TENSOR_DATA_LAYOUT = Keys.LLM.TENSOR_DATA_LAYOUT +KEY_TENSOR_DATA_LAYOUT = Keys.LLM.TENSOR_DATA_LAYOUT # attention -KEY_ATTENTION_HEAD_COUNT = Keys.Attention.HEAD_COUNT -KEY_ATTENTION_HEAD_COUNT_KV = Keys.Attention.HEAD_COUNT_KV -KEY_ATTENTION_MAX_ALIBI_BIAS = Keys.Attention.MAX_ALIBI_BIAS -KEY_ATTENTION_CLAMP_KQV = Keys.Attention.CLAMP_KQV -KEY_ATTENTION_LAYERNORM_EPS = Keys.Attention.LAYERNORM_EPS +KEY_ATTENTION_HEAD_COUNT = Keys.Attention.HEAD_COUNT +KEY_ATTENTION_HEAD_COUNT_KV = Keys.Attention.HEAD_COUNT_KV +KEY_ATTENTION_MAX_ALIBI_BIAS = Keys.Attention.MAX_ALIBI_BIAS +KEY_ATTENTION_CLAMP_KQV = Keys.Attention.CLAMP_KQV +KEY_ATTENTION_LAYERNORM_EPS = Keys.Attention.LAYERNORM_EPS KEY_ATTENTION_LAYERNORM_RMS_EPS = Keys.Attention.LAYERNORM_RMS_EPS # RoPE -KEY_ROPE_DIMENSION_COUNT = Keys.Rope.DIMENSION_COUNT -KEY_ROPE_FREQ_BASE = Keys.Rope.FREQ_BASE -KEY_ROPE_SCALING_TYPE = Keys.Rope.SCALING_TYPE -KEY_ROPE_SCALING_FACTOR = Keys.Rope.SCALING_FACTOR -KEY_ROPE_SCALING_ORIG_CTX_LEN = Keys.Rope.SCALING_ORIG_CTX_LEN -KEY_ROPE_SCALING_FINETUNED = Keys.Rope.SCALING_FINETUNED +KEY_ROPE_DIMENSION_COUNT = Keys.Rope.DIMENSION_COUNT +KEY_ROPE_FREQ_BASE = Keys.Rope.FREQ_BASE +KEY_ROPE_SCALING_TYPE = Keys.Rope.SCALING_TYPE +KEY_ROPE_SCALING_FACTOR = Keys.Rope.SCALING_FACTOR +KEY_ROPE_SCALING_ORIG_CTX_LEN = Keys.Rope.SCALING_ORIG_CTX_LEN +KEY_ROPE_SCALING_FINETUNED = Keys.Rope.SCALING_FINETUNED # SSM -KEY_SSM_CONV_KERNEL = Keys.SSM.CONV_KERNEL -KEY_SSM_INNER_SIZE = Keys.SSM.INNER_SIZE -KEY_SSM_STATE_SIZE = Keys.SSM.STATE_SIZE +KEY_SSM_CONV_KERNEL = Keys.SSM.CONV_KERNEL +KEY_SSM_INNER_SIZE = Keys.SSM.INNER_SIZE +KEY_SSM_STATE_SIZE = Keys.SSM.STATE_SIZE KEY_SSM_TIME_STEP_RANK = Keys.SSM.TIME_STEP_RANK -KEY_SSM_GROUP_COUNT = Keys.SSM.GROUP_COUNT -KEY_SSM_DT_B_C_RMS = Keys.SSM.DT_B_C_RMS +KEY_SSM_GROUP_COUNT = Keys.SSM.GROUP_COUNT +KEY_SSM_DT_B_C_RMS = Keys.SSM.DT_B_C_RMS # KDA -KEY_KDA_HEAD_DIM = Keys.KDA.HEAD_DIM +KEY_KDA_HEAD_DIM = Keys.KDA.HEAD_DIM # tokenization -KEY_TOKENIZER_MODEL = Keys.Tokenizer.MODEL -KEY_TOKENIZER_PRE = Keys.Tokenizer.PRE -KEY_TOKENIZER_LIST = Keys.Tokenizer.LIST +KEY_TOKENIZER_MODEL = Keys.Tokenizer.MODEL +KEY_TOKENIZER_PRE = Keys.Tokenizer.PRE +KEY_TOKENIZER_LIST = Keys.Tokenizer.LIST KEY_TOKENIZER_TOKEN_TYPE = Keys.Tokenizer.TOKEN_TYPE -KEY_TOKENIZER_SCORES = Keys.Tokenizer.SCORES -KEY_TOKENIZER_MERGES = Keys.Tokenizer.MERGES -KEY_TOKENIZER_BOS_ID = Keys.Tokenizer.BOS_ID -KEY_TOKENIZER_EOS_ID = Keys.Tokenizer.EOS_ID -KEY_TOKENIZER_EOT_ID = Keys.Tokenizer.EOT_ID -KEY_TOKENIZER_EOM_ID = Keys.Tokenizer.EOM_ID -KEY_TOKENIZER_UNK_ID = Keys.Tokenizer.UNK_ID -KEY_TOKENIZER_SEP_ID = Keys.Tokenizer.SEP_ID -KEY_TOKENIZER_PAD_ID = Keys.Tokenizer.PAD_ID -KEY_TOKENIZER_MASK_ID = Keys.Tokenizer.MASK_ID -KEY_TOKENIZER_HF_JSON = Keys.Tokenizer.HF_JSON -KEY_TOKENIZER_RWKV = Keys.Tokenizer.RWKV +KEY_TOKENIZER_SCORES = Keys.Tokenizer.SCORES +KEY_TOKENIZER_MERGES = Keys.Tokenizer.MERGES +KEY_TOKENIZER_BOS_ID = Keys.Tokenizer.BOS_ID +KEY_TOKENIZER_EOS_ID = Keys.Tokenizer.EOS_ID +KEY_TOKENIZER_EOT_ID = Keys.Tokenizer.EOT_ID +KEY_TOKENIZER_EOM_ID = Keys.Tokenizer.EOM_ID +KEY_TOKENIZER_UNK_ID = Keys.Tokenizer.UNK_ID +KEY_TOKENIZER_SEP_ID = Keys.Tokenizer.SEP_ID +KEY_TOKENIZER_PAD_ID = Keys.Tokenizer.PAD_ID +KEY_TOKENIZER_MASK_ID = Keys.Tokenizer.MASK_ID +KEY_TOKENIZER_HF_JSON = Keys.Tokenizer.HF_JSON +KEY_TOKENIZER_RWKV = Keys.Tokenizer.RWKV KEY_TOKENIZER_FIM_PRE_ID = Keys.Tokenizer.FIM_PRE_ID KEY_TOKENIZER_FIM_SUF_ID = Keys.Tokenizer.FIM_SUF_ID @@ -4792,6 +4817,6 @@ class VisionProjectorType: KEY_TOKENIZER_FIM_SEP_ID = Keys.Tokenizer.FIM_SEP_ID # deprecated -KEY_TOKENIZER_PREFIX_ID = Keys.Tokenizer.PREFIX_ID -KEY_TOKENIZER_SUFFIX_ID = Keys.Tokenizer.SUFFIX_ID -KEY_TOKENIZER_MIDDLE_ID = Keys.Tokenizer.MIDDLE_ID +KEY_TOKENIZER_PREFIX_ID = Keys.Tokenizer.PREFIX_ID +KEY_TOKENIZER_SUFFIX_ID = Keys.Tokenizer.SUFFIX_ID +KEY_TOKENIZER_MIDDLE_ID = Keys.Tokenizer.MIDDLE_ID diff --git a/include/llama.h b/include/llama.h index b89ab758b28f..63fe85f14b1f 100644 --- a/include/llama.h +++ b/include/llama.h @@ -155,6 +155,14 @@ extern "C" { LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_PT = 41, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_KPT = 42, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_DPT = 43, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_KPT = 44, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_TQ = 45, // except 1d tensors, trellis quantized with RNG codebook + LLAMA_FTYPE_MOSTLY_IQ3_TQ = 46, // except 1d tensors, 3-bit with per-tensor trained grid + LLAMA_FTYPE_MOSTLY_IQ1_BN = 47, // except 1d tensors, 8D vector quantized with trained codebook + LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; diff --git a/pocs/vdot/q8dot.cpp b/pocs/vdot/q8dot.cpp index 3df6e1f42112..bdf6414aad49 100644 --- a/pocs/vdot/q8dot.cpp +++ b/pocs/vdot/q8dot.cpp @@ -157,8 +157,8 @@ int main(int argc, char** argv) { t1 = std::chrono::high_resolution_clock::now(); float fs; - if (type == 0) funcs->vec_dot(kVecSize * QK4_1, &fs, 0, x40.data(), 0, y.data(), 0, 1); - else funcs->vec_dot(kVecSize * QK4_1, &fs, 0, x41.data(), 0, y.data(), 0, 1); + if (type == 0) funcs->vec_dot(kVecSize * QK4_1, &fs, 0, x40.data(), 0, y.data(), 0, 1, nullptr); + else funcs->vec_dot(kVecSize * QK4_1, &fs, 0, x41.data(), 0, y.data(), 0, 1, nullptr); t2 = std::chrono::high_resolution_clock::now(); t = 1e-3*std::chrono::duration_cast(t2-t1).count(); if (iloop > 3) ggml.addResult(fs, t); diff --git a/pocs/vdot/vdot.cpp b/pocs/vdot/vdot.cpp index 2dca62848bca..a78fabc28c0a 100644 --- a/pocs/vdot/vdot.cpp +++ b/pocs/vdot/vdot.cpp @@ -285,8 +285,8 @@ int main(int argc, char** argv) { else { const auto * vdot = ggml_get_type_traits_cpu(funcs_cpu->vec_dot_type); vdot->from_float(y1.data(), q8.data(), kVecSize); - if (useQ4_1) funcs_cpu->vec_dot(kVecSize, &result, 0, q41.data(), 0, q8.data(), 0, 1); - else funcs_cpu->vec_dot(kVecSize, &result, 0, q40.data(), 0, q8.data(), 0, 1); + if (useQ4_1) funcs_cpu->vec_dot(kVecSize, &result, 0, q41.data(), 0, q8.data(), 0, 1, nullptr); + else funcs_cpu->vec_dot(kVecSize, &result, 0, q40.data(), 0, q8.data(), 0, 1, nullptr); } sumq += result; t2 = std::chrono::high_resolution_clock::now(); diff --git a/scripts/analyze-ffn-down.py b/scripts/analyze-ffn-down.py new file mode 100644 index 000000000000..2f43089fac7f --- /dev/null +++ b/scripts/analyze-ffn-down.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python3 +"""Deep analysis of WHY ffn_down is hard to quantize. +Compares structural properties of all weight and activation tensors. +""" + +import numpy as np +import struct +import sys +import os + +DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data") + + +def load_f32_tensor(name): + path = os.path.join(DATA_DIR, name) + with open(path, "rb") as f: + nrow, ncol = struct.unpack("qq", f.read(16)) + data = np.frombuffer(f.read(), dtype=np.float32) + assert len(data) == nrow * ncol, f"Expected {nrow * ncol}, got {len(data)}" + return data.reshape(nrow, ncol) + + +def stats(label, arr): + """Print comprehensive statistics for a flat array.""" + a = arr.ravel() + print(f" {label}:") + print(f" shape={arr.shape}, n={len(a)}") + print(f" mean={a.mean():.6f}, std={a.std():.6f}") + print(f" min={a.min():.6f}, max={a.max():.6f}") + print(f" median={np.median(a):.6f}") + print( + f" |mean|/std = {abs(a.mean()) / (a.std() + 1e-10):.4f} (offset-to-spread ratio)" + ) + # Kurtosis (excess) - how heavy-tailed vs Gaussian + kurt = np.mean(((a - a.mean()) / (a.std() + 1e-10)) ** 4) - 3.0 + # Skewness + skew = np.mean(((a - a.mean()) / (a.std() + 1e-10)) ** 3) + print(f" skewness={skew:.4f}, excess_kurtosis={kurt:.4f}") + # Percentile ranges + pcts = np.percentile(a, [0.1, 1, 5, 25, 50, 75, 95, 99, 99.9]) + print( + f" percentiles: 0.1%={pcts[0]:.4f}, 1%={pcts[1]:.4f}, 5%={pcts[2]:.4f}, " + f"25%={pcts[3]:.4f}, 50%={pcts[4]:.4f}, 75%={pcts[5]:.4f}, " + f"95%={pcts[6]:.4f}, 99%={pcts[7]:.4f}, 99.9%={pcts[8]:.4f}" + ) + # Sparsity + near_zero = np.sum(np.abs(a) < 0.001 * a.std()) / len(a) + print(f" fraction |x| < 0.001*std: {near_zero:.4f}") + return { + "mean": a.mean(), + "std": a.std(), + "skew": skew, + "kurt": kurt, + "min": a.min(), + "max": a.max(), + } + + +# ============================================================================ +# 1. BASIC WEIGHT TENSOR COMPARISON +# ============================================================================ +print("=" * 80) +print("SECTION 1: WEIGHT TENSOR GLOBAL STATISTICS") +print("=" * 80) + +tensors = { + "ffn_gate": ("blk_0_ffn_gate_weight.f32bin", "9728x2560 (wide→narrow proj)"), + "ffn_up": ("blk_0_ffn_up_weight.f32bin", "9728x2560 (wide→narrow proj)"), + "ffn_down": ("blk_0_ffn_down_weight.f32bin", "2560x9728 (narrow→wide proj)"), + "attn_q": ("blk_0_attn_q_weight.f32bin", "4096x2560"), + "attn_k": ("blk_0_attn_k_weight.f32bin", "1024x2560"), + "attn_v": ("blk_0_attn_v_weight.f32bin", "1024x2560"), + "attn_out": ("blk_0_attn_output_weight.f32bin", "2560x4096"), +} + +weight_data = {} +for name, (fname, desc) in tensors.items(): + try: + W = load_f32_tensor(fname) + print(f"\n{'─' * 70}") + print(f" {name} [{desc}] — file: {fname}") + weight_data[name] = W + stats(name, W) + except Exception as e: + print(f" {name}: SKIP ({e})") + +# ============================================================================ +# 2. ROW-LEVEL STATISTICS (each row is a neuron output) +# ============================================================================ +print("\n" + "=" * 80) +print("SECTION 2: ROW-LEVEL VARIABILITY (per-neuron weight statistics)") +print("=" * 80) +print(" Each row of the weight matrix produces one output dimension.") +print(" High row-to-row variability in mean/std means the quantizer") +print(" must handle very different distributions across rows.\n") + +for name, W in weight_data.items(): + row_means = W.mean(axis=1) + row_stds = W.std(axis=1) + row_ranges = W.max(axis=1) - W.min(axis=1) + + print(f"\n {name} ({W.shape[0]} rows × {W.shape[1]} cols):") + print( + f" Row means: mean={row_means.mean():.6f}, std={row_means.std():.6f}, " + f"range=[{row_means.min():.6f}, {row_means.max():.6f}]" + ) + print( + f" Row stds: mean={row_stds.mean():.6f}, std={row_stds.std():.6f}, " + f"range=[{row_stds.min():.6f}, {row_stds.max():.6f}]" + ) + print(f" Row ranges: mean={row_ranges.mean():.6f}, std={row_ranges.std():.6f}") + print( + f" RowMeans CV (std/mean): {row_means.std() / (abs(row_means.mean()) + 1e-10):.4f}" + ) + print(f" RowStds CV: {row_stds.std() / (row_stds.mean() + 1e-10):.4f}") + +# ============================================================================ +# 3. GROUP-LEVEL ANALYSIS (16-element groups, like Q2_K) +# ============================================================================ +print("\n" + "=" * 80) +print("SECTION 3: GROUP-LEVEL ANALYSIS (16-element groups)") +print("=" * 80) +print(" Quantization works on 16-element groups. Key question:") +print(" How much does each group need its own OFFSET (dmin)?\n") + +GS = 16 + +for name, W in weight_data.items(): + # Look at first 256 rows for speed + nr = min(W.shape[0], 256) + nc = W.shape[1] + + group_means = [] + group_stds = [] + group_ranges = [] + group_offsets = [] # |mean| / range — how important is the offset + + for r in range(nr): + for g_start in range(0, nc, GS): + g = W[r, g_start : g_start + GS] + gm = g.mean() + gs = g.std() + gr = g.max() - g.min() + gmin = g.min() + + group_means.append(gm) + group_stds.append(gs) + group_ranges.append(gr) + # Offset importance: how large is the group mean relative to its range? + # If this is high, offset (dmin) matters a lot + if gr > 1e-10: + group_offsets.append(abs(gm) / gr) + else: + group_offsets.append(0) + + gm = np.array(group_means) + gs = np.array(group_stds) + gr = np.array(group_ranges) + go = np.array(group_offsets) + + print(f"\n {name} ({len(group_means)} groups):") + print( + f" Group mean: mean={gm.mean():.6f}, std={gm.std():.6f}, " + f"range=[{gm.min():.6f}, {gm.max():.6f}]" + ) + print(f" Group std: mean={gs.mean():.6f}, std={gs.std():.6f}") + print(f" Group range: mean={gr.mean():.6f}, std={gr.std():.6f}") + print(f" *** OFFSET IMPORTANCE (|group_mean| / range) ***") + print( + f" mean={go.mean():.4f}, median={np.median(go):.4f}, " + f"p90={np.percentile(go, 90):.4f}, max={go.max():.4f}" + ) + print(f" fraction with offset > 0.1: {np.mean(go > 0.1):.3f}") + print(f" fraction with offset > 0.2: {np.mean(go > 0.2):.3f}") + print(f" fraction with offset > 0.3: {np.mean(go > 0.3):.3f}") + + # How well does zeroing the min (Q2_K style, clamping min to 0) work? + # vs keeping the actual min + mse_no_offset = 0 # Assume uniform 4 levels [0,1,2,3] * scale + mse_with_offset = 0 # Assume uniform 4 levels [0,1,2,3] * scale + offset + + for r in range(nr): + for g_start in range(0, nc, GS): + g = W[r, g_start : g_start + GS] + gmin = g.min() + gmax = g.max() + gr = gmax - gmin + if gr < 1e-10: + continue + + # No offset: clamp min to 0, scale = max/3 + if gmin > 0: + scale_no = gmax / 3.0 + min_no = 0 + else: + scale_no = gmax / 3.0 + min_no = 0 # lose the negative offset + # Actually use (gmax - 0)/3 but we're clamping gmin to 0 + + # Better: use actual min/max + scale_w = gr / 3.0 + min_w = gmin + + for val in g: + # No offset quantization + norm_no = val / (scale_no + 1e-10) + idx_no = max(0, min(3, int(round(norm_no)))) + recon_no = scale_no * idx_no + mse_no_offset += (val - recon_no) ** 2 + + # With offset quantization + norm_w = (val - min_w) / (scale_w + 1e-10) + idx_w = max(0, min(3, int(round(norm_w)))) + recon_w = min_w + scale_w * idx_w + mse_with_offset += (val - recon_w) ** 2 + + total_elements = nr * nc + rmse_no = np.sqrt(mse_no_offset / total_elements) + rmse_w = np.sqrt(mse_with_offset / total_elements) + improvement = (rmse_no - rmse_w) / rmse_no * 100 + print(f" Quant RMSE (no offset): {rmse_no:.6f}") + print(f" Quant RMSE (with offset): {rmse_w:.6f}") + print(f" Offset benefit: {improvement:.1f}% RMSE reduction") + +# ============================================================================ +# 4. ACTIVATION ANALYSIS +# ============================================================================ +print("\n" + "=" * 80) +print("SECTION 4: ACTIVATION DISTRIBUTION COMPARISON") +print("=" * 80) + +activations = { + "ffn_input (gate/up)": "act_blk0_ffn_input.f32bin", + "ffn_down_input (swiglu)": "act_blk0_ffn_down_input.f32bin", + "attn_input (q/k/v)": "act_blk0_attn_input.f32bin", + "attn_output_input": "act_blk0_attn_output_input.f32bin", +} + +act_data = {} +for name, fname in activations.items(): + try: + A = load_f32_tensor(fname) + act_data[name] = A + print(f"\n{'─' * 70}") + print(f" {name} — {fname}") + stats(name, A) + except Exception as e: + print(f" {name}: SKIP ({e})") + +# ============================================================================ +# 5. THE CRITICAL QUESTION: PER-DIMENSION ACTIVATION MAGNITUDE +# ============================================================================ +print("\n" + "=" * 80) +print("SECTION 5: PER-DIMENSION ACTIVATION POWER (per-column RMS)") +print("=" * 80) +print(" If activation dimensions have very different magnitudes,") +print(" the quantization error in each weight dimension is weighted differently.") +print(" Dimensions with high activation power amplify weight errors.\n") + +for name, A in act_data.items(): + col_rms = np.sqrt(np.mean(A**2, axis=0)) # RMS per column (dimension) + print(f"\n {name} ({A.shape[1]} dimensions):") + print(f" Col RMS: mean={col_rms.mean():.6f}, std={col_rms.std():.6f}") + print(f" Col RMS range: [{col_rms.min():.6f}, {col_rms.max():.6f}]") + print(f" Col RMS CV (std/mean): {col_rms.std() / (col_rms.mean() + 1e-10):.4f}") + print(f" Max/Min ratio: {col_rms.max() / (col_rms.min() + 1e-10):.1f}x") + + # Top 10 and bottom 10 dimensions by power + top10 = np.argsort(col_rms)[-10:][::-1] + bot10 = np.argsort(col_rms)[:10] + print( + f" Top-10 dims by RMS: {[(int(d), f'{col_rms[d]:.4f}') for d in top10[:5]]}..." + ) + print( + f" Bot-10 dims by RMS: {[(int(d), f'{col_rms[d]:.4f}') for d in bot10[:5]]}..." + ) + + # How much do the top 10% of dimensions contribute to total power? + total_power = np.sum(col_rms**2) + sorted_power = np.sort(col_rms**2)[::-1] + top10pct = int(len(col_rms) * 0.1) + top10pct_power = np.sum(sorted_power[:top10pct]) + top1pct = max(1, int(len(col_rms) * 0.01)) + top1pct_power = np.sum(sorted_power[:top1pct]) + print( + f" Top 10% of dims contribute {top10pct_power / total_power * 100:.1f}% of total power" + ) + print( + f" Top 1% of dims contribute {top1pct_power / total_power * 100:.1f}% of total power" + ) + +# ============================================================================ +# 6. CROSS-CORRELATION: WEIGHT ERROR × ACTIVATION POWER +# ============================================================================ +print("\n" + "=" * 80) +print("SECTION 6: WHERE DO WEIGHT ERRORS MEET HIGH ACTIVATION POWER?") +print("=" * 80) +print(" For each weight dimension, compute: activation_rms[dim] × weight_error[dim]") +print(" This tells us which dimensions contribute most to matmul error.\n") + +# Focus on ffn_down vs ffn_gate for comparison +focus = [ + ("ffn_down", "blk_0_ffn_down_weight.f32bin", "act_blk0_ffn_down_input.f32bin"), + ("ffn_gate", "blk_0_ffn_gate_weight.f32bin", "act_blk0_ffn_input.f32bin"), + ("ffn_up", "blk_0_ffn_up_weight.f32bin", "act_blk0_ffn_input.f32bin"), + ("attn_q", "blk_0_attn_q_weight.f32bin", "act_blk0_attn_input.f32bin"), +] + +for name, wfile, afile in focus: + W = load_f32_tensor(wfile) + A = load_f32_tensor(afile) + + if W.shape[1] != A.shape[1]: + print(f" {name}: dim mismatch W={W.shape[1]} vs A={A.shape[1]}, SKIP") + continue + + nc = W.shape[1] + + # Per-column activation RMS + act_rms = np.sqrt(np.mean(A**2, axis=0)) + + # Per-column weight std and range (how "hard" to quantize) + w_std = W.std(axis=0) + w_range = W.max(axis=0) - W.min(axis=0) + + # Per-column weight kurtosis (heavy tails = harder to quantize) + w_kurt = ( + np.mean(((W - W.mean(axis=0)) / (W.std(axis=0) + 1e-10)) ** 4, axis=0) - 3.0 + ) + + # Weight error proxy: with 2-bit uniform quant on 16-element groups + # Higher variance columns → more error + nr = min(W.shape[0], 256) + + # Simple Q2_K-style error estimate per dimension: + # For each group of 16 in the column direction, quantize and measure error + dim_mse = np.zeros(nc) + for g_start in range(0, nc, GS): + g_end = min(g_start + GS, nc) + for r in range(nr): + g = W[r, g_start:g_end] + gmin = min(g.min(), 0) # Q2_K clamps min to ≤0 + gmax = g.max() + gr = gmax - gmin + if gr < 1e-10: + continue + scale = gr / 3.0 + for i, val in enumerate(g): + norm = (val - gmin) / scale + idx = max(0, min(3, int(round(norm)))) + recon = gmin + scale * idx + dim_mse[g_start + i] += (val - recon) ** 2 + + dim_rmse = np.sqrt(dim_mse / nr) + + # The key metric: dimension-level contribution to matmul error + # matmul_error_contribution[d] ≈ act_rms[d] * weight_rmse[d] + matmul_contrib = act_rms * dim_rmse + + print(f"\n {name} ({nc} dimensions):") + print( + f" act_rms: mean={act_rms.mean():.4f}, CV={act_rms.std() / act_rms.mean():.4f}" + ) + print( + f" w_rmse: mean={dim_rmse.mean():.6f}, CV={dim_rmse.std() / (dim_rmse.mean() + 1e-10):.4f}" + ) + print( + f" matmul_contrib: mean={matmul_contrib.mean():.6f}, " + f"std={matmul_contrib.std():.6f}" + ) + + # Correlation between activation power and weight error + corr = np.corrcoef(act_rms, dim_rmse)[0, 1] + print(f" CORRELATION act_rms ↔ weight_rmse: {corr:.4f}") + print(f" (>0 means high-power dims are also hard to quantize — BAD)") + + # Top contributors to matmul error + top_dims = np.argsort(matmul_contrib)[-20:][::-1] + print(f" Top-5 error-contributing dimensions:") + for d in top_dims[:5]: + print( + f" dim {d}: act_rms={act_rms[d]:.4f}, w_rmse={dim_rmse[d]:.6f}, " + f"contrib={matmul_contrib[d]:.6f}, w_std={w_std[d]:.6f}, w_kurt={w_kurt[d]:.2f}" + ) + + # Distribution of matmul contributions + total_contrib = matmul_contrib.sum() + sorted_contrib = np.sort(matmul_contrib)[::-1] + for pct in [0.01, 0.05, 0.10, 0.25]: + n = max(1, int(nc * pct)) + print( + f" Top {pct * 100:.0f}% dims: {sorted_contrib[:n].sum() / total_contrib * 100:.1f}% " + f"of total matmul error" + ) + +# ============================================================================ +# 7. THE STRUCTURAL ASYMMETRY: COLUMN DIRECTION GROUP ANALYSIS +# ============================================================================ +print("\n" + "=" * 80) +print("SECTION 7: STRUCTURAL ASYMMETRY — COLUMN vs ROW GROUPING") +print("=" * 80) +print(" Quantization groups along the ROW (inner dim). For ffn_down,") +print(" each row has 9728 elements (38 groups of 256).") +print(" For ffn_gate, each row has 2560 elements (10 groups of 256).") +print(" More groups = more metadata (scales/offsets) relative to data bits.\n") + +for name, wfile, afile in focus: + W = load_f32_tensor(wfile) + nc = W.shape[1] + n_groups_per_row = nc // 256 # super-blocks per row + + print(f"\n {name}: {nc} cols → {n_groups_per_row} super-blocks per row") + print(f" Groups per row: {nc // GS} (16-element groups)") + print( + f" With Q2_K (2.625 bpw): {n_groups_per_row * 2} scale+offset bytes per row" + ) + + # How much do group means vary WITHIN a row? + nr = min(W.shape[0], 64) + intra_row_mean_var = [] + for r in range(nr): + group_means = [] + for g_start in range(0, nc, GS): + group_means.append(W[r, g_start : g_start + GS].mean()) + group_means = np.array(group_means) + intra_row_mean_var.append(group_means.std()) + + print( + f" Intra-row group mean variability (avg across rows): " + f"mean={np.mean(intra_row_mean_var):.6f}" + ) + + # How much does the sign of group means vary? + pos_frac = 0 + neg_frac = 0 + total_groups = 0 + for r in range(nr): + for g_start in range(0, nc, GS): + gm = W[r, g_start : g_start + GS].mean() + if gm > 0.001: + pos_frac += 1 + elif gm < -0.001: + neg_frac += 1 + total_groups += 1 + print( + f" Group mean sign: {pos_frac / total_groups * 100:.1f}% positive, " + f"{neg_frac / total_groups * 100:.1f}% negative, " + f"{(1 - pos_frac / total_groups - neg_frac / total_groups) * 100:.1f}% near-zero" + ) + +# ============================================================================ +# 8. THE SWIGLU EFFECT: WHY ffn_down INPUT IS SPECIAL +# ============================================================================ +print("\n" + "=" * 80) +print("SECTION 8: THE SWIGLU EFFECT — ffn_down ACTIVATION STRUCTURE") +print("=" * 80) +print(" ffn_down's activation is the SwiGLU output: silu(gate) * up") +print(" This creates a specific activation pattern that differs from") +print(" raw FFN input (RMSNorm output).\n") + +if "ffn_input (gate/up)" in act_data and "ffn_down_input (swiglu)" in act_data: + A_in = act_data["ffn_input (gate/up)"] + A_swiglu = act_data["ffn_down_input (swiglu)"] + + print(f" FFN input (RMSNorm output): {A_in.shape}") + print(f" SwiGLU output: {A_swiglu.shape}") + + # Per-token analysis + for t in range(min(A_swiglu.shape[0], 3)): + tok_in = A_in[t] + tok_sw = A_swiglu[t] + print(f"\n Token {t}:") + print( + f" FFN input: mean={tok_in.mean():.6f}, std={tok_in.std():.6f}, " + f"|max|={np.abs(tok_in).max():.6f}" + ) + print( + f" SwiGLU out: mean={tok_sw.mean():.6f}, std={tok_sw.std():.6f}, " + f"|max|={np.abs(tok_sw).max():.6f}" + ) + + # SwiGLU creates lots of near-zero values (silu suppresses negatives) + frac_nearzero_sw = np.mean(np.abs(tok_sw) < 0.01 * tok_sw.std()) + frac_nearzero_in = np.mean(np.abs(tok_in) < 0.01 * tok_in.std()) + print( + f" Near-zero fraction: FFN input={frac_nearzero_in:.3f}, " + f"SwiGLU={frac_nearzero_sw:.3f}" + ) + + # Sparsity pattern + frac_neg = np.mean(tok_sw < 0) + print(f" SwiGLU negative fraction: {frac_neg:.3f}") + + # Dimension-level analysis of SwiGLU + print(f"\n Dimension-level SwiGLU properties:") + dim_mean_sw = A_swiglu.mean(axis=0) + dim_std_sw = A_swiglu.std(axis=0) + dim_sparsity = np.mean(A_swiglu < 0, axis=0) # fraction of tokens negative per dim + + print(f" Dim mean range: [{dim_mean_sw.min():.6f}, {dim_mean_sw.max():.6f}]") + print(f" Dim std range: [{dim_std_sw.min():.6f}, {dim_std_sw.max():.6f}]") + print( + f" Dim negative fraction: mean={dim_sparsity.mean():.3f}, " + f"range=[{dim_sparsity.min():.3f}, {dim_sparsity.max():.3f}]" + ) + + # Highly sparse dimensions (mostly near-zero after SwiGLU) + high_sparsity = np.sum(dim_sparsity > 0.7) + low_sparsity = np.sum(dim_sparsity < 0.3) + print(f" Dims with >70% negative tokens: {high_sparsity}/{len(dim_sparsity)}") + print(f" Dims with <30% negative tokens: {low_sparsity}/{len(dim_sparsity)}") + +# ============================================================================ +# 9. QUANTIZATION NOISE × ACTIVATION POWER: THE MATMUL ERROR DECOMPOSITION +# ============================================================================ +print("\n" + "=" * 80) +print("SECTION 9: MATMUL ERROR DECOMPOSITION") +print("=" * 80) +print( + " matmul_error ≈ sum over groups of (activation_power_in_group × " + "weight_mse_in_group)" +) +print( + " If activation power is concentrated in groups with high weight error, " + "matmul error explodes.\n" +) + +# For ffn_down specifically, compare where activation power sits vs weight error +W_down = load_f32_tensor("blk_0_ffn_down_weight.f32bin") +A_swiglu = load_f32_tensor("act_blk0_ffn_down_input.f32bin") + +W_gate = load_f32_tensor("blk_0_ffn_gate_weight.f32bin") +A_ffn_in = load_f32_tensor("act_blk0_ffn_input.f32bin") + +for label, W, A in [("ffn_down", W_down, A_swiglu), ("ffn_gate", W_gate, A_ffn_in)]: + nc = W.shape[1] + nr = min(W.shape[0], 128) + + # Compute per-superblock (256) activation power and weight error + n_sb = nc // 256 + sb_act_power = np.zeros(n_sb) + sb_weight_mse = np.zeros(n_sb) + + for sb in range(n_sb): + s = sb * 256 + e = s + 256 + # Activation power: mean squared activation in this region + sb_act_power[sb] = np.mean(A[:, s:e] ** 2) + + # Weight MSE: Q2_K-style uniform quant error + mse = 0 + count = 0 + for r in range(nr): + for g in range(0, 256, GS): + gvals = W[r, s + g : s + g + GS] + gmin = min(gvals.min(), 0) + gmax = gvals.max() + gr = gmax - gmin + if gr < 1e-10: + continue + scale = gr / 3.0 + for v in gvals: + norm = (v - gmin) / scale + idx = max(0, min(3, int(round(norm)))) + recon = gmin + scale * idx + mse += (v - recon) ** 2 + count += 1 + sb_weight_mse[sb] = mse / max(count, 1) + + # Correlation between activation power and weight error across super-blocks + valid = sb_act_power > 1e-10 + if valid.sum() > 10: + corr = np.corrcoef(np.sqrt(sb_act_power[valid]), np.sqrt(sb_weight_mse[valid]))[ + 0, 1 + ] + else: + corr = 0 + + print(f"\n {label}:") + print(f" Super-blocks: {n_sb}") + print( + f" act_power: mean={sb_act_power.mean():.6f}, " + f"std={np.sqrt(sb_act_power.var()):.6f}, " + f"range=[{sb_act_power.min():.6f}, {sb_act_power.max():.6f}]" + ) + print( + f" weight_mse: mean={sb_weight_mse.mean():.6f}, " + f"range=[{sb_weight_mse.min():.6f}, {sb_weight_mse.max():.6f}]" + ) + print(f" CORRELATION (act_power ↔ weight_mse): {corr:.4f}") + + # Show top-5 super-blocks by contribution to matmul error + contrib = sb_act_power * sb_weight_mse + top5 = np.argsort(contrib)[-5:][::-1] + print(f" Top-5 error-contributing super-blocks (of {n_sb}):") + for idx in top5: + print( + f" SB {idx * 256}-{(idx + 1) * 256 - 1}: act_power={sb_act_power[idx]:.6f}, " + f"weight_mse={sb_weight_mse[idx]:.6f}, contrib={contrib[idx]:.6f}" + ) + +print("\n" + "=" * 80) +print("ANALYSIS COMPLETE") +print("=" * 80) diff --git a/scripts/compute-imatrix.py b/scripts/compute-imatrix.py new file mode 100644 index 000000000000..0b8d394d1b71 --- /dev/null +++ b/scripts/compute-imatrix.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Compute imatrix (importance matrix) from captured activation tensors. + +The imatrix is the per-dimension sum-of-squares of the activations. +It's what upstream llama.cpp uses to weight quantization optimization. + +For each activation file act_blkL_*.f32bin, produces imatrix_blkL_.f32bin +where matches the weight tensor it multiplies with. + +Format: flat float32 array of length n_per_row, one importance value per dimension. +""" + +import numpy as np +import struct +import os + +DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data") + + +def load_f32_tensor(name): + path = os.path.join(DATA_DIR, name) + with open(path, "rb") as f: + nrow, ncol = struct.unpack("qq", f.read(16)) + data = np.frombuffer(f.read(), dtype=np.float32) + assert len(data) == nrow * ncol + return data.reshape(nrow, ncol) + + +def save_imatrix(name, data): + path = os.path.join(DATA_DIR, name) + data.astype(np.float32).tofile(path) + print( + f" Wrote {path}: {len(data)} dims, " + f"min={data.min():.6f}, max={data.max():.6f}, mean={data.mean():.6f}" + ) + + +# Mapping: activation file → imatrix files for each weight it multiplies with +# Each weight tensor's column dimension matches the activation's column dimension +mappings = [ + { + "act_file": "act_blk0_ffn_input.f32bin", + "imatrix_name": "imatrix_blk0_ffn_gate_up.f32bin", + "description": "ffn_gate and ffn_up (both use ffn_input activation)", + }, + { + "act_file": "act_blk0_ffn_down_input.f32bin", + "imatrix_name": "imatrix_blk0_ffn_down.f32bin", + "description": "ffn_down (uses SwiGLU activation)", + }, + { + "act_file": "act_blk0_attn_input.f32bin", + "imatrix_name": "imatrix_blk0_attn_qkv.f32bin", + "description": "attn_q, attn_k, attn_v (all use attn_input activation)", + }, + { + "act_file": "act_blk0_attn_output_input.f32bin", + "imatrix_name": "imatrix_blk0_attn_output.f32bin", + "description": "attn_output (uses kqv_out activation)", + }, +] + +print("Computing imatrix from captured activations") +print("=" * 60) + +for m in mappings: + try: + A = load_f32_tensor(m["act_file"]) + print(f"\n{m['description']}:") + print(f" Activation: {A.shape[0]} tokens × {A.shape[1]} dims") + + # imatrix = sum over tokens of activation^2 + # This is the standard definition used by llama.cpp + imatrix = np.sum(A**2, axis=0) + + # Also compute per-dim RMS for reference + rms = np.sqrt(np.mean(A**2, axis=0)) + + print( + f" Imatrix stats: min={imatrix.min():.6f}, max={imatrix.max():.6f}, " + f"mean={imatrix.mean():.6f}, std={imatrix.std():.6f}" + ) + print( + f" RMS stats: min={rms.min():.6f}, max={rms.max():.6f}, " + f"mean={rms.mean():.6f}" + ) + + # Concentration metrics + total = imatrix.sum() + sorted_im = np.sort(imatrix)[::-1] + top1pct = max(1, int(len(imatrix) * 0.01)) + top10pct = max(1, int(len(imatrix) * 0.10)) + print(f" Power concentration:") + print( + f" Top 1% dims ({top1pct}): {sorted_im[:top1pct].sum() / total * 100:.1f}% of total" + ) + print( + f" Top 10% dims ({top10pct}): {sorted_im[:top10pct].sum() / total * 100:.1f}% of total" + ) + + save_imatrix(m["imatrix_name"], imatrix) + except Exception as e: + print(f" SKIP: {e}") + +print("\nDone.") diff --git a/scripts/extract-activations.py b/scripts/extract-activations.py new file mode 100644 index 000000000000..bc7d2faf1bea --- /dev/null +++ b/scripts/extract-activations.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Extract real activation tensors by running a forward pass through the model. + +Captures the INPUT activations to specific weight tensors (the vectors that get +multiplied by the weight matrix). These are what matter for quantization quality: +quantization error * activation magnitude = output error. + +Usage: + python3 scripts/extract-activations.py MODEL.gguf OUTPUT_DIR [--prompt TEXT] [--layer N] + +Output: + For each target tensor, writes a .f32bin file with header: + int64_t n_rows, int64_t row_len + followed by n_rows * row_len float32 values. + n_rows = number of tokens, row_len = hidden dimension. + +NOTE: This uses a simplified forward pass (no KV cache, single prompt). +Activations are extracted from after the norm layers (the actual matmul inputs). +""" +import sys +import os +import struct +import numpy as np + +script_dir = os.path.dirname(os.path.abspath(__file__)) +repo_root = os.path.dirname(script_dir) +sys.path.insert(0, os.path.join(repo_root, 'gguf-py')) + +from gguf import GGUFReader + + +def bf16_to_f32(raw_bytes): + """Convert raw BF16 bytes to float32 numpy array.""" + bf16 = np.frombuffer(raw_bytes, dtype=np.uint16) + f32_bits = bf16.astype(np.uint32) << 16 + return f32_bits.view(np.float32) + + +def rms_norm(x, weight, eps=1e-6): + """RMS normalization (Qwen3/Llama style).""" + rms = np.sqrt(np.mean(x * x, axis=-1, keepdims=True) + eps) + return (x / rms) * weight + + +def silu(x): + """SiLU activation.""" + return x / (1.0 + np.exp(-np.clip(x, -88, 88))) + + +def softmax(x, axis=-1): + """Numerically stable softmax.""" + x_max = np.max(x, axis=axis, keepdims=True) + e = np.exp(x - x_max) + return e / np.sum(e, axis=axis, keepdims=True) + + +def main(): + if len(sys.argv) < 3: + print(f"Usage: {sys.argv[0]} MODEL.gguf OUTPUT_DIR [--prompt TEXT] [--layer N]") + sys.exit(1) + + model_path = sys.argv[1] + output_dir = sys.argv[2] + prompt_text = "The quick brown fox jumps over the lazy dog. In a distant galaxy, scientists discovered" + target_layer = 16 + + for i in range(3, len(sys.argv)): + if sys.argv[i] == "--prompt" and i + 1 < len(sys.argv): + prompt_text = sys.argv[i + 1] + elif sys.argv[i] == "--layer" and i + 1 < len(sys.argv): + target_layer = int(sys.argv[i + 1]) + + os.makedirs(output_dir, exist_ok=True) + + print(f"Loading {model_path}...") + reader = GGUFReader(model_path) + + # Read model config from metadata + config = {} + for kv in reader.fields.values(): + if hasattr(kv, 'parts') and len(kv.parts) > 0: + name = kv.name + if 'block_count' in name: + config['n_layer'] = int(kv.parts[-1][0]) + elif 'embedding_length' in name: + config['hidden'] = int(kv.parts[-1][0]) + elif 'feed_forward_length' in name: + config['ffn'] = int(kv.parts[-1][0]) + elif 'head_count_kv' in name: + config['n_kv_heads'] = int(kv.parts[-1][0]) + elif 'head_count' in name and 'kv' not in name: + config['n_heads'] = int(kv.parts[-1][0]) + elif 'key_length' in name: + config['head_dim'] = int(kv.parts[-1][0]) + elif 'layer_norm_rms_epsilon' in name: + config['eps'] = float(kv.parts[-1][0]) + + print(f"Config: {config}") + hidden = config['hidden'] + + # Load tensors into a dict + def load_tensor(name): + for t in reader.tensors: + if t.name == name: + raw = bytes(t.data) + shape = [int(s) for s in t.shape] + n_el = int(t.n_elements) + if t.tensor_type.name == 'BF16': + flat = bf16_to_f32(raw) + elif t.tensor_type.name == 'F16': + flat = np.frombuffer(raw, dtype=np.float16).astype(np.float32) + elif t.tensor_type.name == 'F32': + flat = np.frombuffer(raw, dtype=np.float32) + else: + raise ValueError(f"Unsupported type: {t.tensor_type.name}") + assert flat.shape[0] == n_el, f"Expected {n_el} elements, got {flat.shape[0]}" + if len(shape) == 1: + return flat.copy() + return flat.reshape(list(reversed(shape))).copy() + raise KeyError(f"Tensor {name} not found") + + # Create simple token IDs from the prompt (use first few tokens from vocab) + # We just need realistic activations, not perfect tokenization + n_tokens = min(32, len(prompt_text.split())) + print(f"Using {n_tokens} pseudo-tokens for activation extraction") + + # Load token embedding and create input + print("Loading token_embd...") + token_embd = load_tensor("token_embd.weight") # [vocab, hidden] + # Use token IDs 100-131 (arbitrary but avoids special tokens) + token_ids = list(range(100, 100 + n_tokens)) + x = token_embd[token_ids] # [n_tokens, hidden] + print(f"Input shape: {x.shape}") + + # Run forward pass through target layer only (we just need the activations) + layer = target_layer + print(f"\nProcessing layer {layer}...") + + def save_activation(name, data): + """Save activation tensor as f32bin.""" + if data.ndim == 1: + data = data.reshape(1, -1) + n_rows, row_len = data.shape + fname = os.path.join(output_dir, name + ".f32bin") + with open(fname, 'wb') as fp: + fp.write(struct.pack('ths', Q_h, K_h) / np.sqrt(head_dim) + attn_w = softmax(scores, axis=-1) + attn_out = np.einsum('ths,shd->thd', attn_w, V_h).reshape(n_tokens, -1) + + # attn_output weight input + save_activation(f"act_blk{layer}_attn_output_input", attn_out) + + # Project and add residual + attn_proj = attn_out @ W_o.T + x = x + attn_proj + + # FFN norm → input to ffn_gate/ffn_up + ffn_norm_w = load_tensor(f"blk.{layer}.ffn_norm.weight") + x_ffn = rms_norm(x, ffn_norm_w, config.get('eps', 1e-6)) + save_activation(f"act_blk{layer}_ffn_input", x_ffn) + + # FFN: gate and up projections + W_gate = load_tensor(f"blk.{layer}.ffn_gate.weight") # [ffn, hidden] + W_up = load_tensor(f"blk.{layer}.ffn_up.weight") # [ffn, hidden] + W_down = load_tensor(f"blk.{layer}.ffn_down.weight") # [hidden, ffn] + + gate = x_ffn @ W_gate.T + up = x_ffn @ W_up.T + ffn_act = silu(gate) * up # SwiGLU activation + + # ffn_down weight input (the SwiGLU output) + save_activation(f"act_blk{layer}_ffn_down_input", ffn_act) + + print(f"\nDone! Extracted 4 activation tensors to {output_dir}/") + + +if __name__ == "__main__": + main() diff --git a/scripts/extract-tensor-data.py b/scripts/extract-tensor-data.py new file mode 100644 index 000000000000..473cfe108d4c --- /dev/null +++ b/scripts/extract-tensor-data.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Extract tensor data from GGUF as raw f32 binary files for C++ testing. + +Usage: + python3 scripts/extract-tensor-data.py MODEL.gguf pattern1 [pattern2 ...] + +Output: + For each matching tensor, writes a .f32bin file with header: + int64_t n_rows, int64_t row_len + followed by n_rows * row_len float32 values. +""" +import sys +import os +import numpy as np + +# Support running from build/ or repo root +script_dir = os.path.dirname(os.path.abspath(__file__)) +repo_root = os.path.dirname(script_dir) +sys.path.insert(0, os.path.join(repo_root, 'gguf-py')) + +from gguf import GGUFReader + +def main(): + if len(sys.argv) < 3: + print(f"Usage: {sys.argv[0]} MODEL.gguf pattern1 [pattern2 ...]") + print(f" Extracts tensors whose names contain any of the given patterns.") + sys.exit(1) + + model_path = sys.argv[1] + patterns = sys.argv[2:] + + print(f"Reading {model_path}...") + reader = GGUFReader(model_path) + + for tensor in reader.tensors: + if not any(p in tensor.name for p in patterns): + continue + + print(f"\nExtracting: {tensor.name}") + print(f" Shape: {list(tensor.shape)}, type: {tensor.tensor_type.name}") + + # Convert to f32 + raw = np.array(tensor.data, dtype=np.uint8) + + if tensor.tensor_type.name == 'BF16': + bf16_vals = raw.view(np.uint16) + f32_bits = bf16_vals.astype(np.uint32) << 16 + f32_vals = f32_bits.view(np.float32) + elif tensor.tensor_type.name == 'F16': + f16_vals = raw.view(np.float16) + f32_vals = f16_vals.astype(np.float32) + elif tensor.tensor_type.name == 'F32': + f32_vals = raw.view(np.float32) + else: + print(f" SKIP: unsupported type {tensor.tensor_type.name}") + continue + + # Determine layout: GGUF stores shape as [col, row] for 2D + row_len = int(tensor.shape[0]) + n_rows = tensor.n_elements // row_len + + fname = tensor.name.replace(".", "_") + ".f32bin" + with open(fname, 'wb') as fp: + fp.write(np.array([n_rows, row_len], dtype=np.int64).tobytes()) + f32_vals.tofile(fp) + + file_size = os.path.getsize(fname) + print(f" Wrote {fname}: {n_rows} rows x {row_len} cols = {tensor.n_elements} elements") + print(f" File size: {file_size / (1024*1024):.1f} MB") + print(f" Stats: mean={f32_vals.mean():.6f}, std={f32_vals.std():.6f}, " + f"min={f32_vals.min():.6f}, max={f32_vals.max():.6f}") + +if __name__ == "__main__": + main() diff --git a/src/llama-graph.h b/src/llama-graph.h index 4b5b75c632ab..3ecc67b387cc 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -645,6 +645,7 @@ class llm_graph_input_sampling : public llm_graph_input_i { std::map samplers; }; + // // llm_graph_result // diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 55554735dac4..16f4f31f32b5 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -2,6 +2,7 @@ #include "ggml-alloc.h" #include "ggml.h" +#include "llama.h" #include "gguf.h" #include "llama-hparams.h" @@ -61,6 +62,13 @@ const char * llama_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_IQ2_S: name = LLAMA_FTYPE_PREFIX "IQ2_S - 2.5 bpw"; break; case LLAMA_FTYPE_MOSTLY_IQ2_M: name = LLAMA_FTYPE_PREFIX "IQ2_M - 2.7 bpw"; break; case LLAMA_FTYPE_MOSTLY_IQ3_XS: name = LLAMA_FTYPE_PREFIX "IQ3_XS - 3.3 bpw"; break; + case LLAMA_FTYPE_MOSTLY_Q3_PT: name = LLAMA_FTYPE_PREFIX "Q3_PT - 3.25 bpw"; break; + case LLAMA_FTYPE_MOSTLY_Q3_KPT: name = LLAMA_FTYPE_PREFIX "Q3_KPT - Q3_K with learned levels"; break; + case LLAMA_FTYPE_MOSTLY_Q4_DPT: name = LLAMA_FTYPE_PREFIX "Q4_DPT - IQ4_NL with learned levels"; break; + case LLAMA_FTYPE_MOSTLY_Q2_KPT: name = LLAMA_FTYPE_PREFIX "Q2_KPT - Q2_K with learned levels"; break; + case LLAMA_FTYPE_MOSTLY_IQ2_TQ: name = LLAMA_FTYPE_PREFIX "IQ2_TQ - 2.0625 bpw trellis quantized"; break; + case LLAMA_FTYPE_MOSTLY_IQ3_TQ: name = LLAMA_FTYPE_PREFIX "IQ3_TQ - 3.5625 bpw per-tensor trained grid"; break; + case LLAMA_FTYPE_MOSTLY_IQ1_BN: name = LLAMA_FTYPE_PREFIX "IQ1_BN - 1.5625 bpw 8D vector quantized"; break; case LLAMA_FTYPE_MOSTLY_IQ3_XXS: name = LLAMA_FTYPE_PREFIX "IQ3_XXS - 3.0625 bpw"; break; case LLAMA_FTYPE_MOSTLY_IQ1_S: name = LLAMA_FTYPE_PREFIX "IQ1_S - 1.5625 bpw"; break; case LLAMA_FTYPE_MOSTLY_IQ1_M: name = LLAMA_FTYPE_PREFIX "IQ1_M - 1.75 bpw"; break; @@ -765,6 +773,13 @@ llama_model_loader::llama_model_loader( case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break; case GGML_TYPE_IQ4_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ4_XS; break; case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break; + case GGML_TYPE_Q3_PT: ftype = LLAMA_FTYPE_MOSTLY_Q3_PT; break; + case GGML_TYPE_Q3_KPT: ftype = LLAMA_FTYPE_MOSTLY_Q3_KPT; break; + case GGML_TYPE_Q4_DPT: ftype = LLAMA_FTYPE_MOSTLY_Q4_DPT; break; + case GGML_TYPE_Q2_KPT: ftype = LLAMA_FTYPE_MOSTLY_Q2_KPT; break; + case GGML_TYPE_IQ2_TQ: ftype = LLAMA_FTYPE_MOSTLY_IQ2_TQ; break; + case GGML_TYPE_IQ3_TQ: ftype = LLAMA_FTYPE_MOSTLY_IQ3_TQ; break; + case GGML_TYPE_IQ1_BN: ftype = LLAMA_FTYPE_MOSTLY_IQ1_BN; break; case GGML_TYPE_NVFP4: ftype = LLAMA_FTYPE_MOSTLY_NVFP4; break; case GGML_TYPE_Q1_0: ftype = LLAMA_FTYPE_MOSTLY_Q1_0; break; default: diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 8d68cff45b78..457aebf1d6fb 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1630,6 +1630,175 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } } + // Load per-tensor quantization auxiliary data (levels/kvalues) from GGUF metadata. + // Indexed by weight tensor pointer for direct lookup during inference. + { + // Build tensor name to tensor pointer map + std::unordered_map name_to_tensor; + for (auto & [ctx, buf_map] : ctx_buf_maps) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + name_to_tensor[ggml_get_name(t)] = t; + } + } + + struct level_type_info { + ggml_type type; + const char * gguf_key; + size_t n_levels; // number of level values per tensor + size_t elem_bytes; // size of each level value + }; + + const level_type_info level_types[] = { + { GGML_TYPE_Q3_PT, "q3_pt.levels", 8, sizeof(float) }, + { GGML_TYPE_Q3_KPT, "q3_kpt.levels", 8, sizeof(float) }, + { GGML_TYPE_Q4_DPT, "q4_dpt.levels", 16, sizeof(int8_t) }, + }; + + for (const auto & lt : level_types) { + int64_t lv_idx = gguf_find_key(ml.metadata, lt.gguf_key); + if (lv_idx < 0) { continue; } + + const uint8_t * lv_raw = (const uint8_t *)gguf_get_arr_data(ml.metadata, lv_idx); + const size_t lv_arr_n = gguf_get_arr_n(ml.metadata, lv_idx); + + size_t tensor_count = 0; + + // Iterate over GGUF slots to find matching tensors + for (size_t gguf_slot = 0; gguf_slot < lv_arr_n / lt.n_levels; ++gguf_slot) { + std::string tensor_name = gguf_get_tensor_name(ml.metadata, gguf_slot); + auto it = name_to_tensor.find(tensor_name); + if (it == name_to_tensor.end()) { continue; } + + ggml_tensor* t = it->second; + if (t->type != lt.type) { continue; } + + const size_t gguf_offset = gguf_slot * lt.n_levels; + + // Store directly indexed by tensor pointer + auto & aux = tensor_aux_data[t]; + aux.type = lt.type; + aux.host_data.assign( + lv_raw + gguf_offset * lt.elem_bytes, + lv_raw + (gguf_offset + lt.n_levels) * lt.elem_bytes + ); + aux.aux_tensor = nullptr; + + // Set quant_levels directly on the tensor + t->quant_levels = aux.host_data.data(); + + tensor_count++; + } + + if (tensor_count > 0) { + LLAMA_LOG_INFO("%s: loaded %zu %s per-tensor level tables\n", + __func__, tensor_count, lt.gguf_key); + } + } + + // Q2_KPT: per-block levels stored as per-tensor GGUF keys "{tensor_name}.q2kpt_levels" + // Each key holds n_blocks * Q2KPT_N_LEVELS floats for that tensor (4 floats per 256-element block). + { + size_t q2kpt_loaded = 0; + for (auto & [tname, t] : name_to_tensor) { + if (t->type != GGML_TYPE_Q2_KPT) { continue; } + const std::string key = tname + ".q2kpt_levels"; + int64_t lv_idx = gguf_find_key(ml.metadata, key.c_str()); + if (lv_idx < 0) { continue; } + + const uint8_t * lv_raw = (const uint8_t *)gguf_get_arr_data(ml.metadata, lv_idx); + const size_t lv_n = gguf_get_arr_n(ml.metadata, lv_idx); + + auto & aux = tensor_aux_data[t]; + aux.type = GGML_TYPE_Q2_KPT; + aux.host_data.assign(lv_raw, lv_raw + lv_n * sizeof(float)); + aux.aux_tensor = nullptr; + t->quant_levels = aux.host_data.data(); + q2kpt_loaded++; + } + if (q2kpt_loaded > 0) { + LLAMA_LOG_INFO("%s: loaded %zu Q2_KPT per-block level tables\n", __func__, q2kpt_loaded); + } + } + + // IQ2_TQ: per-tensor trained grid (16 × 4 int8 = 64 bytes) + { + size_t iq2tq_loaded = 0; + for (auto & [tname, t] : name_to_tensor) { + if (t->type != GGML_TYPE_IQ2_TQ) { continue; } + + const std::string grid_key = "iq2tq.grid." + tname; + int64_t grid_idx = gguf_find_key(ml.metadata, grid_key.c_str()); + if (grid_idx < 0) { continue; } + + auto & taux = tensor_aux_data[t]; + taux.type = GGML_TYPE_IQ2_TQ; + taux.host_data.resize(64); + const int8_t * grid_data = (const int8_t *)gguf_get_arr_data(ml.metadata, grid_idx); + memcpy(taux.host_data.data(), grid_data, 64); + + t->quant_levels = taux.host_data.data(); + iq2tq_loaded++; + } + if (iq2tq_loaded > 0) { + LLAMA_LOG_INFO("%s: loaded IQ2_TQ grid for %zu tensors\n", __func__, iq2tq_loaded); + } + } + + // IQ3_TQ: per-tensor trained grid (16 × 8 int8 = 128 bytes) + { + size_t iq3tq_loaded = 0; + for (auto & [tname, t] : name_to_tensor) { + if (t->type != GGML_TYPE_IQ3_TQ) { continue; } + + const std::string grid_key = "iq3tq.grid." + tname; + int64_t grid_idx = gguf_find_key(ml.metadata, grid_key.c_str()); + if (grid_idx < 0) { + // backward compat: try old key name + const std::string old_key = "iq3qt.grid." + tname; + grid_idx = gguf_find_key(ml.metadata, old_key.c_str()); + if (grid_idx < 0) { continue; } + } + + auto & taux = tensor_aux_data[t]; + taux.type = GGML_TYPE_IQ3_TQ; + taux.host_data.resize(128); + const int8_t * grid_data = (const int8_t *)gguf_get_arr_data(ml.metadata, grid_idx); + memcpy(taux.host_data.data(), grid_data, 128); + + t->quant_levels = taux.host_data.data(); + iq3tq_loaded++; + } + if (iq3tq_loaded > 0) { + LLAMA_LOG_INFO("%s: loaded IQ3_TQ grid for %zu tensors\n", __func__, iq3tq_loaded); + } + } + + // IQ1_BN: per-tensor trained codebook (32768 bytes) + { + size_t iq1bn_loaded = 0; + for (auto & [tname, t] : name_to_tensor) { + if (t->type != GGML_TYPE_IQ1_BN) { continue; } + + const std::string aux_key = "iq1bn.aux." + tname; + int64_t aux_idx = gguf_find_key(ml.metadata, aux_key.c_str()); + if (aux_idx < 0) { continue; } + + auto & taux = tensor_aux_data[t]; + taux.type = GGML_TYPE_IQ1_BN; + taux.host_data.resize(32768); + const int8_t * aux_data = (const int8_t *)gguf_get_arr_data(ml.metadata, aux_idx); + memcpy(taux.host_data.data(), aux_data, 32768); + + t->quant_levels = taux.host_data.data(); + iq1bn_loaded++; + } + if (iq1bn_loaded > 0) { + LLAMA_LOG_INFO("%s: loaded IQ1_BN codebook for %zu tensors\n", __func__, iq1bn_loaded); + } + } + + } + if (use_mmap_buffer) { for (auto & mapping : ml.mappings) { pimpl->mappings.emplace_back(std::move(mapping)); diff --git a/src/llama-model.h b/src/llama-model.h index 45b054cedf1d..d3d42e3ac763 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -623,6 +623,24 @@ struct llama_model { // for keeping track of associated LoRA adapters std::unordered_set loras; + // host-side auxiliary data for dynamic quantization types (Q4_DPT, Q3_PT, Q3_KPT) + // indexed by weight tensor pointer, allows separate GPU placement of aux data + struct tensor_auxiliary { + ggml_type type; // Quantization type this aux data is for + std::vector host_data; // Host copy of aux data (levels or kvalues) + struct ggml_tensor * aux_tensor; // Separate ggml tensor for backend placement + }; + + // Hash function for ggml_tensor pointers (reuse existing ggml_hash pattern) + struct ggml_tensor_ptr_hash { + size_t operator()(const ggml_tensor* t) const noexcept { + return (size_t)(uintptr_t)t >> 4; // Same as ggml_hash() + } + }; + + // Per-tensor auxiliary data lookup - indexed by WEIGHT tensor pointer + std::unordered_map tensor_aux_data; + // statically allocated context for assigning struct llama_meta_device_get_split_state_userdata get_split_state_ud; diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 847e79f46552..9614bd2f29ae 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -1,6 +1,8 @@ +#include "ggml.h" #include "llama-impl.h" #include "llama-model.h" #include "llama-model-loader.h" +#include "llama.h" #include "llama-ext.h" #include @@ -13,6 +15,98 @@ #include #include +// Q3_PT levels functions (defined in ggml-quants.c) +extern "C" { + void q3pt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float levels_out[8]); + void q3pt_set_levels(const float * levels); +} + +// Q3_KPT levels functions (defined in ggml-quants.c) +extern "C" { + void q3kpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float levels_out[8]); + void q3kpt_set_levels(const float * levels); +} + +// Q4_DPT levels functions (defined in ggml-quants.c) +extern "C" { + void q4dpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, int8_t levels_out[16]); + void q4dpt_set_levels(const int8_t * levels); +} + +// Q2_KPT levels are handled internally by quantize_q2_kpt +#define Q2KPT_N_LEVELS 4 +#define QK_K 256 +extern "C" const float * q2kpt_get_levels(void); +extern "C" void q2kpt_prepare_levels(int64_t nrows, int64_t n_per_row); +extern "C" void q2kpt_free_levels(void); + +// IQ2_TQ functions — per-tensor trained grid +extern "C" size_t quantize_iq2_tq(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +extern "C" void iq2tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t grid_out[64]); +extern "C" void iq2tq_set_grid(const int8_t grid[64]); +extern "C" const int8_t * iq2tq_get_grid(void); + +// IQ3_TQ functions — per-tensor trained grid (3-bit, 128 bytes per tensor) +extern "C" size_t quantize_iq3_tq(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +extern "C" void iq3tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t grid_out[128]); +extern "C" void iq3tq_set_grid(const int8_t grid[128]); +extern "C" const int8_t * iq3tq_get_grid(void); + +// IQ1_BN functions — 8D vector quantized with per-tensor trained 4096-entry codebook (32768 bytes per tensor) +extern "C" size_t quantize_iq1_bn(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +extern "C" void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t aux_out[32768], int nthread); +extern "C" void iq1bn_set_aux(const int8_t aux[32768]); +extern "C" const int8_t * iq1bn_get_aux(void); + +// Q3_PT levels functions (defined in ggml-quants.c) +extern "C" { + void q3pt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float levels_out[8]); + void q3pt_set_levels(const float * levels); +} + +// Q3_KPT levels functions (defined in ggml-quants.c) +extern "C" { + void q3kpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, float levels_out[8]); + void q3kpt_set_levels(const float * levels); +} + +// Q4_DPT levels functions (defined in ggml-quants.c) +extern "C" { + void q4dpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, + const float * imatrix, int8_t levels_out[16]); + void q4dpt_set_levels(const int8_t * levels); +} + +// Q2_KPT levels are handled internally by quantize_q2_kpt +#define Q2KPT_N_LEVELS 4 +#define QK_K 256 +extern "C" const float * q2kpt_get_levels(void); +extern "C" void q2kpt_prepare_levels(int64_t nrows, int64_t n_per_row); +extern "C" void q2kpt_free_levels(void); + +// IQ2_TQ functions — per-tensor trained grid +extern "C" size_t quantize_iq2_tq(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +extern "C" void iq2tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t grid_out[64]); +extern "C" void iq2tq_set_grid(const int8_t grid[64]); +extern "C" const int8_t * iq2tq_get_grid(void); + +// IQ3_TQ functions — per-tensor trained grid (3-bit, 128 bytes per tensor) +extern "C" size_t quantize_iq3_tq(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +extern "C" void iq3tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t grid_out[128]); +extern "C" void iq3tq_set_grid(const int8_t grid[128]); +extern "C" const int8_t * iq3tq_get_grid(void); + +// IQ1_BN functions — 8D vector quantized with per-tensor trained 4096-entry codebook (32768 bytes per tensor) +extern "C" size_t quantize_iq1_bn(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +extern "C" void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t aux_out[32768], int nthread); +extern "C" void iq1bn_set_aux(const int8_t aux[32768]); +extern "C" const int8_t * iq1bn_get_aux(void); + // result of parsing --tensor-type option // (changes to this struct must be reflected in tools/quantize/quantize.cpp) struct tensor_type_option { @@ -234,7 +328,7 @@ static void llama_tensor_dequantize_impl( } else if (tensor->type == GGML_TYPE_BF16) { ggml_bf16_to_fp32_row((ggml_bf16_t *)tensor->data, f32_output, nelements); } else if (ggml_is_quantized(tensor->type)) { - qtype->to_float(tensor->data, f32_output, nelements); + qtype->to_float(tensor->data, f32_output, nelements, tensor->quant_levels); } else { GGML_ABORT("fatal error"); // unreachable } @@ -264,13 +358,14 @@ static void llama_tensor_dequantize_impl( size_t thr_elems = thr_blocks * block_size; // number of elements for this thread size_t thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread - auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) { + const void * quant_levels = tensor->quant_levels; + auto compute = [qtype, quant_levels] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) { if (typ == GGML_TYPE_F16) { ggml_fp16_to_fp32_row((ggml_fp16_t *)inbuf, outbuf, nels); } else if (typ == GGML_TYPE_BF16) { ggml_bf16_to_fp32_row((ggml_bf16_t *)inbuf, outbuf, nels); } else { - qtype->to_float(inbuf, outbuf, nels); + qtype->to_float(inbuf, outbuf, nels, quant_levels); } }; workers.emplace_back(compute, tensor->type, (uint8_t *) tensor->data + in_buff_offs, f32_output + out_buff_offs, thr_elems); @@ -480,6 +575,18 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { new_type = GGML_TYPE_IQ3_S; } + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_PT) { + new_type = GGML_TYPE_IQ4_XS; + } + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_KPT) { + new_type = GGML_TYPE_Q4_K; + } + else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_DPT) { + new_type = GGML_TYPE_IQ4_XS; + } + else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_KPT) { + new_type = GGML_TYPE_Q4_K; + } else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0) { new_type = GGML_TYPE_Q4_K; } @@ -518,13 +625,16 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : !qs.has_imatrix ? GGML_TYPE_IQ3_S : GGML_TYPE_IQ3_XXS; } + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_PT) { + new_type = GGML_TYPE_Q3_PT; + } else if ((ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S) && qs.model.hparams.n_gqa() >= 4) { new_type = GGML_TYPE_Q4_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) { new_type = GGML_TYPE_Q4_K; } - else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) { + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_KPT) { new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; @@ -569,16 +679,17 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type auto info = layer_info(qs.i_ffn_down, qs.n_ffn_down, name.c_str()); int i_layer = info.first, n_layer = info.second; if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_KPT) new_type = GGML_TYPE_Q3_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S) { if (i_layer < n_layer/8) new_type = GGML_TYPE_Q4_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS && !qs.has_imatrix) { new_type = i_layer < n_layer/8 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K; } - else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) { + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_KPT) { new_type = i_layer < n_layer/16 ? GGML_TYPE_Q5_K : arch != LLM_ARCH_FALCON || use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q4_K - : GGML_TYPE_Q3_K; + : (ftype == LLAMA_FTYPE_MOSTLY_Q3_KPT ? GGML_TYPE_Q3_KPT : GGML_TYPE_Q3_K); } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M && (i_layer < n_layer/8 || (qs.model.hparams.n_expert == 8 && use_more_bits(i_layer, n_layer)))) { @@ -587,6 +698,9 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) { new_type = arch == LLM_ARCH_FALCON ? GGML_TYPE_Q4_K : GGML_TYPE_Q5_K; } + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_PT) { + new_type = GGML_TYPE_IQ4_XS; + } else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) { if (arch == LLM_ARCH_FALCON) { new_type = i_layer < n_layer/16 ? GGML_TYPE_Q6_K : @@ -616,13 +730,14 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S || - ftype == LLAMA_FTYPE_MOSTLY_IQ3_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) { + ftype == LLAMA_FTYPE_MOSTLY_IQ3_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS || ftype == LLAMA_FTYPE_MOSTLY_Q3_KPT || + ftype == LLAMA_FTYPE_MOSTLY_Q2_KPT) { new_type = GGML_TYPE_Q5_K; } } else { - if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K ) new_type = GGML_TYPE_Q3_K; + if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_Q2_KPT) new_type = GGML_TYPE_Q3_K; else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) new_type = GGML_TYPE_IQ3_S; - else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M ) new_type = GGML_TYPE_Q4_K; + else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_KPT) new_type = GGML_TYPE_Q4_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L ) new_type = GGML_TYPE_Q5_K; else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M ) new_type = GGML_TYPE_Q4_K; } @@ -828,6 +943,14 @@ ggml_type llama_ftype_get_default_type(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_IQ4_XS: return GGML_TYPE_IQ4_XS; case LLAMA_FTYPE_MOSTLY_IQ3_S: case LLAMA_FTYPE_MOSTLY_IQ3_M: return GGML_TYPE_IQ3_S; + case LLAMA_FTYPE_MOSTLY_Q3_PT: return GGML_TYPE_Q3_PT; + case LLAMA_FTYPE_MOSTLY_Q3_KPT: return GGML_TYPE_Q3_KPT; + case LLAMA_FTYPE_MOSTLY_Q4_DPT: return GGML_TYPE_Q4_DPT; + case LLAMA_FTYPE_MOSTLY_Q2_KPT: return GGML_TYPE_Q2_KPT; + case LLAMA_FTYPE_MOSTLY_IQ2_TQ: return GGML_TYPE_IQ2_TQ; + case LLAMA_FTYPE_MOSTLY_IQ3_TQ: return GGML_TYPE_IQ3_TQ; + case LLAMA_FTYPE_MOSTLY_IQ1_BN: return GGML_TYPE_IQ1_BN; + default: return GGML_TYPE_COUNT; } @@ -1103,6 +1226,615 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: ::zeros(fout, meta_size); }; + // Q3_PT two-pass approach: train all per-tensor levels BEFORE opening the output + // file, so the levels KV entry is already populated at the time of the metadata placeholder. + static const size_t Q3PT_N_LEVELS = 8; + std::vector q3pt_all_levels; // indexed by position in tensors[] + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_PT && !params->dry_run) { + LLAMA_LOG_INFO("%s: Q3_PT pass 1: training per-tensor levels...\n", __func__); + q3pt_all_levels.assign(tensors.size() * Q3PT_N_LEVELS, 0.0f); + + // Temporary dequant buffer for pass 1 (reuse f32_conv_buf / read_data declared below) + std::vector> p1_read_data; + std::vector> p1_f32_buf; + std::vector p1_workers; + p1_workers.reserve(nthread); + + for (size_t ti = 0; ti < tensors.size(); ++ti) { + ggml_tensor * tensor = tensors[ti]->tensor; + const std::string tname = ggml_get_name(tensor); + + // Determine whether this tensor will be Q3_PT (mirror the pass-2 logic) + bool quantize = tname.rfind("weight") == tname.size() - 6; + quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tname.find("_norm.weight") == std::string::npos; + quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; + if (!quantize) { continue; } + + ggml_type new_type = default_type; + if (!params->pure) { + new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); + } + if (new_type != GGML_TYPE_Q3_PT) { continue; } + + // Load tensor data + const size_t tsz = ggml_nbytes(tensor); + if (!ml.use_mmap) { + if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } + tensor->data = p1_read_data.data(); + } + ml.load_data_for(tensor); + + // Dequantize to f32 if needed + const int64_t nelements = ggml_nelements(tensor); + float * f32_data; + if (tensor->type == GGML_TYPE_F32) { + f32_data = (float *) tensor->data; + } else { + llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); + f32_data = (float *) p1_f32_buf.data(); + } + + // Resolve imatrix + const float * imatrix = nullptr; + if (imatrix_data) { + auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); + if (it2 != imatrix_data->end() && + it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { + imatrix = it2->second.data(); + } + } + + const int64_t n_per_row = tensor->ne[0]; + const int64_t nrows = tensor->ne[1]; + + LLAMA_LOG_INFO("%s: Q3_PT levels for [%zu/%zu] %s\n", __func__, ti+1, tensors.size(), tensor->name); + q3pt_train_levels(f32_data, nrows, n_per_row, imatrix, + q3pt_all_levels.data() + ti * Q3PT_N_LEVELS); + } + + // All levels ready — store in GGUF metadata before the file is opened + for (auto & ctx : ctx_outs) { + if (ctx) { + gguf_set_arr_data(ctx.get(), "q3_pt.levels", GGUF_TYPE_FLOAT32, + q3pt_all_levels.data(), q3pt_all_levels.size()); + } + } + LLAMA_LOG_INFO("%s: Q3_PT pass 1 complete.\n", __func__); + } + + // Q3_KPT two-pass approach: train all per-tensor levels BEFORE opening the output + static const size_t Q3KPT_N_LEVELS = 8; + std::vector q3kpt_all_levels; // indexed by position in tensors[] + if (ftype == LLAMA_FTYPE_MOSTLY_Q3_KPT && !params->dry_run) { + LLAMA_LOG_INFO("%s: Q3_KPT pass 1: training per-tensor levels...\n", __func__); + q3kpt_all_levels.assign(tensors.size() * Q3KPT_N_LEVELS, 0.0f); + + // Temporary dequant buffer for pass 1 + std::vector> p1_read_data; + std::vector> p1_f32_buf; + std::vector p1_workers; + p1_workers.reserve(nthread); + + for (size_t ti = 0; ti < tensors.size(); ++ti) { + ggml_tensor * tensor = tensors[ti]->tensor; + const std::string tname = ggml_get_name(tensor); + + // Determine whether this tensor will be Q3_KPT (mirror the pass-2 logic) + bool quantize = tname.rfind("weight") == tname.size() - 6; + quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tname.find("_norm.weight") == std::string::npos; + quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; + if (!quantize) { continue; } + + ggml_type new_type = default_type; + if (!params->pure) { + new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); + } + if (params->token_embedding_type < GGML_TYPE_COUNT && + (tname == "token_embd.weight" || tname == "per_layer_token_embd.weight")) { + new_type = params->token_embedding_type; + } + if (params->output_tensor_type < GGML_TYPE_COUNT && tname == "output.weight") { + new_type = params->output_tensor_type; + } + if (new_type != GGML_TYPE_Q3_KPT) { continue; } + + // Load tensor data + const size_t tsz = ggml_nbytes(tensor); + if (!ml.use_mmap) { + if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } + tensor->data = p1_read_data.data(); + } + ml.load_data_for(tensor); + + // Dequantize to f32 if needed + const int64_t nelements = ggml_nelements(tensor); + float * f32_data; + if (tensor->type == GGML_TYPE_F32) { + f32_data = (float *) tensor->data; + } else { + llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); + f32_data = (float *) p1_f32_buf.data(); + } + + // Resolve imatrix + const float * imatrix = nullptr; + if (imatrix_data) { + auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); + if (it2 != imatrix_data->end() && + it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { + imatrix = it2->second.data(); + } + } + + const int64_t n_per_row = tensor->ne[0]; + const int64_t nrows = tensor->ne[1]; + + LLAMA_LOG_INFO("%s: Q3_KPT levels for [%zu/%zu] %s\n", __func__, ti+1, tensors.size(), tensor->name); + q3kpt_train_levels(f32_data, nrows, n_per_row, imatrix, + q3kpt_all_levels.data() + ti * Q3KPT_N_LEVELS); + } + + // All levels ready — store in GGUF metadata before the file is opened + for (auto & ctx : ctx_outs) { + if (ctx) { + gguf_set_arr_data(ctx.get(), "q3_kpt.levels", GGUF_TYPE_FLOAT32, + q3kpt_all_levels.data(), q3kpt_all_levels.size()); + } + } + LLAMA_LOG_INFO("%s: Q3_KPT pass 1 complete.\n", __func__); + } + + // Q4_DPT two-pass approach: train all per-tensor int8 levels BEFORE opening the output + // file, so the levels KV entry is already populated at the time of the metadata placeholder. + static const size_t Q4DPT_N_LEVELS = 16; + std::vector q4dpt_all_levels; // indexed by position in tensors[] + if (ftype == LLAMA_FTYPE_MOSTLY_Q4_DPT && !params->dry_run) { + LLAMA_LOG_INFO("%s: Q4_DPT pass 1: training per-tensor int8 levels...\n", __func__); + q4dpt_all_levels.assign(tensors.size() * Q4DPT_N_LEVELS, (int8_t)0); + + std::vector> p1_read_data; + std::vector> p1_f32_buf; + std::vector p1_workers; + p1_workers.reserve(nthread); + + for (size_t ti = 0; ti < tensors.size(); ++ti) { + ggml_tensor * tensor = tensors[ti]->tensor; + const std::string tname = ggml_get_name(tensor); + + bool quantize = tname.rfind("weight") == tname.size() - 6; + quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tname.find("_norm.weight") == std::string::npos; + quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; + if (!quantize) { continue; } + + ggml_type new_type = default_type; + if (!params->pure) { + new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); + } + if (params->token_embedding_type < GGML_TYPE_COUNT && + (tname == "token_embd.weight" || tname == "per_layer_token_embd.weight")) { + new_type = params->token_embedding_type; + } + if (params->output_tensor_type < GGML_TYPE_COUNT && tname == "output.weight") { + new_type = params->output_tensor_type; + } + if (new_type != GGML_TYPE_Q4_DPT) { continue; } + + // Load tensor data + const size_t tsz = ggml_nbytes(tensor); + if (!ml.use_mmap) { + if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } + tensor->data = p1_read_data.data(); + } + ml.load_data_for(tensor); + + // Dequantize to f32 if needed + const int64_t nelements = ggml_nelements(tensor); + float * f32_data; + if (tensor->type == GGML_TYPE_F32) { + f32_data = (float *) tensor->data; + } else { + llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); + f32_data = (float *) p1_f32_buf.data(); + } + + // Resolve imatrix + const float * imatrix = nullptr; + if (imatrix_data) { + auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); + if (it2 != imatrix_data->end() && + it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { + imatrix = it2->second.data(); + } + } + + const int64_t n_per_row = tensor->ne[0]; + const int64_t nrows = tensor->ne[1]; + + LLAMA_LOG_INFO("%s: Q4_DPT levels for [%zu/%zu] %s\n", __func__, ti+1, tensors.size(), tensor->name); + q4dpt_train_levels(f32_data, nrows, n_per_row, imatrix, + q4dpt_all_levels.data() + ti * Q4DPT_N_LEVELS); + } + + // Store in GGUF metadata before the file is opened + for (auto & ctx : ctx_outs) { + if (ctx) { + gguf_set_arr_data(ctx.get(), "q4_dpt.levels", GGUF_TYPE_INT8, + q4dpt_all_levels.data(), q4dpt_all_levels.size()); + } + } + LLAMA_LOG_INFO("%s: Q4_DPT pass 1 complete.\n", __func__); + } + + // Q2_KPT two-pass approach: train all per-block levels BEFORE opening the output + // file, so the levels KV entry is already populated at the time of the metadata placeholder. + // Per-block levels: 4 floats per 256-element block. + struct q2kpt_tensor_levels { + std::string name; + std::vector levels; // nrows * (n_per_row / QK_K) * Q2KPT_N_LEVELS floats + }; + std::vector q2kpt_all_levels; + if (ftype == LLAMA_FTYPE_MOSTLY_Q2_KPT && !params->dry_run) { + LLAMA_LOG_INFO("%s: Q2_KPT pass 1: training per-block levels...\n", __func__); + + std::vector> p1_read_data; + std::vector> p1_f32_buf; + std::vector p1_workers; + p1_workers.reserve(nthread); + + for (size_t ti = 0; ti < tensors.size(); ++ti) { + ggml_tensor * tensor = tensors[ti]->tensor; + const std::string tname = ggml_get_name(tensor); + + // Determine whether this tensor will be Q2_KPT (mirror the pass-2 logic) + bool quantize = tname.rfind("weight") == tname.size() - 6; + quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tname.find("_norm.weight") == std::string::npos; + quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; + if (!quantize) { continue; } + + ggml_type new_type = default_type; + if (!params->pure) { + new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); + } + if (params->token_embedding_type < GGML_TYPE_COUNT && + (tname == "token_embd.weight" || tname == "per_layer_token_embd.weight")) { + new_type = params->token_embedding_type; + } + if (params->output_tensor_type < GGML_TYPE_COUNT && tname == "output.weight") { + new_type = params->output_tensor_type; + } + if (new_type != GGML_TYPE_Q2_KPT) { continue; } + + // Load tensor data + const size_t tsz = ggml_nbytes(tensor); + if (!ml.use_mmap) { + if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } + tensor->data = p1_read_data.data(); + } + ml.load_data_for(tensor); + + // Dequantize to f32 if needed + const int64_t nelements = ggml_nelements(tensor); + float * f32_data; + if (tensor->type == GGML_TYPE_F32) { + f32_data = (float *) tensor->data; + } else { + llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); + f32_data = (float *) p1_f32_buf.data(); + } + + // Resolve imatrix + const float * imatrix = nullptr; + if (imatrix_data) { + auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); + if (it2 != imatrix_data->end() && + it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { + imatrix = it2->second.data(); + } + } + + const int64_t n_per_row = tensor->ne[0]; + const int64_t nrows = tensor->ne[1]; + + // Allocate levels buffer for this tensor + const int nb = n_per_row / QK_K; + const size_t n_levels = (size_t)nrows * tensor->ne[2] * nb * Q2KPT_N_LEVELS; + q2kpt_all_levels.push_back({tname, std::vector(n_levels)}); + + LLAMA_LOG_INFO("%s: Q2_KPT levels for [%zu/%zu] %s (%zu floats)\n", + __func__, ti+1, tensors.size(), tensor->name, n_levels); + + // Train levels by running quantization internally + // We need to quantize to f32 -> Q2_KPT -> f32 to get the trained levels + std::vector> p1_qbuf(ggml_nbytes(tensor)); + const size_t row_size = ggml_row_size(GGML_TYPE_Q2_KPT, n_per_row); + + // Prepare levels buffer for this tensor + q2kpt_free_levels(); + q2kpt_prepare_levels(nrows * tensor->ne[2], n_per_row); + + // Quantize each expert slice + const int64_t nelements_matrix = tensor->ne[0] * tensor->ne[1]; + for (int64_t i03 = 0; i03 < tensor->ne[2]; ++i03) { + const float * f32_data_03 = f32_data + i03 * nelements_matrix; + void * q_data_03 = (char *)p1_qbuf.data() + row_size * i03 * nrows; + const float * imatrix_03 = imatrix ? imatrix + i03 * n_per_row : nullptr; + + // start_row must be the absolute row index for correct levels indexing + ggml_quantize_chunk(GGML_TYPE_Q2_KPT, f32_data_03, q_data_03, i03 * nrows, nrows, n_per_row, imatrix_03); + } + + // Copy trained levels to our storage + const float * trained_levels = q2kpt_get_levels(); + if (trained_levels) { + memcpy(q2kpt_all_levels.back().levels.data(), trained_levels, n_levels * sizeof(float)); + } + } + + // Store all levels in GGUF metadata before the file is opened + for (const auto & tl : q2kpt_all_levels) { + for (auto & ctx : ctx_outs) { + if (ctx) { + const std::string key = tl.name + ".q2kpt_levels"; + gguf_set_arr_data(ctx.get(), key.c_str(), GGUF_TYPE_FLOAT32, + tl.levels.data(), tl.levels.size()); + } + } + } + LLAMA_LOG_INFO("%s: Q2_KPT pass 1 complete.\n", __func__); + } + + // IQ2_TQ: train per-tensor grid in pass 1 + struct iq2tq_meta { + std::string tensor_name; + int8_t grid[64]; + }; + std::vector iq2tq_all_meta; + if (params->ftype == LLAMA_FTYPE_MOSTLY_IQ2_TQ) { + const int64_t t_start_p1 = ggml_time_us(); + LLAMA_LOG_INFO("%s: IQ2_TQ pass 1: training per-tensor grids...\n", __func__); + + std::vector> p1_read_data; + std::vector> p1_f32_buf; + std::vector p1_workers; + p1_workers.reserve(nthread); + + for (size_t ti = 0; ti < tensors.size(); ++ti) { + ggml_tensor * tensor = tensors[ti]->tensor; + const std::string tname = ggml_get_name(tensor); + + // Mirror pass-2 logic: only quantize 2D+ weight tensors + bool quantize = tname.rfind("weight") == tname.size() - 6; + quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tname.find("_norm.weight") == std::string::npos; + quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; + if (!quantize) { continue; } + + ggml_type new_type = default_type; + if (!params->pure) { + new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); + } + if (params->token_embedding_type < GGML_TYPE_COUNT && + (tname == "token_embd.weight" || tname == "per_layer_token_embd.weight")) { + new_type = params->token_embedding_type; + } + if (params->output_tensor_type < GGML_TYPE_COUNT && tname == "output.weight") { + new_type = params->output_tensor_type; + } + if (new_type != GGML_TYPE_IQ2_TQ) { continue; } + + // Load tensor data + const size_t tsz = ggml_nbytes(tensor); + if (!ml.use_mmap) { + if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } + tensor->data = p1_read_data.data(); + } + ml.load_data_for(tensor); + + // Dequantize to f32 if needed + const int64_t nelements = ggml_nelements(tensor); + float * f32_data; + if (tensor->type == GGML_TYPE_F32) { + f32_data = (float *) tensor->data; + } else { + llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); + f32_data = (float *) p1_f32_buf.data(); + } + + // Resolve imatrix + const float * imatrix = nullptr; + if (imatrix_data) { + auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); + if (it2 != imatrix_data->end() && + it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { + imatrix = it2->second.data(); + } + } + + const int64_t n_per_row = tensor->ne[0]; + const int64_t nrows = tensor->ne[1]; + + LLAMA_LOG_INFO("%s: IQ2_TQ grid for [%zu/%zu] %s\n", __func__, ti+1, tensors.size(), tensor->name); + + iq2tq_meta meta; + meta.tensor_name = tname; + iq2tq_train_grid(f32_data, nrows, n_per_row, imatrix, meta.grid); + iq2tq_all_meta.push_back(meta); + + // Save to GGUF + std::string grid_key = "iq2tq.grid." + tname; + gguf_set_arr_data(ctx_outs[0].get(), grid_key.c_str(), GGUF_TYPE_INT8, meta.grid, 64); + } + const int64_t t_end_p1 = ggml_time_us(); + LLAMA_LOG_INFO("%s: IQ2_TQ pass 1 complete (%zu tensors trained, %.1f s).\n", + __func__, iq2tq_all_meta.size(), (t_end_p1 - t_start_p1) / 1e6); + } + + // IQ3_TQ: train per-tensor grid in pass 1 (16 entries × 8 levels = 128 bytes) + struct iq3tq_meta { + std::string tensor_name; + int8_t grid[128]; + }; + std::vector iq3tq_all_meta; + if (params->ftype == LLAMA_FTYPE_MOSTLY_IQ3_TQ) { + const int64_t t_start_p1 = ggml_time_us(); + LLAMA_LOG_INFO("%s: IQ3_TQ pass 1: training per-tensor grids...\n", __func__); + + std::vector> p1_read_data; + std::vector> p1_f32_buf; + std::vector p1_workers; + p1_workers.reserve(nthread); + + for (size_t ti = 0; ti < tensors.size(); ++ti) { + ggml_tensor * tensor = tensors[ti]->tensor; + const std::string tname = ggml_get_name(tensor); + + bool quantize = tname.rfind("weight") == tname.size() - 6; + quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tname.find("_norm.weight") == std::string::npos; + quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; + if (!quantize) { continue; } + + ggml_type new_type = default_type; + if (!params->pure) { + new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); + } + if (params->token_embedding_type < GGML_TYPE_COUNT && + (tname == "token_embd.weight" || tname == "per_layer_token_embd.weight")) { + new_type = params->token_embedding_type; + } + if (params->output_tensor_type < GGML_TYPE_COUNT && tname == "output.weight") { + new_type = params->output_tensor_type; + } + if (new_type != GGML_TYPE_IQ3_TQ) { continue; } + + const size_t tsz = ggml_nbytes(tensor); + if (!ml.use_mmap) { + if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } + tensor->data = p1_read_data.data(); + } + ml.load_data_for(tensor); + + const int64_t nelements = ggml_nelements(tensor); + float * f32_data; + if (tensor->type == GGML_TYPE_F32) { + f32_data = (float *) tensor->data; + } else { + llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); + f32_data = (float *) p1_f32_buf.data(); + } + + const float * imatrix = nullptr; + if (imatrix_data) { + auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); + if (it2 != imatrix_data->end() && + it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { + imatrix = it2->second.data(); + } + } + + const int64_t n_per_row = tensor->ne[0]; + const int64_t nrows = tensor->ne[1]; + + LLAMA_LOG_INFO("%s: IQ3_TQ grid for [%zu/%zu] %s\n", __func__, ti+1, tensors.size(), tensor->name); + + iq3tq_meta meta; + meta.tensor_name = tname; + iq3tq_train_grid(f32_data, nrows, n_per_row, imatrix, meta.grid); + iq3tq_all_meta.push_back(meta); + + std::string grid_key = "iq3tq.grid." + tname; + gguf_set_arr_data(ctx_outs[0].get(), grid_key.c_str(), GGUF_TYPE_INT8, meta.grid, 128); + } + const int64_t t_end_p1 = ggml_time_us(); + LLAMA_LOG_INFO("%s: IQ3_TQ pass 1 complete (%zu tensors trained, %.1f s).\n", + __func__, iq3tq_all_meta.size(), (t_end_p1 - t_start_p1) / 1e6); + } + + // IQ1_BN: train per-tensor codebook in pass 1 (4096 × 8D centroids = 32768 bytes) + struct iq1bn_meta { + std::string tensor_name; + int8_t aux[32768]; + }; + std::vector iq1bn_all_meta; + if (params->ftype == LLAMA_FTYPE_MOSTLY_IQ1_BN) { + const int64_t t_start_p1 = ggml_time_us(); + LLAMA_LOG_INFO("%s: IQ1_BN pass 1: training per-tensor codebooks...\n", __func__); + + std::vector> p1_read_data; + std::vector> p1_f32_buf; + std::vector p1_workers; + p1_workers.reserve(nthread); + + for (size_t ti = 0; ti < tensors.size(); ++ti) { + ggml_tensor * tensor = tensors[ti]->tensor; + const std::string tname = ggml_get_name(tensor); + + bool quantize = tname.rfind("weight") == tname.size() - 6; + quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tname.find("_norm.weight") == std::string::npos; + quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; + if (!quantize) { continue; } + + ggml_type new_type = default_type; + if (!params->pure) { + new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); + } + if (params->token_embedding_type < GGML_TYPE_COUNT && + (tname == "token_embd.weight" || tname == "per_layer_token_embd.weight")) { + new_type = params->token_embedding_type; + } + if (params->output_tensor_type < GGML_TYPE_COUNT && tname == "output.weight") { + new_type = params->output_tensor_type; + } + if (new_type != GGML_TYPE_IQ1_BN) { continue; } + + const size_t tsz = ggml_nbytes(tensor); + if (!ml.use_mmap) { + if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } + tensor->data = p1_read_data.data(); + } + ml.load_data_for(tensor); + + const int64_t nelements = ggml_nelements(tensor); + float * f32_data; + if (tensor->type == GGML_TYPE_F32) { + f32_data = (float *) tensor->data; + } else { + llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); + f32_data = (float *) p1_f32_buf.data(); + } + + const float * imatrix = nullptr; + if (imatrix_data) { + auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); + if (it2 != imatrix_data->end() && + it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { + imatrix = it2->second.data(); + } + } + + const int64_t n_per_row = tensor->ne[0]; + const int64_t nrows = tensor->ne[1]; + + LLAMA_LOG_INFO("%s: IQ1_BN codebook for [%zu/%zu] %s\n", __func__, ti+1, tensors.size(), tensor->name); + + iq1bn_meta meta; + meta.tensor_name = tname; + iq1bn_train_codebook(f32_data, nrows, n_per_row, imatrix, meta.aux, nthread); + iq1bn_all_meta.push_back(meta); + + std::string aux_key = "iq1bn.aux." + tname; + gguf_set_arr_data(ctx_outs[0].get(), aux_key.c_str(), GGUF_TYPE_INT8, meta.aux, 32768); + } + const int64_t t_end_p1 = ggml_time_us(); + LLAMA_LOG_INFO("%s: IQ1_BN pass 1 complete (%zu tensors trained, %.1f s).\n", + __func__, iq1bn_all_meta.size(), (t_end_p1 - t_start_p1) / 1e6); + } + // no output file for --dry-run if (!params->dry_run) { new_ofstream(0); @@ -1111,6 +1843,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: // // main loop: iterate over all weights // + size_t tensor_pass2_idx = 0; // index into tensors[], used for Q3_PT levels lookup for (size_t i = 0; i < tensors.size(); ++i) { const auto & weight = *tensors[i]; @@ -1237,6 +1970,75 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: const int64_t nchunk = (nelements_matrix + chunk_size - 1)/chunk_size; const int64_t nthread_use = nthread > 1 ? std::max((int64_t)1, std::min((int64_t)nthread, nchunk)) : 1; + // Q3_PT: set the per-tensor levels (trained in pass 1) as global for quantization + if (new_type == GGML_TYPE_Q3_PT) { + q3pt_set_levels(q3pt_all_levels.data() + tensor_pass2_idx * Q3PT_N_LEVELS); + } + + // Q3_KPT: set the per-tensor levels (trained in pass 1) as global for quantization + if (new_type == GGML_TYPE_Q3_KPT) { + q3kpt_set_levels(q3kpt_all_levels.data() + tensor_pass2_idx * Q3KPT_N_LEVELS); + } + + // Q4_DPT: set the per-tensor levels (trained in pass 1) as global for quantization + if (new_type == GGML_TYPE_Q4_DPT) { + q4dpt_set_levels(q4dpt_all_levels.data() + tensor_pass2_idx * Q4DPT_N_LEVELS); + } + + // IQ2_TQ: set per-tensor trained grid + if (new_type == GGML_TYPE_IQ2_TQ) { + bool found = false; + for (const auto & meta : iq2tq_all_meta) { + if (meta.tensor_name == tm.name) { + iq2tq_set_grid(meta.grid); + found = true; + break; + } + } + if (!found) { + LLAMA_LOG_WARN("%s: WARNING: no trained grid for IQ2_TQ tensor %s\n", __func__, tm.name.c_str()); + } + } + + // IQ3_TQ: set per-tensor trained grid + if (new_type == GGML_TYPE_IQ3_TQ) { + bool found = false; + for (const auto & meta : iq3tq_all_meta) { + if (meta.tensor_name == tm.name) { + iq3tq_set_grid(meta.grid); + found = true; + break; + } + } + if (!found) { + LLAMA_LOG_WARN("%s: WARNING: no trained grid for IQ3_TQ tensor %s\n", __func__, tm.name.c_str()); + } + } + + // IQ1_BN: set per-tensor trained codebook + if (new_type == GGML_TYPE_IQ1_BN) { + bool found = false; + for (const auto & meta : iq1bn_all_meta) { + if (meta.tensor_name == tm.name) { + iq1bn_set_aux(meta.aux); + found = true; + break; + } + } + if (!found) { + LLAMA_LOG_WARN("%s: WARNING: no trained codebook for IQ1_BN tensor %s\n", __func__, tm.name.c_str()); + } + } + + // Q2_KPT: quantize_q2_kpt trains per-block levels internally. + // Levels were already trained and saved to GGUF in pass 1. + // We still need to allocate the levels buffer for quantization to work correctly. + if (new_type == GGML_TYPE_Q2_KPT) { + const int64_t total_rows = nrows * tensor->ne[2]; + q2kpt_free_levels(); // Clear any stale levels from previous tensor + q2kpt_prepare_levels(total_rows, n_per_row); // Allocate for this tensor + } + // quantize each expert separately since they have different importance matrices new_size = 0; for (int64_t i03 = 0; i03 < tensor->ne[2]; ++i03) { @@ -1260,7 +2062,9 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: fout.write((const char *) new_data, new_size); zeros(fout, GGML_PAD(new_size, align) - new_size); } // no --dry-run - } // main loop + + tensor_pass2_idx++; + } // iterate over tensors if (!params->dry_run) { close_ofstream(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d4ea3b96225c..f8bf030a4817 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -266,6 +266,9 @@ if (NOT GGML_BACKEND_DL) llama_build_and_test(test-col2im-1d.cpp) endif() +# Quantization laboratory - tests for 2.5 BPW proposals +llama_build_and_test(test-quant-laboratory.cpp) + # libmtmd set(LLAMA_TEST_NAME test-mtmd-c-api) llama_build_and_test(test-mtmd-c-api.c) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c21675ef5414..f4d6e8040472 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -254,7 +254,7 @@ static std::vector tensor_to_float(const ggml_tensor * t) { } else if (t->type == GGML_TYPE_I8) { tv.push_back((float)*(int8_t *) &buf[i]); } else if (quantized) { - tt->to_float(&buf[i], vq.data(), bs); + tt->to_float(&buf[i], vq.data(), bs, nullptr); tv.insert(tv.end(), vq.begin(), vq.end()); } else { GGML_ABORT("fatal error"); diff --git a/tests/test-quant-laboratory.cpp b/tests/test-quant-laboratory.cpp new file mode 100644 index 000000000000..477461a512b3 --- /dev/null +++ b/tests/test-quant-laboratory.cpp @@ -0,0 +1,355 @@ +// test-quant-laboratory.cpp +// Reusable testing harness for quantization experiments. +// +// Provides: +// - Synthetic data generators (Gaussian, Laplace, uniform) +// - Real tensor data loading (f32bin format with [nrow, ncol] header) +// - Importance matrix loading (flat f32 array) +// - RMSE computation +// - Multi-approach comparison framework (quantize → dequantize → matmul error) +// - ggml graph-level verification skeleton +// +// To add a new experiment: +// 1. Add an approach function: void approach_xxx(const float *W, float *out, +// int64_t nrow, int64_t ncol, +// const float *imatrix) +// 2. Register it in compare_approaches() +// 3. Call test_approach_comparison() from main() + +#include "../ggml/src/ggml-quants.h" +#include "ggml-backend.h" +#include "ggml-alloc.h" +#include "ggml.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// ============================================================================ +// Helper functions +// ============================================================================ + +static float rmse(const float * a, const float * b, size_t n) { + double sum = 0.0; + for (size_t i = 0; i < n; ++i) { + double d = (double) a[i] - (double) b[i]; + sum += d * d; + } + return (float) sqrt(sum / n); +} + +static void fill_gaussian(float * data, size_t n, std::mt19937 & gen, float sigma = 1.0f) { + std::normal_distribution dist(0.0f, sigma); + for (size_t i = 0; i < n; ++i) { + data[i] = dist(gen); + } +} + +static void fill_laplace(float * data, size_t n, std::mt19937 & gen, float b = 1.0f) { + std::uniform_real_distribution u(-0.5f, 0.5f); + for (size_t i = 0; i < n; ++i) { + float v = u(gen); + data[i] = -b * ((v > 0) - (v < 0)) * logf(1.0f - 2.0f * fabsf(v)); + } +} + +static void fill_uniform(float * data, size_t n, std::mt19937 & gen, float range = 1.0f) { + std::uniform_real_distribution dist(-range, range); + for (size_t i = 0; i < n; ++i) { + data[i] = dist(gen); + } +} + +static void fill_offset_gaussian(float * data, size_t n, std::mt19937 & gen, float sigma = 1.0f, float offset = 2.0f) { + std::normal_distribution dist(offset, sigma); + for (size_t i = 0; i < n; ++i) { + data[i] = dist(gen); + } +} + +// ============================================================================ +// Data loading +// ============================================================================ +static bool load_f32_tensor(const char * path, std::vector & data, int64_t & nrow, int64_t & n_per_row) { + FILE * f = fopen(path, "rb"); + if (!f) { + return false; + } + + int64_t header[2]; + if (fread(header, sizeof(int64_t), 2, f) != 2) { + fclose(f); + return false; + } + nrow = header[0]; + n_per_row = header[1]; + + int64_t total = nrow * n_per_row; + data.resize(total); + size_t nread = fread(data.data(), sizeof(float), total, f); + fclose(f); + if ((int64_t) nread != total) { + return false; + } + return true; +} + +// Load imatrix file (flat f32 array, no header, one importance value per column dimension) +// The imatrix is the sum-of-squares of activations per dimension. +static bool load_imatrix(const char * path, std::vector & data, int64_t expected_dims) { + FILE * f = fopen(path, "rb"); + if (!f) { + return false; + } + + // Get file size to determine dimensions + fseek(f, 0, SEEK_END); + long file_size = ftell(f); + fseek(f, 0, SEEK_SET); + + int64_t dims = file_size / sizeof(float); + if (expected_dims > 0 && dims != expected_dims) { + printf(" WARN: imatrix dims %lld != expected %lld\n", (long long) dims, (long long) expected_dims); + fclose(f); + return false; + } + + data.resize(dims); + size_t nread = fread(data.data(), sizeof(float), dims, f); + fclose(f); + if ((int64_t) nread != dims) { + return false; + } + + // Compute stats + float imin = data[0], imax = data[0], isum = 0; + for (int64_t i = 0; i < dims; i++) { + if (data[i] < imin) imin = data[i]; + if (data[i] > imax) imax = data[i]; + isum += data[i]; + } + printf(" Loaded imatrix: %lld dims, min=%.6f, max=%.6f, mean=%.6f\n", + (long long) dims, imin, imax, isum / dims); + + return true; +} + +// ============================================================================ +// Test class +// ============================================================================ + +class QuantLaboratory { + public: + QuantLaboratory() : gen(42) {} + + // ======================================================================== + // MULTI-APPROACH COMPARISON FRAMEWORK + // + // Each "approach" is a function that takes float weights and produces + // dequantized float output. The framework computes: + // - Weight RMSE (dequant vs original) + // - Matmul error (dequant weights x real activations vs f64 reference) + // - Ratio vs first approach (typically Q2_K baseline) + // + // To add a new approach: + // 1. Write: void approach_xxx(const float *W, float *out, + // int64_t nrow, int64_t ncol, + // const float *imatrix) { ... } + // 2. Add it to the `approaches` array in compare_approaches() + // ======================================================================== + + // -- Example approach: Q2_K baseline (via ggml library) -- + // Uncomment and adapt for your experiment: + // + // void approach_q2k(const float * W, float * out, int64_t nrow, int64_t ncol, const float * imatrix) { + // size_t rs = ggml_row_size(GGML_TYPE_Q2_K, ncol); + // std::vector buf(nrow * rs); + // quantize_q2_K(W, buf.data(), nrow, ncol, imatrix); + // auto * tr = ggml_get_type_traits(GGML_TYPE_Q2_K); + // for (int64_t r = 0; r < nrow; r++) { + // tr->to_float(buf.data() + r * rs, out + r * ncol, ncol, NULL); + // } + // } + + void compare_approaches(const float * W, + int64_t w_nrow, + int64_t w_ncol, + const float * A, + int64_t a_nrow, + int64_t a_ncol, + const char * name, + const float * imatrix) { + if (w_ncol != a_ncol) { + return; + } + int64_t nr = std::min(w_nrow, (int64_t) 256); + int64_t nc = w_ncol; + + // Reference matmul (double precision) + std::vector ref(a_nrow * nr); + for (int64_t t = 0; t < a_nrow; t++) { + for (int64_t r = 0; r < nr; r++) { + double s = 0; + for (int64_t c = 0; c < nc; c++) { + s += (double) A[t * a_ncol + c] * (double) W[r * nc + c]; + } + ref[t * nr + r] = s; + } + } + double ref_mag2 = 0; + for (auto v : ref) { + ref_mag2 += v * v; + } + float ref_rms = (float) sqrt(ref_mag2 / (a_nrow * nr)); + (void) ref_rms; + + struct Approach { + const char * name; + float bpw; + std::function fn; + }; + + // ── Register approaches here ── + Approach approaches[] = { + // { "Q2_K (baseline)", 2.625f, + // [&](auto * W, auto * o, auto nr, auto nc, auto * im) { + // approach_q2k(W, o, nr, nc, im); + // } }, + // Add more approaches... + { "placeholder", 0.0f, nullptr }, // remove once real approaches added + }; + + printf("\n %-28s %5s %10s %10s %7s\n", name, "BPW", "RMSE", "MatmulErr", "vs Q2K"); + printf(" %-28s %5s %10s %10s %7s\n", "---", "---", "---", "---", "---"); + + float baseline_matmul_err = 0; + for (auto & ap : approaches) { + if (!ap.fn) { + continue; + } + std::vector dec(nr * nc); + ap.fn(W, dec.data(), nr, nc, imatrix); + + // Weight RMSE + double werr2 = 0; + for (int64_t i = 0; i < nr * nc; i++) { + double d = W[i] - dec[i]; + werr2 += d * d; + } + float wrmse = (float) sqrt(werr2 / (nr * nc)); + + // Matmul error + double merr2 = 0; + for (int64_t t = 0; t < a_nrow; t++) { + for (int64_t r = 0; r < nr; r++) { + double s = 0; + for (int64_t c = 0; c < nc; c++) { + s += (double) A[t * a_ncol + c] * (double) dec[r * nc + c]; + } + double d = s - ref[t * nr + r]; + merr2 += d * d; + } + } + float matmul_rmse = (float) sqrt(merr2 / (a_nrow * nr)); + + if (baseline_matmul_err == 0) { + baseline_matmul_err = matmul_rmse; + } + float ratio = (baseline_matmul_err > 1e-10f) ? matmul_rmse / baseline_matmul_err : 0; + + printf(" %-28s %5.3f %10.6f %10.6f %6.3fx\n", ap.name, ap.bpw, wrmse, matmul_rmse, ratio); + } + } + + // Run comparison on all tensor pairs from data directory + int test_approach_comparison(const char * data_dir) { + printf("\n"); + printf("=======================================================================\n"); + printf(" MULTI-APPROACH COMPARISON (real weights x real activations)\n"); + printf("=======================================================================\n"); + + struct TestPair { + const char * wf; + const char * af; + const char * imf; + const char * name; + } pairs[] = { + { "blk_0_ffn_gate_weight.f32bin", "act_blk0_ffn_input.f32bin", "imatrix_blk0_ffn_gate_up.f32bin", "ffn_gate" }, + { "blk_0_ffn_up_weight.f32bin", "act_blk0_ffn_input.f32bin", "imatrix_blk0_ffn_gate_up.f32bin", "ffn_up" }, + { "blk_0_ffn_down_weight.f32bin", "act_blk0_ffn_down_input.f32bin", "imatrix_blk0_ffn_down.f32bin", "ffn_down" }, + { "blk_0_attn_q_weight.f32bin", "act_blk0_attn_input.f32bin", "imatrix_blk0_attn_qkv.f32bin", "attn_q" }, + }; + + for (auto & p : pairs) { + char wp[512], ap[512], imp[512]; + snprintf(wp, sizeof(wp), "%s/%s", data_dir, p.wf); + snprintf(ap, sizeof(ap), "%s/%s", data_dir, p.af); + snprintf(imp, sizeof(imp), "%s/%s", data_dir, p.imf); + std::vector wd, ad, im; + int64_t wnr, wnc, anr, anc; + if (!load_f32_tensor(wp, wd, wnr, wnc) || !load_f32_tensor(ap, ad, anr, anc)) { + continue; + } + const float * im_ptr = nullptr; + if (load_imatrix(imp, im, wnc)) { + im_ptr = im.data(); + } else { + printf(" [%s] No imatrix found, using uniform weights\n", p.name); + } + compare_approaches(wd.data(), wnr, wnc, ad.data(), anr, anc, p.name, im_ptr); + } + printf("\n"); + return 0; + } + + private: + std::mt19937 gen; +}; + +// ============================================================================ +// Main +// ============================================================================ + +int main(int argc, char ** argv) { + ggml_backend_load_all(); + + QuantLaboratory lab; + int total_fail = 0; + + printf("Quantization Laboratory\n"); + printf("=======================\n"); + + // Real data tests (from data/ directory) + { + const char * data_dir = "data"; + if (argc > 1) { + data_dir = argv[1]; + } + + char probe[512]; + snprintf(probe, sizeof(probe), "%s/blk_0_ffn_gate_weight.f32bin", data_dir); + FILE * fp = fopen(probe, "rb"); + if (fp) { + fclose(fp); + total_fail += lab.test_approach_comparison(data_dir); + } else { + printf("\n=== Real Data Tests SKIPPED ===\n"); + printf(" No data found at %s\n", data_dir); + printf( + " Run: cd data && PYTHONPATH=../gguf-py python3 ../scripts/extract-tensor-data.py MODEL.gguf " + "blk.0.ffn_gate blk.0.ffn_up blk.0.ffn_down blk.0.attn_q\n"); + printf(" And: llama-capture-layer-data -m MODEL.gguf -l 0 -o data\n"); + } + } + + printf("\n\n=== Testing Complete: %d failures ===\n", total_fail); + + return total_fail > 0 ? 1 : 0; +} diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 74978e446a3b..4bda0d4860d2 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -54,7 +54,7 @@ static float total_quantization_error(const ggml_type_traits * qfns, const ggml_ std::vector tmp_out(test_size); qfns_cpu->from_float(test_data, tmp_q.data(), test_size); - qfns->to_float(tmp_q.data(), tmp_out.data(), test_size); + qfns->to_float(tmp_q.data(), tmp_out.data(), test_size, nullptr); return array_rmse(test_data, tmp_out.data(), test_size); } @@ -66,10 +66,10 @@ static float reference_quantization_error(const ggml_type_traits * qfns, const g // FIXME: why is done twice? qfns_cpu->from_float(test_data, tmp_q.data(), test_size); - qfns->to_float(tmp_q.data(), tmp_out.data(), test_size); + qfns->to_float(tmp_q.data(), tmp_out.data(), test_size, nullptr); qfns->from_float_ref(test_data, tmp_q.data(), test_size); - qfns->to_float(tmp_q.data(), tmp_out_ref.data(), test_size); + qfns->to_float(tmp_q.data(), tmp_out_ref.data(), test_size, nullptr); return array_rmse(tmp_out.data(), tmp_out_ref.data(), test_size); } @@ -95,7 +95,7 @@ static float dot_product_error(const ggml_type_traits * qfns, const ggml_type_tr vdot->from_float(test_data2, tmp_q2.data(), test_size); float result = INFINITY; - qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1); + qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1, nullptr); const float dot_ref = dot_product(test_data1, test_data2, test_size); diff --git a/tests/test-quantize-perf.cpp b/tests/test-quantize-perf.cpp index cac0782dee9a..92597a04f462 100644 --- a/tests/test-quantize-perf.cpp +++ b/tests/test-quantize-perf.cpp @@ -309,7 +309,7 @@ int main(int argc, char * argv[]) { for (size_t size : params.test_sizes) { printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024)); auto quantize_fn = [&](void) -> float { - qfns->to_float(test_q1, test_out, size); + qfns->to_float(test_q1, test_out, size, nullptr); return test_out[0]; }; size_t quantized_size = ggml_row_size(type, size); @@ -341,7 +341,7 @@ int main(int argc, char * argv[]) { printf(" %zu values (%.2f MB)\n", size, 4*size/(float)(1024*1024)); auto quantize_fn = [&](void) -> float { float result; - qfns_cpu->vec_dot(size, &result, 0, test_q1, 0, test_q2, 0, 1); + qfns_cpu->vec_dot(size, &result, 0, test_q1, 0, test_q2, 0, 1, nullptr); return result; }; size_t quantized_size = ggml_row_size(type, size); diff --git a/tests/test-quantize-stats.cpp b/tests/test-quantize-stats.cpp index e53a7b355318..9651a54f4919 100644 --- a/tests/test-quantize-stats.cpp +++ b/tests/test-quantize-stats.cpp @@ -161,7 +161,7 @@ static void test_roundtrip_on_chunk( } else { qfns_cpu.from_float(input_scratch, quantized_scratch, chunk_size); } - qfns.to_float(quantized_scratch, output_scratch, chunk_size); + qfns.to_float(quantized_scratch, output_scratch, chunk_size, nullptr); update_error_stats(chunk_size, input_scratch, output_scratch, stats); } diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 780df3266132..8c68ccb21573 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -39,5 +39,6 @@ else() add_subdirectory(export-lora) endif() add_subdirectory(fit-params) + add_subdirectory(capture-layer-data) add_subdirectory(results) endif() diff --git a/tools/capture-layer-data/CMakeLists.txt b/tools/capture-layer-data/CMakeLists.txt new file mode 100644 index 000000000000..4a81272e2ead --- /dev/null +++ b/tools/capture-layer-data/CMakeLists.txt @@ -0,0 +1,9 @@ +set(TARGET llama-capture-layer-data) +add_executable(${TARGET} capture-layer-data.cpp) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_include_directories(${TARGET} PRIVATE ../../common) +target_compile_features(${TARGET} PRIVATE cxx_std_17) + +if(LLAMA_TOOLS_INSTALL) + install(TARGETS ${TARGET} RUNTIME) +endif() diff --git a/tools/capture-layer-data/capture-layer-data.cpp b/tools/capture-layer-data/capture-layer-data.cpp new file mode 100644 index 000000000000..30cfd16b21b4 --- /dev/null +++ b/tools/capture-layer-data/capture-layer-data.cpp @@ -0,0 +1,251 @@ +// capture-layer-data.cpp +// Captures intermediate activation tensors during model inference +// and saves them as .f32bin files for the quantization laboratory. +// +// Usage: +// llama-capture-layer-data -m MODEL_PATH -l LAYER [-p PROMPT] [-o OUTPUT_DIR] +// +// Example: +// llama-capture-layer-data -m /devel/models/Qwen_Qwen3-4B-Instruct-2507-bf16.gguf -l 0 -o data + +#include "arg.h" +#include "common.h" +#include "ggml-backend.h" +#include "ggml.h" +#include "llama.h" +#include "log.h" + +#include +#include +#include +#include +#include +#include +#include + +struct TensorMapping { + const char * graph_name_prefix; + const char * output_suffix; +}; + +static const TensorMapping mappings[] = { + { "attn_norm", "attn_input" }, + { "kqv_out", "attn_output_input" }, + { "ffn_norm", "ffn_input" }, + { "ffn_swiglu", "ffn_down_input" }, +}; +static constexpr int N_MAPPINGS = sizeof(mappings) / sizeof(mappings[0]); + +struct CaptureState { + int target_layer; + std::string output_dir; + int captured_count = 0; + std::string pending_name; + + std::string graph_to_filename(const char * graph_name) const { + for (int i = 0; i < N_MAPPINGS; i++) { + std::string prefix = mappings[i].graph_name_prefix; + if (strncmp(graph_name, prefix.c_str(), prefix.size()) == 0) { + char buf[256]; + snprintf(buf, sizeof(buf), "act_blk%d_%s.f32bin", target_layer, mappings[i].output_suffix); + return std::string(buf); + } + } + return ""; + } +}; + +static CaptureState * g_capture_state = nullptr; + +static void save_tensor_as_f32bin(const ggml_tensor * t, const std::string & filepath) { + int64_t n_rows = t->ne[1]; + int64_t row_len = t->ne[0]; + + int64_t total = 1; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + total *= t->ne[i]; + } + + std::vector f32_data(total); + + if (t->type == GGML_TYPE_F32) { + const float * src = (const float *) t->data; + if (!src) { + LOG_ERR("Tensor %s has null data pointer\n", t->name); + return; + } + memcpy(f32_data.data(), src, total * sizeof(float)); + } else if (t->type == GGML_TYPE_F16) { + const ggml_fp16_t * src = (const ggml_fp16_t *) t->data; + for (int64_t i = 0; i < total; i++) { + f32_data[i] = ggml_fp16_to_fp32(src[i]); + } + } else if (t->type == GGML_TYPE_BF16) { + const ggml_bf16_t * src = (const ggml_bf16_t *) t->data; + for (int64_t i = 0; i < total; i++) { + f32_data[i] = ggml_bf16_to_fp32(src[i]); + } + } else { + LOG_ERR("Unsupported tensor type %s for %s\n", ggml_type_name(t->type), t->name); + return; + } + + std::ofstream file(filepath, std::ios::binary); + if (!file) { + LOG_ERR("Failed to open %s for writing\n", filepath.c_str()); + return; + } + + file.write(reinterpret_cast(&n_rows), sizeof(int64_t)); + file.write(reinterpret_cast(&row_len), sizeof(int64_t)); + file.write(reinterpret_cast(f32_data.data()), total * sizeof(float)); + + file.close(); + LOG(" Captured: %s -> %s (%lld x %lld, %s)\n", t->name, filepath.c_str(), (long long) n_rows, (long long) row_len, + ggml_type_name(t->type)); +} + +static bool capture_callback(ggml_tensor * t, bool ask, void * user_data) { + auto * state = (CaptureState *) user_data; + + if (ask) { + char target[128]; + for (int i = 0; i < N_MAPPINGS; i++) { + snprintf(target, sizeof(target), "%s-%d", mappings[i].graph_name_prefix, state->target_layer); + if (strcmp(t->name, target) == 0) { + state->pending_name = t->name; + return true; + } + } + return false; + } + + if (state->pending_name.empty()) { + return true; + } + if (strcmp(t->name, state->pending_name.c_str()) != 0) { + return true; + } + + if (!ggml_backend_buffer_is_host(t->buffer)) { + size_t nbytes = ggml_nbytes(t); + std::vector tmp(nbytes); + ggml_backend_tensor_get(t, tmp.data(), 0, nbytes); + LOG_WRN("Tensor %s is not host-accessible, data copied via backend\n", t->name); + } + + std::string filename = state->graph_to_filename(t->name); + if (!filename.empty()) { + std::filesystem::create_directories(state->output_dir); + std::string filepath = (std::filesystem::path(state->output_dir) / filename).string(); + save_tensor_as_f32bin(t, filepath); + state->captured_count++; + } + + state->pending_name.clear(); + return true; +} + +static void print_usage(void) { + LOG("Usage: llama-capture-layer-data -m MODEL_PATH [-l LAYER] [-p PROMPT] [-o OUTPUT_DIR]\n"); + LOG("\n"); + LOG(" -m MODEL Path to GGUF model (BF16/F16 recommended)\n"); + LOG(" -l LAYER Target layer index (default: 0)\n"); + LOG(" -p PROMPT Inference prompt (default: \"The quick brown fox...\")\n"); + LOG(" -o DIR Output directory for .f32bin files (default: data)\n"); +} + +int main(int argc, char ** argv) { + if (argc < 3 || (std::string(argv[1]) == "-h" || std::string(argv[1]) == "--help")) { + print_usage(); + return 1; + } + + common_params params; + int layer = 0; + std::string output_dir = "data"; + std::string prompt = "The quick brown fox jumps over the lazy dog."; + std::string model_path; + + for (int i = 1; i < argc; i++) { + std::string arg = argv[i]; + if (arg == "-m" && i + 1 < argc) { + model_path = argv[++i]; + } else if (arg == "-l" && i + 1 < argc) { + layer = atoi(argv[++i]); + } else if (arg == "-p" && i + 1 < argc) { + prompt = argv[++i]; + } else if (arg == "-o" && i + 1 < argc) { + output_dir = argv[++i]; + } + } + + if (model_path.empty()) { + LOG_ERR("Error: -m MODEL_PATH is required\n\n"); + print_usage(); + return 1; + } + + params.model.path = model_path; + params.prompt = prompt; + params.n_batch = 512; + params.n_ubatch = 512; + params.n_gpu_layers = 0; + params.fit_params = false; + + CaptureState state; + state.target_layer = layer; + state.output_dir = output_dir; + g_capture_state = &state; + + params.cb_eval = capture_callback; + params.cb_eval_user_data = &state; + + LOG("Loading model: %s\n", model_path.c_str()); + LOG("Target layer: %d\n", layer); + LOG("Output directory: %s\n", output_dir.c_str()); + + common_init(); + ggml_backend_load_all(); + llama_backend_init(); + llama_numa_init(params.numa); + + auto llama_init = common_init_from_params(params); + if (!llama_init) { + LOG_ERR("Failed to load model\n"); + return 1; + } + + auto * model = llama_init->model(); + auto * ctx = llama_init->context(); + + if (model == nullptr || ctx == nullptr) { + LOG_ERR("Failed to initialize context\n"); + return 1; + } + + LOG("Model loaded successfully\n"); + + const llama_vocab * vocab = llama_model_get_vocab(model); + const bool add_bos = llama_vocab_get_add_bos(vocab); + std::vector tokens = common_tokenize(ctx, params.prompt, add_bos); + + if (tokens.empty()) { + LOG_ERR("No tokens generated from prompt\n"); + return 1; + } + + LOG("Tokenizing prompt: %zu tokens\n", tokens.size()); + LOG("Running inference...\n"); + + if (llama_decode(ctx, llama_batch_get_one(tokens.data(), tokens.size()))) { + LOG_ERR("llama_decode failed\n"); + return 1; + } + + LOG("\nDone. Captured %d tensors to %s/\n", state.captured_count, output_dir.c_str()); + + llama_backend_free(); + + return state.captured_count == 0 ? 1 : 0; +} diff --git a/tools/export-lora/export-lora.cpp b/tools/export-lora/export-lora.cpp index e1bc4a2f3152..b8637512f03b 100644 --- a/tools/export-lora/export-lora.cpp +++ b/tools/export-lora/export-lora.cpp @@ -318,7 +318,7 @@ struct lora_merge_ctx { auto nels = ggml_nelements(inp_base); const auto * qtype = ggml_get_type_traits(base->type); std::vector dequant_buf(nels * sizeof(float)); - qtype->to_float(read_buf.data(), (float *)dequant_buf.data(), nels); + qtype->to_float(read_buf.data(), (float *)dequant_buf.data(), nels, nullptr); ggml_backend_tensor_set(inp_base, dequant_buf.data(), 0, dequant_buf.size()); } else { ggml_backend_tensor_set(inp_base, read_buf.data(), 0, ggml_nbytes(inp_base)); diff --git a/tools/quantize/quantize.cpp b/tools/quantize/quantize.cpp index 840eefc2f5ac..254e556b42ec 100644 --- a/tools/quantize/quantize.cpp +++ b/tools/quantize/quantize.cpp @@ -49,6 +49,13 @@ static const std::vector QUANT_OPTIONS = { { "Q2_K", LLAMA_FTYPE_MOSTLY_Q2_K, " 2.96G, +3.5199 ppl @ Llama-3-8B", }, { "Q2_K_S", LLAMA_FTYPE_MOSTLY_Q2_K_S, " 2.96G, +3.1836 ppl @ Llama-3-8B", }, { "IQ3_XXS", LLAMA_FTYPE_MOSTLY_IQ3_XXS, " 3.06 bpw quantization", }, + { "Q3_PT", LLAMA_FTYPE_MOSTLY_Q3_PT, " 3.25 bpw quantization", }, + { "Q3_KPT", LLAMA_FTYPE_MOSTLY_Q3_KPT, " Q3_K with learned per-tensor levels" }, + { "Q4_DPT", LLAMA_FTYPE_MOSTLY_Q4_DPT, " IQ4_NL with learned per-tensor int8 levels" }, + { "Q2_KPT", LLAMA_FTYPE_MOSTLY_Q2_KPT, " Q2_K with learned per-tensor float levels" }, + { "IQ2_TQ", LLAMA_FTYPE_MOSTLY_IQ2_TQ, " 2.0625 bpw, trellis quantized" }, + { "IQ3_TQ", LLAMA_FTYPE_MOSTLY_IQ3_TQ, " 3.5625 bpw, per-tensor trained grid" }, + { "IQ1_BN", LLAMA_FTYPE_MOSTLY_IQ1_BN, " 1.5625 bpw, 8D vector quantized" }, { "IQ3_S", LLAMA_FTYPE_MOSTLY_IQ3_S, " 3.44 bpw quantization", }, { "IQ3_M", LLAMA_FTYPE_MOSTLY_IQ3_M, " 3.66 bpw quantization mix", }, { "Q3_K", LLAMA_FTYPE_MOSTLY_Q3_K_M, "alias for Q3_K_M" }, @@ -160,6 +167,9 @@ static void usage(const char * executable) { printf(" WARNING: this is an advanced option, use with care.\n"); printf(" --dry-run\n"); printf(" calculate and show the final quantization size without performing quantization\n"); + printf(" --threads n\n"); + printf(" number of threads to use for cross-tensor parallelization (default: 0, use same as within-tensor)\n"); + printf(" when n > 0, enables parallel quantization of multiple tensors simultaneously\n"); printf(" example: llama-quantize --dry-run model-f32.gguf Q4_K\n\n"); printf("note: --include-weights and --exclude-weights cannot be used together\n\n"); printf("-----------------------------------------------------------------------------\n"); @@ -466,6 +476,8 @@ int llama_quantize(int argc, char ** argv) { } } else if (strcmp(argv[arg_idx], "--keep-split") == 0) { params.keep_split = true; + } else if (strcmp(argv[arg_idx], "--keep-split") == 0) { + params.keep_split = true; } else { usage(argv[0]); } From 42fc4d42fd04a0a44706d4caaf572e521f7340ba Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 1 May 2026 16:12:15 +0200 Subject: [PATCH 02/19] The 'fixing compilation' squash --- ggml/src/ggml-cpu/arch-fallback.h | 15 +- ggml/src/ggml-cpu/arch/arm/quants.c | 201 +++++----------- ggml/src/ggml-cpu/arch/loongarch/quants.c | 72 +++--- ggml/src/ggml-cpu/arch/powerpc/quants.c | 84 ++++--- ggml/src/ggml-cpu/arch/riscv/quants.c | 246 +++++++++++--------- ggml/src/ggml-cpu/arch/s390/quants.c | 54 +++-- ggml/src/ggml-cpu/arch/wasm/quants.c | 29 ++- ggml/src/ggml-cpu/ggml-cpu.c | 3 + ggml/src/ggml-cpu/llamafile/sgemm.cpp | 2 + ggml/src/ggml-cpu/quants.c | 6 + ggml/src/ggml-cpu/repack.cpp | 1 - ggml/src/ggml-cpu/simd-mappings.h | 3 + ggml/src/ggml-cpu/vec.cpp | 3 +- ggml/src/ggml-cpu/vec.h | 6 + ggml/src/ggml-cuda/vendors/hip.h | 1 + ggml/src/ggml-cuda/vendors/musa.h | 1 + ggml/src/ggml-openvino/ggml-quants.cpp | 2 +- ggml/src/ggml-quants.c | 123 +++++++--- ggml/src/ggml.c | 2 +- scripts/analyze-ffn-down.py | 253 ++++++++++----------- scripts/compute-imatrix.py | 24 +- scripts/extract-activations.py | 24 +- scripts/extract-tensor-data.py | 22 +- scripts/server-bench.py | 2 +- scripts/server-test-function-call.py | 6 +- scripts/snapdragon/qdc/tests/test_bench.py | 63 +++++ src/llama-model.cpp | 46 +++- src/llama-quant.cpp | 122 +++++----- 28 files changed, 816 insertions(+), 600 deletions(-) create mode 100644 scripts/snapdragon/qdc/tests/test_bench.py diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 90adb7eebdc1..b14c757c3eae 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -35,6 +35,7 @@ #define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K #define ggml_vec_dot_q3_pt_q8_K_generic ggml_vec_dot_q3_pt_q8_K #define ggml_vec_dot_q4_dpt_q8_0_generic ggml_vec_dot_q4_dpt_q8_0 +#define ggml_vec_dot_q2_dpt_q8_0_generic ggml_vec_dot_q2_dpt_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 @@ -73,6 +74,9 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) +// quants.c +#define ggml_vec_dot_q2_dpt_q8_0_generic ggml_vec_dot_q2_dpt_q8_0 +#define ggml_vec_dot_q4_dpt_q8_0_generic ggml_vec_dot_q4_dpt_q8_0 // repack.cpp #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 @@ -118,6 +122,8 @@ #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K #define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K +#define ggml_vec_dot_q2_dpt_q8_0_generic ggml_vec_dot_q2_dpt_q8_0 +#define ggml_vec_dot_q4_dpt_q8_0_generic ggml_vec_dot_q4_dpt_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 @@ -203,15 +209,7 @@ #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__riscv) // quants.c -#define quantize_row_q8_K_generic quantize_row_q8_K -#define ggml_vec_dot_iq2_xxs_q8_K_generic ggml_vec_dot_iq2_xxs_q8_K -#define ggml_vec_dot_iq2_xs_q8_K_generic ggml_vec_dot_iq2_xs_q8_K -#define ggml_vec_dot_iq3_xxs_q8_K_generic ggml_vec_dot_iq3_xxs_q8_K -#define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0 -#define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K -#define ggml_vec_dot_q3_pt_q8_K_generic ggml_vec_dot_q3_pt_q8_K #define ggml_vec_dot_q4_dpt_q8_0_generic ggml_vec_dot_q4_dpt_q8_0 -#define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x1_generic ggml_quantize_mat_q8_0_4x1 @@ -316,6 +314,7 @@ #define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K #define ggml_vec_dot_q3_pt_q8_K_generic ggml_vec_dot_q3_pt_q8_K #define ggml_vec_dot_q4_dpt_q8_0_generic ggml_vec_dot_q4_dpt_q8_0 +#define ggml_vec_dot_q2_dpt_q8_0_generic ggml_vec_dot_q2_dpt_q8_0 #define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c index f45355ecfc1a..9c2c74ba50f0 100644 --- a/ggml/src/ggml-cpu/arch/arm/quants.c +++ b/ggml/src/ggml-cpu/arch/arm/quants.c @@ -338,6 +338,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q4_0 * GGML_RESTRICT x = vx; const block_q8_0 * GGML_RESTRICT y = vy; @@ -631,6 +632,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q4_1 * GGML_RESTRICT x = vx; const block_q8_1 * GGML_RESTRICT y = vy; @@ -776,12 +778,13 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; } -void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); assert(n % QK_MXFP4 == 0); static_assert(QK_MXFP4 == QK8_0, "QK_MXFP4 and QK8_0 must be the same"); @@ -837,117 +840,6 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo *s = sumf; } -void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { - assert(nrc == 1); - UNUSED(nrc); - UNUSED(bx); - UNUSED(by); - UNUSED(bs); - GGML_UNUSED(levels); - assert(n % QK_NVFP4 == 0); - - const block_nvfp4 * GGML_RESTRICT x = vx; - const block_q8_0 * GGML_RESTRICT y = vy; - - // Each NVFP4 super-block (64 elements) spans 2 q8_0 blocks - const int nb = n / QK_NVFP4; - - float sumf = 0; - -#if defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA) - const int8x16_t values = vld1q_s8(kvalues_mxfp4); - const uint8x16_t m4b = vdupq_n_u8(0x0f); - float32x4_t acc = vdupq_n_f32(0.0f); - - for (int ib = 0; ib < nb; ++ib) { - const uint8x16_t q4bits_0 = vld1q_u8(x[ib].qs); - const uint8x16_t q4bits_1 = vld1q_u8(x[ib].qs + 16); - - const int8x16_t q4_lo_0 = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits_0, m4b)); - const int8x16_t q4_hi_0 = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits_0, 4)); - const int8x16_t q4_lo_1 = ggml_vqtbl1q_s8(values, vandq_u8 (q4bits_1, m4b)); - const int8x16_t q4_hi_1 = ggml_vqtbl1q_s8(values, vshrq_n_u8(q4bits_1, 4)); - -#if defined(__ARM_FEATURE_DOTPROD) - const int8x16_t q8_0a = vld1q_s8(y[2*ib].qs); - const int8x16_t q8_0b = vld1q_s8(y[2*ib].qs + 16); - const int8x16_t q8_lo_0 = vcombine_s8(vget_low_s8(q8_0a), vget_low_s8(q8_0b)); - const int8x16_t q8_hi_0 = vcombine_s8(vget_high_s8(q8_0a), vget_high_s8(q8_0b)); - - const int8x16_t q8_1a = vld1q_s8(y[2*ib+1].qs); - const int8x16_t q8_1b = vld1q_s8(y[2*ib+1].qs + 16); - const int8x16_t q8_lo_1 = vcombine_s8(vget_low_s8(q8_1a), vget_low_s8(q8_1b)); - const int8x16_t q8_hi_1 = vcombine_s8(vget_high_s8(q8_1a), vget_high_s8(q8_1b)); - - const int32x4_t p0 = vaddq_s32( - vdotq_s32(vdupq_n_s32(0), q4_lo_0, q8_lo_0), - vdotq_s32(vdupq_n_s32(0), q4_hi_0, q8_hi_0)); - const int32x4_t p1 = vaddq_s32( - vdotq_s32(vdupq_n_s32(0), q4_lo_1, q8_lo_1), - vdotq_s32(vdupq_n_s32(0), q4_hi_1, q8_hi_1)); - - const int32x4_t sumi = vpaddq_s32(p0, p1); -#else - const int8x8_t q4_0_lo = vget_low_s8(q4_lo_0); - const int8x8_t q4_0_hi = vget_low_s8(q4_hi_0); - const int8x8_t q4_1_lo = vget_high_s8(q4_lo_0); - const int8x8_t q4_1_hi = vget_high_s8(q4_hi_0); - const int8x8_t q4_2_lo = vget_low_s8(q4_lo_1); - const int8x8_t q4_2_hi = vget_low_s8(q4_hi_1); - const int8x8_t q4_3_lo = vget_high_s8(q4_lo_1); - const int8x8_t q4_3_hi = vget_high_s8(q4_hi_1); - - const int8x8_t q8_0_lo = vld1_s8(y[2*ib].qs); - const int8x8_t q8_0_hi = vld1_s8(y[2*ib].qs + 8); - const int8x8_t q8_1_lo = vld1_s8(y[2*ib].qs + 16); - const int8x8_t q8_1_hi = vld1_s8(y[2*ib].qs + 24); - const int8x8_t q8_2_lo = vld1_s8(y[2*ib+1].qs); - const int8x8_t q8_2_hi = vld1_s8(y[2*ib+1].qs + 8); - const int8x8_t q8_3_lo = vld1_s8(y[2*ib+1].qs + 16); - const int8x8_t q8_3_hi = vld1_s8(y[2*ib+1].qs + 24); - - const int32x4_t sumi = (int32x4_t){ - vaddvq_s32(ggml_nvfp4_dot8(q4_0_lo, q8_0_lo, q4_0_hi, q8_0_hi)), - vaddvq_s32(ggml_nvfp4_dot8(q4_1_lo, q8_1_lo, q4_1_hi, q8_1_hi)), - vaddvq_s32(ggml_nvfp4_dot8(q4_2_lo, q8_2_lo, q4_2_hi, q8_2_hi)), - vaddvq_s32(ggml_nvfp4_dot8(q4_3_lo, q8_3_lo, q4_3_hi, q8_3_hi)), - }; -#endif - - const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d); - const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d); - const float32x4_t nvsc = { - ggml_ue4m3_to_fp32(x[ib].d[0]), - ggml_ue4m3_to_fp32(x[ib].d[1]), - ggml_ue4m3_to_fp32(x[ib].d[2]), - ggml_ue4m3_to_fp32(x[ib].d[3]) - }; - const float32x4_t scales = vmulq_f32(nvsc, (float32x4_t){dy0, dy0, dy1, dy1}); - - acc = vfmaq_f32(acc, vcvtq_f32_s32(sumi), scales); - } - sumf = vaddvq_f32(acc); -#else - for (int ib = 0; ib < nb; ++ib) { - for (int si = 0; si < 4; ++si) { - const float d = ggml_ue4m3_to_fp32(x[ib].d[si]); - const int q8b = si / 2; - const int q8o = (si % 2) * QK_NVFP4_SUB; - const float dy = GGML_CPU_FP16_TO_FP32(y[2*ib + q8b].d); - - int sumi_lo = 0, sumi_hi = 0; - for (int j = 0; j < QK_NVFP4_SUB/2; ++j) { - const uint8_t qv = x[ib].qs[si*(QK_NVFP4_SUB/2) + j]; - sumi_lo += y[2*ib + q8b].qs[q8o + j + 0] * kvalues_mxfp4[qv & 0xf]; - sumi_hi += y[2*ib + q8b].qs[q8o + j + QK_NVFP4_SUB/2] * kvalues_mxfp4[qv >> 4]; - } - sumf += dy * d * (sumi_lo + sumi_hi); - } - } -#endif - *s = sumf; -} - void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); @@ -1047,6 +939,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q5_0 * GGML_RESTRICT x = vx; const block_q8_0 * GGML_RESTRICT y = vy; @@ -1159,6 +1052,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q5_1 * GGML_RESTRICT x = vx; const block_q8_1 * GGML_RESTRICT y = vy; @@ -1277,6 +1171,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q8_0 * GGML_RESTRICT x = vx; const block_q8_0 * GGML_RESTRICT y = vy; @@ -1510,12 +1405,13 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = sumf; } -void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_tq1_0 * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1683,16 +1579,17 @@ void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_tq2_0 * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1794,16 +1691,17 @@ void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q2_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -2127,17 +2025,18 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const uint32_t kmask1 = 0x03030303; const uint32_t kmask2 = 0x0f0f0f0f; @@ -2422,7 +2321,7 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -2447,7 +2346,7 @@ static inline svuint32_t ggml_decode_q4scales_and_mins_for_mmla(const uint32_t * } #endif -void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); #ifdef __ARM_FEATURE_MATMUL_INT8 assert((nrc == 2) || (nrc == 1)); @@ -2458,6 +2357,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q4_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -2973,17 +2873,18 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(kmask2); UNUSED(kmask3); UNUSED(utmp); - ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q5_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -3073,11 +2974,11 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(kmask2); UNUSED(kmask3); UNUSED(utmp); - ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); #ifdef __ARM_FEATURE_MATMUL_INT8 assert((nrc == 2) || (nrc == 1)); @@ -3088,6 +2989,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q6_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -3703,7 +3605,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -3744,13 +3646,14 @@ static const int8_t keven_signs_q2xs[1024] = { }; #endif -void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_iq2_xxs * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -3802,17 +3705,18 @@ void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_iq2_xs * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -3876,17 +3780,18 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_iq2_s * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -3972,18 +3877,19 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_iq3_xxs * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -4035,17 +3941,18 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_iq3_s * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -4145,21 +4052,22 @@ void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { - ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); +void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); } -void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_iq1_s * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -4215,17 +4123,18 @@ void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_iq1_m * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -4309,16 +4218,17 @@ void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(y); UNUSED(nb); UNUSED(scale); - ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); assert(n % QK4_NL == 0); static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same"); @@ -4373,12 +4283,13 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v *s = sumf; } -void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); assert(n % QK_K == 0); const block_iq4_xs * GGML_RESTRICT x = vx; @@ -4433,7 +4344,7 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } diff --git a/ggml/src/ggml-cpu/arch/loongarch/quants.c b/ggml/src/ggml-cpu/arch/loongarch/quants.c index e25b9c291781..c5c175b3ba8d 100644 --- a/ggml/src/ggml-cpu/arch/loongarch/quants.c +++ b/ggml/src/ggml-cpu/arch/loongarch/quants.c @@ -651,6 +651,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi assert(n % qk == 0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -779,6 +780,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi assert(n % qk == 0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -842,6 +844,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi assert(qk == QK5_0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -895,6 +898,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi assert(qk == QK5_1); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -947,6 +951,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi assert(n % qk == 0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -1016,9 +1021,10 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi #endif } -void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -1092,14 +1098,15 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -1195,14 +1202,15 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -1292,14 +1300,15 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(kmask2); UNUSED(kmask3); UNUSED(utmp); - ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -1395,14 +1404,15 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(kmask2); UNUSED(kmask3); UNUSED(utmp); - ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -1569,7 +1579,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -1610,10 +1620,11 @@ static const int8_t keven_signs_q2xs[1024] = { }; #endif -void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -1669,14 +1680,15 @@ void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -1808,14 +1820,15 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -1903,14 +1916,15 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -1971,14 +1985,15 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -2074,12 +2089,12 @@ void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { - ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); +void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); } #if defined(__loongarch_asx) @@ -2090,10 +2105,11 @@ static inline __m256i mul_add_epi8(const __m256i x, const __m256i y) { } #endif -void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -2162,13 +2178,14 @@ void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -2225,9 +2242,10 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v *s = sumf; } -void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); + UNUSED(levels); UNUSED(bx); UNUSED(by); UNUSED(bs); @@ -2308,6 +2326,6 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } diff --git a/ggml/src/ggml-cpu/arch/powerpc/quants.c b/ggml/src/ggml-cpu/arch/powerpc/quants.c index c8fc0a3766ce..fe8708b8c469 100644 --- a/ggml/src/ggml-cpu/arch/powerpc/quants.c +++ b/ggml/src/ggml-cpu/arch/powerpc/quants.c @@ -151,6 +151,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q4_0 * GGML_RESTRICT x = vx; const block_q8_0 * GGML_RESTRICT y = vy; @@ -221,6 +222,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q4_1 * GGML_RESTRICT x = vx; const block_q8_1 * GGML_RESTRICT y = vy; @@ -278,12 +280,13 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi #endif } -void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); assert(n % QK_MXFP4 == 0); static_assert(QK_MXFP4 == QK8_0, "QK_MXFP4 and QK8_0 must be the same"); @@ -345,6 +348,7 @@ void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo } void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); const int qk = QK8_0; const int nb = n / qk; @@ -417,6 +421,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi } void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); const int qk = QK8_1; const int nb = n / qk; @@ -502,6 +507,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q8_0 * GGML_RESTRICT x = vx; const block_q8_0 * GGML_RESTRICT y = vy; @@ -561,12 +567,13 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi #endif } -void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q2_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -710,20 +717,18 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); - - const uint32_t kmask1 = 0x03030303; - const uint32_t kmask2 = 0x0f0f0f0f; + GGML_UNUSED(levels); const block_q3_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -752,9 +757,6 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi vector float vyd = vec_splats(y[i].d); vector float vd = vec_mul(vxd, vyd); - UNUSED(kmask1); - UNUSED(kmask2); - vector signed char u0 = (vector signed char)vec_xl_len(x[i].scales, 8); vector signed char u1 = vec_and(u0, lowMask1); vector signed char u2 = (vector signed char)vec_xl_len(x[i].scales + 8, 4); @@ -884,22 +886,21 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi *s = vec_extract(vsumf0, 0); #else - UNUSED(kmask1); - UNUSED(kmask2); UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q4_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1057,17 +1058,18 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(kmask2); UNUSED(kmask3); UNUSED(utmp); - ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q5_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1222,17 +1224,18 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(kmask2); UNUSED(kmask3); UNUSED(utmp); - ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q6_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1380,7 +1383,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -1421,13 +1424,14 @@ static const int8_t keven_signs_q2xs[1024] = { }; #endif -void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_iq2_xxs * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1524,17 +1528,18 @@ void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_iq2_xs * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1632,17 +1637,18 @@ void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_iq2_s * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1761,17 +1767,18 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_iq3_xxs * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1867,17 +1874,18 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_iq3_s * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1996,21 +2004,23 @@ void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { - ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); +void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + GGML_UNUSED(levels); + ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); } -void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_iq1_s * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -2116,16 +2126,17 @@ void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); assert(n % QK4_NL == 0); static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same"); @@ -2198,12 +2209,13 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v #endif } -void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); assert(n % QK_K == 0); const block_iq4_xs * GGML_RESTRICT x = vx; @@ -2303,6 +2315,6 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } diff --git a/ggml/src/ggml-cpu/arch/riscv/quants.c b/ggml/src/ggml-cpu/arch/riscv/quants.c index 69e3fba49261..8ce4fb572ed5 100644 --- a/ggml/src/ggml-cpu/arch/riscv/quants.c +++ b/ggml/src/ggml-cpu/arch/riscv/quants.c @@ -196,10 +196,10 @@ void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in // remaining iterations vint8m2_t slid_q = v_q; - for (size_t k = 16; k < vl; k += 16) { + for (size_t j = 16; j < vl; j += 16) { slid_q = __riscv_vslidedown_vx_i8m2(slid_q, 16, vl); - sum_idx = (offset + k) / 16; + sum_idx = (offset + j) / 16; chunk_m1 = __riscv_vget_v_i8m2_i8m1(slid_q, 0); v_sum = __riscv_vwredsum_vs_i8m1_i16m1(chunk_m1, v_zero_sum, 16); @@ -220,6 +220,7 @@ void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in //===================================== Dot products ================================= void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + UNUSED(levels); #if defined(__riscv_v) const int qk = QK8_0; const int nb = n / qk; @@ -275,6 +276,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi } void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + UNUSED(levels); #if defined(__riscv_v) const int qk = QK8_1; const int nb = n / qk; @@ -326,6 +328,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi } void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + UNUSED(levels); #if defined(__riscv_v) const int qk = QK8_0; const int nb = n / qk; @@ -380,6 +383,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi } void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + UNUSED(levels); #if defined(__riscv_v) const int qk = QK8_1; const int nb = n / qk; @@ -433,6 +437,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi } void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + UNUSED(levels); const int qk = QK8_0; const int nb = n / qk; @@ -560,7 +565,7 @@ static NOINLINE void ggml_vec_dot_q1_0_q8_0_vl128(const int n, float * GGML_REST } #endif -void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined(__riscv_v) assert(nrc == 1); @@ -571,20 +576,21 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi } else if (vlen_bits >= 128) { ggml_vec_dot_q1_0_q8_0_vl128(n, s, vx, vy); } else { - ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); } #else - ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } #if defined __riscv_xtheadvector -void ggml_vec_dot_q2_K_q8_K_xtheadvector(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q2_K_q8_K_xtheadvector(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_q2_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -941,9 +947,9 @@ void ggml_vec_dot_q2_K_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, con } #endif -void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_xtheadvector - ggml_vec_dot_q2_K_q8_K_xtheadvector(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q2_K_q8_K_xtheadvector(n, s, bs, vx, bx, vy, by, nrc, levels); #elif defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: @@ -954,18 +960,19 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi break; } #else - ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } #if defined __riscv_xtheadvector -void ggml_vec_dot_q3_K_q8_K_xtheadvector(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q3_K_q8_K_xtheadvector(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const uint32_t kmask1 = 0x03030303; const uint32_t kmask2 = 0x0f0f0f0f; @@ -1604,9 +1611,9 @@ void ggml_vec_dot_q3_K_q8_K_vl1024(int n, float * GGML_RESTRICT s, size_t bs, co } #endif -void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_xtheadvector - ggml_vec_dot_q3_K_q8_K_xtheadvector(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q3_K_q8_K_xtheadvector(n, s, bs, vx, bx, vy, by, nrc, levels); #elif defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: @@ -1622,22 +1629,23 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi ggml_vec_dot_q3_K_q8_K_vl1024(n, s, bs, vx, bx, vy, by, nrc); break; default: - ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); break; } #else - ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } #if defined __riscv_xtheadvector -static NOINLINE void ggml_vec_dot_q4_K_q8_K_xtheadvector(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_q4_K_q8_K_xtheadvector(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_q4_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -2061,9 +2069,9 @@ static NOINLINE void ggml_vec_dot_q4_K_q8_K_vl256(int n, float * GGML_RESTRICT s } #endif -void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_xtheadvector - ggml_vec_dot_q4_K_q8_K_xtheadvector(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_K_q8_K_xtheadvector(n, s, bs, vx, bx, vy, by, nrc, levels); #elif defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: @@ -2074,17 +2082,18 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi break; } #else - ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_q5_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -2192,18 +2201,19 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(nb); UNUSED(utmp); - ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } #if defined __riscv_xtheadvector -static NOINLINE void ggml_vec_dot_q6_K_q8_K_xtheadvector(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_q6_K_q8_K_xtheadvector(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_q6_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -2717,9 +2727,9 @@ static NOINLINE void ggml_vec_dot_q6_K_q8_K_vl1024(int n, float * GGML_RESTRICT } #endif -void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_xtheadvector - ggml_vec_dot_q6_K_q8_K_xtheadvector(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q6_K_q8_K_xtheadvector(n, s, bs, vx, bx, vy, by, nrc, levels); #elif defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: @@ -2735,22 +2745,23 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi ggml_vec_dot_q6_K_q8_K_vl1024(n, s, bs, vx, bx, vy, by, nrc); break; default: - ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); break; } #else - ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } #if defined __riscv_v -static NOINLINE void ggml_vec_dot_iq1_s_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq1_s_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq1_s * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -2856,13 +2867,14 @@ static NOINLINE void ggml_vec_dot_iq1_s_q8_K_vl128(int n, float * GGML_RESTRICT *s = sumf; } -static NOINLINE void ggml_vec_dot_iq1_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq1_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq1_s * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -3133,14 +3145,14 @@ static NOINLINE void ggml_vec_dot_iq1_s_q8_K_vl1024(int n, float * GGML_RESTRICT } #endif -void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: - ggml_vec_dot_iq1_s_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_s_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 256: - ggml_vec_dot_iq1_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 512: ggml_vec_dot_iq1_s_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); @@ -3149,22 +3161,23 @@ void ggml_vec_dot_iq1_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo ggml_vec_dot_iq1_s_q8_K_vl1024(n, s, bs, vx, bx, vy, by, nrc); break; default: - ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); break; } #else - ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } #if defined __riscv_v -static NOINLINE void ggml_vec_dot_iq1_m_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq1_m_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq1_m * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -3325,13 +3338,14 @@ static NOINLINE void ggml_vec_dot_iq1_m_q8_K_vl128(int n, float * GGML_RESTRICT *s = sumf; } -static NOINLINE void ggml_vec_dot_iq1_m_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq1_m_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq1_m * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -3716,14 +3730,14 @@ static NOINLINE void ggml_vec_dot_iq1_m_q8_K_vl1024(int n, float * GGML_RESTRICT } #endif -void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: - ggml_vec_dot_iq1_m_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_m_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 256: - ggml_vec_dot_iq1_m_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_m_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 512: ggml_vec_dot_iq1_m_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); @@ -3732,11 +3746,11 @@ void ggml_vec_dot_iq1_m_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo ggml_vec_dot_iq1_m_q8_K_vl1024(n, s, bs, vx, bx, vy, by, nrc); break; default: - ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); break; } #else - ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq1_m_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -3751,9 +3765,10 @@ static const uint8_t sign_bit_masks_arr[64] = { 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128, 1,2,4,8,16,32,64,128 }; -static NOINLINE void ggml_vec_dot_iq2_s_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq2_s_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq2_s * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -3842,9 +3857,10 @@ static NOINLINE void ggml_vec_dot_iq2_s_q8_K_vl128(int n, float * GGML_RESTRICT *s = 0.125f * sumf; } -static NOINLINE void ggml_vec_dot_iq2_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq2_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq2_s * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -4210,14 +4226,14 @@ static NOINLINE void ggml_vec_dot_iq2_s_q8_K_vl1024(int n, float * GGML_RESTRICT } #endif -void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: - ggml_vec_dot_iq2_s_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_s_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 256: - ggml_vec_dot_iq2_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 512: ggml_vec_dot_iq2_s_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); @@ -4227,7 +4243,7 @@ void ggml_vec_dot_iq2_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const vo break; } #else - ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -4267,13 +4283,14 @@ static const int8_t keven_signs_q2xs[1024] = { 1, 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, }; -static NOINLINE void ggml_vec_dot_iq2_xs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq2_xs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq2_xs * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -4344,13 +4361,14 @@ static NOINLINE void ggml_vec_dot_iq2_xs_q8_K_vl128(int n, float * GGML_RESTRICT *s = 0.125f * sumf; } -static NOINLINE void ggml_vec_dot_iq2_xs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq2_xs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq2_xs * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -4501,32 +4519,33 @@ static NOINLINE void ggml_vec_dot_iq2_xs_q8_K_vl512(int n, float * GGML_RESTRICT } #endif -void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: - ggml_vec_dot_iq2_xs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 256: - ggml_vec_dot_iq2_xs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc, levels); break; default: // 512 and above ggml_vec_dot_iq2_xs_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); break; } #else - ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } #if defined __riscv_v -static NOINLINE void ggml_vec_dot_iq2_xxs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq2_xxs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq2_xxs * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -4611,13 +4630,14 @@ static NOINLINE void ggml_vec_dot_iq2_xxs_q8_K_vl128(int n, float * GGML_RESTRIC *s = 0.125f * sumf; } -static NOINLINE void ggml_vec_dot_iq2_xxs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq2_xxs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq2_xxs * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -4778,28 +4798,29 @@ static NOINLINE void ggml_vec_dot_iq2_xxs_q8_K_vl512(int n, float * GGML_RESTRIC } #endif -void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: - ggml_vec_dot_iq2_xxs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xxs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 256: - ggml_vec_dot_iq2_xxs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xxs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc, levels); break; default: // 512 and above ggml_vec_dot_iq2_xxs_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); break; } #else - ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq2_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } #if defined __riscv_v -static NOINLINE void ggml_vec_dot_iq3_s_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq3_s_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq3_s * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -4892,12 +4913,13 @@ static NOINLINE void ggml_vec_dot_iq3_s_q8_K_vl128(int n, float * GGML_RESTRICT *s = sumf; } -static NOINLINE void ggml_vec_dot_iq3_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq3_s_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq3_s * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -5077,32 +5099,29 @@ static NOINLINE void ggml_vec_dot_iq3_s_q8_K_vl512(int n, float * GGML_RESTRICT } #endif -void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq3_s_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: - ggml_vec_dot_iq3_s_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_s_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 256: - ggml_vec_dot_iq3_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_s_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc, levels); break; default: // 512 and above ggml_vec_dot_iq3_s_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); break; } #else - ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_s_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { - ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); -} - #if defined __riscv_v -static NOINLINE void ggml_vec_dot_iq3_xxs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq3_xxs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq3_xxs * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -5193,13 +5212,14 @@ static NOINLINE void ggml_vec_dot_iq3_xxs_q8_K_vl128(int n, float * GGML_RESTRIC *s = 0.25f * sumf; } -static NOINLINE void ggml_vec_dot_iq3_xxs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq3_xxs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_iq3_xxs * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -5458,14 +5478,14 @@ static NOINLINE void ggml_vec_dot_iq3_xxs_q8_K_vl1024(int n, float * GGML_RESTRI } #endif -void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: - ggml_vec_dot_iq3_xxs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_xxs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 256: - ggml_vec_dot_iq3_xxs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_xxs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 512: ggml_vec_dot_iq3_xxs_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); @@ -5475,17 +5495,18 @@ void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const break; } #else - ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq3_xxs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } #if defined __riscv_v -static NOINLINE void ggml_vec_dot_iq4_nl_q8_0_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq4_nl_q8_0_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); assert(n % QK4_NL == 0); static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same"); @@ -5535,12 +5556,13 @@ static NOINLINE void ggml_vec_dot_iq4_nl_q8_0_vl128(int n, float * GGML_RESTRICT *s = sumf; } -static NOINLINE void ggml_vec_dot_iq4_nl_q8_0_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq4_nl_q8_0_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); assert(n % QK4_NL == 0); static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same"); @@ -5593,28 +5615,29 @@ static NOINLINE void ggml_vec_dot_iq4_nl_q8_0_vl256(int n, float * GGML_RESTRICT } #endif -void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: - ggml_vec_dot_iq4_nl_q8_0_vl128(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_nl_q8_0_vl128(n, s, bs, vx, bx, vy, by, nrc, levels); break; default: // 256 and above - ggml_vec_dot_iq4_nl_q8_0_vl256(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_nl_q8_0_vl256(n, s, bs, vx, bx, vy, by, nrc, levels); break; } #else - ggml_vec_dot_iq4_nl_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_nl_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } #if defined __riscv_v -static NOINLINE void ggml_vec_dot_iq4_xs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq4_xs_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); assert(n % QK_K == 0); const block_iq4_xs * GGML_RESTRICT x = vx; @@ -5675,12 +5698,13 @@ static NOINLINE void ggml_vec_dot_iq4_xs_q8_K_vl128(int n, float * GGML_RESTRICT *s = sumf; } -static NOINLINE void ggml_vec_dot_iq4_xs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_iq4_xs_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); assert(n % QK_K == 0); const block_iq4_xs * GGML_RESTRICT x = vx; @@ -5951,14 +5975,14 @@ static NOINLINE void ggml_vec_dot_iq4_xs_q8_K_vl1024(int n, float * GGML_RESTRIC } #endif -void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: - ggml_vec_dot_iq4_xs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_xs_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 256: - ggml_vec_dot_iq4_xs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_xs_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 512: ggml_vec_dot_iq4_xs_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); @@ -5967,21 +5991,22 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v ggml_vec_dot_iq4_xs_q8_K_vl1024(n, s, bs, vx, bx, vy, by, nrc); break; default: - ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); break; } #else - ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } #if defined __riscv_v -static NOINLINE void ggml_vec_dot_tq1_0_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_tq1_0_q8_K_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_tq1_0 * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -6075,12 +6100,13 @@ static NOINLINE void ggml_vec_dot_tq1_0_q8_K_vl128(int n, float * GGML_RESTRICT *s = sumf; } -static NOINLINE void ggml_vec_dot_tq1_0_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_tq1_0_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_tq1_0 * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -6282,32 +6308,33 @@ static NOINLINE void ggml_vec_dot_tq1_0_q8_K_vl512(int n, float * GGML_RESTRICT } #endif -void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: - ggml_vec_dot_tq1_0_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_tq1_0_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc, levels); break; case 256: - ggml_vec_dot_tq1_0_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_tq1_0_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc, levels); break; default: // 512 and above ggml_vec_dot_tq1_0_q8_K_vl512(n, s, bs, vx, bx, vy, by, nrc); break; } #else - ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_tq1_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } #if defined __riscv_v -static NOINLINE void ggml_vec_dot_tq2_0_q8_K_vl128(const int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_tq2_0_q8_K_vl128(const int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_tq2_0 * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -6383,13 +6410,14 @@ static NOINLINE void ggml_vec_dot_tq2_0_q8_K_vl128(const int n, float * GGML_RES *s = sumf; } -static NOINLINE void ggml_vec_dot_tq2_0_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_tq2_0_q8_K_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_tq2_0 * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -6455,28 +6483,29 @@ static NOINLINE void ggml_vec_dot_tq2_0_q8_K_vl256(int n, float * GGML_RESTRICT } #endif -void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: - ggml_vec_dot_tq2_0_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_tq2_0_q8_K_vl128(n, s, bs, vx, bx, vy, by, nrc, levels); break; default: // 256 and above - ggml_vec_dot_tq2_0_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_tq2_0_q8_K_vl256(n, s, bs, vx, bx, vy, by, nrc, levels); break; } #else - ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_tq2_0_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } #if defined __riscv_v -static NOINLINE void ggml_vec_dot_mxfp4_q8_0_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_mxfp4_q8_0_vl128(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); assert(n % QK_MXFP4 == 0); static_assert(QK_MXFP4 == QK8_0, "QK_MXFP4 and QK8_0 must be the same"); @@ -6526,12 +6555,13 @@ static NOINLINE void ggml_vec_dot_mxfp4_q8_0_vl128(int n, float * GGML_RESTRICT *s = sumf; } -static NOINLINE void ggml_vec_dot_mxfp4_q8_0_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +static NOINLINE void ggml_vec_dot_mxfp4_q8_0_vl256(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); assert(n % QK_MXFP4 == 0); static_assert(QK_MXFP4 == QK8_0, "QK_MXFP4 and QK8_0 must be the same"); @@ -6584,17 +6614,25 @@ static NOINLINE void ggml_vec_dot_mxfp4_q8_0_vl256(int n, float * GGML_RESTRICT } #endif -void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { #if defined __riscv_v switch (__riscv_vlenb() * 8) { case 128: - ggml_vec_dot_mxfp4_q8_0_vl128(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_mxfp4_q8_0_vl128(n, s, bs, vx, bx, vy, by, nrc, levels); break; default: // 256 and above - ggml_vec_dot_mxfp4_q8_0_vl256(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_mxfp4_q8_0_vl256(n, s, bs, vx, bx, vy, by, nrc, levels); break; } #else - ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_mxfp4_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } + +void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); +} + +void ggml_vec_dot_q2_dpt_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + ggml_vec_dot_q2_dpt_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); +} diff --git a/ggml/src/ggml-cpu/arch/s390/quants.c b/ggml/src/ggml-cpu/arch/s390/quants.c index c1186901926e..cff482713d54 100644 --- a/ggml/src/ggml-cpu/arch/s390/quants.c +++ b/ggml/src/ggml-cpu/arch/s390/quants.c @@ -156,6 +156,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q4_0 * GGML_RESTRICT x = vx; const block_q8_0 * GGML_RESTRICT y = vy; @@ -215,6 +216,7 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q4_1 * GGML_RESTRICT x = vx; const block_q8_1 * GGML_RESTRICT y = vy; @@ -262,12 +264,13 @@ void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi #endif } -void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_mxfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); assert(n % QK_MXFP4 == 0); static_assert(QK_MXFP4 == QK8_0, "QK_MXFP4 and QK8_0 must be the same"); @@ -368,6 +371,7 @@ void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q5_0 * GGML_RESTRICT x = vx; const block_q8_0 * GGML_RESTRICT y = vy; @@ -510,6 +514,7 @@ void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q5_1 * GGML_RESTRICT x = vx; const block_q8_1 * GGML_RESTRICT y = vy; @@ -662,6 +667,7 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q8_0 * GGML_RESTRICT x = vx; const block_q8_0 * GGML_RESTRICT y = vy; @@ -702,22 +708,23 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi #endif } -void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); - - const uint32_t kmask1 = 0x03030303; - const uint32_t kmask2 = 0x0f0f0f0f; + GGML_UNUSED(levels); const block_q3_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; const int nb = n / QK_K; + const uint32_t kmask1 = 0x03030303; + const uint32_t kmask2 = 0x0f0f0f0f; + #if defined(__VXE__) || defined(__VXE2__) uint32_t aux[3]; uint32_t utmp[4]; @@ -837,17 +844,18 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q4_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -939,17 +947,18 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(kmask2); UNUSED(kmask3); UNUSED(utmp); - ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q5_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1058,17 +1067,18 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(kmask2); UNUSED(kmask3); UNUSED(utmp); - ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q6_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1204,7 +1214,7 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } @@ -1342,12 +1352,13 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi // #endif // } -void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); assert(n % QK4_NL == 0); static_assert(QK4_NL == QK8_0, "QK4_NL and QK8_0 must be the same"); @@ -1392,12 +1403,13 @@ void ggml_vec_dot_iq4_nl_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const v #endif } -void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); assert(n % QK_K == 0); const block_iq4_xs * GGML_RESTRICT x = vx; @@ -1460,10 +1472,18 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_iq4_xs_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { - ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); +void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); +} + +void ggml_vec_dot_q2_dpt_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + ggml_vec_dot_q2_dpt_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); +} + +void ggml_vec_dot_q4_dpt_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + ggml_vec_dot_q4_dpt_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); } diff --git a/ggml/src/ggml-cpu/arch/wasm/quants.c b/ggml/src/ggml-cpu/arch/wasm/quants.c index d2df68fddc4f..379ca6466ab8 100644 --- a/ggml/src/ggml-cpu/arch/wasm/quants.c +++ b/ggml/src/ggml-cpu/arch/wasm/quants.c @@ -682,12 +682,13 @@ void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi #endif } -void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_q2_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -798,17 +799,18 @@ void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q2_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const uint32_t kmask1 = 0x03030303; const uint32_t kmask2 = 0x0f0f0f0f; @@ -912,18 +914,19 @@ void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q3_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_q4_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1045,16 +1048,17 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(kmask2); UNUSED(kmask3); UNUSED(utmp); - ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q4_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); + UNUSED(levels); UNUSED(bs); const block_q5_K * GGML_RESTRICT x = vx; @@ -1188,17 +1192,18 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(kmask2); UNUSED(kmask3); UNUSED(utmp); - ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q5_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { +void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { assert(n % QK_K == 0); assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); + UNUSED(levels); const block_q6_K * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; @@ -1288,10 +1293,10 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(x); UNUSED(y); UNUSED(nb); - ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q6_K_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } -void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { - ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc); +void ggml_vec_dot_q3_pt_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { + ggml_vec_dot_q3_pt_q8_K_generic(n, s, bs, vx, bx, vy, by, nrc, levels); } diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index b8c459dd410c..f0d3f61bd4aa 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -3485,12 +3485,15 @@ void ggml_cpu_fp32_to_fp16(const float * x, ggml_fp16_t * y, int64_t n) { _mm_storel_epi64((__m128i *)(y + i), y_vec); } #elif defined(__riscv_zvfh) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wpedantic" for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl); vfloat16m1_t vy = __riscv_vfncvt_f_f_w_f16m1(vx, vl); __riscv_vse16_v_f16m1((_Float16 *)&y[i], vy, vl); } + #pragma GCC diagnostic pop #endif for (; i < n; ++i) { y[i] = GGML_CPU_FP32_TO_FP16(x[i]); diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index aa86707bfeb6..91bd94253ba2 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -43,6 +43,7 @@ // [1] J. Tunney, ‘LLaMA Now Goes Faster on CPUs’, Mar. 2024. [Online]. // Available: https://justine.lol/matmul/. [Accessed: 29-Mar-2024]. +#include "ggml.h" #if defined(__GNUC__) #pragma GCC diagnostic ignored "-Wpedantic" #pragma GCC diagnostic ignored "-Wignored-attributes" @@ -3697,6 +3698,7 @@ class tinyBLAS_PPC { bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64_t n, int64_t k, const void *A, int64_t lda, const void *B, int64_t ldb, void *C, int64_t ldc, int Atype, int Btype, int Ctype, const void * quant_levels) { + GGML_UNUSED(quant_levels); assert(m >= 0); assert(n >= 0); diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index 91908ec22c02..6094ca44bb55 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -1384,7 +1384,10 @@ void ggml_vec_dot_iq2_tq_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" const int8_t (*grid)[4] = levels ? (const int8_t (*)[4])levels : (const int8_t (*)[4])iq2tq_grid_cpu; +#pragma GCC diagnostic pop const block_iq2_tq * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; const int nb = n / QK_K; @@ -1441,7 +1444,10 @@ void ggml_vec_dot_iq3_tq_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v assert(nrc == 1); UNUSED(nrc); UNUSED(bx); UNUSED(by); UNUSED(bs); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" const int8_t (*grid)[8] = levels ? (const int8_t (*)[8])levels : (const int8_t (*)[8])iq3tq_grid_cpu; +#pragma GCC diagnostic pop const block_iq3_tq * GGML_RESTRICT x = vx; const block_q8_K * GGML_RESTRICT y = vy; const int nb = n / QK_K; diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp index f18758f16bb6..5db63c82424d 100644 --- a/ggml/src/ggml-cpu/repack.cpp +++ b/ggml/src/ggml-cpu/repack.cpp @@ -94,7 +94,6 @@ void ggml_quantize_mat_q8_K_4x1_generic(const float * GGML_RESTRICT x, void * GG block_q8_Kx4 * GGML_RESTRICT y = (block_q8_Kx4 *) vy; - const int blck_size_interleave = 1; float srcv[4][QK_K]; float iscale[4]; diff --git a/ggml/src/ggml-cpu/simd-mappings.h b/ggml/src/ggml-cpu/simd-mappings.h index be50c25c0aef..12a952e4e25d 100644 --- a/ggml/src/ggml-cpu/simd-mappings.h +++ b/ggml/src/ggml-cpu/simd-mappings.h @@ -93,6 +93,8 @@ extern "C" { return r; } #elif defined(__riscv) && defined(__riscv_zfhmin) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wpedantic" static inline float riscv_compute_fp16_to_fp32(ggml_fp16_t h) { _Float16 hf; memcpy(&hf, &h, sizeof(ggml_fp16_t)); @@ -105,6 +107,7 @@ extern "C" { memcpy(&res, &hf, sizeof(ggml_fp16_t)); return res; } + #pragma GCC diagnostic pop #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) riscv_compute_fp16_to_fp32(x) #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) riscv_compute_fp32_to_fp16(x) diff --git a/ggml/src/ggml-cpu/vec.cpp b/ggml/src/ggml-cpu/vec.cpp index 9aec0684d761..629ea6292fef 100644 --- a/ggml/src/ggml-cpu/vec.cpp +++ b/ggml/src/ggml-cpu/vec.cpp @@ -88,10 +88,9 @@ void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * G #elif defined(__riscv_v_intrinsic) int vl = __riscv_vsetvlmax_e32m8(); vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1); - vfloat32m8_t vsum; + vfloat32m8_t vsum = __riscv_vfmv_v_f_f32m8(0.0f, vl); vfloat32m8_t ax; vfloat32m8_t ay; - vsum = __riscv_vfmv_v_f_f32m8_tu(vsum, 0.0f, vl); for (int i = 0; i < n; i += vl) { vl = __riscv_vsetvl_e32m8(n - i); ax = __riscv_vle32_v_f32m8_tu(ax, &x[i], vl); diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h index 6d35277b4274..ddf797d91a74 100644 --- a/ggml/src/ggml-cpu/vec.h +++ b/ggml/src/ggml-cpu/vec.h @@ -209,6 +209,8 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * GG np = n; #elif defined(__riscv_v_intrinsic) #if defined(__riscv_zvfh) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wpedantic" size_t vl = __riscv_vsetvlmax_e32m4(); // initialize accumulators to all zeroes @@ -272,6 +274,7 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * GG sumf[0] = __riscv_vfmv_f_s_f32m1_f32(redsum0); sumf[1] = __riscv_vfmv_f_s_f32m1_f32(redsum1); np = n; + #pragma GCC diagnostic pop #else const int np = 0; #endif @@ -796,6 +799,8 @@ inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float np = n; #elif defined(__riscv_v_intrinsic) #if defined(__riscv_zvfh) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wpedantic" const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v); const _Float16 scale = *(const _Float16*)(&s); @@ -826,6 +831,7 @@ inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float __riscv_vse16_v_f16m4((_Float16*)y + i, ay0, vl); } np = n; + #pragma GCC diagnostic pop #else // fall to scalar path const int np = 0; diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h index d01f1533abb6..234d76dfd8e0 100644 --- a/ggml/src/ggml-cuda/vendors/hip.h +++ b/ggml/src/ggml-cuda/vendors/hip.h @@ -77,6 +77,7 @@ #define cudaGetDeviceProperties hipGetDeviceProperties #define cudaGetErrorString hipGetErrorString #define cudaGetLastError hipGetLastError +#define cudaGetSymbolAddress hipGetSymbolAddress #define cudaHostRegister hipHostRegister #define cudaHostRegisterPortable hipHostRegisterPortable #define cudaHostRegisterReadOnly hipHostRegisterReadOnly diff --git a/ggml/src/ggml-cuda/vendors/musa.h b/ggml/src/ggml-cuda/vendors/musa.h index 6d725c7ec196..284bd9de4c54 100644 --- a/ggml/src/ggml-cuda/vendors/musa.h +++ b/ggml/src/ggml-cuda/vendors/musa.h @@ -61,6 +61,7 @@ #define cudaGetDeviceProperties musaGetDeviceProperties #define cudaGetErrorString musaGetErrorString #define cudaGetLastError musaGetLastError +#define cudaGetSymbolAddress musaGetSymbolAddress #define cudaHostRegister musaHostRegister #define cudaHostRegisterPortable musaHostRegisterPortable #define cudaHostRegisterReadOnly musaHostRegisterReadOnly diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index 275b95428273..2a93f5634351 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -705,7 +705,7 @@ std::shared_ptr requantize_to_buffers(const ggml_tensor * tensor, // First dequantize to F32 std::vector weights_f32(n_elements); - ggml_get_type_traits(tensor->type)->to_float(data, weights_f32.data(), n_elements); + ggml_get_type_traits(tensor->type)->to_float(data, weights_f32.data(), n_elements, nullptr); // Handle F16 case - just convert and create constant if (requant_type == ExtraQuantType::F16) { diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index a730d29f240a..98432340913a 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -400,6 +400,7 @@ void dequantize_row_q1_0(const block_q1_0 * GGML_RESTRICT x, float * GGML_RESTRI } void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); static const int qk = QK4_0; assert(k % qk == 0); @@ -420,6 +421,7 @@ void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRI } void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); static const int qk = QK4_1; assert(k % qk == 0); @@ -441,6 +443,7 @@ void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRI } void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); static const int qk = QK5_0; assert(k % qk == 0); @@ -467,6 +470,7 @@ void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRI } void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); static const int qk = QK5_1; assert(k % qk == 0); @@ -494,6 +498,7 @@ void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRI } void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); static const int qk = QK8_0; assert(k % qk == 0); @@ -510,6 +515,7 @@ void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRI } void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); static const int qk = QK_MXFP4; assert(k % qk == 0); @@ -912,6 +918,7 @@ void quantize_row_q2_K_ref(const float * GGML_RESTRICT x, block_q2_K * GGML_REST } void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int nb = k / QK_K; @@ -1256,6 +1263,7 @@ void quantize_row_q3_K_ref(const float * GGML_RESTRICT x, block_q3_K * GGML_REST } void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int nb = k / QK_K; @@ -1480,6 +1488,7 @@ void quantize_row_q4_K_ref(const float * GGML_RESTRICT x, block_q4_K * GGML_REST } void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int nb = k / QK_K; @@ -1682,6 +1691,7 @@ void quantize_row_q5_K_ref(const float * GGML_RESTRICT x, block_q5_K * GGML_REST } void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -1890,6 +1900,7 @@ void quantize_row_q6_K_ref(const float * GGML_RESTRICT x, block_q6_K * GGML_REST } void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2365,6 +2376,7 @@ size_t quantize_tq2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, } void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2404,6 +2416,7 @@ void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_REST } void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2425,6 +2438,7 @@ void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_REST // ====================== "True" 2-bit (de)-quantization void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2453,6 +2467,7 @@ void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_ // ====================== 2.3125 bpw (de)-quantization void dequantize_row_iq2_xs(const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2480,6 +2495,7 @@ void dequantize_row_iq2_xs(const block_iq2_xs * GGML_RESTRICT x, float * GGML_RE // ====================== 2.5625 bpw (de)-quantization void dequantize_row_iq2_s(const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2512,6 +2528,7 @@ void dequantize_row_iq2_s(const block_iq2_s * GGML_RESTRICT x, float * GGML_REST // ====================== 3.0625 bpw (de)-quantization void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2544,6 +2561,7 @@ void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_ // ====================== 3.3125 bpw (de)-quantization void dequantize_row_iq3_s(const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2587,6 +2605,7 @@ void dequantize_row_iq3_s(const block_iq3_s * GGML_RESTRICT x, float * GGML_REST // ====================== 1.5625 bpw (de)-quantization void dequantize_row_iq1_s(const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2612,6 +2631,7 @@ void dequantize_row_iq1_s(const block_iq1_s * GGML_RESTRICT x, float * GGML_REST } void dequantize_row_iq1_m(const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2662,6 +2682,7 @@ void dequantize_row_iq1_m(const block_iq1_m * GGML_RESTRICT x, float * GGML_REST } void dequantize_row_iq4_nl(const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK4_NL == 0); const int64_t nb = k / QK4_NL; @@ -2680,6 +2701,7 @@ void dequantize_row_iq4_nl(const block_iq4_nl * GGML_RESTRICT x, float * GGML_RE } void dequantize_row_iq4_xs(const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2744,6 +2766,7 @@ void quantize_row_q8_K_ref(const float * GGML_RESTRICT x, block_q8_K * GGML_REST } void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { + UNUSED(levels); assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -6250,7 +6273,7 @@ size_t quantize_q2_kpt(const float * GGML_RESTRICT src, size_t row_size = ggml_row_size(GGML_TYPE_Q2_KPT, n_per_row); char * qrow = (char *) dst; const int nb = (int)(n_per_row / QK_K); - const size_t total_levels = (size_t)nrow * nb * Q2KPT_N_LEVELS; + UNUSED((size_t)nrow * nb * Q2KPT_N_LEVELS); const size_t levels_needed = (size_t)(start_row + nrow) * nb * Q2KPT_N_LEVELS; // Ensure buffer is large enough (should have been pre-allocated via q2kpt_prepare_levels) @@ -6360,8 +6383,15 @@ static inline int iq2tq_nearest_qi(float xn, const int8_t * g) { // Dequantization — 2-bit with asymmetric grid per group void dequantize_row_iq2_tq(const block_iq2_tq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { +#ifndef _MSC_VER +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif const int8_t (*grid)[4] = levels ? (const int8_t (*)[4])levels : (const int8_t (*)[4])iq2tq_grid_default; +#ifndef _MSC_VER +#pragma GCC diagnostic pop +#endif const int nb = k / QK_K; for (int i = 0; i < nb; ++i) { @@ -6460,10 +6490,10 @@ static void quantize_row_iq2_tq_impl( int g = j / 8; float gval = (float)grid[grid_idx[g]][iq2tq_get_qi(qs, j)]; float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - sumxg += wk * xb[j] * gval; - sumgg += wk * gval * gval; + sumxg += (double)(wk * xb[j] * gval); + sumgg += (double)(wk * gval * gval); } - d = (sumgg > 0) ? (float)(sumxg / (IQ2TQ_GRID_SCALE * sumgg)) : d; + d = (sumgg > 0) ? (float)(sumxg / ((double)IQ2TQ_GRID_SCALE * sumgg)) : d; // Re-optimize grids and re-quantize dq = d * IQ2TQ_GRID_SCALE; @@ -6505,17 +6535,17 @@ static void quantize_row_iq2_tq_impl( int g = j / 8; float gval = (float)grid[grid_idx[g]][iq2tq_get_qi(qs, j)]; float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - sumxg += wk * xb[j] * gval; - sumgg += wk * gval * gval; + sumxg += (double)(wk * xb[j] * gval); + sumgg += (double)(wk * gval * gval); } - d = (sumgg > 0) ? (float)(sumxg / (IQ2TQ_GRID_SCALE * sumgg)) : d; + d = (sumgg > 0) ? (float)(sumxg / ((double)IQ2TQ_GRID_SCALE * sumgg)) : d; } // Multi-d search: try nearby d values and re-optimize grids { float best_d = d; float best_total_err = 1e30f; - uint8_t best_scales[16], best_qs[64]; + uint8_t best_scales[16] = {0}, best_qs[64] = {0}; int best_grid_idx[IQ2TQ_N_GROUPS]; static const float d_factors[] = {0.8f, 0.85f, 0.9f, 0.925f, 0.95f, 0.975f, 1.0f, 1.025f, 1.05f, 1.075f, 1.1f, 1.15f, 1.2f}; static const int n_d_factors = sizeof(d_factors) / sizeof(d_factors[0]); @@ -6577,10 +6607,10 @@ static void quantize_row_iq2_tq_impl( int g = j / 8; float gval = (float)grid[grid_idx[g]][iq2tq_get_qi(qs, j)]; float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - sumxg += wk * xb[j] * gval; - sumgg += wk * gval * gval; + sumxg += (double)(wk * xb[j] * gval); + sumgg += (double)(wk * gval * gval); } - d = (sumgg > 0) ? (float)(sumxg / (IQ2TQ_GRID_SCALE * sumgg)) : d; + d = (sumgg > 0) ? (float)(sumxg / ((double)IQ2TQ_GRID_SCALE * sumgg)) : d; dq = d * IQ2TQ_GRID_SCALE; inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; @@ -6737,8 +6767,8 @@ void iq2tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, for (int i = 0; i < n_sets; i++) { int c = assign[i]; float w = set_weights[i]; - for (int d = 0; d < 4; d++) sum[c][d] += w * sets[i][d]; - wcnt[c] += w; + for (int d = 0; d < 4; d++) sum[c][d] += (double)w * (double)sets[i][d]; + wcnt[c] += (double)w; } for (int c = 0; c < 16; c++) { if (wcnt[c] > 0) { @@ -6850,8 +6880,15 @@ static inline int iq3tq_nearest_qi(float xn, const int8_t * g) { // Dequantization void dequantize_row_iq3_tq(const block_iq3_tq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { +#ifndef _MSC_VER +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif const int8_t (*grid)[IQ3TQ_N_LEVELS] = levels ? (const int8_t (*)[IQ3TQ_N_LEVELS])levels : (const int8_t (*)[IQ3TQ_N_LEVELS])iq3tq_grid_default; +#ifndef _MSC_VER +#pragma GCC diagnostic pop +#endif const int nb = k / QK_K; for (int i = 0; i < nb; ++i) { @@ -6944,10 +6981,10 @@ static void quantize_row_iq3_tq_impl( int g = j / 8; float gval = (float)grid[grid_idx[g]][iq3tq_get_qi(qs, j)]; float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - sumxg += wk * xb[j] * gval; - sumgg += wk * gval * gval; + sumxg += (double)(wk * xb[j] * gval); + sumgg += (double)(wk * gval * gval); } - d = (sumgg > 0) ? (float)(sumxg / (IQ3TQ_GRID_SCALE * sumgg)) : d; + d = (sumgg > 0) ? (float)(sumxg / ((double)IQ3TQ_GRID_SCALE * sumgg)) : d; dq = d * IQ3TQ_GRID_SCALE; inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; @@ -6986,17 +7023,17 @@ static void quantize_row_iq3_tq_impl( int g = j / 8; float gval = (float)grid[grid_idx[g]][iq3tq_get_qi(qs, j)]; float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - sumxg += wk * xb[j] * gval; - sumgg += wk * gval * gval; + sumxg += (double)(wk * xb[j] * gval); + sumgg += (double)(wk * gval * gval); } - d = (sumgg > 0) ? (float)(sumxg / (IQ3TQ_GRID_SCALE * sumgg)) : d; + d = (sumgg > 0) ? (float)(sumxg / ((double)IQ3TQ_GRID_SCALE * sumgg)) : d; } // Multi-d search { float best_d = d; float best_total_err = 1e30f; - uint8_t best_scales[16], best_qs[96]; + uint8_t best_scales[16] = {0}, best_qs[96] = {0}; int best_grid_idx[IQ3TQ_N_GROUPS]; static const float d_factors[] = {0.8f, 0.85f, 0.9f, 0.925f, 0.95f, 0.975f, 1.0f, 1.025f, 1.05f, 1.075f, 1.1f, 1.15f, 1.2f}; static const int n_d_factors = sizeof(d_factors) / sizeof(d_factors[0]); @@ -7058,10 +7095,10 @@ static void quantize_row_iq3_tq_impl( int g = j / 8; float gval = (float)grid[grid_idx[g]][iq3tq_get_qi(qs, j)]; float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - sumxg += wk * xb[j] * gval; - sumgg += wk * gval * gval; + sumxg += (double)(wk * xb[j] * gval); + sumgg += (double)(wk * gval * gval); } - d = (sumgg > 0) ? (float)(sumxg / (IQ3TQ_GRID_SCALE * sumgg)) : d; + d = (sumgg > 0) ? (float)(sumxg / ((double)IQ3TQ_GRID_SCALE * sumgg)) : d; dq = d * IQ3TQ_GRID_SCALE; inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; @@ -7214,8 +7251,8 @@ void iq3tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, for (int i = 0; i < n_sets; i++) { int c = assign[i]; float w = set_weights[i]; - for (int dd = 0; dd < NL; dd++) sum[c][dd] += w * sets[i][dd]; - wcnt[c] += w; + for (int dd = 0; dd < NL; dd++) sum[c][dd] += (double)w * (double)sets[i][dd]; + wcnt[c] += (double)w; } for (int c = 0; c < 16; c++) { if (wcnt[c] > 0) { @@ -7597,7 +7634,9 @@ size_t quantize_iq1_bn(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst } // Thread work structs for parallel K-means +#ifndef _WIN32 #include +#endif // Worker for K-means++ min_dist update (parallel over samples) typedef struct { @@ -7764,10 +7803,24 @@ void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, // Full K-means++ initialization with parallel min_dist update if (nthread < 1) nthread = 1; if (nthread > n_samples) nthread = n_samples; + assert(nthread >= 1); + const size_t nth = (size_t)(unsigned)nthread; + const size_t kmpp_sz = nth * sizeof(iq1bn_kmpp_work_t); + +#ifndef _WIN32 + const size_t threads_sz = nth * sizeof(pthread_t); + pthread_t * threads = (pthread_t *)malloc(threads_sz); + assert(threads != NULL); +#else + nthread = 1; + void * threads = NULL; + const size_t threads_sz = 0; +#endif - // Set up thread pool for K-means++ min_dist update - pthread_t * threads = (pthread_t *)calloc(nthread, sizeof(pthread_t)); - iq1bn_kmpp_work_t * kmpp_workers = (iq1bn_kmpp_work_t *)calloc(nthread, sizeof(iq1bn_kmpp_work_t)); + iq1bn_kmpp_work_t * kmpp_workers = (iq1bn_kmpp_work_t *)malloc(kmpp_sz); + assert(kmpp_workers != NULL); + if (threads != NULL) memset(threads, 0, threads_sz); + memset(kmpp_workers, 0, kmpp_sz); for (int t = 0; t < nthread; ++t) { kmpp_workers[t].samples = samples; kmpp_workers[t].min_dist = min_dist; @@ -7787,12 +7840,18 @@ void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, kmpp_workers[t].centroid = cc; } if (nthread > 1) { +#ifndef _WIN32 for (int t = 0; t < nthread; ++t) { pthread_create(&threads[t], NULL, iq1bn_kmpp_worker, &kmpp_workers[t]); } for (int t = 0; t < nthread; ++t) { pthread_join(threads[t], NULL); } +#else + for (int t = 0; t < nthread; ++t) { + iq1bn_kmpp_worker(&kmpp_workers[t]); + } +#endif } else { iq1bn_kmpp_worker(&kmpp_workers[0]); } @@ -7818,7 +7877,7 @@ void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, // Phase 3: Full-batch K-means with SIMD and pthread parallelism // Allocate per-thread accumulators (reuse threads array from K-means++ init) - iq1bn_kmeans_work_t * workers = (iq1bn_kmeans_work_t *)calloc(nthread, sizeof(iq1bn_kmeans_work_t)); + iq1bn_kmeans_work_t * workers = (iq1bn_kmeans_work_t *)malloc(nth * sizeof(iq1bn_kmeans_work_t)); for (int t = 0; t < nthread; ++t) { workers[t].samples = samples; workers[t].weights = weights; @@ -7836,6 +7895,7 @@ void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, const int n_iters = 30; for (int iter = 0; iter < n_iters; ++iter) { // Launch threads for sample assignment +#ifndef _WIN32 for (int t = 0; t < nthread; ++t) { if (nthread > 1) { pthread_create(&threads[t], NULL, iq1bn_kmeans_worker, &workers[t]); @@ -7848,6 +7908,11 @@ void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, pthread_join(threads[t], NULL); } } +#else + for (int t = 0; t < nthread; ++t) { + iq1bn_kmeans_worker(&workers[t]); + } +#endif // Merge per-thread accumulators int changed = 0; diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index a480e93d58b6..f4cdec262054 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -1859,7 +1859,7 @@ static struct ggml_tensor * ggml_new_tensor_impl( /*.data =*/ obj_alloc_size > 0 ? (void *)(result + 1) : data, /*.name =*/ { 0 }, /*.extra =*/ NULL, - /*.padding =*/ { 0 }, + /*.padding =*/ 0 , }; // TODO: this should not be needed as long as we don't rely on aligned SIMD loads diff --git a/scripts/analyze-ffn-down.py b/scripts/analyze-ffn-down.py index 2f43089fac7f..0081798bbc86 100644 --- a/scripts/analyze-ffn-down.py +++ b/scripts/analyze-ffn-down.py @@ -5,7 +5,6 @@ import numpy as np import struct -import sys import os DATA_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data") @@ -23,29 +22,29 @@ def load_f32_tensor(name): def stats(label, arr): """Print comprehensive statistics for a flat array.""" a = arr.ravel() - print(f" {label}:") - print(f" shape={arr.shape}, n={len(a)}") - print(f" mean={a.mean():.6f}, std={a.std():.6f}") - print(f" min={a.min():.6f}, max={a.max():.6f}") - print(f" median={np.median(a):.6f}") - print( + print(f" {label}:") # noqa: NP100 + print(f" shape={arr.shape}, n={len(a)}") # noqa: NP100 + print(f" mean={a.mean():.6f}, std={a.std():.6f}") # noqa: NP100 + print(f" min={a.min():.6f}, max={a.max():.6f}") # noqa: NP100 + print(f" median={np.median(a):.6f}") # noqa: NP100 + print( # noqa: NP100 f" |mean|/std = {abs(a.mean()) / (a.std() + 1e-10):.4f} (offset-to-spread ratio)" ) # Kurtosis (excess) - how heavy-tailed vs Gaussian kurt = np.mean(((a - a.mean()) / (a.std() + 1e-10)) ** 4) - 3.0 # Skewness skew = np.mean(((a - a.mean()) / (a.std() + 1e-10)) ** 3) - print(f" skewness={skew:.4f}, excess_kurtosis={kurt:.4f}") + print(f" skewness={skew:.4f}, excess_kurtosis={kurt:.4f}") # noqa: NP100 # Percentile ranges pcts = np.percentile(a, [0.1, 1, 5, 25, 50, 75, 95, 99, 99.9]) - print( + print( # noqa: NP100 f" percentiles: 0.1%={pcts[0]:.4f}, 1%={pcts[1]:.4f}, 5%={pcts[2]:.4f}, " f"25%={pcts[3]:.4f}, 50%={pcts[4]:.4f}, 75%={pcts[5]:.4f}, " f"95%={pcts[6]:.4f}, 99%={pcts[7]:.4f}, 99.9%={pcts[8]:.4f}" ) # Sparsity near_zero = np.sum(np.abs(a) < 0.001 * a.std()) / len(a) - print(f" fraction |x| < 0.001*std: {near_zero:.4f}") + print(f" fraction |x| < 0.001*std: {near_zero:.4f}") # noqa: NP100 return { "mean": a.mean(), "std": a.std(), @@ -59,9 +58,9 @@ def stats(label, arr): # ============================================================================ # 1. BASIC WEIGHT TENSOR COMPARISON # ============================================================================ -print("=" * 80) -print("SECTION 1: WEIGHT TENSOR GLOBAL STATISTICS") -print("=" * 80) +print("=" * 80) # noqa: NP100 +print("SECTION 1: WEIGHT TENSOR GLOBAL STATISTICS") # noqa: NP100 +print("=" * 80) # noqa: NP100 tensors = { "ffn_gate": ("blk_0_ffn_gate_weight.f32bin", "9728x2560 (wide→narrow proj)"), @@ -77,51 +76,51 @@ def stats(label, arr): for name, (fname, desc) in tensors.items(): try: W = load_f32_tensor(fname) - print(f"\n{'─' * 70}") - print(f" {name} [{desc}] — file: {fname}") + print(f"\n{'─' * 70}") # noqa: NP100 + print(f" {name} [{desc}] — file: {fname}") # noqa: NP100 weight_data[name] = W stats(name, W) except Exception as e: - print(f" {name}: SKIP ({e})") + print(f" {name}: SKIP ({e})") # noqa: NP100 # ============================================================================ # 2. ROW-LEVEL STATISTICS (each row is a neuron output) # ============================================================================ -print("\n" + "=" * 80) -print("SECTION 2: ROW-LEVEL VARIABILITY (per-neuron weight statistics)") -print("=" * 80) -print(" Each row of the weight matrix produces one output dimension.") -print(" High row-to-row variability in mean/std means the quantizer") -print(" must handle very different distributions across rows.\n") +print("\n" + "=" * 80) # noqa: NP100 +print("SECTION 2: ROW-LEVEL VARIABILITY (per-neuron weight statistics)") # noqa: NP100 +print("=" * 80) # noqa: NP100 +print(" Each row of the weight matrix produces one output dimension.") # noqa: NP100 +print(" High row-to-row variability in mean/std means the quantizer") # noqa: NP100 +print(" must handle very different distributions across rows.\n") # noqa: NP100 for name, W in weight_data.items(): row_means = W.mean(axis=1) row_stds = W.std(axis=1) row_ranges = W.max(axis=1) - W.min(axis=1) - print(f"\n {name} ({W.shape[0]} rows × {W.shape[1]} cols):") - print( + print(f"\n {name} ({W.shape[0]} rows × {W.shape[1]} cols):") # noqa: NP100 + print( # noqa: NP100 f" Row means: mean={row_means.mean():.6f}, std={row_means.std():.6f}, " f"range=[{row_means.min():.6f}, {row_means.max():.6f}]" ) - print( + print( # noqa: NP100 f" Row stds: mean={row_stds.mean():.6f}, std={row_stds.std():.6f}, " f"range=[{row_stds.min():.6f}, {row_stds.max():.6f}]" ) - print(f" Row ranges: mean={row_ranges.mean():.6f}, std={row_ranges.std():.6f}") - print( + print(f" Row ranges: mean={row_ranges.mean():.6f}, std={row_ranges.std():.6f}") # noqa: NP100 + print( # noqa: NP100 f" RowMeans CV (std/mean): {row_means.std() / (abs(row_means.mean()) + 1e-10):.4f}" ) - print(f" RowStds CV: {row_stds.std() / (row_stds.mean() + 1e-10):.4f}") + print(f" RowStds CV: {row_stds.std() / (row_stds.mean() + 1e-10):.4f}") # noqa: NP100 # ============================================================================ # 3. GROUP-LEVEL ANALYSIS (16-element groups, like Q2_K) # ============================================================================ -print("\n" + "=" * 80) -print("SECTION 3: GROUP-LEVEL ANALYSIS (16-element groups)") -print("=" * 80) -print(" Quantization works on 16-element groups. Key question:") -print(" How much does each group need its own OFFSET (dmin)?\n") +print("\n" + "=" * 80) # noqa: NP100 +print("SECTION 3: GROUP-LEVEL ANALYSIS (16-element groups)") # noqa: NP100 +print("=" * 80) # noqa: NP100 +print(" Quantization works on 16-element groups. Key question:") # noqa: NP100 +print(" How much does each group need its own OFFSET (dmin)?\n") # noqa: NP100 GS = 16 @@ -158,21 +157,21 @@ def stats(label, arr): gr = np.array(group_ranges) go = np.array(group_offsets) - print(f"\n {name} ({len(group_means)} groups):") - print( + print(f"\n {name} ({len(group_means)} groups):") # noqa: NP100 + print( # noqa: NP100 f" Group mean: mean={gm.mean():.6f}, std={gm.std():.6f}, " f"range=[{gm.min():.6f}, {gm.max():.6f}]" ) - print(f" Group std: mean={gs.mean():.6f}, std={gs.std():.6f}") - print(f" Group range: mean={gr.mean():.6f}, std={gr.std():.6f}") - print(f" *** OFFSET IMPORTANCE (|group_mean| / range) ***") - print( + print(f" Group std: mean={gs.mean():.6f}, std={gs.std():.6f}") # noqa: NP100 + print(f" Group range: mean={gr.mean():.6f}, std={gr.std():.6f}") # noqa: NP100 + print(" *** OFFSET IMPORTANCE (|group_mean| / range) ***") # noqa: NP100 + print( # noqa: NP100 f" mean={go.mean():.4f}, median={np.median(go):.4f}, " f"p90={np.percentile(go, 90):.4f}, max={go.max():.4f}" ) - print(f" fraction with offset > 0.1: {np.mean(go > 0.1):.3f}") - print(f" fraction with offset > 0.2: {np.mean(go > 0.2):.3f}") - print(f" fraction with offset > 0.3: {np.mean(go > 0.3):.3f}") + print(f" fraction with offset > 0.1: {np.mean(go > 0.1):.3f}") # noqa: NP100 + print(f" fraction with offset > 0.2: {np.mean(go > 0.2):.3f}") # noqa: NP100 + print(f" fraction with offset > 0.3: {np.mean(go > 0.3):.3f}") # noqa: NP100 # How well does zeroing the min (Q2_K style, clamping min to 0) work? # vs keeping the actual min @@ -218,16 +217,16 @@ def stats(label, arr): rmse_no = np.sqrt(mse_no_offset / total_elements) rmse_w = np.sqrt(mse_with_offset / total_elements) improvement = (rmse_no - rmse_w) / rmse_no * 100 - print(f" Quant RMSE (no offset): {rmse_no:.6f}") - print(f" Quant RMSE (with offset): {rmse_w:.6f}") - print(f" Offset benefit: {improvement:.1f}% RMSE reduction") + print(f" Quant RMSE (no offset): {rmse_no:.6f}") # noqa: NP100 + print(f" Quant RMSE (with offset): {rmse_w:.6f}") # noqa: NP100 + print(f" Offset benefit: {improvement:.1f}% RMSE reduction") # noqa: NP100 # ============================================================================ # 4. ACTIVATION ANALYSIS # ============================================================================ -print("\n" + "=" * 80) -print("SECTION 4: ACTIVATION DISTRIBUTION COMPARISON") -print("=" * 80) +print("\n" + "=" * 80) # noqa: NP100 +print("SECTION 4: ACTIVATION DISTRIBUTION COMPARISON") # noqa: NP100 +print("=" * 80) # noqa: NP100 activations = { "ffn_input (gate/up)": "act_blk0_ffn_input.f32bin", @@ -241,37 +240,37 @@ def stats(label, arr): try: A = load_f32_tensor(fname) act_data[name] = A - print(f"\n{'─' * 70}") - print(f" {name} — {fname}") + print(f"\n{'─' * 70}") # noqa: NP100 + print(f" {name} — {fname}") # noqa: NP100 stats(name, A) except Exception as e: - print(f" {name}: SKIP ({e})") + print(f" {name}: SKIP ({e})") # noqa: NP100 # ============================================================================ # 5. THE CRITICAL QUESTION: PER-DIMENSION ACTIVATION MAGNITUDE # ============================================================================ -print("\n" + "=" * 80) -print("SECTION 5: PER-DIMENSION ACTIVATION POWER (per-column RMS)") -print("=" * 80) -print(" If activation dimensions have very different magnitudes,") -print(" the quantization error in each weight dimension is weighted differently.") -print(" Dimensions with high activation power amplify weight errors.\n") +print("\n" + "=" * 80) # noqa: NP100 +print("SECTION 5: PER-DIMENSION ACTIVATION POWER (per-column RMS)") # noqa: NP100 +print("=" * 80) # noqa: NP100 +print(" If activation dimensions have very different magnitudes,") # noqa: NP100 +print(" the quantization error in each weight dimension is weighted differently.") # noqa: NP100 +print(" Dimensions with high activation power amplify weight errors.\n") # noqa: NP100 for name, A in act_data.items(): col_rms = np.sqrt(np.mean(A**2, axis=0)) # RMS per column (dimension) - print(f"\n {name} ({A.shape[1]} dimensions):") - print(f" Col RMS: mean={col_rms.mean():.6f}, std={col_rms.std():.6f}") - print(f" Col RMS range: [{col_rms.min():.6f}, {col_rms.max():.6f}]") - print(f" Col RMS CV (std/mean): {col_rms.std() / (col_rms.mean() + 1e-10):.4f}") - print(f" Max/Min ratio: {col_rms.max() / (col_rms.min() + 1e-10):.1f}x") + print(f"\n {name} ({A.shape[1]} dimensions):") # noqa: NP100 + print(f" Col RMS: mean={col_rms.mean():.6f}, std={col_rms.std():.6f}") # noqa: NP100 + print(f" Col RMS range: [{col_rms.min():.6f}, {col_rms.max():.6f}]") # noqa: NP100 + print(f" Col RMS CV (std/mean): {col_rms.std() / (col_rms.mean() + 1e-10):.4f}") # noqa: NP100 + print(f" Max/Min ratio: {col_rms.max() / (col_rms.min() + 1e-10):.1f}x") # noqa: NP100 # Top 10 and bottom 10 dimensions by power top10 = np.argsort(col_rms)[-10:][::-1] bot10 = np.argsort(col_rms)[:10] - print( + print( # noqa: NP100 f" Top-10 dims by RMS: {[(int(d), f'{col_rms[d]:.4f}') for d in top10[:5]]}..." ) - print( + print( # noqa: NP100 f" Bot-10 dims by RMS: {[(int(d), f'{col_rms[d]:.4f}') for d in bot10[:5]]}..." ) @@ -282,21 +281,21 @@ def stats(label, arr): top10pct_power = np.sum(sorted_power[:top10pct]) top1pct = max(1, int(len(col_rms) * 0.01)) top1pct_power = np.sum(sorted_power[:top1pct]) - print( + print( # noqa: NP100 f" Top 10% of dims contribute {top10pct_power / total_power * 100:.1f}% of total power" ) - print( + print( # noqa: NP100 f" Top 1% of dims contribute {top1pct_power / total_power * 100:.1f}% of total power" ) # ============================================================================ # 6. CROSS-CORRELATION: WEIGHT ERROR × ACTIVATION POWER # ============================================================================ -print("\n" + "=" * 80) -print("SECTION 6: WHERE DO WEIGHT ERRORS MEET HIGH ACTIVATION POWER?") -print("=" * 80) -print(" For each weight dimension, compute: activation_rms[dim] × weight_error[dim]") -print(" This tells us which dimensions contribute most to matmul error.\n") +print("\n" + "=" * 80) # noqa: NP100 +print("SECTION 6: WHERE DO WEIGHT ERRORS MEET HIGH ACTIVATION POWER?") # noqa: NP100 +print("=" * 80) # noqa: NP100 +print(" For each weight dimension, compute: activation_rms[dim] × weight_error[dim]") # noqa: NP100 +print(" This tells us which dimensions contribute most to matmul error.\n") # noqa: NP100 # Focus on ffn_down vs ffn_gate for comparison focus = [ @@ -311,7 +310,7 @@ def stats(label, arr): A = load_f32_tensor(afile) if W.shape[1] != A.shape[1]: - print(f" {name}: dim mismatch W={W.shape[1]} vs A={A.shape[1]}, SKIP") + print(f" {name}: dim mismatch W={W.shape[1]} vs A={A.shape[1]}, SKIP") # noqa: NP100 continue nc = W.shape[1] @@ -357,28 +356,28 @@ def stats(label, arr): # matmul_error_contribution[d] ≈ act_rms[d] * weight_rmse[d] matmul_contrib = act_rms * dim_rmse - print(f"\n {name} ({nc} dimensions):") - print( + print(f"\n {name} ({nc} dimensions):") # noqa: NP100 + print( # noqa: NP100 f" act_rms: mean={act_rms.mean():.4f}, CV={act_rms.std() / act_rms.mean():.4f}" ) - print( + print( # noqa: NP100 f" w_rmse: mean={dim_rmse.mean():.6f}, CV={dim_rmse.std() / (dim_rmse.mean() + 1e-10):.4f}" ) - print( + print( # noqa: NP100 f" matmul_contrib: mean={matmul_contrib.mean():.6f}, " f"std={matmul_contrib.std():.6f}" ) # Correlation between activation power and weight error corr = np.corrcoef(act_rms, dim_rmse)[0, 1] - print(f" CORRELATION act_rms ↔ weight_rmse: {corr:.4f}") - print(f" (>0 means high-power dims are also hard to quantize — BAD)") + print(f" CORRELATION act_rms ↔ weight_rmse: {corr:.4f}") # noqa: NP100 + print(" (>0 means high-power dims are also hard to quantize — BAD)") # noqa: NP100 # Top contributors to matmul error top_dims = np.argsort(matmul_contrib)[-20:][::-1] - print(f" Top-5 error-contributing dimensions:") + print(" Top-5 error-contributing dimensions:") # noqa: NP100 for d in top_dims[:5]: - print( + print( # noqa: NP100 f" dim {d}: act_rms={act_rms[d]:.4f}, w_rmse={dim_rmse[d]:.6f}, " f"contrib={matmul_contrib[d]:.6f}, w_std={w_std[d]:.6f}, w_kurt={w_kurt[d]:.2f}" ) @@ -388,7 +387,7 @@ def stats(label, arr): sorted_contrib = np.sort(matmul_contrib)[::-1] for pct in [0.01, 0.05, 0.10, 0.25]: n = max(1, int(nc * pct)) - print( + print( # noqa: NP100 f" Top {pct * 100:.0f}% dims: {sorted_contrib[:n].sum() / total_contrib * 100:.1f}% " f"of total matmul error" ) @@ -396,22 +395,22 @@ def stats(label, arr): # ============================================================================ # 7. THE STRUCTURAL ASYMMETRY: COLUMN DIRECTION GROUP ANALYSIS # ============================================================================ -print("\n" + "=" * 80) -print("SECTION 7: STRUCTURAL ASYMMETRY — COLUMN vs ROW GROUPING") -print("=" * 80) -print(" Quantization groups along the ROW (inner dim). For ffn_down,") -print(" each row has 9728 elements (38 groups of 256).") -print(" For ffn_gate, each row has 2560 elements (10 groups of 256).") -print(" More groups = more metadata (scales/offsets) relative to data bits.\n") +print("\n" + "=" * 80) # noqa: NP100 +print("SECTION 7: STRUCTURAL ASYMMETRY — COLUMN vs ROW GROUPING") # noqa: NP100 +print("=" * 80) # noqa: NP100 +print(" Quantization groups along the ROW (inner dim). For ffn_down,") # noqa: NP100 +print(" each row has 9728 elements (38 groups of 256).") # noqa: NP100 +print(" For ffn_gate, each row has 2560 elements (10 groups of 256).") # noqa: NP100 +print(" More groups = more metadata (scales/offsets) relative to data bits.\n") # noqa: NP100 for name, wfile, afile in focus: W = load_f32_tensor(wfile) nc = W.shape[1] n_groups_per_row = nc // 256 # super-blocks per row - print(f"\n {name}: {nc} cols → {n_groups_per_row} super-blocks per row") - print(f" Groups per row: {nc // GS} (16-element groups)") - print( + print(f"\n {name}: {nc} cols → {n_groups_per_row} super-blocks per row") # noqa: NP100 + print(f" Groups per row: {nc // GS} (16-element groups)") # noqa: NP100 + print( # noqa: NP100 f" With Q2_K (2.625 bpw): {n_groups_per_row * 2} scale+offset bytes per row" ) @@ -425,7 +424,7 @@ def stats(label, arr): group_means = np.array(group_means) intra_row_mean_var.append(group_means.std()) - print( + print( # noqa: NP100 f" Intra-row group mean variability (avg across rows): " f"mean={np.mean(intra_row_mean_var):.6f}" ) @@ -442,7 +441,7 @@ def stats(label, arr): elif gm < -0.001: neg_frac += 1 total_groups += 1 - print( + print( # noqa: NP100 f" Group mean sign: {pos_frac / total_groups * 100:.1f}% positive, " f"{neg_frac / total_groups * 100:.1f}% negative, " f"{(1 - pos_frac / total_groups - neg_frac / total_groups) * 100:.1f}% near-zero" @@ -451,30 +450,30 @@ def stats(label, arr): # ============================================================================ # 8. THE SWIGLU EFFECT: WHY ffn_down INPUT IS SPECIAL # ============================================================================ -print("\n" + "=" * 80) -print("SECTION 8: THE SWIGLU EFFECT — ffn_down ACTIVATION STRUCTURE") -print("=" * 80) -print(" ffn_down's activation is the SwiGLU output: silu(gate) * up") -print(" This creates a specific activation pattern that differs from") -print(" raw FFN input (RMSNorm output).\n") +print("\n" + "=" * 80) # noqa: NP100 +print("SECTION 8: THE SWIGLU EFFECT — ffn_down ACTIVATION STRUCTURE") # noqa: NP100 +print("=" * 80) # noqa: NP100 +print(" ffn_down's activation is the SwiGLU output: silu(gate) * up") # noqa: NP100 +print(" This creates a specific activation pattern that differs from") # noqa: NP100 +print(" raw FFN input (RMSNorm output).\n") # noqa: NP100 if "ffn_input (gate/up)" in act_data and "ffn_down_input (swiglu)" in act_data: A_in = act_data["ffn_input (gate/up)"] A_swiglu = act_data["ffn_down_input (swiglu)"] - print(f" FFN input (RMSNorm output): {A_in.shape}") - print(f" SwiGLU output: {A_swiglu.shape}") + print(f" FFN input (RMSNorm output): {A_in.shape}") # noqa: NP100 + print(f" SwiGLU output: {A_swiglu.shape}") # noqa: NP100 # Per-token analysis for t in range(min(A_swiglu.shape[0], 3)): tok_in = A_in[t] tok_sw = A_swiglu[t] - print(f"\n Token {t}:") - print( + print(f"\n Token {t}:") # noqa: NP100 + print( # noqa: NP100 f" FFN input: mean={tok_in.mean():.6f}, std={tok_in.std():.6f}, " f"|max|={np.abs(tok_in).max():.6f}" ) - print( + print( # noqa: NP100 f" SwiGLU out: mean={tok_sw.mean():.6f}, std={tok_sw.std():.6f}, " f"|max|={np.abs(tok_sw).max():.6f}" ) @@ -482,24 +481,24 @@ def stats(label, arr): # SwiGLU creates lots of near-zero values (silu suppresses negatives) frac_nearzero_sw = np.mean(np.abs(tok_sw) < 0.01 * tok_sw.std()) frac_nearzero_in = np.mean(np.abs(tok_in) < 0.01 * tok_in.std()) - print( + print( # noqa: NP100 f" Near-zero fraction: FFN input={frac_nearzero_in:.3f}, " f"SwiGLU={frac_nearzero_sw:.3f}" ) # Sparsity pattern frac_neg = np.mean(tok_sw < 0) - print(f" SwiGLU negative fraction: {frac_neg:.3f}") + print(f" SwiGLU negative fraction: {frac_neg:.3f}") # noqa: NP100 # Dimension-level analysis of SwiGLU - print(f"\n Dimension-level SwiGLU properties:") + print("\n Dimension-level SwiGLU properties:") # noqa: NP100 dim_mean_sw = A_swiglu.mean(axis=0) dim_std_sw = A_swiglu.std(axis=0) dim_sparsity = np.mean(A_swiglu < 0, axis=0) # fraction of tokens negative per dim - print(f" Dim mean range: [{dim_mean_sw.min():.6f}, {dim_mean_sw.max():.6f}]") - print(f" Dim std range: [{dim_std_sw.min():.6f}, {dim_std_sw.max():.6f}]") - print( + print(f" Dim mean range: [{dim_mean_sw.min():.6f}, {dim_mean_sw.max():.6f}]") # noqa: NP100 + print(f" Dim std range: [{dim_std_sw.min():.6f}, {dim_std_sw.max():.6f}]") # noqa: NP100 + print( # noqa: NP100 f" Dim negative fraction: mean={dim_sparsity.mean():.3f}, " f"range=[{dim_sparsity.min():.3f}, {dim_sparsity.max():.3f}]" ) @@ -507,20 +506,20 @@ def stats(label, arr): # Highly sparse dimensions (mostly near-zero after SwiGLU) high_sparsity = np.sum(dim_sparsity > 0.7) low_sparsity = np.sum(dim_sparsity < 0.3) - print(f" Dims with >70% negative tokens: {high_sparsity}/{len(dim_sparsity)}") - print(f" Dims with <30% negative tokens: {low_sparsity}/{len(dim_sparsity)}") + print(f" Dims with >70% negative tokens: {high_sparsity}/{len(dim_sparsity)}") # noqa: NP100 + print(f" Dims with <30% negative tokens: {low_sparsity}/{len(dim_sparsity)}") # noqa: NP100 # ============================================================================ # 9. QUANTIZATION NOISE × ACTIVATION POWER: THE MATMUL ERROR DECOMPOSITION # ============================================================================ -print("\n" + "=" * 80) -print("SECTION 9: MATMUL ERROR DECOMPOSITION") -print("=" * 80) -print( +print("\n" + "=" * 80) # noqa: NP100 +print("SECTION 9: MATMUL ERROR DECOMPOSITION") # noqa: NP100 +print("=" * 80) # noqa: NP100 +print( # noqa: NP100 " matmul_error ≈ sum over groups of (activation_power_in_group × " "weight_mse_in_group)" ) -print( +print( # noqa: NP100 " If activation power is concentrated in groups with high weight error, " "matmul error explodes.\n" ) @@ -576,29 +575,29 @@ def stats(label, arr): else: corr = 0 - print(f"\n {label}:") - print(f" Super-blocks: {n_sb}") - print( + print(f"\n {label}:") # noqa: NP100 + print(f" Super-blocks: {n_sb}") # noqa: NP100 + print( # noqa: NP100 f" act_power: mean={sb_act_power.mean():.6f}, " f"std={np.sqrt(sb_act_power.var()):.6f}, " f"range=[{sb_act_power.min():.6f}, {sb_act_power.max():.6f}]" ) - print( + print( # noqa: NP100 f" weight_mse: mean={sb_weight_mse.mean():.6f}, " f"range=[{sb_weight_mse.min():.6f}, {sb_weight_mse.max():.6f}]" ) - print(f" CORRELATION (act_power ↔ weight_mse): {corr:.4f}") + print(f" CORRELATION (act_power ↔ weight_mse): {corr:.4f}") # noqa: NP100 # Show top-5 super-blocks by contribution to matmul error contrib = sb_act_power * sb_weight_mse top5 = np.argsort(contrib)[-5:][::-1] - print(f" Top-5 error-contributing super-blocks (of {n_sb}):") + print(f" Top-5 error-contributing super-blocks (of {n_sb}):") # noqa: NP100 for idx in top5: - print( + print( # noqa: NP100 f" SB {idx * 256}-{(idx + 1) * 256 - 1}: act_power={sb_act_power[idx]:.6f}, " f"weight_mse={sb_weight_mse[idx]:.6f}, contrib={contrib[idx]:.6f}" ) -print("\n" + "=" * 80) -print("ANALYSIS COMPLETE") -print("=" * 80) +print("\n" + "=" * 80) # noqa: NP100 +print("ANALYSIS COMPLETE") # noqa: NP100 +print("=" * 80) # noqa: NP100 diff --git a/scripts/compute-imatrix.py b/scripts/compute-imatrix.py index 0b8d394d1b71..fec059a382a8 100644 --- a/scripts/compute-imatrix.py +++ b/scripts/compute-imatrix.py @@ -29,7 +29,7 @@ def load_f32_tensor(name): def save_imatrix(name, data): path = os.path.join(DATA_DIR, name) data.astype(np.float32).tofile(path) - print( + print( # noqa: NP100 f" Wrote {path}: {len(data)} dims, " f"min={data.min():.6f}, max={data.max():.6f}, mean={data.mean():.6f}" ) @@ -60,14 +60,14 @@ def save_imatrix(name, data): }, ] -print("Computing imatrix from captured activations") -print("=" * 60) +print("Computing imatrix from captured activations") # noqa: NP100 +print("=" * 60) # noqa: NP100 for m in mappings: try: A = load_f32_tensor(m["act_file"]) - print(f"\n{m['description']}:") - print(f" Activation: {A.shape[0]} tokens × {A.shape[1]} dims") + print(f"\n{m['description']}:") # noqa: NP100 + print(f" Activation: {A.shape[0]} tokens × {A.shape[1]} dims") # noqa: NP100 # imatrix = sum over tokens of activation^2 # This is the standard definition used by llama.cpp @@ -76,11 +76,11 @@ def save_imatrix(name, data): # Also compute per-dim RMS for reference rms = np.sqrt(np.mean(A**2, axis=0)) - print( + print( # noqa: NP100 f" Imatrix stats: min={imatrix.min():.6f}, max={imatrix.max():.6f}, " f"mean={imatrix.mean():.6f}, std={imatrix.std():.6f}" ) - print( + print( # noqa: NP100 f" RMS stats: min={rms.min():.6f}, max={rms.max():.6f}, " f"mean={rms.mean():.6f}" ) @@ -90,16 +90,16 @@ def save_imatrix(name, data): sorted_im = np.sort(imatrix)[::-1] top1pct = max(1, int(len(imatrix) * 0.01)) top10pct = max(1, int(len(imatrix) * 0.10)) - print(f" Power concentration:") - print( + print(" Power concentration:") # noqa: NP100 + print( # noqa: NP100 f" Top 1% dims ({top1pct}): {sorted_im[:top1pct].sum() / total * 100:.1f}% of total" ) - print( + print( # noqa: NP100 f" Top 10% dims ({top10pct}): {sorted_im[:top10pct].sum() / total * 100:.1f}% of total" ) save_imatrix(m["imatrix_name"], imatrix) except Exception as e: - print(f" SKIP: {e}") + print(f" SKIP: {e}") # noqa: NP100 -print("\nDone.") +print("\nDone.") # noqa: NP100 diff --git a/scripts/extract-activations.py b/scripts/extract-activations.py index bc7d2faf1bea..0211e3b45fc4 100644 --- a/scripts/extract-activations.py +++ b/scripts/extract-activations.py @@ -26,7 +26,7 @@ repo_root = os.path.dirname(script_dir) sys.path.insert(0, os.path.join(repo_root, 'gguf-py')) -from gguf import GGUFReader +from gguf import GGUFReader # noqa: E402 def bf16_to_f32(raw_bytes): @@ -56,7 +56,7 @@ def softmax(x, axis=-1): def main(): if len(sys.argv) < 3: - print(f"Usage: {sys.argv[0]} MODEL.gguf OUTPUT_DIR [--prompt TEXT] [--layer N]") + print(f"Usage: {sys.argv[0]} MODEL.gguf OUTPUT_DIR [--prompt TEXT] [--layer N]") # noqa: NP100 sys.exit(1) model_path = sys.argv[1] @@ -72,7 +72,7 @@ def main(): os.makedirs(output_dir, exist_ok=True) - print(f"Loading {model_path}...") + print(f"Loading {model_path}...") # noqa: NP100 reader = GGUFReader(model_path) # Read model config from metadata @@ -95,8 +95,8 @@ def main(): elif 'layer_norm_rms_epsilon' in name: config['eps'] = float(kv.parts[-1][0]) - print(f"Config: {config}") - hidden = config['hidden'] + print(f"Config: {config}") # noqa: NP100 + config.get('hidden') # Load tensors into a dict def load_tensor(name): @@ -122,19 +122,19 @@ def load_tensor(name): # Create simple token IDs from the prompt (use first few tokens from vocab) # We just need realistic activations, not perfect tokenization n_tokens = min(32, len(prompt_text.split())) - print(f"Using {n_tokens} pseudo-tokens for activation extraction") + print(f"Using {n_tokens} pseudo-tokens for activation extraction") # noqa: NP100 # Load token embedding and create input - print("Loading token_embd...") + print("Loading token_embd...") # noqa: NP100 token_embd = load_tensor("token_embd.weight") # [vocab, hidden] # Use token IDs 100-131 (arbitrary but avoids special tokens) token_ids = list(range(100, 100 + n_tokens)) x = token_embd[token_ids] # [n_tokens, hidden] - print(f"Input shape: {x.shape}") + print(f"Input shape: {x.shape}") # noqa: NP100 # Run forward pass through target layer only (we just need the activations) layer = target_layer - print(f"\nProcessing layer {layer}...") + print(f"\nProcessing layer {layer}...") # noqa: NP100 def save_activation(name, data): """Save activation tensor as f32bin.""" @@ -145,7 +145,7 @@ def save_activation(name, data): with open(fname, 'wb') as fp: fp.write(struct.pack('type != GGML_TYPE_Q4_DPT) { continue; } + + const std::string levels_key = "q4dpt.levels." + tname; + int64_t levels_idx = gguf_find_key(ml.metadata, levels_key.c_str()); + if (levels_idx < 0) { + // backward compat: try old flat-array key + int64_t flat_idx = gguf_find_key(ml.metadata, "q4_dpt.levels"); + if (flat_idx < 0) { continue; } + const int8_t * flat_data = (const int8_t *)gguf_get_arr_data(ml.metadata, flat_idx); + const size_t flat_n = gguf_get_arr_n(ml.metadata, flat_idx); + + size_t gguf_slot = 0; + for (size_t s = 0; s < (size_t)ml.n_tensors; ++s) { + std::string sname = gguf_get_tensor_name(ml.metadata, s); + if (sname == tname) { gguf_slot = s; break; } + } + if (gguf_slot * 16 + 16 > flat_n) { continue; } + + auto & taux = tensor_aux_data[t]; + taux.type = GGML_TYPE_Q4_DPT; + taux.host_data.resize(16); + memcpy(taux.host_data.data(), flat_data + gguf_slot * 16, 16); + t->quant_levels = taux.host_data.data(); + q4dpt_loaded++; + continue; + } + + auto & taux = tensor_aux_data[t]; + taux.type = GGML_TYPE_Q4_DPT; + taux.host_data.resize(16); + const int8_t * levels_data = (const int8_t *)gguf_get_arr_data(ml.metadata, levels_idx); + memcpy(taux.host_data.data(), levels_data, 16); + + t->quant_levels = taux.host_data.data(); + q4dpt_loaded++; + } + if (q4dpt_loaded > 0) { + LLAMA_LOG_INFO("%s: loaded Q4_DPT levels for %zu tensors\n", __func__, q4dpt_loaded); + } + } + // IQ1_BN: per-tensor trained codebook (32768 bytes) { size_t iq1bn_loaded = 0; diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 9614bd2f29ae..195f984d626b 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -772,6 +772,42 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type return new_type; } +// resolve the effective quantization type for a tensor, including tt_overrides +static ggml_type resolve_tensor_type(quantize_state_impl & qs, const llama_model_quantize_params * params, + const ggml_tensor * tensor, ggml_type default_type, llama_ftype ftype, + const std::string & tname) { + ggml_type new_type = default_type; + if (!params->pure) { + new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); + } + if (params->token_embedding_type < GGML_TYPE_COUNT && + (tname == "token_embd.weight" || tname == "per_layer_token_embd.weight")) { + new_type = params->token_embedding_type; + } + if (params->output_tensor_type < GGML_TYPE_COUNT && tname == "output.weight") { + new_type = params->output_tensor_type; + } + if (!qs.tensor_type_patterns.empty()) { + for (const auto & [pattern, qtype] : qs.tensor_type_patterns) { + if (std::regex_search(tname, pattern)) { + new_type = qtype; + break; + } + } + } + return new_type; +} + +// check if any tt_override pattern targets the given ggml_type +static bool has_override_for_type(const quantize_state_impl & qs, ggml_type target_type) { + for (const auto & [pattern, qtype] : qs.tensor_type_patterns) { + if (qtype == target_type) { + return true; + } + } + return false; +} + // outer wrapper: determine the ggml_type that this tensor should be quantized to static ggml_type llama_tensor_get_type(quantize_state_impl & qs, const llama_model_quantize_params * params, const ggml_tensor * tensor, ggml_type default_type, const tensor_metadata & tm) { if (!tensor_allows_quantization(params, qs.model.arch, tensor)) { @@ -1389,10 +1425,14 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: // Q4_DPT two-pass approach: train all per-tensor int8 levels BEFORE opening the output // file, so the levels KV entry is already populated at the time of the metadata placeholder. static const size_t Q4DPT_N_LEVELS = 16; - std::vector q4dpt_all_levels; // indexed by position in tensors[] - if (ftype == LLAMA_FTYPE_MOSTLY_Q4_DPT && !params->dry_run) { + struct q4dpt_meta { + std::string tensor_name; + int8_t levels[Q4DPT_N_LEVELS]; + }; + std::vector q4dpt_all_meta; + if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_DPT || has_override_for_type(qs, GGML_TYPE_Q4_DPT)) && !params->dry_run) { + const int64_t t_start_p1 = ggml_time_us(); LLAMA_LOG_INFO("%s: Q4_DPT pass 1: training per-tensor int8 levels...\n", __func__); - q4dpt_all_levels.assign(tensors.size() * Q4DPT_N_LEVELS, (int8_t)0); std::vector> p1_read_data; std::vector> p1_f32_buf; @@ -1409,17 +1449,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; if (!quantize) { continue; } - ggml_type new_type = default_type; - if (!params->pure) { - new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); - } - if (params->token_embedding_type < GGML_TYPE_COUNT && - (tname == "token_embd.weight" || tname == "per_layer_token_embd.weight")) { - new_type = params->token_embedding_type; - } - if (params->output_tensor_type < GGML_TYPE_COUNT && tname == "output.weight") { - new_type = params->output_tensor_type; - } + ggml_type new_type = resolve_tensor_type(qs, params, tensor, default_type, ftype, tname); if (new_type != GGML_TYPE_Q4_DPT) { continue; } // Load tensor data @@ -1454,18 +1484,19 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: const int64_t nrows = tensor->ne[1]; LLAMA_LOG_INFO("%s: Q4_DPT levels for [%zu/%zu] %s\n", __func__, ti+1, tensors.size(), tensor->name); - q4dpt_train_levels(f32_data, nrows, n_per_row, imatrix, - q4dpt_all_levels.data() + ti * Q4DPT_N_LEVELS); - } - // Store in GGUF metadata before the file is opened - for (auto & ctx : ctx_outs) { - if (ctx) { - gguf_set_arr_data(ctx.get(), "q4_dpt.levels", GGUF_TYPE_INT8, - q4dpt_all_levels.data(), q4dpt_all_levels.size()); - } + q4dpt_meta meta; + meta.tensor_name = tname; + q4dpt_train_levels(f32_data, nrows, n_per_row, imatrix, meta.levels); + q4dpt_all_meta.push_back(meta); + + // Save to GGUF + std::string levels_key = "q4dpt.levels." + tname; + gguf_set_arr_data(ctx_outs[0].get(), levels_key.c_str(), GGUF_TYPE_INT8, meta.levels, Q4DPT_N_LEVELS); } - LLAMA_LOG_INFO("%s: Q4_DPT pass 1 complete.\n", __func__); + const int64_t t_end_p1 = ggml_time_us(); + LLAMA_LOG_INFO("%s: Q4_DPT pass 1 complete (%zu tensors trained, %.1f s).\n", + __func__, q4dpt_all_meta.size(), (t_end_p1 - t_start_p1) / 1e6); } // Q2_KPT two-pass approach: train all per-block levels BEFORE opening the output @@ -1593,7 +1624,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: int8_t grid[64]; }; std::vector iq2tq_all_meta; - if (params->ftype == LLAMA_FTYPE_MOSTLY_IQ2_TQ) { + if (params->ftype == LLAMA_FTYPE_MOSTLY_IQ2_TQ || has_override_for_type(qs, GGML_TYPE_IQ2_TQ)) { const int64_t t_start_p1 = ggml_time_us(); LLAMA_LOG_INFO("%s: IQ2_TQ pass 1: training per-tensor grids...\n", __func__); @@ -1606,24 +1637,13 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: ggml_tensor * tensor = tensors[ti]->tensor; const std::string tname = ggml_get_name(tensor); - // Mirror pass-2 logic: only quantize 2D+ weight tensors bool quantize = tname.rfind("weight") == tname.size() - 6; quantize &= (ggml_n_dims(tensor) >= 2); quantize &= tname.find("_norm.weight") == std::string::npos; quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; if (!quantize) { continue; } - ggml_type new_type = default_type; - if (!params->pure) { - new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); - } - if (params->token_embedding_type < GGML_TYPE_COUNT && - (tname == "token_embd.weight" || tname == "per_layer_token_embd.weight")) { - new_type = params->token_embedding_type; - } - if (params->output_tensor_type < GGML_TYPE_COUNT && tname == "output.weight") { - new_type = params->output_tensor_type; - } + ggml_type new_type = resolve_tensor_type(qs, params, tensor, default_type, ftype, tname); if (new_type != GGML_TYPE_IQ2_TQ) { continue; } // Load tensor data @@ -1679,7 +1699,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: int8_t grid[128]; }; std::vector iq3tq_all_meta; - if (params->ftype == LLAMA_FTYPE_MOSTLY_IQ3_TQ) { + if (params->ftype == LLAMA_FTYPE_MOSTLY_IQ3_TQ || has_override_for_type(qs, GGML_TYPE_IQ3_TQ)) { const int64_t t_start_p1 = ggml_time_us(); LLAMA_LOG_INFO("%s: IQ3_TQ pass 1: training per-tensor grids...\n", __func__); @@ -1698,17 +1718,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; if (!quantize) { continue; } - ggml_type new_type = default_type; - if (!params->pure) { - new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); - } - if (params->token_embedding_type < GGML_TYPE_COUNT && - (tname == "token_embd.weight" || tname == "per_layer_token_embd.weight")) { - new_type = params->token_embedding_type; - } - if (params->output_tensor_type < GGML_TYPE_COUNT && tname == "output.weight") { - new_type = params->output_tensor_type; - } + ggml_type new_type = resolve_tensor_type(qs, params, tensor, default_type, ftype, tname); if (new_type != GGML_TYPE_IQ3_TQ) { continue; } const size_t tsz = ggml_nbytes(tensor); @@ -1980,9 +1990,19 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: q3kpt_set_levels(q3kpt_all_levels.data() + tensor_pass2_idx * Q3KPT_N_LEVELS); } - // Q4_DPT: set the per-tensor levels (trained in pass 1) as global for quantization + // Q4_DPT: set per-tensor trained levels if (new_type == GGML_TYPE_Q4_DPT) { - q4dpt_set_levels(q4dpt_all_levels.data() + tensor_pass2_idx * Q4DPT_N_LEVELS); + bool found = false; + for (const auto & meta : q4dpt_all_meta) { + if (meta.tensor_name == tm.name) { + q4dpt_set_levels(meta.levels); + found = true; + break; + } + } + if (!found) { + LLAMA_LOG_WARN("%s: WARNING: no trained levels for Q4_DPT tensor %s\n", __func__, tm.name.c_str()); + } } // IQ2_TQ: set per-tensor trained grid From 520262d457987ad7be016760777a970e719a9f51 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 17 Apr 2026 01:24:16 +0200 Subject: [PATCH 03/19] auto-tensor-type --- ggml/include/ggml.h | 7 + ggml/src/ggml-quants.c | 519 ++++-- ggml/src/ggml.c | 27 + tools/CMakeLists.txt | 1 + tools/auto-tensor-type/CMakeLists.txt | 9 + tools/auto-tensor-type/auto-tensor-type.cpp | 1732 +++++++++++++++++++ 6 files changed, 2156 insertions(+), 139 deletions(-) create mode 100644 tools/auto-tensor-type/CMakeLists.txt create mode 100644 tools/auto-tensor-type/auto-tensor-type.cpp diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 84593c5c9b65..430e75a72798 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -757,6 +757,13 @@ extern "C" { GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float "use ggml_row_size() instead"); + // Returns the effective bits per weight for a quantized type. + // For standard types this is type_size*8/blck_size. + // For types with per-tensor trained codebooks/grids/levels, this includes + // the amortized overhead of the auxiliary data stored alongside each tensor. + // The overhead is amortized over a reference tensor of 10M elements. + GGML_API double ggml_get_bpw(enum ggml_type type); + GGML_API const char * ggml_type_name(enum ggml_type type); GGML_API const char * ggml_op_name (enum ggml_op op); GGML_API const char * ggml_op_symbol(enum ggml_op op); diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 98432340913a..8bf4bb81846e 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -6381,17 +6381,113 @@ static inline int iq2tq_nearest_qi(float xn, const int8_t * g) { return 3; } +// AVX2 SIMD horizontal sum of 8 floats in a __m256 +#if defined(__AVX2__) +static inline float hsum_float_8(__m256 v) { + __m128 hi = _mm256_extractf128_ps(v, 1); + __m128 lo = _mm256_castps256_ps128(v); + __m128 sum4 = _mm_add_ps(lo, hi); + __m128 sum2 = _mm_add_ps(sum4, _mm_movehl_ps(sum4, sum4)); + __m128 sum1 = _mm_add_ss(sum2, _mm_movehdup_ps(sum2)); + return _mm_cvtss_f32(sum1); +} + +// AVX2 SIMD grid search for IQ2_TQ: find best grid entry for one group of 8 elements. +// Computes normalized error: sum(wk * (xn - ge[qi])^2) * dq^2 +// xn[k] = xb[k] * inv_dq (normalized values), wk[k] = importance weights +static inline int iq2tq_find_best_grid_avx2( + const float * xn, // [8] normalized values + const float * wk, // [8] importance weights + const float (* grid_f)[4], // [16] grid entries as float + float dq, // scale factor for error + float * best_err_out +) { + __m256 vxn = _mm256_loadu_ps(xn); + __m256 vwk = _mm256_loadu_ps(wk); + __m256 vdq2 = _mm256_set1_ps(dq * dq); + + float best_err = 1e30f; + int best_si = 3; + + for (int si = 0; si < 16; ++si) { + const float * ge = grid_f[si]; + + // Broadcast grid levels + __m256 vge0 = _mm256_set1_ps(ge[0]); + __m256 vge1 = _mm256_set1_ps(ge[1]); + __m256 vge2 = _mm256_set1_ps(ge[2]); + __m256 vge3 = _mm256_set1_ps(ge[3]); + + // Compute midpoints for nearest-qi selection + __m256 vm01 = _mm256_set1_ps(0.5f * (ge[0] + ge[1])); + __m256 vm12 = _mm256_set1_ps(0.5f * (ge[1] + ge[2])); + __m256 vm23 = _mm256_set1_ps(0.5f * (ge[2] + ge[3])); + + // Select reconstruction via cascade of blendv (matches nested if in iq2tq_nearest_qi) + __m256 vrecon = vge3; + vrecon = _mm256_blendv_ps(vge2, vrecon, _mm256_cmp_ps(vxn, vm23, _MM_CMPINT_GT)); + vrecon = _mm256_blendv_ps(vge1, vrecon, _mm256_cmp_ps(vxn, vm12, _MM_CMPINT_GT)); + vrecon = _mm256_blendv_ps(vge0, vrecon, _mm256_cmp_ps(vxn, vm01, _MM_CMPINT_GT)); + + // Weighted squared error: wk * dq^2 * (xn - recon)^2 + __m256 vdiff = _mm256_sub_ps(vxn, vrecon); + __m256 vwerr = _mm256_mul_ps(vwk, _mm256_mul_ps(_mm256_mul_ps(vdiff, vdiff), vdq2)); + + float g_err = hsum_float_8(vwerr); + if (g_err < best_err) { best_err = g_err; best_si = si; } + } + + *best_err_out = best_err; + return best_si; +} + +// AVX2 SIMD grid search for IQ3_TQ: find best grid entry for one group of 8 elements. +// Same approach but with 8 levels (7 midpoints) instead of 4 (3 midpoints). +static inline int iq3tq_find_best_grid_avx2( + const float * xn, // [8] normalized values + const float * wk, // [8] importance weights + const float (* grid_f)[IQ3TQ_N_LEVELS], // [16] grid entries as float + float dq, // scale factor for error + float * best_err_out +) { + __m256 vxn = _mm256_loadu_ps(xn); + __m256 vwk = _mm256_loadu_ps(wk); + __m256 vdq2 = _mm256_set1_ps(dq * dq); + + float best_err = 1e30f; + int best_si = 0; + + for (int si = 0; si < 16; ++si) { + const float * ge = grid_f[si]; + + // Select reconstruction via cascade from ge[7] down to ge[0] + __m256 vrecon = _mm256_set1_ps(ge[7]); + for (int i = 6; i >= 0; --i) { + __m256 vm = _mm256_set1_ps(0.5f * (ge[i] + ge[i + 1])); + vrecon = _mm256_blendv_ps(_mm256_set1_ps(ge[i]), vrecon, + _mm256_cmp_ps(vxn, vm, _MM_CMPINT_GT)); + } + + // Weighted squared error: wk * dq^2 * (xn - recon)^2 + __m256 vdiff = _mm256_sub_ps(vxn, vrecon); + __m256 vwerr = _mm256_mul_ps(vwk, _mm256_mul_ps(_mm256_mul_ps(vdiff, vdiff), vdq2)); + + float g_err = hsum_float_8(vwerr); + if (g_err < best_err) { best_err = g_err; best_si = si; } + } + + *best_err_out = best_err; + return best_si; +} +#endif // __AVX2__ + // Dequantization — 2-bit with asymmetric grid per group void dequantize_row_iq2_tq(const block_iq2_tq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { -#ifndef _MSC_VER #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-qual" -#endif const int8_t (*grid)[4] = levels ? (const int8_t (*)[4])levels : (const int8_t (*)[4])iq2tq_grid_default; -#ifndef _MSC_VER #pragma GCC diagnostic pop -#endif const int nb = k / QK_K; for (int i = 0; i < nb; ++i) { @@ -6411,6 +6507,86 @@ void dequantize_row_iq2_tq(const block_iq2_tq * GGML_RESTRICT x, float * GGML_RE } } +// SIMD K-means assignment: find nearest centroid for each data point +// IQ2_TQ version: 4 dimensions, 16 centroids +#if defined(__AVX2__) +static void kmeans_assign_4d_avx2(const float (*sets)[4], int n_sets, int * assign, + const float centroids[16][4], int * changed) { + // Reorganize centroids into SoA for SIMD: cx[16], cy[16], cz[16], cw[16] + float cx[16], cy[16], cz[16], cw[16]; + for (int c = 0; c < 16; c++) { + cx[c] = centroids[c][0]; cy[c] = centroids[c][1]; + cz[c] = centroids[c][2]; cw[c] = centroids[c][3]; + } + + int local_changed = 0; + for (int i = 0; i < n_sets; i++) { + __m256 vx = _mm256_set1_ps(sets[i][0]); + __m256 vy = _mm256_set1_ps(sets[i][1]); + __m256 vz = _mm256_set1_ps(sets[i][2]); + __m256 vw = _mm256_set1_ps(sets[i][3]); + + float best_dist = 1e30f; + int best_c = 0; + + // Process 8 centroids at a time (2 batches for 16 centroids) + for (int c = 0; c < 16; c += 8) { + __m256 dx = _mm256_sub_ps(vx, _mm256_loadu_ps(cx + c)); + __m256 dy = _mm256_sub_ps(vy, _mm256_loadu_ps(cy + c)); + __m256 dz = _mm256_sub_ps(vz, _mm256_loadu_ps(cz + c)); + __m256 dw = _mm256_sub_ps(vw, _mm256_loadu_ps(cw + c)); + __m256 dist = _mm256_add_ps( + _mm256_add_ps(_mm256_mul_ps(dx, dx), _mm256_mul_ps(dy, dy)), + _mm256_add_ps(_mm256_mul_ps(dz, dz), _mm256_mul_ps(dw, dw))); + + // Extract and find minimum among 8 distances + float d_arr[8]; + _mm256_storeu_ps(d_arr, dist); + for (int j = 0; j < 8; j++) { + if (d_arr[j] < best_dist) { best_dist = d_arr[j]; best_c = c + j; } + } + } + if (assign[i] != best_c) { assign[i] = best_c; local_changed++; } + } + *changed = local_changed; +} + +// IQ3_TQ version: 8 dimensions, 16 centroids +static void kmeans_assign_8d_avx2(const float (*sets)[8], int n_sets, int * assign, + const float centroids[16][8], int * changed) { + // Reorganize centroids into SoA + float csoa[8][16]; + for (int c = 0; c < 16; c++) + for (int d = 0; d < 8; d++) + csoa[d][c] = centroids[c][d]; + + int local_changed = 0; + for (int i = 0; i < n_sets; i++) { + __m256 v[8]; + for (int d = 0; d < 8; d++) v[d] = _mm256_set1_ps(sets[i][d]); + + float best_dist = 1e30f; + int best_c = 0; + + for (int c = 0; c < 16; c += 8) { + __m256 dist = _mm256_setzero_ps(); + for (int d = 0; d < 8; d++) { + __m256 diff = _mm256_sub_ps(v[d], _mm256_loadu_ps(csoa[d] + c)); + dist = _mm256_fmadd_ps(diff, diff, dist); + } + + float d_arr[8]; + _mm256_storeu_ps(d_arr, dist); + for (int j = 0; j < 8; j++) { + if (d_arr[j] < best_dist) { best_dist = d_arr[j]; best_c = c + j; } + } + } + if (assign[i] != best_c) { assign[i] = best_c; local_changed++; } + } + *changed = local_changed; +} +#endif // __AVX2__ + // Reference quantization void quantize_row_iq2_tq_ref(const float * GGML_RESTRICT x, block_iq2_tq * GGML_RESTRICT y, int64_t k) { quantize_iq2_tq(x, y, 1, k, NULL); @@ -6427,6 +6603,14 @@ static void quantize_row_iq2_tq_impl( const int8_t (*grid)[4] = iq2tq_cur_grid(); const int nb = n_per_row / QK_K; +#if defined(__AVX2__) + // Precompute grid as float for SIMD + float grid_f[16][4]; + for (int si = 0; si < 16; ++si) + for (int k = 0; k < 4; ++k) + grid_f[si][k] = (float)grid[si][k]; +#endif + for (int bi = 0; bi < nb; ++bi) { const float * xb = x + bi * QK_K; block_iq2_tq * yb = y + bi; @@ -6447,6 +6631,14 @@ static void quantize_row_iq2_tq_impl( // Initial d: max grid value ~24 in int8, recon = d * 0.125 * 24 = 3d → d = amax/3 float d = amax / 3.0f; + // Precompute importance weights (eliminates sqrtf from hot grid search loops) + float wk_arr[QK_K]; + if (quant_weights) { + for (int j = 0; j < QK_K; ++j) { + wk_arr[j] = quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]); + } + } + uint8_t qs[64] = {0}; uint8_t scales_out[16] = {0}; int grid_idx[IQ2TQ_N_GROUPS]; // chosen grid entry per group @@ -6455,41 +6647,46 @@ static void quantize_row_iq2_tq_impl( float dq = d * IQ2TQ_GRID_SCALE; float inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; for (int g = 0; g < IQ2TQ_N_GROUPS; ++g) { - float best_err = 1e30f; - int best_si = 3; // default: centered medium - + float best_err; + int best_si; + float xn[8], wk[8]; + for (int k = 0; k < 8; ++k) { + xn[k] = xb[g * 8 + k] * inv_dq; + wk[k] = quant_weights ? wk_arr[g * 8 + k] : 1.0f; + } +#if defined(__AVX2__) + best_si = iq2tq_find_best_grid_avx2(xn, wk, grid_f, dq, &best_err); +#else + best_err = 1e30f; best_si = 3; for (int si = 0; si < 16; ++si) { const int8_t * ge = grid[si]; float g_err = 0; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - float xn = xb[j] * inv_dq; - int qi = iq2tq_nearest_qi(xn, ge); + int qi = iq2tq_nearest_qi(xn[k], ge); float recon = dq * (float)ge[qi]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - float err = xb[j] - recon; - g_err += wk * err * err; + float err = xb[g * 8 + k] - recon; + g_err += wk[k] * err * err; } if (g_err < best_err) { best_err = g_err; best_si = si; } } +#endif grid_idx[g] = best_si; // Quantize elements with chosen grid const int8_t * ge = grid[best_si]; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - iq2tq_set_qi(qs, j, iq2tq_nearest_qi(xb[j] * inv_dq, ge)); + iq2tq_set_qi(qs, g * 8 + k, iq2tq_nearest_qi(xn[k], ge)); } } - // Iterative refinement + // Iterative refinement (with convergence early-stop) for (int iter = 0; iter < 12; ++iter) { // Re-fit d via weighted OLS: d = sum(w*x*g) / (GRID_SCALE * sum(w*g*g)) double sumxg = 0, sumgg = 0; for (int j = 0; j < QK_K; ++j) { int g = j / 8; float gval = (float)grid[grid_idx[g]][iq2tq_get_qi(qs, j)]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float wk = quant_weights ? wk_arr[j] : 1.0f; sumxg += (double)(wk * xb[j] * gval); sumgg += (double)(wk * gval * gval); } @@ -6499,33 +6696,41 @@ static void quantize_row_iq2_tq_impl( dq = d * IQ2TQ_GRID_SCALE; inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; memset(scales_out, 0, 16); + int grid_changed = 0; for (int g = 0; g < IQ2TQ_N_GROUPS; ++g) { - float best_err = 1e30f; - int best_si = 3; - + float best_err; + int best_si; + float xn[8], wk[8]; + for (int k = 0; k < 8; ++k) { + xn[k] = xb[g * 8 + k] * inv_dq; + wk[k] = quant_weights ? wk_arr[g * 8 + k] : 1.0f; + } +#if defined(__AVX2__) + best_si = iq2tq_find_best_grid_avx2(xn, wk, grid_f, dq, &best_err); +#else + best_err = 1e30f; best_si = 3; for (int si = 0; si < 16; ++si) { const int8_t * ge = grid[si]; float g_err = 0; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - float xn = xb[j] * inv_dq; - int qi = iq2tq_nearest_qi(xn, ge); + int qi = iq2tq_nearest_qi(xn[k], ge); float recon = dq * (float)ge[qi]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - float err = xb[j] - recon; - g_err += wk * err * err; + float err = xb[g * 8 + k] - recon; + g_err += wk[k] * err * err; } if (g_err < best_err) { best_err = g_err; best_si = si; } } +#endif + if (best_si != grid_idx[g]) grid_changed++; scales_out[g / 2] |= (best_si << (4 * (g % 2))); grid_idx[g] = best_si; const int8_t * ge = grid[best_si]; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - iq2tq_set_qi(qs, j, iq2tq_nearest_qi(xb[j] * inv_dq, ge)); + iq2tq_set_qi(qs, g * 8 + k, iq2tq_nearest_qi(xn[k], ge)); } } + if (grid_changed == 0) break; // converged } // Final OLS d @@ -6534,18 +6739,18 @@ static void quantize_row_iq2_tq_impl( for (int j = 0; j < QK_K; ++j) { int g = j / 8; float gval = (float)grid[grid_idx[g]][iq2tq_get_qi(qs, j)]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float wk = quant_weights ? wk_arr[j] : 1.0f; sumxg += (double)(wk * xb[j] * gval); sumgg += (double)(wk * gval * gval); } d = (sumgg > 0) ? (float)(sumxg / ((double)IQ2TQ_GRID_SCALE * sumgg)) : d; } - // Multi-d search: try nearby d values and re-optimize grids + // Multi-d search: try nearby d values and re-optimize grids (with partial sum pruning) { float best_d = d; float best_total_err = 1e30f; - uint8_t best_scales[16] = {0}, best_qs[64] = {0}; + uint8_t best_scales[16], best_qs[64]; int best_grid_idx[IQ2TQ_N_GROUPS]; static const float d_factors[] = {0.8f, 0.85f, 0.9f, 0.925f, 0.95f, 0.975f, 1.0f, 1.025f, 1.05f, 1.075f, 1.1f, 1.15f, 1.2f}; static const int n_d_factors = sizeof(d_factors) / sizeof(d_factors[0]); @@ -6559,29 +6764,35 @@ static void quantize_row_iq2_tq_impl( int tgrid[IQ2TQ_N_GROUPS]; float total_err = 0; - for (int g = 0; g < IQ2TQ_N_GROUPS; ++g) { - float best_err = 1e30f; - int best_si = 3; + for (int g = 0; g < IQ2TQ_N_GROUPS && total_err <= best_total_err; ++g) { + float best_err; + int best_si; + float xn[8], wk[8]; + for (int k = 0; k < 8; ++k) { + xn[k] = xb[g * 8 + k] * tinv; + wk[k] = quant_weights ? wk_arr[g * 8 + k] : 1.0f; + } +#if defined(__AVX2__) + best_si = iq2tq_find_best_grid_avx2(xn, wk, grid_f, tdq, &best_err); +#else + best_err = 1e30f; best_si = 3; for (int si = 0; si < 16; ++si) { const int8_t * ge = grid[si]; float g_err = 0; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - float xn = xb[j] * tinv; - int qi = iq2tq_nearest_qi(xn, ge); + int qi = iq2tq_nearest_qi(xn[k], ge); float recon = tdq * (float)ge[qi]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - float err = xb[j] - recon; - g_err += wk * err * err; + float err = xb[g * 8 + k] - recon; + g_err += wk[k] * err * err; } if (g_err < best_err) { best_err = g_err; best_si = si; } } +#endif tscales[g / 2] |= (best_si << (4 * (g % 2))); tgrid[g] = best_si; const int8_t * ge = grid[best_si]; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - iq2tq_set_qi(tqs, j, iq2tq_nearest_qi(xb[j] * tinv, ge)); + iq2tq_set_qi(tqs, g * 8 + k, iq2tq_nearest_qi(xn[k], ge)); } total_err += best_err; } @@ -6600,13 +6811,13 @@ static void quantize_row_iq2_tq_impl( memcpy(grid_idx, best_grid_idx, sizeof(grid_idx)); } - // Post multi-d refinement: 2 more OLS+grid iterations from the best d + // Post multi-d refinement (with convergence early-stop) for (int iter = 0; iter < 2; ++iter) { double sumxg = 0, sumgg = 0; for (int j = 0; j < QK_K; ++j) { int g = j / 8; float gval = (float)grid[grid_idx[g]][iq2tq_get_qi(qs, j)]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float wk = quant_weights ? wk_arr[j] : 1.0f; sumxg += (double)(wk * xb[j] * gval); sumgg += (double)(wk * gval * gval); } @@ -6615,31 +6826,40 @@ static void quantize_row_iq2_tq_impl( dq = d * IQ2TQ_GRID_SCALE; inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; memset(scales_out, 0, 16); + int grid_changed = 0; for (int g = 0; g < IQ2TQ_N_GROUPS; ++g) { - float best_err = 1e30f; - int best_si = 3; + float best_err; + int best_si; + float xn[8], wk[8]; + for (int k = 0; k < 8; ++k) { + xn[k] = xb[g * 8 + k] * inv_dq; + wk[k] = quant_weights ? wk_arr[g * 8 + k] : 1.0f; + } +#if defined(__AVX2__) + best_si = iq2tq_find_best_grid_avx2(xn, wk, grid_f, dq, &best_err); +#else + best_err = 1e30f; best_si = 3; for (int si = 0; si < 16; ++si) { const int8_t * ge = grid[si]; float g_err = 0; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - float xn = xb[j] * inv_dq; - int qi = iq2tq_nearest_qi(xn, ge); + int qi = iq2tq_nearest_qi(xn[k], ge); float recon = dq * (float)ge[qi]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - float err = xb[j] - recon; - g_err += wk * err * err; + float err = xb[g * 8 + k] - recon; + g_err += wk[k] * err * err; } if (g_err < best_err) { best_err = g_err; best_si = si; } } +#endif + if (best_si != grid_idx[g]) grid_changed++; scales_out[g / 2] |= (best_si << (4 * (g % 2))); grid_idx[g] = best_si; const int8_t * ge = grid[best_si]; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - iq2tq_set_qi(qs, j, iq2tq_nearest_qi(xb[j] * inv_dq, ge)); + iq2tq_set_qi(qs, g * 8 + k, iq2tq_nearest_qi(xn[k], ge)); } } + if (grid_changed == 0) break; // converged } // Write result @@ -6748,6 +6968,9 @@ void iq2tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, int * assign = (int *)calloc(n_sets, sizeof(int)); for (int iter = 0; iter < 100; iter++) { int changed = 0; +#if defined(__AVX2__) + kmeans_assign_4d_avx2(sets, n_sets, assign, centroids, &changed); +#else for (int i = 0; i < n_sets; i++) { float best_dist = 1e30f; int best_c = 0; @@ -6761,6 +6984,7 @@ void iq2tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, } if (assign[i] != best_c) { assign[i] = best_c; changed++; } } +#endif double sum[16][4] = {{0}}; double wcnt[16] = {0}; @@ -6880,15 +7104,11 @@ static inline int iq3tq_nearest_qi(float xn, const int8_t * g) { // Dequantization void dequantize_row_iq3_tq(const block_iq3_tq * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k, const void * levels) { -#ifndef _MSC_VER #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wcast-qual" -#endif const int8_t (*grid)[IQ3TQ_N_LEVELS] = levels ? (const int8_t (*)[IQ3TQ_N_LEVELS])levels : (const int8_t (*)[IQ3TQ_N_LEVELS])iq3tq_grid_default; -#ifndef _MSC_VER #pragma GCC diagnostic pop -#endif const int nb = k / QK_K; for (int i = 0; i < nb; ++i) { @@ -6923,6 +7143,14 @@ static void quantize_row_iq3_tq_impl( const int8_t (*grid)[IQ3TQ_N_LEVELS] = iq3tq_cur_grid(); const int nb = n_per_row / QK_K; +#if defined(__AVX2__) + // Precompute grid as float for SIMD + float grid_f[16][IQ3TQ_N_LEVELS]; + for (int si = 0; si < 16; ++si) + for (int k = 0; k < IQ3TQ_N_LEVELS; ++k) + grid_f[si][k] = (float)grid[si][k]; +#endif + for (int bi = 0; bi < nb; ++bi) { const float * xb = x + bi * QK_K; block_iq3_tq * yb = y + bi; @@ -6941,6 +7169,14 @@ static void quantize_row_iq3_tq_impl( float d = amax / 3.0f; + // Precompute importance weights (eliminates sqrtf from hot grid search loops) + float wk_arr[QK_K]; + if (quant_weights) { + for (int j = 0; j < QK_K; ++j) { + wk_arr[j] = quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]); + } + } + uint8_t qs[96] = {0}; uint8_t scales_out[16] = {0}; int grid_idx[IQ3TQ_N_GROUPS]; @@ -6949,38 +7185,43 @@ static void quantize_row_iq3_tq_impl( float dq = d * IQ3TQ_GRID_SCALE; float inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; for (int g = 0; g < IQ3TQ_N_GROUPS; ++g) { - float best_err = 1e30f; - int best_si = 0; - + float best_err; + int best_si; + float xn[8], wk[8]; + for (int k = 0; k < 8; ++k) { + xn[k] = xb[g * 8 + k] * inv_dq; + wk[k] = quant_weights ? wk_arr[g * 8 + k] : 1.0f; + } +#if defined(__AVX2__) + best_si = iq3tq_find_best_grid_avx2(xn, wk, grid_f, dq, &best_err); +#else + best_err = 1e30f; best_si = 0; for (int si = 0; si < 16; ++si) { const int8_t * ge = grid[si]; float g_err = 0; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - float xn = xb[j] * inv_dq; - int qi = iq3tq_nearest_qi(xn, ge); + int qi = iq3tq_nearest_qi(xn[k], ge); float recon = dq * (float)ge[qi]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - float err = xb[j] - recon; - g_err += wk * err * err; + float err = xb[g * 8 + k] - recon; + g_err += wk[k] * err * err; } if (g_err < best_err) { best_err = g_err; best_si = si; } } +#endif grid_idx[g] = best_si; const int8_t * ge = grid[best_si]; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - iq3tq_set_qi(qs, j, iq3tq_nearest_qi(xb[j] * inv_dq, ge)); + iq3tq_set_qi(qs, g * 8 + k, iq3tq_nearest_qi(xn[k], ge)); } } - // Iterative refinement + // Iterative refinement (with convergence early-stop) for (int iter = 0; iter < 12; ++iter) { double sumxg = 0, sumgg = 0; for (int j = 0; j < QK_K; ++j) { int g = j / 8; float gval = (float)grid[grid_idx[g]][iq3tq_get_qi(qs, j)]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float wk = quant_weights ? wk_arr[j] : 1.0f; sumxg += (double)(wk * xb[j] * gval); sumgg += (double)(wk * gval * gval); } @@ -6989,31 +7230,40 @@ static void quantize_row_iq3_tq_impl( dq = d * IQ3TQ_GRID_SCALE; inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; memset(scales_out, 0, 16); + int grid_changed = 0; for (int g = 0; g < IQ3TQ_N_GROUPS; ++g) { - float best_err = 1e30f; - int best_si = 0; + float best_err; + int best_si; + float xn[8], wk[8]; + for (int k = 0; k < 8; ++k) { + xn[k] = xb[g * 8 + k] * inv_dq; + wk[k] = quant_weights ? wk_arr[g * 8 + k] : 1.0f; + } +#if defined(__AVX2__) + best_si = iq3tq_find_best_grid_avx2(xn, wk, grid_f, dq, &best_err); +#else + best_err = 1e30f; best_si = 0; for (int si = 0; si < 16; ++si) { const int8_t * ge = grid[si]; float g_err = 0; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - float xn = xb[j] * inv_dq; - int qi = iq3tq_nearest_qi(xn, ge); + int qi = iq3tq_nearest_qi(xn[k], ge); float recon = dq * (float)ge[qi]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - float err = xb[j] - recon; - g_err += wk * err * err; + float err = xb[g * 8 + k] - recon; + g_err += wk[k] * err * err; } if (g_err < best_err) { best_err = g_err; best_si = si; } } +#endif + if (best_si != grid_idx[g]) grid_changed++; scales_out[g / 2] |= (best_si << (4 * (g % 2))); grid_idx[g] = best_si; const int8_t * ge = grid[best_si]; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - iq3tq_set_qi(qs, j, iq3tq_nearest_qi(xb[j] * inv_dq, ge)); + iq3tq_set_qi(qs, g * 8 + k, iq3tq_nearest_qi(xn[k], ge)); } } + if (grid_changed == 0) break; // converged } // Final OLS d @@ -7022,18 +7272,18 @@ static void quantize_row_iq3_tq_impl( for (int j = 0; j < QK_K; ++j) { int g = j / 8; float gval = (float)grid[grid_idx[g]][iq3tq_get_qi(qs, j)]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float wk = quant_weights ? wk_arr[j] : 1.0f; sumxg += (double)(wk * xb[j] * gval); sumgg += (double)(wk * gval * gval); } d = (sumgg > 0) ? (float)(sumxg / ((double)IQ3TQ_GRID_SCALE * sumgg)) : d; } - // Multi-d search + // Multi-d search (with partial sum pruning) { float best_d = d; float best_total_err = 1e30f; - uint8_t best_scales[16] = {0}, best_qs[96] = {0}; + uint8_t best_scales[16], best_qs[96]; int best_grid_idx[IQ3TQ_N_GROUPS]; static const float d_factors[] = {0.8f, 0.85f, 0.9f, 0.925f, 0.95f, 0.975f, 1.0f, 1.025f, 1.05f, 1.075f, 1.1f, 1.15f, 1.2f}; static const int n_d_factors = sizeof(d_factors) / sizeof(d_factors[0]); @@ -7047,29 +7297,35 @@ static void quantize_row_iq3_tq_impl( int tgrid[IQ3TQ_N_GROUPS]; float total_err = 0; - for (int g = 0; g < IQ3TQ_N_GROUPS; ++g) { - float best_err = 1e30f; - int best_si = 0; + for (int g = 0; g < IQ3TQ_N_GROUPS && total_err <= best_total_err; ++g) { + float best_err; + int best_si; + float xn[8], wk[8]; + for (int k = 0; k < 8; ++k) { + xn[k] = xb[g * 8 + k] * tinv; + wk[k] = quant_weights ? wk_arr[g * 8 + k] : 1.0f; + } +#if defined(__AVX2__) + best_si = iq3tq_find_best_grid_avx2(xn, wk, grid_f, tdq, &best_err); +#else + best_err = 1e30f; best_si = 0; for (int si = 0; si < 16; ++si) { const int8_t * ge = grid[si]; float g_err = 0; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - float xn = xb[j] * tinv; - int qi = iq3tq_nearest_qi(xn, ge); + int qi = iq3tq_nearest_qi(xn[k], ge); float recon = tdq * (float)ge[qi]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - float err = xb[j] - recon; - g_err += wk * err * err; + float err = xb[g * 8 + k] - recon; + g_err += wk[k] * err * err; } if (g_err < best_err) { best_err = g_err; best_si = si; } } +#endif tscales[g / 2] |= (best_si << (4 * (g % 2))); tgrid[g] = best_si; const int8_t * ge = grid[best_si]; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - iq3tq_set_qi(tqs, j, iq3tq_nearest_qi(xb[j] * tinv, ge)); + iq3tq_set_qi(tqs, g * 8 + k, iq3tq_nearest_qi(xn[k], ge)); } total_err += best_err; } @@ -7088,13 +7344,13 @@ static void quantize_row_iq3_tq_impl( memcpy(grid_idx, best_grid_idx, sizeof(grid_idx)); } - // Post multi-d refinement + // Post multi-d refinement (with convergence early-stop) for (int iter = 0; iter < 2; ++iter) { double sumxg = 0, sumgg = 0; for (int j = 0; j < QK_K; ++j) { int g = j / 8; float gval = (float)grid[grid_idx[g]][iq3tq_get_qi(qs, j)]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; + float wk = quant_weights ? wk_arr[j] : 1.0f; sumxg += (double)(wk * xb[j] * gval); sumgg += (double)(wk * gval * gval); } @@ -7103,31 +7359,40 @@ static void quantize_row_iq3_tq_impl( dq = d * IQ3TQ_GRID_SCALE; inv_dq = (fabsf(dq) > 1e-15f) ? 1.0f / dq : 0.0f; memset(scales_out, 0, 16); + int grid_changed = 0; for (int g = 0; g < IQ3TQ_N_GROUPS; ++g) { - float best_err = 1e30f; - int best_si = 0; + float best_err; + int best_si; + float xn[8], wk[8]; + for (int k = 0; k < 8; ++k) { + xn[k] = xb[g * 8 + k] * inv_dq; + wk[k] = quant_weights ? wk_arr[g * 8 + k] : 1.0f; + } +#if defined(__AVX2__) + best_si = iq3tq_find_best_grid_avx2(xn, wk, grid_f, dq, &best_err); +#else + best_err = 1e30f; best_si = 0; for (int si = 0; si < 16; ++si) { const int8_t * ge = grid[si]; float g_err = 0; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - float xn = xb[j] * inv_dq; - int qi = iq3tq_nearest_qi(xn, ge); + int qi = iq3tq_nearest_qi(xn[k], ge); float recon = dq * (float)ge[qi]; - float wk = quant_weights ? quant_weights[bi * QK_K + j] * sqrtf(sigma2 + xb[j] * xb[j]) : 1.0f; - float err = xb[j] - recon; - g_err += wk * err * err; + float err = xb[g * 8 + k] - recon; + g_err += wk[k] * err * err; } if (g_err < best_err) { best_err = g_err; best_si = si; } } +#endif + if (best_si != grid_idx[g]) grid_changed++; scales_out[g / 2] |= (best_si << (4 * (g % 2))); grid_idx[g] = best_si; const int8_t * ge = grid[best_si]; for (int k = 0; k < 8; ++k) { - int j = g * 8 + k; - iq3tq_set_qi(qs, j, iq3tq_nearest_qi(xb[j] * inv_dq, ge)); + iq3tq_set_qi(qs, g * 8 + k, iq3tq_nearest_qi(xn[k], ge)); } } + if (grid_changed == 0) break; // converged } yb->d = GGML_FP32_TO_FP16(d); @@ -7230,6 +7495,9 @@ void iq3tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, int * assign = (int *)calloc(n_sets, sizeof(int)); for (int iter = 0; iter < 100; iter++) { int changed = 0; +#if defined(__AVX2__) + kmeans_assign_8d_avx2(sets, n_sets, assign, centroids, &changed); +#else for (int i = 0; i < n_sets; i++) { float best_dist = 1e30f; int best_c = 0; @@ -7243,6 +7511,7 @@ void iq3tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, } if (assign[i] != best_c) { assign[i] = best_c; changed++; } } +#endif double sum[16][IQ3TQ_N_LEVELS]; double wcnt[16]; @@ -7634,9 +7903,7 @@ size_t quantize_iq1_bn(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst } // Thread work structs for parallel K-means -#ifndef _WIN32 #include -#endif // Worker for K-means++ min_dist update (parallel over samples) typedef struct { @@ -7803,24 +8070,10 @@ void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, // Full K-means++ initialization with parallel min_dist update if (nthread < 1) nthread = 1; if (nthread > n_samples) nthread = n_samples; - assert(nthread >= 1); - const size_t nth = (size_t)(unsigned)nthread; - const size_t kmpp_sz = nth * sizeof(iq1bn_kmpp_work_t); - -#ifndef _WIN32 - const size_t threads_sz = nth * sizeof(pthread_t); - pthread_t * threads = (pthread_t *)malloc(threads_sz); - assert(threads != NULL); -#else - nthread = 1; - void * threads = NULL; - const size_t threads_sz = 0; -#endif - iq1bn_kmpp_work_t * kmpp_workers = (iq1bn_kmpp_work_t *)malloc(kmpp_sz); - assert(kmpp_workers != NULL); - if (threads != NULL) memset(threads, 0, threads_sz); - memset(kmpp_workers, 0, kmpp_sz); + // Set up thread pool for K-means++ min_dist update + pthread_t * threads = (pthread_t *)calloc(nthread, sizeof(pthread_t)); + iq1bn_kmpp_work_t * kmpp_workers = (iq1bn_kmpp_work_t *)calloc(nthread, sizeof(iq1bn_kmpp_work_t)); for (int t = 0; t < nthread; ++t) { kmpp_workers[t].samples = samples; kmpp_workers[t].min_dist = min_dist; @@ -7840,18 +8093,12 @@ void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, kmpp_workers[t].centroid = cc; } if (nthread > 1) { -#ifndef _WIN32 for (int t = 0; t < nthread; ++t) { pthread_create(&threads[t], NULL, iq1bn_kmpp_worker, &kmpp_workers[t]); } for (int t = 0; t < nthread; ++t) { pthread_join(threads[t], NULL); } -#else - for (int t = 0; t < nthread; ++t) { - iq1bn_kmpp_worker(&kmpp_workers[t]); - } -#endif } else { iq1bn_kmpp_worker(&kmpp_workers[0]); } @@ -7877,7 +8124,7 @@ void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, // Phase 3: Full-batch K-means with SIMD and pthread parallelism // Allocate per-thread accumulators (reuse threads array from K-means++ init) - iq1bn_kmeans_work_t * workers = (iq1bn_kmeans_work_t *)malloc(nth * sizeof(iq1bn_kmeans_work_t)); + iq1bn_kmeans_work_t * workers = (iq1bn_kmeans_work_t *)calloc(nthread, sizeof(iq1bn_kmeans_work_t)); for (int t = 0; t < nthread; ++t) { workers[t].samples = samples; workers[t].weights = weights; @@ -7895,7 +8142,6 @@ void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, const int n_iters = 30; for (int iter = 0; iter < n_iters; ++iter) { // Launch threads for sample assignment -#ifndef _WIN32 for (int t = 0; t < nthread; ++t) { if (nthread > 1) { pthread_create(&threads[t], NULL, iq1bn_kmeans_worker, &workers[t]); @@ -7908,11 +8154,6 @@ void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, pthread_join(threads[t], NULL); } } -#else - for (int t = 0; t < nthread; ++t) { - iq1bn_kmeans_worker(&workers[t]); - } -#endif // Merge per-thread accumulators int changed = 0; diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index f4cdec262054..1a5db39b220d 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -1397,6 +1397,33 @@ double ggml_type_sizef(enum ggml_type type) { return ((double)(type_traits[type].type_size))/type_traits[type].blck_size; } +// Reference tensor size for amortizing per-tensor codebook/grid/level overhead +#define GGML_BPW_REF_N_ELEMENTS 10000000.0 + +double ggml_get_bpw(enum ggml_type type) { + assert(type >= 0); + assert(type < GGML_TYPE_COUNT); + + const double base_bpw = ((double)(type_traits[type].type_size)) * 8.0 / type_traits[type].blck_size; + + // Per-tensor overhead for types with trained codebooks/grids/levels + // This overhead is amortized over the reference tensor size + double overhead_bits = 0.0; + switch (type) { + case GGML_TYPE_IQ2_TQ: overhead_bits = 64 * 8; break; // int8_t grid[64] + case GGML_TYPE_IQ3_TQ: overhead_bits = 128 * 8; break; // int8_t grid[128] + case GGML_TYPE_IQ1_BN: overhead_bits = 32768 * 8; break; // int8_t codebook[32768] + case GGML_TYPE_Q3_PT: overhead_bits = 8 * 32; break; // float levels[8] + case GGML_TYPE_Q3_KPT: overhead_bits = 8 * 32; break; // float levels[8] + case GGML_TYPE_Q4_DPT: overhead_bits = 16 * 8; break; // int8_t levels[16] + case GGML_TYPE_Q2_DPT: overhead_bits = 16 * 8; break; // int8_t levels[16] + case GGML_TYPE_Q2_KPT: overhead_bits = 4 * 32; break; // float levels[4] + default: break; + } + + return base_bpw + overhead_bits / GGML_BPW_REF_N_ELEMENTS; +} + const char * ggml_type_name(enum ggml_type type) { assert(type >= 0); assert(type < GGML_TYPE_COUNT); diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 8c68ccb21573..5ab6d44b9e7b 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -39,6 +39,7 @@ else() add_subdirectory(export-lora) endif() add_subdirectory(fit-params) + add_subdirectory(auto-tensor-type) add_subdirectory(capture-layer-data) add_subdirectory(results) endif() diff --git a/tools/auto-tensor-type/CMakeLists.txt b/tools/auto-tensor-type/CMakeLists.txt new file mode 100644 index 000000000000..f36f13834223 --- /dev/null +++ b/tools/auto-tensor-type/CMakeLists.txt @@ -0,0 +1,9 @@ +set(TARGET llama-auto-tensor-type) +add_executable(${TARGET} auto-tensor-type.cpp) +target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_include_directories(${TARGET} PRIVATE ../../common ../../ggml/include) +target_compile_features(${TARGET} PRIVATE cxx_std_17) + +if(LLAMA_TOOLS_INSTALL) + install(TARGETS ${TARGET} RUNTIME) +endif() diff --git a/tools/auto-tensor-type/auto-tensor-type.cpp b/tools/auto-tensor-type/auto-tensor-type.cpp new file mode 100644 index 000000000000..3e740bcc0c58 --- /dev/null +++ b/tools/auto-tensor-type/auto-tensor-type.cpp @@ -0,0 +1,1732 @@ +// auto-tensor-type.cpp +// Automatically determine optimal per-tensor-role quantization types to meet a target BPW, +// by measuring the KLD impact of each quant type on MUL_MAT outputs using real activations. +// +// Usage: +// llama-auto-tensor-type -m MODEL -i IMATRIX --quants IQ1_BN,IQ2_TQ,IQ3_TQ,Q4_KPT,Q6_K --target-bpw 3.2 -o output.txt +// +// The tool: +// 1. Loads the model and runs forward passes to capture MUL_MAT activations +// 2. For each (tensor_role, quant_type) pair, measures average KLD of MUL_MAT output +// 3. Optimizes the assignment to minimize total KLD while meeting the BPW target +// 4. Outputs a tensor-type-file compatible with llama-quantize --tensor-type-file + +#include "arg.h" +#include "common.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "ggml.h" +#include "gguf.h" +#include "llama.h" +#include "log.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// ============================================================================ +// Section 1: Extern C declarations for special quant types +// ============================================================================ + +extern "C" { + // IQ2_TQ + size_t quantize_iq2_tq(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + void iq2tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t grid_out[64]); + void iq2tq_set_grid(const int8_t grid[64]); + // IQ3_TQ + size_t quantize_iq3_tq(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + void iq3tq_train_grid(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t grid_out[128]); + void iq3tq_set_grid(const int8_t grid[128]); + // IQ1_BN + size_t quantize_iq1_bn(const float * src, void * dst, int64_t nrows, int64_t n_per_row, const float * imatrix); + void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t aux_out[32768], int nthread); + void iq1bn_set_aux(const int8_t aux[32768]); + // Q3_PT + void q3pt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, float levels_out[8]); + void q3pt_set_levels(const float levels[8]); + // Q3_KPT + void q3kpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, float levels_out[8]); + void q3kpt_set_levels(const float levels[8]); + // Q4_DPT + void q4dpt_train_levels(const float * data, int64_t nrow, int64_t n_per_row, const float * imatrix, int8_t levels_out[16]); + void q4dpt_set_levels(const int8_t levels[16]); + // Q2_KPT + const float * q2kpt_get_levels(void); + void q2kpt_prepare_levels(int64_t nrows, int64_t n_per_row); + void q2kpt_free_levels(void); +} + +// ============================================================================ +// Section 2: Data structures +// ============================================================================ + +struct config { + std::string model_path; + std::string imatrix_path; + std::vector quant_types; + float target_bpw = 0.0f; + float bpw_tol_high = 0.0f; // can be up to this much above target + float bpw_tol_low = 0.2f; // can be up to this much below target + std::string test_data_path; // optional test text file + std::vector test_sizes = {32, 128, 512}; + int64_t min_elements = 40000; + ggml_type output_tensor_type = GGML_TYPE_COUNT; // default: highest from list + int max_iterations = 100; + std::string output_path; + int n_threads = 1; +}; + +struct tensor_info { + std::string name; // e.g., "blk.0.attn_q.weight" + std::string role; // e.g., "attn_q" + int layer; // e.g., 0; -1 for global tensors + ggml_type orig_type; + int64_t ne[4] = {}; + size_t n_elements = 0; +}; + +// One captured MUL_MAT operation +struct mul_mat_capture { + std::string weight_name; // full name, e.g., "blk.0.attn_q.weight" + std::string role; // e.g., "attn_q" + int layer; + + // Weight tensor metadata (needed to read data from file later) + ggml_type weight_type; + int64_t weight_ne0, weight_ne1; // ne0=input features, ne1=output features + + // Captured MUL_MAT input (src[1]) — always F32 after the kernel + std::vector input_data; + int64_t input_ne0, input_ne1; // [ne0=weight_ne0, ne1=n_tokens] + + // Captured reference MUL_MAT output — always F32 + std::vector ref_output_data; + int64_t ref_ne0, ref_ne1; // [ne0=weight_ne1, ne1=n_tokens] +}; + +// Cost of assigning a specific quant type to a role +struct cost_entry { + double kld; // average KLD across all captures for this role + double bpw; // bits per weight for this quant type +}; + +// ============================================================================ +// Section 3: Utility functions +// ============================================================================ + +static std::string extract_role(const std::string & name) { + // "blk.0.attn_q.weight" -> "attn_q" + // "blk.0.ffn_gate.weight" -> "ffn_gate" + // "output.weight" -> "output" + // "token_embd.weight" -> "token_embd" + static const std::regex layer_re("blk\\.\\d+\\.([^.]+)\\.weight"); + static const std::regex global_re("([^.]+)\\.weight"); + + std::smatch m; + if (std::regex_match(name, m, layer_re) && m.size() > 1) { + return m[1].str(); + } + if (std::regex_match(name, m, global_re) && m.size() > 1) { + return m[1].str(); + } + return name; +} + +static bool is_quantizable_weight(const tensor_info & ti, int64_t min_elements) { + // Must end with .weight + if (ti.name.size() < 8 || ti.name.substr(ti.name.size() - 7) != ".weight") return false; + // Must be 2D (matrix) + if (ti.ne[2] != 1 || ti.ne[3] != 1) return false; + // Must have enough elements + if ((int64_t)ti.n_elements < min_elements) return false; + // Skip norms (1D-like: small ne[0] or ne[1] == 1) + if (ti.ne[0] <= 1 || ti.ne[1] <= 1) return false; + return true; +} + +static double compute_bpw(ggml_type type) { + return ggml_get_bpw(type); +} + +// Get the quant types sorted by BPW (lowest first) +static std::vector sorted_by_bpw(const std::vector & types) { + std::vector result = types; + std::sort(result.begin(), result.end(), [](ggml_type a, ggml_type b) { + return compute_bpw(a) < compute_bpw(b); + }); + return result; +} + +// Get the highest-BPW quant type from a list +static ggml_type highest_bpw(const std::vector & types) { + if (types.empty()) return GGML_TYPE_COUNT; + ggml_type best = types[0]; + for (auto t : types) { + if (compute_bpw(t) > compute_bpw(best)) best = t; + } + return best; +} + +// Check if a quant type supports get_rows (needed for token_embd) +// Per-tensor-trained types (Q3_PT, Q3_KPT, Q4_DPT, Q2_KPT, Q2_DPT) do NOT. +static bool supports_get_rows(ggml_type type) { + return type != GGML_TYPE_Q3_PT && type != GGML_TYPE_Q3_KPT && + type != GGML_TYPE_Q4_DPT && type != GGML_TYPE_Q2_KPT && + type != GGML_TYPE_Q2_DPT; +} + +// Get the quant type with BPW closest to a target BPW +static ggml_type closest_bpw(const std::vector & types, float target) { + if (types.empty()) return GGML_TYPE_COUNT; + ggml_type best = types[0]; + double best_diff = fabs(compute_bpw(best) - target); + for (auto t : types) { + double diff = fabs(compute_bpw(t) - target); + if (diff < best_diff) { + best_diff = diff; + best = t; + } + } + return best; +} + +// Get the quant type with BPW closest to target, filtered by a predicate +template +static ggml_type closest_bpw_if(const std::vector & types, float target, Pred pred) { + if (types.empty()) return GGML_TYPE_COUNT; + ggml_type best = GGML_TYPE_COUNT; + double best_diff = 1e30; + for (auto t : types) { + if (!pred(t)) continue; + double diff = fabs(compute_bpw(t) - target); + if (diff < best_diff) { + best_diff = diff; + best = t; + } + } + return best; +} + +// Compute KLD between two rows treated as probability distributions (after softmax) +// p = softmax(ref), q = softmax(quant) +// KLD = sum(p * log(p/q)) +static double compute_kld_row(const float * ref, const float * quant, int64_t n) { + // Find max for numerical stability + float max_ref = -1e30f, max_qt = -1e30f; + for (int64_t i = 0; i < n; i++) { + if (ref[i] > max_ref) max_ref = ref[i]; + if (quant[i] > max_qt) max_qt = quant[i]; + } + + // Compute softmax + std::vector p(n), q(n); + double sum_p = 0, sum_q = 0; + for (int64_t i = 0; i < n; i++) { + p[i] = expf(ref[i] - max_ref); + q[i] = expf(quant[i] - max_qt); + sum_p += p[i]; + sum_q += q[i]; + } + float inv_sum_p = 1.0f / (float)sum_p; + float inv_sum_q = 1.0f / (float)sum_q; + for (int64_t i = 0; i < n; i++) { + p[i] *= inv_sum_p; + q[i] *= inv_sum_q; + } + + // KLD = sum(p * log(p/q)) + const float eps = 1e-10f; + double kld = 0; + for (int64_t i = 0; i < n; i++) { + if (p[i] > eps) { + float q_clipped = std::max(q[i], eps); + kld += (double)p[i] * log((double)p[i] / (double)q_clipped); + } + } + return kld; +} + +// Compute average KLD across all rows of two F32 matrices of the same shape [ne0, ne1] +static double compute_avg_kld(const float * ref, const float * quant, int64_t ne0, int64_t ne1) { + double total_kld = 0; + int64_t valid_rows = 0; + for (int64_t row = 0; row < ne1; row++) { + const float * ref_row = ref + row * ne0; + const float * quant_row = quant + row * ne0; + double kld = compute_kld_row(ref_row, quant_row, ne0); + if (std::isfinite(kld)) { + total_kld += kld; + valid_rows++; + } + } + return valid_rows > 0 ? total_kld / valid_rows : 1e30; +} + +// Read a tensor's raw data from the GGUF file and dequantize to F32 +static bool read_tensor_f32(const std::string & fname, const struct gguf_context * gguf_ctx, + const std::string & tensor_name, std::vector & out) { + int64_t tid = gguf_find_tensor(gguf_ctx, tensor_name.c_str()); + if (tid < 0) { + LOG_ERR("Tensor '%s' not found in GGUF\n", tensor_name.c_str()); + return false; + } + + ggml_type type = gguf_get_tensor_type(gguf_ctx, tid); + size_t offset = gguf_get_tensor_offset(gguf_ctx, tid); + + // Compute total elements from tensor dimensions stored in gguf + // We need to read the ggml context to get the dimensions + // Alternative: just read the raw bytes and dequantize + size_t raw_size = gguf_get_tensor_size(gguf_ctx, tid); + + FILE * f = fopen(fname.c_str(), "rb"); + if (!f) { + LOG_ERR("Failed to open model file: %s\n", fname.c_str()); + return false; + } + + size_t data_offset = gguf_get_data_offset(gguf_ctx); + fseek(f, data_offset + offset, SEEK_SET); + + std::vector raw(raw_size); + if (fread(raw.data(), 1, raw_size, f) != raw_size) { + LOG_ERR("Failed to read tensor data for '%s'\n", tensor_name.c_str()); + fclose(f); + return false; + } + fclose(f); + + // Compute n_elements + size_t type_size = ggml_type_size(type); + int64_t blck_size = ggml_blck_size(type); + size_t n_elements = (raw_size / type_size) * blck_size; + + out.resize(n_elements); + if (type == GGML_TYPE_F32) { + memcpy(out.data(), raw.data(), n_elements * sizeof(float)); + } else if (type == GGML_TYPE_F16) { + const ggml_fp16_t * src = (const ggml_fp16_t *)raw.data(); + for (size_t i = 0; i < n_elements; i++) { + out[i] = ggml_fp16_to_fp32(src[i]); + } + } else if (type == GGML_TYPE_BF16) { + const ggml_bf16_t * src = (const ggml_bf16_t *)raw.data(); + for (size_t i = 0; i < n_elements; i++) { + out[i] = ggml_bf16_to_fp32(src[i]); + } + } else { + // Use ggml's type traits for dequantization + const auto * traits = ggml_get_type_traits(type); + if (traits && traits->to_float) { + traits->to_float(raw.data(), out.data(), n_elements, nullptr); + } else { + LOG_ERR("Cannot dequantize type %s for tensor '%s'\n", ggml_type_name(type), tensor_name.c_str()); + return false; + } + } + + return true; +} + +// Get target layers: first, middle, next-to-last +// Layer signature: sorted list of (role, ne0, ne1) for all quantizable weight tensors +// in a layer. Two layers are equivalent iff their signatures match. +using layer_signature = std::vector>; + +static layer_signature compute_layer_signature(int layer, const std::vector & all_tensors, int64_t min_elements) { + layer_signature sig; + for (const auto & ti : all_tensors) { + if (ti.layer != layer) continue; + if (!is_quantizable_weight(ti, min_elements)) continue; + sig.emplace_back(ti.role, ti.ne[0], ti.ne[1]); + } + std::sort(sig.begin(), sig.end()); + return sig; +} + +// Identify layer equivalence classes and pick representative layers from each. +// Returns: vector of (class_index, representative_layer_indices, all_layer_indices) +struct layer_class_info { + size_t class_index; + std::vector reps; // representative layers (first, middle, last) + std::vector all_layers; // all layers in this class + layer_signature signature; // the signature for diagnostics +}; + +static std::vector get_layer_equivalence_classes( + int n_layer, const std::vector & all_tensors, int64_t min_elements) { + // Compute signatures and group by equivalence class + std::map> class_groups; + for (int l = 0; l < n_layer; l++) { + auto sig = compute_layer_signature(l, all_tensors, min_elements); + if (sig.empty()) continue; // skip layers with no quantizable tensors + class_groups[sig].push_back(l); + } + + // Build result with representatives (first, middle, last of each class) + std::vector result; + size_t class_idx = 0; + for (const auto & [sig, layers] : class_groups) { + layer_class_info info; + info.class_index = class_idx; + info.all_layers = layers; + info.signature = sig; + + info.reps.push_back(layers.front()); + if (layers.size() > 1) { + info.reps.push_back(layers[layers.size() / 2]); + } + if (layers.size() > 2) { + info.reps.push_back(layers.back()); + } + // Deduplicate + std::sort(info.reps.begin(), info.reps.end()); + info.reps.erase(std::unique(info.reps.begin(), info.reps.end()), info.reps.end()); + + result.push_back(std::move(info)); + class_idx++; + } + return result; +} + +// Check if a quant type requires per-tensor training +static bool requires_training(ggml_type type) { + return type == GGML_TYPE_IQ2_TQ || type == GGML_TYPE_IQ3_TQ || + type == GGML_TYPE_IQ1_BN || type == GGML_TYPE_Q3_PT || + type == GGML_TYPE_Q3_KPT || type == GGML_TYPE_Q4_DPT || + type == GGML_TYPE_Q2_KPT; +} + +// ============================================================================ +// Section 4: Eval callback for activation capture +// ============================================================================ + +struct capture_state { + // Set of weight tensor names we want to capture MUL_MAT for + std::unordered_set target_weight_names; + + // Map from weight tensor name -> role + std::unordered_map weight_to_role; + // Map from weight tensor name -> layer index + std::unordered_map weight_to_layer; + + // Captures, organized by role + std::unordered_map> captures_by_role; + + // Temporary buffer for non-host tensor data + std::vector tmp_data; + + // Count of captured MUL_MATs + int captured = 0; +}; + +static bool capture_callback(ggml_tensor * t, bool ask, void * user_data) { + auto * state = (capture_state *) user_data; + + if (t->op != GGML_OP_MUL_MAT) return false; + if (!t->src[0] || !t->src[1]) return false; + + const char * weight_name = t->src[0]->name; + + // Check if this MUL_MAT uses one of our target weight tensors + if (state->target_weight_names.find(weight_name) == state->target_weight_names.end()) { + return false; // Not interested + } + + if (ask) { + return true; // Yes, we want the output data + } + + // ask=false: data is available, capture it + mul_mat_capture cap; + cap.weight_name = weight_name; + cap.role = state->weight_to_role[weight_name]; + cap.layer = state->weight_to_layer[weight_name]; + cap.weight_type = t->src[0]->type; + cap.weight_ne0 = t->src[0]->ne[0]; + cap.weight_ne1 = t->src[0]->ne[1]; + + // Capture input (src[1]) + { + size_t nbytes = ggml_nbytes(t->src[1]); + const float * src_ptr = nullptr; + std::vector host_copy; + + if (ggml_backend_buffer_is_host(t->src[1]->buffer)) { + src_ptr = (const float *) t->src[1]->data; + } else { + host_copy.resize(nbytes / sizeof(float)); + ggml_backend_tensor_get(t->src[1], host_copy.data(), 0, nbytes); + src_ptr = host_copy.data(); + } + + cap.input_ne0 = t->src[1]->ne[0]; + cap.input_ne1 = t->src[1]->ne[1]; + cap.input_data.assign(src_ptr, src_ptr + (nbytes / sizeof(float))); + } + + // Capture reference output + { + size_t nbytes = ggml_nbytes(t); + const float * src_ptr = nullptr; + std::vector host_copy; + + if (ggml_backend_buffer_is_host(t->buffer)) { + src_ptr = (const float *) t->data; + } else { + host_copy.resize(nbytes / sizeof(float)); + ggml_backend_tensor_get(t, host_copy.data(), 0, nbytes); + src_ptr = host_copy.data(); + } + + cap.ref_ne0 = t->ne[0]; + cap.ref_ne1 = t->ne[1]; + cap.ref_output_data.assign(src_ptr, src_ptr + (nbytes / sizeof(float))); + } + + state->captures_by_role[cap.role].push_back(std::move(cap)); + state->captured++; + + return true; +} + +// ============================================================================ +// Section 5: Imatrix loading (simplified from quantize.cpp) +// ============================================================================ + +static bool load_imatrix_data(const std::string & imatrix_file, + std::unordered_map> & imatrix_data) { + struct ggml_context * ctx = nullptr; + struct gguf_init_params meta_params = {false, &ctx}; + struct gguf_context * ctx_gguf = gguf_init_from_file(imatrix_file.c_str(), meta_params); + if (!ctx_gguf) { + LOG_ERR("Failed to open imatrix file: %s\n", imatrix_file.c_str()); + return false; + } + + const std::string sums_suffix{".in_sum2"}; + const std::string counts_suffix{".counts"}; + + std::map> sums_counts_for; + for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) { + std::string name = cur->name; + if (name.empty()) continue; + if (name.size() > sums_suffix.size() && name.substr(name.size() - sums_suffix.size()) == sums_suffix) { + sums_counts_for[name.substr(0, name.size() - sums_suffix.size())].first = cur; + } else if (name.size() > counts_suffix.size() && name.substr(name.size() - counts_suffix.size()) == counts_suffix) { + sums_counts_for[name.substr(0, name.size() - counts_suffix.size())].second = cur; + } + } + + for (const auto & sc : sums_counts_for) { + const auto & name = sc.first; + const auto * sums = sc.second.first; + const auto * counts = sc.second.second; + if (!sums || !counts) continue; + + int64_t ne0 = sums->ne[0]; + int64_t ne1 = sums->ne[1]; + auto & e = imatrix_data[name]; + e.resize(ggml_nelements(sums)); + for (int64_t j = 0; j < ne1; ++j) { + float count = ((const float *) counts->data)[j]; + if (count > 0) { + for (int64_t i = 0; i < ne0; ++i) { + e[j * ne0 + i] = ((const float *) sums->data)[j * ne0 + i] / count; + } + } else { + for (int64_t i = 0; i < ne0; ++i) { + e[j * ne0 + i] = 1.0f; + } + } + } + } + + gguf_free(ctx_gguf); + ggml_free(ctx); + + LOG("Loaded %d imatrix entries from %s\n", (int)imatrix_data.size(), imatrix_file.c_str()); + return true; +} + +// Look up imatrix data for a tensor. Returns a uniform vector of 1s if not found. +static std::vector get_imatrix_for_tensor( + const std::unordered_map> & imatrix_data, + const std::string & tensor_name, int64_t n_per_row) { + auto it = imatrix_data.find(tensor_name); + if (it != imatrix_data.end() && (int64_t)it->second.size() >= n_per_row) { + return it->second; + } + // Not found or wrong size — return uniform + return std::vector(n_per_row, 1.0f); +} + +// ============================================================================ +// Section 6: MUL_MAT evaluation and cost matrix +// ============================================================================ + +// Result of quantizing a weight tensor — includes quantized data and optional per-tensor levels +struct quant_result { + std::vector data; // quantized weight data + std::vector levels; // per-tensor levels/grid (empty if not needed) +}; + +// Train per-tensor params and quantize a weight tensor to the target type. +static quant_result quantize_weight_to_type( + ggml_type target_type, + const float * f32_data, + int64_t nrows, + int64_t n_per_row, + const float * imatrix) { + quant_result result; + size_t quant_size = nrows * ggml_row_size(target_type, n_per_row); + result.data.resize(quant_size); + + // Train per-tensor params if needed + if (target_type == GGML_TYPE_IQ2_TQ) { + int8_t grid[64]; + iq2tq_train_grid(f32_data, nrows, n_per_row, imatrix, grid); + iq2tq_set_grid(grid); + // Store a copy of the grid for MUL_MAT dequantization + result.levels.assign((const uint8_t *)grid, (const uint8_t *)grid + sizeof(grid)); + quantize_iq2_tq(f32_data, result.data.data(), nrows, n_per_row, imatrix); + } else if (target_type == GGML_TYPE_IQ3_TQ) { + int8_t grid[128]; + iq3tq_train_grid(f32_data, nrows, n_per_row, imatrix, grid); + iq3tq_set_grid(grid); + result.levels.assign((const uint8_t *)grid, (const uint8_t *)grid + sizeof(grid)); + quantize_iq3_tq(f32_data, result.data.data(), nrows, n_per_row, imatrix); + } else if (target_type == GGML_TYPE_IQ1_BN) { + int8_t aux[32768]; + iq1bn_train_codebook(f32_data, nrows, n_per_row, imatrix, aux, 1); + iq1bn_set_aux(aux); + result.levels.assign((const uint8_t *)aux, (const uint8_t *)aux + sizeof(aux)); + quantize_iq1_bn(f32_data, result.data.data(), nrows, n_per_row, imatrix); + } else if (target_type == GGML_TYPE_Q3_PT) { + float levels[8]; + q3pt_train_levels(f32_data, nrows, n_per_row, imatrix, levels); + q3pt_set_levels(levels); + // Store a copy of the levels for MUL_MAT dequantization + result.levels.assign((const uint8_t *)levels, (const uint8_t *)levels + sizeof(levels)); + ggml_quantize_chunk(target_type, f32_data, result.data.data(), 0, nrows, n_per_row, imatrix); + } else if (target_type == GGML_TYPE_Q3_KPT) { + float levels[8]; + q3kpt_train_levels(f32_data, nrows, n_per_row, imatrix, levels); + q3kpt_set_levels(levels); + result.levels.assign((const uint8_t *)levels, (const uint8_t *)levels + sizeof(levels)); + ggml_quantize_chunk(target_type, f32_data, result.data.data(), 0, nrows, n_per_row, imatrix); + } else if (target_type == GGML_TYPE_Q4_DPT) { + int8_t levels[16]; + q4dpt_train_levels(f32_data, nrows, n_per_row, imatrix, levels); + q4dpt_set_levels(levels); + result.levels.assign((const uint8_t *)levels, (const uint8_t *)levels + sizeof(levels)); + ggml_quantize_chunk(target_type, f32_data, result.data.data(), 0, nrows, n_per_row, imatrix); + } else if (target_type == GGML_TYPE_Q2_KPT) { + q2kpt_prepare_levels(nrows, n_per_row); + // Store the Q2_KPT levels (they're float[8]) + const float * lv = q2kpt_get_levels(); + if (lv) result.levels.assign((const uint8_t *)lv, (const uint8_t *)lv + 8 * sizeof(float)); + ggml_quantize_chunk(target_type, f32_data, result.data.data(), 0, nrows, n_per_row, imatrix); + } else { + // Standard quant type + ggml_quantize_init(target_type); + ggml_quantize_chunk(target_type, f32_data, result.data.data(), 0, nrows, n_per_row, imatrix); + } + + return result; +} + +// Run MUL_MAT: result = weight^T * input +// weight: [ne0, ne1] in some quantized type +// input: [ne0, n_tokens] in F32 +// result: [ne1, n_tokens] in F32 +// quant_levels: optional per-tensor levels data (for Q3_KPT, Q4_DPT, etc.) +// backend: pre-created backend (CPU or CUDA) — reused across calls +static bool eval_mul_mat(ggml_type weight_type, const void * weight_data, + int64_t weight_ne0, int64_t weight_ne1, + const float * input_data, int64_t input_ne0, int64_t input_ne1, + const std::vector & quant_levels, + std::vector & result_data, + ggml_backend_t backend) { + // Size calculations + size_t weight_bytes = ggml_row_size(weight_type, weight_ne0) * weight_ne1; + size_t input_bytes = (size_t)input_ne0 * input_ne1 * sizeof(float); + + // Create ggml context with no_alloc=true; backend allocator will own tensor data + size_t ctx_size = ggml_tensor_overhead() * 16 + ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE, false) + 4096; + struct ggml_init_params params = {(size_t) ctx_size, NULL, /*.no_alloc =*/ true}; + struct ggml_context * ctx = ggml_init(params); + if (!ctx) { + LOG_ERR("Failed to create ggml context for MUL_MAT eval\n"); + return false; + } + + // Create weight tensor + struct ggml_tensor * w = ggml_new_tensor_2d(ctx, weight_type, weight_ne0, weight_ne1); + + // Set per-tensor levels if provided (needed for Q3_KPT, Q4_DPT, etc.) + if (!quant_levels.empty()) { + w->quant_levels = (void *)quant_levels.data(); + } + + // Create input tensor + struct ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, input_ne0, input_ne1); + + // Create MUL_MAT + struct ggml_tensor * result = ggml_mul_mat(ctx, w, x); + + // Allocate all tensors on the provided backend + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend); + if (!buf) { + LOG_ERR("Failed to allocate tensors for MUL_MAT eval (weight %s [%lld,%lld])\n", + ggml_type_name(weight_type), (long long)weight_ne0, (long long)weight_ne1); + ggml_free(ctx); + return false; + } + + // Verify buffers were assigned + if (!w->buffer || !x->buffer || !result->buffer) { + LOG_ERR("Tensor buffers not set after allocation (w=%p, x=%p, res=%p)\n", + (void *)w->buffer, (void *)x->buffer, (void *)result->buffer); + ggml_backend_buffer_free(buf); + ggml_free(ctx); + return false; + } + + // Upload data to tensors + ggml_backend_tensor_set(w, weight_data, 0, weight_bytes); + ggml_backend_tensor_set(x, input_data, 0, input_bytes); + + // Build graph + struct ggml_cgraph * graph = ggml_new_graph_custom(ctx, GGML_DEFAULT_GRAPH_SIZE, false); + ggml_build_forward_expand(graph, result); + + // Compute + enum ggml_status status = ggml_backend_graph_compute(backend, graph); + if (status != GGML_STATUS_SUCCESS) { + LOG_ERR("ggml_backend_graph_compute failed: %d\n", status); + ggml_backend_buffer_free(buf); + ggml_free(ctx); + return false; + } + + // Download result + result_data.resize((size_t)weight_ne1 * input_ne1); + ggml_backend_tensor_get(result, result_data.data(), 0, ggml_nbytes(result)); + + ggml_backend_buffer_free(buf); + ggml_free(ctx); + return true; +} + +// Build the cost matrix: for each (role, quant_type), compute average KLD +// Quant types are evaluated in parallel per capture (each type has its own +// global state so they don't interfere with each other). +static std::map> build_cost_matrix( + const config & cfg, + const std::vector & /*tensors*/, + const std::unordered_map> & captures_by_role, + const std::string & model_path, + const struct gguf_context * gguf_ctx, + const std::unordered_map> & imatrix_data) { + + std::map> cost_matrix; + + // Accumulate KLD per (role, quant_type) + std::mutex accum_mutex; + std::map> kld_sums; + std::map> kld_counts; + + // Cache for F32 weight data: tensor_name → float data + std::unordered_map> weight_cache; + + const int n_parallel = std::max(1, cfg.n_threads); + + // Create a pool of backends — try CUDA first, fall back to CPU + // Each thread gets its own backend to avoid contention. + struct backend_pool { + std::vector backends; + std::atomic next_idx{0}; + std::string backend_name; + + backend_pool(int count) { + // Use ggml_backend_init_best() — picks GPU if available, falls back to CPU + ggml_backend_t be = ggml_backend_init_best(); + if (!be) { + LOG_ERR("Failed to init any backend\n"); + return; + } + backend_name = ggml_backend_name(be); + // Single shared backend — ggml handles concurrency internally + for (int i = 0; i < count; i++) { + backends.push_back(be); + } + } + ~backend_pool() { + if (!backends.empty()) { + ggml_backend_free(backends[0]); + } + } + ggml_backend_t get() { + return backends[next_idx.fetch_add(1) % (int)backends.size()]; + } + }; + + backend_pool pool(n_parallel); + LOG("Phase 3 backend: %s (%d parallel)\n", + pool.backend_name.c_str(), n_parallel); + + // For each role that has captures + for (const auto & [role, captures] : captures_by_role) { + LOG("Building cost matrix for role '%s' (%zu captures, %d parallel threads)...\n", + role.c_str(), captures.size(), n_parallel); + + for (const auto & cap : captures) { + // Read weight data as F32 (cached) + auto cache_it = weight_cache.find(cap.weight_name); + if (cache_it == weight_cache.end()) { + std::vector weight_f32; + if (!read_tensor_f32(model_path, gguf_ctx, cap.weight_name, weight_f32)) { + LOG_WRN(" Failed to read weight data for '%s', skipping\n", cap.weight_name.c_str()); + continue; + } + weight_cache[cap.weight_name] = std::move(weight_f32); + cache_it = weight_cache.find(cap.weight_name); + } + const auto & weight_f32 = cache_it->second; + + // Get imatrix for this tensor + auto imat = get_imatrix_for_tensor(imatrix_data, cap.weight_name, cap.weight_ne0); + + // Evaluate all quant types for this capture in parallel batches + const size_t n_types = cfg.quant_types.size(); + std::vector>> futures; + futures.reserve(n_parallel); + + for (size_t ti = 0; ti < n_types; /* advanced inside */) { + // Launch up to n_parallel evaluations + futures.clear(); + for (int p = 0; p < n_parallel && ti < n_types; p++, ti++) { + ggml_type qtype = cfg.quant_types[ti]; + ggml_backend_t backend = pool.get(); + futures.push_back(std::async(std::launch::async, + [&cap, &weight_f32, &imat, qtype, backend]() -> std::pair { + // Quantize (sets per-type global state — safe since each type is different) + auto qres = quantize_weight_to_type( + qtype, weight_f32.data(), cap.weight_ne1, cap.weight_ne0, imat.data()); + + // Run MUL_MAT with quantized weight + std::vector quant_output; + if (!eval_mul_mat(qtype, qres.data.data(), + cap.weight_ne0, cap.weight_ne1, + cap.input_data.data(), cap.input_ne0, cap.input_ne1, + qres.levels, + quant_output, + backend)) { + return {qtype, std::numeric_limits::quiet_NaN()}; + } + + // Compute KLD + return {qtype, compute_avg_kld(cap.ref_output_data.data(), quant_output.data(), + cap.ref_ne0, cap.ref_ne1)}; + })); + } + + // Collect results + for (auto & f : futures) { + auto [qtype, kld] = f.get(); + if (std::isfinite(kld)) { + std::lock_guard lock(accum_mutex); + kld_sums[role][qtype] += kld; + kld_counts[role][qtype]++; + } + } + } + } + + // Free cache entries for this role's captures (no longer needed) + for (const auto & cap : captures) { + weight_cache.erase(cap.weight_name); + } + + // Build cost entries from accumulated KLD + for (ggml_type qtype : cfg.quant_types) { + int n_valid = kld_counts[role][qtype]; + cost_entry entry; + entry.kld = n_valid > 0 ? kld_sums[role][qtype] / n_valid : 1e30; + entry.bpw = compute_bpw(qtype); + cost_matrix[role][qtype] = entry; + } + } + + return cost_matrix; +} + +// ============================================================================ +// Section 6b: Fusion map — split fused MUL_MAT scores to component tensors +// ============================================================================ + +// Maps a fused role name to its component roles. +// When the model does a fused MUL_MAT (e.g., QKV projection as one operation), +// the captured KLD is split proportionally among the individual tensor roles +// so the optimizer can assign types to each component independently. +static const std::map> fusion_map = { + {"attn_qkv", {"attn_q", "attn_k", "attn_v"}}, + // Add more fusion patterns here for other models: + // {"ffn_gate_up", {"ffn_gate", "ffn_up"}}, +}; + +// ============================================================================ +// Section 7: Optimization +// ============================================================================ + +struct role_info { + std::string role; + size_t n_elements; // total elements across all layers + size_t n_bytes_orig; // original size in bytes +}; + +// Post-process the cost matrix: for fused roles, split KLD proportionally +// among component roles (by element count). Component roles that already +// have their own captures keep their directly-measured KLD. +static void split_fused_roles( + std::map> & cost_matrix, + const std::map & roles) { + + for (const auto & [fused_role, components] : fusion_map) { + auto fused_it = cost_matrix.find(fused_role); + if (fused_it == cost_matrix.end()) continue; + + // Compute element counts for each component role + size_t total_comp_elements = 0; + std::vector comp_elems; + for (const auto & comp : components) { + auto rit = roles.find(comp); + size_t ne = (rit != roles.end()) ? rit->second.n_elements : 0; + comp_elems.push_back(ne); + total_comp_elements += ne; + } + if (total_comp_elements == 0) continue; + + // For each quant type, create proportional cost entries for component roles + for (const auto & [qtype, fused_entry] : fused_it->second) { + for (size_t i = 0; i < components.size(); i++) { + double fraction = (double)comp_elems[i] / (double)total_comp_elements; + + auto & comp_entries = cost_matrix[components[i]]; + if (comp_entries.count(qtype)) { + // Component already has its own measurement — keep it, don't overwrite + continue; + } + + cost_entry comp_entry; + comp_entry.kld = fused_entry.kld * fraction; + comp_entry.bpw = fused_entry.bpw; + comp_entries[qtype] = comp_entry; + } + } + + LOG("Split fused role '%s' KLD into components:", fused_role.c_str()); + for (size_t i = 0; i < components.size(); i++) { + double fraction = (double)comp_elems[i] / (double)total_comp_elements; + LOG(" %s: %.1f%% (%zu elements)\n", + components[i].c_str(), fraction * 100, comp_elems[i]); + } + } +} + +struct assignment { + std::map role_to_type; + double total_kld; + double total_bpw; +}; + +// Compute the total BPW for an assignment (quantizable roles + non-quantizable overhead) +static double compute_total_bpw( + const std::map & role_to_type, + const std::map & roles, + size_t total_all_elements, + double non_quantizable_bits) { + double weighted_sum = 0; + for (const auto & [role, type] : role_to_type) { + auto it = roles.find(role); + if (it == roles.end()) continue; + weighted_sum += compute_bpw(type) * it->second.n_elements; + } + weighted_sum += non_quantizable_bits; + return total_all_elements > 0 ? weighted_sum / total_all_elements : 0; +} + +// Compute the total KLD for an assignment +static double compute_total_kld( + const std::map & role_to_type, + const std::map> & cost_matrix) { + double total = 0; + for (const auto & [role, type] : role_to_type) { + auto it = cost_matrix.find(role); + if (it == cost_matrix.end()) continue; + auto it2 = it->second.find(type); + if (it2 != it->second.end()) { + total += it2->second.kld; + } + } + return total; +} + +static assignment optimize_assignment( + const config & cfg, + const std::map> & cost_matrix, + const std::map & roles, + size_t total_all_elements, + double non_quantizable_bits) { + + auto types_by_bpw = sorted_by_bpw(cfg.quant_types); + if (types_by_bpw.empty()) { + LOG_ERR("No quant types specified\n"); + return {}; + } + + // Initialize: assign each role the lowest-BPW quant type (greedy fill) + // We start from the bottom and work up until we hit the BPW budget + std::map current; + for (const auto & [role, info] : roles) { + // Start with the lowest-BPW type that has a valid KLD + auto it = cost_matrix.find(role); + if (it == cost_matrix.end()) continue; + ggml_type best = GGML_TYPE_COUNT; + for (auto qt : types_by_bpw) { + auto it2 = it->second.find(qt); + if (it2 != it->second.end() && it2->second.kld < 1e29) { + best = qt; + break; + } + } + if (best == GGML_TYPE_COUNT) continue; + current[role] = best; + } + + // Greedily upgrade roles to higher-quality types until we hit the BPW budget + // For each role, find the "best value" upgrade (most KLD reduction per BPW increase) + bool improved = true; + while (improved) { + improved = false; + (void)0; // cur_bpw used only for debugging + + double best_ratio = 0; + std::string best_role; + ggml_type best_type = GGML_TYPE_COUNT; + + for (const auto & [role, cur_type] : current) { + // Find next higher-BPW type + auto it = cost_matrix.find(role); + if (it == cost_matrix.end()) continue; + + double cur_kld = it->second.count(cur_type) ? it->second.at(cur_type).kld : 1e30; + + for (auto qt : types_by_bpw) { + if (compute_bpw(qt) <= compute_bpw(cur_type)) continue; // skip lower/same + auto it2 = it->second.find(qt); + if (it2 == it->second.end() || it2->second.kld >= 1e29) continue; + + // Check if this upgrade would exceed BPW budget + auto test = current; + test[role] = qt; + double test_bpw = compute_total_bpw(test, roles, total_all_elements, non_quantizable_bits); + if (test_bpw > cfg.target_bpw + cfg.bpw_tol_high) continue; + + // Compute improvement ratio (KLD reduction per BPW increase) + double kld_reduction = cur_kld - it2->second.kld; + double bpw_increase = compute_bpw(qt) - compute_bpw(cur_type); + if (bpw_increase <= 0) continue; + double ratio = kld_reduction / bpw_increase; + + if (ratio > best_ratio) { + best_ratio = ratio; + best_role = role; + best_type = qt; + } + } + } + + if (!best_role.empty() && best_type != GGML_TYPE_COUNT) { + current[best_role] = best_type; + improved = true; + } + } + + // Iterative improvement: try swapping roles up/down + std::set>> visited; + auto make_key = [&](const std::map & a) { + std::vector> v(a.begin(), a.end()); + std::sort(v.begin(), v.end()); + return v; + }; + + assignment best_assign; + best_assign.role_to_type = current; + best_assign.total_bpw = compute_total_bpw(current, roles, total_all_elements, non_quantizable_bits); + best_assign.total_kld = compute_total_kld(current, cost_matrix); + visited.insert(make_key(current)); + + for (int iter = 0; iter < cfg.max_iterations; iter++) { + bool found_improvement = false; + + // Try upgrading each role and downgrading another to compensate + for (const auto & [role_up, cur_type_up] : best_assign.role_to_type) { + auto it_up = cost_matrix.find(role_up); + if (it_up == cost_matrix.end()) continue; + + for (auto qt_up : types_by_bpw) { + if (compute_bpw(qt_up) <= compute_bpw(cur_type_up)) continue; + auto it2_up = it_up->second.find(qt_up); + if (it2_up == it_up->second.end() || it2_up->second.kld >= 1e29) continue; + + (void)0; // bpw_increase no longer used directly + double kld_decrease_up = (it_up->second.count(cur_type_up) ? + it_up->second.at(cur_type_up).kld : 1e30) - it2_up->second.kld; + + // Try downgrading each other role to compensate + for (const auto & [role_dn, cur_type_dn] : best_assign.role_to_type) { + if (role_dn == role_up) continue; + auto it_dn = cost_matrix.find(role_dn); + if (it_dn == cost_matrix.end()) continue; + + for (auto qt_dn : types_by_bpw) { + if (compute_bpw(qt_dn) >= compute_bpw(cur_type_dn)) continue; + auto it2_dn = it_dn->second.find(qt_dn); + if (it2_dn == it_dn->second.end() || it2_dn->second.kld >= 1e29) continue; + + // Check BPW constraint + auto test = best_assign.role_to_type; + test[role_up] = qt_up; + test[role_dn] = qt_dn; + double test_bpw = compute_total_bpw(test, roles, total_all_elements, non_quantizable_bits); + if (test_bpw > cfg.target_bpw + cfg.bpw_tol_high) continue; + if (test_bpw < cfg.target_bpw - cfg.bpw_tol_low) continue; + + // Check if already visited + auto key = make_key(test); + if (visited.count(key)) continue; + + // Compute KLD change + double kld_increase_dn = it2_dn->second.kld - + (it_dn->second.count(cur_type_dn) ? it_dn->second.at(cur_type_dn).kld : 1e30); + double net_kld_change = -kld_decrease_up + kld_increase_dn; + + if (net_kld_change < -1e-10) { + // Improvement found! + double new_kld = best_assign.total_kld + net_kld_change; + visited.insert(key); + + best_assign.role_to_type = test; + best_assign.total_bpw = test_bpw; + best_assign.total_kld = new_kld; + found_improvement = true; + break; + } + visited.insert(key); + } + if (found_improvement) break; + } + if (found_improvement) break; + } + if (found_improvement) break; + } + + if (!found_improvement) { + // Also try single-role upgrades (if BPW budget allows) + for (const auto & [role, cur_type] : best_assign.role_to_type) { + auto it = cost_matrix.find(role); + if (it == cost_matrix.end()) continue; + + for (auto qt : types_by_bpw) { + if (compute_bpw(qt) <= compute_bpw(cur_type)) continue; + auto it2 = it->second.find(qt); + if (it2 == it->second.end() || it2->second.kld >= 1e29) continue; + + auto test = best_assign.role_to_type; + test[role] = qt; + double test_bpw = compute_total_bpw(test, roles, total_all_elements, non_quantizable_bits); + if (test_bpw > cfg.target_bpw + cfg.bpw_tol_high) continue; + + auto key = make_key(test); + if (visited.count(key)) continue; + + double kld_change = it2->second.kld - + (it->second.count(cur_type) ? it->second.at(cur_type).kld : 1e30); + if (kld_change < -1e-10) { + best_assign.role_to_type = test; + best_assign.total_bpw = test_bpw; + best_assign.total_kld += kld_change; + visited.insert(key); + found_improvement = true; + break; + } + visited.insert(key); + } + if (found_improvement) break; + } + } + + if (!found_improvement) break; + + LOG(" Iteration %d: BPW=%.4f, total_KLD=%.6f\n", + iter, best_assign.total_bpw, best_assign.total_kld); + } + + return best_assign; +} + +// ============================================================================ +// Section 8: Output +// ============================================================================ + +static bool write_tensor_type_file(const std::string & path, + const std::map & role_to_type, + ggml_type output_tensor_type, + ggml_type token_embd_tensor_type) { + std::ofstream file(path); + if (!file) { + LOG_ERR("Failed to open output file: %s\n", path.c_str()); + return false; + } + + // Write token_embd type — anchor with ^ so it only matches the global tensor, + // not e.g. some hypothetical "blk.X.something_token_embd.weight" + if (token_embd_tensor_type != GGML_TYPE_COUNT) { + file << "^token_embd=" << ggml_type_name(token_embd_tensor_type) << "\n"; + } + + // Write output tensor type — anchor with ^ so "output" matches "output.weight" + // but NOT "blk.X.attn_output.weight" + if (output_tensor_type != GGML_TYPE_COUNT) { + file << "^output=" << ggml_type_name(output_tensor_type) << "\n"; + } + + // Write per-role types (skip global tensors already written above). + // Use \.ROLE\. patterns so they match between dots, e.g.: + // "ffn_down" → "\.ffn_down\." matches blk.X.ffn_down.weight + // "attn_q" → "\.attn_q\." matches blk.X.attn_q.weight but NOT blk.X.attn_qkv.weight + // "attn_qkv" → "\.attn_qkv\." matches blk.X.attn_qkv.weight + for (const auto & [role, type] : role_to_type) { + if (role == "token_embd" || role == "output") continue; + file << "\\." << role << "\\.=" << ggml_type_name(type) << "\n"; + } + + file.close(); + LOG("Wrote tensor-type-file to %s\n", path.c_str()); + return true; +} + +// ============================================================================ +// Section 9: Argument parsing +// ============================================================================ + +static ggml_type parse_ggml_type_str(const char * arg) { + for (int i = 0; i < GGML_TYPE_COUNT; ++i) { + auto type = (ggml_type)i; + const auto * name = ggml_type_name(type); + if (name && strcasecmp(name, arg) == 0) { + return type; + } + } + return GGML_TYPE_COUNT; +} + +static std::vector parse_quant_list(const std::string & s) { + std::vector result; + std::istringstream ss(s); + std::string token; + while (std::getline(ss, token, ',')) { + // Trim whitespace + token.erase(0, token.find_first_not_of(" \t")); + token.erase(token.find_last_not_of(" \t") + 1); + ggml_type t = parse_ggml_type_str(token.c_str()); + if (t == GGML_TYPE_COUNT) { + LOG_ERR("Unknown quant type: '%s'\n", token.c_str()); + exit(1); + } + result.push_back(t); + } + return result; +} + +static void print_usage(const char * prog) { + LOG("Usage: %s [options]\n", prog); + LOG("\n"); + LOG("Options:\n"); + LOG(" -m, --model PATH Path to input GGUF model (required)\n"); + LOG(" -i, --imatrix PATH Path to importance matrix file (required)\n"); + LOG(" -q, --quants LIST Comma-separated list of candidate quant types (required)\n"); + LOG(" e.g., IQ1_BN,IQ2_TQ,IQ3_TQ,Q4_KPT,Q6_K\n"); + LOG(" -b, --target-bpw N Target bits per weight (required)\n"); + LOG(" -o, --output PATH Output tensor-type-file path (required)\n"); + LOG(" --bpw-tolerance HIGH,LOW BPW tolerance: +HIGH, -LOW from target (default: +0,-0.2)\n"); + LOG(" --test-data PATH Text file for test inputs (optional, synthetic if not given)\n"); + LOG(" --test-sizes S1,S2,S3 Token counts for test inputs (default: 32,128,512)\n"); + LOG(" --min-elements N Skip tensors with fewer elements (default: 40000)\n"); + LOG(" --output-tensor-type T Quant type for output.weight (default: highest from list)\n"); + LOG(" --max-iterations N Max optimization iterations (default: 100)\n"); + LOG(" --threads N Number of threads (default: 1)\n"); +} + +static config parse_args(int argc, char ** argv) { + config cfg; + for (int i = 1; i < argc; i++) { + std::string arg = argv[i]; + if ((arg == "-m" || arg == "--model") && i + 1 < argc) { + cfg.model_path = argv[++i]; + } else if ((arg == "-i" || arg == "--imatrix") && i + 1 < argc) { + cfg.imatrix_path = argv[++i]; + } else if ((arg == "-q" || arg == "--quants") && i + 1 < argc) { + cfg.quant_types = parse_quant_list(argv[++i]); + } else if ((arg == "-b" || arg == "--target-bpw") && i + 1 < argc) { + cfg.target_bpw = std::stof(argv[++i]); + } else if ((arg == "-o" || arg == "--output") && i + 1 < argc) { + cfg.output_path = argv[++i]; + } else if (arg == "--bpw-tolerance" && i + 1 < argc) { + std::string tol = argv[++i]; + // Parse "+HIGH,-LOW" format + size_t comma = tol.find(','); + if (comma != std::string::npos) { + cfg.bpw_tol_high = std::stof(tol.substr(0, comma)); + cfg.bpw_tol_low = std::stof(tol.substr(comma + 1)); + } else { + cfg.bpw_tol_high = std::stof(tol); + } + } else if (arg == "--test-data" && i + 1 < argc) { + cfg.test_data_path = argv[++i]; + } else if (arg == "--test-sizes" && i + 1 < argc) { + std::string sizes = argv[++i]; + std::istringstream ss(sizes); + std::string token; + cfg.test_sizes.clear(); + while (std::getline(ss, token, ',')) { + cfg.test_sizes.push_back(std::stoi(token)); + } + } else if (arg == "--min-elements" && i + 1 < argc) { + cfg.min_elements = std::stoll(argv[++i]); + } else if (arg == "--output-tensor-type" && i + 1 < argc) { + cfg.output_tensor_type = parse_ggml_type_str(argv[++i]); + } else if (arg == "--max-iterations" && i + 1 < argc) { + cfg.max_iterations = std::stoi(argv[++i]); + } else if (arg == "--threads" && i + 1 < argc) { + cfg.n_threads = std::stoi(argv[++i]); + } else if (arg == "-h" || arg == "--help") { + print_usage(argv[0]); + exit(0); + } else { + LOG_ERR("Unknown argument: %s\n\n", arg.c_str()); + print_usage(argv[0]); + exit(1); + } + } + + if (cfg.model_path.empty() || cfg.imatrix_path.empty() || + cfg.quant_types.empty() || cfg.target_bpw <= 0 || cfg.output_path.empty()) { + LOG_ERR("Missing required arguments\n\n"); + print_usage(argv[0]); + exit(1); + } + + if (cfg.output_tensor_type == GGML_TYPE_COUNT) { + cfg.output_tensor_type = highest_bpw(cfg.quant_types); + } + + return cfg; +} + +// ============================================================================ +// Section 10: Main +// ============================================================================ + +int main(int argc, char ** argv) { + config cfg = parse_args(argc, argv); + + LOG("=== llama-auto-tensor-type ===\n"); + LOG("Model: %s\n", cfg.model_path.c_str()); + LOG("Imatrix: %s\n", cfg.imatrix_path.c_str()); + LOG("Target BPW: %.2f (tolerance: +%.2f, -%.2f)\n", + cfg.target_bpw, cfg.bpw_tol_high, cfg.bpw_tol_low); + LOG("Quant types: "); + for (auto t : cfg.quant_types) LOG("%s ", ggml_type_name(t)); + LOG("\n"); + LOG("Output: %s\n", cfg.output_path.c_str()); + LOG("\n"); + + // ---- Phase 1: Load model metadata from GGUF ---- + LOG("--- Phase 1: Loading model metadata ---\n"); + + struct ggml_context * ggml_ctx = nullptr; + struct gguf_init_params gguf_params = {false, &ggml_ctx}; + struct gguf_context * gguf_ctx = gguf_init_from_file(cfg.model_path.c_str(), gguf_params); + if (!gguf_ctx) { + LOG_ERR("Failed to open model: %s\n", cfg.model_path.c_str()); + return 1; + } + + int64_t n_tensors = gguf_get_n_tensors(gguf_ctx); + LOG("Model has %lld tensors\n", (long long)n_tensors); + + // Determine n_layer from tensor names + int n_layer = 0; + for (int64_t i = 0; i < n_tensors; i++) { + const char * name = gguf_get_tensor_name(gguf_ctx, i); + std::string sname(name); + static const std::regex blk_re("blk\\.(\\d+)\\."); + std::smatch m; + if (std::regex_search(sname, m, blk_re) && m.size() > 1) { + int layer = std::stoi(m[1].str()); + if (layer + 1 > n_layer) n_layer = layer + 1; + } + } + LOG("Model has %d layers\n", n_layer); + + // Enumerate tensor info + std::vector all_tensors; + for (int64_t i = 0; i < n_tensors; i++) { + tensor_info ti; + ti.name = gguf_get_tensor_name(gguf_ctx, i); + ti.orig_type = gguf_get_tensor_type(gguf_ctx, i); + ti.role = extract_role(ti.name); + ti.n_elements = gguf_get_tensor_size(gguf_ctx, i) / ggml_type_size(ti.orig_type) * ggml_blck_size(ti.orig_type); + + // Get dimensions from ggml context + struct ggml_tensor * gt = ggml_get_tensor(ggml_ctx, ti.name.c_str()); + if (gt) { + for (int d = 0; d < 4; d++) ti.ne[d] = gt->ne[d]; + } + + // Determine layer from name + static const std::regex blk_re2("blk\\.(\\d+)\\."); + std::smatch m; + if (std::regex_search(ti.name, m, blk_re2) && m.size() > 1) { + ti.layer = std::stoi(m[1].str()); + } else { + ti.layer = -1; // global tensor + } + + all_tensors.push_back(ti); + } + + // Identify quantizable weight tensors + std::vector quantizable; + for (const auto & ti : all_tensors) { + if (is_quantizable_weight(ti, cfg.min_elements)) { + quantizable.push_back(ti); + } + } + LOG("Found %zu quantizable weight tensors (>= %lld elements, 2D)\n", + quantizable.size(), (long long)cfg.min_elements); + + // Get unique roles + std::set unique_roles; + for (const auto & ti : quantizable) { + unique_roles.insert(ti.role); + } + LOG("Found %zu unique tensor roles: ", unique_roles.size()); + for (const auto & r : unique_roles) LOG("%s ", r.c_str()); + LOG("\n"); + + // Determine target layers using layer equivalence classes. + // Hybrid models (e.g., Qwen3.5) have different layer types (dense attention vs SSM/Mamba) + // with different tensor roles and shapes. We group layers by their tensor signature + // and sample representatives from each class. + auto layer_classes = get_layer_equivalence_classes(n_layer, all_tensors, cfg.min_elements); + + // Collect all target layer indices + std::vector target_layers; + for (const auto & lc : layer_classes) { + for (int l : lc.reps) target_layers.push_back(l); + } + std::sort(target_layers.begin(), target_layers.end()); + target_layers.erase(std::unique(target_layers.begin(), target_layers.end()), target_layers.end()); + + LOG("Layer equivalence classes: %zu classes, %zu total target layers\n", + layer_classes.size(), target_layers.size()); + for (const auto & lc : layer_classes) { + std::string roles_str; + for (const auto & [role, ne0, ne1] : lc.signature) { + if (!roles_str.empty()) roles_str += ", "; + roles_str += role; + } + LOG(" Class %zu: %zu layers, reps=[", lc.class_index, lc.all_layers.size()); + for (size_t i = 0; i < lc.reps.size(); i++) { + if (i > 0) LOG(", "); + LOG("%d", lc.reps[i]); + } + LOG("], roles=[%s]\n", roles_str.c_str()); + } + + // Build set of target weight tensor names (for the eval callback) + // NOTE: token_embd uses ggml_get_rows (embedding lookup), NOT ggml_mul_mat, + // so it cannot be measured via MUL_MAT KLD. It gets the highest-BPW type as preset. + std::unordered_set target_weight_names; + for (const auto & ti : quantizable) { + if (ti.role == "token_embd") continue; // skip — not MUL_MAT + if (ti.layer >= 0) { + // Include if this tensor's layer is one of our target layers + if (std::binary_search(target_layers.begin(), target_layers.end(), ti.layer)) { + target_weight_names.insert(ti.name); + } + } else { + // Global tensors (output.weight, etc.) — capture those too + target_weight_names.insert(ti.name); + } + } + + // ---- Phase 2: Capture reference activations ---- + LOG("\n--- Phase 2: Capturing reference activations ---\n"); + + // Load imatrix data + std::unordered_map> imatrix_data; + if (!cfg.imatrix_path.empty()) { + load_imatrix_data(cfg.imatrix_path, imatrix_data); + } + + // Load model with llama API + common_params params; + params.model.path = cfg.model_path; + params.n_gpu_layers = 99; // Offload to GPU if available, falls back to CPU + params.n_batch = 512; + params.n_ubatch = 512; + params.n_ctx = 1024; // enough for test sizes + + capture_state cap_state; + for (const auto & ti : quantizable) { + if (target_weight_names.count(ti.name)) { + cap_state.target_weight_names.insert(ti.name); + cap_state.weight_to_role[ti.name] = ti.role; + cap_state.weight_to_layer[ti.name] = ti.layer; + } + } + + params.cb_eval = capture_callback; + params.cb_eval_user_data = &cap_state; + + common_init(); + ggml_backend_load_all(); + llama_backend_init(); + + auto llama_init = common_init_from_params(params); + if (!llama_init) { + LOG_ERR("Failed to load model\n"); + return 1; + } + + auto * model = llama_init->model(); + auto * ctx = llama_init->context(); + if (!model || !ctx) { + LOG_ERR("Failed to initialize context\n"); + return 1; + } + + const llama_vocab * vocab = llama_model_get_vocab(model); + const bool add_bos = llama_vocab_get_add_bos(vocab); + const int n_vocab = llama_vocab_n_tokens(vocab); + + // Prepare test tokens for each size + std::vector> test_token_sets; + + if (!cfg.test_data_path.empty()) { + // Read test data file + std::ifstream tf(cfg.test_data_path); + if (!tf) { + LOG_ERR("Failed to open test data file: %s\n", cfg.test_data_path.c_str()); + return 1; + } + std::string test_text((std::istreambuf_iterator(tf)), + std::istreambuf_iterator()); + auto all_tokens = common_tokenize(ctx, test_text, add_bos, false); + LOG("Tokenized test data: %zu tokens\n", all_tokens.size()); + + for (int size : cfg.test_sizes) { + int n = std::min(size, (int)all_tokens.size()); + test_token_sets.push_back(std::vector(all_tokens.begin(), all_tokens.begin() + n)); + } + } else { + // Synthetic: random tokens + LOG("Using synthetic test inputs\n"); + srand(42); + for (int size : cfg.test_sizes) { + std::vector tokens; + if (add_bos) tokens.push_back(llama_vocab_bos(vocab)); + while ((int)tokens.size() < size) { + tokens.push_back(rand() % n_vocab); + } + test_token_sets.push_back(tokens); + } + } + + // Run forward passes with the eval callback + for (size_t si = 0; si < test_token_sets.size(); si++) { + const auto & tokens = test_token_sets[si]; + LOG("Running forward pass with %zu tokens (size %d)...\n", + tokens.size(), cfg.test_sizes[si]); + + // Clear KV cache + llama_memory_clear(llama_get_memory(ctx), true); + + // Decode in a single batch + int ret = llama_decode(ctx, llama_batch_get_one(const_cast(tokens.data()), tokens.size())); + if (ret != 0) { + LOG_ERR("llama_decode failed with code %d\n", ret); + // Try smaller batches + int batch_size = 64; + for (int b = 0; b < (int)tokens.size(); b += batch_size) { + int n = std::min(batch_size, (int)tokens.size() - b); + ret = llama_decode(ctx, llama_batch_get_one(const_cast(tokens.data() + b), n)); + if (ret != 0) { + LOG_ERR("llama_decode failed at batch %d with code %d\n", b, ret); + break; + } + } + } + } + + LOG("Captured %d MUL_MAT operations\n", cap_state.captured); + for (const auto & [role, captures] : cap_state.captures_by_role) { + LOG(" Role '%s': %zu captures\n", role.c_str(), captures.size()); + } + + // Free the llama model — we don't need it anymore + llama_init.reset(); + llama_backend_free(); + + // ---- Phase 3: Build cost matrix ---- + LOG("\n--- Phase 3: Building cost matrix ---\n"); + + auto cost_matrix = build_cost_matrix(cfg, quantizable, cap_state.captures_by_role, + cfg.model_path, gguf_ctx, imatrix_data); + + // Print cost matrix + LOG("\nCost matrix (avg KLD by role × quant type):\n"); + LOG("%-16s", "Role"); + for (auto qt : cfg.quant_types) LOG(" %10s", ggml_type_name(qt)); + LOG("\n"); + for (const auto & [role, costs] : cost_matrix) { + LOG("%-16s", role.c_str()); + for (auto qt : cfg.quant_types) { + auto it = costs.find(qt); + if (it != costs.end()) { + LOG(" %10.6f", it->second.kld); + } else { + LOG(" %10s", "N/A"); + } + } + LOG("\n"); + } + + // ---- Phase 4: Optimize assignment ---- + LOG("\n--- Phase 4: Optimizing assignment ---\n"); + + // Determine fixed types for global tensors (before building roles map) + ggml_type token_embd_type = closest_bpw_if(cfg.quant_types, cfg.target_bpw, supports_get_rows); + if (token_embd_type == GGML_TYPE_COUNT) { + token_embd_type = highest_bpw(cfg.quant_types); // fallback + } + ggml_type output_type = cfg.output_tensor_type != GGML_TYPE_COUNT + ? cfg.output_tensor_type + : highest_bpw(cfg.quant_types); + + // Add global tensors to the cost matrix with their fixed types. + // KLD=0 since they're not optimized — we just need their BPW in the budget. + for (ggml_type qtype : cfg.quant_types) { + cost_entry e; + e.kld = (qtype == token_embd_type) ? 0.0 : 1e30; + e.bpw = compute_bpw(qtype); + cost_matrix["token_embd"][qtype] = e; + + e.kld = (qtype == output_type) ? 0.0 : 1e30; + cost_matrix["output"][qtype] = e; + } + LOG("Global tensors: token_embd=%s (%.4f bpw), output=%s (%.4f bpw)\n", + ggml_type_name(token_embd_type), compute_bpw(token_embd_type), + ggml_type_name(output_type), compute_bpw(output_type)); + + // Build role info (n_elements, n_bytes per role) — include ALL quantizable tensors + std::map roles; + size_t total_quant_elements = 0; + for (const auto & ti : quantizable) { + auto & ri = roles[ti.role]; + ri.role = ti.role; + ri.n_elements += ti.n_elements; + ri.n_bytes_orig += ggml_nbytes(ggml_get_tensor(ggml_ctx, ti.name.c_str())); + total_quant_elements += ti.n_elements; + } + + // Compute non-quantizable tensor overhead (small tensors kept at original precision) + // These contribute fixed BPW that must be accounted for in the budget. + size_t total_all_elements = 0; + double non_quantizable_bits = 0; + for (const auto & ti : all_tensors) { + total_all_elements += ti.n_elements; + if (!is_quantizable_weight(ti, cfg.min_elements)) { + non_quantizable_bits += compute_bpw(ti.orig_type) * ti.n_elements; + } + } + + LOG("Total elements in quantizable tensors: %zu\n", total_quant_elements); + LOG("Total elements in all tensors: %zu\n", total_all_elements); + LOG("Non-quantizable overhead: %.4f BPW\n", + total_all_elements > 0 ? non_quantizable_bits / total_all_elements : 0); + for (const auto & [role, ri] : roles) { + LOG(" %s: %zu elements%s\n", role.c_str(), ri.n_elements, + cost_matrix.count(role) ? "" : " [NO CAPTURES]"); + } + + // Warn about any quantizable roles that still have no captures. + // With proper layer equivalence class sampling, this should be rare, + // but can happen for tensors that don't appear as MUL_MAT in any layer type. + { + for (const auto & [role, ri] : roles) { + if (cost_matrix.count(role)) continue; + LOG(" WARNING: Role '%s' has no captures — assigning KLD=0 for all types\n", + role.c_str()); + for (ggml_type qtype : cfg.quant_types) { + cost_entry e; + e.kld = 0.0; + e.bpw = compute_bpw(qtype); + cost_matrix[role][qtype] = e; + } + } + } + auto result = optimize_assignment(cfg, cost_matrix, roles, total_all_elements, non_quantizable_bits); + + LOG("\nOptimal assignment:\n"); + for (const auto & [role, type] : result.role_to_type) { + auto it = cost_matrix.find(role); + double kld = (it != cost_matrix.end() && it->second.count(type)) ? + it->second.at(type).kld : -1; + LOG(" %-16s = %-10s (KLD=%.6f, BPW=%.4f)\n", + role.c_str(), ggml_type_name(type), kld, compute_bpw(type)); + } + LOG("Total BPW: %.4f (target: %.2f +%.2f/-%.2f)\n", + result.total_bpw, cfg.target_bpw, cfg.bpw_tol_high, cfg.bpw_tol_low); + LOG("Total KLD: %.6f\n", result.total_kld); + + // ---- Phase 5: Output tensor-type-file ---- + LOG("\n--- Phase 5: Writing output ---\n"); + write_tensor_type_file(cfg.output_path, result.role_to_type, + output_type, token_embd_type); + + // Cleanup + gguf_free(gguf_ctx); + ggml_free(ggml_ctx); + + LOG("\nDone!\n"); + return 0; +} From 64d60ec4f0d4461de3cc7909e1bb960836c2aebb Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 17 Apr 2026 01:45:05 +0200 Subject: [PATCH 04/19] Fix -Wpedantic warnings --- ggml/src/ggml-quants.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 8bf4bb81846e..80d5c32316c9 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -6398,7 +6398,7 @@ static inline float hsum_float_8(__m256 v) { static inline int iq2tq_find_best_grid_avx2( const float * xn, // [8] normalized values const float * wk, // [8] importance weights - const float (* grid_f)[4], // [16] grid entries as float + float (* grid_f)[4], // [16] grid entries as float float dq, // scale factor for error float * best_err_out ) { @@ -6446,7 +6446,7 @@ static inline int iq2tq_find_best_grid_avx2( static inline int iq3tq_find_best_grid_avx2( const float * xn, // [8] normalized values const float * wk, // [8] importance weights - const float (* grid_f)[IQ3TQ_N_LEVELS], // [16] grid entries as float + float (* grid_f)[IQ3TQ_N_LEVELS], // [16] grid entries as float float dq, // scale factor for error float * best_err_out ) { @@ -6510,8 +6510,8 @@ void dequantize_row_iq2_tq(const block_iq2_tq * GGML_RESTRICT x, float * GGML_RE // SIMD K-means assignment: find nearest centroid for each data point // IQ2_TQ version: 4 dimensions, 16 centroids #if defined(__AVX2__) -static void kmeans_assign_4d_avx2(const float (*sets)[4], int n_sets, int * assign, - const float centroids[16][4], int * changed) { +static void kmeans_assign_4d_avx2(float (*sets)[4], int n_sets, int * assign, + float centroids[16][4], int * changed) { // Reorganize centroids into SoA for SIMD: cx[16], cy[16], cz[16], cw[16] float cx[16], cy[16], cz[16], cw[16]; for (int c = 0; c < 16; c++) { @@ -6552,8 +6552,8 @@ static void kmeans_assign_4d_avx2(const float (*sets)[4], int n_sets, int * assi } // IQ3_TQ version: 8 dimensions, 16 centroids -static void kmeans_assign_8d_avx2(const float (*sets)[8], int n_sets, int * assign, - const float centroids[16][8], int * changed) { +static void kmeans_assign_8d_avx2(float (*sets)[8], int n_sets, int * assign, + float centroids[16][8], int * changed) { // Reorganize centroids into SoA float csoa[8][16]; for (int c = 0; c < 16; c++) From 4911728afcf331365c999c63fcbd22cc02ee750b Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 17 Apr 2026 14:14:03 +0200 Subject: [PATCH 05/19] Kernelz --- ggml/include/ggml.h | 2 +- ggml/src/ggml-backend.cpp | 1 + ggml/src/ggml-cuda/common.cuh | 22 ++++++- ggml/src/ggml-cuda/convert.cu | 89 ++++++++++++++++++++++++++ ggml/src/ggml-cuda/convert.cuh | 6 ++ ggml/src/ggml-cuda/ggml-cuda.cu | 22 +++++++ ggml/src/ggml-cuda/mmvq.cu | 69 ++++++++++++++++++++ ggml/src/ggml-cuda/vecdotq.cuh | 107 ++++++++++++++++++++++++++++++++ tests/CMakeLists.txt | 1 + tests/test-backend-ops.cpp | 81 ++++++++++++++++++++++++ 10 files changed, 397 insertions(+), 3 deletions(-) diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 430e75a72798..9985597def9a 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -434,7 +434,7 @@ extern "C" { GGML_TYPE_Q4_DPT = 44, // IQ4_NL with learned per-tensor int8 levels (4.125 bpw) GGML_TYPE_Q2_DPT = 45, // 2-bit with learned per-tensor int8 levels (2.5 bpw) GGML_TYPE_Q2_KPT = 46, // Q2_K with learned per-tensor float levels (2.625 bpw) - GGML_TYPE_IQ2_TQ = 47, // Trellis quantized with RNG codebook (2.0625 bpw) + GGML_TYPE_IQ2_TQ = 47, // Trellis quantized with RNG codebook (2.5625 bpw) GGML_TYPE_IQ3_TQ = 48, // 3-bit with per-tensor trained grid table (3.5625 bpw) GGML_TYPE_IQ1_BN = 49, // 8D vector quantized with per-tensor trained codebook (1.5625 bpw) GGML_TYPE_COUNT = 50, diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 87615921c09b..4c49cfcef469 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -2023,6 +2023,7 @@ static struct ggml_tensor * graph_copy_dup_tensor(struct ggml_hash_set hash_set, dst->op = src->op; dst->flags = src->flags; memcpy(dst->op_params, src->op_params, sizeof(dst->op_params)); + dst->quant_levels = src->quant_levels; ggml_set_name(dst, src->name); // copy src diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 7e7cfc328f8b..c57ee21b677f 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1112,11 +1112,29 @@ __device__ int8_t q4dpt_levels_cuda[16]; // Per-tensor lookup table for Q2_DPT (4 int8 levels). __device__ int8_t q2dpt_levels_cuda[4]; +__device__ float q3kpt_levels_cuda[Q3KPT_N_LEVELS]; + +__device__ const float * q2kpt_levels_cuda_ptr; + template<> struct ggml_cuda_type_traits { static constexpr int qk = QK2_DPT; - static constexpr int qr = 4; // 4 elements per "quantum" (2-bit) - static constexpr int qi = 1; // 1 uint32 per block + static constexpr int qr = 4; + static constexpr int qi = 1; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR2_K; + static constexpr int qi = QI2_K; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR3_K; + static constexpr int qi = QI3_K; }; template<> diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index d30b51c2830c..02033755a727 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -617,6 +617,18 @@ void ggml_cuda_set_iq3tq_grid(const void * grid, cudaStream_t stream) { CUDA_CHECK(cudaMemcpyAsync(d_grid, grid, 128, cudaMemcpyHostToDevice, stream)); } +void ggml_cuda_set_q3kpt_levels(const float * levels, cudaStream_t stream) { + float * d_levels; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_levels, q3kpt_levels_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_levels, levels, Q3KPT_N_LEVELS * sizeof(float), cudaMemcpyHostToDevice, stream)); +} + +void ggml_cuda_set_q2kpt_levels(const float * levels, size_t n_levels, cudaStream_t stream) { + // levels is a device pointer to the per-block levels array + // q2kpt_levels_cuda_ptr is a device symbol that holds the pointer + CUDA_CHECK(cudaMemcpyToSymbolAsync(q2kpt_levels_cuda_ptr, &levels, sizeof(float *), 0, cudaMemcpyHostToDevice, stream)); +} + void ggml_cuda_set_iq1bn_aux(const void * aux, cudaStream_t stream) { int8_t * d_cb; @@ -677,6 +689,75 @@ static void dequantize_row_q2_dpt_cuda(const void * vx, dst_t * y, const int64_t dequantize_block_q2_dpt<<>>(vx, y); } +template +static __global__ void dequantize_block_q3_kpt(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; + const block_q3_kpt * x = (const block_q3_kpt *) vx; + + const int64_t r = threadIdx.x/4; + const int64_t tid = r/2; + const int64_t is0 = r%2; + const int64_t l0 = 16*is0 + 4*(threadIdx.x%4); + const int64_t n = tid / 4; + const int64_t j = tid - 4*n; + + uint8_t m = 1 << (4*n + j); + int64_t is = 8*n + 2*j + is0; + int shift = 2*j; + + int8_t us = is < 4 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+8] >> 0) & 3) << 4) : + is < 8 ? (x[i].scales[is-0] & 0xF) | (((x[i].scales[is+4] >> 2) & 3) << 4) : + is < 12 ? (x[i].scales[is-8] >> 4) | (((x[i].scales[is+0] >> 4) & 3) << 4) : + (x[i].scales[is-8] >> 4) | (((x[i].scales[is-4] >> 6) & 3) << 4); + float d_all = x[i].d; + float dl = d_all * (us - 32); + + dst_t * y = yy + i*QK_K + 128*n + 32*j; + const uint8_t * q = x[i].qs + 32*n; + const uint8_t * hm = x[i].hmask; + + for (int l = l0; l < l0+4; ++l) { + int k_idx = ((q[l] >> shift) & 3) + ((hm[l] & m) ? 4 : 0); + y[l] = dl * (q3kpt_levels_cuda[k_idx] * 7.0f - 4.0f); + } +} + +template +static void dequantize_row_q3_kpt_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_q3_kpt<<>>(vx, y); +} + +template +static __global__ void dequantize_block_q2_kpt(const void * __restrict__ vx, dst_t * __restrict__ yy) { + const int64_t i = blockIdx.x; + const block_q2_kpt * x = (const block_q2_kpt *) vx; + + const int64_t tid = threadIdx.x; + const int64_t n = tid/32; + const int64_t l = tid - 32*n; + const int64_t is = 8*n + l/16; + + const uint8_t q = x[i].qs[32*n + l]; + dst_t * y = yy + i*QK_K + 128*n; + + float dall = __low2half(x[i].dm); + float dmin = __high2half(x[i].dm); + + const float * block_lv = q2kpt_levels_cuda_ptr + i * Q2KPT_N_LEVELS; + + y[l+ 0] = dall * (x[i].scales[is+0] & 0xF) * (block_lv[(q >> 0) & 3] * 3.0f) - dmin * (x[i].scales[is+0] >> 4); + y[l+32] = dall * (x[i].scales[is+2] & 0xF) * (block_lv[(q >> 2) & 3] * 3.0f) - dmin * (x[i].scales[is+2] >> 4); + y[l+64] = dall * (x[i].scales[is+4] & 0xF) * (block_lv[(q >> 4) & 3] * 3.0f) - dmin * (x[i].scales[is+4] >> 4); + y[l+96] = dall * (x[i].scales[is+6] & 0xF) * (block_lv[(q >> 6) & 3] * 3.0f) - dmin * (x[i].scales[is+6] >> 4); +} + +template +static void dequantize_row_q2_kpt_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_q2_kpt<<>>(vx, y); +} + template static __global__ void dequantize_block_iq2_tq(const void * __restrict__ vx, dst_t * __restrict__ yy) { const int64_t i = blockIdx.x; @@ -935,6 +1016,10 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { return dequantize_row_iq3_tq_cuda; case GGML_TYPE_IQ1_BN: return dequantize_row_iq1_bn_cuda; + case GGML_TYPE_Q3_KPT: + return dequantize_row_q3_kpt_cuda; + case GGML_TYPE_Q2_KPT: + return dequantize_row_q2_kpt_cuda; case GGML_TYPE_IQ4_XS: return dequantize_row_iq4_xs_cuda; case GGML_TYPE_IQ3_S: @@ -1000,6 +1085,10 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { return dequantize_row_iq3_tq_cuda; case GGML_TYPE_IQ1_BN: return dequantize_row_iq1_bn_cuda; + case GGML_TYPE_Q3_KPT: + return dequantize_row_q3_kpt_cuda; + case GGML_TYPE_Q2_KPT: + return dequantize_row_q2_kpt_cuda; case GGML_TYPE_IQ4_XS: return dequantize_row_iq4_xs_cuda; case GGML_TYPE_IQ3_S: diff --git a/ggml/src/ggml-cuda/convert.cuh b/ggml/src/ggml-cuda/convert.cuh index 3b96662dd35f..1181ff7604e0 100644 --- a/ggml/src/ggml-cuda/convert.cuh +++ b/ggml/src/ggml-cuda/convert.cuh @@ -47,6 +47,12 @@ void ggml_cuda_set_iq3tq_grid(const void * grid, cudaStream_t stream); // Set the IQ1_BN per-tensor codebook+scale (2064 bytes). void ggml_cuda_set_iq1bn_aux(const void * aux, cudaStream_t stream); +// Set the Q3_KPT per-tensor levels (8 floats). +void ggml_cuda_set_q3kpt_levels(const float * levels, cudaStream_t stream); + +// Set the Q2_KPT per-block levels pointer. +void ggml_cuda_set_q2kpt_levels(const float * levels, size_t n_levels, cudaStream_t stream); + template __host__ __device__ inline dst_t ggml_cuda_cast(src_t x) { if constexpr (std::is_same_v) { diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 8b75e95f3480..c91fa48a8acf 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -1709,6 +1709,25 @@ static void ggml_cuda_op_mul_mat_cublas( ggml_cuda_set_iq1bn_aux(src0->quant_levels, stream); } + // Q3_KPT per-tensor levels (8 floats) + if (src0->type == GGML_TYPE_Q3_KPT) { + GGML_ASSERT(src0->quant_levels && "Q3_KPT MUL_MAT requires levels (set tensor->quant_levels)"); + ggml_cuda_set_q3kpt_levels((const float *)src0->quant_levels, stream); + } + + // Q2_KPT per-block levels + ggml_cuda_pool_alloc q2kpt_levels_alloc(ctx.pool(id)); + if (src0->type == GGML_TYPE_Q2_KPT) { + GGML_ASSERT(src0->quant_levels && "Q2_KPT MUL_MAT requires levels (set tensor->quant_levels)"); + const int blocks_per_row = ne00 / QK_K; + const int total_blocks = row_diff * blocks_per_row; + const int level_offset = row_low * blocks_per_row * Q2KPT_N_LEVELS; + const size_t n_levels = total_blocks * Q2KPT_N_LEVELS; + float * d_levels = q2kpt_levels_alloc.alloc(n_levels); + CUDA_CHECK(cudaMemcpyAsync(d_levels, (const float *)src0->quant_levels + level_offset, n_levels * sizeof(float), cudaMemcpyHostToDevice, stream)); + ggml_cuda_set_q2kpt_levels(d_levels, n_levels, stream); + } + if (supports_bf16 && src0->type == GGML_TYPE_BF16 && ggml_is_contiguous(src0) && row_diff == src0->ne[1]) { ggml_cuda_pool_alloc src1_as_bf16(ctx.pool(id)); if (src1->type != GGML_TYPE_BF16) { @@ -5314,6 +5333,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_IQ2_TQ: case GGML_TYPE_IQ3_TQ: case GGML_TYPE_IQ1_BN: + case GGML_TYPE_Q2_DPT: + case GGML_TYPE_Q3_KPT: + case GGML_TYPE_Q2_KPT: case GGML_TYPE_IQ4_XS: case GGML_TYPE_BF16: return true; diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 2cc7d76a0e54..58e2efdc28d5 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -38,6 +38,8 @@ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) case GGML_TYPE_IQ1_BN: return vec_dot_iq1_bn_q8_1; case GGML_TYPE_IQ4_XS: return vec_dot_iq4_xs_q8_1; case GGML_TYPE_IQ3_S: return vec_dot_iq3_s_q8_1; + case GGML_TYPE_Q2_KPT: return vec_dot_q2_kpt_q8_K; + case GGML_TYPE_Q3_KPT: return vec_dot_q3_kpt_q8_K; default: return nullptr; } } @@ -69,6 +71,8 @@ static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) { case GGML_TYPE_IQ3_TQ: return VDR_IQ3_TQ_Q8_1_MMVQ; case GGML_TYPE_IQ1_BN: return VDR_IQ1_BN_Q8_1_MMVQ; case GGML_TYPE_IQ4_XS: return VDR_IQ4_XS_Q8_1_MMVQ; + case GGML_TYPE_Q2_KPT: return VDR_Q2_KPT_Q8_1_MMVQ; + case GGML_TYPE_Q3_KPT: return VDR_Q3_KPT_Q8_1_MMVQ; default: return 1; } } @@ -1150,6 +1154,18 @@ static void mul_mat_vec_q_switch_type( nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); break; + case GGML_TYPE_Q2_KPT: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; + case GGML_TYPE_Q3_KPT: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; default: GGML_ABORT("fatal error"); break; @@ -1206,6 +1222,26 @@ void ggml_cuda_mul_mat_vec_q( ggml_cuda_set_iq1bn_aux(src0->quant_levels, stream); } + // Set Q3_KPT per-tensor levels (8 floats) + if (src0->type == GGML_TYPE_Q3_KPT) { + GGML_ASSERT(src0->quant_levels && "Q3_KPT MUL_MAT requires levels (set tensor->quant_levels)"); + float * d_levels; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_levels, q3kpt_levels_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_levels, src0->quant_levels, Q3KPT_N_LEVELS * sizeof(float), cudaMemcpyHostToDevice, stream)); + } + + // Set Q2_KPT per-block levels + ggml_cuda_pool_alloc q2kpt_levels_alloc(ctx.pool()); + if (src0->type == GGML_TYPE_Q2_KPT) { + GGML_ASSERT(src0->quant_levels && "Q2_KPT MUL_MAT requires levels (set tensor->quant_levels)"); + const int blocks_per_row = ne00 / QK_K; + const int total_blocks = ne03*ne02*ne01 * blocks_per_row; + const size_t n_levels = total_blocks * Q2KPT_N_LEVELS; + float * d_levels = q2kpt_levels_alloc.alloc(n_levels); + CUDA_CHECK(cudaMemcpyAsync(d_levels, src0->quant_levels, n_levels * sizeof(float), cudaMemcpyHostToDevice, stream)); + CUDA_CHECK(cudaMemcpyToSymbolAsync(q2kpt_levels_cuda_ptr, &d_levels, sizeof(float *), 0, cudaMemcpyHostToDevice, stream)); + } + const size_t ts_src0 = ggml_type_size(src0->type); const size_t ts_src1 = ggml_type_size(src1->type); const size_t ts_dst = ggml_type_size(dst->type); @@ -1319,6 +1355,39 @@ void ggml_cuda_op_mul_mat_vec_q( const int stride_col_y = src1_padded_row_size / QK8_1; ggml_cuda_mm_fusion_args_device fusion_local{}; + + // Upload per-tensor levels for DPT/KPT types + if (src0->type == GGML_TYPE_Q4_DPT) { + GGML_ASSERT(src0->quant_levels && "Q4_DPT MUL_MAT requires levels"); + int8_t * d_levels; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_levels, q4dpt_levels_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_levels, src0->quant_levels, Q4DPT_N_LEVELS * sizeof(int8_t), cudaMemcpyHostToDevice, stream)); + } + if (src0->type == GGML_TYPE_Q2_DPT) { + GGML_ASSERT(src0->quant_levels && "Q2_DPT MUL_MAT requires levels"); + int8_t * d_levels; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_levels, q2dpt_levels_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_levels, src0->quant_levels, Q2DPT_N_LEVELS * sizeof(int8_t), cudaMemcpyHostToDevice, stream)); + } + if (src0->type == GGML_TYPE_Q3_KPT) { + GGML_ASSERT(src0->quant_levels && "Q3_KPT MUL_MAT requires levels"); + float * d_levels; + CUDA_CHECK(cudaGetSymbolAddress((void **)&d_levels, q3kpt_levels_cuda)); + CUDA_CHECK(cudaMemcpyAsync(d_levels, src0->quant_levels, Q3KPT_N_LEVELS * sizeof(float), cudaMemcpyHostToDevice, stream)); + } + + ggml_cuda_pool_alloc q2kpt_levels_op_alloc(ctx.pool()); + if (src0->type == GGML_TYPE_Q2_KPT) { + GGML_ASSERT(src0->quant_levels && "Q2_KPT MUL_MAT requires levels"); + const int blocks_per_row = ne00 / QK_K; + const int total_blocks = row_diff * blocks_per_row; + const int level_offset = row_low * blocks_per_row * Q2KPT_N_LEVELS; + const size_t n_levels = total_blocks * Q2KPT_N_LEVELS; + float * d_levels = q2kpt_levels_op_alloc.alloc(n_levels); + CUDA_CHECK(cudaMemcpyAsync(d_levels, (const float *)src0->quant_levels + level_offset, n_levels * sizeof(float), cudaMemcpyHostToDevice, stream)); + CUDA_CHECK(cudaMemcpyToSymbolAsync(q2kpt_levels_cuda_ptr, &d_levels, sizeof(float *), 0, cudaMemcpyHostToDevice, stream)); + } + mul_mat_vec_q_switch_type( src0_dd_i, src0->type, src1_ddq_i, nullptr, fusion_local, dst_dd_i, ne00, row_diff, src1_ncols, stride_row_x, stride_col_y, nrows_dst, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, stream); diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuh index fddc969dc1d6..df3c7a6f50a2 100644 --- a/ggml/src/ggml-cuda/vecdotq.cuh +++ b/ggml/src/ggml-cuda/vecdotq.cuh @@ -1503,3 +1503,110 @@ static __device__ __forceinline__ float vec_dot_iq4_xs_q8_1( const float d = __half2float(bq4->d) * __low2float(bq8_1[iqs/4].ds); return d * sumi; } + +#define VDR_Q3_KPT_Q8_1_MMVQ 1 +#define VDR_Q3_KPT_Q8_1_MMQ 2 + +static __device__ __forceinline__ float vec_dot_q3_kpt_q8_K( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q3_kpt * bq3_K = (const block_q3_kpt *) vbq + kbx; + + const int bq8_offset = QR3_K * (iqs / (QI3_K/2)); + const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2); + + const float d = bq3_K->d; + + const int vl = get_int_b2(bq3_K->qs, iqs); + const int vh = get_int_b2(bq3_K->hmask, iqs % (QI3_K/2)) >> bq8_offset; + + float sumf = 0.0f; + +#pragma unroll + for (int i = 0; i < QR3_K; ++i) { + const int isc = scale_offset + 2*i; + + const int isc_low = isc % (QK_K/32); + const int sc_shift_low = 4 * (isc / (QK_K/32)); + const int sc_low = (bq3_K->scales[isc_low] >> sc_shift_low) & 0xF; + + const int isc_high = isc % (QK_K/64); + const int sc_shift_high = 2 * (isc / (QK_K/64)); + const int sc_high = ((bq3_K->scales[(QK_K/32) + isc_high] >> sc_shift_high) & 3) << 4; + + const int sc = (sc_low | sc_high) - 32; + + const int vil = (vl >> (2*i)) & 0x03030303; + const int vih = ((vh >> i) << 2) & 0x04040404; + + const int u_i = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1); + const float d8_i = __low2float(bq8_1[bq8_offset + i].ds); + + float dot = 0.0f; +#pragma unroll + for (int b = 0; b < 4; ++b) { + const int shift = 8*b; + const int low2 = (vil >> shift) & 3; + const int high = (vih >> shift) & 4; + const int k_idx = low2 + high; + const float level_val = q3kpt_levels_cuda[k_idx] * 7.0f - 4.0f; + const int q8_val = (int)(int8_t)((u_i >> shift) & 0xFF); + dot += level_val * (float)q8_val; + } + + sumf += d8_i * (dot * sc); + } + + return d * sumf; +} + +#define VDR_Q2_KPT_Q8_1_MMVQ 1 +#define VDR_Q2_KPT_Q8_1_MMQ 4 + +static __device__ __forceinline__ float vec_dot_q2_kpt_q8_K( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + + const block_q2_kpt * bq2_K = (const block_q2_kpt *) vbq + kbx; + + const int bq8_offset = QR2_K * (iqs / QI8_1); + const int scale_offset = iqs - iqs % QI8_1 + (iqs % QI8_1) / (QI8_1/2); + + const uint8_t * scales = bq2_K->scales + scale_offset; + + const int v = get_int_b4(bq2_K->qs, iqs); + + const float * block_lv = q2kpt_levels_cuda_ptr + kbx * Q2KPT_N_LEVELS; + + const float2 dm2f = __half22float2(bq2_K->dm); + + float sumf_d = 0.0f; + float sumf_m = 0.0f; + +#pragma unroll + for (int i = 0; i < QR2_K; ++i) { + const int sc = scales[2*i]; + const float d8_i = __low2float(bq8_1[bq8_offset + i].ds); + const int u_i = get_int_b4(bq8_1[bq8_offset + i].qs, iqs % QI8_1); + + const int vi = (v >> (2*i)) & 0x03030303; + + float dot = 0.0f; +#pragma unroll + for (int b = 0; b < 4; ++b) { + const int shift = 8*b; + const int idx = (vi >> shift) & 3; + const float level_val = block_lv[idx] * 3.0f; + const int q8_val = (int)(int8_t)((u_i >> shift) & 0xFF); + dot += level_val * (float)q8_val; + } + + sumf_d += d8_i * (dot * (sc & 0xF)); + + int m = sc >> 4; + m |= m << 8; + m |= m << 16; + sumf_m += d8_i * ggml_cuda_dp4a(m, u_i, 0); + } + + return dm2f.x*sumf_d - dm2f.y*sumf_m; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f8bf030a4817..a4e922485e57 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -240,6 +240,7 @@ if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC) endif() llama_build_and_test(test-gguf.cpp) llama_build_and_test(test-backend-ops.cpp) +target_include_directories(test-backend-ops PRIVATE ${CMAKE_SOURCE_DIR}/ggml/src) llama_build_and_test(test-model-load-cancel.cpp LABEL "model") llama_build_and_test(test-autorelease.cpp LABEL "model") diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index f4d6e8040472..069c606a0a1e 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -4183,6 +4184,77 @@ struct test_mul_mat_hadamard : public test_mul_mat { } }; +struct test_mul_mat_kpt : public test_mul_mat { + using test_mul_mat::test_mul_mat; + + double max_nmse_err() override { + // Q2_KPT needs slightly higher tolerance due to 2-bit quantization + return type_a == GGML_TYPE_Q2_KPT ? 5e-3 : 3e-3; + } + + void initialize_tensors(ggml_context * ctx) override { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (t->type == GGML_TYPE_Q3_KPT) { + init_tensor_q3_kpt(t); + } else if (t->type == GGML_TYPE_Q2_KPT) { + init_tensor_q2_kpt(t); + } else { + init_tensor_uniform(t); + } + } + } + +private: + void init_tensor_q3_kpt(ggml_tensor * tensor) { + const int64_t nels = ggml_nelements(tensor); + const int64_t n_per_row = tensor->ne[0]; + const int64_t nrows = nels / n_per_row; + + std::vector data(nels); + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution dis(-1.0f, 1.0f); + for (int64_t i = 0; i < nels; ++i) data[i] = dis(gen); + + float levels[Q3KPT_N_LEVELS]; + q3kpt_train_levels(data.data(), nrows, n_per_row, nullptr, levels); + q3kpt_set_levels(levels); + + static std::vector stored_levels; + stored_levels.assign(levels, levels + Q3KPT_N_LEVELS); + tensor->quant_levels = stored_levels.data(); + + std::vector dataq(ggml_row_size(tensor->type, nels)); + ggml_quantize_chunk(tensor->type, data.data(), dataq.data(), 0, nrows, n_per_row, nullptr); + ggml_backend_tensor_set(tensor, dataq.data(), 0, dataq.size()); + } + + void init_tensor_q2_kpt(ggml_tensor * tensor) { + const int64_t nels = ggml_nelements(tensor); + const int64_t n_per_row = tensor->ne[0]; + const int64_t nrows = nels / n_per_row; + const int nb = n_per_row / QK_K; + + std::vector data(nels); + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution dis(-1.0f, 1.0f); + for (int64_t i = 0; i < nels; ++i) data[i] = dis(gen); + + q2kpt_prepare_levels(nrows, n_per_row); + + std::vector dataq(ggml_row_size(tensor->type, nels)); + ggml_quantize_chunk(tensor->type, data.data(), dataq.data(), 0, nrows, n_per_row, nullptr); + ggml_backend_tensor_set(tensor, dataq.data(), 0, dataq.size()); + + static std::vector stored_block_levels; + const float * block_levels = q2kpt_get_levels(); + const size_t total_levels = nrows * nb * Q2KPT_N_LEVELS; + stored_block_levels.assign(block_levels, block_levels + total_levels); + tensor->quant_levels = stored_block_levels.data(); + } +}; + static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats) { std::random_device rd; std::default_random_engine rng(rd()); @@ -8539,6 +8611,15 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(type_a, type_b, 16, 1, 256, {1, 1}, {1, 1})); } } + // KPT types: require level training before quantization + for (ggml_type type_a : {GGML_TYPE_Q3_KPT, GGML_TYPE_Q2_KPT}) { + for (ggml_type type_b : {GGML_TYPE_F32}) { + test_cases.emplace_back(new test_mul_mat_kpt(type_a, type_b, 16, 1, 256, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat_kpt(type_a, type_b, 16, 1, 256, {1, 1}, {2, 1})); + test_cases.emplace_back(new test_mul_mat_kpt(type_a, type_b, 16, 16, 256, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat_kpt(type_a, type_b, 16, 1, 256, {3, 1}, {1, 1})); + } + } #else // m = a rows // n = b rows From 1d6ad3394f45a7901e8cee10a90a3686b3583e46 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sat, 18 Apr 2026 23:03:51 +0200 Subject: [PATCH 06/19] Add alpha, fix bugs, add docs --- docs/auto-tensor-type.md | 213 ++++++++++++ ggml/src/ggml-cuda/convert.cu | 4 +- ggml/src/ggml-quants.c | 14 +- tools/auto-tensor-type/auto-tensor-type.cpp | 356 +++++++++++++++----- 4 files changed, 484 insertions(+), 103 deletions(-) create mode 100644 docs/auto-tensor-type.md diff --git a/docs/auto-tensor-type.md b/docs/auto-tensor-type.md new file mode 100644 index 000000000000..678eaa173717 --- /dev/null +++ b/docs/auto-tensor-type.md @@ -0,0 +1,213 @@ +# `llama-auto-tensor-type` + +A tool that picks per-tensor-role quantization types to meet a target bits-per-weight +(BPW) budget while minimizing the quality impact of quantization. It produces a +`tensor-type-file` that `llama-quantize --tensor-type-file` consumes directly. + +## Why this tool exists + +For a given target BPW, the question of "how do I distribute bits across tensor roles?" +does not have a universal answer — the right allocation depends on the model's +architecture, the role's sensitivity to quantization, and the set of available quant +types. Hand-crafted recipes (e.g. "Q6_K for attention output, IQ3_TQ for FFN gate/up") +embed implicit judgments about which tensors are load-bearing. This tool tries to +derive those judgments from measurements and produce a numerically-justified recipe. + +## Pipeline + +The tool runs in five phases: + +1. **Metadata load.** Opens the source GGUF with `no_alloc=true` (we only need tensor + names/shapes/offsets; raw weight data is read later on demand). Enumerates tensor + roles via regex (`blk.N.ROLE.weight` → `ROLE`; global tensors like `token_embd` + keep their full name). + +2. **Activation capture.** Loads the model via the llama API, runs one forward pass per + `--test-size` over either synthetic tokens or real text (`--test-data`), and + captures the input (`src[1]`) and reference output of every `MUL_MAT` op whose + `src[0]` weight is a quantization target. Captures are organized per role. + +3. **Cost matrix.** For each (role, candidate type) pair, re-quantizes each captured + weight to the candidate type, replays the captured input through `MUL_MAT`, and + measures the KL divergence of the output against the reference. See + [KLD measurement](#kld-measurement) below. Results are averaged over captures + within the role. + +4. **Optimization.** Greedily upgrades role→type assignments to minimize (possibly + energy-weighted) total KLD subject to the BPW budget. See + [Optimizer](#optimizer). + +5. **Emit recipe.** Writes a `tensor-type-file` with anchored regexes: + `\.ROLE\.=TYPE` for layer tensors, `^ROLE=TYPE` for globals. + +## KLD measurement + +For each captured MUL_MAT output (a `[ne_out, n_tokens]` F32 matrix), per-row KLD is +computed as + +``` +p = softmax(ref_row), q = softmax(quant_row) +KLD(p || q) = sum_i p_i * log(p_i / q_i) +``` + +and averaged over rows and captures. This is the signal the optimizer minimizes. + +### Caveats + +- Softmax over an intermediate MUL_MAT output is not a meaningful probability + distribution by itself. Rows with large absolute magnitudes (typical of + attention Q/K/QKV projections, which read from the post-norm residual stream) + produce visibly peaked softmaxes and dramatic-looking KLDs. Rows with small + magnitudes (post-gate FFN inputs, post-attention values) produce flatter + softmaxes and small KLDs. **This systematically under-weights residual-path + tensors like `ffn_down` and `attn_output`**, which often carry more downstream + importance than their local KLD suggests. The `--importance-alpha` correction + (below) compensates for this. +- Local per-tensor KLD is summed as if tensors were independent, but errors + compound non-linearly through residuals and norms. No attempt is made to model + that propagation directly. +- Captures use the model's own (unquantized) activations. For trained-type + quantizations (IQ2_TQ, IQ3_TQ, Q4_DPT, ...), the measured KLD uses the levels + trained on the same data the imatrix was built from, which is a mildly + optimistic estimate of end-use behavior. + +## Importance-alpha bias correction + +To counter the local-KLD bias above, each role's KLD is optionally multiplied by a +weight derived from its imatrix energy: + +``` +role_energy[r] = mean over tensors in r of mean(imatrix[tensor]) +geomean = geometric mean of role_energy[r] across roles with data +role_weight[r] = (geomean / role_energy[r]) ^ alpha +weighted_kld[r] = local_kld[r] * role_weight[r] +``` + +`alpha = 0` disables the correction; `alpha = 1` is a full inverse-energy flip. +The weights are displayed in the run log. + +### Why imatrix energy? + +The imatrix records per-column input activation energy. Residual-path inputs +(post-gate FFN, post-attention) have low activation energy because the non-linearity +or attention weighting compresses the signal into a subspace. Those same tensors +write back into the residual stream, where errors accumulate. Inverse-energy +weighting promotes exactly those tensors. Empirically (see +[Calibration](#calibration)) this aligns with hand-tuned recipes. + +### Calibration + +Measured on `Qwen3.5-0.8B-BF16` and `Nanbeige4.1-3B-BF16` at target BPW 4.5, +quant-list `IQ3_TQ,Q4_DPT,IQ4_XS,Q4_K,Q5_K,Q6_K`, real-text test data: + +| α | Qwen3.5-0.8B PPL | Nanbeige-3B PPL | +|---|---|---| +| 0.0 | 54.2149 | 45.4793 | +| 0.5 | 54.2657 | 46.7783 (worse than baseline) | +| **1.0** | **53.2286** | **44.8957** | +| 1.5 | 52.1232 | (same recipe as 1.0) | +| 2.0 | 52.4972 (starts regressing) | (same recipe as 1.0) | + +`alpha=1.0` is the default. It's the first setting where the correction reliably +flips allocations on residual-path tensors (promotes `ffn_down`, `ssm_out`, +`attn_output`) on both tested architectures. Lower values (`0.5`) are marginal and +can regress on some architectures. Higher values (`1.5+`) sometimes win on dense +architectures but over-correct on sparser ones. + +## Optimizer + +Greedy, two-stage, on a (role × type) cost matrix: + +1. **Seed.** Every role starts at the lowest-BPW type in the candidate list that has + a finite KLD. +2. **Greedy upgrade.** Repeatedly pick the single role→type upgrade with the highest + `Δkld / Δbpw` ratio that still fits the BPW budget (`target + bpw_tol_high`), + until no improving move exists. +3. **Swap improvement.** Up to `--max-iterations` rounds, try upgrading one role + while downgrading another to stay in budget and reduce total weighted KLD. + Terminates on convergence. + +BPW accounting uses `ggml_get_bpw` (block size + per-tensor-trained overhead). +Global tensors (`token_embd`, `output`) are pinned before optimization begins: +- `output` → `--output-tensor-type`, or highest-BPW candidate if unset. +- `token_embd` → if tied embeddings detected (no distinct `output.weight` in the + model), force-matches `output_type` because `llama-quantize` will do the same + upgrade silently regardless of what we ask for. Otherwise → closest-BPW + `get_rows`-compatible candidate to target. + +## Layer equivalence classes + +Hybrid models (e.g. Qwen3.5's SSM + attention mix) have different tensor signatures +per layer. To avoid measuring every layer: + +1. Compute per-layer signature `sorted([(role, ne0, ne1)])`. +2. Group layers by signature into equivalence classes. +3. Sample representative layers per class (first, middle, last; deduped). + +Only representative layers are captured in Phase 2. Per-role KLDs are averaged across +all representatives within their class. + +## Fusion map + +When a model fuses projections (e.g. `attn_qkv` = Q‖K‖V as one MUL_MAT), the capture +belongs to the fused role. A `fusion_map` in the source splits the measured KLD +proportionally (by element count) to the component roles that lack their own +captures. Current entries: + +- `attn_qkv` → `{attn_q, attn_k, attn_v}` + +Extend as needed per arch. + +## Implementation notes + +- **Per-(weight, qtype) quant caching.** Same weight captured at multiple test + sizes is quantized once per type; the cached `quant_result` is reused by every + capture's `eval_mul_mat`. This is ~3× faster than the naive per-capture approach + and also avoids a latent CUDA race on per-tensor grid/level device symbols when + multiple captures of the same (weight, qtype) would otherwise run concurrently. +- **Parallelism.** `--threads N` launches N `std::async` workers per (weight, + capture) batch. The CUDA backend pool pre-creates one backend per thread + because the VMM pool requires strict LIFO alloc/free per backend. +- **Memory.** After Phase 2 finishes, the llama model is reset and the capture + scaffolding (`target_weight_names`, `weight_to_role`, tokenized test inputs) is + freed. During Phase 3 each role's capture buffers are released as soon as the + corresponding cost-matrix row is populated. The GGUF context is opened with + `no_alloc=true` to avoid duplicating the model in host RAM. + +## CLI reference + +``` +-m, --model PATH Input GGUF model (required) +-i, --imatrix PATH Importance matrix (required, also used for bias correction) +-q, --quants LIST Comma-separated candidate quant types (required) +-b, --target-bpw N Target bits per weight (required) +-o, --output PATH Output tensor-type-file (required) +--bpw-tolerance H,L BPW tolerance: +H, -L from target (default: +0,-0.2) +--test-data PATH Text file for test inputs (recommended; synthetic otherwise) +--test-sizes S1,S2,S3 Token counts for test inputs (default: 32,128,512) +--min-elements N Skip KLD measurement for tensors below this size (default: 40000) +--output-tensor-type T Quant type for output.weight (default: highest from list) +--max-iterations N Max swap-improvement iterations (default: 100) +--threads N Parallel workers (default: 1) +--importance-alpha A Imatrix-energy bias exponent (default: 1.0, range 0..2) +``` + +## Known limitations + +- **BPW estimate ignores GGUF metadata.** Tokenizer vocab, model config, and + alignment padding contribute ~2% to the file size. The tool's predicted BPW is + tensor-only. Expect final file size ≈ predicted size × 1.02. +- **Tied embeddings force `token_embd` up.** As noted above, the tool now detects + the tied case and reports the promoted BPW correctly. On such models, raising + `target_bpw` or overriding `--output-tensor-type` with a cheaper type is the only + way to free budget for other roles. +- **Small matrix weights are best-effort.** 2D `.weight` tensors below + `--min-elements` aren't measured for KLD; they're assumed to be quantized to the + candidate type closest to the target BPW for accounting purposes. On most archs + this is within ~1 MB of reality but can drift for unusual tensor shapes. +- **Synthetic tokens are worse than real text.** Default is `rand() % n_vocab`, + which has different activation statistics from real inputs. Always pass + `--test-data` when quality matters. +- **Local KLD is a proxy.** See the caveats above. For critical quantizations, + verify with `llama-perplexity` on held-out text and compare against a known-good + reference. diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index 02033755a727..96557545614f 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -596,13 +596,13 @@ static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int64_t void ggml_cuda_set_q4dpt_levels(const int8_t * levels, cudaStream_t stream) { int8_t * d_q4dpt_levels; CUDA_CHECK(cudaGetSymbolAddress((void **)&d_q4dpt_levels, q4dpt_levels_cuda)); - CUDA_CHECK(cudaMemcpyAsync(d_q4dpt_levels, levels, 16, cudaMemcpyDeviceToDevice, stream)); + CUDA_CHECK(cudaMemcpyAsync(d_q4dpt_levels, levels, 16, cudaMemcpyHostToDevice, stream)); } void ggml_cuda_set_q2dpt_levels(const int8_t * levels, cudaStream_t stream) { int8_t * d_q2dpt_levels; CUDA_CHECK(cudaGetSymbolAddress((void **)&d_q2dpt_levels, q2dpt_levels_cuda)); - CUDA_CHECK(cudaMemcpyAsync(d_q2dpt_levels, levels, 4, cudaMemcpyDeviceToDevice, stream)); + CUDA_CHECK(cudaMemcpyAsync(d_q2dpt_levels, levels, 4, cudaMemcpyHostToDevice, stream)); } void ggml_cuda_set_iq2tq_grid(const void * grid, cudaStream_t stream) { diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 80d5c32316c9..d8324073702d 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -6873,11 +6873,10 @@ static void quantize_row_iq2_tq_impl( size_t quantize_iq2_tq(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) { size_t row_size = ggml_row_size(GGML_TYPE_IQ2_TQ, n_per_row); - char * qrow = (char *)dst; for (int64_t row = 0; row < nrows; ++row) { - quantize_row_iq2_tq_impl(src, (block_iq2_tq *)qrow, n_per_row, imatrix); - src += n_per_row; - qrow += row_size; + const float * src_row = src + row * n_per_row; + char * qrow = (char *)dst + row * row_size; + quantize_row_iq2_tq_impl(src_row, (block_iq2_tq *)qrow, n_per_row, imatrix); } return nrows * row_size; } @@ -7404,11 +7403,10 @@ static void quantize_row_iq3_tq_impl( size_t quantize_iq3_tq(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) { size_t row_size = ggml_row_size(GGML_TYPE_IQ3_TQ, n_per_row); - char * qrow = (char *)dst; for (int64_t row = 0; row < nrows; ++row) { - quantize_row_iq3_tq_impl(src, (block_iq3_tq *)qrow, n_per_row, imatrix); - src += n_per_row; - qrow += row_size; + const float * src_row = src + row * n_per_row; + char * qrow = (char *)dst + row * row_size; + quantize_row_iq3_tq_impl(src_row, (block_iq3_tq *)qrow, n_per_row, imatrix); } return nrows * row_size; } diff --git a/tools/auto-tensor-type/auto-tensor-type.cpp b/tools/auto-tensor-type/auto-tensor-type.cpp index 3e740bcc0c58..4131e6e1c23c 100644 --- a/tools/auto-tensor-type/auto-tensor-type.cpp +++ b/tools/auto-tensor-type/auto-tensor-type.cpp @@ -87,6 +87,14 @@ struct config { int max_iterations = 100; std::string output_path; int n_threads = 1; + // Imatrix-energy bias correction: per-role KLD is multiplied by + // (geomean_energy / role_energy)^alpha + // to counter the local-KLD signal's systematic under-weighting of + // low-input-magnitude residual-path tensors (ffn_down, attn_output, ssm_out). + // alpha = 0 disables the correction; alpha = 1 is a full inverse-energy flip. + // Default chosen from empirical PPL sweeps on Qwen3.5-0.8B and Nanbeige-3B + // (see docs/auto-tensor-type.md); 0.5 was fragile on some architectures. + float importance_alpha = 1.0f; }; struct tensor_info { @@ -145,18 +153,23 @@ static std::string extract_role(const std::string & name) { return name; } -static bool is_quantizable_weight(const tensor_info & ti, int64_t min_elements) { - // Must end with .weight +// 2D .weight matrix, regardless of size. Used for BPW accounting — llama-quantize +// will quantize these to the fallback ftype even if we don't measure them. +static bool is_matrix_weight(const tensor_info & ti) { if (ti.name.size() < 8 || ti.name.substr(ti.name.size() - 7) != ".weight") return false; - // Must be 2D (matrix) if (ti.ne[2] != 1 || ti.ne[3] != 1) return false; - // Must have enough elements - if ((int64_t)ti.n_elements < min_elements) return false; - // Skip norms (1D-like: small ne[0] or ne[1] == 1) if (ti.ne[0] <= 1 || ti.ne[1] <= 1) return false; return true; } +static bool is_quantizable_weight(const tensor_info & ti, int64_t min_elements) { + // Must be a 2D matrix weight… + if (!is_matrix_weight(ti)) return false; + // …and large enough to justify per-role KLD measurement. + if ((int64_t)ti.n_elements < min_elements) return false; + return true; +} + static double compute_bpw(ggml_type type) { return ggml_get_bpw(type); } @@ -738,7 +751,7 @@ static bool eval_mul_mat(ggml_type weight_type, const void * weight_data, static std::map> build_cost_matrix( const config & cfg, const std::vector & /*tensors*/, - const std::unordered_map> & captures_by_role, + std::unordered_map> & captures_by_role, const std::string & model_path, const struct gguf_context * gguf_ctx, const std::unordered_map> & imatrix_data) { @@ -750,34 +763,45 @@ static std::map> build_cost_matrix( std::map> kld_sums; std::map> kld_counts; - // Cache for F32 weight data: tensor_name → float data - std::unordered_map> weight_cache; - const int n_parallel = std::max(1, cfg.n_threads); // Create a pool of backends — try CUDA first, fall back to CPU - // Each thread gets its own backend to avoid contention. + // Each thread gets its own backend to avoid contention (VMM pool requires LIFO alloc/free). struct backend_pool { std::vector backends; std::atomic next_idx{0}; std::string backend_name; backend_pool(int count) { - // Use ggml_backend_init_best() — picks GPU if available, falls back to CPU + // Initialize first backend to determine type ggml_backend_t be = ggml_backend_init_best(); if (!be) { LOG_ERR("Failed to init any backend\n"); return; } backend_name = ggml_backend_name(be); - // Single shared backend — ggml handles concurrency internally - for (int i = 0; i < count; i++) { - backends.push_back(be); + backends.push_back(be); + + // Create separate backends for remaining threads + // This is required because the CUDA VMM pool is a stack allocator + // that requires strict LIFO allocation/deallocation order per pool. + for (int i = 1; i < count; i++) { + ggml_backend_t be_i = ggml_backend_init_best(); + if (be_i) { + backends.push_back(be_i); + } else { + LOG_WRN("Failed to init backend %d, reusing backend 0\n", i); + backends.push_back(be); // Fallback to sharing + } } } ~backend_pool() { - if (!backends.empty()) { - ggml_backend_free(backends[0]); + // Free all unique backends + std::unordered_set freed; + for (auto be : backends) { + if (freed.insert(be).second) { + ggml_backend_free(be); + } } } ggml_backend_t get() { @@ -790,76 +814,99 @@ static std::map> build_cost_matrix( pool.backend_name.c_str(), n_parallel); // For each role that has captures - for (const auto & [role, captures] : captures_by_role) { - LOG("Building cost matrix for role '%s' (%zu captures, %d parallel threads)...\n", - role.c_str(), captures.size(), n_parallel); - + for (auto & [role, captures] : captures_by_role) { + // Group captures by weight_name. quantize_weight_to_type() depends only on + // (weight_name, qtype) — with multiple test_sizes the same weight is captured + // N times, so we quantize once per unique weight and reuse across captures. + std::unordered_map> caps_by_weight; for (const auto & cap : captures) { - // Read weight data as F32 (cached) - auto cache_it = weight_cache.find(cap.weight_name); - if (cache_it == weight_cache.end()) { - std::vector weight_f32; - if (!read_tensor_f32(model_path, gguf_ctx, cap.weight_name, weight_f32)) { - LOG_WRN(" Failed to read weight data for '%s', skipping\n", cap.weight_name.c_str()); - continue; - } - weight_cache[cap.weight_name] = std::move(weight_f32); - cache_it = weight_cache.find(cap.weight_name); + caps_by_weight[cap.weight_name].push_back(&cap); + } + + LOG("Building cost matrix for role '%s' (%zu captures across %zu unique weights, %d parallel threads)...\n", + role.c_str(), captures.size(), caps_by_weight.size(), n_parallel); + + const size_t n_types = cfg.quant_types.size(); + + for (const auto & [weight_name, caps_for_weight] : caps_by_weight) { + // Read weight data as F32 — scoped to this weight, freed at end of iteration + std::vector weight_f32; + if (!read_tensor_f32(model_path, gguf_ctx, weight_name, weight_f32)) { + LOG_WRN(" Failed to read weight data for '%s', skipping\n", weight_name.c_str()); + continue; } - const auto & weight_f32 = cache_it->second; - - // Get imatrix for this tensor - auto imat = get_imatrix_for_tensor(imatrix_data, cap.weight_name, cap.weight_ne0); - - // Evaluate all quant types for this capture in parallel batches - const size_t n_types = cfg.quant_types.size(); - std::vector>> futures; - futures.reserve(n_parallel); - - for (size_t ti = 0; ti < n_types; /* advanced inside */) { - // Launch up to n_parallel evaluations - futures.clear(); - for (int p = 0; p < n_parallel && ti < n_types; p++, ti++) { - ggml_type qtype = cfg.quant_types[ti]; - ggml_backend_t backend = pool.get(); - futures.push_back(std::async(std::launch::async, - [&cap, &weight_f32, &imat, qtype, backend]() -> std::pair { - // Quantize (sets per-type global state — safe since each type is different) - auto qres = quantize_weight_to_type( - qtype, weight_f32.data(), cap.weight_ne1, cap.weight_ne0, imat.data()); - - // Run MUL_MAT with quantized weight - std::vector quant_output; - if (!eval_mul_mat(qtype, qres.data.data(), - cap.weight_ne0, cap.weight_ne1, - cap.input_data.data(), cap.input_ne0, cap.input_ne1, - qres.levels, - quant_output, - backend)) { - return {qtype, std::numeric_limits::quiet_NaN()}; - } - - // Compute KLD - return {qtype, compute_avg_kld(cap.ref_output_data.data(), quant_output.data(), - cap.ref_ne0, cap.ref_ne1)}; - })); - } - // Collect results - for (auto & f : futures) { - auto [qtype, kld] = f.get(); - if (std::isfinite(kld)) { - std::lock_guard lock(accum_mutex); - kld_sums[role][qtype] += kld; - kld_counts[role][qtype]++; + const mul_mat_capture & first_cap = *caps_for_weight.front(); + auto imat = get_imatrix_for_tensor(imatrix_data, weight_name, first_cap.weight_ne0); + + // Quantize each qtype once for this weight (parallel across qtypes). + // Different qtypes use disjoint per-type globals, so parallel training is safe. + std::unordered_map quant_cache; + std::mutex quant_cache_mutex; + { + std::vector> qfutures; + qfutures.reserve(n_parallel); + for (size_t ti = 0; ti < n_types; /* advanced inside */) { + qfutures.clear(); + for (int p = 0; p < n_parallel && ti < n_types; p++, ti++) { + ggml_type qtype = cfg.quant_types[ti]; + qfutures.push_back(std::async(std::launch::async, + [&first_cap, &weight_f32, &imat, qtype, &quant_cache, &quant_cache_mutex]() { + auto qres = quantize_weight_to_type( + qtype, weight_f32.data(), + first_cap.weight_ne1, first_cap.weight_ne0, imat.data()); + std::lock_guard lock(quant_cache_mutex); + quant_cache.emplace(qtype, std::move(qres)); + })); } + for (auto & f : qfutures) f.get(); } } - } - // Free cache entries for this role's captures (no longer needed) - for (const auto & cap : captures) { - weight_cache.erase(cap.weight_name); + // For each capture × qtype, run MUL_MAT + KLD using the cached quant_result. + // eval_mul_mat reads the trained grid/levels from tensor->quant_levels + // (set from quant_result::levels), not from per-type globals, so a stale + // global state from later quantize calls is irrelevant here. + for (const auto * cap_ptr : caps_for_weight) { + const auto & cap = *cap_ptr; + + std::vector>> futures; + futures.reserve(n_parallel); + + for (size_t ti = 0; ti < n_types; /* advanced inside */) { + futures.clear(); + for (int p = 0; p < n_parallel && ti < n_types; p++, ti++) { + ggml_type qtype = cfg.quant_types[ti]; + auto qit = quant_cache.find(qtype); + if (qit == quant_cache.end()) continue; // quantization failed earlier + const quant_result * qres = &qit->second; + ggml_backend_t backend = pool.get(); + futures.push_back(std::async(std::launch::async, + [&cap, qtype, qres, backend]() -> std::pair { + std::vector quant_output; + if (!eval_mul_mat(qtype, qres->data.data(), + cap.weight_ne0, cap.weight_ne1, + cap.input_data.data(), cap.input_ne0, cap.input_ne1, + qres->levels, + quant_output, + backend)) { + return {qtype, std::numeric_limits::quiet_NaN()}; + } + return {qtype, compute_avg_kld(cap.ref_output_data.data(), quant_output.data(), + cap.ref_ne0, cap.ref_ne1)}; + })); + } + + for (auto & f : futures) { + auto [qtype, kld] = f.get(); + if (std::isfinite(kld)) { + std::lock_guard lock(accum_mutex); + kld_sums[role][qtype] += kld; + kld_counts[role][qtype]++; + } + } + } + } } // Build cost entries from accumulated KLD @@ -870,6 +917,10 @@ static std::map> build_cost_matrix( entry.bpw = compute_bpw(qtype); cost_matrix[role][qtype] = entry; } + + // Release this role's captured input/output buffers — largest single chunk of + // Phase-2 memory, no longer needed once the cost matrix row is populated. + std::vector().swap(captures); } return cost_matrix; @@ -1281,6 +1332,9 @@ static void print_usage(const char * prog) { LOG(" --output-tensor-type T Quant type for output.weight (default: highest from list)\n"); LOG(" --max-iterations N Max optimization iterations (default: 100)\n"); LOG(" --threads N Number of threads (default: 1)\n"); + LOG(" --importance-alpha A Imatrix-energy bias for per-role KLD (default: 1.0)\n"); + LOG(" weight[r] = (geomean_energy / role_energy[r])^A\n"); + LOG(" 0 = no bias, 1 = full inverse-energy reweighting\n"); } static config parse_args(int argc, char ** argv) { @@ -1325,6 +1379,8 @@ static config parse_args(int argc, char ** argv) { cfg.max_iterations = std::stoi(argv[++i]); } else if (arg == "--threads" && i + 1 < argc) { cfg.n_threads = std::stoi(argv[++i]); + } else if (arg == "--importance-alpha" && i + 1 < argc) { + cfg.importance_alpha = std::stof(argv[++i]); } else if (arg == "-h" || arg == "--help") { print_usage(argv[0]); exit(0); @@ -1370,8 +1426,11 @@ int main(int argc, char ** argv) { // ---- Phase 1: Load model metadata from GGUF ---- LOG("--- Phase 1: Loading model metadata ---\n"); + // no_alloc=true: we only need tensor metadata (dims, type, offset); raw weight data + // is read from disk in Phase 3 via read_tensor_f32. Loading it here would duplicate + // the entire model in host RAM alongside the llama-backend copy. struct ggml_context * ggml_ctx = nullptr; - struct gguf_init_params gguf_params = {false, &ggml_ctx}; + struct gguf_init_params gguf_params = {true, &ggml_ctx}; struct gguf_context * gguf_ctx = gguf_init_from_file(cfg.model_path.c_str(), gguf_params); if (!gguf_ctx) { LOG_ERR("Failed to open model: %s\n", cfg.model_path.c_str()); @@ -1501,9 +1560,14 @@ int main(int argc, char ** argv) { common_params params; params.model.path = cfg.model_path; params.n_gpu_layers = 99; // Offload to GPU if available, falls back to CPU - params.n_batch = 512; - params.n_ubatch = 512; - params.n_ctx = 1024; // enough for test sizes + // Size batch/context to the largest requested test size (+BOS); n_batch must be + // >= n_tokens_all or llama_decode asserts. + int max_test_size = 0; + for (int s : cfg.test_sizes) max_test_size = std::max(max_test_size, s); + const int min_batch = std::max(512, max_test_size + 1); + params.n_batch = min_batch; + params.n_ubatch = min_batch; + params.n_ctx = std::max(1024, max_test_size + 1); capture_state cap_state; for (const auto & ti : quantizable) { @@ -1606,6 +1670,14 @@ int main(int argc, char ** argv) { llama_init.reset(); llama_backend_free(); + // Free Phase 2 scaffolding: only the captured input/output tensors are needed from here. + // swap-with-empty forces the underlying bucket arrays to actually return memory. + decltype(cap_state.target_weight_names)().swap(cap_state.target_weight_names); + decltype(cap_state.weight_to_role)().swap(cap_state.weight_to_role); + decltype(cap_state.weight_to_layer)().swap(cap_state.weight_to_layer); + std::vector().swap(cap_state.tmp_data); + std::vector>().swap(test_token_sets); + // ---- Phase 3: Build cost matrix ---- LOG("\n--- Phase 3: Building cost matrix ---\n"); @@ -1634,14 +1706,33 @@ int main(int argc, char ** argv) { LOG("\n--- Phase 4: Optimizing assignment ---\n"); // Determine fixed types for global tensors (before building roles map) - ggml_type token_embd_type = closest_bpw_if(cfg.quant_types, cfg.target_bpw, supports_get_rows); - if (token_embd_type == GGML_TYPE_COUNT) { - token_embd_type = highest_bpw(cfg.quant_types); // fallback - } ggml_type output_type = cfg.output_tensor_type != GGML_TYPE_COUNT ? cfg.output_tensor_type : highest_bpw(cfg.quant_types); + // Detect tied embeddings: llama-quantize (src/llama-quant.cpp:~534) silently + // promotes token_embd to output_tensor_type when the arch lacks a distinct + // output.weight. If we pick a different (lower-bpw) token_embd type in that + // case, our BPW estimate is off by a lot (e.g. Q4_K vs Q6_K on a 150k-vocab + // embedding table is ~60MB). Match llama-quantize's behavior up front. + bool has_output_weight = false; + for (const auto & ti : all_tensors) { + if (ti.name == "output.weight") { has_output_weight = true; break; } + } + const bool has_tied_embeddings = !has_output_weight; + + ggml_type token_embd_type; + if (has_tied_embeddings) { + token_embd_type = output_type; + LOG("Detected tied embeddings: token_embd will be quantized with output_type (%s)\n", + ggml_type_name(token_embd_type)); + } else { + token_embd_type = closest_bpw_if(cfg.quant_types, cfg.target_bpw, supports_get_rows); + if (token_embd_type == GGML_TYPE_COUNT) { + token_embd_type = highest_bpw(cfg.quant_types); // fallback + } + } + // Add global tensors to the cost matrix with their fixed types. // KLD=0 since they're not optimized — we just need their BPW in the budget. for (ggml_type qtype : cfg.quant_types) { @@ -1668,13 +1759,27 @@ int main(int argc, char ** argv) { total_quant_elements += ti.n_elements; } - // Compute non-quantizable tensor overhead (small tensors kept at original precision) - // These contribute fixed BPW that must be accounted for in the budget. + // Small 2D .weight tensors below min_elements aren't measured for KLD, but + // llama-quantize will still quantize them (to the fallback ftype passed on + // the CLI). We don't know the user's chosen fallback, so we approximate it + // as the listed type closest to the target BPW. + ggml_type small_matrix_type = closest_bpw(cfg.quant_types, cfg.target_bpw); + if (small_matrix_type == GGML_TYPE_COUNT) small_matrix_type = token_embd_type; + size_t total_all_elements = 0; double non_quantizable_bits = 0; + size_t small_matrix_elements = 0; for (const auto & ti : all_tensors) { total_all_elements += ti.n_elements; - if (!is_quantizable_weight(ti, cfg.min_elements)) { + if (is_quantizable_weight(ti, cfg.min_elements)) { + continue; // counted via role_to_type in compute_total_bpw + } + if (is_matrix_weight(ti)) { + // Small 2D weight: llama-quantize will quantize it with the fallback ftype. + non_quantizable_bits += compute_bpw(small_matrix_type) * ti.n_elements; + small_matrix_elements += ti.n_elements; + } else { + // 1D, biases, norms: stay at original precision. non_quantizable_bits += compute_bpw(ti.orig_type) * ti.n_elements; } } @@ -1683,6 +1788,10 @@ int main(int argc, char ** argv) { LOG("Total elements in all tensors: %zu\n", total_all_elements); LOG("Non-quantizable overhead: %.4f BPW\n", total_all_elements > 0 ? non_quantizable_bits / total_all_elements : 0); + if (small_matrix_elements > 0) { + LOG("Small matrix weights below --min-elements (%zu elements) assumed %s for BPW estimate\n", + small_matrix_elements, ggml_type_name(small_matrix_type)); + } for (const auto & [role, ri] : roles) { LOG(" %s: %zu elements%s\n", role.c_str(), ri.n_elements, cost_matrix.count(role) ? "" : " [NO CAPTURES]"); @@ -1704,7 +1813,60 @@ int main(int argc, char ** argv) { } } } - auto result = optimize_assignment(cfg, cost_matrix, roles, total_all_elements, non_quantizable_bits); + // Imatrix-energy bias correction. Local MUL_MAT KLD systematically under-weights + // residual-path tensors with small-magnitude inputs (ffn_down, attn_output, ssm_out) + // because softmax over a small-output row is flatter. We compensate by scaling each + // role's KLD by (geomean_energy / role_energy)^alpha, computed from the imatrix. + std::map role_weight; + if (cfg.importance_alpha > 0.0f && !imatrix_data.empty()) { + std::map> role_mean_by_tensor; + for (const auto & [tname, imat] : imatrix_data) { + if (imat.empty()) continue; + double s = 0; + for (float v : imat) s += v; + role_mean_by_tensor[extract_role(tname)].push_back(s / imat.size()); + } + + std::map role_energy; + double log_sum = 0; + int n_energies = 0; + for (const auto & [role, vs] : role_mean_by_tensor) { + double sum = 0; + for (double v : vs) sum += v; + double mean = vs.empty() ? 0.0 : sum / vs.size(); + role_energy[role] = mean; + if (mean > 1e-20) { + log_sum += std::log(mean); + n_energies++; + } + } + + const double geo_mean = n_energies > 0 ? std::exp(log_sum / n_energies) : 1.0; + for (const auto & [role, e] : role_energy) { + role_weight[role] = e > 1e-20 + ? std::pow(geo_mean / e, (double)cfg.importance_alpha) + : 1.0; + } + + LOG("\nRole importance weights (alpha=%.2f, geomean_energy=%.4g):\n", cfg.importance_alpha, geo_mean); + for (const auto & [role, w] : role_weight) { + LOG(" %-16s energy=%10.4g weight=%6.3f\n", role.c_str(), role_energy[role], w); + } + } + + // Apply weights to a cost-matrix copy used by the optimizer; preserve raw values + // for display and for the final KLD we report to the user. + auto weighted_cost = cost_matrix; + for (auto & [role, row] : weighted_cost) { + auto wit = role_weight.find(role); + if (wit == role_weight.end()) continue; + double w = wit->second; + for (auto & [qt, ce] : row) { + if (ce.kld < 1e29) ce.kld *= w; // keep sentinels intact + } + } + + auto result = optimize_assignment(cfg, weighted_cost, roles, total_all_elements, non_quantizable_bits); LOG("\nOptimal assignment:\n"); for (const auto & [role, type] : result.role_to_type) { @@ -1716,7 +1878,15 @@ int main(int argc, char ** argv) { } LOG("Total BPW: %.4f (target: %.2f +%.2f/-%.2f)\n", result.total_bpw, cfg.target_bpw, cfg.bpw_tol_high, cfg.bpw_tol_low); - LOG("Total KLD: %.6f\n", result.total_kld); + // Raw total KLD is the unweighted sum from cost_matrix; the optimizer's + // result.total_kld is in weighted units when importance_alpha > 0. + double raw_total_kld = compute_total_kld(result.role_to_type, cost_matrix); + if (cfg.importance_alpha > 0.0f && !role_weight.empty()) { + LOG("Total KLD: %.6f raw (%.6f weighted, used by optimizer)\n", + raw_total_kld, result.total_kld); + } else { + LOG("Total KLD: %.6f\n", raw_total_kld); + } // ---- Phase 5: Output tensor-type-file ---- LOG("\n--- Phase 5: Writing output ---\n"); From bcc37d9283dcaf46e3310226b5ecfcf9761bf483 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Mon, 20 Apr 2026 15:55:57 +0200 Subject: [PATCH 07/19] allow multisample --- tools/auto-tensor-type/auto-tensor-type.cpp | 74 +++++++++++++++++---- 1 file changed, 61 insertions(+), 13 deletions(-) diff --git a/tools/auto-tensor-type/auto-tensor-type.cpp b/tools/auto-tensor-type/auto-tensor-type.cpp index 4131e6e1c23c..3bdec9ea0ffe 100644 --- a/tools/auto-tensor-type/auto-tensor-type.cpp +++ b/tools/auto-tensor-type/auto-tensor-type.cpp @@ -81,6 +81,7 @@ struct config { float bpw_tol_high = 0.0f; // can be up to this much above target float bpw_tol_low = 0.2f; // can be up to this much below target std::string test_data_path; // optional test text file + int n_test_samples = 1; // number of samples per test size std::vector test_sizes = {32, 128, 512}; int64_t min_elements = 40000; ggml_type output_tensor_type = GGML_TYPE_COUNT; // default: highest from list @@ -692,7 +693,7 @@ static bool eval_mul_mat(ggml_type weight_type, const void * weight_data, // Set per-tensor levels if provided (needed for Q3_KPT, Q4_DPT, etc.) if (!quant_levels.empty()) { - w->quant_levels = (void *)quant_levels.data(); + w->quant_levels = const_cast(static_cast(quant_levels.data())); } // Create input tensor @@ -1327,6 +1328,7 @@ static void print_usage(const char * prog) { LOG(" -o, --output PATH Output tensor-type-file path (required)\n"); LOG(" --bpw-tolerance HIGH,LOW BPW tolerance: +HIGH, -LOW from target (default: +0,-0.2)\n"); LOG(" --test-data PATH Text file for test inputs (optional, synthetic if not given)\n"); + LOG(" --test-samples N Number of samples per test size (default: 1)\n"); LOG(" --test-sizes S1,S2,S3 Token counts for test inputs (default: 32,128,512)\n"); LOG(" --min-elements N Skip tensors with fewer elements (default: 40000)\n"); LOG(" --output-tensor-type T Quant type for output.weight (default: highest from list)\n"); @@ -1363,6 +1365,12 @@ static config parse_args(int argc, char ** argv) { } } else if (arg == "--test-data" && i + 1 < argc) { cfg.test_data_path = argv[++i]; + } else if (arg == "--test-samples" && i + 1 < argc) { + cfg.n_test_samples = std::stoi(argv[++i]); + if (cfg.n_test_samples < 1) { + LOG_ERR("--test-samples must be >= 1\n"); + exit(1); + } } else if (arg == "--test-sizes" && i + 1 < argc) { std::string sizes = argv[++i]; std::istringstream ss(sizes); @@ -1602,11 +1610,19 @@ int main(int argc, char ** argv) { const bool add_bos = llama_vocab_get_add_bos(vocab); const int n_vocab = llama_vocab_n_tokens(vocab); - // Prepare test tokens for each size + // Prepare test tokens for each (size, sample) combination. + // Tokens are taken as consecutive non-overlapping chunks from the file: + // [size0_sample0][size0_sample1]...[size1_sample0][size1_sample1]... + // With --test-sizes 64,128 --test-samples 3, the layout is: + // [64][64][64][128][128][128] (6 consecutive chunks) + struct test_set_info { + int size; + int sample; + }; std::vector> test_token_sets; + std::vector test_set_infos; if (!cfg.test_data_path.empty()) { - // Read test data file std::ifstream tf(cfg.test_data_path); if (!tf) { LOG_ERR("Failed to open test data file: %s\n", cfg.test_data_path.c_str()); @@ -1617,29 +1633,60 @@ int main(int argc, char ** argv) { auto all_tokens = common_tokenize(ctx, test_text, add_bos, false); LOG("Tokenized test data: %zu tokens\n", all_tokens.size()); + size_t total_needed = 0; for (int size : cfg.test_sizes) { - int n = std::min(size, (int)all_tokens.size()); - test_token_sets.push_back(std::vector(all_tokens.begin(), all_tokens.begin() + n)); + total_needed += (size_t)size * cfg.n_test_samples; + } + if (all_tokens.size() < total_needed) { + LOG_ERR("Test data file too small: need %zu tokens (%d samples × %d sizes: ", + total_needed, cfg.n_test_samples, (int)cfg.test_sizes.size()); + for (size_t i = 0; i < cfg.test_sizes.size(); i++) { + if (i > 0) LOG_ERR(" + "); + LOG_ERR("%d×%d", cfg.n_test_samples, cfg.test_sizes[i]); + } + LOG_ERR("), but only %zu tokens available in '%s'\n", + all_tokens.size(), cfg.test_data_path.c_str()); + return 1; + } + + size_t offset = 0; + for (int size : cfg.test_sizes) { + for (int sample = 0; sample < cfg.n_test_samples; sample++) { + test_token_sets.emplace_back(all_tokens.begin() + offset, + all_tokens.begin() + offset + size); + test_set_infos.push_back({size, sample}); + offset += size; + } + } + LOG("Prepared %zu test sets from file (%zu tokens consumed):\n", + test_token_sets.size(), offset); + for (size_t i = 0; i < test_set_infos.size(); i++) { + LOG(" [%zu] size=%d, sample=%d, tokens=%zu\n", + i, test_set_infos[i].size, test_set_infos[i].sample + 1, + test_token_sets[i].size()); } } else { - // Synthetic: random tokens LOG("Using synthetic test inputs\n"); srand(42); for (int size : cfg.test_sizes) { - std::vector tokens; - if (add_bos) tokens.push_back(llama_vocab_bos(vocab)); - while ((int)tokens.size() < size) { - tokens.push_back(rand() % n_vocab); + for (int sample = 0; sample < cfg.n_test_samples; sample++) { + std::vector tokens; + if (add_bos) tokens.push_back(llama_vocab_bos(vocab)); + while ((int)tokens.size() < size) { + tokens.push_back(rand() % n_vocab); + } + test_token_sets.push_back(tokens); + test_set_infos.push_back({size, sample}); } - test_token_sets.push_back(tokens); } } // Run forward passes with the eval callback for (size_t si = 0; si < test_token_sets.size(); si++) { const auto & tokens = test_token_sets[si]; - LOG("Running forward pass with %zu tokens (size %d)...\n", - tokens.size(), cfg.test_sizes[si]); + const auto & info = test_set_infos[si]; + LOG("Running forward pass %zu/%zu: size=%d, sample=%d, %zu tokens...\n", + si + 1, test_token_sets.size(), info.size, info.sample + 1, tokens.size()); // Clear KV cache llama_memory_clear(llama_get_memory(ctx), true); @@ -1677,6 +1724,7 @@ int main(int argc, char ** argv) { decltype(cap_state.weight_to_layer)().swap(cap_state.weight_to_layer); std::vector().swap(cap_state.tmp_data); std::vector>().swap(test_token_sets); + std::vector().swap(test_set_infos); // ---- Phase 3: Build cost matrix ---- LOG("\n--- Phase 3: Building cost matrix ---\n"); From 22b6044a3fd6b8c778af54c0c874a13c756511ba Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Tue, 21 Apr 2026 21:37:24 +0200 Subject: [PATCH 08/19] Algorithm refinement --- tools/auto-tensor-type/auto-tensor-type.cpp | 746 ++++++++++---------- 1 file changed, 377 insertions(+), 369 deletions(-) diff --git a/tools/auto-tensor-type/auto-tensor-type.cpp b/tools/auto-tensor-type/auto-tensor-type.cpp index 3bdec9ea0ffe..61e2cc492ec1 100644 --- a/tools/auto-tensor-type/auto-tensor-type.cpp +++ b/tools/auto-tensor-type/auto-tensor-type.cpp @@ -88,14 +88,11 @@ struct config { int max_iterations = 100; std::string output_path; int n_threads = 1; - // Imatrix-energy bias correction: per-role KLD is multiplied by - // (geomean_energy / role_energy)^alpha - // to counter the local-KLD signal's systematic under-weighting of - // low-input-magnitude residual-path tensors (ffn_down, attn_output, ssm_out). - // alpha = 0 disables the correction; alpha = 1 is a full inverse-energy flip. - // Default chosen from empirical PPL sweeps on Qwen3.5-0.8B and Nanbeige-3B - // (see docs/auto-tensor-type.md); 0.5 was fragile on some architectures. - float importance_alpha = 1.0f; + // Number of layer buckets per equivalence class. Captures/assignments are + // made per (role, bucket), so early/middle/late layers of the same role + // can get different quant types (matches hand-tuned recipes in llama-quant.cpp). + // Set to 1 to reproduce the older per-role-only behavior. + int n_layer_buckets = 3; }; struct tensor_info { @@ -126,12 +123,45 @@ struct mul_mat_capture { int64_t ref_ne0, ref_ne1; // [ne0=weight_ne1, ne1=n_tokens] }; -// Cost of assigning a specific quant type to a role +// Cost of assigning a specific quant type to a (role, bucket) struct cost_entry { - double kld; // average KLD across all captures for this role + double kld; // average relative-L2 error across captures for this role-bucket double bpw; // bits per weight for this quant type }; +// Key for the cost matrix: role + layer-bucket index. +// Bucket -1 is reserved for globals (token_embd, output). +// Encoded as "role\x01" — \x01 is not a character that appears in tensor names. +// Kept as std::string so existing std::map types are unaffected. +static constexpr char RB_SEP = '\x01'; +static std::string make_rb_key(const std::string & role, int bucket) { + return role + RB_SEP + std::to_string(bucket); +} +static std::string rb_role(const std::string & key) { + auto p = key.find(RB_SEP); + return (p == std::string::npos) ? key : key.substr(0, p); +} +static int rb_bucket(const std::string & key) { + auto p = key.find(RB_SEP); + return (p == std::string::npos) ? 0 : std::stoi(key.substr(p + 1)); +} +// Human-readable form for logs: "ffn_down[0]" or "output[G]" +static std::string rb_display(const std::string & key) { + std::string r = rb_role(key); + int b = rb_bucket(key); + if (b < 0) return r + "[G]"; + return r + "[" + std::to_string(b) + "]"; +} +// Bucket index of a layer given its position within its equivalence class. +// Returns a bucket in [0, n_buckets). +static int compute_bucket(int pos_in_class, int n_in_class, int n_buckets) { + if (n_in_class <= 1 || n_buckets <= 1) return 0; + int b = (int)(((long long)n_buckets * pos_in_class) / n_in_class); + if (b >= n_buckets) b = n_buckets - 1; + if (b < 0) b = 0; + return b; +} + // ============================================================================ // Section 3: Utility functions // ============================================================================ @@ -234,59 +264,49 @@ static ggml_type closest_bpw_if(const std::vector & types, float targ return best; } -// Compute KLD between two rows treated as probability distributions (after softmax) -// p = softmax(ref), q = softmax(quant) -// KLD = sum(p * log(p/q)) +// Compute relative squared L2 error between two rows of size n: +// err = ||ref - quant||^2 / (||ref||^2 + eps) +// Why this metric instead of softmax-KLD? +// Softmax over an intermediate MUL_MAT output is not a real distribution, and +// its peakedness depends on the row's absolute magnitude. Rows with large +// magnitudes (attn_q/k/v read from the post-norm residual) produce near one-hot +// softmaxes and dramatic KLDs; rows with small magnitudes (ffn_down/attn_output +// write into the residual) produce flat softmaxes and tiny KLDs. The signal is +// thus inversely correlated with the downstream cost of quantizing the tensor, +// and the `importance_alpha` hack is a scalar fix for a metric-level problem. +// Relative L2 is scale-free per-row: it measures fractional output perturbation +// directly, which is a much better proxy for how the error propagates through +// the residual stream. static double compute_kld_row(const float * ref, const float * quant, int64_t n) { - // Find max for numerical stability - float max_ref = -1e30f, max_qt = -1e30f; + double num = 0; + double den = 0; for (int64_t i = 0; i < n; i++) { - if (ref[i] > max_ref) max_ref = ref[i]; - if (quant[i] > max_qt) max_qt = quant[i]; + double r = (double)ref[i]; + double d = r - (double)quant[i]; + num += d * d; + den += r * r; } - - // Compute softmax - std::vector p(n), q(n); - double sum_p = 0, sum_q = 0; - for (int64_t i = 0; i < n; i++) { - p[i] = expf(ref[i] - max_ref); - q[i] = expf(quant[i] - max_qt); - sum_p += p[i]; - sum_q += q[i]; - } - float inv_sum_p = 1.0f / (float)sum_p; - float inv_sum_q = 1.0f / (float)sum_q; - for (int64_t i = 0; i < n; i++) { - p[i] *= inv_sum_p; - q[i] *= inv_sum_q; - } - - // KLD = sum(p * log(p/q)) - const float eps = 1e-10f; - double kld = 0; - for (int64_t i = 0; i < n; i++) { - if (p[i] > eps) { - float q_clipped = std::max(q[i], eps); - kld += (double)p[i] * log((double)p[i] / (double)q_clipped); - } + // Guard rows whose reference is all-zero (degenerate, no signal to match). + if (den <= 1e-20) { + return num > 1e-20 ? 1.0 : 0.0; } - return kld; + return num / den; } -// Compute average KLD across all rows of two F32 matrices of the same shape [ne0, ne1] +// Compute average relative-L2 across all rows of two F32 matrices of shape [ne0, ne1] static double compute_avg_kld(const float * ref, const float * quant, int64_t ne0, int64_t ne1) { - double total_kld = 0; + double total = 0; int64_t valid_rows = 0; for (int64_t row = 0; row < ne1; row++) { const float * ref_row = ref + row * ne0; const float * quant_row = quant + row * ne0; - double kld = compute_kld_row(ref_row, quant_row, ne0); - if (std::isfinite(kld)) { - total_kld += kld; + double e = compute_kld_row(ref_row, quant_row, ne0); + if (std::isfinite(e)) { + total += e; valid_rows++; } } - return valid_rows > 0 ? total_kld / valid_rows : 1e30; + return valid_rows > 0 ? total / valid_rows : 1e30; } // Read a tensor's raw data from the GGUF file and dequantize to F32 @@ -436,8 +456,10 @@ struct capture_state { std::unordered_map weight_to_role; // Map from weight tensor name -> layer index std::unordered_map weight_to_layer; + // Map from weight tensor name -> layer bucket (0..n_buckets-1, or -1 for globals) + std::unordered_map weight_to_bucket; - // Captures, organized by role + // Captures, organized by rb_key = make_rb_key(role, bucket) std::unordered_map> captures_by_role; // Temporary buffer for non-host tensor data @@ -472,6 +494,8 @@ static bool capture_callback(ggml_tensor * t, bool ask, void * user_data) { cap.weight_type = t->src[0]->type; cap.weight_ne0 = t->src[0]->ne[0]; cap.weight_ne1 = t->src[0]->ne[1]; + const int cap_bucket = state->weight_to_bucket.count(weight_name) + ? state->weight_to_bucket[weight_name] : (cap.layer < 0 ? -1 : 0); // Capture input (src[1]) { @@ -511,7 +535,7 @@ static bool capture_callback(ggml_tensor * t, bool ask, void * user_data) { cap.ref_output_data.assign(src_ptr, src_ptr + (nbytes / sizeof(float))); } - state->captures_by_role[cap.role].push_back(std::move(cap)); + state->captures_by_role[make_rb_key(cap.role, cap_bucket)].push_back(std::move(cap)); state->captured++; return true; @@ -824,8 +848,8 @@ static std::map> build_cost_matrix( caps_by_weight[cap.weight_name].push_back(&cap); } - LOG("Building cost matrix for role '%s' (%zu captures across %zu unique weights, %d parallel threads)...\n", - role.c_str(), captures.size(), caps_by_weight.size(), n_parallel); + LOG("Building cost matrix for %s (%zu captures across %zu unique weights, %d parallel threads)...\n", + rb_display(role).c_str(), captures.size(), caps_by_weight.size(), n_parallel); const size_t n_types = cfg.quant_types.size(); @@ -951,38 +975,45 @@ struct role_info { size_t n_bytes_orig; // original size in bytes }; -// Post-process the cost matrix: for fused roles, split KLD proportionally -// among component roles (by element count). Component roles that already -// have their own captures keep their directly-measured KLD. +// Post-process the cost matrix: for fused roles, split the per-(role, bucket) +// error proportionally among component roles (by element count) at the same +// bucket. Component role-buckets that already have their own captures keep +// their directly-measured value. static void split_fused_roles( std::map> & cost_matrix, const std::map & roles) { - for (const auto & [fused_role, components] : fusion_map) { - auto fused_it = cost_matrix.find(fused_role); - if (fused_it == cost_matrix.end()) continue; + // Collect rb_keys to split first — we'll mutate cost_matrix while iterating. + std::vector fused_keys; + for (const auto & [k, _] : cost_matrix) { + std::string role = rb_role(k); + if (fusion_map.count(role)) fused_keys.push_back(k); + } + + for (const auto & fused_key : fused_keys) { + std::string fused_role = rb_role(fused_key); + int bucket = rb_bucket(fused_key); + const auto & components = fusion_map.at(fused_role); - // Compute element counts for each component role + // Element counts for each component role at THIS bucket. size_t total_comp_elements = 0; std::vector comp_elems; for (const auto & comp : components) { - auto rit = roles.find(comp); + auto rit = roles.find(make_rb_key(comp, bucket)); size_t ne = (rit != roles.end()) ? rit->second.n_elements : 0; comp_elems.push_back(ne); total_comp_elements += ne; } if (total_comp_elements == 0) continue; - // For each quant type, create proportional cost entries for component roles - for (const auto & [qtype, fused_entry] : fused_it->second) { + const auto fused_entries = cost_matrix[fused_key]; // copy — we'll be mutating + + for (const auto & [qtype, fused_entry] : fused_entries) { for (size_t i = 0; i < components.size(); i++) { double fraction = (double)comp_elems[i] / (double)total_comp_elements; - - auto & comp_entries = cost_matrix[components[i]]; - if (comp_entries.count(qtype)) { - // Component already has its own measurement — keep it, don't overwrite - continue; - } + std::string comp_key = make_rb_key(components[i], bucket); + auto & comp_entries = cost_matrix[comp_key]; + if (comp_entries.count(qtype)) continue; // direct measurement wins cost_entry comp_entry; comp_entry.kld = fused_entry.kld * fraction; @@ -991,11 +1022,12 @@ static void split_fused_roles( } } - LOG("Split fused role '%s' KLD into components:", fused_role.c_str()); + LOG("Split fused %s error into components:\n", rb_display(fused_key).c_str()); for (size_t i = 0; i < components.size(); i++) { double fraction = (double)comp_elems[i] / (double)total_comp_elements; LOG(" %s: %.1f%% (%zu elements)\n", - components[i].c_str(), fraction * 100, comp_elems[i]); + rb_display(make_rb_key(components[i], bucket)).c_str(), + fraction * 100, comp_elems[i]); } } } @@ -1038,6 +1070,13 @@ static double compute_total_kld( return total; } +// Exact multi-choice knapsack DP over per-(role, bucket) items. +// Each item has a set of (qtype, bpw, error) choices; pick one per item, +// minimize total error, constrained by total BPW in [target - tol_low, target + tol_high]. +// +// Budget is discretized: 1 DP unit ≈ total_all_elements / N_UNITS bits. At +// N_UNITS = 16384 on an 800M-param model, 1 unit ≈ 50K bits ≈ 0.00006 BPW, +// well below any realistic tolerance. Table size is O(n_items × budget_units). static assignment optimize_assignment( const config & cfg, const std::map> & cost_matrix, @@ -1045,237 +1084,205 @@ static assignment optimize_assignment( size_t total_all_elements, double non_quantizable_bits) { - auto types_by_bpw = sorted_by_bpw(cfg.quant_types); - if (types_by_bpw.empty()) { - LOG_ERR("No quant types specified\n"); + struct dp_choice { ggml_type qt; int units; double kld; }; + struct dp_item { std::string key; size_t n_elements; std::vector choices; }; + + constexpr int N_UNITS = 16384; + const double bits_per_unit = (double)total_all_elements / (double)N_UNITS; + + const double hi_bpw = cfg.target_bpw + cfg.bpw_tol_high; + const double lo_bpw = std::max(0.0, (double)cfg.target_bpw - (double)cfg.bpw_tol_low); + const int budget_hi = (int)std::ceil(hi_bpw * N_UNITS); + const int budget_lo = (int)std::floor(lo_bpw * N_UNITS); + const int non_quant_units = + (int)std::round(non_quantizable_bits / bits_per_unit); + + // DP state axis is sized to allow up to 2 BPW of overshoot beyond the upper + // tolerance. This matters when the forced minimum (e.g. token_embd at Q6_K + // with tied embeddings) already overshoots the target — we still want to + // report the best achievable assignment rather than fail. + const int overshoot_cap = (int)std::ceil(2.0 * N_UNITS); + const int B = budget_hi + overshoot_cap + 1; + + // Build items from the cost matrix (skip items with zero elements — they + // correspond to roles with no actual tensors, e.g. fused-role entries + // materialized only through split_fused_roles). + // + // Per-choice cost is element-weighted: n_elements * relative_L2. The raw + // per-row relative-L2 is dimensionless and treats all roles equally, which + // makes the DP happy to spend Q6_K on a tiny tensor (4M-param attn_output) + // while starving a 37M-param attn_qkv to IQ2_TQ — same bit cost per role, + // very different aggregate quality impact. Element-weighting aligns the + // objective with "total number of parameters perturbed, weighted by + // fractional output error" and keeps big tensors from being sacrificed. + std::vector items; + items.reserve(cost_matrix.size()); + for (const auto & [key, row] : cost_matrix) { + auto rit = roles.find(key); + size_t ne = (rit != roles.end()) ? rit->second.n_elements : 0; + if (ne == 0) continue; + dp_item it; + it.key = key; + it.n_elements = ne; + for (auto qt : cfg.quant_types) { + auto cit = row.find(qt); + if (cit == row.end()) continue; + if (cit->second.kld >= 1e29) continue; // sentinel: forbidden choice + dp_choice c; + c.qt = qt; + c.units = (int)std::round((double)ne * compute_bpw(qt) / bits_per_unit); + if (c.units < 1) c.units = 1; + c.kld = cit->second.kld * (double)ne; + it.choices.push_back(c); + } + if (it.choices.empty()) continue; + items.push_back(std::move(it)); + } + if (items.empty()) { + LOG_ERR("No optimization items — empty cost matrix?\n"); return {}; } - // Initialize: assign each role the lowest-BPW quant type (greedy fill) - // We start from the bottom and work up until we hit the BPW budget - std::map current; - for (const auto & [role, info] : roles) { - // Start with the lowest-BPW type that has a valid KLD - auto it = cost_matrix.find(role); - if (it == cost_matrix.end()) continue; - ggml_type best = GGML_TYPE_COUNT; - for (auto qt : types_by_bpw) { - auto it2 = it->second.find(qt); - if (it2 != it->second.end() && it2->second.kld < 1e29) { - best = qt; - break; - } - } - if (best == GGML_TYPE_COUNT) continue; - current[role] = best; - } - - // Greedily upgrade roles to higher-quality types until we hit the BPW budget - // For each role, find the "best value" upgrade (most KLD reduction per BPW increase) - bool improved = true; - while (improved) { - improved = false; - (void)0; // cur_bpw used only for debugging - - double best_ratio = 0; - std::string best_role; - ggml_type best_type = GGML_TYPE_COUNT; - - for (const auto & [role, cur_type] : current) { - // Find next higher-BPW type - auto it = cost_matrix.find(role); - if (it == cost_matrix.end()) continue; - - double cur_kld = it->second.count(cur_type) ? it->second.at(cur_type).kld : 1e30; - - for (auto qt : types_by_bpw) { - if (compute_bpw(qt) <= compute_bpw(cur_type)) continue; // skip lower/same - auto it2 = it->second.find(qt); - if (it2 == it->second.end() || it2->second.kld >= 1e29) continue; - - // Check if this upgrade would exceed BPW budget - auto test = current; - test[role] = qt; - double test_bpw = compute_total_bpw(test, roles, total_all_elements, non_quantizable_bits); - if (test_bpw > cfg.target_bpw + cfg.bpw_tol_high) continue; - - // Compute improvement ratio (KLD reduction per BPW increase) - double kld_reduction = cur_kld - it2->second.kld; - double bpw_increase = compute_bpw(qt) - compute_bpw(cur_type); - if (bpw_increase <= 0) continue; - double ratio = kld_reduction / bpw_increase; - - if (ratio > best_ratio) { - best_ratio = ratio; - best_role = role; - best_type = qt; + const int n = (int)items.size(); + constexpr double INF = 1e300; + + std::vector dp(B, INF); + std::vector next_dp(B, INF); + // choice_taken[i][u] = which choice index was used at item i to reach state u + // prev_u[i][u] = the u state this came from + std::vector> choice_taken(n, std::vector(B, -1)); + std::vector> prev_u(n, std::vector(B, -1)); + + if (non_quant_units >= 0 && non_quant_units < B) dp[non_quant_units] = 0.0; + + for (int i = 0; i < n; i++) { + std::fill(next_dp.begin(), next_dp.end(), INF); + for (int u = 0; u < B; u++) { + if (dp[u] >= INF) continue; + for (int ci = 0; ci < (int)items[i].choices.size(); ci++) { + const auto & c = items[i].choices[ci]; + int nu = u + c.units; + if (nu >= B) continue; + double nk = dp[u] + c.kld; + if (nk < next_dp[nu]) { + next_dp[nu] = nk; + choice_taken[i][nu] = ci; + prev_u[i][nu] = u; } } } + dp.swap(next_dp); + } - if (!best_role.empty() && best_type != GGML_TYPE_COUNT) { - current[best_role] = best_type; - improved = true; + // Preferred: best terminal state in [budget_lo, budget_hi]. + int best_u = -1; + double best_kld = INF; + for (int u = std::max(0, budget_lo); u <= budget_hi && u < B; u++) { + if (dp[u] < best_kld) { best_kld = dp[u]; best_u = u; } + } + // Fallback 1: lower bound unreachable (e.g. almost everything overshoots + // the floor). Take any u in [0, budget_hi]. + if (best_u < 0) { + for (int u = 0; u <= budget_hi && u < B; u++) { + if (dp[u] < best_kld) { best_kld = dp[u]; best_u = u; } } } - - // Iterative improvement: try swapping roles up/down - std::set>> visited; - auto make_key = [&](const std::map & a) { - std::vector> v(a.begin(), a.end()); - std::sort(v.begin(), v.end()); - return v; - }; - - assignment best_assign; - best_assign.role_to_type = current; - best_assign.total_bpw = compute_total_bpw(current, roles, total_all_elements, non_quantizable_bits); - best_assign.total_kld = compute_total_kld(current, cost_matrix); - visited.insert(make_key(current)); - - for (int iter = 0; iter < cfg.max_iterations; iter++) { - bool found_improvement = false; - - // Try upgrading each role and downgrading another to compensate - for (const auto & [role_up, cur_type_up] : best_assign.role_to_type) { - auto it_up = cost_matrix.find(role_up); - if (it_up == cost_matrix.end()) continue; - - for (auto qt_up : types_by_bpw) { - if (compute_bpw(qt_up) <= compute_bpw(cur_type_up)) continue; - auto it2_up = it_up->second.find(qt_up); - if (it2_up == it_up->second.end() || it2_up->second.kld >= 1e29) continue; - - (void)0; // bpw_increase no longer used directly - double kld_decrease_up = (it_up->second.count(cur_type_up) ? - it_up->second.at(cur_type_up).kld : 1e30) - it2_up->second.kld; - - // Try downgrading each other role to compensate - for (const auto & [role_dn, cur_type_dn] : best_assign.role_to_type) { - if (role_dn == role_up) continue; - auto it_dn = cost_matrix.find(role_dn); - if (it_dn == cost_matrix.end()) continue; - - for (auto qt_dn : types_by_bpw) { - if (compute_bpw(qt_dn) >= compute_bpw(cur_type_dn)) continue; - auto it2_dn = it_dn->second.find(qt_dn); - if (it2_dn == it_dn->second.end() || it2_dn->second.kld >= 1e29) continue; - - // Check BPW constraint - auto test = best_assign.role_to_type; - test[role_up] = qt_up; - test[role_dn] = qt_dn; - double test_bpw = compute_total_bpw(test, roles, total_all_elements, non_quantizable_bits); - if (test_bpw > cfg.target_bpw + cfg.bpw_tol_high) continue; - if (test_bpw < cfg.target_bpw - cfg.bpw_tol_low) continue; - - // Check if already visited - auto key = make_key(test); - if (visited.count(key)) continue; - - // Compute KLD change - double kld_increase_dn = it2_dn->second.kld - - (it_dn->second.count(cur_type_dn) ? it_dn->second.at(cur_type_dn).kld : 1e30); - double net_kld_change = -kld_decrease_up + kld_increase_dn; - - if (net_kld_change < -1e-10) { - // Improvement found! - double new_kld = best_assign.total_kld + net_kld_change; - visited.insert(key); - - best_assign.role_to_type = test; - best_assign.total_bpw = test_bpw; - best_assign.total_kld = new_kld; - found_improvement = true; - break; - } - visited.insert(key); - } - if (found_improvement) break; - } - if (found_improvement) break; - } - if (found_improvement) break; + // Fallback 2: target itself is infeasible (common with small/tied-embedding + // models where token_embd at Q6_K alone exceeds the target). Report best + // overshoot — smallest u with finite dp, tie-break on lower KLD. + if (best_u < 0) { + int min_u = -1; + for (int u = 0; u < B; u++) { + if (dp[u] < INF) { min_u = u; break; } } - - if (!found_improvement) { - // Also try single-role upgrades (if BPW budget allows) - for (const auto & [role, cur_type] : best_assign.role_to_type) { - auto it = cost_matrix.find(role); - if (it == cost_matrix.end()) continue; - - for (auto qt : types_by_bpw) { - if (compute_bpw(qt) <= compute_bpw(cur_type)) continue; - auto it2 = it->second.find(qt); - if (it2 == it->second.end() || it2->second.kld >= 1e29) continue; - - auto test = best_assign.role_to_type; - test[role] = qt; - double test_bpw = compute_total_bpw(test, roles, total_all_elements, non_quantizable_bits); - if (test_bpw > cfg.target_bpw + cfg.bpw_tol_high) continue; - - auto key = make_key(test); - if (visited.count(key)) continue; - - double kld_change = it2->second.kld - - (it->second.count(cur_type) ? it->second.at(cur_type).kld : 1e30); - if (kld_change < -1e-10) { - best_assign.role_to_type = test; - best_assign.total_bpw = test_bpw; - best_assign.total_kld += kld_change; - visited.insert(key); - found_improvement = true; - break; - } - visited.insert(key); - } - if (found_improvement) break; - } + if (min_u >= 0) { + best_u = min_u; + best_kld = dp[min_u]; + LOG(" [warning] target BPW %.4f is infeasible — falling back to lowest achievable at %.4f BPW\n", + cfg.target_bpw, (double)min_u / (double)N_UNITS); } + } + if (best_u < 0) { + LOG_ERR("No feasible DP assignment at all — something is wrong with cost matrix\n"); + return {}; + } - if (!found_improvement) break; - - LOG(" Iteration %d: BPW=%.4f, total_KLD=%.6f\n", - iter, best_assign.total_bpw, best_assign.total_kld); + // Reconstruct the assignment by walking back through the choice tables. + assignment out; + out.total_kld = best_kld; + int cur_u = best_u; + for (int i = n - 1; i >= 0; i--) { + int ci = choice_taken[i][cur_u]; + if (ci < 0) { + LOG_ERR("DP reconstruction failed at item %d (u=%d)\n", i, cur_u); + return {}; + } + out.role_to_type[items[i].key] = items[i].choices[ci].qt; + cur_u = prev_u[i][cur_u]; } + out.total_bpw = compute_total_bpw(out.role_to_type, roles, total_all_elements, non_quantizable_bits); + + LOG("DP optimizer: %d items, budget units=[%d, %d] (step=%.6f bpw), " + "optimum at u=%d (bpw=%.4f, element-weighted error=%.3e)\n", + n, budget_lo, budget_hi, 1.0 / (double)N_UNITS, best_u, + (double)best_u / (double)N_UNITS, best_kld); - return best_assign; + return out; } // ============================================================================ // Section 8: Output // ============================================================================ +// Build a regex chunk like "(0|3|27)" from a sorted layer list. +static std::string layer_alternation(const std::vector & layers) { + std::string s = "("; + for (size_t i = 0; i < layers.size(); i++) { + if (i) s += "|"; + s += std::to_string(layers[i]); + } + s += ")"; + return s; +} + static bool write_tensor_type_file(const std::string & path, const std::map & role_to_type, ggml_type output_tensor_type, - ggml_type token_embd_tensor_type) { + ggml_type token_embd_tensor_type, + const std::map> & bucket_layers) { std::ofstream file(path); if (!file) { LOG_ERR("Failed to open output file: %s\n", path.c_str()); return false; } - // Write token_embd type — anchor with ^ so it only matches the global tensor, - // not e.g. some hypothetical "blk.X.something_token_embd.weight" + // Globals first. Anchored with ^ so they don't collide with layer tensors. if (token_embd_tensor_type != GGML_TYPE_COUNT) { file << "^token_embd=" << ggml_type_name(token_embd_tensor_type) << "\n"; } - - // Write output tensor type — anchor with ^ so "output" matches "output.weight" - // but NOT "blk.X.attn_output.weight" if (output_tensor_type != GGML_TYPE_COUNT) { file << "^output=" << ggml_type_name(output_tensor_type) << "\n"; } - // Write per-role types (skip global tensors already written above). - // Use \.ROLE\. patterns so they match between dots, e.g.: - // "ffn_down" → "\.ffn_down\." matches blk.X.ffn_down.weight - // "attn_q" → "\.attn_q\." matches blk.X.attn_q.weight but NOT blk.X.attn_qkv.weight - // "attn_qkv" → "\.attn_qkv\." matches blk.X.attn_qkv.weight - for (const auto & [role, type] : role_to_type) { + // Per-(role, bucket) entries. For each entry we list the concrete layer + // indices from bucket_layers so the regex matches exactly the layers in + // this bucket: + // blk\.(0|1|2)\.ffn_down\.=Q5_K + // blk\.(3|4|...)\.ffn_down\.=Q4_K + // blk\.(28|29|30|31)\.ffn_down\.=Q6_K + // If bucket_layers has no entry for a key (e.g. single-bucket fallback), + // we emit the role-only pattern as before. + for (const auto & [key, type] : role_to_type) { + std::string role = rb_role(key); if (role == "token_embd" || role == "output") continue; - file << "\\." << role << "\\.=" << ggml_type_name(type) << "\n"; + + auto bit = bucket_layers.find(key); + if (bit == bucket_layers.end() || bit->second.empty()) { + file << "\\." << role << "\\.=" << ggml_type_name(type) << "\n"; + } else { + file << "blk\\." << layer_alternation(bit->second) + << "\\." << role << "\\.=" << ggml_type_name(type) << "\n"; + } } file.close(); @@ -1334,9 +1341,9 @@ static void print_usage(const char * prog) { LOG(" --output-tensor-type T Quant type for output.weight (default: highest from list)\n"); LOG(" --max-iterations N Max optimization iterations (default: 100)\n"); LOG(" --threads N Number of threads (default: 1)\n"); - LOG(" --importance-alpha A Imatrix-energy bias for per-role KLD (default: 1.0)\n"); - LOG(" weight[r] = (geomean_energy / role_energy[r])^A\n"); - LOG(" 0 = no bias, 1 = full inverse-energy reweighting\n"); + LOG(" --layer-buckets N Per-class layer buckets for independent quant assignment\n"); + LOG(" (default: 3 — first/middle/last third).\n"); + LOG(" 1 reproduces the older per-role-only behavior.\n"); } static config parse_args(int argc, char ** argv) { @@ -1387,8 +1394,12 @@ static config parse_args(int argc, char ** argv) { cfg.max_iterations = std::stoi(argv[++i]); } else if (arg == "--threads" && i + 1 < argc) { cfg.n_threads = std::stoi(argv[++i]); - } else if (arg == "--importance-alpha" && i + 1 < argc) { - cfg.importance_alpha = std::stof(argv[++i]); + } else if (arg == "--layer-buckets" && i + 1 < argc) { + cfg.n_layer_buckets = std::stoi(argv[++i]); + if (cfg.n_layer_buckets < 1) { + LOG_ERR("--layer-buckets must be >= 1\n"); + exit(1); + } } else if (arg == "-h" || arg == "--help") { print_usage(argv[0]); exit(0); @@ -1538,6 +1549,46 @@ int main(int argc, char ** argv) { LOG("], roles=[%s]\n", roles_str.c_str()); } + // Compute per-layer bucket (0..n_layer_buckets-1) based on position within + // the layer's equivalence class. Layers outside any class (i.e. no quantizable + // weights) get bucket 0 — they won't be sampled anyway. Globals use -1. + std::map layer_to_bucket; + for (const auto & lc : layer_classes) { + const int n = (int)lc.all_layers.size(); + for (int i = 0; i < n; i++) { + layer_to_bucket[lc.all_layers[i]] = compute_bucket(i, n, cfg.n_layer_buckets); + } + } + // For each (role, bucket), list the concrete layer indices that belong to it + // (used at emission to write layer-alternation regexes). + std::map> bucket_layers; + for (const auto & ti : quantizable) { + if (ti.layer < 0) continue; // globals handled separately + auto bit = layer_to_bucket.find(ti.layer); + if (bit == layer_to_bucket.end()) continue; + std::string key = make_rb_key(ti.role, bit->second); + auto & v = bucket_layers[key]; + if (std::find(v.begin(), v.end(), ti.layer) == v.end()) v.push_back(ti.layer); + } + for (auto & [k, v] : bucket_layers) std::sort(v.begin(), v.end()); + + if (cfg.n_layer_buckets > 1) { + LOG("Layer bucketing (n_buckets=%d):\n", cfg.n_layer_buckets); + for (const auto & lc : layer_classes) { + LOG(" Class %zu buckets:", lc.class_index); + for (int b = 0; b < cfg.n_layer_buckets; b++) { + std::string layers_str; + for (int L : lc.all_layers) { + if (layer_to_bucket[L] != b) continue; + if (!layers_str.empty()) layers_str += ","; + layers_str += std::to_string(L); + } + if (!layers_str.empty()) LOG(" [%d]={%s}", b, layers_str.c_str()); + } + LOG("\n"); + } + } + // Build set of target weight tensor names (for the eval callback) // NOTE: token_embd uses ggml_get_rows (embedding lookup), NOT ggml_mul_mat, // so it cannot be measured via MUL_MAT KLD. It gets the highest-BPW type as preset. @@ -1583,6 +1634,13 @@ int main(int argc, char ** argv) { cap_state.target_weight_names.insert(ti.name); cap_state.weight_to_role[ti.name] = ti.role; cap_state.weight_to_layer[ti.name] = ti.layer; + if (ti.layer < 0) { + cap_state.weight_to_bucket[ti.name] = -1; + } else { + auto bit = layer_to_bucket.find(ti.layer); + cap_state.weight_to_bucket[ti.name] = + (bit != layer_to_bucket.end()) ? bit->second : 0; + } } } @@ -1709,8 +1767,8 @@ int main(int argc, char ** argv) { } LOG("Captured %d MUL_MAT operations\n", cap_state.captured); - for (const auto & [role, captures] : cap_state.captures_by_role) { - LOG(" Role '%s': %zu captures\n", role.c_str(), captures.size()); + for (const auto & [rb, captures] : cap_state.captures_by_role) { + LOG(" %s: %zu captures\n", rb_display(rb).c_str(), captures.size()); } // Free the llama model — we don't need it anymore @@ -1722,6 +1780,7 @@ int main(int argc, char ** argv) { decltype(cap_state.target_weight_names)().swap(cap_state.target_weight_names); decltype(cap_state.weight_to_role)().swap(cap_state.weight_to_role); decltype(cap_state.weight_to_layer)().swap(cap_state.weight_to_layer); + decltype(cap_state.weight_to_bucket)().swap(cap_state.weight_to_bucket); std::vector().swap(cap_state.tmp_data); std::vector>().swap(test_token_sets); std::vector().swap(test_set_infos); @@ -1732,13 +1791,31 @@ int main(int argc, char ** argv) { auto cost_matrix = build_cost_matrix(cfg, quantizable, cap_state.captures_by_role, cfg.model_path, gguf_ctx, imatrix_data); + // Build preliminary role_info map (per-bucket) so split_fused_roles can + // apportion fused error by element count at the same bucket. + std::map roles; + size_t total_quant_elements = 0; + for (const auto & ti : quantizable) { + int bucket = (ti.layer < 0) ? -1 : + (layer_to_bucket.count(ti.layer) ? layer_to_bucket[ti.layer] : 0); + std::string key = make_rb_key(ti.role, bucket); + auto & ri = roles[key]; + ri.role = key; + ri.n_elements += ti.n_elements; + ri.n_bytes_orig += ggml_nbytes(ggml_get_tensor(ggml_ctx, ti.name.c_str())); + total_quant_elements += ti.n_elements; + } + + // Split fused-MUL_MAT captures (e.g. attn_qkv) into their component roles. + split_fused_roles(cost_matrix, roles); + // Print cost matrix - LOG("\nCost matrix (avg KLD by role × quant type):\n"); - LOG("%-16s", "Role"); + LOG("\nCost matrix (avg relative-L2 error by role-bucket × quant type):\n"); + LOG("%-20s", "Role[bucket]"); for (auto qt : cfg.quant_types) LOG(" %10s", ggml_type_name(qt)); LOG("\n"); - for (const auto & [role, costs] : cost_matrix) { - LOG("%-16s", role.c_str()); + for (const auto & [key, costs] : cost_matrix) { + LOG("%-20s", rb_display(key).c_str()); for (auto qt : cfg.quant_types) { auto it = costs.find(qt); if (it != costs.end()) { @@ -1781,32 +1858,23 @@ int main(int argc, char ** argv) { } } - // Add global tensors to the cost matrix with their fixed types. + // Add global tensors (bucket -1) to the cost matrix with their fixed types. // KLD=0 since they're not optimized — we just need their BPW in the budget. + const std::string key_tok_embd = make_rb_key("token_embd", -1); + const std::string key_output = make_rb_key("output", -1); for (ggml_type qtype : cfg.quant_types) { cost_entry e; e.kld = (qtype == token_embd_type) ? 0.0 : 1e30; e.bpw = compute_bpw(qtype); - cost_matrix["token_embd"][qtype] = e; + cost_matrix[key_tok_embd][qtype] = e; e.kld = (qtype == output_type) ? 0.0 : 1e30; - cost_matrix["output"][qtype] = e; + cost_matrix[key_output][qtype] = e; } LOG("Global tensors: token_embd=%s (%.4f bpw), output=%s (%.4f bpw)\n", ggml_type_name(token_embd_type), compute_bpw(token_embd_type), ggml_type_name(output_type), compute_bpw(output_type)); - // Build role info (n_elements, n_bytes per role) — include ALL quantizable tensors - std::map roles; - size_t total_quant_elements = 0; - for (const auto & ti : quantizable) { - auto & ri = roles[ti.role]; - ri.role = ti.role; - ri.n_elements += ti.n_elements; - ri.n_bytes_orig += ggml_nbytes(ggml_get_tensor(ggml_ctx, ti.name.c_str())); - total_quant_elements += ti.n_elements; - } - // Small 2D .weight tensors below min_elements aren't measured for KLD, but // llama-quantize will still quantize them (to the fallback ftype passed on // the CLI). We don't know the user's chosen fallback, so we approximate it @@ -1840,106 +1908,46 @@ int main(int argc, char ** argv) { LOG("Small matrix weights below --min-elements (%zu elements) assumed %s for BPW estimate\n", small_matrix_elements, ggml_type_name(small_matrix_type)); } - for (const auto & [role, ri] : roles) { - LOG(" %s: %zu elements%s\n", role.c_str(), ri.n_elements, - cost_matrix.count(role) ? "" : " [NO CAPTURES]"); + for (const auto & [key, ri] : roles) { + LOG(" %s: %zu elements%s\n", rb_display(key).c_str(), ri.n_elements, + cost_matrix.count(key) ? "" : " [NO CAPTURES]"); } - // Warn about any quantizable roles that still have no captures. - // With proper layer equivalence class sampling, this should be rare, - // but can happen for tensors that don't appear as MUL_MAT in any layer type. + // Warn about any quantizable (role, bucket) pairs that still have no captures. + // With proper layer equivalence class sampling this should be rare, but can + // happen for tensors that don't appear as MUL_MAT in any layer type. { - for (const auto & [role, ri] : roles) { - if (cost_matrix.count(role)) continue; - LOG(" WARNING: Role '%s' has no captures — assigning KLD=0 for all types\n", - role.c_str()); + for (const auto & [key, ri] : roles) { + if (cost_matrix.count(key)) continue; + LOG(" WARNING: %s has no captures — assigning error=0 for all types\n", + rb_display(key).c_str()); for (ggml_type qtype : cfg.quant_types) { cost_entry e; e.kld = 0.0; e.bpw = compute_bpw(qtype); - cost_matrix[role][qtype] = e; + cost_matrix[key][qtype] = e; } } } - // Imatrix-energy bias correction. Local MUL_MAT KLD systematically under-weights - // residual-path tensors with small-magnitude inputs (ffn_down, attn_output, ssm_out) - // because softmax over a small-output row is flatter. We compensate by scaling each - // role's KLD by (geomean_energy / role_energy)^alpha, computed from the imatrix. - std::map role_weight; - if (cfg.importance_alpha > 0.0f && !imatrix_data.empty()) { - std::map> role_mean_by_tensor; - for (const auto & [tname, imat] : imatrix_data) { - if (imat.empty()) continue; - double s = 0; - for (float v : imat) s += v; - role_mean_by_tensor[extract_role(tname)].push_back(s / imat.size()); - } - - std::map role_energy; - double log_sum = 0; - int n_energies = 0; - for (const auto & [role, vs] : role_mean_by_tensor) { - double sum = 0; - for (double v : vs) sum += v; - double mean = vs.empty() ? 0.0 : sum / vs.size(); - role_energy[role] = mean; - if (mean > 1e-20) { - log_sum += std::log(mean); - n_energies++; - } - } - - const double geo_mean = n_energies > 0 ? std::exp(log_sum / n_energies) : 1.0; - for (const auto & [role, e] : role_energy) { - role_weight[role] = e > 1e-20 - ? std::pow(geo_mean / e, (double)cfg.importance_alpha) - : 1.0; - } - - LOG("\nRole importance weights (alpha=%.2f, geomean_energy=%.4g):\n", cfg.importance_alpha, geo_mean); - for (const auto & [role, w] : role_weight) { - LOG(" %-16s energy=%10.4g weight=%6.3f\n", role.c_str(), role_energy[role], w); - } - } - - // Apply weights to a cost-matrix copy used by the optimizer; preserve raw values - // for display and for the final KLD we report to the user. - auto weighted_cost = cost_matrix; - for (auto & [role, row] : weighted_cost) { - auto wit = role_weight.find(role); - if (wit == role_weight.end()) continue; - double w = wit->second; - for (auto & [qt, ce] : row) { - if (ce.kld < 1e29) ce.kld *= w; // keep sentinels intact - } - } - - auto result = optimize_assignment(cfg, weighted_cost, roles, total_all_elements, non_quantizable_bits); + auto result = optimize_assignment(cfg, cost_matrix, roles, total_all_elements, non_quantizable_bits); LOG("\nOptimal assignment:\n"); - for (const auto & [role, type] : result.role_to_type) { - auto it = cost_matrix.find(role); + for (const auto & [key, type] : result.role_to_type) { + auto it = cost_matrix.find(key); double kld = (it != cost_matrix.end() && it->second.count(type)) ? it->second.at(type).kld : -1; - LOG(" %-16s = %-10s (KLD=%.6f, BPW=%.4f)\n", - role.c_str(), ggml_type_name(type), kld, compute_bpw(type)); + LOG(" %-20s = %-10s (error=%.6f, BPW=%.4f)\n", + rb_display(key).c_str(), ggml_type_name(type), kld, compute_bpw(type)); } LOG("Total BPW: %.4f (target: %.2f +%.2f/-%.2f)\n", result.total_bpw, cfg.target_bpw, cfg.bpw_tol_high, cfg.bpw_tol_low); - // Raw total KLD is the unweighted sum from cost_matrix; the optimizer's - // result.total_kld is in weighted units when importance_alpha > 0. - double raw_total_kld = compute_total_kld(result.role_to_type, cost_matrix); - if (cfg.importance_alpha > 0.0f && !role_weight.empty()) { - LOG("Total KLD: %.6f raw (%.6f weighted, used by optimizer)\n", - raw_total_kld, result.total_kld); - } else { - LOG("Total KLD: %.6f\n", raw_total_kld); - } + LOG("Total error (unweighted sum): %.6f\n", + compute_total_kld(result.role_to_type, cost_matrix)); // ---- Phase 5: Output tensor-type-file ---- LOG("\n--- Phase 5: Writing output ---\n"); write_tensor_type_file(cfg.output_path, result.role_to_type, - output_type, token_embd_type); + output_type, token_embd_type, bucket_layers); // Cleanup gguf_free(gguf_ctx); From 2f0def137ed1611b68aed47b4b430a2dc4750045 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Sat, 2 May 2026 17:49:44 +0200 Subject: [PATCH 09/19] Update working auto-tensor-type --- docs/auto-tensor-type.md | 376 ++++++++++----- ggml/src/ggml-cpu/arch/x86/quants.c | 150 +----- src/llama-quant.cpp | 5 + tools/auto-tensor-type/CMakeLists.txt | 2 +- tools/auto-tensor-type/auto-tensor-type.cpp | 489 ++++++++++++++++++-- tools/capture-layer-data/CMakeLists.txt | 2 +- 6 files changed, 716 insertions(+), 308 deletions(-) diff --git a/docs/auto-tensor-type.md b/docs/auto-tensor-type.md index 678eaa173717..2835711e7be0 100644 --- a/docs/auto-tensor-type.md +++ b/docs/auto-tensor-type.md @@ -13,6 +13,44 @@ types. Hand-crafted recipes (e.g. "Q6_K for attention output, IQ3_TQ for FFN gat embed implicit judgments about which tensors are load-bearing. This tool tries to derive those judgments from measurements and produce a numerically-justified recipe. +## Quick start + +``` +build/bin/llama-auto-tensor-type \ + -m /path/to/model.gguf \ + -i /path/to/model.imatrix.gguf \ + -q IQ3_TQ,Q4_DPT,IQ4_XS,Q4_K,Q5_K,Q6_K \ + -b 4.25 \ + -o /tmp/recipe.types \ + --test-data /path/to/calibration.txt \ + --test-sizes 128,512 \ + --threads 32 \ + --cost-matrix-cache /tmp/model.cm.cache +``` + +Then quantize and validate: + +``` +build/bin/llama-quantize \ + --imatrix /path/to/model.imatrix.gguf \ + --tensor-type-file /tmp/recipe.types \ + /path/to/model.gguf \ + /tmp/quantized.gguf \ + IQ4_XS 32 + +build/bin/llama-perplexity \ + -m /tmp/quantized.gguf \ + --kl-divergence-base /path/to/reference.kld \ + --kl-divergence \ + -ngl 99 -t 32 -fa 1 -c 512 -b 512 -ub 512 \ + -f /path/to/calibration.txt +``` + +The `IQ4_XS 32` positional args at the end of `llama-quantize` are the **fallback +ftype** and thread count. Tensors not matched by the recipe (small 2D weights below +`--min-elements`) get the fallback ftype. Pick a fallback whose BPW is close to +your target so the unmeasured tensors don't push BPW off-budget. + ## Pipeline The tool runs in five phases: @@ -20,154 +58,221 @@ The tool runs in five phases: 1. **Metadata load.** Opens the source GGUF with `no_alloc=true` (we only need tensor names/shapes/offsets; raw weight data is read later on demand). Enumerates tensor roles via regex (`blk.N.ROLE.weight` → `ROLE`; global tensors like `token_embd` - keep their full name). - -2. **Activation capture.** Loads the model via the llama API, runs one forward pass per - `--test-size` over either synthetic tokens or real text (`--test-data`), and - captures the input (`src[1]`) and reference output of every `MUL_MAT` op whose - `src[0]` weight is a quantization target. Captures are organized per role. - -3. **Cost matrix.** For each (role, candidate type) pair, re-quantizes each captured - weight to the candidate type, replays the captured input through `MUL_MAT`, and - measures the KL divergence of the output against the reference. See - [KLD measurement](#kld-measurement) below. Results are averaged over captures - within the role. - -4. **Optimization.** Greedily upgrades role→type assignments to minimize (possibly - energy-weighted) total KLD subject to the BPW budget. See - [Optimizer](#optimizer). + keep their full name). Computes layer equivalence classes and per-(class, bucket) + layer assignments. + +2. **Activation capture.** Loads the model via the llama API, runs one forward pass + per `--test-size` × `--test-samples` over either real text (`--test-data`, + recommended) or synthetic tokens, and captures the input (`src[1]`) and reference + output of every `MUL_MAT` op whose `src[0]` weight is a quantization target. + Captures are organized per (role, bucket). + +3. **Cost matrix.** For each (role, bucket, candidate type) cell, re-quantizes each + captured weight to the candidate type, replays the captured input through + `MUL_MAT`, and measures the relative L2 error of the output against the reference + (see [Cost metric](#cost-metric)). Results are averaged over captures within the + cell. **This is the slow phase** — typically 30-50 minutes for a 9B model with + 7 candidate types — and is the target of the [cost-matrix cache](#cost-matrix-cache). + +4. **Optimization.** Exact multi-choice knapsack DP over per-(role, bucket) items. + Each item picks one candidate type; the objective is the element-weighted sum of + per-cell relative-L2 errors, subject to the BPW budget (`target - tol_low ≤ BPW ≤ + target + tol_high`). See [Optimizer](#optimizer). 5. **Emit recipe.** Writes a `tensor-type-file` with anchored regexes: - `\.ROLE\.=TYPE` for layer tensors, `^ROLE=TYPE` for globals. + `blk\.()\.ROLE\.=TYPE` for layer tensors, `^ROLE=TYPE` for globals. -## KLD measurement +## Cost metric -For each captured MUL_MAT output (a `[ne_out, n_tokens]` F32 matrix), per-row KLD is -computed as +For each captured MUL_MAT output (a `[ne_out, n_tokens]` F32 matrix), per-row error +is computed as ``` -p = softmax(ref_row), q = softmax(quant_row) -KLD(p || q) = sum_i p_i * log(p_i / q_i) +err(ref_row, quant_row) = ||ref - quant||² / ||ref||² ``` -and averaged over rows and captures. This is the signal the optimizer minimizes. +(scale-free relative squared L2 per row), then averaged over rows and captures. This +is the per-cell value the optimizer minimizes. + +### Why relative L2 and not softmax-KLD + +Softmax over an intermediate MUL_MAT output is not a meaningful probability +distribution by itself, and its peakedness depends on the row's absolute magnitude. +Rows with large magnitudes (typical of attention Q/K/V/QKV projections, which read +from the post-norm residual stream) produce near one-hot softmaxes and dramatic +KLDs; rows with small magnitudes (post-gate FFN inputs, post-attention values) +produce flat softmaxes and tiny KLDs. The signal is thus **inversely correlated** +with the downstream cost of quantizing the tensor — softmax-KLD penalises exactly +the wrong tensors. + +Relative L2 is scale-free per-row: it measures fractional output perturbation +directly, which is a better proxy for how the error propagates through the residual +stream. The tool used softmax-KLD historically (with an `--importance-alpha` hack +to compensate for the inverse-correlation); the relative-L2 metric obviates the +hack for typical mid-BPW targets, but [importance-alpha](#importance-alpha-bias-correction) +is still available as an opt-in tuning knob. ### Caveats -- Softmax over an intermediate MUL_MAT output is not a meaningful probability - distribution by itself. Rows with large absolute magnitudes (typical of - attention Q/K/QKV projections, which read from the post-norm residual stream) - produce visibly peaked softmaxes and dramatic-looking KLDs. Rows with small - magnitudes (post-gate FFN inputs, post-attention values) produce flatter - softmaxes and small KLDs. **This systematically under-weights residual-path - tensors like `ffn_down` and `attn_output`**, which often carry more downstream - importance than their local KLD suggests. The `--importance-alpha` correction - (below) compensates for this. -- Local per-tensor KLD is summed as if tensors were independent, but errors - compound non-linearly through residuals and norms. No attempt is made to model - that propagation directly. -- Captures use the model's own (unquantized) activations. For trained-type - quantizations (IQ2_TQ, IQ3_TQ, Q4_DPT, ...), the measured KLD uses the levels - trained on the same data the imatrix was built from, which is a mildly - optimistic estimate of end-use behavior. - -## Importance-alpha bias correction - -To counter the local-KLD bias above, each role's KLD is optionally multiplied by a -weight derived from its imatrix energy: +- **Local metric ≠ global impact.** Per-cell relative L2 measures the fractional + perturbation the tensor introduces *into its own output*. It does not model how + that perturbation amplifies through downstream nonlinearities. Pre-softmax + tensors (`attn_q`, `attn_k`, `attn_qkv`) are particularly under-weighted: a small + pre-softmax error becomes a large attention-weight shift after exp + normalize. + This shows up at low BPW targets (≤ ~3.75), where the DP cheerfully pushes + IQ2_TQ onto these roles based on local metric alone. +- **Errors compound across layers, not summed.** The DP minimises a sum-of-cells + objective. Real model output KLD compounds through the residual stream and + norms; the sum is a useful proxy but not the true loss. +- **Trained-quant captures use the model's own (unquantized) activations.** For + trained types (IQ2_TQ, IQ3_TQ, Q4_DPT, Q3_PT, Q3_KPT, Q2_KPT), the measured + error uses levels trained on the same data the imatrix was built from, which + is a mildly optimistic estimate of end-use behavior. + +## Layer equivalence classes and bucketing -``` -role_energy[r] = mean over tensors in r of mean(imatrix[tensor]) -geomean = geometric mean of role_energy[r] across roles with data -role_weight[r] = (geomean / role_energy[r]) ^ alpha -weighted_kld[r] = local_kld[r] * role_weight[r] -``` +Hybrid models (e.g. Qwen3.5's SSM + attention mix) have different tensor signatures +per layer. To avoid measuring every layer: -`alpha = 0` disables the correction; `alpha = 1` is a full inverse-energy flip. -The weights are displayed in the run log. +1. Compute per-layer signature `sorted([(role, ne0, ne1)])` over all quantizable + weights in the layer. +2. Group layers by signature into equivalence classes. +3. Within each class, divide layers into `--layer-buckets` (default 3) — first + third, middle third, last third by position. Different buckets of the same + role can get different quant types (matches hand-tuned recipes that protect + first/last layers). +4. Sample `--reps-per-bucket` (default 3) layers per (class, bucket) for activation + capture, evenly stratified. The first and last layer of each class are *always* + pinned regardless of `K` — boundary layers are most sensitive (cf. + `llama-quant.cpp:use_more_bits`). + +Per-(role, bucket) cost-matrix entries are averaged across all sampled captures +for that cell. Larger `--reps-per-bucket` reduces noise at the cost of more +Phase-3 quantize+eval work. -### Why imatrix energy? +## Optimizer -The imatrix records per-column input activation energy. Residual-path inputs -(post-gate FFN, post-attention) have low activation energy because the non-linearity -or attention weighting compresses the signal into a subspace. Those same tensors -write back into the residual stream, where errors accumulate. Inverse-energy -weighting promotes exactly those tensors. Empirically (see -[Calibration](#calibration)) this aligns with hand-tuned recipes. +Exact dynamic-programming multi-choice knapsack over per-(role, bucket) items: + +- One item per (role, bucket) cell with a positive element count. Globals + (`token_embd`, `output`) are pinned before the DP runs (see [Globals](#globals)) + and enter as items with one valid choice and forbidden alternatives. +- One choice per candidate `qtype` in `--quants`. Each choice has: + - bits = `n_elements * compute_bpw(qtype)` + - cost = `n_elements * relative_L2_avg(role, bucket, qtype)` (element-weighted) +- The DP maximises -cost (minimises element-weighted error) subject to total + bits ∈ [`(target - tol_low) * total_elements`, `(target + tol_high) * total_elements`], + with overhead from non-quantizable tensors (norms, biases, small 2D matrices) + added to the budget floor. +- Budget axis discretised at 16384 units (~5×10⁻⁵ BPW step). Table size is + `O(n_items × budget_units)` — fast even for 36-cell hybrid models. + +If the lower budget bound is unreachable (rare; happens with tied embeddings +forcing `token_embd` to a high BPW), the DP falls back to the lowest-cost +assignment within the upper bound. If the target itself is infeasible (e.g. +embedding alone exceeds it), the tool reports the smallest achievable BPW. + +### Globals + +`token_embd` and `output` are not measured for KLD — `token_embd` uses +`ggml_get_rows` (not MUL_MAT, so no signal), and `output` is force-pinned regardless +of measurement, so capturing it would waste Phase-3 work on a typically-huge tensor. + +- `output` → `--output-tensor-type` if set, else `default_output_type_for_target()`: + the lowest candidate with `bpw ≥ max(target_bpw + 1.0, 5.0)`, falling back to the + highest candidate. The 5.0-bpw floor reflects the empirical sweet spot for output + on Qwen3.5-9B (Q5_K beats Q6_K by 20% Mean KLD at 4.25 BPW target with the same + total budget) and matches hand-tuned recipes (bartowski/unsloth use Q5_K for + output across IQ4_XS, Q4_K_S, Q3_K_M). +- `token_embd` → if tied embeddings detected (no distinct `output.weight` in the + model), force-matches `output_type` because `llama-quantize` will do the same + silently regardless of what we ask. Otherwise → closest-BPW `get_rows`-compatible + candidate to target. -### Calibration +## Cost-matrix cache -Measured on `Qwen3.5-0.8B-BF16` and `Nanbeige4.1-3B-BF16` at target BPW 4.5, -quant-list `IQ3_TQ,Q4_DPT,IQ4_XS,Q4_K,Q5_K,Q6_K`, real-text test data: +Phase 3 is the wall-time bottleneck (30-50 min on a 9B model with 7 candidate types, +of which 2-3 are CPU-bound trained types). For BPW sweeps on the same model the +cost matrix doesn't change between runs — only the DP target shifts. -| α | Qwen3.5-0.8B PPL | Nanbeige-3B PPL | -|---|---|---| -| 0.0 | 54.2149 | 45.4793 | -| 0.5 | 54.2657 | 46.7783 (worse than baseline) | -| **1.0** | **53.2286** | **44.8957** | -| 1.5 | 52.1232 | (same recipe as 1.0) | -| 2.0 | 52.4972 (starts regressing) | (same recipe as 1.0) | +Pass `--cost-matrix-cache PATH` to read+write a text-format cache: -`alpha=1.0` is the default. It's the first setting where the correction reliably -flips allocations on residual-path tensors (promotes `ffn_down`, `ssm_out`, -`attn_output`) on both tested architectures. Lower values (`0.5`) are marginal and -can regress on some architectures. Higher values (`1.5+`) sometimes win on dense -architectures but over-correct on sparser ones. +``` +build/bin/llama-auto-tensor-type ... -b 4.5 --cost-matrix-cache /tmp/m.cm.cache +build/bin/llama-auto-tensor-type ... -b 4.25 --cost-matrix-cache /tmp/m.cm.cache # cache hit, ~0.2 s +build/bin/llama-auto-tensor-type ... -b 4.0 --cost-matrix-cache /tmp/m.cm.cache # cache hit, ~0.2 s +``` -## Optimizer +Cache invalidation is exact: the header records every parameter that affects what +the cost matrix would contain, and **any** mismatch invalidates the cache (the tool +runs Phase 2+3 normally and overwrites the file). Validated fields: -Greedy, two-stage, on a (role × type) cost matrix: +- `model_path`, `model_size`, `model_mtime` +- `quant_types` (sorted) +- `min_elements` +- `n_layer_buckets`, `n_reps_per_bucket` +- `test_data`, `test_data_size` +- `test_sizes`, `n_test_samples` -1. **Seed.** Every role starts at the lowest-BPW type in the candidate list that has - a finite KLD. -2. **Greedy upgrade.** Repeatedly pick the single role→type upgrade with the highest - `Δkld / Δbpw` ratio that still fits the BPW budget (`target + bpw_tol_high`), - until no improving move exists. -3. **Swap improvement.** Up to `--max-iterations` rounds, try upgrading one role - while downgrading another to stay in budget and reduce total weighted KLD. - Terminates on convergence. +`--target-bpw`, `--bpw-tolerance`, `--output-tensor-type`, `--importance-alpha`, +and `--max-iterations` are *not* part of the cache key — these only affect +Phase 4, which always re-runs (it's near-instant). That's the whole point of the +cache: tune them freely. -BPW accounting uses `ggml_get_bpw` (block size + per-tensor-trained overhead). -Global tensors (`token_embd`, `output`) are pinned before optimization begins: -- `output` → `--output-tensor-type`, or highest-BPW candidate if unset. -- `token_embd` → if tied embeddings detected (no distinct `output.weight` in the - model), force-matches `output_type` because `llama-quantize` will do the same - upgrade silently regardless of what we ask for. Otherwise → closest-BPW - `get_rows`-compatible candidate to target. +The cache is a plain text file (one entry per `(role, bucket, qtype)` line); safe +to inspect, diff, or delete. -## Layer equivalence classes +## Importance-alpha bias correction (optional) -Hybrid models (e.g. Qwen3.5's SSM + attention mix) have different tensor signatures -per layer. To avoid measuring every layer: +Each role's per-cell error can be multiplied by an imatrix-energy weight: -1. Compute per-layer signature `sorted([(role, ne0, ne1)])`. -2. Group layers by signature into equivalence classes. -3. Sample representative layers per class (first, middle, last; deduped). +``` +role_energy[r] = mean over tensors in r of mean(imatrix[tensor]) +geomean = geometric mean of role_energy[r] across roles with data +role_weight[r] = (geomean / role_energy[r]) ^ alpha +weighted_err[r] = local_err[r] * role_weight[r] +``` -Only representative layers are captured in Phase 2. Per-role KLDs are averaged across -all representatives within their class. +`--importance-alpha 0` (default) disables the correction; `1` is full inverse-energy. +With the relative-L2 metric the correction is much less critical than under +softmax-KLD (where it was on by default), but it still has empirical use when the +target BPW is mid-range and the imatrix is high-quality. Setting α = 1 promotes +low-energy residual-write tensors (`ffn_down`, `attn_output`, `ssm_out`) at the +expense of high-energy pre-softmax tensors. The weights are displayed in the run +log. ## Fusion map When a model fuses projections (e.g. `attn_qkv` = Q‖K‖V as one MUL_MAT), the capture -belongs to the fused role. A `fusion_map` in the source splits the measured KLD +belongs to the fused role. A `fusion_map` in the source splits the measured error proportionally (by element count) to the component roles that lack their own captures. Current entries: - `attn_qkv` → `{attn_q, attn_k, attn_v}` -Extend as needed per arch. +Extend in source as needed per arch. ## Implementation notes - **Per-(weight, qtype) quant caching.** Same weight captured at multiple test sizes is quantized once per type; the cached `quant_result` is reused by every - capture's `eval_mul_mat`. This is ~3× faster than the naive per-capture approach - and also avoids a latent CUDA race on per-tensor grid/level device symbols when - multiple captures of the same (weight, qtype) would otherwise run concurrently. + capture's `eval_mul_mat`. This avoids a latent CUDA race on per-tensor grid/level + device symbols when multiple captures of the same (weight, qtype) would + otherwise run concurrently. - **Parallelism.** `--threads N` launches N `std::async` workers per (weight, - capture) batch. The CUDA backend pool pre-creates one backend per thread - because the VMM pool requires strict LIFO alloc/free per backend. + capture) batch. The CUDA backend pool pre-creates one backend per thread because + the VMM pool requires strict LIFO alloc/free per backend. Effective quantize + parallelism is bounded by `n_quant_types` (the outer weight loop is sequential); + eval parallelism scales with `n_threads`. +- **Output skip.** `output.weight` is excluded from Phase-2 capture because it's + force-pinned in Phase 4 and the per-tensor KLD measurement would be discarded + — significant savings on models with large vocab × hidden output projections. +- **llama-quantize override fix.** `llama-quant.cpp` historically treated + `--tensor-type-file` overrides equal to the ftype default as "no override" and + then ran the auto-upgrade rules (e.g. n_gqa ≥ 4 promotes `attn_v` IQ4_XS → Q5_K). + The fix: regex match counts as `manual = true` regardless of whether the type + literally changes, so user intent is honoured. Without this, recipes that + happen to coincide with the ftype default get silently re-mapped. - **Memory.** After Phase 2 finishes, the llama model is reset and the capture scaffolding (`target_weight_names`, `weight_to_role`, tokenized test inputs) is freed. During Phase 3 each role's capture buffers are released as soon as the @@ -178,36 +283,67 @@ Extend as needed per arch. ``` -m, --model PATH Input GGUF model (required) --i, --imatrix PATH Importance matrix (required, also used for bias correction) +-i, --imatrix PATH Importance matrix (required) -q, --quants LIST Comma-separated candidate quant types (required) -b, --target-bpw N Target bits per weight (required) -o, --output PATH Output tensor-type-file (required) --bpw-tolerance H,L BPW tolerance: +H, -L from target (default: +0,-0.2) --test-data PATH Text file for test inputs (recommended; synthetic otherwise) +--test-samples N Samples per test size (default: 1) --test-sizes S1,S2,S3 Token counts for test inputs (default: 32,128,512) --min-elements N Skip KLD measurement for tensors below this size (default: 40000) ---output-tensor-type T Quant type for output.weight (default: highest from list) ---max-iterations N Max swap-improvement iterations (default: 100) +--output-tensor-type T Quant type for output.weight (default: lowest candidate with + bpw >= max(target_bpw + 1.0, 5.0), else highest) +--max-iterations N Reserved (the DP is exact, not iterative; default: 100) --threads N Parallel workers (default: 1) ---importance-alpha A Imatrix-energy bias exponent (default: 1.0, range 0..2) +--layer-buckets N Per-class layer buckets for independent quant assignment + (default: 3 — first/middle/last third). 1 reproduces + the older per-role-only behavior. +--reps-per-bucket K Layers sampled per (class, bucket) for activation capture + (default: 3). Boundary layers always sampled additionally. +--importance-alpha A Imatrix-energy bias exponent (default: 0; opt-in). +--cost-matrix-cache PATH Read/write Phase-3 cost matrix (skip Phase 2+3 on hit). ``` +## Empirical results + +Measured 2026-05-01 on `Qwen3.5-9B-BF16.gguf` with imatrix calibrated on +`calibration_data_v5_rc.txt`, KLD basis from the unquantized BF16 model on +`eval_dataset_260412-0326.txt` (103 chunks). Quants list: +`IQ2_TQ,IQ3_TQ,Q4_DPT,IQ4_XS,Q4_K,Q5_K,Q6_K`. Settings: `--test-sizes 128,512 +--threads 32` (defaults otherwise). + +| Target BPW | Mean KLD | Median | 95% | 99% | Max | Notes | +|---|---|---|---|---|---|---| +| 4.5 | **0.0275** | 0.0172 | 0.0798 | 0.1862 | 11.96 | output Q5_K | +| 4.25 | **0.0366** | 0.0223 | 0.1088 | 0.2768 | 2.57 | output Q5_K | +| 4.0 | 0.0608 | 0.0363 | 0.1857 | 0.4318 | 9.48 | IQ2_TQ on 6 cells | +| 3.75 | 0.0830 | 0.0473 | 0.2550 | 0.6368 | 13.35 | IQ2_TQ on 8 cells | +| 3.5 | 0.1243 | 0.0720 | 0.3932 | 0.9375 | 9.62 | IQ2_TQ on 16 cells | + +The 4.5 and 4.25 results clear the user-stated targets (< 0.03 and < 0.04 +respectively). Quality degrades super-linearly below 4.0 BPW because the DP +exhausts low-error options and pushes IQ2_TQ onto pre-softmax tensors +(attn_q, attn_qkv) — exactly the case where the local relative-L2 metric most +under-predicts global impact (see [Caveats](#caveats)). + ## Known limitations - **BPW estimate ignores GGUF metadata.** Tokenizer vocab, model config, and alignment padding contribute ~2% to the file size. The tool's predicted BPW is tensor-only. Expect final file size ≈ predicted size × 1.02. -- **Tied embeddings force `token_embd` up.** As noted above, the tool now detects - the tied case and reports the promoted BPW correctly. On such models, raising - `target_bpw` or overriding `--output-tensor-type` with a cheaper type is the only - way to free budget for other roles. +- **Tied embeddings force `token_embd` up.** As noted in [Globals](#globals), the + tool detects the tied case and reports the promoted BPW correctly. On such + models, raising `target_bpw` or overriding `--output-tensor-type` with a cheaper + type is the only way to free budget for other roles. - **Small matrix weights are best-effort.** 2D `.weight` tensors below - `--min-elements` aren't measured for KLD; they're assumed to be quantized to the - candidate type closest to the target BPW for accounting purposes. On most archs - this is within ~1 MB of reality but can drift for unusual tensor shapes. + `--min-elements` aren't measured for KLD; they're assumed to be quantized to + the candidate type closest to the target BPW for accounting purposes. On most + archs this is within ~1 MB of reality but can drift for unusual tensor shapes. - **Synthetic tokens are worse than real text.** Default is `rand() % n_vocab`, which has different activation statistics from real inputs. Always pass `--test-data` when quality matters. -- **Local KLD is a proxy.** See the caveats above. For critical quantizations, - verify with `llama-perplexity` on held-out text and compare against a known-good - reference. +- **Local error is a proxy.** See the caveats above — pre-softmax tensors at low + BPW are the most exposed. For critical quantizations, verify with + `llama-perplexity --kl-divergence` on held-out text and compare against a + known-good reference. diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c index 2740abfed479..24d9919c1636 100644 --- a/ggml/src/ggml-cpu/arch/x86/quants.c +++ b/ggml/src/ggml-cpu/arch/x86/quants.c @@ -552,153 +552,7 @@ static inline __m128i get_scale_shuffle(int i) { } #endif -void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { - const int qk = QK1_0; - const int nb = n / qk; - - assert(n % qk == 0); - assert(nrc == 1); - UNUSED(nrc); - UNUSED(bx); - UNUSED(by); - UNUSED(bs); - - const block_q1_0 * GGML_RESTRICT x = vx; - const block_q8_0 * GGML_RESTRICT y = vy; - -#if defined(__AVX2__) - const __m256i ones_8 = _mm256_set1_epi8(1); - const __m256i ones_16 = _mm256_set1_epi16(1); - const __m256i byte_shuf = _mm256_setr_epi8( - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3); - const __m256i bit_masks = _mm256_setr_epi8( - 1, 2, 4, 8, 16, 32, 64, (char) -128, 1, 2, 4, 8, 16, 32, 64, (char) -128, - 1, 2, 4, 8, 16, 32, 64, (char) -128, 1, 2, 4, 8, 16, 32, 64, (char) -128); - const __m256i zero = _mm256_setzero_si256(); - __m256 acc = _mm256_setzero_ps(); - - for (int ib = 0; ib < nb; ++ib) { - const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d); - const uint32_t * GGML_RESTRICT qs32 = (const uint32_t *) x[ib].qs; - const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4]; - - __m256 acc_block; - { - const __m256i qy = _mm256_loadu_si256((const __m256i *) y_ptr[0].qs); - const __m256i sm = _mm256_cmpeq_epi8( - _mm256_and_si256(_mm256_shuffle_epi8(_mm256_set1_epi32((int) qs32[0]), byte_shuf), bit_masks), zero); - const __m256i sy = _mm256_sub_epi8(_mm256_xor_si256(qy, sm), sm); - const __m256i s32 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy), ones_16); - acc_block = _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[0].d)), _mm256_cvtepi32_ps(s32)); - } - for (int K = 1; K < 4; ++K) { - const __m256i qy = _mm256_loadu_si256((const __m256i *) y_ptr[K].qs); - const __m256i sm = _mm256_cmpeq_epi8( - _mm256_and_si256(_mm256_shuffle_epi8(_mm256_set1_epi32((int) qs32[K]), byte_shuf), bit_masks), zero); - const __m256i sy = _mm256_sub_epi8(_mm256_xor_si256(qy, sm), sm); - const __m256i s32 = _mm256_madd_epi16(_mm256_maddubs_epi16(ones_8, sy), ones_16); - acc_block = _mm256_fmadd_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[K].d)), _mm256_cvtepi32_ps(s32), acc_block); - } - acc = _mm256_fmadd_ps(_mm256_set1_ps(d0), acc_block, acc); - } - - *s = hsum_float_8(acc); -#elif defined(__AVX__) - const __m128i ones_8 = _mm_set1_epi8(1); - const __m128i ones_16 = _mm_set1_epi16(1); - const __m128i zero = _mm_setzero_si128(); - __m256 acc = _mm256_setzero_ps(); - - for (int ib = 0; ib < nb; ++ib) { - const float d0 = GGML_CPU_FP16_TO_FP32(x[ib].d); - const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4]; - __m256 acc_block; - { - const __m256i bit_mask = bytes_from_bits_32(&x[ib].qs[0]); - const __m128i bit_mask_0 = _mm256_castsi256_si128(bit_mask); - const __m128i bit_mask_1 = _mm256_extractf128_si256(bit_mask, 1); - const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[0].qs[0]); - const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[0].qs[16]); - const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero); - const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero); - const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0); - const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1); - const __m128i sum16_0 = _mm_maddubs_epi16(ones_8, sy_0); - const __m128i sum16_1 = _mm_maddubs_epi16(ones_8, sy_1); - const __m128i sum32_0 = _mm_madd_epi16(sum16_0, ones_16); - const __m128i sum32_1 = _mm_madd_epi16(sum16_1, ones_16); - const __m256 q = _mm256_cvtepi32_ps(MM256_SET_M128I(sum32_1, sum32_0)); - acc_block = _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[0].d)), q); - } - for(int K = 1; K < 4; ++K) { - const __m256i bit_mask = bytes_from_bits_32(&x[ib].qs[(K) * 4]); - const __m128i bit_mask_0 = _mm256_castsi256_si128(bit_mask); - const __m128i bit_mask_1 = _mm256_extractf128_si256(bit_mask, 1); - const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[(K)].qs[0]); - const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[(K)].qs[16]); - const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero); - const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero); - const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0); - const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1); - const __m128i sum16_0 = _mm_maddubs_epi16(ones_8, sy_0); - const __m128i sum16_1 = _mm_maddubs_epi16(ones_8, sy_1); - const __m128i sum32_0 = _mm_madd_epi16(sum16_0, ones_16); - const __m128i sum32_1 = _mm_madd_epi16(sum16_1, ones_16); - const __m256 q = _mm256_cvtepi32_ps(MM256_SET_M128I(sum32_1, sum32_0)); - acc_block = _mm256_add_ps(acc_block, _mm256_mul_ps(_mm256_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[(K)].d)), q)); - } -#undef Q1_AVX_BLOCK - - acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_set1_ps(d0), acc_block)); - } - - *s = hsum_float_8(acc); -#elif defined(__SSSE3__) - const __m128i ones_8 = _mm_set1_epi8(1); - const __m128i ones_16 = _mm_set1_epi16(1); - const __m128i zero = _mm_setzero_si128(); - __m128 acc_0 = _mm_setzero_ps(); - __m128 acc_1 = _mm_setzero_ps(); - __m128 acc_2 = _mm_setzero_ps(); - __m128 acc_3 = _mm_setzero_ps(); - - for (int ib = 0; ib < nb; ++ib) { - const __m128 d0 = _mm_set1_ps(GGML_CPU_FP16_TO_FP32(x[ib].d)); - const block_q8_0 * GGML_RESTRICT y_ptr = &y[ib * 4]; - -#define Q1_SSSE3_BLOCK(QS_OFF, Y_IDX, ACC) \ - { \ - const __m128i bit_mask_0 = bytes_from_bits_16(&x[ib].qs[(QS_OFF) + 0]); \ - const __m128i bit_mask_1 = bytes_from_bits_16(&x[ib].qs[(QS_OFF) + 2]); \ - const __m128i qy_0 = _mm_loadu_si128((const __m128i *) &y_ptr[(Y_IDX)].qs[0]); \ - const __m128i qy_1 = _mm_loadu_si128((const __m128i *) &y_ptr[(Y_IDX)].qs[16]); \ - const __m128i sign_mask_0 = _mm_cmpeq_epi8(bit_mask_0, zero); \ - const __m128i sign_mask_1 = _mm_cmpeq_epi8(bit_mask_1, zero); \ - const __m128i sy_0 = _mm_sub_epi8(_mm_xor_si128(qy_0, sign_mask_0), sign_mask_0); \ - const __m128i sy_1 = _mm_sub_epi8(_mm_xor_si128(qy_1, sign_mask_1), sign_mask_1); \ - const __m128i sum_0 = _mm_madd_epi16(_mm_maddubs_epi16(ones_8, sy_0), ones_16); \ - const __m128i sum_1 = _mm_madd_epi16(_mm_maddubs_epi16(ones_8, sy_1), ones_16); \ - const __m128 q = _mm_cvtepi32_ps(_mm_add_epi32(sum_0, sum_1)); \ - (ACC) = _mm_add_ps((ACC), _mm_mul_ps(_mm_mul_ps(d0, _mm_set1_ps(GGML_CPU_FP16_TO_FP32(y_ptr[(Y_IDX)].d))), q)); \ - } - Q1_SSSE3_BLOCK(0, 0, acc_0) - Q1_SSSE3_BLOCK(4, 1, acc_1) - Q1_SSSE3_BLOCK(8, 2, acc_2) - Q1_SSSE3_BLOCK(12, 3, acc_3) -#undef Q1_SSSE3_BLOCK - } - - *s = hsum_float_4x4(acc_0, acc_1, acc_2, acc_3); -#else - UNUSED(nb); - UNUSED(x); - UNUSED(y); - ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); -#endif -} - -void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { +void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { GGML_UNUSED(levels); const int qk = QK1_0; const int nb = n / qk; @@ -841,7 +695,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(nb); UNUSED(x); UNUSED(y); - ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 195f984d626b..d07a46f049f8 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -830,6 +830,11 @@ static ggml_type llama_tensor_get_type(quantize_state_impl & qs, const llama_mod const std::string tensor_name(tensor->name); for (const auto & [pattern, qtype] : qs.tensor_type_patterns) { if (std::regex_search(tensor_name, pattern)) { + // The user explicitly specified a type for this tensor; + // honor it and skip the auto-upgrade rules below, even if + // the chosen type happens to equal the ftype default. + // Otherwise rules like the n_gqa>=4 attn_v upgrade would + // silently flip the user's choice. if (qtype != new_type) { LLAMA_LOG_WARN("%s: %-36s - applying manual override: %s -> %s\n", __func__, tensor_name.c_str(), ggml_type_name(new_type), ggml_type_name(qtype)); diff --git a/tools/auto-tensor-type/CMakeLists.txt b/tools/auto-tensor-type/CMakeLists.txt index f36f13834223..2b264bd2ab44 100644 --- a/tools/auto-tensor-type/CMakeLists.txt +++ b/tools/auto-tensor-type/CMakeLists.txt @@ -1,6 +1,6 @@ set(TARGET llama-auto-tensor-type) add_executable(${TARGET} auto-tensor-type.cpp) -target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(${TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT}) target_include_directories(${TARGET} PRIVATE ../../common ../../ggml/include) target_compile_features(${TARGET} PRIVATE cxx_std_17) diff --git a/tools/auto-tensor-type/auto-tensor-type.cpp b/tools/auto-tensor-type/auto-tensor-type.cpp index 61e2cc492ec1..d6b06c7fa283 100644 --- a/tools/auto-tensor-type/auto-tensor-type.cpp +++ b/tools/auto-tensor-type/auto-tensor-type.cpp @@ -20,6 +20,8 @@ #include "llama.h" #include "log.h" +#include + #include #include #include @@ -27,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -84,7 +87,7 @@ struct config { int n_test_samples = 1; // number of samples per test size std::vector test_sizes = {32, 128, 512}; int64_t min_elements = 40000; - ggml_type output_tensor_type = GGML_TYPE_COUNT; // default: highest from list + ggml_type output_tensor_type = GGML_TYPE_COUNT; // default: default_output_type_for_target() int max_iterations = 100; std::string output_path; int n_threads = 1; @@ -93,6 +96,30 @@ struct config { // can get different quant types (matches hand-tuned recipes in llama-quant.cpp). // Set to 1 to reproduce the older per-role-only behavior. int n_layer_buckets = 3; + // Number of representative layers sampled per (class, bucket) for activation + // capture. Larger values reduce per-bucket cost-matrix noise at the cost of + // more Phase-3 quantize+eval work. The first and last layer of each class + // are always sampled regardless of K (boundary layers are most sensitive). + int n_reps_per_bucket = 3; + // Imatrix-energy bias correction. Each role's KLD is multiplied by + // role_weight[r] = (geomean_role_energy / role_energy[r])^alpha + // where role_energy[r] is the mean per-input-channel imatrix value averaged + // across that role's tensors. Promotes residual-write tensors (ffn_down, + // attn_output, ssm_out) whose inputs are post-nonlinearity / post-attention + // and therefore low-energy, but whose errors compound through the residual + // stream. alpha = 0 disables the correction (current default — relative-L2 + // metric already mitigates the worst magnitude bias). Reintroduced on top + // of the relative-L2 metric to capture role-level architectural sensitivity. + float importance_alpha = 0.0f; + // Path to a Phase-3 cost-matrix cache file. If the file exists at startup + // and its header matches the current run's parameters (model size+mtime, + // sorted quant_types, min_elements, n_layer_buckets, n_reps_per_bucket, + // test_data, test_sizes, n_test_samples), Phase 2 + Phase 3 are skipped and + // the cached cost matrix is used directly. On miss/mismatch, the file is + // overwritten after Phase 3 completes. Empty = no cache. + // Useful for sweeping --target-bpw and --output-tensor-type without redoing + // ~45 minutes of trained-quant training each iteration. + std::string cost_matrix_cache; }; struct tensor_info { @@ -224,6 +251,52 @@ static ggml_type highest_bpw(const std::vector & types) { return best; } +// Heuristic for the default output_tensor_type when the user does not pass +// --output-tensor-type. Picks the lowest-BPW candidate at or above +// max(target_bpw + 1.0, 5.0) +// falling back to the highest if none qualifies. +// +// Why not just `highest_bpw`? The output projection is large (vocab × hidden, +// often ~10-20% of model params on modern tokenizers) and force-pinned (we +// can't measure its per-tensor KLD via MUL_MAT — the model output KLD captures +// it end-to-end). Empirically, picking Q6_K when target is 4.25 BPW wastes +// ~0.11 BPW that the DP can spend on architecturally-amplified tensors +// (attn_v, attn_output, ffn residual writes) for a much larger quality win. +// Measured 2026-05-01 on Qwen3.5-9B: switching default from Q6_K to Q5_K cut +// Mean KLD 0.0456 → 0.0366 (-19.6%) and Max KLD 8.78 → 2.57 (-71%) at the +// same total 4.25 BPW. +// +// Why the 5.0-bpw floor? Both winning experiments (4.25 and 4.5 BPW targets) +// landed on Q5_K (5.5 bpw) for output. Hand-tuned recipes (bartowski/unsloth +// IQ4_XS, Q4_K_S, Q3_K_M) consistently use Q5_K for output across a wide +// range of layer-tensor BPW targets — output sensitivity does not scale +// linearly with the rest of the budget. A naive "smallest above target" +// rule would pick Q4_K (4.5) for target 4.25 — untested but plausibly worse +// than Q5_K, since it removes the small-but-meaningful premium that protects +// the head against tail-token errors. Using the floor keeps output at Q5_K +// for low/mid targets and lets it scale up only when target itself goes high. +// +// Resolved choices on the standard {IQ3_TQ, Q4_DPT, IQ4_XS, Q4_K, Q5_K, Q6_K}: +// target 3.0 → Q5_K (5.5), target 4.25 → Q5_K (5.5), +// target 4.5 → Q5_K (5.5), target 5.0 → Q6_K (6.5), +// target 5.5 → Q6_K (6.5), target 6.5 → Q6_K (fallback to highest). +static ggml_type default_output_type_for_target(const std::vector & types, float target_bpw) { + const double min_output_bpw = std::max((double)target_bpw + 1.0, 5.0); + ggml_type best = GGML_TYPE_COUNT; + double best_bpw = 1e30; + for (auto t : types) { + const double bpw = compute_bpw(t); + if (bpw >= min_output_bpw && bpw < best_bpw) { + best = t; + best_bpw = bpw; + } + } + if (best == GGML_TYPE_COUNT) { + best = highest_bpw(types); + } + return best; +} + // Check if a quant type supports get_rows (needed for token_embd) // Per-tensor-trained types (Q3_PT, Q3_KPT, Q4_DPT, Q2_KPT, Q2_DPT) do NOT. static bool supports_get_rows(ggml_type type) { @@ -1032,6 +1105,70 @@ static void split_fused_roles( } } +// Mean per-input-channel imatrix energy for each role. The imatrix records +// per-column sum of x_i^2 / counts ≈ E[x_i^2]; averaging across columns gives +// a per-tensor scalar, then we average across tensors of the same role. +// Tensors not present in the imatrix are skipped (they fall back to weight=1). +static std::map compute_role_energy( + const std::vector & quantizable, + const std::unordered_map> & imatrix_data, + int64_t min_elements) { + std::map> sums; // role -> (sum_per_tensor_mean, n_tensors) + for (const auto & ti : quantizable) { + if (!is_quantizable_weight(ti, min_elements)) continue; + auto it = imatrix_data.find(ti.name); + if (it == imatrix_data.end() || it->second.empty()) continue; + double s = 0; + for (float v : it->second) s += (double)v; + double mean_energy = s / (double)it->second.size(); + if (!std::isfinite(mean_energy) || mean_energy <= 0) continue; + sums[ti.role].first += mean_energy; + sums[ti.role].second += 1; + } + std::map out; + for (auto & [r, p] : sums) { + if (p.second > 0) out[r] = p.first / (double)p.second; + } + return out; +} + +// Apply role_weight = (geomean / role_energy)^alpha to every (role, bucket) cell +// in the cost matrix. The same multiplicative factor is applied to all qtypes for +// a given role-bucket, so it preserves the within-role qtype ordering and only +// changes the BPW-vs-error trade-off the DP optimizer sees across roles. +static void apply_importance_alpha( + std::map> & cost_matrix, + const std::map & role_energy, + float alpha) { + if (alpha == 0.0f || role_energy.empty()) return; + + // Geometric mean over roles that have a positive energy reading. + double sum_log = 0; + int n = 0; + for (const auto & [r, e] : role_energy) { + if (e > 0) { sum_log += std::log(e); n++; } + } + if (n == 0) return; + const double gmean = std::exp(sum_log / (double)n); + + LOG("Applying importance-alpha=%.2f role weights (geomean energy=%.6g):\n", alpha, gmean); + std::map role_weight; + for (const auto & [r, e] : role_energy) { + if (e <= 0) { role_weight[r] = 1.0; continue; } + role_weight[r] = std::pow(gmean / e, (double)alpha); + LOG(" %-12s energy=%.6g weight=%.4f\n", r.c_str(), e, role_weight[r]); + } + + for (auto & [key, costs] : cost_matrix) { + auto wit = role_weight.find(rb_role(key)); + if (wit == role_weight.end()) continue; + const double w = wit->second; + for (auto & [qt, ce] : costs) { + if (ce.kld < 1e29) ce.kld *= w; // skip the forbidden-choice sentinel + } + } +} + struct assignment { std::map role_to_type; double total_kld; @@ -1338,12 +1475,29 @@ static void print_usage(const char * prog) { LOG(" --test-samples N Number of samples per test size (default: 1)\n"); LOG(" --test-sizes S1,S2,S3 Token counts for test inputs (default: 32,128,512)\n"); LOG(" --min-elements N Skip tensors with fewer elements (default: 40000)\n"); - LOG(" --output-tensor-type T Quant type for output.weight (default: highest from list)\n"); + LOG(" --output-tensor-type T Quant type for output.weight (default: lowest candidate\n"); + LOG(" with bpw >= max(target_bpw + 1.0, 5.0), else highest.\n"); + LOG(" Avoids spending top-shelf bits on a force-pinned tensor\n"); + LOG(" whose budget the DP spends better on amplified roles.)\n"); LOG(" --max-iterations N Max optimization iterations (default: 100)\n"); LOG(" --threads N Number of threads (default: 1)\n"); LOG(" --layer-buckets N Per-class layer buckets for independent quant assignment\n"); LOG(" (default: 3 — first/middle/last third).\n"); LOG(" 1 reproduces the older per-role-only behavior.\n"); + LOG(" --reps-per-bucket K Number of representative layers sampled per (class, bucket)\n"); + LOG(" for activation capture (default: 3). Boundary layers\n"); + LOG(" (first/last of each class) are always sampled in\n"); + LOG(" addition. Larger K reduces cost-matrix noise at\n"); + LOG(" the cost of more Phase-3 quantize+eval work.\n"); + LOG(" --importance-alpha A Imatrix-energy bias exponent (default: 0). Multiplies\n"); + LOG(" each role's KLD by (geomean_energy/role_energy)^A,\n"); + LOG(" promoting low-energy residual-write tensors\n"); + LOG(" (ffn_down, attn_output, ssm_out). 0 = disabled,\n"); + LOG(" 1 = full inverse-energy. Useful range 0..1.5.\n"); + LOG(" --cost-matrix-cache PATH Read/write the Phase-3 cost matrix to PATH. On hit,\n"); + LOG(" skips Phase 2+3 entirely (~50min savings) when the\n"); + LOG(" cache header matches model+quants+test-data+sampling.\n"); + LOG(" On miss, runs Phase 3 normally and overwrites PATH.\n"); } static config parse_args(int argc, char ** argv) { @@ -1400,6 +1554,20 @@ static config parse_args(int argc, char ** argv) { LOG_ERR("--layer-buckets must be >= 1\n"); exit(1); } + } else if (arg == "--reps-per-bucket" && i + 1 < argc) { + cfg.n_reps_per_bucket = std::stoi(argv[++i]); + if (cfg.n_reps_per_bucket < 1) { + LOG_ERR("--reps-per-bucket must be >= 1\n"); + exit(1); + } + } else if (arg == "--importance-alpha" && i + 1 < argc) { + cfg.importance_alpha = std::stof(argv[++i]); + if (cfg.importance_alpha < 0.0f) { + LOG_ERR("--importance-alpha must be >= 0\n"); + exit(1); + } + } else if (arg == "--cost-matrix-cache" && i + 1 < argc) { + cfg.cost_matrix_cache = argv[++i]; } else if (arg == "-h" || arg == "--help") { print_usage(argv[0]); exit(0); @@ -1418,12 +1586,176 @@ static config parse_args(int argc, char ** argv) { } if (cfg.output_tensor_type == GGML_TYPE_COUNT) { - cfg.output_tensor_type = highest_bpw(cfg.quant_types); + cfg.output_tensor_type = default_output_type_for_target(cfg.quant_types, cfg.target_bpw); } return cfg; } +// ============================================================================ +// Section 9b: Cost-matrix cache (skip Phase 2+3 across reruns) +// ============================================================================ + +static long long file_size_or_neg(const std::string & path) { + if (path.empty()) return -1; + struct stat st; + return stat(path.c_str(), &st) == 0 ? (long long)st.st_size : -1; +} +static long long file_mtime_or_neg(const std::string & path) { + if (path.empty()) return -1; + struct stat st; + return stat(path.c_str(), &st) == 0 ? (long long)st.st_mtime : -1; +} +static std::string sorted_qtypes_str(const std::vector & types) { + std::vector names; + names.reserve(types.size()); + for (auto t : types) names.emplace_back(ggml_type_name(t)); + std::sort(names.begin(), names.end()); + std::string s; + for (size_t i = 0; i < names.size(); i++) { + if (i) s += ","; + s += names[i]; + } + return s; +} +static std::string ints_str(const std::vector & v) { + std::string s; + for (size_t i = 0; i < v.size(); i++) { + if (i) s += ","; + s += std::to_string(v[i]); + } + return s; +} + +// Trim leading whitespace (used to handle "key value" parsing where value may +// have a leading space after the key). +static std::string ltrim(const std::string & s) { + size_t i = 0; + while (i < s.size() && (s[i] == ' ' || s[i] == '\t')) i++; + return s.substr(i); +} + +// Write the Phase-3 cost matrix to PATH along with a header capturing every +// parameter that affects what the cost matrix would contain. Subsequent runs +// can short-circuit Phase 2 + Phase 3 if their headers match exactly. +static bool save_cost_matrix_cache( + const std::string & path, const config & cfg, + const std::map> & cost_matrix) { + std::ofstream out(path); + if (!out) { + LOG_ERR("Failed to open cost-matrix cache for writing: %s\n", path.c_str()); + return false; + } + out << "# auto-tensor-type cost matrix cache v1\n"; + out << "model_path " << cfg.model_path << "\n"; + out << "model_size " << file_size_or_neg(cfg.model_path) << "\n"; + out << "model_mtime " << file_mtime_or_neg(cfg.model_path) << "\n"; + out << "quant_types " << sorted_qtypes_str(cfg.quant_types) << "\n"; + out << "min_elements " << cfg.min_elements << "\n"; + out << "n_layer_buckets " << cfg.n_layer_buckets << "\n"; + out << "n_reps_per_bucket " << cfg.n_reps_per_bucket << "\n"; + out << "test_data " << cfg.test_data_path << "\n"; + out << "test_data_size " << file_size_or_neg(cfg.test_data_path) << "\n"; + out << "test_sizes " << ints_str(cfg.test_sizes) << "\n"; + out << "n_test_samples " << cfg.n_test_samples << "\n"; + out << "[entries]\n"; + out << std::scientific << std::setprecision(10); + int n = 0; + for (const auto & [key, row] : cost_matrix) { + const std::string role = rb_role(key); + const int bucket = rb_bucket(key); + for (const auto & [qt, e] : row) { + out << role << " " << bucket << " " << ggml_type_name(qt) + << " " << e.kld << " " << e.bpw << "\n"; + n++; + } + } + if (!out) { + LOG_ERR("Failed while writing cost-matrix cache: %s\n", path.c_str()); + return false; + } + LOG("Wrote cost-matrix cache (%d entries) to %s\n", n, path.c_str()); + return true; +} + +// Try to load a cached cost matrix from PATH. Returns true and fills cost_matrix +// only if the file exists AND every header line matches the current cfg. On any +// header mismatch we log the offending field and return false (caller will run +// Phase 2 + Phase 3 normally). Missing-file is silent (not an error). +static bool load_cost_matrix_cache( + const std::string & path, const config & cfg, + std::map> & cost_matrix) { + std::ifstream in(path); + if (!in) return false; + + std::string line; + if (!std::getline(in, line) || line != "# auto-tensor-type cost matrix cache v1") { + LOG_WRN("Cost-matrix cache %s: bad header; ignoring\n", path.c_str()); + return false; + } + + auto expect_kv = [&](const std::string & key, const std::string & expected) -> bool { + if (!std::getline(in, line)) { + LOG_WRN("Cost-matrix cache %s: truncated at '%s'\n", path.c_str(), key.c_str()); + return false; + } + const size_t sp = line.find(' '); + const std::string k = (sp == std::string::npos) ? line : line.substr(0, sp); + const std::string v = (sp == std::string::npos) ? std::string() : ltrim(line.substr(sp + 1)); + if (k != key) { + LOG_WRN("Cost-matrix cache %s: header key mismatch (expected '%s', got '%s')\n", + path.c_str(), key.c_str(), k.c_str()); + return false; + } + if (v != expected) { + LOG_WRN("Cost-matrix cache %s: header mismatch on '%s' (cached='%s', current='%s')\n", + path.c_str(), key.c_str(), v.c_str(), expected.c_str()); + return false; + } + return true; + }; + + if (!expect_kv("model_path", cfg.model_path)) return false; + if (!expect_kv("model_size", std::to_string(file_size_or_neg(cfg.model_path)))) return false; + if (!expect_kv("model_mtime", std::to_string(file_mtime_or_neg(cfg.model_path)))) return false; + if (!expect_kv("quant_types", sorted_qtypes_str(cfg.quant_types))) return false; + if (!expect_kv("min_elements", std::to_string(cfg.min_elements))) return false; + if (!expect_kv("n_layer_buckets", std::to_string(cfg.n_layer_buckets))) return false; + if (!expect_kv("n_reps_per_bucket", std::to_string(cfg.n_reps_per_bucket))) return false; + if (!expect_kv("test_data", cfg.test_data_path)) return false; + if (!expect_kv("test_data_size", std::to_string(file_size_or_neg(cfg.test_data_path)))) return false; + if (!expect_kv("test_sizes", ints_str(cfg.test_sizes))) return false; + if (!expect_kv("n_test_samples", std::to_string(cfg.n_test_samples))) return false; + + if (!std::getline(in, line) || line != "[entries]") { + LOG_WRN("Cost-matrix cache %s: missing [entries] marker\n", path.c_str()); + return false; + } + + int n = 0; + while (std::getline(in, line)) { + if (line.empty()) continue; + std::istringstream ss(line); + std::string role, qt_name; + int bucket; + double kld, bpw; + if (!(ss >> role >> bucket >> qt_name >> kld >> bpw)) { + LOG_WRN("Cost-matrix cache %s: parse failure on line: %s\n", path.c_str(), line.c_str()); + return false; + } + const ggml_type qt = parse_ggml_type_str(qt_name.c_str()); + if (qt == GGML_TYPE_COUNT) { + LOG_WRN("Cost-matrix cache %s: unknown qtype '%s'\n", path.c_str(), qt_name.c_str()); + return false; + } + cost_entry e; e.kld = kld; e.bpw = bpw; + cost_matrix[make_rb_key(role, bucket)][qt] = e; + n++; + } + LOG("Loaded cost-matrix cache (%d entries) from %s\n", n, path.c_str()); + return true; +} + // ============================================================================ // Section 10: Main // ============================================================================ @@ -1439,6 +1771,8 @@ int main(int argc, char ** argv) { LOG("Quant types: "); for (auto t : cfg.quant_types) LOG("%s ", ggml_type_name(t)); LOG("\n"); + LOG("Output tensor type: %s (%.4f bpw)\n", + ggml_type_name(cfg.output_tensor_type), compute_bpw(cfg.output_tensor_type)); LOG("Output: %s\n", cfg.output_path.c_str()); LOG("\n"); @@ -1525,39 +1859,80 @@ int main(int argc, char ** argv) { // and sample representatives from each class. auto layer_classes = get_layer_equivalence_classes(n_layer, all_tensors, cfg.min_elements); - // Collect all target layer indices + // Compute per-layer bucket (0..n_layer_buckets-1) based on position within + // the layer's equivalence class. Layers outside any class (i.e. no quantizable + // weights) get bucket 0 — they won't be sampled anyway. Globals use -1. + // Computed BEFORE target_layers so we can sample reps per bucket. + std::map layer_to_bucket; + for (const auto & lc : layer_classes) { + const int n = (int)lc.all_layers.size(); + for (int i = 0; i < n; i++) { + layer_to_bucket[lc.all_layers[i]] = compute_bucket(i, n, cfg.n_layer_buckets); + } + } + + // Sample target layers: K reps per (class, bucket), evenly stratified across + // the bucket's layers. Boundary layers (first and last of each class) are + // always pinned regardless of K — they correspond to the most-sensitive + // positions in the residual stream (post-embedding read, pre-output write) + // and the hand-tuned recipes in llama-quant.cpp:513 (`use_more_bits`) treat + // them specially. Pinning them here keeps the (role, bucket-0) and + // (role, bucket-last) cost-matrix entries grounded in the actual boundary + // layers' KLD rather than only the bucket midpoint's. std::vector target_layers; for (const auto & lc : layer_classes) { - for (int l : lc.reps) target_layers.push_back(l); + if (lc.all_layers.empty()) continue; + target_layers.push_back(lc.all_layers.front()); + if (lc.all_layers.size() > 1) target_layers.push_back(lc.all_layers.back()); + + // Group this class's layers by bucket, then evenly sample K from each. + std::map> bucket_to_layers; + for (int l : lc.all_layers) { + bucket_to_layers[layer_to_bucket.at(l)].push_back(l); + } + for (auto & [b, layers] : bucket_to_layers) { + const int n = (int)layers.size(); + const int k = std::min(n, std::max(1, cfg.n_reps_per_bucket)); + if (k == 1) { + target_layers.push_back(layers[n / 2]); // middle of bucket + } else { + for (int i = 0; i < k; i++) { + int idx = (int)(((long long)i * (n - 1)) / (k - 1)); // 0..n-1 evenly + target_layers.push_back(layers[idx]); + } + } + } } std::sort(target_layers.begin(), target_layers.end()); target_layers.erase(std::unique(target_layers.begin(), target_layers.end()), target_layers.end()); - LOG("Layer equivalence classes: %zu classes, %zu total target layers\n", - layer_classes.size(), target_layers.size()); + LOG("Layer equivalence classes: %zu classes, %zu total target layers (K=%d reps/bucket)\n", + layer_classes.size(), target_layers.size(), cfg.n_reps_per_bucket); for (const auto & lc : layer_classes) { std::string roles_str; for (const auto & [role, ne0, ne1] : lc.signature) { if (!roles_str.empty()) roles_str += ", "; roles_str += role; } - LOG(" Class %zu: %zu layers, reps=[", lc.class_index, lc.all_layers.size()); - for (size_t i = 0; i < lc.reps.size(); i++) { - if (i > 0) LOG(", "); - LOG("%d", lc.reps[i]); + // Per-bucket rep listing: which sampled layers fall into each bucket. + std::map> sampled_by_bucket; + for (int l : lc.all_layers) { + if (std::binary_search(target_layers.begin(), target_layers.end(), l)) { + sampled_by_bucket[layer_to_bucket.at(l)].push_back(l); + } } - LOG("], roles=[%s]\n", roles_str.c_str()); - } - - // Compute per-layer bucket (0..n_layer_buckets-1) based on position within - // the layer's equivalence class. Layers outside any class (i.e. no quantizable - // weights) get bucket 0 — they won't be sampled anyway. Globals use -1. - std::map layer_to_bucket; - for (const auto & lc : layer_classes) { - const int n = (int)lc.all_layers.size(); - for (int i = 0; i < n; i++) { - layer_to_bucket[lc.all_layers[i]] = compute_bucket(i, n, cfg.n_layer_buckets); + std::string reps_str; + for (const auto & [b, layers] : sampled_by_bucket) { + if (!reps_str.empty()) reps_str += " "; + reps_str += "[" + std::to_string(b) + "]={"; + for (size_t i = 0; i < layers.size(); i++) { + if (i) reps_str += ","; + reps_str += std::to_string(layers[i]); + } + reps_str += "}"; } + LOG(" Class %zu: %zu layers, sampled %s, roles=[%s]\n", + lc.class_index, lc.all_layers.size(), reps_str.c_str(), roles_str.c_str()); } // For each (role, bucket), list the concrete layer indices that belong to it // (used at emission to write layer-alternation regexes). @@ -1589,32 +1964,54 @@ int main(int argc, char ** argv) { } } - // Build set of target weight tensor names (for the eval callback) - // NOTE: token_embd uses ggml_get_rows (embedding lookup), NOT ggml_mul_mat, - // so it cannot be measured via MUL_MAT KLD. It gets the highest-BPW type as preset. + // Build set of target weight tensor names (for the eval callback). + // Skip BOTH token_embd and output: + // - token_embd uses ggml_get_rows (embedding lookup), NOT ggml_mul_mat, + // so it cannot be measured via MUL_MAT KLD. + // - output IS a MUL_MAT, but Phase 4 force-pins it to `output_type` and + // overwrites the cost matrix entry regardless of measurement (see + // "Add global tensors to the cost matrix with their fixed types"), + // so measuring its KLD is wasted work — and it's typically the largest + // tensor in the model (vocab × hidden), so the waste is significant. + // Both globals get their types preset before optimization. std::unordered_set target_weight_names; for (const auto & ti : quantizable) { - if (ti.role == "token_embd") continue; // skip — not MUL_MAT + if (ti.role == "token_embd" || ti.role == "output") continue; if (ti.layer >= 0) { // Include if this tensor's layer is one of our target layers if (std::binary_search(target_layers.begin(), target_layers.end(), ti.layer)) { target_weight_names.insert(ti.name); } } else { - // Global tensors (output.weight, etc.) — capture those too + // Other global tensors (rare) — capture those too target_weight_names.insert(ti.name); } } - // ---- Phase 2: Capture reference activations ---- - LOG("\n--- Phase 2: Capturing reference activations ---\n"); - - // Load imatrix data + // Load imatrix data unconditionally — needed for Phase 3 trained-quant + // training (cache-miss path) AND for --importance-alpha role weights in + // Phase 4 (cache-hit path). std::unordered_map> imatrix_data; if (!cfg.imatrix_path.empty()) { load_imatrix_data(cfg.imatrix_path, imatrix_data); } + // ---- Cache check: try to skip Phase 2 + Phase 3 ---- + std::map> cost_matrix; + capture_state cap_state; // populated only on cache miss; declared here so + // its swap-with-empty cleanup later still compiles + bool cache_hit = false; + if (!cfg.cost_matrix_cache.empty()) { + cache_hit = load_cost_matrix_cache(cfg.cost_matrix_cache, cfg, cost_matrix); + } + + if (cache_hit) { + LOG("\n--- Skipping Phase 2 + Phase 3 (cost-matrix cache hit) ---\n"); + } else { + + // ---- Phase 2: Capture reference activations ---- + LOG("\n--- Phase 2: Capturing reference activations ---\n"); + // Load model with llama API common_params params; params.model.path = cfg.model_path; @@ -1628,7 +2025,6 @@ int main(int argc, char ** argv) { params.n_ubatch = min_batch; params.n_ctx = std::max(1024, max_test_size + 1); - capture_state cap_state; for (const auto & ti : quantizable) { if (target_weight_names.count(ti.name)) { cap_state.target_weight_names.insert(ti.name); @@ -1788,8 +2184,15 @@ int main(int argc, char ** argv) { // ---- Phase 3: Build cost matrix ---- LOG("\n--- Phase 3: Building cost matrix ---\n"); - auto cost_matrix = build_cost_matrix(cfg, quantizable, cap_state.captures_by_role, - cfg.model_path, gguf_ctx, imatrix_data); + cost_matrix = build_cost_matrix(cfg, quantizable, cap_state.captures_by_role, + cfg.model_path, gguf_ctx, imatrix_data); + + // Persist for future runs (cache invalidation is by header-field match). + if (!cfg.cost_matrix_cache.empty()) { + save_cost_matrix_cache(cfg.cost_matrix_cache, cfg, cost_matrix); + } + + } // end of cache-miss block (Phase 2 + Phase 3) // Build preliminary role_info map (per-bucket) so split_fused_roles can // apportion fused error by element count at the same bucket. @@ -1809,6 +2212,15 @@ int main(int argc, char ** argv) { // Split fused-MUL_MAT captures (e.g. attn_qkv) into their component roles. split_fused_roles(cost_matrix, roles); + // Apply imatrix-energy bias correction (if --importance-alpha > 0). Promotes + // low-energy residual-write tensors that the relative-L2 metric still + // under-weights. Done after split_fused_roles so the bias applies to the + // component roles (which actually carry the elements), not the fused entry. + if (cfg.importance_alpha > 0.0f) { + auto role_energy = compute_role_energy(quantizable, imatrix_data, cfg.min_elements); + apply_importance_alpha(cost_matrix, role_energy, cfg.importance_alpha); + } + // Print cost matrix LOG("\nCost matrix (avg relative-L2 error by role-bucket × quant type):\n"); LOG("%-20s", "Role[bucket]"); @@ -1830,10 +2242,11 @@ int main(int argc, char ** argv) { // ---- Phase 4: Optimize assignment ---- LOG("\n--- Phase 4: Optimizing assignment ---\n"); - // Determine fixed types for global tensors (before building roles map) - ggml_type output_type = cfg.output_tensor_type != GGML_TYPE_COUNT - ? cfg.output_tensor_type - : highest_bpw(cfg.quant_types); + // Determine fixed types for global tensors (before building roles map). + // parse_args has already set cfg.output_tensor_type (either from the user's + // --output-tensor-type or via default_output_type_for_target) so this is + // always non-COUNT here. + ggml_type output_type = cfg.output_tensor_type; // Detect tied embeddings: llama-quantize (src/llama-quant.cpp:~534) silently // promotes token_embd to output_tensor_type when the arch lacks a distinct diff --git a/tools/capture-layer-data/CMakeLists.txt b/tools/capture-layer-data/CMakeLists.txt index 4a81272e2ead..284c5ad53220 100644 --- a/tools/capture-layer-data/CMakeLists.txt +++ b/tools/capture-layer-data/CMakeLists.txt @@ -1,6 +1,6 @@ set(TARGET llama-capture-layer-data) add_executable(${TARGET} capture-layer-data.cpp) -target_link_libraries(${TARGET} PRIVATE common llama ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(${TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT}) target_include_directories(${TARGET} PRIVATE ../../common) target_compile_features(${TARGET} PRIVATE cxx_std_17) From 7209efbcecd5a9b9e40a6fbe22c349c32d719f53 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Thu, 14 May 2026 17:35:33 +0200 Subject: [PATCH 10/19] Add fitting support --- tools/auto-tensor-type/auto-tensor-type.cpp | 200 +++++++++++++------- 1 file changed, 134 insertions(+), 66 deletions(-) diff --git a/tools/auto-tensor-type/auto-tensor-type.cpp b/tools/auto-tensor-type/auto-tensor-type.cpp index d6b06c7fa283..5b022d35b0f3 100644 --- a/tools/auto-tensor-type/auto-tensor-type.cpp +++ b/tools/auto-tensor-type/auto-tensor-type.cpp @@ -1460,11 +1460,10 @@ static std::vector parse_quant_list(const std::string & s) { return result; } -static void print_usage(const char * prog) { - LOG("Usage: %s [options]\n", prog); +static void print_usage(int /*argc*/, char ** argv) { + LOG("Usage: %s [options]\n", argv[0]); LOG("\n"); - LOG("Options:\n"); - LOG(" -m, --model PATH Path to input GGUF model (required)\n"); + LOG("Tool-specific options:\n"); LOG(" -i, --imatrix PATH Path to importance matrix file (required)\n"); LOG(" -q, --quants LIST Comma-separated list of candidate quant types (required)\n"); LOG(" e.g., IQ1_BN,IQ2_TQ,IQ3_TQ,Q4_KPT,Q6_K\n"); @@ -1480,7 +1479,6 @@ static void print_usage(const char * prog) { LOG(" Avoids spending top-shelf bits on a force-pinned tensor\n"); LOG(" whose budget the DP spends better on amplified roles.)\n"); LOG(" --max-iterations N Max optimization iterations (default: 100)\n"); - LOG(" --threads N Number of threads (default: 1)\n"); LOG(" --layer-buckets N Per-class layer buckets for independent quant assignment\n"); LOG(" (default: 3 — first/middle/last third).\n"); LOG(" 1 reproduces the older per-role-only behavior.\n"); @@ -1498,90 +1496,154 @@ static void print_usage(const char * prog) { LOG(" skips Phase 2+3 entirely (~50min savings) when the\n"); LOG(" cache header matches model+quants+test-data+sampling.\n"); LOG(" On miss, runs Phase 3 normally and overwrites PATH.\n"); + LOG("\n"); + LOG("Model / offload options (forwarded to the standard llama.cpp parser; pass --help\n"); + LOG("to see the full list of supported common args, including caching, NUMA, etc.):\n"); + LOG(" -m, --model PATH Path to input GGUF model (required)\n"); + LOG(" -ngl, --gpu-layers N Layers to offload to GPU; 'auto' or 'all' (default: all)\n"); + LOG(" -ot, --override-tensor PAT=BUF Per-tensor buffer-type override (e.g. 'exps=CPU')\n"); + LOG(" -cmoe, --cpu-moe Keep all MoE expert weights on CPU\n"); + LOG(" -ncmoe, --n-cpu-moe N Keep MoE expert weights of the first N layers on CPU\n"); + LOG(" -ts, --tensor-split A,B,... Fraction of model per GPU\n"); + LOG(" -mg, --main-gpu IDX Main GPU index\n"); + LOG(" -sm, --split-mode MODE GPU split mode: none|layer|row|tensor\n"); + LOG(" -t, --threads N CPU threads (also drives Phase-3 parallelism)\n"); + LOG(" --no-mmap, --mlock, --numa MODE Standard memory-locking / NUMA controls\n"); } -static config parse_args(int argc, char ** argv) { +// Args owned by this tool. Everything else in argv is forwarded to the standard +// llama.cpp parser (common_params_parse), which handles -m/--model, -ngl, -ot, +// --cpu-moe, --n-cpu-moe, -ts, -mg, -sm, -t/--threads, -c, --no-mmap, --mlock, +// --numa, -h/--help, etc. — exactly the same offloading/fitting machinery used +// by llama-cli, llama-imatrix, llama-perplexity, llama-bench. +static const std::set & tool_args_with_value() { + static const std::set s = { + "-q", "--quants", + "-b", "--target-bpw", + "-o", "--output", + "-i", "--imatrix", + "--bpw-tolerance", + "--test-data", + "--test-samples", + "--test-sizes", + "--min-elements", + "--output-tensor-type", + "--max-iterations", + "--layer-buckets", + "--reps-per-bucket", + "--importance-alpha", + "--cost-matrix-cache", + }; + return s; +} + +static config parse_args(int argc, char ** argv, common_params & params) { config cfg; + + // First pass: split argv into (tool-specific args we own) and (everything else, + // forwarded to the common parser). Stripping tool-specific args is required — + // common_params_parse() throws on unknown flags. + std::vector forwarded; + forwarded.reserve(argc); + forwarded.push_back(argv[0]); + + auto need_value = [&](int i, const std::string & arg) { + if (i + 1 >= argc) { + LOG_ERR("Missing value for argument: %s\n", arg.c_str()); + exit(1); + } + }; + + const auto & owned = tool_args_with_value(); for (int i = 1; i < argc; i++) { std::string arg = argv[i]; - if ((arg == "-m" || arg == "--model") && i + 1 < argc) { - cfg.model_path = argv[++i]; - } else if ((arg == "-i" || arg == "--imatrix") && i + 1 < argc) { - cfg.imatrix_path = argv[++i]; - } else if ((arg == "-q" || arg == "--quants") && i + 1 < argc) { - cfg.quant_types = parse_quant_list(argv[++i]); - } else if ((arg == "-b" || arg == "--target-bpw") && i + 1 < argc) { - cfg.target_bpw = std::stof(argv[++i]); - } else if ((arg == "-o" || arg == "--output") && i + 1 < argc) { - cfg.output_path = argv[++i]; - } else if (arg == "--bpw-tolerance" && i + 1 < argc) { - std::string tol = argv[++i]; - // Parse "+HIGH,-LOW" format + if (!owned.count(arg)) { + forwarded.push_back(argv[i]); + continue; + } + need_value(i, arg); + const char * val = argv[++i]; + if (arg == "-q" || arg == "--quants") { + cfg.quant_types = parse_quant_list(val); + } else if (arg == "-b" || arg == "--target-bpw") { + cfg.target_bpw = std::stof(val); + } else if (arg == "-o" || arg == "--output") { + cfg.output_path = val; + } else if (arg == "-i" || arg == "--imatrix") { + cfg.imatrix_path = val; + } else if (arg == "--bpw-tolerance") { + std::string tol = val; size_t comma = tol.find(','); if (comma != std::string::npos) { cfg.bpw_tol_high = std::stof(tol.substr(0, comma)); - cfg.bpw_tol_low = std::stof(tol.substr(comma + 1)); + cfg.bpw_tol_low = std::stof(tol.substr(comma + 1)); } else { cfg.bpw_tol_high = std::stof(tol); } - } else if (arg == "--test-data" && i + 1 < argc) { - cfg.test_data_path = argv[++i]; - } else if (arg == "--test-samples" && i + 1 < argc) { - cfg.n_test_samples = std::stoi(argv[++i]); + } else if (arg == "--test-data") { + cfg.test_data_path = val; + } else if (arg == "--test-samples") { + cfg.n_test_samples = std::stoi(val); if (cfg.n_test_samples < 1) { LOG_ERR("--test-samples must be >= 1\n"); exit(1); } - } else if (arg == "--test-sizes" && i + 1 < argc) { - std::string sizes = argv[++i]; - std::istringstream ss(sizes); + } else if (arg == "--test-sizes") { + std::istringstream ss(val); std::string token; cfg.test_sizes.clear(); while (std::getline(ss, token, ',')) { cfg.test_sizes.push_back(std::stoi(token)); } - } else if (arg == "--min-elements" && i + 1 < argc) { - cfg.min_elements = std::stoll(argv[++i]); - } else if (arg == "--output-tensor-type" && i + 1 < argc) { - cfg.output_tensor_type = parse_ggml_type_str(argv[++i]); - } else if (arg == "--max-iterations" && i + 1 < argc) { - cfg.max_iterations = std::stoi(argv[++i]); - } else if (arg == "--threads" && i + 1 < argc) { - cfg.n_threads = std::stoi(argv[++i]); - } else if (arg == "--layer-buckets" && i + 1 < argc) { - cfg.n_layer_buckets = std::stoi(argv[++i]); + } else if (arg == "--min-elements") { + cfg.min_elements = std::stoll(val); + } else if (arg == "--output-tensor-type") { + cfg.output_tensor_type = parse_ggml_type_str(val); + } else if (arg == "--max-iterations") { + cfg.max_iterations = std::stoi(val); + } else if (arg == "--layer-buckets") { + cfg.n_layer_buckets = std::stoi(val); if (cfg.n_layer_buckets < 1) { LOG_ERR("--layer-buckets must be >= 1\n"); exit(1); } - } else if (arg == "--reps-per-bucket" && i + 1 < argc) { - cfg.n_reps_per_bucket = std::stoi(argv[++i]); + } else if (arg == "--reps-per-bucket") { + cfg.n_reps_per_bucket = std::stoi(val); if (cfg.n_reps_per_bucket < 1) { LOG_ERR("--reps-per-bucket must be >= 1\n"); exit(1); } - } else if (arg == "--importance-alpha" && i + 1 < argc) { - cfg.importance_alpha = std::stof(argv[++i]); + } else if (arg == "--importance-alpha") { + cfg.importance_alpha = std::stof(val); if (cfg.importance_alpha < 0.0f) { LOG_ERR("--importance-alpha must be >= 0\n"); exit(1); } - } else if (arg == "--cost-matrix-cache" && i + 1 < argc) { - cfg.cost_matrix_cache = argv[++i]; - } else if (arg == "-h" || arg == "--help") { - print_usage(argv[0]); - exit(0); - } else { - LOG_ERR("Unknown argument: %s\n\n", arg.c_str()); - print_usage(argv[0]); - exit(1); + } else if (arg == "--cost-matrix-cache") { + cfg.cost_matrix_cache = val; } } + // Hand the rest to the standard llama.cpp arg parser. This wires up -m, + // -ngl/--gpu-layers, -ot/--override-tensor, --cpu-moe/--n-cpu-moe, + // -ts/--tensor-split, -mg/--main-gpu, -sm/--split-mode, -t/--threads, + // -c/--ctx-size, --no-mmap, --mlock, --numa, -h/--help, etc. + // Default n_gpu_layers stays at common's default (-1 = auto/all layers), + // which matches the previous hardcoded n_gpu_layers=99 for models that + // fit in VRAM, and is overridable here by passing -ngl. + if (!common_params_parse((int)forwarded.size(), forwarded.data(), params, + LLAMA_EXAMPLE_COMMON, print_usage)) { + exit(1); + } + + // Pull values needed by the tool out of the parsed common_params. + cfg.model_path = params.model.path; + cfg.n_threads = params.cpuparams.n_threads; + if (cfg.model_path.empty() || cfg.imatrix_path.empty() || cfg.quant_types.empty() || cfg.target_bpw <= 0 || cfg.output_path.empty()) { LOG_ERR("Missing required arguments\n\n"); - print_usage(argv[0]); + print_usage(0, argv); exit(1); } @@ -1589,6 +1651,20 @@ static config parse_args(int argc, char ** argv) { cfg.output_tensor_type = default_output_type_for_target(cfg.quant_types, cfg.target_bpw); } + // Set n_ctx / n_batch / n_ubatch from --test-sizes so the fitting algorithm + // (which runs inside common_init_from_params in Phase 2) uses the correct + // memory footprint for its estimates. n_batch must be >= the longest sequence + // or llama_decode asserts; n_ctx just needs to hold that many tokens. + // fit_params_min_ctx is set to the same value so the fitter's floor matches + // what the tests actually require. + int max_test_size = 0; + for (int s : cfg.test_sizes) { max_test_size = std::max(max_test_size, s); } + const int min_ctx_batch = std::max(512, max_test_size + 1); + params.n_batch = min_ctx_batch; + params.n_ubatch = min_ctx_batch; + params.n_ctx = std::max(1024, max_test_size + 1); + params.fit_params_min_ctx = (uint32_t)params.n_ctx; + return cfg; } @@ -1761,7 +1837,10 @@ static bool load_cost_matrix_cache( // ============================================================================ int main(int argc, char ** argv) { - config cfg = parse_args(argc, argv); + common_init(); + + common_params params; + config cfg = parse_args(argc, argv, params); LOG("=== llama-auto-tensor-type ===\n"); LOG("Model: %s\n", cfg.model_path.c_str()); @@ -2012,19 +2091,8 @@ int main(int argc, char ** argv) { // ---- Phase 2: Capture reference activations ---- LOG("\n--- Phase 2: Capturing reference activations ---\n"); - // Load model with llama API - common_params params; - params.model.path = cfg.model_path; - params.n_gpu_layers = 99; // Offload to GPU if available, falls back to CPU - // Size batch/context to the largest requested test size (+BOS); n_batch must be - // >= n_tokens_all or llama_decode asserts. - int max_test_size = 0; - for (int s : cfg.test_sizes) max_test_size = std::max(max_test_size, s); - const int min_batch = std::max(512, max_test_size + 1); - params.n_batch = min_batch; - params.n_ubatch = min_batch; - params.n_ctx = std::max(1024, max_test_size + 1); - + // params.n_ctx / n_batch / n_ubatch were set in parse_args from --test-sizes + // so the fitting algorithm already used them. Just install the eval callback. for (const auto & ti : quantizable) { if (target_weight_names.count(ti.name)) { cap_state.target_weight_names.insert(ti.name); @@ -2043,9 +2111,9 @@ int main(int argc, char ** argv) { params.cb_eval = capture_callback; params.cb_eval_user_data = &cap_state; - common_init(); ggml_backend_load_all(); llama_backend_init(); + llama_numa_init(params.numa); auto llama_init = common_init_from_params(params); if (!llama_init) { From eae0e41635c2ba568e64c9d82083f8eac8e9a7e7 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 3 Jul 2026 15:43:34 +0200 Subject: [PATCH 11/19] imatrix: fix grouped-matmul weights (DeepSeek4 attn_output_a) DeepSeek4 stores attn_output_a (wo_a) as a 2D weight but reshapes it into an o_groups grouped matmul at inference. This broke imatrix twice: 1. Naming: ggml_reshape_3d renames the view " (reshaped)", and filter_tensor_name only stripped the backend#..#split wrapper, so the importance stats were saved under "...weight (reshaped)" and never matched at quant time ("Missing importance matrix"). Now also strip trailing (reshaped)/(view)/(cont)/(transposed)/(permuted) annotations. 2. Shape: the collected imatrix is one vector per group, i.e. an integer multiple of ne[0]*ne[2], which tripped the size check ("imatrix size different"). The quantizer now accepts a size that is a whole multiple of the per-slice size, treats it as size/ne[0] contiguous row-groups, and quantizes each group with its own vector. This is a strict generalization of the per-ne[2]-slice loop: expert (ne[2]>1) and normal (size==ne[0]) cases are byte-identical to before. No model or GGUF metadata change; wo_a stays a normal 2D quantized tensor. Also adds fix_reshaped_imatrix.py to recover imatrix files computed before fix (1) by stripping the annotations from the stored tensor keys. Assisted-By: Claude Opus 4.8 --- src/llama-quant.cpp | 37 ++++++++--- tools/imatrix/fix_reshaped_imatrix.py | 89 +++++++++++++++++++++++++++ tools/imatrix/imatrix.cpp | 12 +++- 3 files changed, 128 insertions(+), 10 deletions(-) create mode 100644 tools/imatrix/fix_reshaped_imatrix.py diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index d07a46f049f8..761f68cf161b 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -1926,16 +1926,29 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: const int64_t nelements = ggml_nelements(tensor); const float * imatrix = nullptr; + // number of importance-matrix row-groups. Normally one vector per ne[2] slice (3D experts), + // but a weight that is reshaped into groups before its matmul (e.g. DeepSeek4 attn_output_a, + // stored 2D but used as a grouped matmul) contributes one vector per group, so the imatrix + // can be an integer multiple of the per-slice size ne[0]*ne[2]. + int64_t imatrix_n_groups = tensor->ne[2]; if (imatrix_data) { auto it = imatrix_data->find(tm.remapped_imatrix_name); if (it == imatrix_data->end()) { LLAMA_LOG_INFO("\n====== %s: did not find weights for %s\n", __func__, tensor->name); } else { - if (it->second.size() == (size_t)tensor->ne[0]*tensor->ne[2]) { + const size_t imx_size = it->second.size(); + const size_t slice_size = (size_t) tensor->ne[0] * tensor->ne[2]; + // accept an imatrix whose size is a whole multiple of the per-slice size, provided the + // resulting groups tile the tensor rows evenly - each group then covers a contiguous + // block of ne[1]*ne[2]/n_groups rows, matching how the grouped matmul collected them + const bool size_ok = slice_size > 0 && imx_size % slice_size == 0 && + (size_t)(tensor->ne[1]*tensor->ne[2]) % (imx_size / (size_t)tensor->ne[0]) == 0; + if (size_ok) { imatrix = it->second.data(); + imatrix_n_groups = (int64_t)(imx_size / (size_t)tensor->ne[0]); } else { LLAMA_LOG_INFO("\n====== %s: imatrix size %d is different from tensor size %d for %s\n", __func__, - int(it->second.size()), int(tensor->ne[0]*tensor->ne[2]), tensor->name); + int(imx_size), int(slice_size), tensor->name); // this can happen when quantizing an old mixtral model with split tensors with a new incompatible imatrix // this is a significant error and it may be good idea to abort the process if this happens, @@ -1943,7 +1956,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: // tok_embd should be ignored in this case, since it always causes this warning if (!tensor_name_match_token_embd(tensor->name)) { throw std::runtime_error(format("imatrix size %d is different from tensor size %d for %s", - int(it->second.size()), int(tensor->ne[0]*tensor->ne[2]), tensor->name)); + int(imx_size), int(slice_size), tensor->name)); } } } @@ -2064,14 +2077,20 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: q2kpt_prepare_levels(total_rows, n_per_row); // Allocate for this tensor } - // quantize each expert separately since they have different importance matrices + // quantize each imatrix group separately since they have different importance matrices. + // for 3D experts this is one group per ne[2] slice; for a grouped-matmul weight like + // attn_output_a it is one group per matmul group (imatrix_n_groups > ne[2]), each covering + // a contiguous block of rows. when no imatrix is present, fall back to per-ne[2]-slice. + const int64_t n_groups_q = imatrix ? imatrix_n_groups : tensor->ne[2]; + const int64_t n_rows_total = tensor->ne[1] * tensor->ne[2]; + const int64_t rows_per_group = n_rows_total / n_groups_q; new_size = 0; - for (int64_t i03 = 0; i03 < tensor->ne[2]; ++i03) { - const float * f32_data_03 = f32_data + i03 * nelements_matrix; - void * new_data_03 = (char *)new_data + ggml_row_size(new_type, n_per_row) * i03 * nrows; - const float * imatrix_03 = imatrix ? imatrix + i03 * n_per_row : nullptr; + for (int64_t ig = 0; ig < n_groups_q; ++ig) { + const float * f32_data_g = f32_data + ig * rows_per_group * n_per_row; + void * new_data_g = (char *)new_data + ggml_row_size(new_type, n_per_row) * ig * rows_per_group; + const float * imatrix_g = imatrix ? imatrix + ig * n_per_row : nullptr; - new_size += llama_tensor_quantize_impl(new_type, f32_data_03, new_data_03, chunk_size, nrows, n_per_row, imatrix_03, workers, nthread_use); + new_size += llama_tensor_quantize_impl(new_type, f32_data_g, new_data_g, chunk_size, rows_per_group, n_per_row, imatrix_g, workers, nthread_use); } LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB\n", tensor_size/1024.0/1024.0, new_size/1024.0/1024.0); } diff --git a/tools/imatrix/fix_reshaped_imatrix.py b/tools/imatrix/fix_reshaped_imatrix.py new file mode 100644 index 000000000000..c00c2be9a89b --- /dev/null +++ b/tools/imatrix/fix_reshaped_imatrix.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +""" +Strip ggml view/reshape annotations (e.g. " (reshaped)") from imatrix tensor +names so they match the real model weight names at quantization time. + +Background: when a weight is reshaped inline before ggml_mul_mat (as DeepSeek4 +does for blk.N.attn_output_a.weight), the imatrix collector keys the importance +stats by the derived tensor name " (reshaped)". Quantization looks up the +plain weight name and fails with "Missing importance matrix ...". The data is +present under the wrong key; this rewrites the keys in place (into a new file). + +Usage: fix_reshaped_imatrix.py input.imatrix.gguf output.imatrix.gguf +""" +from __future__ import annotations + +import os +import sys +from pathlib import Path + +# Load the local gguf package (repo checkout), like the other gguf scripts do. +if "NO_LOCAL_GGUF" not in os.environ and (Path(__file__).parent.parent.parent / "gguf-py").exists(): + sys.path.insert(0, str(Path(__file__).parent.parent.parent / "gguf-py")) + +import gguf # noqa: E402 + +# ggml suffixes appended to derived tensors (see ggml_format_name in ggml.c) +VIEW_SUFFIXES = (" (reshaped)", " (view)", " (cont)", " (transposed)", " (permuted)") +# imatrix stores two tensors per entry +STAT_SUFFIXES = (".in_sum2", ".counts") + + +def clean_name(name: str) -> str: + stat = next((s for s in STAT_SUFFIXES if name.endswith(s)), "") + base = name[: -len(stat)] if stat else name + changed = True + while changed: + changed = False + for s in VIEW_SUFFIXES: + if base.endswith(s): + base = base[: -len(s)] + changed = True + return base + stat + + +def main() -> None: + if len(sys.argv) != 3: + print(__doc__) + sys.exit(1) + + inp, outp = Path(sys.argv[1]), Path(sys.argv[2]) + print(f"* Loading: {inp}") + reader = gguf.GGUFReader(inp, "r") + + # imatrix files carry no general.architecture; keep the copy faithful. + writer = gguf.GGUFWriter(outp, arch="", endianess=reader.endianess) + writer.kv_data[0].pop(gguf.Keys.General.ARCHITECTURE, None) + + alignment = reader.get_field(gguf.Keys.General.ALIGNMENT) + if alignment is not None: + writer.data_alignment = alignment.contents() + + # copy every KV field verbatim (skip virtual GGUF.* and the auto-added arch) + for field in reader.fields.values(): + if field.name == gguf.Keys.General.ARCHITECTURE or field.name.startswith("GGUF."): + continue + val_type = field.types[0] + sub_type = field.types[-1] if val_type == gguf.GGUFValueType.ARRAY else None + writer.add_key_value(field.name, field.contents(), val_type, sub_type=sub_type) + + renamed = 0 + for tensor in reader.tensors: + new_name = clean_name(tensor.name) + if new_name != tensor.name: + renamed += 1 + writer.add_tensor_info(new_name, tensor.data.shape, tensor.data.dtype, tensor.data.nbytes, tensor.tensor_type) + + print(f"* Renaming {renamed} of {len(reader.tensors)} tensors") + print(f"* Writing: {outp}") + writer.write_header_to_file() + writer.write_kv_data_to_file() + writer.write_ti_data_to_file() + for tensor in reader.tensors: + writer.write_tensor_data(tensor.data, tensor_endianess=reader.endianess) + writer.close() + print("* Done") + + +if __name__ == "__main__": + main() diff --git a/tools/imatrix/imatrix.cpp b/tools/imatrix/imatrix.cpp index 3431a4eca84b..a4ce017d09bf 100644 --- a/tools/imatrix/imatrix.cpp +++ b/tools/imatrix/imatrix.cpp @@ -75,7 +75,8 @@ class IMatrixCollector { }; // remove any prefix and suffixes from the name -// CUDA0#blk.0.attn_k.weight#0 => blk.0.attn_k.weight +// CUDA0#blk.0.attn_k.weight#0 => blk.0.attn_k.weight +// blk.0.attn_output_a.weight (reshaped) => blk.0.attn_output_a.weight static std::string filter_tensor_name(const char * name) { std::string wname; const char * p = strchr(name, '#'); @@ -90,6 +91,15 @@ static std::string filter_tensor_name(const char * name) { } else { wname = name; } + // strip trailing view/reshape annotations that ggml appends to derived tensors + // (e.g. a weight reshaped inline before ggml_mul_mat), so importance stats are + // keyed by the original weight name and match at quantization time + for (const char * suffix : { " (reshaped)", " (view)", " (cont)", " (transposed)", " (permuted)" }) { + const size_t slen = strlen(suffix); + if (wname.size() >= slen && wname.compare(wname.size() - slen, slen, suffix) == 0) { + wname.resize(wname.size() - slen); + } + } return wname; } From dbd2f0afd8da81660181191a02972d481db1a819 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 3 Jul 2026 15:47:30 +0200 Subject: [PATCH 12/19] imatrix: make pass-1 level trainers group-aware The pass-2 quantizer already applies a grouped imatrix per row-group for weights like DeepSeek4 attn_output_a. The pass-1 per-tensor trainers (Q3_PT/Q3_KPT/Q4_DPT/Q2_KPT/IQ2_TQ/IQ3_TQ/IQ1_BN) still gated the imatrix on size == ne[0]*ne[2], so for a grouped weight they silently trained levels/grids/codebooks without any importance weighting. Route all seven resolutions through a shared resolve_level_imatrix() helper: when the stored imatrix holds multiple groups over a 2D weight, average the group vectors into one per-column vector for these single-vector trainers; otherwise return the data unchanged. This is a strict generalization - the per-slice (size == ne[0]*ne[2]) and genuine-mismatch cases behave exactly as before. Assisted-By: Claude Opus 4.8 --- src/llama-quant.cpp | 114 ++++++++++++++++++++++---------------------- 1 file changed, 58 insertions(+), 56 deletions(-) diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 761f68cf161b..744b7a5f8dc7 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -1014,6 +1014,50 @@ static void init_quantize_state_counters(quantize_state_impl & qs, std::vector> * imatrix_data, + const std::string & name, const ggml_tensor * tensor, std::vector & buf) { + if (!imatrix_data) { + return nullptr; + } + auto it = imatrix_data->find(name); + if (it == imatrix_data->end()) { + return nullptr; + } + const std::vector & imx = it->second; + const size_t n_per_row = (size_t) tensor->ne[0]; + const size_t slice_size = n_per_row * (size_t) tensor->ne[2]; + if (slice_size == 0) { + return nullptr; + } + if (imx.size() == slice_size) { + return imx.data(); // one vector per ne[2] slice - existing behaviour + } + // grouped-matmul weight: the imatrix holds n_groups vectors over a 2D weight; + // average them into a single per-column vector for the per-tensor trainer + if (tensor->ne[2] == 1 && imx.size() % n_per_row == 0 && + (size_t) tensor->ne[1] % (imx.size() / n_per_row) == 0) { + const size_t n_groups = imx.size() / n_per_row; + buf.assign(n_per_row, 0.0f); + for (size_t g = 0; g < n_groups; ++g) { + for (size_t j = 0; j < n_per_row; ++j) { + buf[j] += imx[g * n_per_row + j]; + } + } + const float inv = 1.0f / (float) n_groups; + for (size_t j = 0; j < n_per_row; ++j) { + buf[j] *= inv; + } + return buf.data(); + } + return nullptr; // genuine size mismatch - train without imatrix (unchanged) +} + // // main quantization driver // @@ -1317,14 +1361,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: } // Resolve imatrix - const float * imatrix = nullptr; - if (imatrix_data) { - auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); - if (it2 != imatrix_data->end() && - it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { - imatrix = it2->second.data(); - } - } + std::vector imatrix_buf; + const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); const int64_t n_per_row = tensor->ne[0]; const int64_t nrows = tensor->ne[1]; @@ -1400,14 +1438,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: } // Resolve imatrix - const float * imatrix = nullptr; - if (imatrix_data) { - auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); - if (it2 != imatrix_data->end() && - it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { - imatrix = it2->second.data(); - } - } + std::vector imatrix_buf; + const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); const int64_t n_per_row = tensor->ne[0]; const int64_t nrows = tensor->ne[1]; @@ -1476,14 +1508,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: } // Resolve imatrix - const float * imatrix = nullptr; - if (imatrix_data) { - auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); - if (it2 != imatrix_data->end() && - it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { - imatrix = it2->second.data(); - } - } + std::vector imatrix_buf; + const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); const int64_t n_per_row = tensor->ne[0]; const int64_t nrows = tensor->ne[1]; @@ -1563,14 +1589,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: } // Resolve imatrix - const float * imatrix = nullptr; - if (imatrix_data) { - auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); - if (it2 != imatrix_data->end() && - it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { - imatrix = it2->second.data(); - } - } + std::vector imatrix_buf; + const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); const int64_t n_per_row = tensor->ne[0]; const int64_t nrows = tensor->ne[1]; @@ -1670,14 +1690,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: } // Resolve imatrix - const float * imatrix = nullptr; - if (imatrix_data) { - auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); - if (it2 != imatrix_data->end() && - it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { - imatrix = it2->second.data(); - } - } + std::vector imatrix_buf; + const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); const int64_t n_per_row = tensor->ne[0]; const int64_t nrows = tensor->ne[1]; @@ -1742,14 +1756,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: f32_data = (float *) p1_f32_buf.data(); } - const float * imatrix = nullptr; - if (imatrix_data) { - auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); - if (it2 != imatrix_data->end() && - it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { - imatrix = it2->second.data(); - } - } + std::vector imatrix_buf; + const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); const int64_t n_per_row = tensor->ne[0]; const int64_t nrows = tensor->ne[1]; @@ -1823,14 +1831,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: f32_data = (float *) p1_f32_buf.data(); } - const float * imatrix = nullptr; - if (imatrix_data) { - auto it2 = imatrix_data->find(remap_imatrix(tensor->name, mapped)); - if (it2 != imatrix_data->end() && - it2->second.size() == (size_t)tensor->ne[0] * tensor->ne[2]) { - imatrix = it2->second.data(); - } - } + std::vector imatrix_buf; + const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); const int64_t n_per_row = tensor->ne[0]; const int64_t nrows = tensor->ne[1]; From f39ba96e974e68c4a1a686989fc515d40809920b Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 3 Jul 2026 15:58:21 +0200 Subject: [PATCH 13/19] quant: never quantize integer id/index tensors DeepSeek4 MoE routing tables (ffn_gate_tid2eid / ffn_gate_eid2tid) are i32 id maps, not weights. Their names end in ".weight" and they are 2D, so the quantize gate selected them and quantization then failed with "cannot dequantize/convert tensor type i32". Skip any I8/I16/I32/I64 tensor in tensor_allows_quantization (pass-2) and in the pass-1 trainer gates, so these tensors are copied through unchanged. Assisted-By: Claude Opus 4.8 --- src/llama-quant.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 744b7a5f8dc7..91bbd644a368 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -387,6 +387,14 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param // quantize only 2D and 3D tensors (experts) if (ggml_n_dims(tensor) < 2) return false; + // never quantize integer tensors: these are id/index tables, not weights + // (e.g. MoE routing maps like ffn_gate_tid2eid / ffn_gate_eid2tid) and cannot + // be dequantized/converted - pass them through unchanged + if (tensor->type == GGML_TYPE_I8 || tensor->type == GGML_TYPE_I16 || + tensor->type == GGML_TYPE_I32 || tensor->type == GGML_TYPE_I64) { + return false; + } + const std::string name = ggml_get_name(tensor); // This used to be a regex, but has an extreme cost to compile times. @@ -1332,6 +1340,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: // Determine whether this tensor will be Q3_PT (mirror the pass-2 logic) bool quantize = tname.rfind("weight") == tname.size() - 6; quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && + tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; quantize &= tname.find("_norm.weight") == std::string::npos; quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; if (!quantize) { continue; } @@ -1402,6 +1412,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: // Determine whether this tensor will be Q3_KPT (mirror the pass-2 logic) bool quantize = tname.rfind("weight") == tname.size() - 6; quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && + tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; quantize &= tname.find("_norm.weight") == std::string::npos; quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; if (!quantize) { continue; } @@ -1482,6 +1494,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: bool quantize = tname.rfind("weight") == tname.size() - 6; quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && + tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; quantize &= tname.find("_norm.weight") == std::string::npos; quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; if (!quantize) { continue; } @@ -1553,6 +1567,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: // Determine whether this tensor will be Q2_KPT (mirror the pass-2 logic) bool quantize = tname.rfind("weight") == tname.size() - 6; quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && + tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; quantize &= tname.find("_norm.weight") == std::string::npos; quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; if (!quantize) { continue; } @@ -1664,6 +1680,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: bool quantize = tname.rfind("weight") == tname.size() - 6; quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && + tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; quantize &= tname.find("_norm.weight") == std::string::npos; quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; if (!quantize) { continue; } @@ -1733,6 +1751,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: bool quantize = tname.rfind("weight") == tname.size() - 6; quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && + tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; quantize &= tname.find("_norm.weight") == std::string::npos; quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; if (!quantize) { continue; } @@ -1798,6 +1818,8 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: bool quantize = tname.rfind("weight") == tname.size() - 6; quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && + tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; quantize &= tname.find("_norm.weight") == std::string::npos; quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; if (!quantize) { continue; } From 9f573b87fb3c04f4ce8bfa50ca33b54a9cb9ff6d Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 3 Jul 2026 16:10:04 +0200 Subject: [PATCH 14/19] quant: factor out shared pass-1 trainer boilerplate The seven pass-1 per-tensor trainers (Q3_PT/Q3_KPT/Q4_DPT/Q2_KPT/IQ2_TQ/ IQ3_TQ/IQ1_BN) each repeated ~40 lines of identical setup: the quantize gate, target-type resolution, tensor load, f32 dequantize, and imatrix resolution. Extract this into a reusable pass1_setup driver with a prepare(tensor, want_type) method that returns the f32 data, resolved (possibly grouped) imatrix, name and dims, so each loop keeps only its genuinely type-specific training call. Net -237 lines. This also unifies the target-type resolution: Q3_PT/Q3_KPT/Q2_KPT/IQ1_BN were still using divergent inline copies (Q3_PT even omitted the token_embedding_type/output_tensor_type overrides, and none honored the --tensor-type regex patterns). They now all go through resolve_tensor_type like Q4_DPT/IQ2_TQ/IQ3_TQ already did, matching pass-2. Verified with a Q3_PT quantize of DeepSeek4-Flash: pass-1 trains every tensor (including grouped attn_output_a) and pass-2 runs, with no errors. Assisted-By: Claude Opus 4.8 --- src/llama-quant.cpp | 461 +++++++++++--------------------------------- 1 file changed, 112 insertions(+), 349 deletions(-) diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 91bbd644a368..f5206ebdd31c 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -1066,6 +1066,85 @@ static const float * resolve_level_imatrix( return nullptr; // genuine size mismatch - train without imatrix (unchanged) } +// Shared driver for the pass-1 per-tensor trainers (Q3_PT/Q3_KPT/Q4_DPT/Q2_KPT/ +// IQ2_TQ/IQ3_TQ/IQ1_BN). Holds the reusable dequant scratch, and for each tensor +// decides whether it will be quantized to `want_type`; if so it loads and +// dequantizes it to f32 and resolves its (possibly grouped) imatrix. Keeps the +// seven trainer loops down to their genuinely type-specific work. +struct pass1_setup { + quantize_state_impl & qs; + const llama_model_quantize_params * params; + llama_model_loader & ml; + ggml_type default_type; + llama_ftype ftype; + int nthread; + const std::map & mapped; + const std::unordered_map> * imatrix_data; + + // reusable scratch + std::vector> read_data; + std::vector> f32_buf; + std::vector workers; + + // outputs, valid only when prepare() returns true + std::string name; + float * f32_data = nullptr; + const float * imatrix = nullptr; + std::vector imatrix_buf; + int64_t n_per_row = 0; + int64_t nrows = 0; + + pass1_setup(quantize_state_impl & qs_, const llama_model_quantize_params * params_, + llama_model_loader & ml_, ggml_type default_type_, llama_ftype ftype_, int nthread_, + const std::map & mapped_, + const std::unordered_map> * imatrix_data_) + : qs(qs_), params(params_), ml(ml_), default_type(default_type_), ftype(ftype_), + nthread(nthread_), mapped(mapped_), imatrix_data(imatrix_data_) { + workers.reserve(nthread); + } + + // returns true if `tensor` will be quantized to `want_type` and is now loaded as f32 + bool prepare(ggml_tensor * tensor, ggml_type want_type) { + name = ggml_get_name(tensor); + + // mirror the pass-2 quantize gate + bool quantize = name.rfind("weight") == name.size() - 6; + quantize &= (ggml_n_dims(tensor) >= 2); + quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && + tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; + quantize &= name.find("_norm.weight") == std::string::npos; + quantize &= name.find("ffn_gate_inp.weight") == std::string::npos; + if (!quantize) { + return false; + } + if (resolve_tensor_type(qs, params, tensor, default_type, ftype, name) != want_type) { + return false; + } + + // load tensor data + const size_t tsz = ggml_nbytes(tensor); + if (!ml.use_mmap) { + if (read_data.size() < tsz) { read_data.resize(tsz); } + tensor->data = read_data.data(); + } + ml.load_data_for(tensor); + + // dequantize to f32 if needed + const int64_t nelements = ggml_nelements(tensor); + if (tensor->type == GGML_TYPE_F32) { + f32_data = (float *) tensor->data; + } else { + llama_tensor_dequantize_impl(tensor, f32_buf, workers, nelements, nthread); + f32_data = (float *) f32_buf.data(); + } + + imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); + n_per_row = tensor->ne[0]; + nrows = tensor->ne[1]; + return true; + } +}; + // // main quantization driver // @@ -1319,6 +1398,9 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: ::zeros(fout, meta_size); }; + // shared driver for all pass-1 per-tensor trainers below (dequant scratch + gate + imatrix) + pass1_setup p1(qs, params, ml, default_type, ftype, nthread, mapped, imatrix_data); + // Q3_PT two-pass approach: train all per-tensor levels BEFORE opening the output // file, so the levels KV entry is already populated at the time of the metadata placeholder. static const size_t Q3PT_N_LEVELS = 8; @@ -1327,58 +1409,12 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: LLAMA_LOG_INFO("%s: Q3_PT pass 1: training per-tensor levels...\n", __func__); q3pt_all_levels.assign(tensors.size() * Q3PT_N_LEVELS, 0.0f); - // Temporary dequant buffer for pass 1 (reuse f32_conv_buf / read_data declared below) - std::vector> p1_read_data; - std::vector> p1_f32_buf; - std::vector p1_workers; - p1_workers.reserve(nthread); - for (size_t ti = 0; ti < tensors.size(); ++ti) { ggml_tensor * tensor = tensors[ti]->tensor; - const std::string tname = ggml_get_name(tensor); - - // Determine whether this tensor will be Q3_PT (mirror the pass-2 logic) - bool quantize = tname.rfind("weight") == tname.size() - 6; - quantize &= (ggml_n_dims(tensor) >= 2); - quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && - tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; - quantize &= tname.find("_norm.weight") == std::string::npos; - quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; - if (!quantize) { continue; } - - ggml_type new_type = default_type; - if (!params->pure) { - new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); - } - if (new_type != GGML_TYPE_Q3_PT) { continue; } - - // Load tensor data - const size_t tsz = ggml_nbytes(tensor); - if (!ml.use_mmap) { - if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } - tensor->data = p1_read_data.data(); - } - ml.load_data_for(tensor); - - // Dequantize to f32 if needed - const int64_t nelements = ggml_nelements(tensor); - float * f32_data; - if (tensor->type == GGML_TYPE_F32) { - f32_data = (float *) tensor->data; - } else { - llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); - f32_data = (float *) p1_f32_buf.data(); - } - - // Resolve imatrix - std::vector imatrix_buf; - const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); - - const int64_t n_per_row = tensor->ne[0]; - const int64_t nrows = tensor->ne[1]; + if (!p1.prepare(tensor, GGML_TYPE_Q3_PT)) { continue; } LLAMA_LOG_INFO("%s: Q3_PT levels for [%zu/%zu] %s\n", __func__, ti+1, tensors.size(), tensor->name); - q3pt_train_levels(f32_data, nrows, n_per_row, imatrix, + q3pt_train_levels(p1.f32_data, p1.nrows, p1.n_per_row, p1.imatrix, q3pt_all_levels.data() + ti * Q3PT_N_LEVELS); } @@ -1399,65 +1435,12 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: LLAMA_LOG_INFO("%s: Q3_KPT pass 1: training per-tensor levels...\n", __func__); q3kpt_all_levels.assign(tensors.size() * Q3KPT_N_LEVELS, 0.0f); - // Temporary dequant buffer for pass 1 - std::vector> p1_read_data; - std::vector> p1_f32_buf; - std::vector p1_workers; - p1_workers.reserve(nthread); - for (size_t ti = 0; ti < tensors.size(); ++ti) { ggml_tensor * tensor = tensors[ti]->tensor; - const std::string tname = ggml_get_name(tensor); - - // Determine whether this tensor will be Q3_KPT (mirror the pass-2 logic) - bool quantize = tname.rfind("weight") == tname.size() - 6; - quantize &= (ggml_n_dims(tensor) >= 2); - quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && - tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; - quantize &= tname.find("_norm.weight") == std::string::npos; - quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; - if (!quantize) { continue; } - - ggml_type new_type = default_type; - if (!params->pure) { - new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); - } - if (params->token_embedding_type < GGML_TYPE_COUNT && - (tname == "token_embd.weight" || tname == "per_layer_token_embd.weight")) { - new_type = params->token_embedding_type; - } - if (params->output_tensor_type < GGML_TYPE_COUNT && tname == "output.weight") { - new_type = params->output_tensor_type; - } - if (new_type != GGML_TYPE_Q3_KPT) { continue; } - - // Load tensor data - const size_t tsz = ggml_nbytes(tensor); - if (!ml.use_mmap) { - if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } - tensor->data = p1_read_data.data(); - } - ml.load_data_for(tensor); - - // Dequantize to f32 if needed - const int64_t nelements = ggml_nelements(tensor); - float * f32_data; - if (tensor->type == GGML_TYPE_F32) { - f32_data = (float *) tensor->data; - } else { - llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); - f32_data = (float *) p1_f32_buf.data(); - } - - // Resolve imatrix - std::vector imatrix_buf; - const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); - - const int64_t n_per_row = tensor->ne[0]; - const int64_t nrows = tensor->ne[1]; + if (!p1.prepare(tensor, GGML_TYPE_Q3_KPT)) { continue; } LLAMA_LOG_INFO("%s: Q3_KPT levels for [%zu/%zu] %s\n", __func__, ti+1, tensors.size(), tensor->name); - q3kpt_train_levels(f32_data, nrows, n_per_row, imatrix, + q3kpt_train_levels(p1.f32_data, p1.nrows, p1.n_per_row, p1.imatrix, q3kpt_all_levels.data() + ti * Q3KPT_N_LEVELS); } @@ -1483,60 +1466,19 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: const int64_t t_start_p1 = ggml_time_us(); LLAMA_LOG_INFO("%s: Q4_DPT pass 1: training per-tensor int8 levels...\n", __func__); - std::vector> p1_read_data; - std::vector> p1_f32_buf; - std::vector p1_workers; - p1_workers.reserve(nthread); - for (size_t ti = 0; ti < tensors.size(); ++ti) { ggml_tensor * tensor = tensors[ti]->tensor; - const std::string tname = ggml_get_name(tensor); - - bool quantize = tname.rfind("weight") == tname.size() - 6; - quantize &= (ggml_n_dims(tensor) >= 2); - quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && - tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; - quantize &= tname.find("_norm.weight") == std::string::npos; - quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; - if (!quantize) { continue; } - - ggml_type new_type = resolve_tensor_type(qs, params, tensor, default_type, ftype, tname); - if (new_type != GGML_TYPE_Q4_DPT) { continue; } - - // Load tensor data - const size_t tsz = ggml_nbytes(tensor); - if (!ml.use_mmap) { - if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } - tensor->data = p1_read_data.data(); - } - ml.load_data_for(tensor); - - // Dequantize to f32 if needed - const int64_t nelements = ggml_nelements(tensor); - float * f32_data; - if (tensor->type == GGML_TYPE_F32) { - f32_data = (float *) tensor->data; - } else { - llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); - f32_data = (float *) p1_f32_buf.data(); - } - - // Resolve imatrix - std::vector imatrix_buf; - const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); - - const int64_t n_per_row = tensor->ne[0]; - const int64_t nrows = tensor->ne[1]; + if (!p1.prepare(tensor, GGML_TYPE_Q4_DPT)) { continue; } LLAMA_LOG_INFO("%s: Q4_DPT levels for [%zu/%zu] %s\n", __func__, ti+1, tensors.size(), tensor->name); q4dpt_meta meta; - meta.tensor_name = tname; - q4dpt_train_levels(f32_data, nrows, n_per_row, imatrix, meta.levels); + meta.tensor_name = p1.name; + q4dpt_train_levels(p1.f32_data, p1.nrows, p1.n_per_row, p1.imatrix, meta.levels); q4dpt_all_meta.push_back(meta); // Save to GGUF - std::string levels_key = "q4dpt.levels." + tname; + std::string levels_key = "q4dpt.levels." + p1.name; gguf_set_arr_data(ctx_outs[0].get(), levels_key.c_str(), GGUF_TYPE_INT8, meta.levels, Q4DPT_N_LEVELS); } const int64_t t_end_p1 = ggml_time_us(); @@ -1555,66 +1497,14 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: if (ftype == LLAMA_FTYPE_MOSTLY_Q2_KPT && !params->dry_run) { LLAMA_LOG_INFO("%s: Q2_KPT pass 1: training per-block levels...\n", __func__); - std::vector> p1_read_data; - std::vector> p1_f32_buf; - std::vector p1_workers; - p1_workers.reserve(nthread); - for (size_t ti = 0; ti < tensors.size(); ++ti) { ggml_tensor * tensor = tensors[ti]->tensor; - const std::string tname = ggml_get_name(tensor); - - // Determine whether this tensor will be Q2_KPT (mirror the pass-2 logic) - bool quantize = tname.rfind("weight") == tname.size() - 6; - quantize &= (ggml_n_dims(tensor) >= 2); - quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && - tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; - quantize &= tname.find("_norm.weight") == std::string::npos; - quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; - if (!quantize) { continue; } - - ggml_type new_type = default_type; - if (!params->pure) { - new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); - } - if (params->token_embedding_type < GGML_TYPE_COUNT && - (tname == "token_embd.weight" || tname == "per_layer_token_embd.weight")) { - new_type = params->token_embedding_type; - } - if (params->output_tensor_type < GGML_TYPE_COUNT && tname == "output.weight") { - new_type = params->output_tensor_type; - } - if (new_type != GGML_TYPE_Q2_KPT) { continue; } - - // Load tensor data - const size_t tsz = ggml_nbytes(tensor); - if (!ml.use_mmap) { - if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } - tensor->data = p1_read_data.data(); - } - ml.load_data_for(tensor); - - // Dequantize to f32 if needed - const int64_t nelements = ggml_nelements(tensor); - float * f32_data; - if (tensor->type == GGML_TYPE_F32) { - f32_data = (float *) tensor->data; - } else { - llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); - f32_data = (float *) p1_f32_buf.data(); - } - - // Resolve imatrix - std::vector imatrix_buf; - const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); - - const int64_t n_per_row = tensor->ne[0]; - const int64_t nrows = tensor->ne[1]; + if (!p1.prepare(tensor, GGML_TYPE_Q2_KPT)) { continue; } // Allocate levels buffer for this tensor - const int nb = n_per_row / QK_K; - const size_t n_levels = (size_t)nrows * tensor->ne[2] * nb * Q2KPT_N_LEVELS; - q2kpt_all_levels.push_back({tname, std::vector(n_levels)}); + const int nb = p1.n_per_row / QK_K; + const size_t n_levels = (size_t)p1.nrows * tensor->ne[2] * nb * Q2KPT_N_LEVELS; + q2kpt_all_levels.push_back({p1.name, std::vector(n_levels)}); LLAMA_LOG_INFO("%s: Q2_KPT levels for [%zu/%zu] %s (%zu floats)\n", __func__, ti+1, tensors.size(), tensor->name, n_levels); @@ -1622,21 +1512,21 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: // Train levels by running quantization internally // We need to quantize to f32 -> Q2_KPT -> f32 to get the trained levels std::vector> p1_qbuf(ggml_nbytes(tensor)); - const size_t row_size = ggml_row_size(GGML_TYPE_Q2_KPT, n_per_row); + const size_t row_size = ggml_row_size(GGML_TYPE_Q2_KPT, p1.n_per_row); // Prepare levels buffer for this tensor q2kpt_free_levels(); - q2kpt_prepare_levels(nrows * tensor->ne[2], n_per_row); + q2kpt_prepare_levels(p1.nrows * tensor->ne[2], p1.n_per_row); // Quantize each expert slice const int64_t nelements_matrix = tensor->ne[0] * tensor->ne[1]; for (int64_t i03 = 0; i03 < tensor->ne[2]; ++i03) { - const float * f32_data_03 = f32_data + i03 * nelements_matrix; - void * q_data_03 = (char *)p1_qbuf.data() + row_size * i03 * nrows; - const float * imatrix_03 = imatrix ? imatrix + i03 * n_per_row : nullptr; + const float * f32_data_03 = p1.f32_data + i03 * nelements_matrix; + void * q_data_03 = (char *)p1_qbuf.data() + row_size * i03 * p1.nrows; + const float * imatrix_03 = p1.imatrix ? p1.imatrix + i03 * p1.n_per_row : nullptr; // start_row must be the absolute row index for correct levels indexing - ggml_quantize_chunk(GGML_TYPE_Q2_KPT, f32_data_03, q_data_03, i03 * nrows, nrows, n_per_row, imatrix_03); + ggml_quantize_chunk(GGML_TYPE_Q2_KPT, f32_data_03, q_data_03, i03 * p1.nrows, p1.nrows, p1.n_per_row, imatrix_03); } // Copy trained levels to our storage @@ -1669,60 +1559,19 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: const int64_t t_start_p1 = ggml_time_us(); LLAMA_LOG_INFO("%s: IQ2_TQ pass 1: training per-tensor grids...\n", __func__); - std::vector> p1_read_data; - std::vector> p1_f32_buf; - std::vector p1_workers; - p1_workers.reserve(nthread); - for (size_t ti = 0; ti < tensors.size(); ++ti) { ggml_tensor * tensor = tensors[ti]->tensor; - const std::string tname = ggml_get_name(tensor); - - bool quantize = tname.rfind("weight") == tname.size() - 6; - quantize &= (ggml_n_dims(tensor) >= 2); - quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && - tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; - quantize &= tname.find("_norm.weight") == std::string::npos; - quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; - if (!quantize) { continue; } - - ggml_type new_type = resolve_tensor_type(qs, params, tensor, default_type, ftype, tname); - if (new_type != GGML_TYPE_IQ2_TQ) { continue; } - - // Load tensor data - const size_t tsz = ggml_nbytes(tensor); - if (!ml.use_mmap) { - if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } - tensor->data = p1_read_data.data(); - } - ml.load_data_for(tensor); - - // Dequantize to f32 if needed - const int64_t nelements = ggml_nelements(tensor); - float * f32_data; - if (tensor->type == GGML_TYPE_F32) { - f32_data = (float *) tensor->data; - } else { - llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); - f32_data = (float *) p1_f32_buf.data(); - } - - // Resolve imatrix - std::vector imatrix_buf; - const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); - - const int64_t n_per_row = tensor->ne[0]; - const int64_t nrows = tensor->ne[1]; + if (!p1.prepare(tensor, GGML_TYPE_IQ2_TQ)) { continue; } LLAMA_LOG_INFO("%s: IQ2_TQ grid for [%zu/%zu] %s\n", __func__, ti+1, tensors.size(), tensor->name); iq2tq_meta meta; - meta.tensor_name = tname; - iq2tq_train_grid(f32_data, nrows, n_per_row, imatrix, meta.grid); + meta.tensor_name = p1.name; + iq2tq_train_grid(p1.f32_data, p1.nrows, p1.n_per_row, p1.imatrix, meta.grid); iq2tq_all_meta.push_back(meta); // Save to GGUF - std::string grid_key = "iq2tq.grid." + tname; + std::string grid_key = "iq2tq.grid." + p1.name; gguf_set_arr_data(ctx_outs[0].get(), grid_key.c_str(), GGUF_TYPE_INT8, meta.grid, 64); } const int64_t t_end_p1 = ggml_time_us(); @@ -1740,56 +1589,18 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: const int64_t t_start_p1 = ggml_time_us(); LLAMA_LOG_INFO("%s: IQ3_TQ pass 1: training per-tensor grids...\n", __func__); - std::vector> p1_read_data; - std::vector> p1_f32_buf; - std::vector p1_workers; - p1_workers.reserve(nthread); - for (size_t ti = 0; ti < tensors.size(); ++ti) { ggml_tensor * tensor = tensors[ti]->tensor; - const std::string tname = ggml_get_name(tensor); - - bool quantize = tname.rfind("weight") == tname.size() - 6; - quantize &= (ggml_n_dims(tensor) >= 2); - quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && - tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; - quantize &= tname.find("_norm.weight") == std::string::npos; - quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; - if (!quantize) { continue; } - - ggml_type new_type = resolve_tensor_type(qs, params, tensor, default_type, ftype, tname); - if (new_type != GGML_TYPE_IQ3_TQ) { continue; } - - const size_t tsz = ggml_nbytes(tensor); - if (!ml.use_mmap) { - if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } - tensor->data = p1_read_data.data(); - } - ml.load_data_for(tensor); - - const int64_t nelements = ggml_nelements(tensor); - float * f32_data; - if (tensor->type == GGML_TYPE_F32) { - f32_data = (float *) tensor->data; - } else { - llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); - f32_data = (float *) p1_f32_buf.data(); - } - - std::vector imatrix_buf; - const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); - - const int64_t n_per_row = tensor->ne[0]; - const int64_t nrows = tensor->ne[1]; + if (!p1.prepare(tensor, GGML_TYPE_IQ3_TQ)) { continue; } LLAMA_LOG_INFO("%s: IQ3_TQ grid for [%zu/%zu] %s\n", __func__, ti+1, tensors.size(), tensor->name); iq3tq_meta meta; - meta.tensor_name = tname; - iq3tq_train_grid(f32_data, nrows, n_per_row, imatrix, meta.grid); + meta.tensor_name = p1.name; + iq3tq_train_grid(p1.f32_data, p1.nrows, p1.n_per_row, p1.imatrix, meta.grid); iq3tq_all_meta.push_back(meta); - std::string grid_key = "iq3tq.grid." + tname; + std::string grid_key = "iq3tq.grid." + p1.name; gguf_set_arr_data(ctx_outs[0].get(), grid_key.c_str(), GGUF_TYPE_INT8, meta.grid, 128); } const int64_t t_end_p1 = ggml_time_us(); @@ -1807,66 +1618,18 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: const int64_t t_start_p1 = ggml_time_us(); LLAMA_LOG_INFO("%s: IQ1_BN pass 1: training per-tensor codebooks...\n", __func__); - std::vector> p1_read_data; - std::vector> p1_f32_buf; - std::vector p1_workers; - p1_workers.reserve(nthread); - for (size_t ti = 0; ti < tensors.size(); ++ti) { ggml_tensor * tensor = tensors[ti]->tensor; - const std::string tname = ggml_get_name(tensor); - - bool quantize = tname.rfind("weight") == tname.size() - 6; - quantize &= (ggml_n_dims(tensor) >= 2); - quantize &= tensor->type != GGML_TYPE_I8 && tensor->type != GGML_TYPE_I16 && - tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; - quantize &= tname.find("_norm.weight") == std::string::npos; - quantize &= tname.find("ffn_gate_inp.weight") == std::string::npos; - if (!quantize) { continue; } - - ggml_type new_type = default_type; - if (!params->pure) { - new_type = llama_tensor_get_type_impl(qs, new_type, tensor, ftype, tensor_get_category(tname)); - } - if (params->token_embedding_type < GGML_TYPE_COUNT && - (tname == "token_embd.weight" || tname == "per_layer_token_embd.weight")) { - new_type = params->token_embedding_type; - } - if (params->output_tensor_type < GGML_TYPE_COUNT && tname == "output.weight") { - new_type = params->output_tensor_type; - } - if (new_type != GGML_TYPE_IQ1_BN) { continue; } - - const size_t tsz = ggml_nbytes(tensor); - if (!ml.use_mmap) { - if (p1_read_data.size() < tsz) { p1_read_data.resize(tsz); } - tensor->data = p1_read_data.data(); - } - ml.load_data_for(tensor); - - const int64_t nelements = ggml_nelements(tensor); - float * f32_data; - if (tensor->type == GGML_TYPE_F32) { - f32_data = (float *) tensor->data; - } else { - llama_tensor_dequantize_impl(tensor, p1_f32_buf, p1_workers, nelements, nthread); - f32_data = (float *) p1_f32_buf.data(); - } - - std::vector imatrix_buf; - const float * imatrix = resolve_level_imatrix(imatrix_data, remap_imatrix(tensor->name, mapped), tensor, imatrix_buf); - - const int64_t n_per_row = tensor->ne[0]; - const int64_t nrows = tensor->ne[1]; + if (!p1.prepare(tensor, GGML_TYPE_IQ1_BN)) { continue; } LLAMA_LOG_INFO("%s: IQ1_BN codebook for [%zu/%zu] %s\n", __func__, ti+1, tensors.size(), tensor->name); iq1bn_meta meta; - meta.tensor_name = tname; - iq1bn_train_codebook(f32_data, nrows, n_per_row, imatrix, meta.aux, nthread); + meta.tensor_name = p1.name; + iq1bn_train_codebook(p1.f32_data, p1.nrows, p1.n_per_row, p1.imatrix, meta.aux, nthread); iq1bn_all_meta.push_back(meta); - std::string aux_key = "iq1bn.aux." + tname; + std::string aux_key = "iq1bn.aux." + p1.name; gguf_set_arr_data(ctx_outs[0].get(), aux_key.c_str(), GGUF_TYPE_INT8, meta.aux, 32768); } const int64_t t_end_p1 = ggml_time_us(); From fd213c69a9bcf17a8ffcfbf4e4fc437e5268085e Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Fri, 3 Jul 2026 16:22:12 +0200 Subject: [PATCH 15/19] quant: don't quantize DeepSeek4 position-embedding (APE) tensors blk.N.attn_compressor_ape / indexer_compressor_ape are absolute position embedding tables, added as a positional bias via get_rows/add rather than used as matmul weights, so the imatrix carries no data for them ("did not find weights ..."). At a very low-bit target that requires imatrix this would then abort with "Missing importance matrix". They are tiny position tables, so skip them in quantization (both the pass-2 gate and the pass-1 trainer gate), mirroring the existing .position_embd handling. They pass through at their original precision. Assisted-By: Claude Opus 4.8 --- src/llama-quant.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index f5206ebdd31c..697a83bc512e 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -445,6 +445,10 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param // do not quantize relative position bias (T5) quantize &= name.find("attn_rel_b.weight") == std::string::npos; + // do not quantize DeepSeek4 absolute position embeddings (attn/indexer compressor APE): + // these are position tables added via get_rows, not matmul weights, so they carry no imatrix + quantize &= name.find("compressor_ape") == std::string::npos; + // do not quantize specific multimodal tensors quantize &= name.find(".position_embd") == std::string::npos; quantize &= name.find("sam.pos_embd") == std::string::npos; @@ -1114,6 +1118,7 @@ struct pass1_setup { tensor->type != GGML_TYPE_I32 && tensor->type != GGML_TYPE_I64; quantize &= name.find("_norm.weight") == std::string::npos; quantize &= name.find("ffn_gate_inp.weight") == std::string::npos; + quantize &= name.find("compressor_ape") == std::string::npos; // position embeddings, not weights if (!quantize) { return false; } From d5161cc200049afd47fca1af58d30a184014a073 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Mon, 6 Jul 2026 10:20:50 +0200 Subject: [PATCH 16/19] fixes to auto-type --- src/llama-quant.cpp | 5 + tools/auto-tensor-type/auto-tensor-type.cpp | 466 ++++++++++++++++---- 2 files changed, 391 insertions(+), 80 deletions(-) diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 697a83bc512e..3ceeadc3bffa 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -449,6 +449,10 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param // these are position tables added via get_rows, not matmul weights, so they carry no imatrix quantize &= name.find("compressor_ape") == std::string::npos; + // do not quantize the DeepSeek4 sparse-attention indexer projection: it selects which + // tokens attention attends to, has no redundancy, and is highly sensitive - keep it native + quantize &= name.find("indexer.proj") == std::string::npos; + // do not quantize specific multimodal tensors quantize &= name.find(".position_embd") == std::string::npos; quantize &= name.find("sam.pos_embd") == std::string::npos; @@ -1119,6 +1123,7 @@ struct pass1_setup { quantize &= name.find("_norm.weight") == std::string::npos; quantize &= name.find("ffn_gate_inp.weight") == std::string::npos; quantize &= name.find("compressor_ape") == std::string::npos; // position embeddings, not weights + quantize &= name.find("indexer.proj") == std::string::npos; // sparse-attn selector, keep native if (!quantize) { return false; } diff --git a/tools/auto-tensor-type/auto-tensor-type.cpp b/tools/auto-tensor-type/auto-tensor-type.cpp index 5b022d35b0f3..14f3849f1698 100644 --- a/tools/auto-tensor-type/auto-tensor-type.cpp +++ b/tools/auto-tensor-type/auto-tensor-type.cpp @@ -140,14 +140,26 @@ struct mul_mat_capture { // Weight tensor metadata (needed to read data from file later) ggml_type weight_type; int64_t weight_ne0, weight_ne1; // ne0=input features, ne1=output features + int64_t weight_ne2 = 1; // n_expert (>1 only for MUL_MAT_ID experts) - // Captured MUL_MAT input (src[1]) — always F32 after the kernel + // True if this capture is a GGML_OP_MUL_MAT_ID (MoE routed experts). The + // replay path then uses ggml_mul_mat_id with the captured input + ids. + bool is_id = false; + + // Captured MUL_MAT input (src[1]) — always F32 after the kernel. + // For MUL_MAT_ID this is 3D: [ne0=weight_ne0, ne1 (1 or n_expert_used), ne2=n_tokens]. std::vector input_data; - int64_t input_ne0, input_ne1; // [ne0=weight_ne0, ne1=n_tokens] + int64_t input_ne0, input_ne1; + int64_t input_ne2 = 1; + + // Expert-selection ids (src[2]) for MUL_MAT_ID — I32, shape [n_expert_used, n_tokens]. + std::vector ids_data; + int64_t ids_ne0 = 0, ids_ne1 = 0; - // Captured reference MUL_MAT output — always F32 + // Captured reference MUL_MAT output — always F32. Flattened to rows of ref_ne0: + // ref_ne1 = total_elements / ref_ne0 (covers the 3D MUL_MAT_ID output too). std::vector ref_output_data; - int64_t ref_ne0, ref_ne1; // [ne0=weight_ne1, ne1=n_tokens] + int64_t ref_ne0, ref_ne1; }; // Cost of assigning a specific quant type to a (role, bucket) @@ -196,10 +208,11 @@ static int compute_bucket(int pos_in_class, int n_in_class, int n_buckets) { static std::string extract_role(const std::string & name) { // "blk.0.attn_q.weight" -> "attn_q" // "blk.0.ffn_gate.weight" -> "ffn_gate" + // "blk.0.indexer.proj.weight" -> "indexer.proj" (role portion may contain dots) // "output.weight" -> "output" // "token_embd.weight" -> "token_embd" - static const std::regex layer_re("blk\\.\\d+\\.([^.]+)\\.weight"); - static const std::regex global_re("([^.]+)\\.weight"); + static const std::regex layer_re("blk\\.\\d+\\.(.+)\\.weight"); + static const std::regex global_re("(.+)\\.weight"); std::smatch m; if (std::regex_match(name, m, layer_re) && m.size() > 1) { @@ -220,9 +233,25 @@ static bool is_matrix_weight(const tensor_info & ti) { return true; } +// 3D stacked-expert weight (MoE routed experts): one 2D matrix per expert along +// ne[2], e.g. blk.N.ffn_down_exps.weight [n_in, n_out, n_expert]. These flow +// through GGML_OP_MUL_MAT_ID (not MUL_MAT) and are usually the bulk of a MoE +// model's parameters, so they must be measured and optimized like any other +// weight — not booked as fixed-precision overhead. +static bool is_expert_weight(const tensor_info & ti) { + if (ti.name.size() < 8 || ti.name.substr(ti.name.size() - 7) != ".weight") return false; + if (ti.ne[3] != 1) return false; + if (ti.ne[2] <= 1) return false; // must have >1 expert stacked along ne[2] + if (ti.ne[0] <= 1 || ti.ne[1] <= 1) return false; + return true; +} + static bool is_quantizable_weight(const tensor_info & ti, int64_t min_elements) { - // Must be a 2D matrix weight… - if (!is_matrix_weight(ti)) return false; + // Must be a 2D matrix weight or a 3D stacked-expert weight… + if (!is_matrix_weight(ti) && !is_expert_weight(ti)) return false; + // The DeepSeek4 sparse-attention indexer projection is kept native by + // llama-quantize (too sensitive to quantize), so don't measure or emit it. + if (ti.name.find("indexer.proj") != std::string::npos) return false; // …and large enough to justify per-role KLD measurement. if ((int64_t)ti.n_elements < min_elements) return false; return true; @@ -542,15 +571,43 @@ struct capture_state { int captured = 0; }; +// Copy a tensor's logical (de-strided) contents into `dst`, which must hold +// ggml_nelements(t) elements. Captured src tensors may be non-contiguous views: +// e.g. ggml_argsort_top_k slices the expert ids to [n_expert_used, n_tokens] but +// keeps the parent argsort's row stride (n_expert), so a flat +// ggml_backend_tensor_get would read interleaved garbage. We gather row-by-row +// honoring nb[] so the destination is tightly packed [ne0, ne1, ne2, ne3]. +// Assumes the innermost dim is contiguous (nb[0] == element size), which holds +// for every tensor this tool captures (MUL_MAT/MUL_MAT_ID inputs, ids, outputs). +static void capture_tensor_get(const ggml_tensor * t, void * dst) { + if (ggml_is_contiguous(t)) { + ggml_backend_tensor_get(t, dst, 0, ggml_nbytes(t)); + return; + } + const size_t row_bytes = (size_t) t->ne[0] * ggml_type_size(t->type); + char * d = (char *) dst; + for (int64_t i3 = 0; i3 < t->ne[3]; i3++) { + for (int64_t i2 = 0; i2 < t->ne[2]; i2++) { + for (int64_t i1 = 0; i1 < t->ne[1]; i1++) { + size_t off = i1 * t->nb[1] + i2 * t->nb[2] + i3 * t->nb[3]; + ggml_backend_tensor_get(t, d, off, row_bytes); + d += row_bytes; + } + } + } +} + static bool capture_callback(ggml_tensor * t, bool ask, void * user_data) { auto * state = (capture_state *) user_data; - if (t->op != GGML_OP_MUL_MAT) return false; + if (t->op != GGML_OP_MUL_MAT && t->op != GGML_OP_MUL_MAT_ID) return false; if (!t->src[0] || !t->src[1]) return false; + const bool is_id = (t->op == GGML_OP_MUL_MAT_ID); + if (is_id && !t->src[2]) return false; const char * weight_name = t->src[0]->name; - // Check if this MUL_MAT uses one of our target weight tensors + // Check if this MUL_MAT(_ID) uses one of our target weight tensors if (state->target_weight_names.find(weight_name) == state->target_weight_names.end()) { return false; // Not interested } @@ -567,45 +624,39 @@ static bool capture_callback(ggml_tensor * t, bool ask, void * user_data) { cap.weight_type = t->src[0]->type; cap.weight_ne0 = t->src[0]->ne[0]; cap.weight_ne1 = t->src[0]->ne[1]; + cap.weight_ne2 = t->src[0]->ne[2]; // n_expert (1 for plain MUL_MAT) + cap.is_id = is_id; const int cap_bucket = state->weight_to_bucket.count(weight_name) ? state->weight_to_bucket[weight_name] : (cap.layer < 0 ? -1 : 0); - // Capture input (src[1]) + // Capture input (src[1]) — may be a non-contiguous view, so de-stride. { - size_t nbytes = ggml_nbytes(t->src[1]); - const float * src_ptr = nullptr; - std::vector host_copy; - - if (ggml_backend_buffer_is_host(t->src[1]->buffer)) { - src_ptr = (const float *) t->src[1]->data; - } else { - host_copy.resize(nbytes / sizeof(float)); - ggml_backend_tensor_get(t->src[1], host_copy.data(), 0, nbytes); - src_ptr = host_copy.data(); - } - cap.input_ne0 = t->src[1]->ne[0]; cap.input_ne1 = t->src[1]->ne[1]; - cap.input_data.assign(src_ptr, src_ptr + (nbytes / sizeof(float))); + cap.input_ne2 = t->src[1]->ne[2]; + cap.input_data.resize(ggml_nelements(t->src[1])); + capture_tensor_get(t->src[1], cap.input_data.data()); } - // Capture reference output - { - size_t nbytes = ggml_nbytes(t); - const float * src_ptr = nullptr; - std::vector host_copy; - - if (ggml_backend_buffer_is_host(t->buffer)) { - src_ptr = (const float *) t->data; - } else { - host_copy.resize(nbytes / sizeof(float)); - ggml_backend_tensor_get(t, host_copy.data(), 0, nbytes); - src_ptr = host_copy.data(); - } + // Capture expert-selection ids (src[2]) for MUL_MAT_ID — always I32. + // selected_experts (ggml_argsort_top_k) is a sliced view with the parent's + // row stride, so de-striding here is essential to capture the real routing. + if (is_id) { + ggml_tensor * idt = t->src[2]; + cap.ids_ne0 = idt->ne[0]; + cap.ids_ne1 = idt->ne[1]; + cap.ids_data.resize(ggml_nelements(idt)); + capture_tensor_get(idt, cap.ids_data.data()); + } + // Capture reference output. For MUL_MAT_ID the output is 3D + // [n_out, n_expert_used, n_tokens]; flatten everything past ne0 into rows so + // compute_avg_kld measures per-(output-row) relative-L2 uniformly. + { cap.ref_ne0 = t->ne[0]; - cap.ref_ne1 = t->ne[1]; - cap.ref_output_data.assign(src_ptr, src_ptr + (nbytes / sizeof(float))); + cap.ref_ne1 = ggml_nelements(t) / t->ne[0]; + cap.ref_output_data.resize(ggml_nelements(t)); + capture_tensor_get(t, cap.ref_output_data.data()); } state->captures_by_role[make_rb_key(cap.role, cap_bucket)].push_back(std::move(cap)); @@ -696,13 +747,43 @@ struct quant_result { }; // Train per-tensor params and quantize a weight tensor to the target type. +// +// n_expert > 1 selects the stacked-expert path (MUL_MAT_ID weights): the tensor +// is [n_per_row, nrows, n_expert] and each expert slice is quantized +// independently — matching llama-quantize's per-expert loop (src/llama-quant.cpp). +// imatrix_per_expert indicates the imatrix holds one n_per_row row per expert +// (length n_per_row*n_expert); otherwise the same imatrix row is reused for all +// experts. Per-tensor-trained quant types are not supported for experts and +// return an empty result (signals "skip this qtype for this role"). static quant_result quantize_weight_to_type( ggml_type target_type, const float * f32_data, int64_t nrows, int64_t n_per_row, - const float * imatrix) { + const float * imatrix, + int64_t n_expert = 1, + bool imatrix_per_expert = false) { quant_result result; + + if (n_expert > 1) { + if (requires_training(target_type)) { + // Per-tensor-trained types carry per-tensor grids/levels that the + // MUL_MAT_ID replay path can't thread per-expert. Leave empty. + return result; + } + const size_t row_size = ggml_row_size(target_type, n_per_row); + result.data.resize(row_size * (size_t)nrows * (size_t)n_expert); + ggml_quantize_init(target_type); + const int64_t nelements_matrix = nrows * n_per_row; + for (int64_t i03 = 0; i03 < n_expert; ++i03) { + const float * imat03 = + imatrix ? (imatrix_per_expert ? imatrix + i03 * n_per_row : imatrix) : nullptr; + ggml_quantize_chunk(target_type, f32_data, result.data.data(), + i03 * nelements_matrix, nrows, n_per_row, imat03); + } + return result; + } + size_t quant_size = nrows * ggml_row_size(target_type, n_per_row); result.data.resize(quant_size); @@ -843,6 +924,72 @@ static bool eval_mul_mat(ggml_type weight_type, const void * weight_data, return true; } +// Run MUL_MAT_ID (MoE routed experts): result = mul_mat_id(weights, input, ids) +// weights: [w_ne0, w_ne1, n_expert] in a quantized type (all experts stacked) +// input: [w_ne0, in_ne1, n_tokens] in F32 (in_ne1 is 1 or n_expert_used) +// ids: [n_expert_used, n_tokens] in I32 +// result: [w_ne1, n_expert_used, n_tokens] in F32 +// Mirrors eval_mul_mat but builds the 3-input MUL_MAT_ID node. +static bool eval_mul_mat_id(ggml_type weight_type, const void * weight_data, + int64_t w_ne0, int64_t w_ne1, int64_t w_ne2, + const float * input_data, int64_t in_ne0, int64_t in_ne1, int64_t in_ne2, + const int32_t * ids_data, int64_t ids_ne0, int64_t ids_ne1, + std::vector & result_data, + ggml_backend_t backend) { + size_t weight_bytes = ggml_row_size(weight_type, w_ne0) * (size_t)w_ne1 * (size_t)w_ne2; + size_t input_bytes = (size_t)in_ne0 * in_ne1 * in_ne2 * sizeof(float); + size_t ids_bytes = (size_t)ids_ne0 * ids_ne1 * sizeof(int32_t); + + size_t ctx_size = ggml_tensor_overhead() * 16 + ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE, false) + 4096; + struct ggml_init_params params = {(size_t) ctx_size, NULL, /*.no_alloc =*/ true}; + struct ggml_context * ctx = ggml_init(params); + if (!ctx) { + LOG_ERR("Failed to create ggml context for MUL_MAT_ID eval\n"); + return false; + } + + struct ggml_tensor * w = ggml_new_tensor_3d(ctx, weight_type, w_ne0, w_ne1, w_ne2); + struct ggml_tensor * x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, in_ne0, in_ne1, in_ne2); + struct ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, ids_ne0, ids_ne1); + struct ggml_tensor * result = ggml_mul_mat_id(ctx, w, x, ids); + + ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx, backend); + if (!buf) { + LOG_ERR("Failed to allocate tensors for MUL_MAT_ID eval (weight %s [%lld,%lld,%lld])\n", + ggml_type_name(weight_type), (long long)w_ne0, (long long)w_ne1, (long long)w_ne2); + ggml_free(ctx); + return false; + } + if (!w->buffer || !x->buffer || !ids->buffer || !result->buffer) { + LOG_ERR("Tensor buffers not set after allocation (MUL_MAT_ID)\n"); + ggml_backend_buffer_free(buf); + ggml_free(ctx); + return false; + } + + ggml_backend_tensor_set(w, weight_data, 0, weight_bytes); + ggml_backend_tensor_set(x, input_data, 0, input_bytes); + ggml_backend_tensor_set(ids, ids_data, 0, ids_bytes); + + struct ggml_cgraph * graph = ggml_new_graph_custom(ctx, GGML_DEFAULT_GRAPH_SIZE, false); + ggml_build_forward_expand(graph, result); + + enum ggml_status status = ggml_backend_graph_compute(backend, graph); + if (status != GGML_STATUS_SUCCESS) { + LOG_ERR("ggml_backend_graph_compute failed (MUL_MAT_ID): %d\n", status); + ggml_backend_buffer_free(buf); + ggml_free(ctx); + return false; + } + + result_data.resize(ggml_nelements(result)); + ggml_backend_tensor_get(result, result_data.data(), 0, ggml_nbytes(result)); + + ggml_backend_buffer_free(buf); + ggml_free(ctx); + return true; +} + // Build the cost matrix: for each (role, quant_type), compute average KLD // Quant types are evaluated in parallel per capture (each type has its own // global state so they don't interfere with each other). @@ -863,35 +1010,90 @@ static std::map> build_cost_matrix( const int n_parallel = std::max(1, cfg.n_threads); - // Create a pool of backends — try CUDA first, fall back to CPU - // Each thread gets its own backend to avoid contention (VMM pool requires LIFO alloc/free). + // Create a pool of backends — spread across all GPUs with free memory, else CPU. + // Each concurrent eval needs its own backend (the CUDA VMM pool is a per-backend + // LIFO stack allocator), so the pool must hold at least as many backends as the + // peak eval concurrency. MUL_MAT_ID expert weights are large (all experts stacked, + // ~200-300 MiB each), and each backend's CUDA pool caches freed blocks for the + // duration of Phase 3 — so piling every backend onto device 0 exhausts it. Round- + // robin the backends over every GPU that has free headroom instead. struct backend_pool { std::vector backends; std::atomic next_idx{0}; std::string backend_name; backend_pool(int count) { - // Initialize first backend to determine type - ggml_backend_t be = ggml_backend_init_best(); - if (!be) { - LOG_ERR("Failed to init any backend\n"); - return; + // Determine the preferred backend registry (e.g. CUDA) from the best + // device, so we don't mix backend types or double-count a physical GPU + // that is also exposed through another backend (e.g. the same cards + // showing up as both CUDA* and Vulkan*). + std::string pref_reg; + { + ggml_backend_t probe = ggml_backend_init_best(); + if (probe) { + ggml_backend_dev_t d = ggml_backend_get_device(probe); + ggml_backend_reg_t r = d ? ggml_backend_dev_backend_reg(d) : nullptr; + if (r) pref_reg = ggml_backend_reg_name(r); + ggml_backend_free(probe); + } } - backend_name = ggml_backend_name(be); - backends.push_back(be); - - // Create separate backends for remaining threads - // This is required because the CUDA VMM pool is a stack allocator - // that requires strict LIFO allocation/deallocation order per pool. - for (int i = 1; i < count; i++) { - ggml_backend_t be_i = ggml_backend_init_best(); - if (be_i) { - backends.push_back(be_i); - } else { - LOG_WRN("Failed to init backend %d, reusing backend 0\n", i); - backends.push_back(be); // Fallback to sharing + + // Collect GPU devices (of the preferred registry) with enough free + // memory to hold a few expert weights. + std::vector gpu_devs; + const size_t min_free = (size_t) 1024 * 1024 * 1024; // 1 GiB headroom + for (size_t i = 0; i < ggml_backend_dev_count(); i++) { + ggml_backend_dev_t dev = ggml_backend_dev_get(i); + if (ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) continue; + if (!pref_reg.empty()) { + ggml_backend_reg_t r = ggml_backend_dev_backend_reg(dev); + if (!r || pref_reg != ggml_backend_reg_name(r)) continue; + } + size_t free = 0, total = 0; + ggml_backend_dev_memory(dev, &free, &total); + if (free < min_free) { + LOG_WRN("Phase 3: skipping %s (%.0f MiB free < 1 GiB)\n", + ggml_backend_dev_name(dev), free / 1048576.0); + continue; + } + gpu_devs.push_back(dev); + } + + if (!gpu_devs.empty()) { + std::string devs; + for (auto d : gpu_devs) { + if (!devs.empty()) devs += ", "; + devs += ggml_backend_dev_name(d); + } + LOG("Phase 3: %zu GPU(s) with free memory [%s]\n", gpu_devs.size(), devs.c_str()); + for (int i = 0; i < count; i++) { + ggml_backend_dev_t dev = gpu_devs[i % gpu_devs.size()]; + ggml_backend_t be = ggml_backend_dev_init(dev, nullptr); + if (be) { + backends.push_back(be); + } else if (!backends.empty()) { + LOG_WRN("Failed to init backend %d on %s, sharing backend 0\n", + i, ggml_backend_dev_name(dev)); + backends.push_back(backends[0]); + } } } + + if (backends.empty()) { + // No eligible GPU — fall back to best available (typically CPU). + ggml_backend_t be = ggml_backend_init_best(); + if (!be) { + LOG_ERR("Failed to init any backend\n"); + return; + } + backends.push_back(be); + for (int i = 1; i < count; i++) { + ggml_backend_t be_i = ggml_backend_init_best(); + backends.push_back(be_i ? be_i : be); + } + } + + backend_name = ggml_backend_name(backends[0]); } ~backend_pool() { // Free all unique backends @@ -907,9 +1109,13 @@ static std::map> build_cost_matrix( } }; - backend_pool pool(n_parallel); - LOG("Phase 3 backend: %s (%d parallel)\n", - pool.backend_name.c_str(), n_parallel); + // The eval/quantize inner loops never run more than n_types tasks at once + // (they stop at ti < n_types), so the pool only needs that many backends — + // creating one per CPU thread would just spread idle CUDA pools over every GPU. + const int pool_size = std::max(1, std::min(n_parallel, (int) cfg.quant_types.size())); + backend_pool pool(pool_size); + LOG("Phase 3 backend: %s (%d eval backends, %d parallel quantize threads)\n", + pool.backend_name.c_str(), pool_size, n_parallel); // For each role that has captures for (auto & [role, captures] : captures_by_role) { @@ -935,13 +1141,81 @@ static std::map> build_cost_matrix( } const mul_mat_capture & first_cap = *caps_for_weight.front(); - auto imat = get_imatrix_for_tensor(imatrix_data, weight_name, first_cap.weight_ne0); + const bool is_id = first_cap.is_id; + const int64_t n_expert = is_id ? first_cap.weight_ne2 : 1; + + // Resolve the imatrix. Plain weights want one n_per_row row; expert + // weights may carry one row per expert (length n_per_row*n_expert) — + // if so, quantize each expert with its own row, else broadcast a + // uniform row to all experts. + std::vector imat; + bool imat_per_expert = false; + if (is_id) { + auto it = imatrix_data.find(weight_name); + if (it != imatrix_data.end() && + (int64_t)it->second.size() >= first_cap.weight_ne0 * n_expert) { + imat = it->second; + imat_per_expert = true; + } else { + imat.assign(first_cap.weight_ne0, 1.0f); + } + } else { + imat = get_imatrix_for_tensor(imatrix_data, weight_name, first_cap.weight_ne0); + } - // Quantize each qtype once for this weight (parallel across qtypes). - // Different qtypes use disjoint per-type globals, so parallel training is safe. + // Quantize each qtype once for this weight; the result is reused across + // all of this weight's captures in the eval loop below. std::unordered_map quant_cache; std::mutex quant_cache_mutex; - { + if (is_id && n_expert > 1) { + // Expert (MUL_MAT_ID) weights dominate cost and are huge: quantizing all + // n_expert slices for one qtype serially, while only parallelizing across + // the ~11 qtypes, leaves most cores idle. Expert slices never use the + // trained-quant path (quantize_weight_to_type bails on requires_training), + // so this is pure per-slice ggml_quantize_chunk — fan out over the full + // (qtype x expert) grid so all n_parallel threads are used. + const int64_t nrows = first_cap.weight_ne1; + const int64_t n_per_row = first_cap.weight_ne0; + const int64_t nel_matrix = nrows * n_per_row; + + // Pre-size each non-trained qtype's buffer and init its tables once, + // serially (ggml_quantize_init is not safe to call concurrently; the + // grids it fills are read-only during quantization). + std::vector etypes; + std::vector out; // resolved dst pointers (no map lookup in workers) + for (ggml_type qt : cfg.quant_types) { + if (requires_training(qt)) continue; // unsupported for experts (forbidden sentinel) + const size_t row_size = ggml_row_size(qt, n_per_row); + quant_result qr; + qr.data.resize(row_size * (size_t) nrows * (size_t) n_expert); + ggml_quantize_init(qt); + auto res = quant_cache.emplace(qt, std::move(qr)); + etypes.push_back(qt); + out.push_back(res.first->second.data.data()); + } + + // Flat task pool over (qtype, expert): workers pull from an atomic counter. + // Each task writes a disjoint slice, so no locking is needed. + const int64_t n_tasks = (int64_t) etypes.size() * n_expert; + std::atomic next{0}; + auto worker = [&]() { + for (int64_t k = next.fetch_add(1); k < n_tasks; k = next.fetch_add(1)) { + const int64_t ci = k / n_expert; + const int64_t e = k % n_expert; + const float * imat_e = imat.empty() ? nullptr + : (imat_per_expert ? imat.data() + e * n_per_row : imat.data()); + ggml_quantize_chunk(etypes[ci], weight_f32.data(), out[ci], + e * nel_matrix, nrows, n_per_row, imat_e); + } + }; + std::vector> ws; + ws.reserve(n_parallel); + for (int p = 0; p < n_parallel; p++) ws.push_back(std::async(std::launch::async, worker)); + for (auto & f : ws) f.get(); + } else { + // Plain weights: parallelize across qtypes. Different qtypes use disjoint + // per-type trained globals, so cross-type parallelism is safe; a single + // plain matrix is small enough that per-qtype parallelism suffices. std::vector> qfutures; qfutures.reserve(n_parallel); for (size_t ti = 0; ti < n_types; /* advanced inside */) { @@ -949,10 +1223,16 @@ static std::map> build_cost_matrix( for (int p = 0; p < n_parallel && ti < n_types; p++, ti++) { ggml_type qtype = cfg.quant_types[ti]; qfutures.push_back(std::async(std::launch::async, - [&first_cap, &weight_f32, &imat, qtype, &quant_cache, &quant_cache_mutex]() { + [&first_cap, &weight_f32, &imat, qtype, n_expert, imat_per_expert, + &quant_cache, &quant_cache_mutex]() { auto qres = quantize_weight_to_type( qtype, weight_f32.data(), - first_cap.weight_ne1, first_cap.weight_ne0, imat.data()); + first_cap.weight_ne1, first_cap.weight_ne0, imat.data(), + n_expert, imat_per_expert); + // Empty data = qtype unsupported for this weight (e.g. a + // trained quant on experts); leave it out of the cache so + // the cost entry falls back to the forbidden sentinel. + if (qres.data.empty()) return; std::lock_guard lock(quant_cache_mutex); quant_cache.emplace(qtype, std::move(qres)); })); @@ -982,12 +1262,21 @@ static std::map> build_cost_matrix( futures.push_back(std::async(std::launch::async, [&cap, qtype, qres, backend]() -> std::pair { std::vector quant_output; - if (!eval_mul_mat(qtype, qres->data.data(), - cap.weight_ne0, cap.weight_ne1, - cap.input_data.data(), cap.input_ne0, cap.input_ne1, - qres->levels, - quant_output, - backend)) { + bool ok; + if (cap.is_id) { + ok = eval_mul_mat_id(qtype, qres->data.data(), + cap.weight_ne0, cap.weight_ne1, cap.weight_ne2, + cap.input_data.data(), cap.input_ne0, cap.input_ne1, cap.input_ne2, + cap.ids_data.data(), cap.ids_ne0, cap.ids_ne1, + quant_output, backend); + } else { + ok = eval_mul_mat(qtype, qres->data.data(), + cap.weight_ne0, cap.weight_ne1, + cap.input_data.data(), cap.input_ne0, cap.input_ne1, + qres->levels, + quant_output, backend); + } + if (!ok) { return {qtype, std::numeric_limits::quiet_NaN()}; } return {qtype, compute_avg_kld(cap.ref_output_data.data(), quant_output.data(), @@ -1722,7 +2011,7 @@ static bool save_cost_matrix_cache( LOG_ERR("Failed to open cost-matrix cache for writing: %s\n", path.c_str()); return false; } - out << "# auto-tensor-type cost matrix cache v1\n"; + out << "# auto-tensor-type cost matrix cache v2\n"; out << "model_path " << cfg.model_path << "\n"; out << "model_size " << file_size_or_neg(cfg.model_path) << "\n"; out << "model_mtime " << file_mtime_or_neg(cfg.model_path) << "\n"; @@ -1765,7 +2054,7 @@ static bool load_cost_matrix_cache( if (!in) return false; std::string line; - if (!std::getline(in, line) || line != "# auto-tensor-type cost matrix cache v1") { + if (!std::getline(in, line) || line != "# auto-tensor-type cost matrix cache v2") { LOG_WRN("Cost-matrix cache %s: bad header; ignoring\n", path.c_str()); return false; } @@ -1920,7 +2209,7 @@ int main(int argc, char ** argv) { quantizable.push_back(ti); } } - LOG("Found %zu quantizable weight tensors (>= %lld elements, 2D)\n", + LOG("Found %zu quantizable weight tensors (>= %lld elements, 2D or 3D-expert)\n", quantizable.size(), (long long)cfg.min_elements); // Get unique roles @@ -2339,17 +2628,34 @@ int main(int argc, char ** argv) { } } + // token_embd is a get_rows lookup table with no imatrix and is highly sensitive; + // never quantize it below Q5_K, regardless of the target bpw or --quants list. + if (compute_bpw(token_embd_type) < compute_bpw(GGML_TYPE_Q5_K)) { + LOG("token_embd type %s (%.4f bpw) is below Q5_K; flooring to Q5_K\n", + ggml_type_name(token_embd_type), compute_bpw(token_embd_type)); + token_embd_type = GGML_TYPE_Q5_K; + } + // Add global tensors (bucket -1) to the cost matrix with their fixed types. // KLD=0 since they're not optimized — we just need their BPW in the budget. + // token_embd_type may have been floored to a type outside cfg.quant_types + // (Q5_K), so make sure it is always a selectable option for that tensor. const std::string key_tok_embd = make_rb_key("token_embd", -1); const std::string key_output = make_rb_key("output", -1); - for (ggml_type qtype : cfg.quant_types) { + std::vector tok_embd_types = cfg.quant_types; + if (std::find(tok_embd_types.begin(), tok_embd_types.end(), token_embd_type) == tok_embd_types.end()) { + tok_embd_types.push_back(token_embd_type); + } + for (ggml_type qtype : tok_embd_types) { cost_entry e; e.kld = (qtype == token_embd_type) ? 0.0 : 1e30; e.bpw = compute_bpw(qtype); cost_matrix[key_tok_embd][qtype] = e; - + } + for (ggml_type qtype : cfg.quant_types) { + cost_entry e; e.kld = (qtype == output_type) ? 0.0 : 1e30; + e.bpw = compute_bpw(qtype); cost_matrix[key_output][qtype] = e; } LOG("Global tensors: token_embd=%s (%.4f bpw), output=%s (%.4f bpw)\n", From f3ce9b7e4112304229f2109b2298b9843c2c105a Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Mon, 6 Jul 2026 11:04:37 +0200 Subject: [PATCH 17/19] ggml : fix cross-platform build errors for learned-levels vec_dot - arm: forward the new `levels` arg in the q1_0 generic fallback call - arch-fallback: add missing q4_dpt/q2_dpt generic renames for loongarch (no native impl -> undefined symbol), and drop the erroneous q3_pt rename for wasm (native impl present -> duplicate symbol) - test-quantize-fns: pass the new `levels` arg (nullptr) to f32 vec_dot Assisted-By: Claude Opus 4.8 (1M context) --- ggml/src/ggml-cpu/arch-fallback.h | 3 ++- ggml/src/ggml-cpu/arch/arm/quants.c | 2 +- tests/test-quantize-fns.cpp | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index b14c757c3eae..15012822394a 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -170,6 +170,8 @@ #define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q1_0_q8_0_generic ggml_vec_dot_q1_0_q8_0 +#define ggml_vec_dot_q4_dpt_q8_0_generic ggml_vec_dot_q4_dpt_q8_0 +#define ggml_vec_dot_q2_dpt_q8_0_generic ggml_vec_dot_q2_dpt_q8_0 // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 #define ggml_quantize_mat_q8_0_4x8_generic ggml_quantize_mat_q8_0_4x8 @@ -312,7 +314,6 @@ #define ggml_vec_dot_iq1_m_q8_K_generic ggml_vec_dot_iq1_m_q8_K #define ggml_vec_dot_iq4_nl_q8_0_generic ggml_vec_dot_iq4_nl_q8_0 #define ggml_vec_dot_iq4_xs_q8_K_generic ggml_vec_dot_iq4_xs_q8_K -#define ggml_vec_dot_q3_pt_q8_K_generic ggml_vec_dot_q3_pt_q8_K #define ggml_vec_dot_q4_dpt_q8_0_generic ggml_vec_dot_q4_dpt_q8_0 #define ggml_vec_dot_q2_dpt_q8_0_generic ggml_vec_dot_q2_dpt_q8_0 #define ggml_vec_dot_mxfp4_q8_0_generic ggml_vec_dot_mxfp4_q8_0 diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c index 9c2c74ba50f0..4d0106b9fb25 100644 --- a/ggml/src/ggml-cpu/arch/arm/quants.c +++ b/ggml/src/ggml-cpu/arch/arm/quants.c @@ -319,7 +319,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(nb); UNUSED(x); UNUSED(y); - ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc); + ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); #endif } diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 4bda0d4860d2..82a80ab81538 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -112,7 +112,7 @@ static int test_vec_dot_f32(bool verbose) { generate_data(1.0, n, b.data()); float result = 0.0f; - f32->vec_dot(n, &result, 0, a.data(), 0, b.data(), 0, 1); + f32->vec_dot(n, &result, 0, a.data(), 0, b.data(), 0, 1, nullptr); const float ref = dot_product(a.data(), b.data(), n); const float error = fabsf(result - ref) / n; From f305c2db2c49339bf82a6d6517aa1ee39befefc2 Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Mon, 6 Jul 2026 11:15:15 +0200 Subject: [PATCH 18/19] ggml-quants : make IQ1_BN k-means threading build on Windows ggml-quants.c is part of ggml-base and is compiled on all platforms, but the IQ1_BN k-means code used raw , which does not exist on Windows/MSVC (fatal error: 'pthread.h' file not found on the arm64/x64 windows-llvm builds). Add the same thin Win32 pthread shim ggml-cpu.c uses (HANDLE-based pthread_create/pthread_join, DWORD thread_ret_t) behind an #if _WIN32 guard, and give the k-means workers the portable thread_ret_t return type. POSIX keeps using pthreads unchanged. Assisted-By: Claude Opus 4.8 (1M context) --- ggml/src/ggml-quants.c | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index d8324073702d..5ac25e1e3894 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -7901,7 +7901,38 @@ size_t quantize_iq1_bn(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst } // Thread work structs for parallel K-means +// Cross-platform threading shim: use pthreads on POSIX and a thin Win32 +// wrapper elsewhere (mirrors the shim in ggml-cpu.c) so this file also +// builds on Windows, where is unavailable. +#if defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +typedef HANDLE pthread_t; +typedef DWORD thread_ret_t; +static int pthread_create(pthread_t * out, void * unused, thread_ret_t (*func)(void *), void * arg) { + (void) unused; + HANDLE handle = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) func, arg, 0, NULL); + if (handle == NULL) { + return -1; + } + *out = handle; + return 0; +} +static int pthread_join(pthread_t thread, void * unused) { + (void) unused; + int ret = (int) WaitForSingleObject(thread, INFINITE); + CloseHandle(thread); + return ret; +} +#else #include +typedef void * thread_ret_t; +#endif // Worker for K-means++ min_dist update (parallel over samples) typedef struct { @@ -7913,7 +7944,7 @@ typedef struct { int dim; } iq1bn_kmpp_work_t; -static void * iq1bn_kmpp_worker(void * arg) { +static thread_ret_t iq1bn_kmpp_worker(void * arg) { iq1bn_kmpp_work_t * w = (iq1bn_kmpp_work_t *)arg; const float * cc = w->centroid; const int dim = w->dim; @@ -7928,7 +7959,7 @@ static void * iq1bn_kmpp_worker(void * arg) { } if (dist < w->min_dist[i]) w->min_dist[i] = dist; } - return NULL; + return (thread_ret_t) 0; } // Worker for K-means iteration (parallel over samples) @@ -7946,7 +7977,7 @@ typedef struct { int changed; } iq1bn_kmeans_work_t; -static void * iq1bn_kmeans_worker(void * arg) { +static thread_ret_t iq1bn_kmeans_worker(void * arg) { iq1bn_kmeans_work_t * w = (iq1bn_kmeans_work_t *)arg; const int K = w->K; const int dim = w->dim; @@ -7984,7 +8015,7 @@ static void * iq1bn_kmeans_worker(void * arg) { w->cwt[best_c] += wt; } w->changed = changed; - return NULL; + return (thread_ret_t) 0; } // K-means codebook training (K=4096, random init) From 123a1bcced8b18f7113d9b10a8b8b5fb4ad22c0a Mon Sep 17 00:00:00 2001 From: Piotr Wilkin Date: Mon, 6 Jul 2026 12:15:36 +0200 Subject: [PATCH 19/19] ggml : fix more cross-platform build errors from the quant-lab changes Fixes surfaced by the ARM, HIP, MUSA, CUDA, OpenCL and Windows CI jobs: - arm/quants.c: drop a duplicate, misnamed copy of the q1_0 kernel that was defined as ggml_vec_dot_q4_0_q8_0 and collided with the real q4_0 (redefinition + unused-parameter errors on every ARM build) - wasm/quants.c: GGML_UNUSED(levels) in q4_0 (-Werror=unused-parameter) - ggml-quants.c: clamp the k-means thread count to n_samples before flooring to 1 so nthread is provably >= 1 and calloc() no longer trips -Werror=alloc-size-larger-than on gcc - ggml-cuda/common.cuh: use -FLT_MAX / -HALF_MAX instead of -INFINITY for the MAX block-reduce sentinel (-INFINITY is rejected under the HIP -ffast-math build via -Werror,-Wnan-infinity-disabled) - ggml-cuda/convert.cu: GGML_UNUSED(n_levels) in ggml_cuda_set_q2kpt_levels - ggml-cuda/vendors/{hip,musa}.h: map cudaMemcpyToSymbolAsync to the hip/musa equivalents so convert.cu and mmvq.cu build on HIP and MUSA - ggml-opencl.cpp: pass the new levels arg (nullptr) to the to_float trait - auto-tensor-type: define strcasecmp -> _stricmp on Windows (MSVC/clang) Assisted-By: Claude Opus 4.8 (1M context) --- ggml/src/ggml-cpu/arch/arm/quants.c | 83 --------------------- ggml/src/ggml-cpu/arch/wasm/quants.c | 1 + ggml/src/ggml-cuda/common.cuh | 7 +- ggml/src/ggml-cuda/convert.cu | 1 + ggml/src/ggml-cuda/vendors/hip.h | 1 + ggml/src/ggml-cuda/vendors/musa.h | 1 + ggml/src/ggml-opencl/ggml-opencl.cpp | 2 +- ggml/src/ggml-quants.c | 4 +- tools/auto-tensor-type/auto-tensor-type.cpp | 5 ++ 9 files changed, 18 insertions(+), 87 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c index 4d0106b9fb25..9622f6cf498f 100644 --- a/ggml/src/ggml-cpu/arch/arm/quants.c +++ b/ggml/src/ggml-cpu/arch/arm/quants.c @@ -241,89 +241,6 @@ void ggml_vec_dot_q1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi } -void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { - const int qk = QK1_0; // 128 - const int nb = n / qk; - - assert(n % qk == 0); - assert(nrc == 1); - UNUSED(nrc); - UNUSED(bx); - UNUSED(by); - UNUSED(bs); - - const block_q1_0 * GGML_RESTRICT x = vx; - const block_q8_0 * GGML_RESTRICT y = vy; - -#if defined(__ARM_NEON) - float32x4_t sumv = vdupq_n_f32(0.0f); - - for (int i = 0; i < nb; i++) { - const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); - - // Process 4 Q8_0 blocks (each has 32 elements) - for (int k = 0; k < 4; k++) { - const block_q8_0 * GGML_RESTRICT yb = &y[i * 4 + k]; - const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); - - // Get the 4 bytes of bits for this Q8_0 block (32 bits = 4 bytes) - // Bits are at offset k*4 bytes in x[i].qs - const uint8_t * bits = &x[i].qs[k * 4]; - - // Load 32 int8 values from y - const int8x16_t y0 = vld1q_s8(yb->qs); - const int8x16_t y1 = vld1q_s8(yb->qs + 16); - - // Byte 0-1: bits for y0[0..15] - const uint64_t expand0 = table_b2b_0[bits[0]]; - const uint64_t expand1 = table_b2b_0[bits[1]]; - // Byte 2-3: bits for y1[0..15] - const uint64_t expand2 = table_b2b_0[bits[2]]; - const uint64_t expand3 = table_b2b_0[bits[3]]; - - // Build the sign vectors by reinterpreting the table values - uint8x8_t e0 = vcreate_u8(expand0); - uint8x8_t e1 = vcreate_u8(expand1); - uint8x8_t e2 = vcreate_u8(expand2); - uint8x8_t e3 = vcreate_u8(expand3); - - // Shift right by 4 to get 0 or 1 - int8x8_t s0 = vreinterpret_s8_u8(vshr_n_u8(e0, 4)); - int8x8_t s1 = vreinterpret_s8_u8(vshr_n_u8(e1, 4)); - int8x8_t s2 = vreinterpret_s8_u8(vshr_n_u8(e2, 4)); - int8x8_t s3 = vreinterpret_s8_u8(vshr_n_u8(e3, 4)); - - // Convert 0/1 to -1/+1: sign = 2*val - 1 - int8x8_t one = vdup_n_s8(1); - s0 = vsub_s8(vadd_s8(s0, s0), one); // 2*s0 - 1 - s1 = vsub_s8(vadd_s8(s1, s1), one); - s2 = vsub_s8(vadd_s8(s2, s2), one); - s3 = vsub_s8(vadd_s8(s3, s3), one); - - // Combine into 16-element vectors - int8x16_t signs0 = vcombine_s8(s0, s1); - int8x16_t signs1 = vcombine_s8(s2, s3); - - // Multiply signs with y values and accumulate - // dot(signs, y) where signs are +1/-1 - int32x4_t p0 = ggml_vdotq_s32(vdupq_n_s32(0), signs0, y0); - int32x4_t p1 = ggml_vdotq_s32(p0, signs1, y1); - - // Scale by d1 and accumulate - sumv = vmlaq_n_f32(sumv, vcvtq_f32_s32(p1), d0 * d1); - } - } - - *s = vaddvq_f32(sumv); -#else - UNUSED(nb); - UNUSED(x); - UNUSED(y); - ggml_vec_dot_q1_0_q8_0_generic(n, s, bs, vx, bx, vy, by, nrc, levels); -#endif -} - - void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc, const void * levels) { const int qk = QK8_0; const int nb = n / qk; diff --git a/ggml/src/ggml-cpu/arch/wasm/quants.c b/ggml/src/ggml-cpu/arch/wasm/quants.c index 379ca6466ab8..da67337d04d3 100644 --- a/ggml/src/ggml-cpu/arch/wasm/quants.c +++ b/ggml/src/ggml-cpu/arch/wasm/quants.c @@ -239,6 +239,7 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi UNUSED(bx); UNUSED(by); UNUSED(bs); + GGML_UNUSED(levels); const block_q4_0 * GGML_RESTRICT x = vx; const block_q8_0 * GGML_RESTRICT y = vy; diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index c57ee21b677f..f1e4b7a64632 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -607,10 +607,13 @@ template struct block_reduce_policy { } static __device__ T sentinel() { + // Use the lowest finite value of each type as the max-reduction identity. + // Avoids the INFINITY macro, which is rejected under -ffast-math / + // -ffinite-math-only (e.g. the HIP build) via -Werror,-Wnan-infinity-disabled. if constexpr (std::is_same_v) { - return -INFINITY; + return -FLT_MAX; } else if constexpr (std::is_same_v) { - return make_half2(-INFINITY, -INFINITY); + return make_half2(-65504.0f, -65504.0f); // -HALF_MAX } else { static_assert(ggml_cuda_dependent_false_v, "Unsupported type for block reduce max"); } diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index 96557545614f..61368e2f8081 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -624,6 +624,7 @@ void ggml_cuda_set_q3kpt_levels(const float * levels, cudaStream_t stream) { } void ggml_cuda_set_q2kpt_levels(const float * levels, size_t n_levels, cudaStream_t stream) { + GGML_UNUSED(n_levels); // levels is a device pointer to the per-block levels array // q2kpt_levels_cuda_ptr is a device symbol that holds the pointer CUDA_CHECK(cudaMemcpyToSymbolAsync(q2kpt_levels_cuda_ptr, &levels, sizeof(float *), 0, cudaMemcpyHostToDevice, stream)); diff --git a/ggml/src/ggml-cuda/vendors/hip.h b/ggml/src/ggml-cuda/vendors/hip.h index 234d76dfd8e0..e20ad4a136ff 100644 --- a/ggml/src/ggml-cuda/vendors/hip.h +++ b/ggml/src/ggml-cuda/vendors/hip.h @@ -78,6 +78,7 @@ #define cudaGetErrorString hipGetErrorString #define cudaGetLastError hipGetLastError #define cudaGetSymbolAddress hipGetSymbolAddress +#define cudaMemcpyToSymbolAsync hipMemcpyToSymbolAsync #define cudaHostRegister hipHostRegister #define cudaHostRegisterPortable hipHostRegisterPortable #define cudaHostRegisterReadOnly hipHostRegisterReadOnly diff --git a/ggml/src/ggml-cuda/vendors/musa.h b/ggml/src/ggml-cuda/vendors/musa.h index 284bd9de4c54..919b35fd1f07 100644 --- a/ggml/src/ggml-cuda/vendors/musa.h +++ b/ggml/src/ggml-cuda/vendors/musa.h @@ -62,6 +62,7 @@ #define cudaGetErrorString musaGetErrorString #define cudaGetLastError musaGetLastError #define cudaGetSymbolAddress musaGetSymbolAddress +#define cudaMemcpyToSymbolAsync musaMemcpyToSymbolAsync #define cudaHostRegister musaHostRegister #define cudaHostRegisterPortable musaHostRegisterPortable #define cudaHostRegisterReadOnly musaHostRegisterReadOnly diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 10f9cc22d88c..df2b1b210520 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -12830,7 +12830,7 @@ static bool ggml_cl_flash_attn_prepare_quantized_tensor( row_bytes, host_quant.data(), total_bytes); std::vector host_f32(n); - ggml_get_type_traits(tensor->type)->to_float(host_quant.data(), host_f32.data(), n); + ggml_get_type_traits(tensor->type)->to_float(host_quant.data(), host_f32.data(), n, nullptr); const size_t bytes_per_elem = ggml_type_size(target_type); const size_t buffer_size = (size_t) n * bytes_per_elem; diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 5ac25e1e3894..0c6e05c27c7b 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -8097,8 +8097,10 @@ void iq1bn_train_codebook(const float * data, int64_t nrow, int64_t n_per_row, } // Full K-means++ initialization with parallel min_dist update - if (nthread < 1) nthread = 1; + // Clamp to n_samples first, then floor to 1 so nthread is provably >= 1 + // (a negative n_samples must not leak a negative count into calloc()). if (nthread > n_samples) nthread = n_samples; + if (nthread < 1) nthread = 1; // Set up thread pool for K-means++ min_dist update pthread_t * threads = (pthread_t *)calloc(nthread, sizeof(pthread_t)); diff --git a/tools/auto-tensor-type/auto-tensor-type.cpp b/tools/auto-tensor-type/auto-tensor-type.cpp index 14f3849f1698..5316a36b0b22 100644 --- a/tools/auto-tensor-type/auto-tensor-type.cpp +++ b/tools/auto-tensor-type/auto-tensor-type.cpp @@ -40,6 +40,11 @@ #include #include +#if defined(_WIN32) +// strcasecmp is POSIX; MSVC/clang-windows provide _stricmp via the CRT. +#define strcasecmp _stricmp +#endif + // ============================================================================ // Section 1: Extern C declarations for special quant types // ============================================================================