diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e54fa057..86e78fbd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -187,6 +187,7 @@ target_link_libraries(flm PUBLIC phi4_npu nanbeige_npu dequant + dequant_new gemm lm_head mha diff --git a/src/include/modules/dequant.hpp b/src/include/modules/dequant.hpp index 43596d08..f1c236ac 100644 --- a/src/include/modules/dequant.hpp +++ b/src/include/modules/dequant.hpp @@ -21,12 +21,17 @@ class Dequant{ Dequant(LM_Config& config); ~Dequant(); - /// @brief generate the dequant sequence - /// @param seq: the sequence - /// @param D_in: input dimension of the projection weight - /// @param D_out: output dimension of the projection weight - /// @param weight_offset: the weight offset in byte - /// @param mode: dequant output mode + typedef enum: int { + Q4_1 = 0, + Q8_0 = 1 + } quant_block_t; + + void reorder_cpy(u8 *dst, buffer &src, + quant_block_t quant_block_type, + const int quant_matrix_row, + const int quant_matrix_col, + const int vertical_blocks=2, + const int vetrical_block_interleave_byte_size=-1); void generate_dequant_q4_1_seq(npu_sequence* seq, const uint32_t D_in, const uint32_t D_out, const uint32_t weight_offset, int mode); void generate_dequant_q80_packed_in_q4nx_seq(npu_sequence* seq, const uint32_t D_in, const uint32_t D_out, const uint32_t weight_offset, int mode); private: diff --git a/src/lib/libdequant_new.so b/src/lib/libdequant_new.so new file mode 100755 index 00000000..fc02a73a Binary files /dev/null and b/src/lib/libdequant_new.so differ diff --git a/src/lib/libllama_npu.so b/src/lib/libllama_npu.so index ef423e72..b5c3ce29 100755 Binary files a/src/lib/libllama_npu.so and b/src/lib/libllama_npu.so differ diff --git a/src/lib/libphi4_npu.so b/src/lib/libphi4_npu.so index 13cd76c2..40065d39 100755 Binary files a/src/lib/libphi4_npu.so and b/src/lib/libphi4_npu.so differ diff --git a/src/lib/libq4_npu_eXpress.so b/src/lib/libq4_npu_eXpress.so index f0d0bd6b..5dd4e8f5 100755 Binary files a/src/lib/libq4_npu_eXpress.so and b/src/lib/libq4_npu_eXpress.so differ diff --git a/src/lib/libqwen3_5vl_npu.so b/src/lib/libqwen3_5vl_npu.so index f1aa361e..837d87ee 100755 Binary files a/src/lib/libqwen3_5vl_npu.so and b/src/lib/libqwen3_5vl_npu.so differ diff --git a/src/lib/libqwen3_npu.so b/src/lib/libqwen3_npu.so index f023f1f0..14db01d4 100755 Binary files a/src/lib/libqwen3_npu.so and b/src/lib/libqwen3_npu.so differ diff --git a/src/lib/libqwen3vl_npu.so b/src/lib/libqwen3vl_npu.so index 19e9dc3d..ebd7f1c4 100755 Binary files a/src/lib/libqwen3vl_npu.so and b/src/lib/libqwen3vl_npu.so differ diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 8d925855..5873e8f9 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -110,6 +110,7 @@ function(add_npu_test TARGET_NAME OUTPUT_SUBDIR) q4_npu_eXpress lm_head dequant + dequant_new gemm mha tokenizers_cpp diff --git a/src/xclbins/Qwen3.5-0.8B-NPU2/dequant_q41.xclbin b/src/xclbins/Qwen3.5-0.8B-NPU2/dequant_q41.xclbin new file mode 100644 index 00000000..18e227f5 Binary files /dev/null and b/src/xclbins/Qwen3.5-0.8B-NPU2/dequant_q41.xclbin differ diff --git a/src/xclbins/Qwen3.5-0.8B-NPU2/dequant_q80.xclbin b/src/xclbins/Qwen3.5-0.8B-NPU2/dequant_q80.xclbin new file mode 100644 index 00000000..e3143417 Binary files /dev/null and b/src/xclbins/Qwen3.5-0.8B-NPU2/dequant_q80.xclbin differ diff --git a/src/xclbins/Qwen3.5-0.8B-NPU2/layer.xclbin b/src/xclbins/Qwen3.5-0.8B-NPU2/layer.xclbin index 909d5432..9ad7a95a 100644 Binary files a/src/xclbins/Qwen3.5-0.8B-NPU2/layer.xclbin and b/src/xclbins/Qwen3.5-0.8B-NPU2/layer.xclbin differ diff --git a/src/xclbins/Qwen3.5-0.8B-NPU2/lm_head.xclbin b/src/xclbins/Qwen3.5-0.8B-NPU2/lm_head.xclbin index dc119b32..e645706e 100644 Binary files a/src/xclbins/Qwen3.5-0.8B-NPU2/lm_head.xclbin and b/src/xclbins/Qwen3.5-0.8B-NPU2/lm_head.xclbin differ