diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 00000000..a0665727 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,36 @@ +{ + "permissions": { + "allow": [ + "Bash(g++ -std=c++20 -fsyntax-only -DUSEAVX2=1 -D__WINDOWS__ -DDISABLE_ABI_CHECK=1 -DDEV_BUILD -I include test/qwen3_5_omni_npu/test.cpp)", + "Read(//scratch/alfxu/FastFlowLM_IRON/FLM_DLL/**)", + "Read(//scratch/alfxu/FastFlowLM_IRON/**)", + "Bash(nm -DC /scratch/alfxu/FastFlowLM/src/lib/libqwen3_5_omni_npu.so)", + "Bash(cd /scratch/alfxu/FastFlowLM_IRON/FLM_DLL; echo \"=== qwen3_5_omni_npu.cpp \\(public wrapper\\) ===\"; wc -l detail/qwen3_5_omni/qwen3_5_omni_npu.cpp; echo; grep -n \"prefill\\\\|payload\\\\|image_soft\\\\|soft_token\\\\|forward\\\\|say\\\\|image_payload\\\\|audio_payload\\\\|num_soft\\\\|image_token\\\\|248\\\\|preprocess\" detail/qwen3_5_omni/qwen3_5_omni_npu.cpp | head -60)", + "Read(//scratch/alfxu/FastFlowLM_IRON/FLM_DLL/detail/qwen3_5_omni/**)", + "Bash(cd /scratch/alfxu/FastFlowLM_IRON/FLM_DLL; echo \"=== thinker hpp ===\"; sed -n '1,80p' detail/qwen3_5_omni/qwen3_5_omni_thinker/qwen3_5_omni_thinker.hpp 2>/dev/null; echo \"=== grep patch constants in omni thinker/image embed ===\"; grep -rn \"PATCH_SIZE\\\\|MERGE_SIZE\\\\|SHORTEST_EDGE\\\\|LONGEST_EDGE\\\\|TEMPORAL\\\\|RESCALE\\\\|IMAGE_MEAN\\\\|IMAGE_STD\\\\|soft_token\\\\|image_soft\\\\|248\" detail/qwen3_5_omni/ include/models/qwen3_5_omni/ 2>/dev/null | grep -iv \"//\" | head -40)", + "Bash(cd /scratch/alfxu/FastFlowLM_IRON/FLM_DLL; echo \"=== image embed hpp ===\"; cat detail/qwen3_5_omni/qwen3_5_omni_thinker/qwen3_5_omni_image_embed/qwen3_5_omni_image_embed.hpp 2>/dev/null | head -80; echo \"=== vision_config in LM_Config / omni ===\"; grep -rn \"vision_config\\\\|_vision_config\\\\|PATCH_SIZE\\\\|patch_size\\\\|merge_size\\\\|temporal\\\\|shortest\\\\|longest\\\\|image_mean\\\\|image_std\\\\|rescale\" detail/qwen3_5_omni/ 2>/dev/null | head -40)", + "Bash(cd /scratch/alfxu/FastFlowLM; echo \"=== LM_Config public members ===\"; grep -n \"_json_config\\\\|_vision_config\\\\|from_pretrained\\\\|vocab_size\\\\|struct\\\\|class LM_Config\\\\|public\\\\|nlohmann\" src/include/lm_config.hpp | head -40; echo \"=== omni model config.json vision/image processor ===\"; find /scratch/alfxu -maxdepth 4 -name \"config.json\" -path \"*mni*\" 2>/dev/null | head; find /scratch/alfxu -maxdepth 4 -iname \"*omni*\" -type d 2>/dev/null | head)", + "Read(//scratch/alfxu/**)", + "Bash(python3 -c ' *)", + "Bash(python3 -c \"import json,sys; d=json.load\\(sys.stdin\\); print\\(json.dumps\\({k:d[k] for k in d if k in ['image_mean','image_std','rescale_factor','patch_size','temporal_patch_size','merge_size','min_pixels','max_pixels','size','shortest_edge','longest_edge']},indent=1\\)\\)\")", + "Read(//scratch/alfxu/FastFlowLM_IRON/FLM_DLL/detail/qwen3_5_omni/qwen3_5_omni_talker/**)", + "Bash(cat /scratch/alfxu/models/Qwen3.5-omni-NPU2/config.json 2>/dev/null | grep -iE \"audio|vision|image|token_id\" | head -50)", + "Read(//scratch/alfxu/models/Qwen3.5-omni-NPU2/**)", + "Bash(grep -rniE \"write_wav|write.wav|\\\\.wav|save.*wav|pcm\" /scratch/alfxu/FastFlowLM/src /scratch/alfxu/FastFlowLM_IRON 2>/dev/null | grep -viE \"\\\\.o:|Binary\" | head -40)", + "Bash(grep -n \"audio_end_token_id\\\\|audio_start_token_id\\\\|audio_token_id\\\\|codec_bos\\\\|codec_eos\\\\|tts\" /scratch/alfxu/models/Qwen3.5-omni-NPU2/config.json)", + "Bash(grep -nE \"\\\\\"thinker_config\\\\\"|\\\\\"talker_config\\\\\"|\\\\\"text_config\\\\\"|\\\\\"vision_config\\\\\"|\\\\\"audio_config\\\\\"|audio_end_token_id|audio_start_token_id|audio_token_id\" /scratch/alfxu/models/Qwen3.5-omni-NPU2/config.json | head -40)", + "Bash(cd /scratch/alfxu/FastFlowLM/src; echo \"=== audio_reader.hpp API ===\"; grep -n \"load_audio\\\\|audio_data_t\\\\|MonoDownmix\\\\|sample_rate\\\\|num_frames\\\\|num_samples\\\\|struct\\\\|class AudioReader\" include/audio/audio_reader.hpp | head -30; echo \"=== omni engine resample rate / audio config ===\"; grep -rn \"resample\\\\|sample_rate\\\\|16000\\\\|Audio_resample\" /scratch/alfxu/FastFlowLM_IRON/FLM_DLL/detail/qwen3_5_omni/qwen3_5_omni_thinker/qwen3_5_omni_audio_embed/*.cpp 2>/dev/null | head)", + "Read(//scratch/alfxu/FastFlowLM_IRON/FLM_DLL/detail/qwen3_5_omni/qwen3_5_omni_thinker/qwen3_5_omni_audio_embed/**)", + "Bash(cd /scratch/alfxu/FastFlowLM_IRON/FLM_DLL/detail/qwen3_5_omni/qwen3_5_omni_thinker/qwen3_5_omni_audio_embed; ls; echo \"=== audio embed hpp ===\"; cat *.hpp 2>/dev/null | head -60)", + "Bash(grep -rn \"say\\\\|talker\\\\|codec\\\\|thinker_res\" /scratch/alfxu/FastFlowLM/src/include/models/qwen3_5_omni/qwen3_5_omni.hpp; echo \"===say impl===\"; grep -rn \"::say\\\\|buffer say\\\\|say\\(\" /scratch/alfxu/FastFlowLM/src /scratch/alfxu/FastFlowLM_IRON 2>/dev/null | grep -v \"\\\\.o:\" | head)", + "Bash(make clean *)", + "Bash(cd /scratch/alfxu/FastFlowLM/src; grep -rn \"model.q4nx\\\\|\\\\.q4nx\\\\|safetensors\" common/tensor_utils/*.cpp 2>/dev/null | head; echo \"=== how other models' model dirs look \\(do they have model.q4nx?\\) ===\"; ls /scratch/alfxu/models/Qwen3.5-4B-NPU2/ 2>/dev/null | head)", + "Read(//scratch/alfxu/models/Qwen3.5-4B-NPU2/**)", + "Bash(g++ -std=c++20 -fsyntax-only -I include -I . common/qwen3_5_omni/modeling_qwen3_5_omni.cpp)", + "Bash(g++ -std=c++20 -fsyntax-only -I include -I . test/qwen3_5_omni_npu/test.cpp)", + "Bash(g++ -std=c++20 -fsyntax-only -I include -I . common/AutoModel/automodel.cpp)", + "Bash(g++ -std=c++20 -fsyntax-only -D__FLM_VERSION__='\"test\"' -I include -I . test/qwen3_5_omni_npu/test.cpp)", + "Bash(g++ -std=c++20 -fsyntax-only -I include -I . common/AutoModel/modeling_qwen3_5_omni.cpp)" + ] + } +} diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index ab4ee4b5..b291a22f 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -483,7 +483,7 @@ void AutoModel::set_sampler(sampler_config& sampler_config) { if (this->sampler != nullptr) { this->sampler.reset(); } - this->sampler = std::make_unique(this->lm_config->vocab_size, sampler_config); + this->sampler = std::make_unique(this->lm_config->get("vocab_size"), sampler_config); } /// \brief Set the max length @@ -575,9 +575,9 @@ void AutoModel::set_topk(int topk) { header_print("WARNING", "Top-k must be greater than 0"); return; } - if (topk > this->lm_config->vocab_size) { - header_print("WARNING", "Top-k is greater than vocab size, set to vocab size: " << this->lm_config->vocab_size); - topk = this->lm_config->vocab_size; + if (topk > this->lm_config->get("vocab_size")) { + header_print("WARNING", "Top-k is greater than vocab size, set to vocab size: " << this->lm_config->get("vocab_size")); + topk = this->lm_config->get("vocab_size"); } this->sampler->top_k = topk; diff --git a/src/common/AutoModel/modeling_qwen2.cpp b/src/common/AutoModel/modeling_qwen2.cpp index 8f8513b5..681d4f45 100644 --- a/src/common/AutoModel/modeling_qwen2.cpp +++ b/src/common/AutoModel/modeling_qwen2.cpp @@ -16,7 +16,7 @@ void Qwen2::load_model(std::string model_path, json model_info, int default_cont this->q4nx = std::make_unique(this->model_path); - // lm_config->model_type == qwen2 + // lm_config->get("model_type", "") == qwen2 this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); this->lm_engine->load_weights(*this->q4nx); diff --git a/src/common/AutoModel/modeling_qwen2vl.cpp b/src/common/AutoModel/modeling_qwen2vl.cpp index bc6ceba7..fb01b153 100644 --- a/src/common/AutoModel/modeling_qwen2vl.cpp +++ b/src/common/AutoModel/modeling_qwen2vl.cpp @@ -18,7 +18,7 @@ void Qwen2VL::load_model(std::string model_path, json model_info, int default_co this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); this->q4nx = std::make_unique(this->model_path); - // lm_config->model_type == qwen2 + // lm_config->get("model_type", "") == qwen2 this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); this->lm_engine->load_weights(*this->q4nx); diff --git a/src/common/AutoModel/modeling_qwen3.cpp b/src/common/AutoModel/modeling_qwen3.cpp index 299c4101..f27af84b 100644 --- a/src/common/AutoModel/modeling_qwen3.cpp +++ b/src/common/AutoModel/modeling_qwen3.cpp @@ -16,7 +16,7 @@ void Qwen3::load_model(std::string model_path, json model_info, int default_cont this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); this->q4nx = std::make_unique(this->model_path); - // lm_config->model_type == qwen3 + // lm_config->get("model_type", "") == qwen3 this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); this->lm_engine->load_weights(*this->q4nx); @@ -314,7 +314,7 @@ void Qwen3_IT::load_model(std::string model_path, json model_info, int default_c this->q4nx = std::make_unique(this->model_path); - // lm_config->model_type == qwen3 + // lm_config->get("model_type", "") == qwen3 this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); this->lm_engine->load_weights(*this->q4nx); @@ -465,7 +465,7 @@ void Qwen3_TK::load_model(std::string model_path, json model_info, int default_c this->q4nx = std::make_unique(this->model_path); - // lm_config->model_type == qwen3 + // lm_config->get("model_type", "") == qwen3 this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); this->lm_engine->load_weights(*this->q4nx); diff --git a/src/common/AutoModel/modeling_qwen3_5_omni.cpp b/src/common/AutoModel/modeling_qwen3_5_omni.cpp index 43649e45..89ef50d1 100644 --- a/src/common/AutoModel/modeling_qwen3_5_omni.cpp +++ b/src/common/AutoModel/modeling_qwen3_5_omni.cpp @@ -75,13 +75,13 @@ void Qwen3_5_Omni::load_model(std::string model_path, json model_info, int defau this->lm_config->from_pretrained(this->model_path); // The omni thinker vocab lives in thinker_config.text_config, so the - // top-level "vocab_size" is 0. Resolve it here so every downstream reader - // (set_sampler, set_topk, ...) sees the real vocab instead of 0. + // top-level "vocab_size" is 0. Republish it at the top level so every + // downstream reader (set_sampler, set_topk, ...) sees the real vocab. { - const auto& jc = this->lm_config->_json_config; + auto& jc = this->lm_config->_json_config; if (jc.contains("thinker_config") && jc["thinker_config"].contains("text_config")) { - this->lm_config->vocab_size = - json_u32(jc["thinker_config"]["text_config"], "vocab_size", this->lm_config->vocab_size); + jc["vocab_size"] = + json_u32(jc["thinker_config"]["text_config"], "vocab_size", this->lm_config->get("vocab_size")); } } @@ -694,7 +694,7 @@ buffer Qwen3_5_Omni::say(std::string wav_out_path) { void Qwen3_5_Omni::set_sampler(sampler_config& sampler_config) { if (this->sampler != nullptr) this->sampler.reset(); - this->sampler = std::make_unique((int)this->lm_config->vocab_size, sampler_config); + this->sampler = std::make_unique((int)this->lm_config->get("vocab_size"), sampler_config); } void Qwen3_5_Omni::clear_context() { diff --git a/src/common/AutoModel/modeling_qwen3_5vl.cpp b/src/common/AutoModel/modeling_qwen3_5vl.cpp index 37900818..f55d7e0b 100644 --- a/src/common/AutoModel/modeling_qwen3_5vl.cpp +++ b/src/common/AutoModel/modeling_qwen3_5vl.cpp @@ -17,7 +17,7 @@ void Qwen3_5VL::load_model(std::string model_path, json model_info, int default_ this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); this->q4nx = std::make_unique(this->model_path); - // lm_config->model_type == qwen3 + // lm_config->get("model_type", "") == qwen3 this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); this->lm_engine->load_weights(*this->q4nx); diff --git a/src/common/AutoModel/modeling_qwen3_6_moe.cpp b/src/common/AutoModel/modeling_qwen3_6_moe.cpp index 11af490c..5251ad6b 100644 --- a/src/common/AutoModel/modeling_qwen3_6_moe.cpp +++ b/src/common/AutoModel/modeling_qwen3_6_moe.cpp @@ -17,7 +17,7 @@ void Qwen3_6_MOE::load_model(std::string model_path, json model_info, int defaul this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); this->q4nx = std::make_unique(this->model_path); - // lm_config->model_type == qwen3 + // lm_config->get("model_type", "") == qwen3 this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); this->lm_engine->load_weights(*this->q4nx); diff --git a/src/common/AutoModel/modeling_qwen3vl.cpp b/src/common/AutoModel/modeling_qwen3vl.cpp index 1b3b81dc..08958229 100644 --- a/src/common/AutoModel/modeling_qwen3vl.cpp +++ b/src/common/AutoModel/modeling_qwen3vl.cpp @@ -17,7 +17,7 @@ void Qwen3VL::load_model(std::string model_path, json model_info, int default_co this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); this->q4nx = std::make_unique(this->model_path); - // lm_config->model_type == qwen3 + // lm_config->get("model_type", "") == qwen3 this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); this->lm_engine->load_weights(*this->q4nx); diff --git a/src/common/whisper/modeling_whisper.cpp b/src/common/whisper/modeling_whisper.cpp index afac93c7..b347eb8a 100644 --- a/src/common/whisper/modeling_whisper.cpp +++ b/src/common/whisper/modeling_whisper.cpp @@ -50,7 +50,7 @@ void Whisper::load_model(std::string model_path, nlohmann::ordered_json model_in s_config.min_p = 0.1; s_config.temperature = 0.4; - this->sampler = std::make_unique(this->lm_config->vocab_size, s_config); + this->sampler = std::make_unique(this->lm_config->get("vocab_size"), s_config); } void Whisper::setup_tokenizer(std::string model_path) { diff --git a/src/include/buffer.hpp b/src/include/buffer.hpp index dde591fa..5387c10b 100644 --- a/src/include/buffer.hpp +++ b/src/include/buffer.hpp @@ -1,534 +1,525 @@ -/// \file buffer.hpp -/// \brief Buffer and bytes class for memory management -/// \author FastFlowLM Team -/// \date 2025-06-24 -/// \version 0.9.10 -/// \note This class is used to manage the memory. -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define __XRT__ - -#ifdef __XRT__ -#include "xrt/xrt_bo.h" -#include "xrt/xrt_kernel.h" -#include "xrt/xrt_device.h" -#include "xrt/experimental/xrt_ext.h" -#endif - -#include "utils/debug_utils.hpp" - -/// \brief bytes class -/// \note This is a buffer wrapper that maps to a bo_buffer or other memory without performing a deep copy. -/// \note A copy (or mapping) does not duplicate the underlying memory; it only maps the pointer. -class bytes { -protected: - std::unique_ptr owned_data_; - uint8_t* data_; - size_t size_; - bool is_owner_; -#ifdef __XRT__ - bool is_bo_owner_; - xrt::bo* bo_; - std::unique_ptr owned_bo_; -#endif - -public: - /// \brief constructor - /// \note This is a buffer wrapper that maps to a bo_buffer or other memory without performing a deep copy. - /// \note A copy (or mapping) does not duplicate the underlying memory; it only maps the pointer. - bytes() : data_(nullptr), size_(0), is_owner_(false) -#ifdef __XRT__ - , is_bo_owner_(false), bo_(nullptr), owned_bo_(nullptr) -#endif - {} - - /// \brief copy constructor - /// \param other the other bytes - bytes(const bytes& other) : owned_data_(nullptr), data_(other.data_), size_(other.size_), is_owner_(false) -#ifdef __XRT__ - , is_bo_owner_(false), bo_(other.bo_), owned_bo_(nullptr) -#endif - {} - - /// \brief move constructor - /// \param other the other bytes - bytes(bytes&& other) noexcept - : owned_data_(std::move(other.owned_data_)), data_(other.data_), size_(other.size_), is_owner_(other.is_owner_) -#ifdef __XRT__ - , is_bo_owner_(other.is_bo_owner_), bo_(other.bo_), owned_bo_(std::move(other.owned_bo_)) -#endif - { - other.data_ = nullptr; - other.size_ = 0; - other.is_owner_ = false; -#ifdef __XRT__ - other.is_bo_owner_ = false; - other.bo_ = nullptr; - other.owned_bo_ = nullptr; -#endif - } - - /// \brief constructor - /// \param size the size - bytes(size_t size) - : size_(size), is_owner_(true) -#ifdef __XRT__ - , is_bo_owner_(false), bo_(nullptr), owned_bo_(nullptr) -#endif - { - if (size > 0 && size < 8ull * 1024 * 1024 * 1024){ - try { - owned_data_ = std::make_unique(size); - } - catch (const std::bad_alloc& e) { - throw std::runtime_error(std::string("Failed to allocate bytes of size ") + std::to_string(size) + ": " + e.what()); - } - data_ = owned_data_.get(); - } - else{ - throw std::runtime_error("Invalid size for bytes allocation"); - } - } - - /// \brief constructor - /// \param data the data - /// \param size the size - bytes(uint8_t* data, size_t size) - : owned_data_(nullptr), data_(data), size_(size), is_owner_(false) -#ifdef __XRT__ - , is_bo_owner_(false), bo_(nullptr), owned_bo_(nullptr) -#endif - {} - -#ifdef __XRT__ - /// \brief constructor - /// \param bo the bo - bytes(xrt::bo& bo) - : owned_data_(nullptr), data_(bo.map()), size_(bo.size()), is_owner_(false), is_bo_owner_(false), bo_(&bo), owned_bo_(nullptr) - {} - - /// \brief constructor - /// \param size the size - /// \param device the device - /// \param kernel the kernel - /// \param group_id the group id - /// \param flags the flags - bytes(xrt::device& device, size_t size) - : owned_data_(nullptr), size_(size), is_owner_(false), is_bo_owner_(true) - { - if (size > 3ull * 1024 * 1024 * 1024 || size == 0){ - throw std::runtime_error("Invalid size for bytes allocation"); - } - size_t alignment = 1024 * 1024; - int padded_size = (size + alignment - 1) / alignment * alignment; // 4KB alignment, , (xrt::ext::bo::access_mode)(xrt::ext::bo::access_mode::read_write | xrt::ext::bo::access_mode::process) - - try { - owned_bo_ = std::make_unique(device, padded_size); - } - catch (const std::exception& e) { - throw std::runtime_error(std::string("Failed to allocate xrt::ext::bo: ") + e.what()); - } - - // uint64_t bo_address = reinterpret_cast(owned_bo_->map()); - // while ( ((bo_address & 0xF0000000) == 0x60000000) || - // ((bo_address & 0xF0000000) == 0x70000000) ) { - - // owned_bo_ = std::make_unique(device, padded_size); - // //header_print("info", "Re-allocating proj_weights for layer " + std::to_string(i) + " to avoid address in 0x60000000 - 0x7FFFFFFF, new address: " + std::to_string(reinterpret_cast(proj_weights[i].data()))); - // bo_address = reinterpret_cast(owned_bo_->map()); - // } - - data_ = owned_bo_->map(); - bo_ = owned_bo_.get(); - } -#endif - - /// \brief destructor - virtual ~bytes() { - if (is_owner_) { - owned_data_.reset(); - } - data_ = nullptr; -#ifdef __XRT__ - if (is_bo_owner_) { - owned_bo_.reset(); - } - bo_ = nullptr; -#endif - } - - /// \brief copy assignment operator - /// \param other the other bytes - bytes& operator=(const bytes& other) { - if (this != &other) { - if (is_owner_){ - owned_data_.reset(); - } - data_ = other.data_; - size_ = other.size_; - is_owner_ = false; -#ifdef __XRT__ - if (is_bo_owner_){ - owned_bo_.reset(); - } - is_bo_owner_ = false; - bo_ = other.bo_; -#endif - } - return *this; - } - - /// \brief move assignment operator - /// \param other the other bytes - bytes& operator=(bytes&& other) noexcept { - if (this != &other) { - if (is_owner_){ - owned_data_.reset(); - } - owned_data_ = std::move(other.owned_data_); - data_ = other.data_; - size_ = other.size_; - is_owner_ = other.is_owner_; -#ifdef __XRT__ - if (is_bo_owner_){ - owned_bo_.reset(); - } - is_bo_owner_ = other.is_bo_owner_; - owned_bo_ = std::move(other.owned_bo_); - bo_ = other.bo_; - other.bo_ = nullptr; - other.is_bo_owner_ = false; -#endif - other.data_ = nullptr; - other.size_ = 0; - other.is_owner_ = false; - } - return *this; - } - - /// \brief operator [] - /// \param index the index - /// \return the value - uint8_t& operator[](size_t index) { - assert(data_ && index < size_); - return data_[index]; - } - - /// \brief operator [] - /// \param index the index - /// \return the value - const uint8_t& operator[](size_t index) const { - assert(data_ && index < size_); - return data_[index]; - } - - size_t size() const { return size_; } - uint8_t* data() const { return data_; } - uint8_t* bdata() const { return data_; } - uint8_t* begin() const { return data_; } - uint8_t* end() const { return data_ + size_; } - - /// \brief copy from - /// \param src the source - /// \param size the size - void copy_from(const uint8_t* src, size_t size) { - assert(size <= size_); - std::memcpy(data_, src, size); - } - - /// \brief resize - /// \param new_size the new size - void resize(size_t new_size) { -#ifdef __XRT__ - assert(!is_bo_owner_); -#endif - if (data_ != nullptr && !is_owner_) { - throw std::runtime_error("Cannot resize a non-owner buffer"); - } - if (new_size == 0) { - throw std::runtime_error("Cannot resize to zero size"); - } - try { - owned_data_.reset(new uint8_t[new_size]); - } - catch (const std::bad_alloc& e) { - throw std::runtime_error(std::string("Failed to allocate bytes of size ") + std::to_string(new_size) + ": " + e.what()); - } - data_ = owned_data_.get(); - size_ = new_size; - is_owner_ = true; - } - - /// \brief free, release the memory or the bo - void free() { -#ifdef __XRT__ - assert(!is_bo_owner_); -#endif - if (is_owner_){ - owned_data_.reset(); - } - data_ = nullptr; - size_ = 0; - is_owner_ = false; -#ifdef __XRT__ - if (is_bo_owner_){ - owned_bo_.reset(); - } - is_bo_owner_ = false; - bo_ = nullptr; -#endif - } - - /// \brief reserve - /// \param size the size - void reserve(size_t size) { resize(size); } - - /// \brief release - void release() { free(); } - - /// \brief is owner - /// \return the is owner - bool is_owner() const { return is_owner_; } -#ifdef __XRT__ - /// \brief is bo owner - /// \return the is bo owner - bool is_bo_owner() const { return is_bo_owner_; } - - /// \brief sync to device - void sync_to_device() { assert(bo_); bo_->sync(XCL_BO_SYNC_BO_TO_DEVICE); } - - /// \brief sync from device - void sync_from_device() { assert(bo_); bo_->sync(XCL_BO_SYNC_BO_FROM_DEVICE); } - - /// \brief bo - /// \return the bo - xrt::bo& bo() { assert(bo_); return *bo_; } -#endif - - /// \brief from file - /// \param filename the filename - /// \param offset the offset - /// \param size the size - void from_file(const std::string& filename, size_t offset = 0, size_t size = 0) { - std::ifstream file(filename, std::ios::binary); - if (!file.is_open()) { - throw std::runtime_error("Failed to open file: " + filename); - } - file.seekg(0, std::ios::end); - size_t file_size = file.tellg(); - file.seekg(0, std::ios::beg); - if (size == 0) size = file_size; - assert(size <= file_size); - assert(offset + size <= size_); - file.read(reinterpret_cast(data_) + offset, size); - file.close(); - } -}; - -/// \brief buffer class -/// \note This class wraps a data type T over the underlying byte buffer. -template -class buffer : public bytes { -public: - /// \brief constructor - buffer() : bytes() {} - - /// \brief constructor - /// \param count the count - buffer(size_t count) : bytes(count * sizeof(T)) {} - - /// \brief constructor - /// \param data the data - /// \param count the count - buffer(T* data, size_t count) - : bytes(reinterpret_cast(data), count * sizeof(T)) {} - - /// \brief shallow copy constructor - /// \param other the other buffer - buffer(const buffer& other) : bytes(other) {} - - /// \brief move constructor - /// \param other the other buffer - /// \note Transfers ownership (owned_data_/owned_bo_) so a returned buffer does not dangle. - buffer(buffer&& other) noexcept : bytes(std::move(other)) {} - -#ifdef __XRT__ - /// \brief constructor - /// \param bo the bo - buffer(xrt::bo& bo) : bytes(bo) {} - - /// \brief constructor - /// \param count the count - /// \param device the device - /// \param kernel the kernel - /// \param group_id the group id - /// \param flags the flags - buffer(xrt::device& device, size_t count) - : bytes(device, count * sizeof(T)) {} -#endif - - /// \brief constructor - /// \param vec the vector - buffer(const std::vector& vec) - : bytes(reinterpret_cast(const_cast(vec.data())), vec.size() * sizeof(T)) - { - } - - /// \brief constructor - /// \param vec the vector - /// \warning This also creates a shallow mapping. - /// \warning The caller must ensure that the vector is not used (and remains valid) - /// \warning after constructing this buffer. - buffer(std::vector&& vec) - : bytes(reinterpret_cast(vec.data()), vec.size() * sizeof(T)) - { - } - - /// \brief copy from - /// \param vec the vector - void copy_from(const std::vector& vec) { - if (vec.size() * sizeof(T) != this->size_) { - throw std::runtime_error("Size mismatch in copy_from(vector)"); - } - std::memcpy(data_, vec.data(), size_); - } - - /// \brief cast to another type - /// \tparam U the type - /// \return the buffer - template - buffer cast_to() { - size_t newCount = size_ / sizeof(U); - return buffer(reinterpret_cast(data_), newCount); - } - - /// \brief as bytes - /// \return the bytes - const bytes as_bytes() const { - return *this; - } - - /// \brief move assignment operator - /// \param other the other buffer - /// \return the buffer - buffer& operator=(buffer&& other) noexcept { - bytes::operator=(std::move(other)); - return *this; - } - - /// \brief copy assignment operator - /// \param other the other buffer - /// \return the buffer - buffer& operator=(const buffer& other) { - bytes::operator=(other); - return *this; - } - - /// \brief operator [] - /// \param index the index - /// \return the value - T& operator[](size_t index) { - assert(data_ != nullptr); - assert(index < size()); - assert(index >= 0); - return reinterpret_cast(data_)[index]; - } - - /// \brief operator [] - /// \param index the index - /// \return the value - const T& operator[](size_t index) const { - assert(data_ != nullptr); - assert(index < size()); - assert(index >= 0); - return reinterpret_cast(data_)[index]; - } - - /// \brief size, number of elements - /// \return the size - size_t size() const { return size_ / sizeof(T); } - - /// \brief data - /// \return the data - T* data() const { return reinterpret_cast(data_); } - - /// \brief begin - /// \return the pointer to the first element - T* begin() const { return reinterpret_cast(data_); } - - /// \brief end - /// \return the pointer to the last element - T* end() const { return reinterpret_cast(data_) + size(); } - - /// \brief resize - /// \param count: the number of elements - void resize(size_t count) { - bytes::resize(count * sizeof(T)); - } - - /// \brief reserve - /// \param count: the number of elements - void reserve(size_t count) { - bytes::reserve(count * sizeof(T)); - } - - /// \brief memset - /// \param value the value - void memset(T value) { - T* ptr = data(); - for (size_t i = 0; i < size(); i++) { - ptr[i] = value; - } - } - - /// \brief copy from - /// \param other the other bytes - void copy_from(const bytes& other) { - if (size_ != other.size()) { - throw std::runtime_error("Size mismatch in copy_from(bytes)"); - } - memcpy(data_, other.data(), size_); // size_ is already in bytes. - } - - /// \brief copy from - /// \param other the other buffer - void copy_from(const buffer& other) { - if (size() != other.size()) { - throw std::runtime_error("Size mismatch in copy_from(buffer)"); - } - memcpy(data_, other.bdata(), size_); // size_ is already in bytes. - } - - /// \brief copy from - /// \param data the data - /// \param size the number of elements - void copy_from(T* data, size_t size) { - if (size > this->size()) { - throw std::runtime_error("Size mismatch in copy_from(pointer)"); - } - memcpy(data_, data, size * sizeof(T)); - } - - /// \brief as bytes - /// \return the bytes - bytes& as_bytes() { - return *this; - } - - /// \brief from file - /// \param filename the filename - /// \param offset the offset - /// \param size the number of elements - void from_file(const std::string& filename, size_t offset = 0, size_t size = 0) { - bytes::from_file(filename, offset * sizeof(T), size * sizeof(T)); - } -}; +/// \file buffer.hpp +/// \brief Buffer and bytes class for memory management +/// \author FastFlowLM Team +/// \date 2025-06-24 +/// \version 0.9.10 +/// \note This class is used to manage the memory. +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define __XRT__ + +#ifdef __XRT__ +#include "xrt/xrt_bo.h" +#include "xrt/xrt_kernel.h" +#include "xrt/xrt_device.h" +#include "xrt/experimental/xrt_ext.h" +#endif + +#include "utils/debug_utils.hpp" + +/// \brief bytes class +/// \note This is a buffer wrapper that maps to a bo_buffer or other memory without performing a deep copy. +/// \note A copy (or mapping) does not duplicate the underlying memory; it only maps the pointer. +class bytes { +protected: + std::unique_ptr owned_data_; + uint8_t* data_; + size_t size_; + bool is_owner_; +#ifdef __XRT__ + bool is_bo_owner_; + xrt::bo* bo_; + std::unique_ptr owned_bo_; +#endif + +public: + /// \brief constructor + /// \note This is a buffer wrapper that maps to a bo_buffer or other memory without performing a deep copy. + /// \note A copy (or mapping) does not duplicate the underlying memory; it only maps the pointer. + bytes() : data_(nullptr), size_(0), is_owner_(false) +#ifdef __XRT__ + , is_bo_owner_(false), bo_(nullptr), owned_bo_(nullptr) +#endif + {} + + /// \brief copy constructor + /// \param other the other bytes + bytes(const bytes& other) : owned_data_(nullptr), data_(other.data_), size_(other.size_), is_owner_(false) +#ifdef __XRT__ + , is_bo_owner_(false), bo_(other.bo_), owned_bo_(nullptr) +#endif + {} + + /// \brief move constructor + /// \param other the other bytes + bytes(bytes&& other) noexcept + : owned_data_(std::move(other.owned_data_)), data_(other.data_), size_(other.size_), is_owner_(other.is_owner_) +#ifdef __XRT__ + , is_bo_owner_(other.is_bo_owner_), bo_(other.bo_), owned_bo_(std::move(other.owned_bo_)) +#endif + { + other.data_ = nullptr; + other.size_ = 0; + other.is_owner_ = false; +#ifdef __XRT__ + other.is_bo_owner_ = false; + other.bo_ = nullptr; + other.owned_bo_ = nullptr; +#endif + } + + /// \brief constructor + /// \param size the size + bytes(size_t size) + : size_(size), is_owner_(true) +#ifdef __XRT__ + , is_bo_owner_(false), bo_(nullptr), owned_bo_(nullptr) +#endif + { + if (size > 0 && size < 8ull * 1024 * 1024 * 1024){ + try { + owned_data_ = std::make_unique(size); + } + catch (const std::bad_alloc& e) { + throw std::runtime_error(std::string("Failed to allocate bytes of size ") + std::to_string(size) + ": " + e.what()); + } + data_ = owned_data_.get(); + } + else{ + throw std::runtime_error("Invalid size for bytes allocation"); + } + } + + /// \brief constructor + /// \param data the data + /// \param size the size + bytes(uint8_t* data, size_t size) + : owned_data_(nullptr), data_(data), size_(size), is_owner_(false) +#ifdef __XRT__ + , is_bo_owner_(false), bo_(nullptr), owned_bo_(nullptr) +#endif + {} + +#ifdef __XRT__ + /// \brief constructor + /// \param bo the bo + bytes(xrt::bo& bo) + : owned_data_(nullptr), data_(bo.map()), size_(bo.size()), is_owner_(false), is_bo_owner_(false), bo_(&bo), owned_bo_(nullptr) + {} + + /// \brief constructor + /// \param size the size + /// \param device the device + /// \param kernel the kernel + /// \param group_id the group id + /// \param flags the flags + bytes(xrt::device& device, size_t size) + : owned_data_(nullptr), size_(size), is_owner_(false), is_bo_owner_(true) + { + if (size > 3ull * 1024 * 1024 * 1024 || size == 0){ + throw std::runtime_error("Invalid size for bytes allocation"); + } + size_t alignment = 1024 * 1024; + int padded_size = (size + alignment - 1) / alignment * alignment; // 4KB alignment, , (xrt::ext::bo::access_mode)(xrt::ext::bo::access_mode::read_write | xrt::ext::bo::access_mode::process) + + try { + owned_bo_ = std::make_unique(device, padded_size); + } + catch (const std::exception& e) { + throw std::runtime_error(std::string("Failed to allocate xrt::ext::bo: ") + e.what()); + } + + data_ = owned_bo_->map(); + bo_ = owned_bo_.get(); + } +#endif + + /// \brief destructor + virtual ~bytes() { + if (is_owner_) { + owned_data_.reset(); + } + data_ = nullptr; +#ifdef __XRT__ + if (is_bo_owner_) { + owned_bo_.reset(); + } + bo_ = nullptr; +#endif + } + + /// \brief copy assignment operator + /// \param other the other bytes + bytes& operator=(const bytes& other) { + if (this != &other) { + if (is_owner_){ + owned_data_.reset(); + } + data_ = other.data_; + size_ = other.size_; + is_owner_ = false; +#ifdef __XRT__ + if (is_bo_owner_){ + owned_bo_.reset(); + } + is_bo_owner_ = false; + bo_ = other.bo_; +#endif + } + return *this; + } + + /// \brief move assignment operator + /// \param other the other bytes + bytes& operator=(bytes&& other) noexcept { + if (this != &other) { + if (is_owner_){ + owned_data_.reset(); + } + owned_data_ = std::move(other.owned_data_); + data_ = other.data_; + size_ = other.size_; + is_owner_ = other.is_owner_; +#ifdef __XRT__ + if (is_bo_owner_){ + owned_bo_.reset(); + } + is_bo_owner_ = other.is_bo_owner_; + owned_bo_ = std::move(other.owned_bo_); + bo_ = other.bo_; + other.bo_ = nullptr; + other.is_bo_owner_ = false; +#endif + other.data_ = nullptr; + other.size_ = 0; + other.is_owner_ = false; + } + return *this; + } + + /// \brief operator [] + /// \param index the index + /// \return the value + uint8_t& operator[](size_t index) { + assert(data_ && index < size_); + return data_[index]; + } + + /// \brief operator [] + /// \param index the index + /// \return the value + const uint8_t& operator[](size_t index) const { + assert(data_ && index < size_); + return data_[index]; + } + + size_t size() const { return size_; } + uint8_t* data() const { return data_; } + uint8_t* bdata() const { return data_; } + uint8_t* begin() const { return data_; } + uint8_t* end() const { return data_ + size_; } + + /// \brief copy from + /// \param src the source + /// \param size the size + void copy_from(const uint8_t* src, size_t size) { + assert(size <= size_); + std::memcpy(data_, src, size); + } + + /// \brief resize + /// \param new_size the new size + void resize(size_t new_size) { +#ifdef __XRT__ + assert(!is_bo_owner_); +#endif + if (data_ != nullptr && !is_owner_) { + throw std::runtime_error("Cannot resize a non-owner buffer"); + } + if (new_size == 0) { + throw std::runtime_error("Cannot resize to zero size"); + } + try { + owned_data_.reset(new uint8_t[new_size]); + } + catch (const std::bad_alloc& e) { + throw std::runtime_error(std::string("Failed to allocate bytes of size ") + std::to_string(new_size) + ": " + e.what()); + } + data_ = owned_data_.get(); + size_ = new_size; + is_owner_ = true; + } + + /// \brief free, release the memory or the bo + void free() { +#ifdef __XRT__ + assert(!is_bo_owner_); +#endif + if (is_owner_){ + owned_data_.reset(); + } + data_ = nullptr; + size_ = 0; + is_owner_ = false; +#ifdef __XRT__ + if (is_bo_owner_){ + owned_bo_.reset(); + } + is_bo_owner_ = false; + bo_ = nullptr; +#endif + } + + /// \brief reserve + /// \param size the size + void reserve(size_t size) { resize(size); } + + /// \brief release + void release() { free(); } + + /// \brief is owner + /// \return the is owner + bool is_owner() const { return is_owner_; } +#ifdef __XRT__ + /// \brief is bo owner + /// \return the is bo owner + bool is_bo_owner() const { return is_bo_owner_; } + + /// \brief sync to device + void sync_to_device() { assert(bo_); bo_->sync(XCL_BO_SYNC_BO_TO_DEVICE); } + + /// \brief sync from device + void sync_from_device() { assert(bo_); bo_->sync(XCL_BO_SYNC_BO_FROM_DEVICE); } + + /// \brief bo + /// \return the bo + xrt::bo& bo() { assert(bo_); return *bo_; } +#endif + + /// \brief from file + /// \param filename the filename + /// \param offset the offset + /// \param size the size + void from_file(const std::string& filename, size_t offset = 0, size_t size = 0) { + std::ifstream file(filename, std::ios::binary); + if (!file.is_open()) { + throw std::runtime_error("Failed to open file: " + filename); + } + file.seekg(0, std::ios::end); + size_t file_size = file.tellg(); + file.seekg(0, std::ios::beg); + if (size == 0) size = file_size; + assert(size <= file_size); + assert(offset + size <= size_); + file.read(reinterpret_cast(data_) + offset, size); + file.close(); + } +}; + +/// \brief buffer class +/// \note This class wraps a data type T over the underlying byte buffer. +template +class buffer : public bytes { +public: + /// \brief constructor + buffer() : bytes() {} + + /// \brief constructor + /// \param count the count + buffer(size_t count) : bytes(count * sizeof(T)) {} + + /// \brief constructor + /// \param data the data + /// \param count the count + buffer(T* data, size_t count) + : bytes(reinterpret_cast(data), count * sizeof(T)) {} + + /// \brief shallow copy constructor + /// \param other the other buffer + buffer(const buffer& other) : bytes(other) {} + + /// \brief move constructor + /// \param other the other buffer + /// \note Transfers ownership (owned_data_/owned_bo_) so a returned buffer does not dangle. + buffer(buffer&& other) noexcept : bytes(std::move(other)) {} + +#ifdef __XRT__ + /// \brief constructor + /// \param bo the bo + buffer(xrt::bo& bo) : bytes(bo) {} + + /// \brief constructor + /// \param count the count + /// \param device the device + /// \param kernel the kernel + /// \param group_id the group id + /// \param flags the flags + buffer(xrt::device& device, size_t count) + : bytes(device, count * sizeof(T)) {} +#endif + + /// \brief constructor + /// \param vec the vector + buffer(const std::vector& vec) + : bytes(reinterpret_cast(const_cast(vec.data())), vec.size() * sizeof(T)) + { + } + + /// \brief constructor + /// \param vec the vector + /// \warning This also creates a shallow mapping. + /// \warning The caller must ensure that the vector is not used (and remains valid) + /// \warning after constructing this buffer. + buffer(std::vector&& vec) + : bytes(reinterpret_cast(vec.data()), vec.size() * sizeof(T)) + { + } + + /// \brief copy from + /// \param vec the vector + void copy_from(const std::vector& vec) { + if (vec.size() * sizeof(T) != this->size_) { + throw std::runtime_error("Size mismatch in copy_from(vector)"); + } + std::memcpy(data_, vec.data(), size_); + } + + /// \brief cast to another type + /// \tparam U the type + /// \return the buffer + template + buffer cast_to() { + size_t newCount = size_ / sizeof(U); + return buffer(reinterpret_cast(data_), newCount); + } + + /// \brief as bytes + /// \return the bytes + const bytes as_bytes() const { + return *this; + } + + /// \brief move assignment operator + /// \param other the other buffer + /// \return the buffer + buffer& operator=(buffer&& other) noexcept { + bytes::operator=(std::move(other)); + return *this; + } + + /// \brief copy assignment operator + /// \param other the other buffer + /// \return the buffer + buffer& operator=(const buffer& other) { + bytes::operator=(other); + return *this; + } + + /// \brief operator [] + /// \param index the index + /// \return the value + T& operator[](size_t index) { + assert(data_ != nullptr); + assert(index < size()); + assert(index >= 0); + return reinterpret_cast(data_)[index]; + } + + /// \brief operator [] + /// \param index the index + /// \return the value + const T& operator[](size_t index) const { + assert(data_ != nullptr); + assert(index < size()); + assert(index >= 0); + return reinterpret_cast(data_)[index]; + } + + /// \brief size, number of elements + /// \return the size + size_t size() const { return size_ / sizeof(T); } + + /// \brief data + /// \return the data + T* data() const { return reinterpret_cast(data_); } + + /// \brief begin + /// \return the pointer to the first element + T* begin() const { return reinterpret_cast(data_); } + + /// \brief end + /// \return the pointer to the last element + T* end() const { return reinterpret_cast(data_) + size(); } + + /// \brief resize + /// \param count: the number of elements + void resize(size_t count) { + bytes::resize(count * sizeof(T)); + } + + /// \brief reserve + /// \param count: the number of elements + void reserve(size_t count) { + bytes::reserve(count * sizeof(T)); + } + + /// \brief memset + /// \param value the value + void memset(T value) { + T* ptr = data(); + for (size_t i = 0; i < size(); i++) { + ptr[i] = value; + } + } + + /// \brief copy from + /// \param other the other bytes + void copy_from(const bytes& other) { + if (size_ != other.size()) { + throw std::runtime_error("Size mismatch in copy_from(bytes)"); + } + memcpy(data_, other.data(), size_); // size_ is already in bytes. + } + + /// \brief copy from + /// \param other the other buffer + void copy_from(const buffer& other) { + if (size() != other.size()) { + throw std::runtime_error("Size mismatch in copy_from(buffer)"); + } + memcpy(data_, other.bdata(), size_); // size_ is already in bytes. + } + + /// \brief copy from + /// \param data the data + /// \param size the number of elements + void copy_from(T* data, size_t size) { + if (size > this->size()) { + throw std::runtime_error("Size mismatch in copy_from(pointer)"); + } + memcpy(data_, data, size * sizeof(T)); + } + + /// \brief as bytes + /// \return the bytes + bytes& as_bytes() { + return *this; + } + + /// \brief from file + /// \param filename the filename + /// \param offset the offset + /// \param size the number of elements + void from_file(const std::string& filename, size_t offset = 0, size_t size = 0) { + bytes::from_file(filename, offset * sizeof(T), size * sizeof(T)); + } +}; \ No newline at end of file diff --git a/src/include/lm_config.hpp b/src/include/lm_config.hpp index 98d304f6..20721361 100644 --- a/src/include/lm_config.hpp +++ b/src/include/lm_config.hpp @@ -11,52 +11,70 @@ #include "nlohmann/json.hpp" #include +/// \brief read one parameter out of a config json +/// \note Same semantics as JSON_GET (missing OR null falls back to the default), +/// in expression form. Plain nlohmann .value() throws on a null, and real +/// configs do carry nulls (e.g. "sliding_window": null on qwen3). +template +inline T cfg_get(const nlohmann::json& jc, const char* key, T default_value){ + if (jc.contains(key) && !jc[key].is_null()){ + return T(jc[key]); + } + return default_value; +} + /// \brief LM_Config class +/// \note Model parameters are NOT cached as members. Everything comes from the +/// model's config.json, so every consumer reads what it needs straight out +/// of _json_config with JSON_GET. from_pretrained() only locates the file +/// and normalizes it, so that all readers share one canonical key set. +/// \note This class is passed by value across the FLM_DLL boundary. Its layout +/// must stay identical to FLM_DLL/include/lm_config.hpp. class LM_Config{ public: std::string model_path; std::string model_name; - std::string model_type; - u32 head_dim; - u32 hidden_size; - std::string hidden_act; - u32 intermediate_size; - u32 num_attention_heads; - u32 num_hidden_layers; - u32 num_key_value_heads; - u32 linear_conv_kernel_dim; - u32 linear_key_head_dim; - u32 linear_num_key_heads; - u32 linear_num_value_heads; - u32 linear_value_head_dim; - u32 pretraining_tp; - f32 rms_norm_eps; - f32 rope_theta; - u32 vocab_size; - u32 sliding_window; - u32 sliding_window_pattern; - u32 addr_qk; - u32 addr_kv; - u32 addr_l_begin_mha; - u32 addr_l_end_mha; - u32 addr_kk; - std::string flm_version; std::string exec_path; + std::string flm_version; - //vision specific - std::string vision_model_weight; - nlohmann::json _vision_config; - std::string audio_model_weight; - nlohmann::json _audio_config; - bool is_vlm; - bool is_audio; + nlohmann::json _json_config; + /// \brief read one model parameter out of config.json + /// \note Defaults to u32 because most parameters are dimensions: + /// config.get("head_dim"), config.get("rms_norm_eps", 0.0f), + /// config.get("vision_model_weight", ""). + template + T get(const char* key, T default_value = T(0)) const { + return cfg_get(this->_json_config, key, default_value); + } - nlohmann::json _json_config; + /// \brief read a nested config object (vision_config, audio_config, ...) + /// \return the sub-object, or an empty object when absent/null + const nlohmann::json& sub(const char* key) const { + static const nlohmann::json empty = nlohmann::json::object(); + if (this->_json_config.contains(key) && !this->_json_config[key].is_null()){ + return this->_json_config[key]; + } + return empty; + } /// \brief from pretrained /// \param model_name the model name void from_pretrained(std::string model_name){ + this->_resolve_paths(model_name); + this->_load_json(); + this->_normalize_multi_modal(); + JSON_GET(this->flm_version, this->_json_config, "flm_version", "0.0.0", std::string); + } + + std::string _str(){ + return this->_str_from(this->_json_config); + } + LM_Config(){} + + protected: + /// \brief resolve model_path / model_name / exec_path + void _resolve_paths(const std::string& model_name){ // #define DEV_BUILD #ifdef DEV_BUILD #ifdef __WINDOWS__ @@ -69,86 +87,87 @@ class LM_Config{ #endif this->model_path = model_name; this->model_name = std::filesystem::path(model_name).filename().string(); + } + + /// \brief read model_path/config.json into _json_config + void _load_json(){ std::ifstream file(this->model_path + "/config.json"); if (!file.is_open()){ - std::cerr << "Failed to open file: " << model_name << std::endl; + std::cerr << "Failed to open file: " << this->model_path << std::endl; exit(1); } // read the json file as a string std::string json_str((std::istreambuf_iterator(file)), std::istreambuf_iterator()); this->_json_config = nlohmann::json::parse(json_str); - JSON_GET(this->sliding_window, this->_json_config, "sliding_window", 0, u32); - JSON_GET(this->sliding_window_pattern, this->_json_config, "sliding_window_pattern", 0, u32); - JSON_GET(this->model_type, this->_json_config, "model_type", "", std::string); - JSON_GET(this->head_dim, this->_json_config, "head_dim", 0, u32); - JSON_GET(this->hidden_size, this->_json_config, "hidden_size", 0, u32); - JSON_GET(this->hidden_act, this->_json_config, "hidden_act", "", std::string); - JSON_GET(this->intermediate_size, this->_json_config, "intermediate_size", 0, u32); - JSON_GET(this->num_attention_heads, this->_json_config, "num_attention_heads", 0, u32); - JSON_GET(this->linear_conv_kernel_dim, this->_json_config, "linear_conv_kernel_dim", 0, u32); - JSON_GET(this->linear_key_head_dim, this->_json_config, "linear_key_head_dim", 0, u32); - JSON_GET(this->linear_num_key_heads, this->_json_config, "linear_num_key_heads", 0, u32); - JSON_GET(this->linear_num_value_heads, this->_json_config, "linear_num_value_heads", 0, u32); - JSON_GET(this->linear_value_head_dim, this->_json_config, "linear_value_head_dim", 0, u32); - - JSON_GET(this->num_hidden_layers, this->_json_config, "num_hidden_layers", 0, u32); - JSON_GET(this->num_key_value_heads, this->_json_config, "num_key_value_heads", 0, u32); - JSON_GET(this->pretraining_tp, this->_json_config, "pretraining_tp", 0, u32); - JSON_GET(this->rms_norm_eps, this->_json_config, "rms_norm_eps", 0.0, f32); - JSON_GET(this->rope_theta, this->_json_config, "rope_theta", 0.0, f32); - JSON_GET(this->vocab_size, this->_json_config, "vocab_size", 0, u32); - JSON_GET(this->addr_qk, this->_json_config, "addr_qk", 0, u32); - JSON_GET(this->addr_kv, this->_json_config, "addr_kv", 0, u32); - JSON_GET(this->addr_l_begin_mha, this->_json_config, "addr_l_begin_mha", 0, u32); - JSON_GET(this->addr_l_end_mha, this->_json_config, "addr_l_end_mha", 0, u32); - JSON_GET(this->addr_kk, this->_json_config, "addr_kk", 0, u32); - - - // config for vision - { - JSON_GET(this->vision_model_weight, this->_json_config, "vision_model_weight", "", std::string); - JSON_GET(this->_vision_config, this->_json_config, "vision_config", nlohmann::json::object(), nlohmann::json); - JSON_GET(this->_audio_config, this->_json_config, "audio_config", nlohmann::json::object(), nlohmann::json); - JSON_GET(this->audio_model_weight, this->_json_config, "audio_model_weight", "", std::string); + } + + /// \brief turn the relative weight names into full paths and publish the + /// derived is_vlm / is_audio flags, so consumers can read them + /// from json like every other parameter. + void _normalize_multi_modal(){ + std::string vision_model_weight; + std::string audio_model_weight; + JSON_GET(vision_model_weight, this->_json_config, "vision_model_weight", "", std::string); + JSON_GET(audio_model_weight, this->_json_config, "audio_model_weight", "", std::string); + this->_json_config["is_vlm"] = !vision_model_weight.empty(); + this->_json_config["is_audio"] = !audio_model_weight.empty(); + if (!vision_model_weight.empty()){ + this->_json_config["vision_model_weight"] = this->model_path + "/" + vision_model_weight; } - this->is_vlm = this->vision_model_weight != ""; - this->is_audio = this->audio_model_weight != ""; + if (!audio_model_weight.empty()){ + this->_json_config["audio_model_weight"] = this->model_path + "/" + audio_model_weight; + } + } + /// \brief shared pretty printer, reads everything from the normalized json + std::string _str_from(const nlohmann::json& jc){ + u32 head_dim, hidden_size, intermediate_size; + u32 num_attention_heads, num_hidden_layers, num_key_value_heads; + u32 pretraining_tp, sliding_window, sliding_window_pattern; + f32 rms_norm_eps; + std::string hidden_act, vision_model_weight; + bool is_vlm; + JSON_GET(head_dim, jc, "head_dim", 0, u32); + JSON_GET(hidden_size, jc, "hidden_size", 0, u32); + JSON_GET(hidden_act, jc, "hidden_act", "", std::string); + JSON_GET(intermediate_size, jc, "intermediate_size", 0, u32); + JSON_GET(num_attention_heads, jc, "num_attention_heads", 0, u32); + JSON_GET(num_hidden_layers, jc, "num_hidden_layers", 0, u32); + JSON_GET(num_key_value_heads, jc, "num_key_value_heads", 0, u32); + JSON_GET(pretraining_tp, jc, "pretraining_tp", 0, u32); + JSON_GET(rms_norm_eps, jc, "rms_norm_eps", 0.0, f32); + JSON_GET(sliding_window, jc, "sliding_window", 0, u32); + JSON_GET(sliding_window_pattern, jc, "sliding_window_pattern", 0, u32); + JSON_GET(is_vlm, jc, "is_vlm", false, bool); + JSON_GET(vision_model_weight, jc, "vision_model_weight", "", std::string); - JSON_GET(this->flm_version, this->_json_config, "flm_version", "0.0.0", std::string); - - this->vision_model_weight = this->model_path + "/" + this->vision_model_weight; - this->audio_model_weight = this->model_path + "/" + this->audio_model_weight; - } - std::string _str(){ std::stringstream ss; ss << " Model: " << std::endl; ss << " model_name: " << this->model_name << std::endl; ss << " compatible_flm_version: >= " << this->flm_version << std::endl; - ss << " head_dim: " << this->head_dim << std::endl; - ss << " hidden_size: " << this->hidden_size << std::endl; - if (this->hidden_act != ""){ - ss << " hidden_act: " << this->hidden_act << std::endl; + ss << " head_dim: " << head_dim << std::endl; + ss << " hidden_size: " << hidden_size << std::endl; + if (hidden_act != ""){ + ss << " hidden_act: " << hidden_act << std::endl; } - ss << " intermediate_size: " << this->intermediate_size << std::endl; - ss << " num_attention_heads: " << this->num_attention_heads << std::endl; - ss << " num_hidden_layers: " << this->num_hidden_layers << std::endl; - ss << " num_key_value_heads: " << this->num_key_value_heads << std::endl; - ss << " pretraining_tp: " << this->pretraining_tp << std::endl; - ss << " rms_norm_eps: " << this->rms_norm_eps << std::endl; - if (this->sliding_window > 0){ - ss << " sliding_window: " << this->sliding_window << std::endl; - ss << " sliding_window_pattern: " << this->sliding_window_pattern << std::endl; + ss << " intermediate_size: " << intermediate_size << std::endl; + ss << " num_attention_heads: " << num_attention_heads << std::endl; + ss << " num_hidden_layers: " << num_hidden_layers << std::endl; + ss << " num_key_value_heads: " << num_key_value_heads << std::endl; + ss << " pretraining_tp: " << pretraining_tp << std::endl; + ss << " rms_norm_eps: " << rms_norm_eps << std::endl; + if (sliding_window > 0){ + ss << " sliding_window: " << sliding_window << std::endl; + ss << " sliding_window_pattern: " << sliding_window_pattern << std::endl; } - if(this->is_vlm){ + if (is_vlm){ ss << " Vision: " << std::endl; - ss << " vision_model_weight: " << this->vision_model_weight << std::endl; + ss << " vision_model_weight: " << vision_model_weight << std::endl; } return ss.str(); } - LM_Config(){} }; class Whisper_Config : public LM_Config{ @@ -156,73 +175,26 @@ class Whisper_Config : public LM_Config{ /// \brief from pretrained /// \param model_name the model name void from_pretrained(std::string model_name){ - this->model_path = model_name; - #ifdef DEV_BUILD - #ifdef __WINDOWS__ - this->exec_path = "..\\..\\..\\"; - #else - this->exec_path = "../../../"; - #endif - #else - this->exec_path = utils::find_xclbin_path(); - #endif - this->model_name = std::filesystem::path(model_name).filename().string(); - std::ifstream file(model_name + "/config.json"); - if (!file.is_open()){ - std::cerr << "Failed to open file: " << model_name << std::endl; - exit(1); - } - // read the json file as a string - std::string json_str((std::istreambuf_iterator(file)), std::istreambuf_iterator()); - this->_json_config = nlohmann::json::parse(json_str); - JSON_GET(this->sliding_window, this->_json_config, "sliding_window", 0, u32); - JSON_GET(this->sliding_window_pattern, this->_json_config, "sliding_window_pattern", 0, u32); - JSON_GET(this->model_type, this->_json_config, "model_type", "", std::string); - JSON_GET(this->head_dim, this->_json_config, "head_dim", 0, u32); - JSON_GET(this->hidden_size, this->_json_config, "d_model", 0, u32); - JSON_GET(this->hidden_act, this->_json_config, "hidden_act", "", std::string); - JSON_GET(this->intermediate_size, this->_json_config, "decoder_ffn_dim", 0, u32); - JSON_GET(this->num_attention_heads, this->_json_config, "num_attention_heads", 0, u32); - JSON_GET(this->num_hidden_layers, this->_json_config, "num_hidden_layers", 0, u32); - JSON_GET(this->num_key_value_heads, this->_json_config, "num_key_value_heads", 0, u32); - JSON_GET(this->pretraining_tp, this->_json_config, "pretraining_tp", 0, u32); - JSON_GET(this->rms_norm_eps, this->_json_config, "rms_norm_eps", 0.0, f32); - JSON_GET(this->rope_theta, this->_json_config, "rope_theta", 0.0, f32); - JSON_GET(this->vocab_size, this->_json_config, "vocab_size", 0, u32); - JSON_GET(this->addr_qk, this->_json_config, "addr_qk", 0, u32); - JSON_GET(this->addr_kv, this->_json_config, "addr_kv", 0, u32); - JSON_GET(this->addr_l_begin_mha, this->_json_config, "addr_l_begin_mha", 0, u32); - JSON_GET(this->addr_l_end_mha, this->_json_config, "addr_l_end_mha", 0, u32); - JSON_GET(this->addr_kk, this->_json_config, "addr_kk", 0, u32); - - this->is_vlm = false; - this->vocab_size = (this->vocab_size + 31) / 32 * 32; + this->_resolve_paths(model_name); + this->_load_json(); + + // whisper spells the common dimensions differently; republish them under + // the canonical keys so downstream readers stay model agnostic. + u32 hidden_size, intermediate_size, vocab_size; + JSON_GET(hidden_size, this->_json_config, "d_model", 0, u32); + JSON_GET(intermediate_size, this->_json_config, "decoder_ffn_dim", 0, u32); + JSON_GET(vocab_size, this->_json_config, "vocab_size", 0, u32); + this->_json_config["hidden_size"] = hidden_size; + this->_json_config["intermediate_size"] = intermediate_size; + this->_json_config["vocab_size"] = (vocab_size + 31) / 32 * 32; + + this->_json_config["is_vlm"] = false; + this->_json_config["is_audio"] = false; JSON_GET(this->flm_version, this->_json_config, "flm_version", "0.0.0", std::string); - - this->vision_model_weight = this->model_path + "/" + this->vision_model_weight; } std::string _str(){ - std::stringstream ss; - ss << " Model: " << std::endl; - ss << " model_name: " << this->model_name << std::endl; - ss << " compatible_flm_version: >= " << this->flm_version << std::endl; - ss << " head_dim: " << this->head_dim << std::endl; - ss << " hidden_size: " << this->hidden_size << std::endl; - if (this->hidden_act != ""){ - ss << " hidden_act: " << this->hidden_act << std::endl; - } - ss << " intermediate_size: " << this->intermediate_size << std::endl; - ss << " num_attention_heads: " << this->num_attention_heads << std::endl; - ss << " num_hidden_layers: " << this->num_hidden_layers << std::endl; - ss << " num_key_value_heads: " << this->num_key_value_heads << std::endl; - ss << " pretraining_tp: " << this->pretraining_tp << std::endl; - ss << " rms_norm_eps: " << this->rms_norm_eps << std::endl; - if (this->sliding_window > 0){ - ss << " sliding_window: " << this->sliding_window << std::endl; - ss << " sliding_window_pattern: " << this->sliding_window_pattern << std::endl; - } - return ss.str(); + return this->_str_from(this->_json_config); } Whisper_Config(){} -}; \ No newline at end of file +}; diff --git a/src/include/models/gemma4e/gemma4e_npu.hpp b/src/include/models/gemma4e/gemma4e_npu.hpp index 9bdf4971..573d6031 100644 --- a/src/include/models/gemma4e/gemma4e_npu.hpp +++ b/src/include/models/gemma4e/gemma4e_npu.hpp @@ -178,45 +178,45 @@ class gemma4e_npu : public causal_lm{ inline void load_vision_preprocess_parameters(LM_Config& config){ // Note: this should be called by Impl:: constructor - GEMMA4E_VISION_MAX_POSITION_EMBEDDINGS = config._vision_config.value("GEMMA4E_VISION_MAX_POSITION_EMBEDDINGS", -1); - GEMMA4E_VISION_NUM_HIDDEN_LAYERS = config._vision_config.value("GEMMA4E_VISION_NUM_HIDDEN_LAYERS", -1); - GEMMA4E_VISION_NUM_ATTENTION_HEADS = config._vision_config.value("GEMMA4E_VISION_NUM_ATTENTION_HEADS", -1); - GEMMA4E_VISION_HIDDEN_SIZE = config._vision_config.value("GEMMA4E_VISION_HIDDEN_SIZE", -1); - GEMMA4E_VISION_INTERMEDIATE_SIZE = config._vision_config.value("GEMMA4E_VISION_INTERMEDIATE_SIZE", -1); - GEMMA4E_VISION_HEAD_DIM = config._vision_config.value("GEMMA4E_VISION_HEAD_DIM", -1); - GEMMA4E_VISION_PATCH_SIZE = config._vision_config.value("GEMMA4E_VISION_PATCH_SIZE", -1); - GEMMA4E_ROPE_THETA = config._vision_config.value("GEMMA4E_ROPE_THETA", -1.0f); - GEMMA4E_POOLING_KERNEL_SIZE = config._vision_config.value("GEMMA4E_POOLING_KERNEL_SIZE", -1); - GEMMA4E_POSITION_EMBEDDING_SIZE = config._vision_config.value("GEMMA4E_POSITION_EMBEDDING_SIZE", -1); - GEMMA4E_VISION_IMAGE_OUTPUT_SIZE = config._vision_config.value("GEMMA4E_VISION_IMAGE_OUTPUT_SIZE", -1); - GEMMA4E_VISION_RESCALE_FACTOR = config._vision_config.value("GEMMA4E_VISION_RESCALE_FACTOR", -1.0f); - GEMMA4E_VISION_IMAGE_MEAN = config._vision_config.value("GEMMA4E_VISION_IMAGE_MEAN", -1.0f); - GEMMA4E_VISION_IMAGE_STD = config._vision_config.value("GEMMA4E_VISION_IMAGE_STD", -1.0f); + GEMMA4E_VISION_MAX_POSITION_EMBEDDINGS = config.sub("vision_config").value("GEMMA4E_VISION_MAX_POSITION_EMBEDDINGS", -1); + GEMMA4E_VISION_NUM_HIDDEN_LAYERS = config.sub("vision_config").value("GEMMA4E_VISION_NUM_HIDDEN_LAYERS", -1); + GEMMA4E_VISION_NUM_ATTENTION_HEADS = config.sub("vision_config").value("GEMMA4E_VISION_NUM_ATTENTION_HEADS", -1); + GEMMA4E_VISION_HIDDEN_SIZE = config.sub("vision_config").value("GEMMA4E_VISION_HIDDEN_SIZE", -1); + GEMMA4E_VISION_INTERMEDIATE_SIZE = config.sub("vision_config").value("GEMMA4E_VISION_INTERMEDIATE_SIZE", -1); + GEMMA4E_VISION_HEAD_DIM = config.sub("vision_config").value("GEMMA4E_VISION_HEAD_DIM", -1); + GEMMA4E_VISION_PATCH_SIZE = config.sub("vision_config").value("GEMMA4E_VISION_PATCH_SIZE", -1); + GEMMA4E_ROPE_THETA = config.sub("vision_config").value("GEMMA4E_ROPE_THETA", -1.0f); + GEMMA4E_POOLING_KERNEL_SIZE = config.sub("vision_config").value("GEMMA4E_POOLING_KERNEL_SIZE", -1); + GEMMA4E_POSITION_EMBEDDING_SIZE = config.sub("vision_config").value("GEMMA4E_POSITION_EMBEDDING_SIZE", -1); + GEMMA4E_VISION_IMAGE_OUTPUT_SIZE = config.sub("vision_config").value("GEMMA4E_VISION_IMAGE_OUTPUT_SIZE", -1); + GEMMA4E_VISION_RESCALE_FACTOR = config.sub("vision_config").value("GEMMA4E_VISION_RESCALE_FACTOR", -1.0f); + GEMMA4E_VISION_IMAGE_MEAN = config.sub("vision_config").value("GEMMA4E_VISION_IMAGE_MEAN", -1.0f); + GEMMA4E_VISION_IMAGE_STD = config.sub("vision_config").value("GEMMA4E_VISION_IMAGE_STD", -1.0f); } inline void load_audio_preprocess_parameters(LM_Config& config){ - Audio_MM_TILE_M = config._audio_config.value("Audio_MM_TILE_M", 128); - Audio_MM_TILE_K = config._audio_config.value("Audio_MM_TILE_K", 512); - Audio_MM_TILE_N = config._audio_config.value("Audio_MM_TILE_N", 64); - Gemma4E_Audio_resample_rate = config._audio_config.value("Gemma4E_Audio_audio_resample_rate", -1); - Gemma4E_Audio_gradient_clipping = config._audio_config.value("Gemma4E_Audio_gradient_clipping", -1.0f); - Gemma4E_Audio_Multimodal_Output_SIZE = config._audio_config.value("Gemma4E_Audio_Multimodal_Output_SIZE", -1); - Gemma4E_Audio_language_projection_output_size = config._audio_config.value("Gemma4E_Audio_language_projection_output_size", -1); - Gemma4E_Audio_HIDDEN_SIZE = config._audio_config.value("Gemma4E_Audio_HIDDEN_SIZE", -1); - Gemma4E_Audio_INTERMEDIATE_SIZE = config._audio_config.value("Gemma4E_Audio_INTERMEDIATE_SIZE", -1); - Gemma4E_Audio_attention_chunk_size = config._audio_config.value("Gemma4E_Audio_attention_chunk_size", -1); - Gemma4E_Audio_attention_context_left = config._audio_config.value("Gemma4E_Audio_attention_context_left", -1); - Gemma4E_Audio_attention_context_right = config._audio_config.value("Gemma4E_Audio_attention_context_right", -1); - Gemma4E_Audio_num_attention_heads = config._audio_config.value("Gemma4E_Audio_num_attention_heads", -1); - Gemma4E_Audio_num_attention_layers = config._audio_config.value("Gemma4E_Audio_num_attention_layers", -1); - Gemma4E_Audio_conv1d_kernel_size = config._audio_config.value("Gemma4E_Audio_conv1d_kernel_size", -1); - Gemma4E_Audio_conv1d_stride = config._audio_config.value("Gemma4E_Audio_conv1d_stride", -1); - Gemma4E_Audio_conv2d_kernel_size = config._audio_config.value("Gemma4E_conv2d_kernel_size", -1); - Gemma4E_Audio_conv2d_Stride = config._audio_config.value("Gemma4E_conv2d_Stride", -1); - Gemma4e_Audio_conv2d_Padding = config._audio_config.value("Gemma4e_conv2d_Padding", -1); - Gemma4E_Audio_subsampling_conv_channels_0 = config._audio_config.value("Gemma4E_Audio_subsampling_conv_channels_0", -1); - Gemma4E_Audio_subsampling_conv_channels_1 = config._audio_config.value("Gemma4E_Audio_subsampling_conv_channels_1", -1); - Gemma4E_Audio_attention_softcap = config._audio_config.value("Gemma4E_Audio_attention_softcap", -1.0f); + Audio_MM_TILE_M = config.sub("audio_config").value("Audio_MM_TILE_M", 128); + Audio_MM_TILE_K = config.sub("audio_config").value("Audio_MM_TILE_K", 512); + Audio_MM_TILE_N = config.sub("audio_config").value("Audio_MM_TILE_N", 64); + Gemma4E_Audio_resample_rate = config.sub("audio_config").value("Gemma4E_Audio_audio_resample_rate", -1); + Gemma4E_Audio_gradient_clipping = config.sub("audio_config").value("Gemma4E_Audio_gradient_clipping", -1.0f); + Gemma4E_Audio_Multimodal_Output_SIZE = config.sub("audio_config").value("Gemma4E_Audio_Multimodal_Output_SIZE", -1); + Gemma4E_Audio_language_projection_output_size = config.sub("audio_config").value("Gemma4E_Audio_language_projection_output_size", -1); + Gemma4E_Audio_HIDDEN_SIZE = config.sub("audio_config").value("Gemma4E_Audio_HIDDEN_SIZE", -1); + Gemma4E_Audio_INTERMEDIATE_SIZE = config.sub("audio_config").value("Gemma4E_Audio_INTERMEDIATE_SIZE", -1); + Gemma4E_Audio_attention_chunk_size = config.sub("audio_config").value("Gemma4E_Audio_attention_chunk_size", -1); + Gemma4E_Audio_attention_context_left = config.sub("audio_config").value("Gemma4E_Audio_attention_context_left", -1); + Gemma4E_Audio_attention_context_right = config.sub("audio_config").value("Gemma4E_Audio_attention_context_right", -1); + Gemma4E_Audio_num_attention_heads = config.sub("audio_config").value("Gemma4E_Audio_num_attention_heads", -1); + Gemma4E_Audio_num_attention_layers = config.sub("audio_config").value("Gemma4E_Audio_num_attention_layers", -1); + Gemma4E_Audio_conv1d_kernel_size = config.sub("audio_config").value("Gemma4E_Audio_conv1d_kernel_size", -1); + Gemma4E_Audio_conv1d_stride = config.sub("audio_config").value("Gemma4E_Audio_conv1d_stride", -1); + Gemma4E_Audio_conv2d_kernel_size = config.sub("audio_config").value("Gemma4E_conv2d_kernel_size", -1); + Gemma4E_Audio_conv2d_Stride = config.sub("audio_config").value("Gemma4E_conv2d_Stride", -1); + Gemma4e_Audio_conv2d_Padding = config.sub("audio_config").value("Gemma4e_conv2d_Padding", -1); + Gemma4E_Audio_subsampling_conv_channels_0 = config.sub("audio_config").value("Gemma4E_Audio_subsampling_conv_channels_0", -1); + Gemma4E_Audio_subsampling_conv_channels_1 = config.sub("audio_config").value("Gemma4E_Audio_subsampling_conv_channels_1", -1); + Gemma4E_Audio_attention_softcap = config.sub("audio_config").value("Gemma4E_Audio_attention_softcap", -1.0f); } private: diff --git a/src/include/models/qwen3_5vl/qwen3_5vl_npu.hpp b/src/include/models/qwen3_5vl/qwen3_5vl_npu.hpp index 83d83ef0..7dc70fd7 100644 --- a/src/include/models/qwen3_5vl/qwen3_5vl_npu.hpp +++ b/src/include/models/qwen3_5vl/qwen3_5vl_npu.hpp @@ -126,15 +126,15 @@ class qwen3_5vl_npu : public causal_lm{ inline void load_vision_preprocess_parameters(LM_Config& config){ // Note: this should be called by Impl:: constructor - QWEN3_5_PATCH_SIZE = config._vision_config.value("QWEN3_5_PATCH_SIZE", -1); - QWEN3_5_IMAGE_MERGE_SIZE = config._vision_config.value("QWEN3_5_IMAGE_MERGE_SIZE", -1); - QWEN3_5_SPATIAL_MERGE_SIZE = config._vision_config.value("QWEN3_5_SPATIAL_MERGE_SIZE", -1); - QWEN3_5_SHORTEST_EDGE = config._vision_config.value("QWEN3_5_SHORTEST_EDGE", -1); - QWEN3_5_LONGEST_EDGE = config._vision_config.value("QWEN3_5_LONGEST_EDGE", -1); - QWEN3_5_VISION_RESCALE_FACTOR = config._vision_config.value("QWEN3_5_VISION_RESCALE_FACTOR", -1.0f); - QWEN3_5_VISION_RESCALE_IMAGE_MEAN = config._vision_config.value("QWEN3_5_VISION_RESCALE_IMAGE_MEAN", -1.0f); - QWEN3_5_VISION_RESCALE_IMAGE_STD = config._vision_config.value("QWEN3_5_VISION_RESCALE_IMAGE_STD", -1.0f); - QWEN3_5_TEMPORAL_PATCH_SIZE = config._vision_config.value("QWEN3_5_TEMPORAL_PATCH_SIZE", -1); + QWEN3_5_PATCH_SIZE = config.sub("vision_config").value("QWEN3_5_PATCH_SIZE", -1); + QWEN3_5_IMAGE_MERGE_SIZE = config.sub("vision_config").value("QWEN3_5_IMAGE_MERGE_SIZE", -1); + QWEN3_5_SPATIAL_MERGE_SIZE = config.sub("vision_config").value("QWEN3_5_SPATIAL_MERGE_SIZE", -1); + QWEN3_5_SHORTEST_EDGE = config.sub("vision_config").value("QWEN3_5_SHORTEST_EDGE", -1); + QWEN3_5_LONGEST_EDGE = config.sub("vision_config").value("QWEN3_5_LONGEST_EDGE", -1); + QWEN3_5_VISION_RESCALE_FACTOR = config.sub("vision_config").value("QWEN3_5_VISION_RESCALE_FACTOR", -1.0f); + QWEN3_5_VISION_RESCALE_IMAGE_MEAN = config.sub("vision_config").value("QWEN3_5_VISION_RESCALE_IMAGE_MEAN", -1.0f); + QWEN3_5_VISION_RESCALE_IMAGE_STD = config.sub("vision_config").value("QWEN3_5_VISION_RESCALE_IMAGE_STD", -1.0f); + QWEN3_5_TEMPORAL_PATCH_SIZE = config.sub("vision_config").value("QWEN3_5_TEMPORAL_PATCH_SIZE", -1); QWEN3_5_MERGE_SIZE = QWEN3_5_IMAGE_MERGE_SIZE; diff --git a/src/include/models/qwen3_6_moe/qwen3_6_moe_npu.hpp b/src/include/models/qwen3_6_moe/qwen3_6_moe_npu.hpp index 2ea080bc..4f67b037 100644 --- a/src/include/models/qwen3_6_moe/qwen3_6_moe_npu.hpp +++ b/src/include/models/qwen3_6_moe/qwen3_6_moe_npu.hpp @@ -124,15 +124,15 @@ class qwen3_6_moe_npu : public causal_lm{ inline void load_vision_preprocess_parameters(LM_Config& config){ // Note: this should be called by Impl:: constructor - QWEN3_6_MOE_PATCH_SIZE = config._vision_config.value("QWEN3_6_MOE_PATCH_SIZE", -1); - QWEN3_6_MOE_IMAGE_MERGE_SIZE = config._vision_config.value("QWEN3_6_MOE_IMAGE_MERGE_SIZE", -1); - QWEN3_6_MOE_SPATIAL_MERGE_SIZE = config._vision_config.value("QWEN3_6_MOE_SPATIAL_MERGE_SIZE", -1); - QWEN3_6_MOE_SHORTEST_EDGE = config._vision_config.value("QWEN3_6_MOE_SHORTEST_EDGE", -1); - QWEN3_6_MOE_LONGEST_EDGE = config._vision_config.value("QWEN3_6_MOE_LONGEST_EDGE", -1); - QWEN3_6_MOE_VISION_RESCALE_FACTOR = config._vision_config.value("QWEN3_6_MOE_VISION_RESCALE_FACTOR", -1.0f); - QWEN3_6_MOE_VISION_RESCALE_IMAGE_MEAN = config._vision_config.value("QWEN3_6_MOE_VISION_RESCALE_IMAGE_MEAN", -1.0f); - QWEN3_6_MOE_VISION_RESCALE_IMAGE_STD = config._vision_config.value("QWEN3_6_MOE_VISION_RESCALE_IMAGE_STD", -1.0f); - QWEN3_6_MOE_TEMPORAL_PATCH_SIZE = config._vision_config.value("QWEN3_6_MOE_TEMPORAL_PATCH_SIZE", -1); + QWEN3_6_MOE_PATCH_SIZE = config.sub("vision_config").value("QWEN3_6_MOE_PATCH_SIZE", -1); + QWEN3_6_MOE_IMAGE_MERGE_SIZE = config.sub("vision_config").value("QWEN3_6_MOE_IMAGE_MERGE_SIZE", -1); + QWEN3_6_MOE_SPATIAL_MERGE_SIZE = config.sub("vision_config").value("QWEN3_6_MOE_SPATIAL_MERGE_SIZE", -1); + QWEN3_6_MOE_SHORTEST_EDGE = config.sub("vision_config").value("QWEN3_6_MOE_SHORTEST_EDGE", -1); + QWEN3_6_MOE_LONGEST_EDGE = config.sub("vision_config").value("QWEN3_6_MOE_LONGEST_EDGE", -1); + QWEN3_6_MOE_VISION_RESCALE_FACTOR = config.sub("vision_config").value("QWEN3_6_MOE_VISION_RESCALE_FACTOR", -1.0f); + QWEN3_6_MOE_VISION_RESCALE_IMAGE_MEAN = config.sub("vision_config").value("QWEN3_6_MOE_VISION_RESCALE_IMAGE_MEAN", -1.0f); + QWEN3_6_MOE_VISION_RESCALE_IMAGE_STD = config.sub("vision_config").value("QWEN3_6_MOE_VISION_RESCALE_IMAGE_STD", -1.0f); + QWEN3_6_MOE_TEMPORAL_PATCH_SIZE = config.sub("vision_config").value("QWEN3_6_MOE_TEMPORAL_PATCH_SIZE", -1); QWEN3_6_MOE_MERGE_SIZE = QWEN3_6_MOE_IMAGE_MERGE_SIZE; diff --git a/src/lib/dequant_new.dll b/src/lib/dequant_new.dll deleted file mode 100644 index 79cf0014..00000000 Binary files a/src/lib/dequant_new.dll and /dev/null differ diff --git a/src/lib/dequant_new.lib b/src/lib/dequant_new.lib deleted file mode 100644 index ef5bdf8f..00000000 Binary files a/src/lib/dequant_new.lib and /dev/null differ diff --git a/src/lib/libdequant.so b/src/lib/libdequant.so index 12378b38..46da90c9 100755 Binary files a/src/lib/libdequant.so and b/src/lib/libdequant.so differ diff --git a/src/lib/libdequant_new.so b/src/lib/libdequant_new.so deleted file mode 100755 index f1df287c..00000000 Binary files a/src/lib/libdequant_new.so and /dev/null differ diff --git a/src/lib/libgemm.so b/src/lib/libgemm.so index 5efde387..ed54dbc7 100755 Binary files a/src/lib/libgemm.so and b/src/lib/libgemm.so differ diff --git a/src/lib/libgemma4e_npu.so b/src/lib/libgemma4e_npu.so index 3bf595ce..fd4311ba 100755 Binary files a/src/lib/libgemma4e_npu.so and b/src/lib/libgemma4e_npu.so differ diff --git a/src/lib/libgemma_embedding.so b/src/lib/libgemma_embedding.so index 0348435b..ed862f5e 100755 Binary files a/src/lib/libgemma_embedding.so and b/src/lib/libgemma_embedding.so differ diff --git a/src/lib/libgemma_npu.so b/src/lib/libgemma_npu.so index f1e1432b..47a4d2f3 100755 Binary files a/src/lib/libgemma_npu.so and b/src/lib/libgemma_npu.so differ diff --git a/src/lib/libgemma_text_npu.so b/src/lib/libgemma_text_npu.so index 1215c7a1..1c4394ee 100755 Binary files a/src/lib/libgemma_text_npu.so and b/src/lib/libgemma_text_npu.so differ diff --git a/src/lib/libgpt_oss_npu.so b/src/lib/libgpt_oss_npu.so index 34ce9920..e5d0650b 100755 Binary files a/src/lib/libgpt_oss_npu.so and b/src/lib/libgpt_oss_npu.so differ diff --git a/src/lib/liblfm2_npu.so b/src/lib/liblfm2_npu.so index 79485ff7..27bc0c04 100755 Binary files a/src/lib/liblfm2_npu.so and b/src/lib/liblfm2_npu.so differ diff --git a/src/lib/libllama_npu.so b/src/lib/libllama_npu.so index 8391fe8c..82c5779d 100755 Binary files a/src/lib/libllama_npu.so and b/src/lib/libllama_npu.so differ diff --git a/src/lib/liblm_head.so b/src/lib/liblm_head.so index e8089758..fef6a2f4 100755 Binary files a/src/lib/liblm_head.so and b/src/lib/liblm_head.so differ diff --git a/src/lib/libmha.so b/src/lib/libmha.so index 41544eb2..3fa3474a 100755 Binary files a/src/lib/libmha.so and b/src/lib/libmha.so differ diff --git a/src/lib/libnanbeige_npu.so b/src/lib/libnanbeige_npu.so index 0dbd71b5..a2702b45 100755 Binary files a/src/lib/libnanbeige_npu.so and b/src/lib/libnanbeige_npu.so differ diff --git a/src/lib/libphi4_npu.so b/src/lib/libphi4_npu.so index 820ff7df..9d34e043 100755 Binary files a/src/lib/libphi4_npu.so and b/src/lib/libphi4_npu.so differ diff --git a/src/lib/libq4_npu_eXpress.so b/src/lib/libq4_npu_eXpress.so index ac0b1d74..d72b3fe6 100755 Binary files a/src/lib/libq4_npu_eXpress.so and b/src/lib/libq4_npu_eXpress.so differ diff --git a/src/lib/libqwen2_npu.so b/src/lib/libqwen2_npu.so index ae18498f..cb21daf6 100755 Binary files a/src/lib/libqwen2_npu.so and b/src/lib/libqwen2_npu.so differ diff --git a/src/lib/libqwen2vl_npu.so b/src/lib/libqwen2vl_npu.so index 9ac6c058..6e6d8a01 100755 Binary files a/src/lib/libqwen2vl_npu.so and b/src/lib/libqwen2vl_npu.so differ diff --git a/src/lib/libqwen3_5_omni_npu.so b/src/lib/libqwen3_5_omni_npu.so index deca80f3..1e138656 100644 Binary files a/src/lib/libqwen3_5_omni_npu.so and b/src/lib/libqwen3_5_omni_npu.so differ diff --git a/src/lib/libqwen3_5vl_npu.so b/src/lib/libqwen3_5vl_npu.so index 89d4edf0..33ea08e0 100755 Binary files a/src/lib/libqwen3_5vl_npu.so and b/src/lib/libqwen3_5vl_npu.so differ diff --git a/src/lib/libqwen3_6_moe_npu.so b/src/lib/libqwen3_6_moe_npu.so index 0838a11d..1feb3d15 100755 Binary files a/src/lib/libqwen3_6_moe_npu.so and b/src/lib/libqwen3_6_moe_npu.so differ diff --git a/src/lib/libqwen3_npu.so b/src/lib/libqwen3_npu.so index c82e2f0d..a54f8010 100755 Binary files a/src/lib/libqwen3_npu.so and b/src/lib/libqwen3_npu.so differ diff --git a/src/lib/libqwen3vl_npu.so b/src/lib/libqwen3vl_npu.so index 2cf34aa1..e381c721 100755 Binary files a/src/lib/libqwen3vl_npu.so and b/src/lib/libqwen3vl_npu.so differ diff --git a/src/lib/libwhisper_npu.so b/src/lib/libwhisper_npu.so index 19cf7880..0fde6d33 100755 Binary files a/src/lib/libwhisper_npu.so and b/src/lib/libwhisper_npu.so differ diff --git a/src/test/gemma_npu/test.cpp b/src/test/gemma_npu/test.cpp index 5b59b52b..88ea9a20 100644 --- a/src/test/gemma_npu/test.cpp +++ b/src/test/gemma_npu/test.cpp @@ -50,6 +50,7 @@ int main(int argc, char* argv[]) { if (short_prompt) { uniformed_input.prompt = "Describe this?"; uniformed_input.images.push_back("../../../tb_files/panda.png"); + uniformed_input.images.push_back("../../../tb_files/pcb.jpg"); uniformed_input.image_payload_types.push_back(input_payload_type_t::FILE_NAME); std::cout << "Prompt: " << uniformed_input.prompt << std::endl; std::cout << "Response: "; @@ -129,4 +130,4 @@ int main(int argc, char* argv[]) { std::cout << std::endl; return 0; -} \ No newline at end of file +} diff --git a/src/test/qwen2vl_npu/test.cpp b/src/test/qwen2vl_npu/test.cpp index 9526bcb3..9237e1fd 100644 --- a/src/test/qwen2vl_npu/test.cpp +++ b/src/test/qwen2vl_npu/test.cpp @@ -54,9 +54,11 @@ int main(int argc, char* argv[]) { lm_uniform_input_t uniformed_input; if (short_prompt) { - uniformed_input.prompt = "What is this?"; + uniformed_input.prompt = "What are these?"; std::cout << "Prompt: " << uniformed_input.prompt << std::endl; uniformed_input.images.push_back("../../../tb_files/panda.png"); + uniformed_input.images.push_back("../../../tb_files/pcb.jpg"); + uniformed_input.images.push_back("../../../tb_files/error.png"); std::cout << "Response: "; chat->start_total_timer(); std::string response = chat->generate_with_prompt(meta_info, uniformed_input, 1024, std::cout); @@ -64,27 +66,6 @@ int main(int argc, char* argv[]) { std::cout << std::endl; std::cout << std::endl; std::cout << chat->show_profile() << std::endl; - // uniformed_input.prompt = "How are you"; - uniformed_input.images.clear(); - uniformed_input.images.push_back("../../../tb_files/panda.png"); - std::cout << "Prompt: " << uniformed_input.prompt << std::endl; - std::cout << "Response: " << std::endl; - chat->start_total_timer(); - response = chat->generate_with_prompt(meta_info, uniformed_input, 1024, std::cout); - chat->stop_total_timer(); - std::cout << std::endl; - std::cout << std::endl; - std::cout << chat->show_profile() << std::endl; - uniformed_input.images.clear(); - uniformed_input.images.push_back("../../../tb_files/panda.png"); - std::cout << "Prompt: " << uniformed_input.prompt << std::endl; - std::cout << "Response: " << std::endl; - chat->start_total_timer(); - response = chat->generate_with_prompt(meta_info, uniformed_input, 1024, std::cout); - chat->stop_total_timer(); - std::cout << std::endl; - std::cout << std::endl; - std::cout << chat->show_profile() << std::endl; } else{ std::ifstream file("../../../../prompt.txt", std::ios::binary); @@ -116,4 +97,4 @@ int main(int argc, char* argv[]) { std::cout << std::endl; return 0; -} \ No newline at end of file +}