diff --git a/hrx-integration/fetch-hrx-release.sh b/hrx-integration/fetch-hrx-release.sh index 8b5f238f4..b763f329b 100755 --- a/hrx-integration/fetch-hrx-release.sh +++ b/hrx-integration/fetch-hrx-release.sh @@ -80,15 +80,18 @@ actual_sha="$(sha256sum "$tarball" | awk '{print $1}')" # Public package ships as .tar.zst (fall back to gzip for legacy assets). # Decompress zstd via the standalone binary piped into tar, so extraction does # not depend on tar being built with its zstd plugin, and fail with a clear -# message when the zstd package is not installed. +# message when the zstd package is not installed. Do not preserve archive mtimes: +# some HRX archives can carry future-dated entries, and CMake records imported +# package configs as regeneration inputs. Future config mtimes make Ninja rerun +# CMake until it aborts with "build.ninja still dirty after 100 tries". case "$HRX_RELEASE_ASSET" in *.tar.zst) command -v zstd >/dev/null 2>&1 || die "zstd is required to extract $HRX_RELEASE_ASSET but was not found on PATH (install the 'zstd' package)" - zstd -dc "$tarball" | tar -C "$out_dir" -xf - + zstd -dc "$tarball" | tar -m -C "$out_dir" -xf - ;; - *.tar.gz|*.tgz) tar -C "$out_dir" -xzf "$tarball" ;; - *) tar -C "$out_dir" -xf "$tarball" ;; + *.tar.gz|*.tgz) tar -m -C "$out_dir" -xzf "$tarball" ;; + *) tar -m -C "$out_dir" -xf "$tarball" ;; esac # Locate the HRX CMake package config; its dir feeds find_package(hrx CONFIG). diff --git a/hrx-integration/hrx-release.env b/hrx-integration/hrx-release.env index bfe659996..33e7b0b1c 100644 --- a/hrx-integration/hrx-release.env +++ b/hrx-integration/hrx-release.env @@ -3,10 +3,10 @@ # Pinned HRX (amdxdna) release consumed by the HRX runtime path. HRX_RELEASE_REPO="jtuyls/hrx" -HRX_RELEASE_TAG="flm-hrx-amdxdna-v2026.07.30" +HRX_RELEASE_TAG="flm-hrx-amdxdna-v2026.09.15" # For Linux. -HRX_RELEASE_ASSET="hrx-amdxdna-2026.07.30-amdxdna-hal-native-rel-eb0b39f-linux-x86_64.tar.zst" -HRX_RELEASE_SHA256="661ed94051cc6ad04f53739b2df7a791aecb658bc435bd5a6ff3c46716696345" +HRX_RELEASE_ASSET="hrx-amdxdna-2026.09.15-amdxdna-hal-native-rel-c023c3e-linux-x86_64.tar.zst" +HRX_RELEASE_SHA256="0a3437af76f6e2b5dcf0df52f26505c9f318d583328fd141c7f377170d14fe5b" # For Windows. -HRX_RELEASE_ASSET_WINDOWS="hrx-amdxdna-2026.07.30-amdxdna-hal-native-rel-eb0b39f-windows-x86_64.zip" -HRX_RELEASE_SHA256_WINDOWS="b3af74c9b393ce49ef71a3cbec2cbe92be6d40601858b41009d2bda57eadeb43" +HRX_RELEASE_ASSET_WINDOWS="hrx-amdxdna-2026.09.15-amdxdna-hal-native-rel-c023c3e-windows-x86_64.zip" +HRX_RELEASE_SHA256_WINDOWS="be7d3bc882b61234776be35aa98ee6cd0d1822e8281f64885472eed888170105" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 588149974..ab5bffe44 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -61,6 +61,10 @@ endif() # and HRX-built .so/.dll/.lib never collide (see src/lib/xrt and src/lib/hrx). set(FLM_ENGINE_LIB_DIR "${CMAKE_SOURCE_DIR}/lib/${FLM_RUNTIME_NAME}") +# The flash engine libs (qwen3vl_flash, gemma4e_flash) are always shipped in +# src/lib/${FLM_RUNTIME_NAME} for every backend, so their model families are +# always compiled in -- no probe/gate/macro needed. + # ——————————————————————————————————————————————— # NPU runtime discovery. # HRX: consumed from its public CMake package via find_package(hrx). @@ -403,22 +407,29 @@ if(MSVC) target_link_libraries(flm PUBLIC ${STATIC_LIBS}) endif() -# Link your custom libraries (these may still be DLLs if no static versions available) -target_link_libraries(flm PUBLIC +set(FLM_ENGINE_LINK_LIBS q4_npu_eXpress llama_npu qwen2_npu qwen2vl_npu qwen3_npu qwen3vl_npu - qwen3vl_flash +) + +list(APPEND FLM_ENGINE_LINK_LIBS qwen3vl_flash) + +list(APPEND FLM_ENGINE_LINK_LIBS qwen3_5vl_npu qwen3_5_omni_npu qwen3_6_moe_npu gemma_npu gemma_text_npu gemma4e_npu - gemma4e_flash +) + +list(APPEND FLM_ENGINE_LINK_LIBS gemma4e_flash) + +list(APPEND FLM_ENGINE_LINK_LIBS gemma4_12b_npu gpt_oss_npu whisper_npu @@ -427,6 +438,11 @@ target_link_libraries(flm PUBLIC phi4_npu nanbeige_npu hunyuan_npu +) + +# Link your custom libraries (these may still be DLLs if no static versions available) +target_link_libraries(flm PUBLIC + ${FLM_ENGINE_LINK_LIBS} # flm's own code calls nothing in the four shared-runtime libs below -- the # engine .so above do -- so the linker's default --as-needed drops them from # DT_NEEDED entirely. The engines then have to find MHA/Dequant/... in some diff --git a/src/common/AutoModel/modeling_gemma4e.cpp b/src/common/AutoModel/modeling_gemma4e.cpp index 99ae20672..73dba973b 100644 --- a/src/common/AutoModel/modeling_gemma4e.cpp +++ b/src/common/AutoModel/modeling_gemma4e.cpp @@ -1412,4 +1412,4 @@ StreamResult Gemma4e::parse_stream_content_impl(const std::string content, bool result.type = current_mode_; return result; -} \ No newline at end of file +} diff --git a/src/common/AutoModel/modeling_hunyuan.cpp b/src/common/AutoModel/modeling_hunyuan.cpp index 5d7795cff..eefdac321 100644 --- a/src/common/AutoModel/modeling_hunyuan.cpp +++ b/src/common/AutoModel/modeling_hunyuan.cpp @@ -11,9 +11,12 @@ /************ hunyuan-dense family **************/ Hunyuan::Hunyuan(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Hunyuan") { this->single_turn = true; - // the translator emits one short line per turn and the caller already has it - // from the stream / return value, so the raw dump would only double the log - this->log_raw_output = false; + // The translator streams one short line per turn, so echoing the + // "Model RAW Output:" dump only doubles the interactive log; keep it off by + // default. Diagnostics (qualification, numerical-match) anchor per-turn + // parsing on that marker, so FLM_LOG_RAW_OUTPUT=1 forces it back on for them + // without changing the interactive default. + this->log_raw_output = env_forces_raw_output(); // every turn either rewinds to the pinned prefix or clears, so the kv state // the trailing eos forward exists to preserve is discarded either way this->forward_on_eos = false; diff --git a/src/common/AutoModel/modeling_qwen3vl.cpp b/src/common/AutoModel/modeling_qwen3vl.cpp index bd69d6e67..136d82d65 100644 --- a/src/common/AutoModel/modeling_qwen3vl.cpp +++ b/src/common/AutoModel/modeling_qwen3vl.cpp @@ -17,10 +17,6 @@ void Qwen3VL::create_engine() { this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); } -void Qwen3VL_Flash::create_engine() { - this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); -} - void Qwen3VL::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); @@ -430,6 +426,10 @@ StreamResult Qwen3VL::parse_stream_content(const std::string content) { /************ Qwen3VL_Flash **************/ +void Qwen3VL_Flash::create_engine() { + this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); +} + int Qwen3VL_Flash::_pin_system_prefix(const std::string& system_text) { // Full clear — drop any previous pin before building the new one. this->lm_engine->clear_context(); diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index 45a6a63b7..69b21bb3b 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -16,9 +16,12 @@ #include #include #include +#include +#include #include "typedef.hpp" #include "causal_lm.hpp" #include "lm_config.hpp" + #include "models/llama/llama_npu.hpp" #include "models/qwen2/qwen2_npu.hpp" #include "models/qwen3/qwen3_npu.hpp" @@ -162,8 +165,18 @@ class AutoModel { std::vector checkpoint_his; /// \brief dump the undecorated model output to stdout once a turn ends /// \note on by default; models whose turns are short and driven in bulk (the - /// hunyuan translator) turn it off so the log is not doubled. + /// hunyuan translator) turn it off so the log is not doubled. Those + /// models should seed this from env_forces_raw_output() so diagnostics + /// can opt back in without changing the interactive default. bool log_raw_output = true; + /// \brief whether FLM_LOG_RAW_OUTPUT forces the "Model RAW Output:" dump on + /// \note the qualification harness and numerical-match anchor per-turn parsing + /// on that marker, so they set FLM_LOG_RAW_OUTPUT=1 to re-enable it for + /// models that quiet it by default; unset/empty/"0" leave the default. + static bool env_forces_raw_output() { + const char* value = std::getenv("FLM_LOG_RAW_OUTPUT"); + return value != nullptr && value[0] != '\0' && std::strcmp(value, "0") != 0; + } /// \brief run one more forward on the eos token once a turn ends /// \note this keeps the kv cache aligned with token_history so a following /// turn can append to it. Models that rewind or clear between turns diff --git a/src/include/AutoModel/modeling_qwen3vl.hpp b/src/include/AutoModel/modeling_qwen3vl.hpp index 6e6311a3f..b6e1a22d4 100644 --- a/src/include/AutoModel/modeling_qwen3vl.hpp +++ b/src/include/AutoModel/modeling_qwen3vl.hpp @@ -225,4 +225,4 @@ class Qwen3VL_Thinking : public Qwen3VL { } std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; - }; \ No newline at end of file + }; diff --git a/src/include/buffer.hpp b/src/include/buffer.hpp index 53c14c614..d44e99af8 100644 --- a/src/include/buffer.hpp +++ b/src/include/buffer.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -30,15 +31,25 @@ /// \note A copy (or mapping) does not duplicate the underlying memory; it only maps the pointer. class bytes { protected: + // HRX engines (IRON) use shared_ptr so copies keep the BO mapped. Unique_ptr + // copies drop ownership and leave data_ dangling across the .so boundary. +#if defined(FLM_USE_HRX) + std::shared_ptr owned_data_; +#else std::unique_ptr owned_data_; +#endif uint8_t* data_; size_t size_; bool is_owner_; #ifdef FLM_DEVICE_BUFFER bool is_bo_owner_; flm_rt::bo* bo_; +#if defined(FLM_USE_HRX) + std::shared_ptr owned_bo_; +#else std::unique_ptr owned_bo_; #endif +#endif public: /// \brief constructor @@ -52,11 +63,19 @@ class bytes { /// \brief copy constructor /// \param other the other bytes +#if defined(FLM_USE_HRX) + bytes(const bytes& other) : owned_data_(other.owned_data_), data_(other.data_), size_(other.size_), is_owner_(other.is_owner_) +#ifdef FLM_DEVICE_BUFFER + , is_bo_owner_(other.is_bo_owner_), bo_(other.bo_), owned_bo_(other.owned_bo_) +#endif + {} +#else bytes(const bytes& other) : owned_data_(nullptr), data_(other.data_), size_(other.size_), is_owner_(false) #ifdef FLM_DEVICE_BUFFER , is_bo_owner_(false), bo_(other.bo_), owned_bo_(nullptr) #endif {} +#endif /// \brief move constructor /// \param other the other bytes @@ -86,7 +105,11 @@ class bytes { { if (size > 0 && size < 8ull * 1024 * 1024 * 1024){ try { +#if defined(FLM_USE_HRX) + owned_data_ = std::shared_ptr(new uint8_t[size]()); +#else owned_data_ = std::make_unique(size); +#endif } catch (const std::bad_alloc& e) { throw std::runtime_error(std::string("Failed to allocate bytes of size ") + std::to_string(size) + ": " + e.what()); @@ -128,10 +151,14 @@ class bytes { throw std::runtime_error("Invalid size for bytes allocation"); } size_t alignment = 1024 * 1024; - int padded_size = (size + alignment - 1) / alignment * alignment; // 1MB alignment + size_t padded_size = (size + alignment - 1) / alignment * alignment; // 1MB alignment try { +#if defined(FLM_USE_HRX) + owned_bo_ = std::make_shared(device, padded_size); +#else owned_bo_ = std::make_unique(device, padded_size); +#endif } catch (const std::exception& e) { throw std::runtime_error(std::string("Failed to allocate flm_rt::ext::bo: ") + e.what()); @@ -169,6 +196,17 @@ class bytes { /// \param other the other bytes bytes& operator=(const bytes& other) { if (this != &other) { +#if defined(FLM_USE_HRX) + owned_data_ = other.owned_data_; + data_ = other.data_; + size_ = other.size_; + is_owner_ = other.is_owner_; +#ifdef FLM_DEVICE_BUFFER + owned_bo_ = other.owned_bo_; + is_bo_owner_ = other.is_bo_owner_; + bo_ = other.bo_; +#endif +#else if (is_owner_){ owned_data_.reset(); } @@ -181,6 +219,7 @@ class bytes { } is_bo_owner_ = false; bo_ = other.bo_; +#endif #endif } return *this; @@ -257,7 +296,11 @@ class bytes { throw std::runtime_error("Cannot resize to zero size"); } try { +#if defined(FLM_USE_HRX) + owned_data_.reset(new uint8_t[new_size]()); +#else owned_data_.reset(new uint8_t[new_size]); +#endif } catch (const std::bad_alloc& e) { throw std::runtime_error(std::string("Failed to allocate bytes of size ") + std::to_string(new_size) + ": " + e.what()); @@ -302,6 +345,8 @@ class bytes { /// \return the is bo owner bool is_bo_owner() const { return is_bo_owner_; } + bool has_bo() const { return bo_ != nullptr; } + /// \brief sync to device (host writes -> device) #if defined(FLM_USE_HRX) void sync_to_device() { assert(bo_); bo_->flush(); } diff --git a/src/include/hrx_cpp/hrx_cpp.hpp b/src/include/hrx_cpp/hrx_cpp.hpp index 13fbf79c7..b4c4fe4d3 100644 --- a/src/include/hrx_cpp/hrx_cpp.hpp +++ b/src/include/hrx_cpp/hrx_cpp.hpp @@ -113,10 +113,32 @@ inline bool hrx_report(hrx_status_t s, const char* where) { // Executable cache: build one HRX XADX executable per distinct executable // identity (xclbin + control program + host patch table) and resolve its export // ordinal once. +// +// Ownership / lifetime (context-exhaustion fix) +// --------------------------------------------------------------------------- +// Every executable is created with HRX_AMDXDNA_CONTEXT_MODE_CREATE, i.e. it is +// backed by its own NPU hardware context. That pool is small: on the Windows +// MCDM path the 314 driver runs out after enough distinct executables and +// hrx_stream_wait then fails with +// D3DKMTCreateContextVirtual failed with 0xc01e0009 +// (the Linux amdxdna path fails the analogous BO allocation with errno 11 / +// EAGAIN). A single long generation stays bounded because decode revisits the +// same control sequences, but a `flm serve` sweep that loads many models used +// to leak every model's executables forever: the cache was a process-wide map +// that was never pruned on model unload, so contexts accumulated across the +// sweep until the driver was exhausted -- exactly the "running out of contexts" +// failure seen at gemma4-e4b after many prior models. +// +// Executables are therefore reference counted. Each npu_app that dispatches a +// given (xclbin, ctrl_seq) holds exactly one reference for it; when the owning +// model's engine objects are destroyed (model unload / model switch in serve) +// the npu_app releases its references and, once the last owner drops, the +// executable is destroyed and its hardware context returned to the driver. // --------------------------------------------------------------------------- struct CachedExe { hrx_executable_t exe = nullptr; uint32_t ord = 0; + size_t refs = 0; // number of npu_app owners; released when it reaches 0 }; inline void append_key_bytes(std::string& key, const void* data, size_t byte_count) { @@ -127,20 +149,37 @@ inline void append_key_bytes(std::string& key, const void* data, size_t byte_cou } } -inline hrx_executable_t build_or_get_executable( - const std::vector& xclbin_bytes, const uint32_t* cc, size_t n, - uint32_t* ord_out) { - static std::mutex mu; - static std::unordered_map cache; +inline std::mutex& exe_cache_mu() { + static std::mutex m; + return m; +} +inline std::unordered_map& exe_cache() { + static std::unordered_map c; + return c; +} +inline std::string make_exe_key(const std::vector& xclbin_bytes, + const uint32_t* cc, size_t n) { std::string key; - key.reserve(xclbin_bytes.size() + n * sizeof(uint32_t) + - 2 * sizeof(uint64_t)); + key.reserve(xclbin_bytes.size() + n * sizeof(uint32_t) + 2 * sizeof(uint64_t)); append_key_bytes(key, xclbin_bytes.data(), xclbin_bytes.size()); append_key_bytes(key, cc, n * sizeof(uint32_t)); - std::lock_guard lk(mu); + return key; +} + +// Build (or fetch the cached) executable for (xclbin, cc). Does NOT change the +// reference count; callers manage ownership with retain_executable / +// release_executable below. Returns nullptr on create failure (never cached, so +// a later call can retry instead of dispatching a silent no-op). +inline hrx_executable_t lookup_or_build_executable( + const std::vector& xclbin_bytes, const uint32_t* cc, size_t n, + uint32_t* ord_out, std::string* key_out) { + std::string key = make_exe_key(xclbin_bytes, cc, n); + std::lock_guard lk(exe_cache_mu()); + auto& cache = exe_cache(); auto it = cache.find(key); if (it != cache.end()) { if (ord_out) *ord_out = it->second.ord; + if (key_out) *key_out = std::move(key); return it->second.exe; } hrx_const_byte_span_t xclbin = {xclbin_bytes.data(), xclbin_bytes.size()}; @@ -176,11 +215,51 @@ inline hrx_executable_t build_or_get_executable( hrx_executable_release(exe); exe = nullptr; } - cache.emplace(std::move(key), CachedExe{exe, ord}); + if (exe) { + cache.emplace(key, CachedExe{exe, ord, 0}); + } if (ord_out) *ord_out = ord; + if (key_out) *key_out = std::move(key); return exe; } +// Add one owner reference to the cached executable identified by key. +inline void retain_executable(const std::string& key) { + std::lock_guard lk(exe_cache_mu()); + auto& cache = exe_cache(); + auto it = cache.find(key); + if (it != cache.end()) ++it->second.refs; +} + +// Drop one owner reference; when the last owner drops the executable is +// destroyed and its hardware context is returned to the driver. +inline void release_executable(const std::string& key) { + hrx_executable_t to_free = nullptr; + { + std::lock_guard lk(exe_cache_mu()); + auto& cache = exe_cache(); + auto it = cache.find(key); + if (it == cache.end()) return; + if (it->second.refs > 0) --it->second.refs; + if (it->second.refs == 0) { + to_free = it->second.exe; + cache.erase(it); + } + } + if (to_free) { + hrx_executable_release(to_free); + } +} + +// Backward-compatible entry point (no ownership tracking). Retained for any +// caller that does not manage executable lifetime; prefer the +// lookup_or_build/retain/release trio used by npu_app. +inline hrx_executable_t build_or_get_executable( + const std::vector& xclbin_bytes, const uint32_t* cc, size_t n, + uint32_t* ord_out) { + return lookup_or_build_executable(xclbin_bytes, cc, n, ord_out, nullptr); +} + // --------------------------------------------------------------------------- // uuid / xclbin / device / hw_context // --------------------------------------------------------------------------- diff --git a/src/include/npu_utils/npu_utils_hrx.hpp b/src/include/npu_utils/npu_utils_hrx.hpp index 6268d698f..1923ffddb 100644 --- a/src/include/npu_utils/npu_utils_hrx.hpp +++ b/src/include/npu_utils/npu_utils_hrx.hpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #ifndef __WINDOWS__ #include @@ -77,6 +78,30 @@ class npu_app { hrx_executable_t exe; // HRX direct executable built from ctrl_seq->dump() uint32_t exe_ord; // resolved "MLIR_AIE" export ordinal std::unique_ptr ctrl_seq; + // Executable-cache keys this app owns a reference for. Each distinct + // (xclbin, ctrl_seq) this app dispatches contributes one reference; they are + // all released in the destructor so the backing NPU hardware contexts are + // returned when the owning model is unloaded (see hrx_cpp.hpp). + std::unordered_set exe_keys_; + + ///@brief Transfer all state from `o` and leave it empty (owning no + /// executable references) so its destructor releases nothing. + void move_from(npu_app&& o) noexcept { + device_gen = o.device_gen; + device = o.device; + context = o.context; + kernel_name = std::move(o.kernel_name); + enable_preemption = o.enable_preemption; + module_valid = o.module_valid; + module_version = o.module_version; + exe = o.exe; + exe_ord = o.exe_ord; + ctrl_seq = std::move(o.ctrl_seq); + exe_keys_ = std::move(o.exe_keys_); + o.exe = nullptr; + o.module_valid = false; + o.exe_keys_.clear(); + } ///@brief Setup the kernel ///@note Builds (or fetches a cached) HRX XADX executable directly from the @@ -86,13 +111,20 @@ class npu_app { assert(data.first != nullptr); assert(data.second > 0); assert(this->context != nullptr); - this->exe = hrx::build_or_get_executable( + std::string exe_key; + this->exe = hrx::lookup_or_build_executable( this->context->xclbin_bytes(), data.first, data.second, - &this->exe_ord); + &this->exe_ord, &exe_key); if (this->exe == nullptr){ header_print_r("ERROR", "Failed to build HRX executable from ctrl_seq"); exit(1); } + // Take one reference the first time this app uses this executable; the + // destructor drops all of them so the hardware context is freed on + // model unload instead of leaking for the process lifetime. + if (this->exe_keys_.insert(exe_key).second) { + hrx::retain_executable(exe_key); + } this->module_valid = true; this->module_version = this->ctrl_seq->sequence_version(); } @@ -154,6 +186,31 @@ class npu_app { this->module_version = 0xFF; } + ///@brief Destructor: release this app's executable references so their NPU + /// hardware contexts are freed when the owning model is unloaded. + ~npu_app(){ + for (const auto& k : this->exe_keys_) hrx::release_executable(k); + } + + // npu_app owns HRX executable references, so it is move-only (copies are + // already deleted by the unique_ptr member). Declaring the destructor above + // suppresses the implicit move operations, so define them explicitly and + // transfer ownership of the executable keys, leaving the moved-from object + // with none (so it releases nothing). + npu_app(const npu_app&) = delete; + npu_app& operator=(const npu_app&) = delete; + + npu_app(npu_app&& other) noexcept { this->move_from(std::move(other)); } + + npu_app& operator=(npu_app&& other) noexcept { + if (this != &other) { + for (const auto& k : this->exe_keys_) hrx::release_executable(k); + this->exe_keys_.clear(); + this->move_from(std::move(other)); + } + return *this; + } + void update_ctrl_seq(){ assert(this->ctrl_seq != nullptr); diff --git a/src/lib/hrx/dequant.dll b/src/lib/hrx/dequant.dll index d6614115d..c0202eccd 100644 Binary files a/src/lib/hrx/dequant.dll and b/src/lib/hrx/dequant.dll differ diff --git a/src/lib/hrx/dequant.lib b/src/lib/hrx/dequant.lib index 595bb7429..420233d74 100644 Binary files a/src/lib/hrx/dequant.lib and b/src/lib/hrx/dequant.lib differ diff --git a/src/lib/hrx/dequant_new.dll b/src/lib/hrx/dequant_new.dll deleted file mode 100644 index 8a82d4aea..000000000 Binary files a/src/lib/hrx/dequant_new.dll and /dev/null differ diff --git a/src/lib/hrx/dequant_new.lib b/src/lib/hrx/dequant_new.lib deleted file mode 100644 index af2f9584e..000000000 Binary files a/src/lib/hrx/dequant_new.lib and /dev/null differ diff --git a/src/lib/hrx/gemm.dll b/src/lib/hrx/gemm.dll index ce51d217a..8ffe920a9 100644 Binary files a/src/lib/hrx/gemm.dll and b/src/lib/hrx/gemm.dll differ diff --git a/src/lib/hrx/gemm.lib b/src/lib/hrx/gemm.lib index 717084934..afd6c330a 100644 Binary files a/src/lib/hrx/gemm.lib and b/src/lib/hrx/gemm.lib differ diff --git a/src/lib/hrx/gemma4_12b_npu.dll b/src/lib/hrx/gemma4_12b_npu.dll index 5eba1c4e9..6ce0542e7 100644 Binary files a/src/lib/hrx/gemma4_12b_npu.dll and b/src/lib/hrx/gemma4_12b_npu.dll differ diff --git a/src/lib/hrx/gemma4_12b_npu.lib b/src/lib/hrx/gemma4_12b_npu.lib index d3f189e99..e9f9453fd 100644 Binary files a/src/lib/hrx/gemma4_12b_npu.lib and b/src/lib/hrx/gemma4_12b_npu.lib differ diff --git a/src/lib/hrx/gemma4e_flash.dll b/src/lib/hrx/gemma4e_flash.dll new file mode 100644 index 000000000..0bcb4471b Binary files /dev/null and b/src/lib/hrx/gemma4e_flash.dll differ diff --git a/src/lib/hrx/gemma4e_flash.lib b/src/lib/hrx/gemma4e_flash.lib new file mode 100644 index 000000000..d6bc2b4b8 Binary files /dev/null and b/src/lib/hrx/gemma4e_flash.lib differ diff --git a/src/lib/hrx/gemma4e_npu.dll b/src/lib/hrx/gemma4e_npu.dll index 5b4cdec1b..eed97db2a 100644 Binary files a/src/lib/hrx/gemma4e_npu.dll and b/src/lib/hrx/gemma4e_npu.dll differ diff --git a/src/lib/hrx/gemma4e_npu.lib b/src/lib/hrx/gemma4e_npu.lib index bab48c3dd..ec2014f1b 100644 Binary files a/src/lib/hrx/gemma4e_npu.lib and b/src/lib/hrx/gemma4e_npu.lib differ diff --git a/src/lib/hrx/gemma_embedding.dll b/src/lib/hrx/gemma_embedding.dll index 476cf1399..37fc43816 100644 Binary files a/src/lib/hrx/gemma_embedding.dll and b/src/lib/hrx/gemma_embedding.dll differ diff --git a/src/lib/hrx/gemma_embedding.lib b/src/lib/hrx/gemma_embedding.lib index b5c9af6dc..a54fbf355 100644 Binary files a/src/lib/hrx/gemma_embedding.lib and b/src/lib/hrx/gemma_embedding.lib differ diff --git a/src/lib/hrx/gemma_npu.dll b/src/lib/hrx/gemma_npu.dll index 50123227f..c50f36f89 100644 Binary files a/src/lib/hrx/gemma_npu.dll and b/src/lib/hrx/gemma_npu.dll differ diff --git a/src/lib/hrx/gemma_npu.lib b/src/lib/hrx/gemma_npu.lib index 79c76282f..41b94e6d9 100644 Binary files a/src/lib/hrx/gemma_npu.lib and b/src/lib/hrx/gemma_npu.lib differ diff --git a/src/lib/hrx/gemma_text_npu.dll b/src/lib/hrx/gemma_text_npu.dll index b7b3887c7..ab764cf71 100644 Binary files a/src/lib/hrx/gemma_text_npu.dll and b/src/lib/hrx/gemma_text_npu.dll differ diff --git a/src/lib/hrx/gemma_text_npu.lib b/src/lib/hrx/gemma_text_npu.lib index ca0cc2fd9..07e189d61 100644 Binary files a/src/lib/hrx/gemma_text_npu.lib and b/src/lib/hrx/gemma_text_npu.lib differ diff --git a/src/lib/hrx/gpt_oss_npu.dll b/src/lib/hrx/gpt_oss_npu.dll index 3e8f4f9ff..81e020510 100644 Binary files a/src/lib/hrx/gpt_oss_npu.dll and b/src/lib/hrx/gpt_oss_npu.dll differ diff --git a/src/lib/hrx/gpt_oss_npu.lib b/src/lib/hrx/gpt_oss_npu.lib index c193c663d..32ae08119 100644 Binary files a/src/lib/hrx/gpt_oss_npu.lib and b/src/lib/hrx/gpt_oss_npu.lib differ diff --git a/src/lib/hrx/hunyuan_npu.dll b/src/lib/hrx/hunyuan_npu.dll index 007154b71..9836904fa 100644 Binary files a/src/lib/hrx/hunyuan_npu.dll and b/src/lib/hrx/hunyuan_npu.dll differ diff --git a/src/lib/hrx/hunyuan_npu.lib b/src/lib/hrx/hunyuan_npu.lib index 44830bd11..1ead4d766 100644 Binary files a/src/lib/hrx/hunyuan_npu.lib and b/src/lib/hrx/hunyuan_npu.lib differ diff --git a/src/lib/hrx/lfm2_npu.dll b/src/lib/hrx/lfm2_npu.dll index ac94c580d..f6f2cd108 100644 Binary files a/src/lib/hrx/lfm2_npu.dll and b/src/lib/hrx/lfm2_npu.dll differ diff --git a/src/lib/hrx/lfm2_npu.lib b/src/lib/hrx/lfm2_npu.lib index eefd034b8..1d2f7f1ed 100644 Binary files a/src/lib/hrx/lfm2_npu.lib and b/src/lib/hrx/lfm2_npu.lib differ diff --git a/src/lib/hrx/libdequant.so b/src/lib/hrx/libdequant.so old mode 100755 new mode 100644 index 42b859fa6..c17086128 Binary files a/src/lib/hrx/libdequant.so and b/src/lib/hrx/libdequant.so differ diff --git a/src/lib/hrx/libdequant_new.so b/src/lib/hrx/libdequant_new.so deleted file mode 100755 index 4851f5b9c..000000000 Binary files a/src/lib/hrx/libdequant_new.so and /dev/null differ diff --git a/src/lib/hrx/libgemm.so b/src/lib/hrx/libgemm.so old mode 100755 new mode 100644 index 4093c244e..9fef70ff1 Binary files a/src/lib/hrx/libgemm.so and b/src/lib/hrx/libgemm.so differ diff --git a/src/lib/hrx/libgemma4_12b_npu.so b/src/lib/hrx/libgemma4_12b_npu.so old mode 100644 new mode 100755 index aa347a5a7..6fbaf0581 Binary files a/src/lib/hrx/libgemma4_12b_npu.so and b/src/lib/hrx/libgemma4_12b_npu.so differ diff --git a/src/lib/hrx/libgemma4e_flash.so b/src/lib/hrx/libgemma4e_flash.so new file mode 100644 index 000000000..e029161fe Binary files /dev/null and b/src/lib/hrx/libgemma4e_flash.so differ diff --git a/src/lib/hrx/libgemma4e_npu.so b/src/lib/hrx/libgemma4e_npu.so index 23353a2b6..5aa21be09 100755 Binary files a/src/lib/hrx/libgemma4e_npu.so and b/src/lib/hrx/libgemma4e_npu.so differ diff --git a/src/lib/hrx/libgemma_embedding.so b/src/lib/hrx/libgemma_embedding.so index ddb74daac..8f3afb1a1 100755 Binary files a/src/lib/hrx/libgemma_embedding.so and b/src/lib/hrx/libgemma_embedding.so differ diff --git a/src/lib/hrx/libgemma_npu.so b/src/lib/hrx/libgemma_npu.so index 154f265b6..1ecabf305 100755 Binary files a/src/lib/hrx/libgemma_npu.so and b/src/lib/hrx/libgemma_npu.so differ diff --git a/src/lib/hrx/libgemma_text_npu.so b/src/lib/hrx/libgemma_text_npu.so index ad25e60c9..81ff5f14a 100755 Binary files a/src/lib/hrx/libgemma_text_npu.so and b/src/lib/hrx/libgemma_text_npu.so differ diff --git a/src/lib/hrx/libgpt_oss_npu.so b/src/lib/hrx/libgpt_oss_npu.so index 26fc19853..b01886334 100755 Binary files a/src/lib/hrx/libgpt_oss_npu.so and b/src/lib/hrx/libgpt_oss_npu.so differ diff --git a/src/lib/hrx/libhunyuan_npu.so b/src/lib/hrx/libhunyuan_npu.so old mode 100644 new mode 100755 index 7079f0c15..a2e04db14 Binary files a/src/lib/hrx/libhunyuan_npu.so and b/src/lib/hrx/libhunyuan_npu.so differ diff --git a/src/lib/hrx/liblfm2_npu.so b/src/lib/hrx/liblfm2_npu.so index 8c568ff8e..bca8a2204 100755 Binary files a/src/lib/hrx/liblfm2_npu.so and b/src/lib/hrx/liblfm2_npu.so differ diff --git a/src/lib/hrx/libllama_npu.so b/src/lib/hrx/libllama_npu.so index 53a6e4025..0ac2b07ee 100755 Binary files a/src/lib/hrx/libllama_npu.so and b/src/lib/hrx/libllama_npu.so differ diff --git a/src/lib/hrx/liblm_head.so b/src/lib/hrx/liblm_head.so old mode 100755 new mode 100644 index 2666d258c..181f4bd60 Binary files a/src/lib/hrx/liblm_head.so and b/src/lib/hrx/liblm_head.so differ diff --git a/src/lib/hrx/libmha.so b/src/lib/hrx/libmha.so old mode 100755 new mode 100644 index daa108bb8..c7147a1cf Binary files a/src/lib/hrx/libmha.so and b/src/lib/hrx/libmha.so differ diff --git a/src/lib/hrx/libnanbeige_npu.so b/src/lib/hrx/libnanbeige_npu.so index 2bb6fa876..927a6dde8 100755 Binary files a/src/lib/hrx/libnanbeige_npu.so and b/src/lib/hrx/libnanbeige_npu.so differ diff --git a/src/lib/hrx/libphi4_npu.so b/src/lib/hrx/libphi4_npu.so index 67ca6e61a..2db1059c6 100755 Binary files a/src/lib/hrx/libphi4_npu.so and b/src/lib/hrx/libphi4_npu.so differ diff --git a/src/lib/hrx/libq4_npu_eXpress.so b/src/lib/hrx/libq4_npu_eXpress.so old mode 100755 new mode 100644 index 49074f5d8..327626399 Binary files a/src/lib/hrx/libq4_npu_eXpress.so and b/src/lib/hrx/libq4_npu_eXpress.so differ diff --git a/src/lib/hrx/libqwen2_npu.so b/src/lib/hrx/libqwen2_npu.so index 340e5fa89..e7421dfb2 100755 Binary files a/src/lib/hrx/libqwen2_npu.so and b/src/lib/hrx/libqwen2_npu.so differ diff --git a/src/lib/hrx/libqwen2vl_npu.so b/src/lib/hrx/libqwen2vl_npu.so index d48450012..e008a8391 100755 Binary files a/src/lib/hrx/libqwen2vl_npu.so and b/src/lib/hrx/libqwen2vl_npu.so differ diff --git a/src/lib/hrx/libqwen3_5_omni_npu.so b/src/lib/hrx/libqwen3_5_omni_npu.so index 5e13c89f0..802d0c865 100755 Binary files a/src/lib/hrx/libqwen3_5_omni_npu.so and b/src/lib/hrx/libqwen3_5_omni_npu.so differ diff --git a/src/lib/hrx/libqwen3_5vl_npu.so b/src/lib/hrx/libqwen3_5vl_npu.so index 3b7f9d8a3..b76eea76f 100755 Binary files a/src/lib/hrx/libqwen3_5vl_npu.so and b/src/lib/hrx/libqwen3_5vl_npu.so differ diff --git a/src/lib/hrx/libqwen3_6_moe_npu.so b/src/lib/hrx/libqwen3_6_moe_npu.so index 6ea305cd9..f05330dcf 100755 Binary files a/src/lib/hrx/libqwen3_6_moe_npu.so and b/src/lib/hrx/libqwen3_6_moe_npu.so differ diff --git a/src/lib/hrx/libqwen3_npu.so b/src/lib/hrx/libqwen3_npu.so index 0842d6805..debfa2455 100755 Binary files a/src/lib/hrx/libqwen3_npu.so and b/src/lib/hrx/libqwen3_npu.so differ diff --git a/src/lib/hrx/libqwen3vl_flash.so b/src/lib/hrx/libqwen3vl_flash.so old mode 100755 new mode 100644 index 4677eaf1c..40cf6ae62 Binary files a/src/lib/hrx/libqwen3vl_flash.so and b/src/lib/hrx/libqwen3vl_flash.so differ diff --git a/src/lib/hrx/libqwen3vl_npu.so b/src/lib/hrx/libqwen3vl_npu.so index 5b47b72a4..cdc55fcf6 100755 Binary files a/src/lib/hrx/libqwen3vl_npu.so and b/src/lib/hrx/libqwen3vl_npu.so differ diff --git a/src/lib/hrx/libwhisper_npu.so b/src/lib/hrx/libwhisper_npu.so index 71df18de3..545a31ec1 100755 Binary files a/src/lib/hrx/libwhisper_npu.so and b/src/lib/hrx/libwhisper_npu.so differ diff --git a/src/lib/hrx/llama_npu.dll b/src/lib/hrx/llama_npu.dll index aa5157cb0..04da329a2 100644 Binary files a/src/lib/hrx/llama_npu.dll and b/src/lib/hrx/llama_npu.dll differ diff --git a/src/lib/hrx/llama_npu.lib b/src/lib/hrx/llama_npu.lib index a128ec96e..c9a12e5ea 100644 Binary files a/src/lib/hrx/llama_npu.lib and b/src/lib/hrx/llama_npu.lib differ diff --git a/src/lib/hrx/lm_head.dll b/src/lib/hrx/lm_head.dll index d83bee843..08f808a94 100644 Binary files a/src/lib/hrx/lm_head.dll and b/src/lib/hrx/lm_head.dll differ diff --git a/src/lib/hrx/lm_head.lib b/src/lib/hrx/lm_head.lib index 311d63a8d..dd3fb5f46 100644 Binary files a/src/lib/hrx/lm_head.lib and b/src/lib/hrx/lm_head.lib differ diff --git a/src/lib/hrx/mha.dll b/src/lib/hrx/mha.dll index fe5119326..4f2135874 100644 Binary files a/src/lib/hrx/mha.dll and b/src/lib/hrx/mha.dll differ diff --git a/src/lib/hrx/mha.lib b/src/lib/hrx/mha.lib index 817427778..01fdb324d 100644 Binary files a/src/lib/hrx/mha.lib and b/src/lib/hrx/mha.lib differ diff --git a/src/lib/hrx/nanbeige_npu.dll b/src/lib/hrx/nanbeige_npu.dll index 59b5165f5..da87e8eb7 100644 Binary files a/src/lib/hrx/nanbeige_npu.dll and b/src/lib/hrx/nanbeige_npu.dll differ diff --git a/src/lib/hrx/nanbeige_npu.lib b/src/lib/hrx/nanbeige_npu.lib index 49a806e4e..d5964499b 100644 Binary files a/src/lib/hrx/nanbeige_npu.lib and b/src/lib/hrx/nanbeige_npu.lib differ diff --git a/src/lib/hrx/phi4_npu.dll b/src/lib/hrx/phi4_npu.dll index a21ac4684..3c480a0b7 100644 Binary files a/src/lib/hrx/phi4_npu.dll and b/src/lib/hrx/phi4_npu.dll differ diff --git a/src/lib/hrx/phi4_npu.lib b/src/lib/hrx/phi4_npu.lib index d78609953..2f746e87a 100644 Binary files a/src/lib/hrx/phi4_npu.lib and b/src/lib/hrx/phi4_npu.lib differ diff --git a/src/lib/hrx/q4_npu_eXpress.dll b/src/lib/hrx/q4_npu_eXpress.dll index b669c0c04..dda73dee7 100644 Binary files a/src/lib/hrx/q4_npu_eXpress.dll and b/src/lib/hrx/q4_npu_eXpress.dll differ diff --git a/src/lib/hrx/q4_npu_eXpress.lib b/src/lib/hrx/q4_npu_eXpress.lib index 963d31c22..07c41b52d 100644 Binary files a/src/lib/hrx/q4_npu_eXpress.lib and b/src/lib/hrx/q4_npu_eXpress.lib differ diff --git a/src/lib/hrx/qwen2_npu.dll b/src/lib/hrx/qwen2_npu.dll index d83814dad..b23dcb75c 100644 Binary files a/src/lib/hrx/qwen2_npu.dll and b/src/lib/hrx/qwen2_npu.dll differ diff --git a/src/lib/hrx/qwen2_npu.lib b/src/lib/hrx/qwen2_npu.lib index 866af63f6..9eb85b532 100644 Binary files a/src/lib/hrx/qwen2_npu.lib and b/src/lib/hrx/qwen2_npu.lib differ diff --git a/src/lib/hrx/qwen2vl_npu.dll b/src/lib/hrx/qwen2vl_npu.dll index a7519f368..27bda827b 100644 Binary files a/src/lib/hrx/qwen2vl_npu.dll and b/src/lib/hrx/qwen2vl_npu.dll differ diff --git a/src/lib/hrx/qwen2vl_npu.lib b/src/lib/hrx/qwen2vl_npu.lib index 9d08e116e..739beaa7f 100644 Binary files a/src/lib/hrx/qwen2vl_npu.lib and b/src/lib/hrx/qwen2vl_npu.lib differ diff --git a/src/lib/hrx/qwen3_5_omni_npu.dll b/src/lib/hrx/qwen3_5_omni_npu.dll index d37c28be1..86b942ccf 100644 Binary files a/src/lib/hrx/qwen3_5_omni_npu.dll and b/src/lib/hrx/qwen3_5_omni_npu.dll differ diff --git a/src/lib/hrx/qwen3_5_omni_npu.lib b/src/lib/hrx/qwen3_5_omni_npu.lib index a401cb885..e2e7f6f07 100644 Binary files a/src/lib/hrx/qwen3_5_omni_npu.lib and b/src/lib/hrx/qwen3_5_omni_npu.lib differ diff --git a/src/lib/hrx/qwen3_5vl_npu.dll b/src/lib/hrx/qwen3_5vl_npu.dll index a35449e3a..c7b59beeb 100644 Binary files a/src/lib/hrx/qwen3_5vl_npu.dll and b/src/lib/hrx/qwen3_5vl_npu.dll differ diff --git a/src/lib/hrx/qwen3_5vl_npu.lib b/src/lib/hrx/qwen3_5vl_npu.lib index 9a05301dd..9a133638d 100644 Binary files a/src/lib/hrx/qwen3_5vl_npu.lib and b/src/lib/hrx/qwen3_5vl_npu.lib differ diff --git a/src/lib/hrx/qwen3_6_moe_npu.dll b/src/lib/hrx/qwen3_6_moe_npu.dll index 72cb8758c..fdc4919c1 100644 Binary files a/src/lib/hrx/qwen3_6_moe_npu.dll and b/src/lib/hrx/qwen3_6_moe_npu.dll differ diff --git a/src/lib/hrx/qwen3_6_moe_npu.lib b/src/lib/hrx/qwen3_6_moe_npu.lib index 204aa5be5..f84b2cdbe 100644 Binary files a/src/lib/hrx/qwen3_6_moe_npu.lib and b/src/lib/hrx/qwen3_6_moe_npu.lib differ diff --git a/src/lib/hrx/qwen3_npu.dll b/src/lib/hrx/qwen3_npu.dll index 5fb3d3779..c800f5e61 100644 Binary files a/src/lib/hrx/qwen3_npu.dll and b/src/lib/hrx/qwen3_npu.dll differ diff --git a/src/lib/hrx/qwen3_npu.lib b/src/lib/hrx/qwen3_npu.lib index 1352763ea..6fdbf7983 100644 Binary files a/src/lib/hrx/qwen3_npu.lib and b/src/lib/hrx/qwen3_npu.lib differ diff --git a/src/lib/hrx/qwen3vl_flash.dll b/src/lib/hrx/qwen3vl_flash.dll new file mode 100644 index 000000000..9949fbcfc Binary files /dev/null and b/src/lib/hrx/qwen3vl_flash.dll differ diff --git a/src/lib/hrx/qwen3vl_flash.lib b/src/lib/hrx/qwen3vl_flash.lib new file mode 100644 index 000000000..26085ca06 Binary files /dev/null and b/src/lib/hrx/qwen3vl_flash.lib differ diff --git a/src/lib/hrx/qwen3vl_npu.dll b/src/lib/hrx/qwen3vl_npu.dll index cecb88fb6..ce8f7d177 100644 Binary files a/src/lib/hrx/qwen3vl_npu.dll and b/src/lib/hrx/qwen3vl_npu.dll differ diff --git a/src/lib/hrx/qwen3vl_npu.lib b/src/lib/hrx/qwen3vl_npu.lib index 1ddbdfc44..0fc30d960 100644 Binary files a/src/lib/hrx/qwen3vl_npu.lib and b/src/lib/hrx/qwen3vl_npu.lib differ diff --git a/src/lib/hrx/whisper_npu.dll b/src/lib/hrx/whisper_npu.dll index dd7867e79..9a8d33cf1 100644 Binary files a/src/lib/hrx/whisper_npu.dll and b/src/lib/hrx/whisper_npu.dll differ diff --git a/src/lib/hrx/whisper_npu.lib b/src/lib/hrx/whisper_npu.lib index 3fb1dd32f..761313d42 100644 Binary files a/src/lib/hrx/whisper_npu.lib and b/src/lib/hrx/whisper_npu.lib differ