diff --git a/sdk/CMakeLists.txt b/sdk/CMakeLists.txt index 69400fd89..9b6b30ee0 100644 --- a/sdk/CMakeLists.txt +++ b/sdk/CMakeLists.txt @@ -150,6 +150,12 @@ enable_testing() # per-model (llama-hexagon-power-mode.patch alone only reads it once, from # GGML_HEXAGON_POWER_MODE, at backend-registry construction). See # sdk/plugins/llama_cpp/src/htp_session.cpp for the caller. +# * llama-hexagon-hmx-f16-batched-prefetch.patch fixes the prefetch source of +# the second weight chunk in the batched F16 HMX matmul (GQA broadcast). It +# read one row past the first chunk, so the second chunk's output was wrong +# whenever the weights span more than one chunk (attention prefill without +# flash attention at long KV). Upstream ggml-org/llama.cpp#29626 carries the +# same fix; delete this file once the submodule is bumped past it. # * llama-opencl-adreno-cpy-constreg.patch works around an Adreno OpenCL # compiler crash: kernel_cpy_f32_f32_pack's min(ne00, get_local_size(0)) # emits an IMINrr with two constant-class operands, which the driver's @@ -175,7 +181,8 @@ if(GENIEX_PLUGIN_LLAMA_CPP) list(APPEND _LLAMA_PATCHES "${CMAKE_CURRENT_SOURCE_DIR}/patches/llama-hexagon-release-sessions.patch" "${CMAKE_CURRENT_SOURCE_DIR}/patches/llama-hexagon-power-mode.patch" - "${CMAKE_CURRENT_SOURCE_DIR}/patches/llama-hexagon-power-mode-setter.patch") + "${CMAKE_CURRENT_SOURCE_DIR}/patches/llama-hexagon-power-mode-setter.patch" + "${CMAKE_CURRENT_SOURCE_DIR}/patches/llama-hexagon-hmx-f16-batched-prefetch.patch") endif() if(GGML_OPENCL) list(APPEND _LLAMA_PATCHES diff --git a/sdk/patches/llama-hexagon-hmx-f16-batched-prefetch.patch b/sdk/patches/llama-hexagon-hmx-f16-batched-prefetch.patch new file mode 100644 index 000000000..ff831ae06 --- /dev/null +++ b/sdk/patches/llama-hexagon-hmx-f16-batched-prefetch.patch @@ -0,0 +1,13 @@ +diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c +index a09bc7a28..526f200b0 100644 +--- a/ggml/src/ggml-hexagon/htp/matmul-ops.c ++++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c +@@ -3356,7 +3356,7 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_ + } + if (n_chunk_n_cols < (size_t) params->n) { + const size_t n_cols_second = hex_smin((size_t) params->n - n_chunk_n_cols, n_chunk_n_cols); +- dma_queue_push(weight_dma, dma_make_data(vtcm_scratch1, weight_group + params->weight_stride * sizeof(__fp16)), ++ dma_queue_push(weight_dma, dma_make_data(vtcm_scratch1, weight_group + n_chunk_n_cols * params->weight_stride * sizeof(__fp16)), + fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_second); + } +