Skip to content

fix(sdk): prefetch the right weight chunk in batched F16 HMX matmul #1505

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Tadayuki Okada (TadayukiOkada) wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(sdk): prefetch the right weight chunk in batched F16 HMX matmul #1505

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion sdk/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,12 @@ enable_testing()
# per-model (llama-hexagon-power-mode.patch alone only reads it once, from
# GGML_HEXAGON_POWER_MODE, at backend-registry construction). See
# sdk/plugins/llama_cpp/src/htp_session.cpp for the caller.
# * llama-hexagon-hmx-f16-batched-prefetch.patch fixes the prefetch source of
# the second weight chunk in the batched F16 HMX matmul (GQA broadcast). It
# read one row past the first chunk, so the second chunk's output was wrong
# whenever the weights span more than one chunk (attention prefill without
# flash attention at long KV). Upstream ggml-org/llama.cpp#29626 carries the
# same fix; delete this file once the submodule is bumped past it.
# * llama-opencl-adreno-cpy-constreg.patch works around an Adreno OpenCL
# compiler crash: kernel_cpy_f32_f32_pack's min(ne00, get_local_size(0))
# emits an IMINrr with two constant-class operands, which the driver's
Expand All @@ -175,7 +181,8 @@ if(GENIEX_PLUGIN_LLAMA_CPP)
list(APPEND _LLAMA_PATCHES
"${CMAKE_CURRENT_SOURCE_DIR}/patches/llama-hexagon-release-sessions.patch"
"${CMAKE_CURRENT_SOURCE_DIR}/patches/llama-hexagon-power-mode.patch"
"${CMAKE_CURRENT_SOURCE_DIR}/patches/llama-hexagon-power-mode-setter.patch")
"${CMAKE_CURRENT_SOURCE_DIR}/patches/llama-hexagon-power-mode-setter.patch"
"${CMAKE_CURRENT_SOURCE_DIR}/patches/llama-hexagon-hmx-f16-batched-prefetch.patch")
endif()
if(GGML_OPENCL)
list(APPEND _LLAMA_PATCHES
Expand Down
13 changes: 13 additions & 0 deletions sdk/patches/llama-hexagon-hmx-f16-batched-prefetch.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
diff --git a/ggml/src/ggml-hexagon/htp/matmul-ops.c b/ggml/src/ggml-hexagon/htp/matmul-ops.c
index a09bc7a28..526f200b0 100644
--- a/ggml/src/ggml-hexagon/htp/matmul-ops.c
+++ b/ggml/src/ggml-hexagon/htp/matmul-ops.c
@@ -3356,7 +3356,7 @@ static int hmx_mm_f16_f32_batched(struct htp_context *ctx, const hmx_mm_f16_f32_
}
if (n_chunk_n_cols < (size_t) params->n) {
const size_t n_cols_second = hex_smin((size_t) params->n - n_chunk_n_cols, n_chunk_n_cols);
- dma_queue_push(weight_dma, dma_make_data(vtcm_scratch1, weight_group + params->weight_stride * sizeof(__fp16)),
+ dma_queue_push(weight_dma, dma_make_data(vtcm_scratch1, weight_group + n_chunk_n_cols * params->weight_stride * sizeof(__fp16)),
fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_second);
}