Skip to content

Commit 0eca4d4

Browse files
cuda : prevent integer truncation and overflow errors when using KQ mask strides in flash_attn_mask_to_KV_max kernel (ggml-org#24945)
Co-authored-by: Stanisław Szymczyk <sszymczy@gmail.com>
1 parent 4f31eed commit 0eca4d4

1 file changed

Lines changed: 3 additions & 3 deletions

File tree

ggml/src/ggml-cuda/fattn-common.cuh

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -664,7 +664,7 @@ constexpr __device__ dequantize_V_t get_dequantize_V() {
664664
template <int ncols1>
665665
__launch_bounds__(FATTN_KQ_STRIDE/2, 1)
666666
static __global__ void flash_attn_mask_to_KV_max(
667-
const half2 * __restrict__ mask, int * __restrict__ KV_max, const int ne30, const int s31, const int s33) {
667+
const half2 * __restrict__ mask, int * __restrict__ KV_max, const int ne30, const int64_t s31, const int64_t s33) {
668668
const int ne31 = gridDim.x;
669669
const int tid = threadIdx.x;
670670
const int sequence = blockIdx.y;
@@ -1089,8 +1089,8 @@ void launch_fattn(
10891089
// Only worth the overhead if there is at lease one FATTN_KQ_STRIDE x FATTN_KQ_STRIDE square to be skipped or
10901090
// multiple sequences of possibly different lengths.
10911091
if (mask && K->ne[1] % FATTN_KQ_STRIDE == 0 && (Q->ne[1] >= 1024 || Q->ne[3] > 1)) {
1092-
const int s31 = mask->nb[1] / sizeof(half2);
1093-
const int s33 = mask->nb[3] / sizeof(half2);
1092+
const int64_t s31 = mask->nb[1] / sizeof(half2);
1093+
const int64_t s33 = mask->nb[3] / sizeof(half2);
10941094

10951095
const dim3 blocks_num_KV_max(ntiles_x, Q->ne[3], 1);
10961096
const dim3 block_dim_KV_max(FATTN_KQ_STRIDE/2, 1, 1);

0 commit comments

Comments
 (0)