diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 85a1ae7ae208..357262a96d58 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -430,7 +430,12 @@ extern "C" { GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, GGML_TYPE_Q2_0 = 42, - GGML_TYPE_COUNT = 43, + // Prism-private ternary at group size 128, ported from PrismML-Eng/llama.cpp + // (Ternary-Bonsai GGUFs). High id so it slots above upstream types; type_traits + // is sized to COUNT with 43..142 unused. PQ2_0 (142) is deliberately not ported: + // it has no Vulkan kernels upstream of us either. + GGML_TYPE_PTQ1_0 = 143, + GGML_TYPE_COUNT = 144, }; // [TAG_GGML_PREC] @@ -486,6 +491,7 @@ extern "C" { GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors + GGML_FTYPE_MOSTLY_PTQ1_0 = 129, // except 1d tensors (Prism-private group-128 ternary) }; // available tensor operations: diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h index 1dbbe326d0fc..684bf5dce065 100644 --- a/ggml/src/ggml-common.h +++ b/ggml/src/ggml-common.h @@ -191,6 +191,19 @@ typedef struct { } block_q2_0; static_assert(sizeof(block_q2_0) == sizeof(ggml_half) + QK2_0 / 4, "wrong q2_0 block size/padding"); +// PTQ1_0: Prism-private ternary at group size 128 (ported from PrismML-Eng/llama.cpp). +// Same base-3 trit packing as upstream TQ1_0 (type 34) but one fp16 scale per 128 +// weights instead of per 256. 1.75 bpw, and lossless for checkpoints that are already +// ternary at group 128 -- TQ1_0 cannot represent those, because a 256-wide scale has +// to discard one of the two group scales it straddles. +#define QK_PTQ1_0 128 +typedef struct { + uint8_t qs[(QK_PTQ1_0 - 4*QK_PTQ1_0/64)/5]; // 24 B, 5 trits per byte -> 120 values + uint8_t qh[QK_PTQ1_0/64]; // 2 B, 4 trits per byte -> 8 values + ggml_half d; // scale +} block_ptq1_0; +static_assert(sizeof(block_ptq1_0) == sizeof(ggml_half) + QK_PTQ1_0/64 + (QK_PTQ1_0 - 4*QK_PTQ1_0/64)/5, "wrong ptq1_0 block size/padding"); + #define QK4_0 32 typedef struct { ggml_half d; // delta diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 2b9a426577c7..9aa99e5b87e2 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -5,6 +5,8 @@ // This effectively selects the generic implementation. #if defined(GGML_CPU_GENERIC) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define quantize_row_q8_0_generic quantize_row_q8_0 #define quantize_row_q8_1_generic quantize_row_q8_1 @@ -72,6 +74,8 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 @@ -82,6 +86,8 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp @@ -110,6 +116,8 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__POWERPC__) || defined(__powerpc__) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // ref: https://github.com/ggml-org/llama.cpp/pull/14146#issuecomment-2972561679 // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K @@ -157,6 +165,8 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__loongarch64) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K @@ -204,6 +214,8 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__riscv) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 @@ -244,6 +256,8 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__s390x__) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 @@ -293,6 +307,8 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__wasm__) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 87a329f26975..45cc15a115dc 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -237,6 +237,12 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_0, .nrows = 1, }, + [GGML_TYPE_PTQ1_0] = { + .from_float = quantize_row_ptq1_0, + .vec_dot = ggml_vec_dot_ptq1_0_q8_0, + .vec_dot_type = GGML_TYPE_Q8_0, + .nrows = 1, + }, [GGML_TYPE_Q4_0] = { .from_float = quantize_row_q4_0, .vec_dot = ggml_vec_dot_q4_0_q8_0, diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 266261c5e5a4..e739decd5f56 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -666,6 +666,7 @@ void ggml_compute_forward_add( } break; case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -1117,6 +1118,7 @@ void ggml_compute_forward_add1( } break; case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -1248,6 +1250,7 @@ void ggml_compute_forward_acc( case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4651,6 +4654,7 @@ void ggml_compute_forward_out_prod( switch (src0->type) { case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4927,6 +4931,7 @@ void ggml_compute_forward_set( case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -5152,6 +5157,7 @@ void ggml_compute_forward_get_rows( switch (src0->type) { case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -5909,6 +5915,7 @@ void ggml_compute_forward_clamp( case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index 5e36459f8cbc..0f9c8d8683d5 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -30,6 +30,10 @@ void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in quantize_row_q2_0_ref(x, y, k); } +void quantize_row_ptq1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + quantize_row_ptq1_0_ref(x, y, k); +} + void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { quantize_row_q4_0_ref(x, y, k); } @@ -1337,3 +1341,71 @@ void quantize_row_iq4_xs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, assert(k % QK_K == 0); quantize_iq4_xs(x, y, 1, k, NULL); } + +// PTQ1_0 x Q8_0. The trits are stored base-3 interleaved rather than in element +// order, so decode a block into element order first using the same traversal as +// dequantize_row_ptq1_0 -- that keeps the two provably in step. Four Q8_0 blocks +// cover one 128-wide PTQ1_0 block. +void ggml_vec_dot_ptq1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK_PTQ1_0; + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_ptq1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + static const uint8_t pow3[6] = {1, 3, 9, 27, 81, 243}; + static const size_t stages[3] = {32, 16, 8}; + + float sumf = 0.0f; + + for (int i = 0; i < nb; i++) { + int8_t q[QK_PTQ1_0]; + int o = 0; + + size_t j = 0; + for (size_t st = 0; st < 3; ++st) { + const size_t c = stages[st]; + for (; j + c <= sizeof(x->qs); j += c) { + for (size_t nn = 0; nn < 5; ++nn) { + for (size_t m = 0; m < c; ++m) { + const uint8_t v = x[i].qs[j + m] * pow3[nn]; + const int16_t xi = ((uint16_t) v * 3) >> 8; + q[o++] = (int8_t) (xi - 1); + } + } + } + } + for (size_t nn = 0; nn < 4; ++nn) { + for (size_t h = 0; h < sizeof(x->qh); ++h) { + const uint8_t v = x[i].qh[h] * pow3[nn]; + const int16_t xi = ((uint16_t) v * 3) >> 8; + q[o++] = (int8_t) (xi - 1); + } + } + assert(o == QK_PTQ1_0); + + const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); + float sumi = 0.0f; + + for (int k = 0; k < 4; k++) { + const block_q8_0 * GGML_RESTRICT yb = &y[i * 4 + k]; + const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); + int sumi_block = 0; + for (int b = 0; b < 32; ++b) { + sumi_block += (int) q[k*32 + b] * (int) yb->qs[b]; + } + sumi += d1 * sumi_block; + } + + sumf += d0 * sumi; + } + + *s = sumf; +} diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h index 93ea7eeffe5b..8bfe4d9467d7 100644 --- a/ggml/src/ggml-cpu/quants.h +++ b/ggml/src/ggml-cpu/quants.h @@ -32,6 +32,7 @@ void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_tq1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +void quantize_row_ptq1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); @@ -56,6 +57,7 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_ptq1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); @@ -84,6 +86,7 @@ void ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_ptq1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 1ebc50a763f1..9fde3209e519 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -2125,6 +2125,107 @@ size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, return nrow * row_size; } +// ====================== PTQ1_0 (Prism ternary, group 128) ====================== +// Base-3 trit packing identical to upstream TQ1_0, but at block 128 so one fp16 +// scale covers 128 weights. qs is 24 bytes, which TQ1_0's fixed 32-then-16 byte +// staging cannot cover, so the stages are generalised to 32/16/8; at TQ1_0's +// 48-byte qs this reduces to exactly its original 32-then-16 behaviour. +static const size_t ptq1_0_stages[3] = {32, 16, 8}; + +void quantize_row_ptq1_0_ref(const float * GGML_RESTRICT x, block_ptq1_0 * GGML_RESTRICT y, int64_t k) { + assert(k % QK_PTQ1_0 == 0); + const int64_t nb = k / QK_PTQ1_0; + + for (int64_t i = 0; i < nb; i++) { + float amax = 0.0f; + for (int j = 0; j < QK_PTQ1_0; j++) { + amax = MAX(amax, fabsf(x[j])); + } + + const float d = amax; + const float id = d ? 1.0f/d : 0.0f; + + y[i].d = GGML_FP32_TO_FP16(d); + + size_t j = 0; + for (size_t s = 0; s < 3; ++s) { + const size_t c = ptq1_0_stages[s]; + for (; j + c <= sizeof(y->qs); j += c) { + for (size_t m = 0; m < c; ++m) { + uint8_t q = 0; + for (size_t n = 0; n < 5; ++n) { + int xi = lroundf(x[m + n*c] * id) + 1; // -1, 0, 1 -> 0, 1, 2 + q *= 3; + q += xi; + } + // ceiling division (243 == pow(3, 5)) + q = ((uint16_t)q * 256 + (243 - 1)) / 243; + y[i].qs[j + m] = q; + } + x += 5*c; + } + } + // 4 elements per byte + for (size_t h = 0; h < sizeof(y->qh); ++h) { + uint8_t q = 0; + for (size_t m = 0; m < 4; ++m) { + int xi = lroundf(x[h + m*sizeof(y->qh)] * id) + 1; + q *= 3; + q += xi; + } + // shift the first value to the most significant trit + q *= 3; + q = ((uint16_t)q * 256 + (243 - 1)) / 243; + y[i].qh[h] = q; + } + x += 4*sizeof(y->qh); + } +} + +void dequantize_row_ptq1_0(const block_ptq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_PTQ1_0 == 0); + const int64_t nb = k / QK_PTQ1_0; + + const uint8_t pow3[6] = {1, 3, 9, 27, 81, 243}; + + for (int64_t i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d); + + size_t j = 0; + for (size_t s = 0; s < 3; ++s) { + const size_t c = ptq1_0_stages[s]; + for (; j + c <= sizeof(x->qs); j += c) { + for (size_t n = 0; n < 5; ++n) { + for (size_t m = 0; m < c; ++m) { + uint8_t q = x[i].qs[j + m] * pow3[n]; + int16_t xi = ((uint16_t) q * 3) >> 8; + *y++ = (float) (xi - 1) * d; + } + } + } + } + for (size_t n = 0; n < 4; ++n) { + for (size_t h = 0; h < sizeof(x->qh); ++h) { + uint8_t q = x[i].qh[h] * pow3[n]; + int16_t xi = ((uint16_t) q * 3) >> 8; + *y++ = (float) (xi - 1) * d; + } + } + } +} + +size_t quantize_ptq1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + (void)quant_weights; // ternary codes come from the weights themselves; an imatrix has no role + const size_t row_size = ggml_row_size(GGML_TYPE_PTQ1_0, n_per_row); + char * qrow = (char *)dst; + for (int64_t row = 0; row < nrow; ++row) { + quantize_row_ptq1_0_ref(src, (block_ptq1_0 *)qrow, n_per_row); + src += n_per_row; + qrow += row_size; + } + return nrow * row_size; +} + size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { if (!quant_weights) { quantize_row_q4_0_ref(src, dst, (int64_t)nrow*n_per_row); diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h index 75188f1af180..87ce496d1a63 100644 --- a/ggml/src/ggml-quants.h +++ b/ggml/src/ggml-quants.h @@ -34,6 +34,7 @@ GGML_API void quantize_row_q6_K_ref(const float * GGML_RESTRICT x, block_q6_K * GGML_API void quantize_row_q8_K_ref(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_tq1_0_ref(const float * GGML_RESTRICT x, block_tq1_0 * GGML_RESTRICT y, int64_t k); +GGML_API void quantize_row_ptq1_0_ref(const float * GGML_RESTRICT x, block_ptq1_0 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_tq2_0_ref(const float * GGML_RESTRICT x, block_tq2_0 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_iq3_xxs_ref(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k); @@ -63,6 +64,7 @@ GGML_API void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GG GGML_API void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_ptq1_0(const block_ptq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); @@ -87,6 +89,7 @@ GGML_API size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RE GGML_API size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_tq1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_ptq1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_tq2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index bafbec6c5cdc..6fbe2d3b9efe 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -5165,6 +5165,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { cm1_create({TYPE, GGML_TYPE_F16, false, false}, tc_mmq, "matmul_" #tstr "_f16", matmul_##tstr##_f16_cm1_len, matmul_##tstr##_f16_cm1_data, sizeof(vk_mat_mat_push_constants), 3); \ } FOR_EACH_LUT_TYPE_NONFP4(X_CM1) + X_CM1(GGML_TYPE_PTQ1_0, ptq1_0) #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) if (device->ocp_fp4) { #define X_CM1_OCP(TYPE, tstr) \ @@ -5222,6 +5223,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { cm1_create({TYPE, GGML_TYPE_F16, true, false}, tc_mmq_id, "matmul_id_subgroup_" #tstr "_f16", matmul_id_subgroup_##tstr##_f16_cm1_len, matmul_id_subgroup_##tstr##_f16_cm1_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ } FOR_EACH_LUT_TYPE_NONFP4(X_CM1_ID) + X_CM1_ID(GGML_TYPE_PTQ1_0, ptq1_0) #if defined(GGML_VULKAN_FLOAT_E2M1_GLSLC_SUPPORT) && defined(GGML_VULKAN_FLOAT_E4M3_GLSLC_SUPPORT) if (device->ocp_fp4) { #define X_CM1_ID_OCP(TYPE, tstr) \ @@ -5299,6 +5301,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { sg_create({TYPE, GGML_TYPE_F32, false, true}, tc_mmq, "matmul_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_##tstr##_f32), sizeof(vk_mat_mat_push_constants), 3); \ sg_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", SPV_DOT2(matmul_##tstr##_f32), sizeof(vk_mat_mat_push_constants), 3); FOR_EACH_LUT_TYPE(X_SG) + X_SG(GGML_TYPE_PTQ1_0, ptq1_0) #undef X_SG #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) @@ -5336,6 +5339,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { sg_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_subgroup_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); \ sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32", SPV_DOT2(matmul_id_subgroup_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); FOR_EACH_LUT_TYPE(X_SG_ID_SUB) + X_SG_ID_SUB(GGML_TYPE_PTQ1_0, ptq1_0) #undef X_SG_ID_SUB #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { @@ -5371,6 +5375,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { sg_create({TYPE, GGML_TYPE_F32, true, true}, tc_mmqid, "matmul_id_" #tstr "_f32_f16acc", SPV_DOT2_F16ACC(matmul_id_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); \ sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_" #tstr "_f32", SPV_DOT2(matmul_id_##tstr##_f32), sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); FOR_EACH_LUT_TYPE(X_SG_ID) + X_SG_ID(GGML_TYPE_PTQ1_0, ptq1_0) #undef X_SG_ID #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) if (device->integer_dot_product) { @@ -5407,6 +5412,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { #define X_SG_FP32(TYPE, tstr) \ sg_create({TYPE, GGML_TYPE_F32, false, false}, tc_mmq, "matmul_" #tstr "_f32", matmul_##tstr##_f32_fp32_len, matmul_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_push_constants), 3); FOR_EACH_LUT_TYPE(X_SG_FP32) + X_SG_FP32(GGML_TYPE_PTQ1_0, ptq1_0) #undef X_SG_FP32 #if defined(GGML_VULKAN_INTEGER_DOT_GLSLC_SUPPORT) @@ -5438,6 +5444,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { #define X_SG_ID_SUB_FP32(TYPE, tstr) \ sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_subgroup_" #tstr "_f32", matmul_id_subgroup_##tstr##_f32_fp32_len, matmul_id_subgroup_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count, mul_mat_subgroup_size); FOR_EACH_LUT_TYPE(X_SG_ID_SUB_FP32) + X_SG_ID_SUB_FP32(GGML_TYPE_PTQ1_0, ptq1_0) #undef X_SG_ID_SUB_FP32 } else { sg_create({GGML_TYPE_F32, GGML_TYPE_F32, true, false}, tc_mm, "matmul_id_f32_f32", matmul_id_f32_f32_fp32_len, matmul_id_f32_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); @@ -5450,6 +5457,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { #define X_SG_ID_FP32(TYPE, tstr) \ sg_create({TYPE, GGML_TYPE_F32, true, false}, tc_mmqid, "matmul_id_" #tstr "_f32", matmul_id_##tstr##_f32_fp32_len, matmul_id_##tstr##_f32_fp32_data, sizeof(vk_mat_mat_id_push_constants), mul_mat_id_param_count); FOR_EACH_LUT_TYPE(X_SG_ID_FP32) + X_SG_ID_FP32(GGML_TYPE_PTQ1_0, ptq1_0) #undef X_SG_ID_FP32 } } @@ -5594,6 +5602,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f32_f32", arr_dmmv_f16_f32_f32_len[reduc], arr_dmmv_f16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f32_f32", arr_dmmv_bf16_f32_f32_len[reduc], arr_dmmv_bf16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f32_f32", arr_dmmv_q1_0_f32_f32_len[reduc], arr_dmmv_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_PTQ1_0][i], "mul_mat_vec_ptq1_0_f32_f32", arr_dmmv_ptq1_0_f32_f32_len[reduc], arr_dmmv_ptq1_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f32_f32", arr_dmmv_q2_0_f32_f32_len[reduc], arr_dmmv_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f32_f32", arr_dmmv_q4_0_f32_f32_len[reduc], arr_dmmv_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f32_f32", arr_dmmv_q4_1_f32_f32_len[reduc], arr_dmmv_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); @@ -5623,6 +5632,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f16_f32", arr_dmmv_f16_f16_f32_len[reduc], arr_dmmv_f16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f16_f32", arr_dmmv_bf16_f16_f32_len[reduc], arr_dmmv_bf16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f16_f32", arr_dmmv_q1_0_f16_f32_len[reduc], arr_dmmv_q1_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_PTQ1_0][i], "mul_mat_vec_ptq1_0_f16_f32", arr_dmmv_ptq1_0_f16_f32_len[reduc], arr_dmmv_ptq1_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f16_f32", arr_dmmv_q2_0_f16_f32_len[reduc], arr_dmmv_q2_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f16_f32", arr_dmmv_q4_0_f16_f32_len[reduc], arr_dmmv_q4_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f16_f32", arr_dmmv_q4_1_f16_f32_len[reduc], arr_dmmv_q4_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); @@ -5679,6 +5689,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_F16 ], "mul_mat_vec_id_f16_f32", arr_dmmv_id_f16_f32_f32_len[reduc], arr_dmmv_id_f16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2, 1, 1}, {wg_size_subgroup, 2}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_BF16], "mul_mat_vec_id_bf16_f32", arr_dmmv_id_bf16_f32_f32_len[reduc], arr_dmmv_id_bf16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2, 1, 1}, {wg_size_subgroup, 2}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q1_0], "mul_mat_vec_id_q1_0_f32", arr_dmmv_id_q1_0_f32_f32_len[reduc], arr_dmmv_id_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_PTQ1_0], "mul_mat_vec_id_ptq1_0_f32", arr_dmmv_id_ptq1_0_f32_f32_len[reduc], arr_dmmv_id_ptq1_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_0], "mul_mat_vec_id_q2_0_f32", arr_dmmv_id_q2_0_f32_f32_len[reduc], arr_dmmv_id_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_0], "mul_mat_vec_id_q4_0_f32", arr_dmmv_id_q4_0_f32_f32_len[reduc], arr_dmmv_id_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_1], "mul_mat_vec_id_q4_1_f32", arr_dmmv_id_q4_1_f32_f32_len[reduc], arr_dmmv_id_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); @@ -5745,6 +5756,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { // dequant shaders ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_F32 ], "f32_to_f16", dequant_f32_len, dequant_f32_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q1_0], "dequant_q1_0", dequant_q1_0_len, dequant_q1_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 8, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_PTQ1_0], "dequant_ptq1_0", dequant_ptq1_0_len, dequant_ptq1_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 8, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q2_0], "dequant_q2_0", dequant_q2_0_len, dequant_q2_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_0], "dequant_q4_0", dequant_q4_0_len, dequant_q4_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_1], "dequant_q4_1", dequant_q4_1_len, dequant_q4_1_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); @@ -5776,6 +5788,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_F16 ], "get_rows_f16", get_rows_f16_len, get_rows_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_BF16], "get_rows_bf16", get_rows_bf16_len, get_rows_bf16_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q1_0], "get_rows_q1_0", get_rows_q1_0_len, get_rows_q1_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_PTQ1_0], "get_rows_ptq1_0", get_rows_ptq1_0_len, get_rows_ptq1_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q2_0], "get_rows_q2_0", get_rows_q2_0_len, get_rows_q2_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_0], "get_rows_q4_0", get_rows_q4_0_len, get_rows_q4_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_1], "get_rows_q4_1", get_rows_q4_1_len, get_rows_q4_1_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); @@ -5806,6 +5819,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_F16 ], "get_rows_f16_f32", get_rows_f16_f32_len, get_rows_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_BF16], "get_rows_bf16_f32", get_rows_bf16_f32_len, get_rows_bf16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q1_0], "get_rows_q1_0_f32", get_rows_q1_0_f32_len, get_rows_q1_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_PTQ1_0], "get_rows_ptq1_0_f32", get_rows_ptq1_0_f32_len, get_rows_ptq1_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q2_0], "get_rows_q2_0_f32", get_rows_q2_0_f32_len, get_rows_q2_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_0], "get_rows_q4_0_f32", get_rows_q4_0_f32_len, get_rows_q4_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_1], "get_rows_q4_1_f32", get_rows_q4_1_f32_len, get_rows_q4_1_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); @@ -8083,6 +8097,7 @@ static vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type switch (type) { case GGML_TYPE_F32: case GGML_TYPE_Q1_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -8147,6 +8162,7 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -8234,6 +8250,7 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec_id(ggml_backend_vk_context case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -19376,6 +19393,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -19483,6 +19501,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl index 9df66cb44f9d..d22a1c45dc26 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl @@ -126,6 +126,17 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { } #endif +#if defined(DATA_A_PTQ1_0) +#include "ptq1_0.glsl" + +vec2 dequantize(uint ib, uint iqs, uint a_offset) { + return ptq1_0_trits2(ib, a_offset, iqs); +} +vec4 dequantize4(uint ib, uint iqs, uint a_offset) { + return ptq1_0_trits4(ib, a_offset, iqs); +} +#endif + #if defined(DATA_A_Q1_0) vec2 dequantize(uint ib, uint iqs, uint a_offset) { const uint bits = uint(data_a[a_offset + ib].qs[iqs / 8u]) >> (iqs % 8u); @@ -564,6 +575,13 @@ vec2 get_dm(uint ib, uint a_offset) { } #endif +#if defined(DATA_A_PTQ1_0) +vec2 get_dm(uint ib, uint a_offset) { + const float d = float(data_a[a_offset + ib].d); + return vec2(d, 0); +} +#endif + #if defined(DATA_A_Q1_0) vec2 get_dm(uint ib, uint a_offset) { const float d = float(data_a[a_offset + ib].d); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp new file mode 100644 index 000000000000..995235d041ce --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp @@ -0,0 +1,65 @@ +#version 450 + +#include "dequant_head.glsl" + +layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer A {block_ptq1_0 data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; + +// One invocation per 8 elements: 16 invocations cover a 128-weight block. Element +// order follows the CPU codec (16-byte qs chunk, 8-byte qs chunk, then qh), so the +// index maths lives in one place rather than being reproduced per shader. +// 3^n for n in [0,4]. Written as selects rather than a const array: a +// dynamically indexed local array can land in scratch memory on some drivers, +// and this sits on the hot path of every ternary matmul. +uint ptq1_0_pow3(uint n) { + return n < 2u ? (n == 0u ? 1u : 3u) + : (n == 2u ? 9u : (n == 3u ? 27u : 81u)); +} + +float ptq1_0_trit_l(uint ib, uint e) { + uint b; + uint n; + if (e < 80u) { + b = uint(data_a[ib].qs[e & 15u]); + n = e >> 4u; + } else if (e < 120u) { + const uint t = e - 80u; + b = uint(data_a[ib].qs[16u + (t & 7u)]); + n = t >> 3u; + } else { + const uint t = e - 120u; + b = uint(data_a[ib].qh[t & 1u]); + n = t >> 1u; + } + + // Mod-256 multiplication is associative, so the repeated v = (v*3)&0xFF that + // walks to trit n collapses to a single multiply by 3^n -- which is what + // dequantize_row_ptq1_0 in ggml-quants.c already does. Verified identical over + // all 256 byte values x 5 trit positions. The serial form cost up to 4 + // dependent multiplies per element with a trip count that varies by element + // position, so lanes in a subgroup diverged; this is branch-free and uniform. + const uint v = (b * ptq1_0_pow3(n)) & 0xFFu; + return float(int((v * 3u) >> 8u) - 1); +} + +void main() { + const uint i = gl_WorkGroupID.x * 4 + gl_LocalInvocationID.x / 64; + + const uint tid = gl_LocalInvocationID.x % 64; + const uint il = tid / 4; + const uint ir = tid % 4; + const uint ib = 4*i + ir; + if (ib >= p.nel / 128) { + return; + } + + const uint b_idx = 512*i + 128*ir + 8*il; + + const float d = float(data_a[ib].d); + + [[unroll]] for (uint l = 0; l < 8; ++l) { + data_b[b_idx + l] = D_TYPE(ptq1_0_trit_l(ib, 8*il + l) * d); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp index 5a9d0e778fdf..dec377c15ff1 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp @@ -74,8 +74,13 @@ void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const ibi += p.ncols; #if K_PER_ITER == 8 +#if defined(DATA_A_PTQ1_0) + vec4 v, v2; + ptq1_0_trits8(ib, a_offset, iqs, v, v2); +#else vec4 v = dequantize4(ib, iqs, a_offset); vec4 v2 = dequantize4(ib, iqs+(4/QUANT_R), a_offset); +#endif const vec2 dm = get_dm(ib, a_offset); if (dm.y != 0) { // quant has min component @@ -138,6 +143,54 @@ void iter_aligned_nonquant(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint } #endif +#if defined(DATA_A_PTQ1_0) +// Eight lanes own a whole block: six packed qs words, one qh tail, one +// inactive lane. Each packed byte is loaded once and supplies all its trits. +// Keep the common reduction and fusion path, including partial output tiles. +void accumulate_ptq1_0(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], + uint first_row, uint num_rows, uint tid) { + const uint lane = tid & 7u; + const uint blocks = p.ncols / 128u; + if (lane == 7u) { + return; + } + for (uint block = tid / 8u; block < blocks; block += BLOCK_SIZE / 8u) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint base = j*p.batch_stride_b + b_offset + block*128u; + if (lane < 6u) { + const uint start = lane < 4u ? lane*4u : 80u + (lane-4u)*4u; + const uint step = lane < 4u ? 16u : 8u; + vec4 activation[5]; + [[unroll]] for (uint t = 0; t < 5u; ++t) { + activation[t] = vec4(data_b_v4[(base + start + t*step)/4u]); + } + [[unroll]] for (uint row = 0; row < num_rows; ++row) { + const uint ib = a_offset + (first_row + row)*blocks + block; + const uint packed = data_a_packed32[ib].qs[lane]; + uvec4 bytes = (uvec4(packed) >> uvec4(0u, 8u, 16u, 24u)) & 255u; + float sum = 0.0f; + [[unroll]] for (uint t = 0; t < 5u; ++t) { + bytes *= 3u; + sum += dot(activation[t], vec4(bytes >> 8u) - 1.0f); + bytes &= 255u; + } + temp[j][row] += sum * get_dm(ib - a_offset, a_offset).x; + } + } else { + const vec4 b0 = vec4(data_b_v4[(base + 120u)/4u]); + const vec4 b1 = vec4(data_b_v4[(base + 124u)/4u]); + [[unroll]] for (uint row = 0; row < num_rows; ++row) { + const uint ib = (first_row + row)*blocks + block; + vec4 lo, hi; + ptq1_0_trits8(ib, a_offset, 120u, lo, hi); + temp[j][row] += (dot(b0, lo) + dot(b1, hi)) * get_dm(ib, a_offset).x; + } + } + } + } +} +#endif + void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { const uint tid = gl_LocalInvocationID.x; @@ -157,6 +210,9 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { } } +#if defined(DATA_A_PTQ1_0) + accumulate_ptq1_0(temp, first_row, num_rows, tid); +#else uint num_iters = p.ncols / (K_PER_ITER * BLOCK_SIZE); if (num_iters * K_PER_ITER * BLOCK_SIZE + K_PER_ITER*tid < p.ncols) { num_iters++; @@ -242,6 +298,7 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { } #endif +#endif // DATA_A_PTQ1_0 reduce_result(temp, d_offset, first_row, num_rows, tid); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp index 90e4e11cdec7..e88d10014770 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp @@ -200,6 +200,9 @@ shared FLOAT_TYPEV2 buf_b[BN * SHMEM_STRIDE]; shared ACC_TYPE coopmat_stage[TM * TN * NUM_WARPS]; #endif +#if defined(DATA_A_PTQ1_0) +#include "ptq1_0.glsl" +#endif #include "mul_mm_id_funcs.glsl" #include "mul_mm_funcs.glsl" diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl index dd05fb1bde69..13d37cfd7967 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl @@ -69,6 +69,24 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin } else { store_a(col, row, FLOAT_TYPEV2(0.0f)); } +#elif defined(DATA_A_PTQ1_0) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + + const uint ib = idx / 16; + const uint grp = idx & 0xfu; // which 8-element group inside the block + const uint e0 = grp * 8u; + + const float d = float(data_a[ib].d); + + const uint k_pair = row * LOAD_VEC_A / 2; + vec4 lo, hi; + ptq1_0_trits8(ib, 0u, e0, lo, hi); + lo *= d; + hi *= d; + store_a(col, k_pair, FLOAT_TYPEV2(lo.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(lo.zw)); + store_a(col, k_pair + 2, FLOAT_TYPEV2(hi.xy)); + store_a(col, k_pair + 3, FLOAT_TYPEV2(hi.zw)); #elif defined(DATA_A_IQ1_S) const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; const uint k_pair = row * LOAD_VEC_A / 2; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl new file mode 100644 index 000000000000..0f4da53c3752 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl @@ -0,0 +1,102 @@ +#ifndef PTQ1_0_GLSL +#define PTQ1_0_GLSL + +// Shared PTQ1_0 trit accessor. Lives in its own header because two consumers need it +// from different include chains: dequant_funcs.glsl (mul_mat_vec, get_rows, +// copy_from_quant) and mul_mm.comp (via mul_mm_funcs.glsl), and mul_mm.comp does not +// include dequant_funcs.glsl. Duplicating it would leave two copies that must stay in +// step with the CPU codec in ggml-quants.c, where a divergence shows up as wrong +// matmul results rather than a build error. +// +// Element order is not positional: a 16-byte chunk of qs where byte j carries +// elements t*16+j, then an 8-byte chunk carrying 80 + t*8 + (j-16), then qh at four +// trits per byte carrying 120 + t*2 + h. Trits come out by the base-3 remainder +// recurrence t = (v*3)>>8, v = (v*3)&0xFF. +// 3^n for n in [0,4]. Written as selects rather than a const array: a +// dynamically indexed local array can land in scratch memory on some drivers, +// and this sits on the hot path of every ternary matmul. +uint ptq1_0_pow3(uint n) { + return n < 2u ? (n == 0u ? 1u : 3u) + : (n == 2u ? 9u : (n == 3u ? 27u : 81u)); +} + +float ptq1_0_trit(uint ib, uint a_offset, uint e) { + uint b; + uint n; + if (e < 80u) { + b = uint(data_a[a_offset + ib].qs[e & 15u]); + n = e >> 4u; + } else if (e < 120u) { + const uint t = e - 80u; + b = uint(data_a[a_offset + ib].qs[16u + (t & 7u)]); + n = t >> 3u; + } else { + const uint t = e - 120u; + b = uint(data_a[a_offset + ib].qh[t & 1u]); + n = t >> 1u; + } + + // Mod-256 multiplication is associative, so the repeated v = (v*3)&0xFF that + // walks to trit n collapses to a single multiply by 3^n -- which is what + // dequantize_row_ptq1_0 in ggml-quants.c already does. Verified identical over + // all 256 byte values x 5 trit positions. The serial form cost up to 4 + // dependent multiplies per element with a trip count that varies by element + // position, so lanes in a subgroup diverged; this is branch-free and uniform. + const uint v = (b * ptq1_0_pow3(n)) & 0xFFu; + return float(int((v * 3u) >> 8u) - 1); +} + +// Decode four independent bytes at the same trit position. One aligned word +// load replaces four byte loads; mask before multiplying by 3, not afterwards. +vec4 ptq1_0_decode4(uint packed, uint power) { + const uvec4 bytes = (uvec4(packed) >> uvec4(0u, 8u, 16u, 24u)) & 255u; + return vec4((((bytes * power) & 255u) * 3u) >> 8u) - 1.0f; +} + +// e must be even. All qs region/position boundaries are multiples of eight. +vec2 ptq1_0_trits2(uint ib, uint a_offset, uint e) { + uint packed; + uint n; + if (e < 120u) { + const uint q = e < 80u ? (e & 15u) : (16u + (e & 7u)); + n = e < 80u ? (e >> 4u) : ((e - 80u) >> 3u); + packed = data_a_packed32[a_offset + ib].qs[q >> 2u] >> ((q & 2u) * 8u); + } else { + packed = uint(data_a_packed32[a_offset + ib].qh); + n = (e - 120u) >> 1u; + } + const uvec2 bytes = uvec2(packed, packed >> 8u) & 255u; + return vec2((((bytes * ptq1_0_pow3(n)) & 255u) * 3u) >> 8u) - 1.0f; +} + +// e must be four-aligned. The qh tail repeats two bytes at successive powers. +vec4 ptq1_0_trits4(uint ib, uint a_offset, uint e) { + if (e < 120u) { + const uint q = e < 80u ? (e & 15u) : (16u + (e & 7u)); + const uint n = e < 80u ? (e >> 4u) : ((e - 80u) >> 3u); + return ptq1_0_decode4(data_a_packed32[a_offset + ib].qs[q >> 2u], ptq1_0_pow3(n)); + } + const uint packed = uint(data_a_packed32[a_offset + ib].qh); + const uvec2 bytes = uvec2(packed, packed >> 8u) & 255u; + const uvec4 powers = e == 120u ? uvec4(1u, 1u, 3u, 3u) : uvec4(9u, 9u, 27u, 27u); + return vec4((((bytes.xyxy * powers) & 255u) * 3u) >> 8u) - 1.0f; +} + +// Shared hot path for matvec and matmul: one region decision and one power +// selection per eight outputs, with just two word loads (one halfword for qh). +void ptq1_0_trits8(uint ib, uint a_offset, uint e, out vec4 lo, out vec4 hi) { + if (e < 120u) { + const uint q = e < 80u ? (e & 15u) : 16u; + const uint n = e < 80u ? (e >> 4u) : ((e - 80u) >> 3u); + const uint power = ptq1_0_pow3(n); + lo = ptq1_0_decode4(data_a_packed32[a_offset + ib].qs[q >> 2u], power); + hi = ptq1_0_decode4(data_a_packed32[a_offset + ib].qs[(q >> 2u) + 1u], power); + } else { + const uint packed = uint(data_a_packed32[a_offset + ib].qh); + const uvec2 bytes = uvec2(packed, packed >> 8u) & 255u; + lo = vec4((((bytes.xyxy * uvec4(1u, 1u, 3u, 3u)) & 255u) * 3u) >> 8u) - 1.0f; + hi = vec4((((bytes.xyxy * uvec4(9u, 9u, 27u, 27u)) & 255u) * 3u) >> 8u) - 1.0f; + } +} + +#endif // PTQ1_0_GLSL diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index 21d601d0eb61..0c1837e6c4b7 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -211,6 +211,36 @@ struct block_q1_0 #define A_TYPE block_q1_0 #endif +// PTQ1_0: ternary at group 128, base-3 packed five trits per byte. +// Field order mirrors block_ptq1_0 in ggml-common.h EXACTLY -- qs, then qh, then d. +// Unlike q1_0 the scale is LAST, and getting that wrong silently misindexes every +// block rather than failing loudly. +#define QUANT_K_PTQ1_0 128 +#define QUANT_R_PTQ1_0 1 + +struct block_ptq1_0 +{ + uint8_t qs[24]; + uint8_t qh[2]; + float16_t d; +}; + +// Same 28-byte stride: qs at 0, qh at 24, fp16 scale at 26. +struct block_ptq1_0_packed32 +{ + uint32_t qs[6]; + uint16_t qh; + float16_t d; +}; + +#if defined(DATA_A_PTQ1_0) +#define A_TYPE_PACKED32 block_ptq1_0_packed32 +#define QUANT_K QUANT_K_PTQ1_0 +#define QUANT_R QUANT_R_PTQ1_0 +#define QUANT_AUXF 1 +#define A_TYPE block_ptq1_0 +#endif + #define QUANT_K_Q2_0 64 #define QUANT_R_Q2_0 1 diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index d3f425968df2..12f8ef75369f 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -50,6 +50,7 @@ const std::vector type_names = { "f32", "f16", "q1_0", + "ptq1_0", "q2_0", "q4_0", "q4_1", @@ -597,6 +598,13 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c if (tname == "bf16") { continue; } + // PTQ1_0 has no coopmat2 decoder: dequant_funcs_cm2.glsl carries no PTQ1_0 entry, + // so emitting mul_mm_cm2 for it fails shader compilation and takes the whole + // Vulkan build down, not just this type. Skip it; it falls back to the scalar and + // coopmat1 matmul paths, which are the ones implemented and tested. + if (coopmat2 && tname == "ptq1_0") { + continue; + } // Float types keep per-type compilation (different accumulation loop structure) if (tname == "f32" || tname == "f16") { @@ -629,8 +637,8 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c } #endif - if (is_lut_quant(tname)) { - std::string lva = lut_load_vec_a(tname); + if (is_lut_quant(tname) || tname == "ptq1_0") { + std::string lva = tname == "ptq1_0" ? "8" : lut_load_vec_a(tname); string_to_spv(shader_name + "_" + tname + "_f16" + dot2_sfx, source_name, merge_maps(merge_maps(base_dict, float_type_dict), {{data_a_key, "1"}, {"LOAD_VEC_A", lva}, {"LOAD_VEC_B", load_vec}, {"B_TYPE", aligned_b_type_f16}, {"B_TYPE_SCALAR", "float16_t"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}}), fp16, coopmat, coopmat2, f16acc); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 5ef03e190e34..c13ed60fb831 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -690,6 +690,14 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .to_float = (ggml_to_float_t) dequantize_row_q2_0, .from_float_ref = (ggml_from_float_t) quantize_row_q2_0_ref, }, + [GGML_TYPE_PTQ1_0] = { + .type_name = "ptq1_0", + .blck_size = QK_PTQ1_0, + .type_size = sizeof(block_ptq1_0), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_ptq1_0, + .from_float_ref = (ggml_from_float_t) quantize_row_ptq1_0_ref, + }, [GGML_TYPE_Q4_0] = { .type_name = "q4_0", .blck_size = QK4_0, @@ -8065,6 +8073,7 @@ size_t ggml_quantize_chunk( case GGML_TYPE_Q4_K: result = quantize_q4_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q5_K: result = quantize_q5_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q6_K: result = quantize_q6_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_PTQ1_0: result = quantize_ptq1_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_TQ1_0: result = quantize_tq1_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_TQ2_0: result = quantize_tq2_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_IQ2_XXS: result = quantize_iq2_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; diff --git a/include/llama.h b/include/llama.h index 2bafdde0c986..b5b63eb01777 100644 --- a/include/llama.h +++ b/include/llama.h @@ -156,6 +156,7 @@ extern "C" { LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors + LLAMA_FTYPE_MOSTLY_PTQ1_0 = 143, // except 1d tensors (Prism group-128 ternary, 1.75 bpw) LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 221c2d2185ac..0df4a2946e25 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -24,6 +24,78 @@ // llama_context // +// Verify that every Hadamard-folded weight consumed by the graph receives its +// activation-side transform, and every latent lookup table gets the inverse. +// An architecture whose matmul path bypasses the transform helpers would +// otherwise load cleanly and silently compute wrong results. +static void llama_verify_hadamard_graph( + ggml_cgraph * gf, + const llama_hadamard_rotations & rotations, + const llama_hadamard_rotations & inverses) { + auto unwrap = [](const ggml_tensor * t) { + while (t && (t->op == GGML_OP_RESHAPE || t->op == GGML_OP_VIEW)) { + t = t->src[0]; + } + return t; + }; + + std::map lookups; // get_rows results of latent tables + + for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) { + const ggml_tensor * node = ggml_graph_node(gf, i); + + if (node->op == GGML_OP_GET_ROWS && inverses.count(node->src[0])) { + lookups.emplace(node, false); + continue; + } + + if (node->op != GGML_OP_MUL_MAT && node->op != GGML_OP_MUL_MAT_ID) { + continue; + } + + if (node->op == GGML_OP_MUL_MAT && ((const int32_t *) node->op_params)[1] == GGML_HINT_SRC0_IS_HADAMARD) { + const auto lk = lookups.find(unwrap(node->src[1])); + if (lk != lookups.end()) { + lk->second = true; + } + continue; + } + + const auto it = rotations.find(node->src[0]); + if (it == rotations.end()) { + continue; + } + const ggml_tensor * src = unwrap(node->src[1]); + const bool transformed = src && src->op == GGML_OP_MUL_MAT && + ((const int32_t *) src->op_params)[1] == GGML_HINT_SRC0_IS_HADAMARD && + src->src[0] == it->second.rot; + if (!transformed) { + throw std::runtime_error(format( + "Hadamard-folded weight '%s' is consumed without its activation transform; " + "this graph's matmul path does not support prism.hadamard folding", + node->src[0]->name)); + } + } + + for (const auto & [node, ok] : lookups) { + if (!ok) { + for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) { + const ggml_tensor * n2 = ggml_graph_node(gf, i); + for (int s = 0; s < GGML_MAX_SRC && n2->src[s]; ++s) { + if (unwrap(n2->src[s]) == node) { + LLAMA_LOG_WARN("%s: latent lookup '%s' consumed by op=%s name='%s' src%d hint=%d\n", + __func__, node->name, ggml_op_name(n2->op), n2->name, s, + ((const int32_t *) n2->op_params)[1]); + } + } + } + throw std::runtime_error(format( + "Hadamard-latent table '%s' is read without the inverse transform", + node->src[0]->name)); + } + } +} + static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) { switch (ctx_type) { case LLAMA_CONTEXT_TYPE_DEFAULT: return LLM_GRAPH_TYPE_DEFAULT; @@ -2467,6 +2539,13 @@ ggml_cgraph * llama_context::graph_reserve( auto * gf = model.build_graph(gparams); + // verify transform coverage on the pristine graph: after scheduling, + // cross-backend copies break the producer chain the check follows + if (!hadamard_verified && gf && (!model.hadamard_rotations.empty() || !model.hadamard_inverses.empty())) { + llama_verify_hadamard_graph(gf, model.hadamard_rotations, model.hadamard_inverses); + hadamard_verified = true; + } + this->n_outputs = save_n_outputs; // initialize scheduler with the specified graph @@ -2502,6 +2581,8 @@ llm_graph_params llama_context::graph_params( /*.loras =*/ loras.get(), /*.mctx =*/ mctx, /*.cross =*/ &cross, + /*.hadamard_rotations =*/&model.hadamard_rotations, + /*.hadamard_inverses =*/&model.hadamard_inverses, /*.samplers =*/ sampling.samplers, /*.n_outputs =*/ n_outputs, /*.cb =*/ graph_get_cb(), diff --git a/src/llama-context.h b/src/llama-context.h index b089e1267aaf..848fdc3f6857 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -376,6 +376,9 @@ struct llama_context { bool has_evaluated_once = false; + // prism.hadamard transform coverage is checked once, on the first built graph + bool hadamard_verified = false; + // env: LLAMA_GRAPH_REUSE_DISABLE bool graph_reuse_disable = false; diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 657ea80f7f6e..41fb24c9ccf2 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1490,6 +1490,8 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) : loras (params.loras), mctx (params.mctx), cross (params.cross), + hadamard_rotations(params.hadamard_rotations), + hadamard_inverses(params.hadamard_inverses), samplers (params.samplers), cb_func (params.cb), res (params.res), @@ -1516,7 +1518,35 @@ ggml_tensor * llm_graph_context::build_lora_mm( ggml_tensor * w, ggml_tensor * cur, ggml_tensor * w_s) const { - ggml_tensor * res = ggml_mul_mat(ctx0, w, cur); + ggml_tensor * cur_mm = cur; + if (hadamard_rotations) { + const auto it = hadamard_rotations->find(w); + if (it != hadamard_rotations->end()) { + const auto & t = it->second; + // another folded weight on this same activation already built the transform + const auto memo_key = std::make_pair((const ggml_tensor *) cur, (const ggml_tensor *) t.rot); + const auto memo_it = hadamard_memo.find(memo_key); + if (memo_it != hadamard_memo.end()) { + cur_mm = memo_it->second; + } else { + if (t.perm_rep > 1) { + // tiled [hd, nk, rep] -> grouped [hd, rep, nk] feature order + ggml_tensor * x = ggml_is_contiguous(cur_mm) ? cur_mm : ggml_cont(ctx0, cur_mm); + const int64_t ne1 = x->ne[1], ne2 = x->ne[2], ne3 = x->ne[3]; + x = ggml_reshape_4d(ctx0, x, t.perm_hd, t.perm_nk, t.perm_rep, ne1*ne2*ne3); + x = ggml_cont(ctx0, ggml_permute(ctx0, x, 0, 2, 1, 3)); + cur_mm = ggml_reshape_4d(ctx0, x, t.perm_hd*t.perm_nk*t.perm_rep, ne1, ne2, ne3); + } + if (t.signs) { + cur_mm = ggml_mul(ctx0, cur_mm, t.signs); + } + cur_mm = llama_mul_mat_hadamard(ctx0, cur_mm, t.rot); + hadamard_memo[memo_key] = cur_mm; + } + } + } + + ggml_tensor * res = ggml_mul_mat(ctx0, w, cur_mm); if (w_s) { res = ggml_mul(ctx0, res, w_s); @@ -1548,7 +1578,35 @@ ggml_tensor * llm_graph_context::build_lora_mm_id( ggml_tensor * cur, // ggml_tensor * b ggml_tensor * ids, ggml_tensor * w_s) const { - ggml_tensor * res = ggml_mul_mat_id(ctx0, w, cur, ids); + ggml_tensor * cur_mm = cur; + if (hadamard_rotations) { + const auto it = hadamard_rotations->find(w); + if (it != hadamard_rotations->end()) { + const auto & t = it->second; + // another folded weight on this same activation already built the transform + const auto memo_key = std::make_pair((const ggml_tensor *) cur, (const ggml_tensor *) t.rot); + const auto memo_it = hadamard_memo.find(memo_key); + if (memo_it != hadamard_memo.end()) { + cur_mm = memo_it->second; + } else { + if (t.perm_rep > 1) { + // tiled [hd, nk, rep] -> grouped [hd, rep, nk] feature order + ggml_tensor * x = ggml_is_contiguous(cur_mm) ? cur_mm : ggml_cont(ctx0, cur_mm); + const int64_t ne1 = x->ne[1], ne2 = x->ne[2], ne3 = x->ne[3]; + x = ggml_reshape_4d(ctx0, x, t.perm_hd, t.perm_nk, t.perm_rep, ne1*ne2*ne3); + x = ggml_cont(ctx0, ggml_permute(ctx0, x, 0, 2, 1, 3)); + cur_mm = ggml_reshape_4d(ctx0, x, t.perm_hd*t.perm_nk*t.perm_rep, ne1, ne2, ne3); + } + if (t.signs) { + cur_mm = ggml_mul(ctx0, cur_mm, t.signs); + } + cur_mm = llama_mul_mat_hadamard(ctx0, cur_mm, t.rot); + hadamard_memo[memo_key] = cur_mm; + } + } + } + + ggml_tensor * res = ggml_mul_mat_id(ctx0, w, cur_mm, ids); if (w_s) { const int64_t n_expert = w_s->ne[0]; @@ -2383,6 +2441,18 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { cur = ggml_get_rows(ctx0, tok_embd, inp->tokens); + // a Hadamard-latent embedding table stores rotated rows; restore the + // primal basis right after the lookup: h = s * (H z) + if (hadamard_inverses) { + const auto it = hadamard_inverses->find(tok_embd); + if (it != hadamard_inverses->end()) { + cur = llama_mul_mat_hadamard(ctx0, cur, it->second.rot); + if (it->second.signs) { + cur = ggml_mul(ctx0, cur, it->second.signs); + } + } + } + // apply lora for embedding tokens if needed for (const auto & lora : *loras) { llama_adapter_lora_weight * lw = lora.first->get_weight(tok_embd); diff --git a/src/llama-graph.h b/src/llama-graph.h index cc4110639d4b..b87d7313f75d 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -17,6 +17,21 @@ struct ggml_cgraph; struct ggml_context; struct ggml_tensor; +// Maps a folded model weight to the activation-side transform applied +// immediately before the matmul: optional sign flip, then the normalized +// blockwise Hadamard rotation. +struct llama_hadamard_transform { + ggml_tensor * rot; + ggml_tensor * signs; // nullptr for identity sign mode + // when perm_rep > 1 the activation arrives with its feature axis in tiled + // head order [hd, nk, rep] and must be permuted to the grouped order + // the folded weight expects + int64_t perm_hd = 0; + int64_t perm_nk = 0; + int64_t perm_rep = 0; +}; +using llama_hadamard_rotations = std::unordered_map; + struct llama_cparams; struct llama_layer; @@ -786,6 +801,8 @@ struct llm_graph_params { const llama_adapter_loras * loras; const llama_memory_context_i * mctx; const llama_cross * cross; + const llama_hadamard_rotations * hadamard_rotations; + const llama_hadamard_rotations * hadamard_inverses; std::map samplers; @@ -1026,6 +1043,12 @@ struct llm_graph_context { const llama_adapter_loras * loras; const llama_memory_context_i * mctx; const llama_cross * cross; + const llama_hadamard_rotations * hadamard_rotations; + const llama_hadamard_rotations * hadamard_inverses; + + // Transforms shared by folded weights on the same activation. Key is (input, rotation); + // both must match. Valid for one graph build only. + mutable std::map, ggml_tensor *> hadamard_memo; std::map samplers; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 89ae60d2d571..6fd092525100 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -39,6 +39,7 @@ const char * llama_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_BF16: name = LLAMA_FTYPE_PREFIX "BF16"; break; case LLAMA_FTYPE_MOSTLY_Q1_0: name = LLAMA_FTYPE_PREFIX "Q1_0"; break; case LLAMA_FTYPE_MOSTLY_Q2_0: name = LLAMA_FTYPE_PREFIX "Q2_0"; break; + case LLAMA_FTYPE_MOSTLY_PTQ1_0: name = LLAMA_FTYPE_PREFIX "PTQ1_0 - 1.75 bpw ternary (group 128)"; break; case LLAMA_FTYPE_MOSTLY_Q4_0: name = LLAMA_FTYPE_PREFIX "Q4_0"; break; case LLAMA_FTYPE_MOSTLY_Q4_1: name = LLAMA_FTYPE_PREFIX "Q4_1"; break; case LLAMA_FTYPE_MOSTLY_Q5_0: name = LLAMA_FTYPE_PREFIX "Q5_0"; break; @@ -408,6 +409,9 @@ namespace GGUFMeta { return get_arr(llm_kv(kid), result, required); } + // string-key overloads, used by the prism.hadamard.* metadata + template bool llama_model_loader::get_arr(const std::string & key, std::vector & result, bool required); + template bool llama_model_loader::get_arr(const std::string & key, std::vector & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::vector & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::array & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::vector & result, bool required); @@ -438,6 +442,7 @@ namespace GGUFMeta { } template bool llama_model_loader::get_key (enum llm_kv kid, bool & result, bool required); + template bool llama_model_loader::get_key (const std::string & key, bool & result, bool required); template bool llama_model_loader::get_key (enum llm_kv kid, float & result, bool required); template bool llama_model_loader::get_key (enum llm_kv kid, uint32_t & result, bool required); template bool llama_model_loader::get_key(enum llm_kv kid, std::string & result, bool required); @@ -775,6 +780,7 @@ llama_model_loader::llama_model_loader( case GGML_TYPE_NVFP4: ftype = LLAMA_FTYPE_MOSTLY_NVFP4; break; case GGML_TYPE_Q1_0: ftype = LLAMA_FTYPE_MOSTLY_Q1_0; break; case GGML_TYPE_Q2_0: ftype = LLAMA_FTYPE_MOSTLY_Q2_0; break; + case GGML_TYPE_PTQ1_0: ftype = LLAMA_FTYPE_MOSTLY_PTQ1_0; break; default: { LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max)); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index a7a163e27cf9..5298bf327929 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1217,6 +1217,146 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { gguf_kv.emplace(name, value); } + uint32_t hadamard_version = 0; + if (ml.get_key("prism.hadamard.version", hadamard_version, false)) { + if (hadamard_version != 1) { + throw std::runtime_error(format("unsupported prism.hadamard.version: %u", hadamard_version)); + } + + uint32_t block_size = 0; + std::string transform; + std::string axis; + std::string sign_mode; + std::vector weight_names; + + ml.get_key("prism.hadamard.block_size", block_size); + ml.get_key("prism.hadamard.transform", transform); + ml.get_key("prism.hadamard.axis", axis); + ml.get_key("prism.hadamard.sign_mode", sign_mode); + ml.get_arr("prism.hadamard.weight_names", weight_names); + + if (block_size == 0 || (block_size & (block_size - 1)) != 0) { + throw std::runtime_error(format("invalid prism.hadamard.block_size: %u", block_size)); + } + if (transform != "normalized-sylvester-walsh-hadamard") { + throw std::runtime_error(format("unsupported prism.hadamard.transform: %s", transform.c_str())); + } + if (axis != "input-last-dimension") { + throw std::runtime_error(format("unsupported prism.hadamard.axis: %s", axis.c_str())); + } + if (sign_mode != "identity" && sign_mode != "explicit") { + throw std::runtime_error(format("unsupported prism.hadamard.sign_mode: %s", sign_mode.c_str())); + } + if (weight_names.empty()) { + throw std::runtime_error("prism.hadamard.weight_names is empty"); + } + + if (sign_mode == "explicit") { + std::vector sign_widths; + std::vector sign_values; + ml.get_arr("prism.hadamard.sign_widths", sign_widths); + ml.get_arr("prism.hadamard.sign_values", sign_values); + // explicit mode with no widths would leave the sign table empty, which reads + // as identity later and silently changes the model function + if (sign_widths.empty()) { + throw std::runtime_error("prism.hadamard.sign_mode is explicit but sign_widths is empty"); + } + size_t off = 0; + for (const int32_t width : sign_widths) { + if (width <= 0 || (uint32_t) width % block_size != 0 || off + width > sign_values.size()) { + throw std::runtime_error(format("invalid prism.hadamard sign width: %d", width)); + } + auto & vec = hadamard_sign_data[width]; + vec.assign(sign_values.begin() + off, sign_values.begin() + off + width); + for (const int32_t v : vec) { + if (v != 1 && v != -1) { + throw std::runtime_error("prism.hadamard sign values must be +/-1"); + } + } + off += width; + } + if (off != sign_values.size()) { + throw std::runtime_error("prism.hadamard.sign_values length mismatch"); + } + } + + ml.get_key("prism.hadamard.gdn_v_grouped", hadamard_gdn_v_grouped, false); + + // the activation-side transform is applied only by build_lora_mm/build_lora_mm_id; + // refuse to load folded weights for architectures or tensor kinds that are not + // verified to route every matmul through those helpers, rather than run wrong math + switch (arch) { + case LLM_ARCH_LLAMA: + case LLM_ARCH_QWEN3: + case LLM_ARCH_QWEN3MOE: + case LLM_ARCH_QWEN35: + case LLM_ARCH_QWEN35MOE: + case LLM_ARCH_QWEN3NEXT: + break; + default: + throw std::runtime_error(format( + "prism.hadamard: arch '%s' is not verified to apply the activation transform to all folded weights", + llm_arch_name(arch))); + } + + const auto is_foldable_weight = [](const std::string & name) { + static const char * kinds[] = { + "attn_q", "attn_k", "attn_v", "attn_qkv", "attn_gate", "attn_output", + "ffn_gate", "ffn_up", "ffn_down", + "ffn_gate_exps", "ffn_up_exps", "ffn_down_exps", "ffn_gate_up_exps", + "ffn_gate_shexp", "ffn_up_shexp", "ffn_down_shexp", + "ssm_out", + }; + if (name == "output.weight") { + return true; // the output head is built through build_lora_mm in every arch + } + if (name.compare(0, 4, "blk.") != 0) { + return false; + } + size_t pos = 4; + while (pos < name.size() && isdigit((unsigned char) name[pos])) { + pos++; + } + if (pos == 4 || pos >= name.size() || name[pos] != '.') { + return false; + } + pos++; + for (const char * kind : kinds) { + const std::string suffix = std::string(kind) + ".weight"; + if (name.compare(pos, std::string::npos, suffix) == 0) { + return true; + } + } + return false; + }; + + for (const auto & weight_name : weight_names) { + if (!is_foldable_weight(weight_name)) { + throw std::runtime_error(format( + "prism.hadamard: weight '%s' is not on a verified Hadamard-aware matmul path", weight_name.c_str())); + } + if (!hadamard_weight_blocks.emplace(weight_name, block_size).second) { + throw std::runtime_error(format("duplicate prism.hadamard weight: %s", weight_name.c_str())); + } + } + + // tensors consumed by row lookup store latent rows and need the + // inverse transform applied to the lookup result instead + std::vector inverse_names; + ml.get_arr("prism.hadamard.inverse_weight_names", inverse_names, false); + for (const auto & name : inverse_names) { + // the graph applies the inverse only to the token-embedding lookup; any + // other latent table would load and silently stay rotated + if (name != "token_embd.weight") { + throw std::runtime_error(format( + "prism.hadamard: weight '%s' is not a verified inverse-after-lookup table", name.c_str())); + } + if (hadamard_weight_blocks.count(name) || !hadamard_inverse_blocks.emplace(name, block_size).second) { + throw std::runtime_error(format("duplicate prism.hadamard inverse weight: %s", name.c_str())); + } + } + } + // get general kv ml.get_key(LLM_KV_GENERAL_NAME, name, false); @@ -1854,6 +1994,163 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } } + if (!hadamard_weight_blocks.empty() || !hadamard_inverse_blocks.empty()) { + struct hadamard_rotation { + uint32_t block_size; + ggml_backend_buffer_type_t buft; + ggml_tensor * tensor; + }; + + std::vector rotations; + std::map, ggml_tensor *> sign_tensors; + + const std::pair *, llama_hadamard_rotations *> groups[] = { + { &hadamard_weight_blocks, &hadamard_rotations }, + { &hadamard_inverse_blocks, &hadamard_inverses }, + }; + // inverse (lookup-side) transforms must not inherit a host buffer + // type from a CPU-mapped table: the per-token transform would then + // ping-pong across the PCIe boundary. Prefer the buffer type the + // forward rotations live on (the GPU when layers are offloaded). + ggml_backend_buffer_type_t preferred_buft = nullptr; + + for (const auto & [blocks, target] : groups) + for (const auto & entry : *blocks) { + const std::string & weight_name = entry.first; + const uint32_t block_size = entry.second; + const ggml_tensor * weight = get_tensor(weight_name.c_str()); + if (weight == nullptr) { + throw std::runtime_error(format("prism.hadamard weight not found: %s", weight_name.c_str())); + } + if (weight->ne[0] % block_size != 0) { + throw std::runtime_error(format( + "prism.hadamard block size %u does not divide input dimension %lld for %s", + block_size, (long long) weight->ne[0], weight_name.c_str())); + } + if (weight->buffer == nullptr) { + throw std::runtime_error(format("prism.hadamard weight has no buffer: %s", weight_name.c_str())); + } + + ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(weight->buffer); + if (target == &hadamard_rotations) { + preferred_buft = buft; + } else if (preferred_buft) { + buft = preferred_buft; + } + auto it = std::find_if(rotations.begin(), rotations.end(), + [block_size, buft](const hadamard_rotation & rotation) { + return rotation.block_size == block_size && rotation.buft == buft; + }); + + if (it == rotations.end()) { + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx { ggml_init(params) }; + if (!ctx) { + throw std::runtime_error("failed to create Hadamard rotation context"); + } + + ggml_tensor * rotation = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_F32, block_size, block_size); + char rotation_name[GGML_MAX_NAME]; + snprintf(rotation_name, sizeof(rotation_name), "prism.hadamard.%u", block_size); + ggml_set_name(rotation, rotation_name); + + ggml_backend_buffer_ptr buffer { ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft) }; + if (!buffer) { + throw std::runtime_error(format("unable to allocate %s Hadamard rotation buffer", ggml_backend_buft_name(buft))); + } + ggml_backend_buffer_set_usage(buffer.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + std::vector data((size_t) block_size * block_size); + const float scale = 1.0f / sqrtf((float) block_size); + for (uint32_t row = 0; row < block_size; ++row) { + for (uint32_t col = 0; col < block_size; ++col) { + uint32_t parity = row & col; + parity ^= parity >> 16; + parity ^= parity >> 8; + parity ^= parity >> 4; + parity ^= parity >> 2; + parity ^= parity >> 1; + data[(size_t) row * block_size + col] = (parity & 1) ? -scale : scale; + } + } + ggml_backend_tensor_set(rotation, data.data(), 0, data.size() * sizeof(float)); + + std::vector buffers; + buffers.emplace_back(std::move(buffer)); + pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers)); + rotations.push_back({ block_size, buft, rotation }); + it = std::prev(rotations.end()); + } + + ggml_tensor * sign_tensor = nullptr; + if (!hadamard_sign_data.empty()) { + const uint32_t width = (uint32_t) weight->ne[0]; + const auto sd = hadamard_sign_data.find(width); + if (sd == hadamard_sign_data.end()) { + throw std::runtime_error(format( + "prism.hadamard has no sign vector for width %u (%s)", width, weight_name.c_str())); + } + const auto key = std::make_pair(width, buft); + auto st = sign_tensors.find(key); + if (st == sign_tensors.end()) { + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx { ggml_init(params) }; + if (!ctx) { + throw std::runtime_error("failed to create Hadamard sign context"); + } + + ggml_tensor * signs = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, width); + char sign_name[GGML_MAX_NAME]; + snprintf(sign_name, sizeof(sign_name), "prism.hadamard.signs.%u", width); + ggml_set_name(signs, sign_name); + + ggml_backend_buffer_ptr buffer { ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft) }; + if (!buffer) { + throw std::runtime_error(format("unable to allocate %s Hadamard sign buffer", ggml_backend_buft_name(buft))); + } + ggml_backend_buffer_set_usage(buffer.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + std::vector data(width); + for (uint32_t i = 0; i < width; ++i) { + data[i] = (float) sd->second[i]; + } + ggml_backend_tensor_set(signs, data.data(), 0, data.size() * sizeof(float)); + + std::vector buffers; + buffers.emplace_back(std::move(buffer)); + pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers)); + st = sign_tensors.emplace(key, signs).first; + } + sign_tensor = st->second; + } + + llama_hadamard_transform transform { it->tensor, sign_tensor }; + if (hadamard_gdn_v_grouped && weight_name.find(".ssm_out.") != std::string::npos) { + const int64_t n_v = hparams.ssm_dt_rank; + const int64_t n_k = hparams.ssm_n_group; + if (n_k <= 0 || n_v <= 0 || n_v % n_k != 0 || weight->ne[0] % n_v != 0) { + throw std::runtime_error(format("prism.hadamard: bad GDN head geometry for %s", weight_name.c_str())); + } + transform.perm_hd = weight->ne[0] / n_v; + transform.perm_nk = n_k; + transform.perm_rep = n_v / n_k; + } + target->emplace(weight, transform); + } + + LLAMA_LOG_INFO("%s: loaded %zu Hadamard-folded weight(s) (%zu inverse-lookup) using %zu rotation(s) and %zu sign vector(s)\n", + __func__, hadamard_rotations.size() + hadamard_inverses.size(), hadamard_inverses.size(), + rotations.size(), sign_tensors.size()); + } + return true; } diff --git a/src/llama-model.h b/src/llama-model.h index a02b30ca7ada..5d8e06abd0ea 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -700,6 +700,18 @@ struct llama_model { // gguf metadata std::unordered_map gguf_kv; + // Hadamard-folded GGUF weights are matched with persistent model tensors + // containing the activation-side transform. The string map is populated + // from GGUF metadata while loading hparams; the pointer map is populated + // after model buffers have been allocated. In explicit sign mode the + // per-width sign vectors come from GGUF metadata as well. + std::unordered_map hadamard_weight_blocks; + std::unordered_map hadamard_inverse_blocks; + std::map> hadamard_sign_data; + bool hadamard_gdn_v_grouped = false; + llama_hadamard_rotations hadamard_rotations; + llama_hadamard_rotations hadamard_inverses; + // list of devices used in this model std::vector devices; diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 34ff25db57e6..b3664bb7f21b 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -391,6 +391,7 @@ static ggml_type tensor_type_fallback(quantize_state_impl & qs, const ggml_tenso case GGML_TYPE_IQ3_S: // types on the right: block size 32 case GGML_TYPE_IQ4_XS: return_type = GGML_TYPE_IQ4_NL; break; case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_TQ1_0: @@ -502,7 +503,7 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { new_type = GGML_TYPE_IQ3_S; } - else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0 || ftype == LLAMA_FTYPE_MOSTLY_Q2_0) { + else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0 || ftype == LLAMA_FTYPE_MOSTLY_Q2_0 || ftype == LLAMA_FTYPE_MOSTLY_PTQ1_0) { new_type = GGML_TYPE_Q4_K; } } @@ -856,6 +857,7 @@ ggml_type llama_ftype_get_default_type(llama_ftype ftype) { case LLAMA_FTYPE_ALL_F32: return GGML_TYPE_F32; case LLAMA_FTYPE_MOSTLY_Q1_0: return GGML_TYPE_Q1_0; case LLAMA_FTYPE_MOSTLY_Q2_0: return GGML_TYPE_Q2_0; + case LLAMA_FTYPE_MOSTLY_PTQ1_0: return GGML_TYPE_PTQ1_0; case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return GGML_TYPE_MXFP4; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 27adea031cfe..1913637c0419 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8927,6 +8927,7 @@ static const ggml_type all_types[] = { GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, GGML_TYPE_Q1_0, + GGML_TYPE_PTQ1_0, GGML_TYPE_Q2_0, GGML_TYPE_MXFP4, GGML_TYPE_NVFP4, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, @@ -8943,6 +8944,7 @@ static const ggml_type base_types[] = { GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q8_0, // for I8MM tests GGML_TYPE_Q1_0, + GGML_TYPE_PTQ1_0, GGML_TYPE_Q2_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, // for I8MM tests @@ -8956,6 +8958,7 @@ static const ggml_type other_types[] = { GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, GGML_TYPE_Q1_0, + GGML_TYPE_PTQ1_0, GGML_TYPE_Q2_0, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q5_K, @@ -9894,6 +9897,18 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8192, 512, 5120, {128, 1}, {1, 1})); #endif + // PTQ has independent scales every 128 elements and a two-byte qh tail. + // Exercise odd block counts, partial output tiles, and batched row offsets + // through both matvec and matmul, including non-contiguous weight views. + for (int64_t k : {128, 384}) { + for (int64_t n : {1, 3, 8, 32}) { + for (int64_t k_v : {int64_t(0), k + 128}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_PTQ1_0, GGML_TYPE_F32, + 17, n, k, {2, 1}, {2, 1}, {0, 1, 2, 3}, k_v)); + } + } + } + for (ggml_type type_a : all_types) { for (int i = 1; i < 10; ++i) { test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, i, 1*256, { 1, 1}, {1, 1})); diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 570fca89a876..725f8001c67b 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -155,6 +155,37 @@ static int test_vec_dot_f32(bool verbose) { return num_failed; } +// Check arithmetic separately from the unavoidable loss of ternary quantization. +static int test_vec_dot_ptq1_0(bool verbose) { + const auto * q = ggml_get_type_traits(GGML_TYPE_PTQ1_0); + const auto * cpu = ggml_get_type_traits_cpu(GGML_TYPE_PTQ1_0); + const auto * v = ggml_get_type_traits(cpu->vec_dot_type); + const auto * vcpu = ggml_get_type_traits_cpu(cpu->vec_dot_type); + int num_failed = 0; + for (int n : {128, 256, 384, 4096, 5120, 17408}) { + std::vector a(n), b(n), da(n), db(n); + std::vector qa(ggml_row_size(GGML_TYPE_PTQ1_0, n)); + std::vector qb(ggml_row_size(cpu->vec_dot_type, n)); + generate_data(0.0f, n, a.data()); + generate_data(1.0f, n, b.data()); + cpu->from_float(a.data(), qa.data(), n); + vcpu->from_float(b.data(), qb.data(), n); + q->to_float(qa.data(), da.data(), n); + v->to_float(qb.data(), db.data(), n); + float result = INFINITY; + cpu->vec_dot(n, &result, 0, qa.data(), 0, qb.data(), 0, 1); + const float ref = dot_product(da.data(), db.data(), n); + const float error = fabsf(result - ref) / n; + const bool failed = !(error < MAX_QUANTIZATION_REFERENCE_ERROR); + num_failed += failed; + if (failed || verbose) { + printf("ptq1_0 dequantized dot n=%5d: %s (ref=%f got=%f err=%f)\n", + n, RESULT_STR[failed], ref, result, error); + } + } + return num_failed; +} + static int test_vec_dot_q(bool verbose) { int num_failed = 0; @@ -189,6 +220,7 @@ static int test_vec_dot_q(bool verbose) { const float total_error = total_quantization_error(qfns, qfns_cpu, test_size, test_data.data()); const float max_quantization_error = type == GGML_TYPE_Q1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_BINARY : + type == GGML_TYPE_PTQ1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY : type == GGML_TYPE_TQ1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY : type == GGML_TYPE_TQ2_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY : type == GGML_TYPE_Q2_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY : @@ -217,7 +249,7 @@ static int test_vec_dot_q(bool verbose) { ? MAX_DOT_PRODUCT_ERROR_LOWBIT : type == GGML_TYPE_Q1_0 ? MAX_DOT_PRODUCT_ERROR_BINARY - : type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0 || type == GGML_TYPE_Q2_0 + : type == GGML_TYPE_PTQ1_0 || type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0 || type == GGML_TYPE_Q2_0 ? MAX_DOT_PRODUCT_ERROR_TERNARY : type == GGML_TYPE_NVFP4 ? MAX_DOT_PRODUCT_ERROR_FP4 @@ -264,6 +296,7 @@ int main(int argc, char * argv[]) { num_failed += test_vec_dot_f32(verbose); num_failed += test_vec_dot_q(verbose); + num_failed += test_vec_dot_ptq1_0(verbose); if (num_failed || verbose) { printf("%d tests failed\n", num_failed); diff --git a/tools/quantize/quantize.cpp b/tools/quantize/quantize.cpp index 38950036cd82..d8161228bdfd 100644 --- a/tools/quantize/quantize.cpp +++ b/tools/quantize/quantize.cpp @@ -34,6 +34,7 @@ struct quant_option { static const std::vector QUANT_OPTIONS = { { "Q1_0", LLAMA_FTYPE_MOSTLY_Q1_0, " 1.125 bpw quantization", }, { "Q2_0", LLAMA_FTYPE_MOSTLY_Q2_0, " 2.25 bpw quantization (group 64)", }, + { "PTQ1_0", LLAMA_FTYPE_MOSTLY_PTQ1_0, " 1.75 bpw ternarization (group 128, Prism)", }, { "Q4_0", LLAMA_FTYPE_MOSTLY_Q4_0, " 4.34G, +0.4685 ppl @ Llama-3-8B", }, { "Q4_1", LLAMA_FTYPE_MOSTLY_Q4_1, " 4.78G, +0.4511 ppl @ Llama-3-8B", }, { "MXFP4_MOE",LLAMA_FTYPE_MOSTLY_MXFP4_MOE," MXFP4 MoE", },