From dd7a782028e530993f65cf729788504754aa5e6b Mon Sep 17 00:00:00 2001 From: Laurent Zuijdwijk Date: Fri, 18 Sep 2026 09:36:13 +0100 Subject: [PATCH 1/4] ptq1_0 : port Prism ternary type (1.75 bpw, group 128) with Vulkan kernels Ports GGML_TYPE_PTQ1_0 (143) from PrismML-Eng/llama.cpp, the packing used by prism-ml/Ternary-Bonsai-2-27B-gguf. The format is upstream TQ1_0's base-3 trit packing at group 128 instead of 256: block_ptq1_0 is 28 B per 128 weights (24 B qs at 5 trits/byte + 2 B qh at 4 trits/byte + fp16 scale) = 1.75 bpw. Covered: ggml type + ftype plumbing, the CPU reference codec and vec_dot, and the full Vulkan path (dequant, get_rows, mul_mat_vec f32/f16, matmul and matmul_id including subgroup and coopmat1 variants) -- 50 SPIR-V shaders. coopmat2 is skipped deliberately: dequant_funcs_cm2.glsl has no PTQ1_0 decoder and emitting cm2 for it fails the whole Vulkan build, so it falls back to the scalar and coopmat1 paths. PTQ1_0 has no arch-specific vec_dot on any target, so arch-fallback.h routes every architecture to the generic one -- including x86, which is not covered by the tq1_0 fallback list. PQ2_0 (142) is not ported: it has no Vulkan kernels in the source fork either. Known incomplete: the prism.hadamard.* weight-rotation runtime is NOT ported yet. Bonsai weights are stored in a rotated basis, so a Bonsai GGUF currently loads and reports its ftype correctly but generates garbage. The activation transform (llama-model.cpp metadata + validation, the build_lora_mm hook in llama-graph.cpp, and the token_embd inverse) is the remaining piece; the FWHT primitive and llama_mul_mat_hadamard already exist here from the KV work. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01K3NeoeVLYx9mNqMSEJHRvS --- ggml/include/ggml.h | 8 +- ggml/src/ggml-common.h | 13 +++ ggml/src/ggml-cpu/arch-fallback.h | 16 +++ ggml/src/ggml-cpu/ggml-cpu.c | 6 ++ ggml/src/ggml-cpu/ops.cpp | 7 ++ ggml/src/ggml-cpu/quants.c | 72 +++++++++++++ ggml/src/ggml-cpu/quants.h | 3 + ggml/src/ggml-quants.c | 101 ++++++++++++++++++ ggml/src/ggml-quants.h | 3 + ggml/src/ggml-vulkan/ggml-vulkan.cpp | 21 ++++ .../vulkan-shaders/dequant_funcs.glsl | 21 ++++ .../vulkan-shaders/dequant_ptq1_0.comp | 54 ++++++++++ .../ggml-vulkan/vulkan-shaders/mul_mm.comp | 3 + .../vulkan-shaders/mul_mm_funcs.glsl | 15 +++ .../ggml-vulkan/vulkan-shaders/ptq1_0.glsl | 38 +++++++ .../src/ggml-vulkan/vulkan-shaders/types.glsl | 21 ++++ .../vulkan-shaders/vulkan-shaders-gen.cpp | 10 +- ggml/src/ggml.c | 8 ++ include/llama.h | 1 + src/llama-model-loader.cpp | 2 + src/llama-quant.cpp | 4 +- tools/quantize/quantize.cpp | 1 + 22 files changed, 425 insertions(+), 3 deletions(-) create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp create mode 100644 ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index b88b7e54a5ea..1b5a47ec385f 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -430,7 +430,12 @@ extern "C" { GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, GGML_TYPE_Q2_0 = 42, - GGML_TYPE_COUNT = 43, + // Prism-private ternary at group size 128, ported from PrismML-Eng/llama.cpp + // (Ternary-Bonsai GGUFs). High id so it slots above upstream types; type_traits + // is sized to COUNT with 43..142 unused. PQ2_0 (142) is deliberately not ported: + // it has no Vulkan kernels upstream of us either. + GGML_TYPE_PTQ1_0 = 143, + GGML_TYPE_COUNT = 144, }; // precision @@ -475,6 +480,7 @@ extern "C" { GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors + GGML_FTYPE_MOSTLY_PTQ1_0 = 129, // except 1d tensors (Prism-private group-128 ternary) }; // available tensor operations: diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h index 1dbbe326d0fc..684bf5dce065 100644 --- a/ggml/src/ggml-common.h +++ b/ggml/src/ggml-common.h @@ -191,6 +191,19 @@ typedef struct { } block_q2_0; static_assert(sizeof(block_q2_0) == sizeof(ggml_half) + QK2_0 / 4, "wrong q2_0 block size/padding"); +// PTQ1_0: Prism-private ternary at group size 128 (ported from PrismML-Eng/llama.cpp). +// Same base-3 trit packing as upstream TQ1_0 (type 34) but one fp16 scale per 128 +// weights instead of per 256. 1.75 bpw, and lossless for checkpoints that are already +// ternary at group 128 -- TQ1_0 cannot represent those, because a 256-wide scale has +// to discard one of the two group scales it straddles. +#define QK_PTQ1_0 128 +typedef struct { + uint8_t qs[(QK_PTQ1_0 - 4*QK_PTQ1_0/64)/5]; // 24 B, 5 trits per byte -> 120 values + uint8_t qh[QK_PTQ1_0/64]; // 2 B, 4 trits per byte -> 8 values + ggml_half d; // scale +} block_ptq1_0; +static_assert(sizeof(block_ptq1_0) == sizeof(ggml_half) + QK_PTQ1_0/64 + (QK_PTQ1_0 - 4*QK_PTQ1_0/64)/5, "wrong ptq1_0 block size/padding"); + #define QK4_0 32 typedef struct { ggml_half d; // delta diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 152e0bac99b0..1fcf709ebb83 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -5,6 +5,8 @@ // This effectively selects the generic implementation. #if defined(GGML_CPU_GENERIC) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define quantize_row_q8_0_generic quantize_row_q8_0 #define quantize_row_q8_1_generic quantize_row_q8_1 @@ -72,6 +74,8 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // repack.cpp #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 @@ -82,6 +86,8 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 // repack.cpp @@ -110,6 +116,8 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__POWERPC__) || defined(__powerpc__) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // ref: https://github.com/ggml-org/llama.cpp/pull/14146#issuecomment-2972561679 // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K @@ -157,6 +165,8 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__loongarch64) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K @@ -204,6 +214,8 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__riscv) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 #define ggml_vec_dot_q2_0_q8_0_generic ggml_vec_dot_q2_0_q8_0 @@ -244,6 +256,8 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__s390x__) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 @@ -297,6 +311,8 @@ #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 #elif defined(__wasm__) +// PTQ1_0 has no arch-specific kernel on any target; always use the generic one. +#define ggml_vec_dot_ptq1_0_q8_0_generic ggml_vec_dot_ptq1_0_q8_0 // quants.c #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K #define ggml_vec_dot_tq2_0_q8_K_generic ggml_vec_dot_tq2_0_q8_K diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 87a329f26975..45cc15a115dc 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -237,6 +237,12 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_0, .nrows = 1, }, + [GGML_TYPE_PTQ1_0] = { + .from_float = quantize_row_ptq1_0, + .vec_dot = ggml_vec_dot_ptq1_0_q8_0, + .vec_dot_type = GGML_TYPE_Q8_0, + .nrows = 1, + }, [GGML_TYPE_Q4_0] = { .from_float = quantize_row_q4_0, .vec_dot = ggml_vec_dot_q4_0_q8_0, diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 266261c5e5a4..e739decd5f56 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -666,6 +666,7 @@ void ggml_compute_forward_add( } break; case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -1117,6 +1118,7 @@ void ggml_compute_forward_add1( } break; case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -1248,6 +1250,7 @@ void ggml_compute_forward_acc( case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4651,6 +4654,7 @@ void ggml_compute_forward_out_prod( switch (src0->type) { case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -4927,6 +4931,7 @@ void ggml_compute_forward_set( case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -5152,6 +5157,7 @@ void ggml_compute_forward_get_rows( switch (src0->type) { case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -5909,6 +5915,7 @@ void ggml_compute_forward_clamp( case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index 5e36459f8cbc..0f9c8d8683d5 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -30,6 +30,10 @@ void quantize_row_q2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in quantize_row_q2_0_ref(x, y, k); } +void quantize_row_ptq1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + quantize_row_ptq1_0_ref(x, y, k); +} + void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { quantize_row_q4_0_ref(x, y, k); } @@ -1337,3 +1341,71 @@ void quantize_row_iq4_xs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, assert(k % QK_K == 0); quantize_iq4_xs(x, y, 1, k, NULL); } + +// PTQ1_0 x Q8_0. The trits are stored base-3 interleaved rather than in element +// order, so decode a block into element order first using the same traversal as +// dequantize_row_ptq1_0 -- that keeps the two provably in step. Four Q8_0 blocks +// cover one 128-wide PTQ1_0 block. +void ggml_vec_dot_ptq1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + const int qk = QK_PTQ1_0; + const int nb = n / qk; + + assert(n % qk == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_ptq1_0 * GGML_RESTRICT x = vx; + const block_q8_0 * GGML_RESTRICT y = vy; + + static const uint8_t pow3[6] = {1, 3, 9, 27, 81, 243}; + static const size_t stages[3] = {32, 16, 8}; + + float sumf = 0.0f; + + for (int i = 0; i < nb; i++) { + int8_t q[QK_PTQ1_0]; + int o = 0; + + size_t j = 0; + for (size_t st = 0; st < 3; ++st) { + const size_t c = stages[st]; + for (; j + c <= sizeof(x->qs); j += c) { + for (size_t nn = 0; nn < 5; ++nn) { + for (size_t m = 0; m < c; ++m) { + const uint8_t v = x[i].qs[j + m] * pow3[nn]; + const int16_t xi = ((uint16_t) v * 3) >> 8; + q[o++] = (int8_t) (xi - 1); + } + } + } + } + for (size_t nn = 0; nn < 4; ++nn) { + for (size_t h = 0; h < sizeof(x->qh); ++h) { + const uint8_t v = x[i].qh[h] * pow3[nn]; + const int16_t xi = ((uint16_t) v * 3) >> 8; + q[o++] = (int8_t) (xi - 1); + } + } + assert(o == QK_PTQ1_0); + + const float d0 = GGML_CPU_FP16_TO_FP32(x[i].d); + float sumi = 0.0f; + + for (int k = 0; k < 4; k++) { + const block_q8_0 * GGML_RESTRICT yb = &y[i * 4 + k]; + const float d1 = GGML_CPU_FP16_TO_FP32(yb->d); + int sumi_block = 0; + for (int b = 0; b < 32; ++b) { + sumi_block += (int) q[k*32 + b] * (int) yb->qs[b]; + } + sumi += d1 * sumi_block; + } + + sumf += d0 * sumi; + } + + *s = sumf; +} diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h index 93ea7eeffe5b..8bfe4d9467d7 100644 --- a/ggml/src/ggml-cpu/quants.h +++ b/ggml/src/ggml-cpu/quants.h @@ -32,6 +32,7 @@ void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, in void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_tq1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +void quantize_row_ptq1_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); @@ -56,6 +57,7 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_tq1_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_ptq1_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_tq2_0_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); @@ -84,6 +86,7 @@ void ggml_vec_dot_mxfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, void ggml_vec_dot_nvfp4_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_tq1_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_ptq1_0_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_tq2_0_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_q2_K_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 1ebc50a763f1..9fde3209e519 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -2125,6 +2125,107 @@ size_t quantize_q2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, return nrow * row_size; } +// ====================== PTQ1_0 (Prism ternary, group 128) ====================== +// Base-3 trit packing identical to upstream TQ1_0, but at block 128 so one fp16 +// scale covers 128 weights. qs is 24 bytes, which TQ1_0's fixed 32-then-16 byte +// staging cannot cover, so the stages are generalised to 32/16/8; at TQ1_0's +// 48-byte qs this reduces to exactly its original 32-then-16 behaviour. +static const size_t ptq1_0_stages[3] = {32, 16, 8}; + +void quantize_row_ptq1_0_ref(const float * GGML_RESTRICT x, block_ptq1_0 * GGML_RESTRICT y, int64_t k) { + assert(k % QK_PTQ1_0 == 0); + const int64_t nb = k / QK_PTQ1_0; + + for (int64_t i = 0; i < nb; i++) { + float amax = 0.0f; + for (int j = 0; j < QK_PTQ1_0; j++) { + amax = MAX(amax, fabsf(x[j])); + } + + const float d = amax; + const float id = d ? 1.0f/d : 0.0f; + + y[i].d = GGML_FP32_TO_FP16(d); + + size_t j = 0; + for (size_t s = 0; s < 3; ++s) { + const size_t c = ptq1_0_stages[s]; + for (; j + c <= sizeof(y->qs); j += c) { + for (size_t m = 0; m < c; ++m) { + uint8_t q = 0; + for (size_t n = 0; n < 5; ++n) { + int xi = lroundf(x[m + n*c] * id) + 1; // -1, 0, 1 -> 0, 1, 2 + q *= 3; + q += xi; + } + // ceiling division (243 == pow(3, 5)) + q = ((uint16_t)q * 256 + (243 - 1)) / 243; + y[i].qs[j + m] = q; + } + x += 5*c; + } + } + // 4 elements per byte + for (size_t h = 0; h < sizeof(y->qh); ++h) { + uint8_t q = 0; + for (size_t m = 0; m < 4; ++m) { + int xi = lroundf(x[h + m*sizeof(y->qh)] * id) + 1; + q *= 3; + q += xi; + } + // shift the first value to the most significant trit + q *= 3; + q = ((uint16_t)q * 256 + (243 - 1)) / 243; + y[i].qh[h] = q; + } + x += 4*sizeof(y->qh); + } +} + +void dequantize_row_ptq1_0(const block_ptq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_PTQ1_0 == 0); + const int64_t nb = k / QK_PTQ1_0; + + const uint8_t pow3[6] = {1, 3, 9, 27, 81, 243}; + + for (int64_t i = 0; i < nb; ++i) { + const float d = GGML_FP16_TO_FP32(x[i].d); + + size_t j = 0; + for (size_t s = 0; s < 3; ++s) { + const size_t c = ptq1_0_stages[s]; + for (; j + c <= sizeof(x->qs); j += c) { + for (size_t n = 0; n < 5; ++n) { + for (size_t m = 0; m < c; ++m) { + uint8_t q = x[i].qs[j + m] * pow3[n]; + int16_t xi = ((uint16_t) q * 3) >> 8; + *y++ = (float) (xi - 1) * d; + } + } + } + } + for (size_t n = 0; n < 4; ++n) { + for (size_t h = 0; h < sizeof(x->qh); ++h) { + uint8_t q = x[i].qh[h] * pow3[n]; + int16_t xi = ((uint16_t) q * 3) >> 8; + *y++ = (float) (xi - 1) * d; + } + } + } +} + +size_t quantize_ptq1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + (void)quant_weights; // ternary codes come from the weights themselves; an imatrix has no role + const size_t row_size = ggml_row_size(GGML_TYPE_PTQ1_0, n_per_row); + char * qrow = (char *)dst; + for (int64_t row = 0; row < nrow; ++row) { + quantize_row_ptq1_0_ref(src, (block_ptq1_0 *)qrow, n_per_row); + src += n_per_row; + qrow += row_size; + } + return nrow * row_size; +} + size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { if (!quant_weights) { quantize_row_q4_0_ref(src, dst, (int64_t)nrow*n_per_row); diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h index 75188f1af180..87ce496d1a63 100644 --- a/ggml/src/ggml-quants.h +++ b/ggml/src/ggml-quants.h @@ -34,6 +34,7 @@ GGML_API void quantize_row_q6_K_ref(const float * GGML_RESTRICT x, block_q6_K * GGML_API void quantize_row_q8_K_ref(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_tq1_0_ref(const float * GGML_RESTRICT x, block_tq1_0 * GGML_RESTRICT y, int64_t k); +GGML_API void quantize_row_ptq1_0_ref(const float * GGML_RESTRICT x, block_ptq1_0 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_tq2_0_ref(const float * GGML_RESTRICT x, block_tq2_0 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_iq3_xxs_ref(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k); @@ -63,6 +64,7 @@ GGML_API void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GG GGML_API void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_tq1_0(const block_tq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_ptq1_0(const block_ptq1_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_tq2_0(const block_tq2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); @@ -87,6 +89,7 @@ GGML_API size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RE GGML_API size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_tq1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_ptq1_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_tq2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 81136b152d16..3cc47f484769 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -4874,6 +4874,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { #endif CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0], matmul_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); + CREATE_MM2(GGML_TYPE_PTQ1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_PTQ1_0], matmul_ptq1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0], matmul_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0], matmul_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1], matmul_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, ); @@ -4920,6 +4921,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { #endif CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); + CREATE_MM2(GGML_TYPE_PTQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_PTQ1_0], matmul_id_subgroup_ptq1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id); @@ -5011,6 +5013,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0], matmul_q1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); + CREATE_MM2(GGML_TYPE_PTQ1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_PTQ1_0], matmul_ptq1_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0], matmul_q2_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0], matmul_q4_0_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1], matmul_q4_1_f32, mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); @@ -5060,6 +5063,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_subgroup_f16_f32, wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_subgroup_q1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); + CREATE_MM2(GGML_TYPE_PTQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_PTQ1_0], matmul_id_subgroup_ptq1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_subgroup_q2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_subgroup_q4_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_subgroup_q4_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); @@ -5108,6 +5112,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { CREATE_MM2(GGML_TYPE_F16, pipeline_matmul_id_f16_f32, matmul_id_f16_f32, wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); CREATE_MM_NODOT2(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); CREATE_MM2(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0], matmul_id_q1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); + CREATE_MM2(GGML_TYPE_PTQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_PTQ1_0], matmul_id_ptq1_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); CREATE_MM2(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0], matmul_id_q2_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); CREATE_MM2(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0], matmul_id_q4_0_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); CREATE_MM2(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1], matmul_id_q4_1_f32, mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); @@ -5187,6 +5192,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_bf16, matmul_bf16, , wg_denoms, warptile, vk_mat_mat_push_constants, 3, , 0); CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q1_0].f32acc, matmul_q1_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); + CREATE_MM(GGML_TYPE_PTQ1_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_PTQ1_0].f32acc, matmul_ptq1_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q2_0].f32acc, matmul_q2_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_0].f32acc, matmul_q4_0_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat[GGML_TYPE_Q4_1].f32acc, matmul_q4_1_f32, , mmq_wg_denoms, warptile_mmq, vk_mat_mat_push_constants, 3, , 0); @@ -5236,6 +5242,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_subgroup_bf16, , wg_denoms, warptile_id, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size_16); CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0].f32acc, matmul_id_subgroup_q1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); + CREATE_MM(GGML_TYPE_PTQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_PTQ1_0].f32acc, matmul_id_subgroup_ptq1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0].f32acc, matmul_id_subgroup_q2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0].f32acc, matmul_id_subgroup_q4_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1].f32acc, matmul_id_subgroup_q4_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, mul_mat_subgroup_size); @@ -5266,6 +5273,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { CREATE_MM(GGML_TYPE_BF16, pipeline_matmul_id_bf16, matmul_id_bf16, , wg_denoms, warptile, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); CREATE_MM(GGML_TYPE_Q1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q1_0].f32acc, matmul_id_q1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); + CREATE_MM(GGML_TYPE_PTQ1_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_PTQ1_0].f32acc, matmul_id_ptq1_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); CREATE_MM(GGML_TYPE_Q2_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q2_0].f32acc, matmul_id_q2_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); CREATE_MM(GGML_TYPE_Q4_0, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_0].f32acc, matmul_id_q4_0_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); CREATE_MM(GGML_TYPE_Q4_1, pipeline_dequant_mul_mat_mat_id[GGML_TYPE_Q4_1].f32acc, matmul_id_q4_1_f32, , mmq_wg_denoms, warptile_mmqid, vk_mat_mat_id_push_constants, mul_mat_id_param_count, _id, 0); @@ -5375,6 +5383,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f32_f32", arr_dmmv_f16_f32_f32_len[reduc], arr_dmmv_f16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f32_f32", arr_dmmv_bf16_f32_f32_len[reduc], arr_dmmv_bf16_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f32_f32", arr_dmmv_q1_0_f32_f32_len[reduc], arr_dmmv_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_PTQ1_0][i], "mul_mat_vec_ptq1_0_f32_f32", arr_dmmv_ptq1_0_f32_f32_len[reduc], arr_dmmv_ptq1_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f32_f32", arr_dmmv_q2_0_f32_f32_len[reduc], arr_dmmv_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f32_f32", arr_dmmv_q4_0_f32_f32_len[reduc], arr_dmmv_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f32_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f32_f32", arr_dmmv_q4_1_f32_f32_len[reduc], arr_dmmv_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); @@ -5403,6 +5412,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_F16 ][i], "mul_mat_vec_f16_f16_f32", arr_dmmv_f16_f16_f32_len[reduc], arr_dmmv_f16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_BF16][i], "mul_mat_vec_bf16_f16_f32", arr_dmmv_bf16_f16_f32_len[reduc], arr_dmmv_bf16_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2, 1, 1}, {wg_size_subgroup, 2, i+1}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q1_0][i], "mul_mat_vec_q1_0_f16_f32", arr_dmmv_q1_0_f16_f32_len[reduc], arr_dmmv_q1_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_PTQ1_0][i], "mul_mat_vec_ptq1_0_f16_f32", arr_dmmv_ptq1_0_f16_f32_len[reduc], arr_dmmv_ptq1_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q2_0][i], "mul_mat_vec_q2_0_f16_f32", arr_dmmv_q2_0_f16_f32_len[reduc], arr_dmmv_q2_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_0][i], "mul_mat_vec_q4_0_f16_f32", arr_dmmv_q4_0_f16_f32_len[reduc], arr_dmmv_q4_0_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_f16_f32[w][GGML_TYPE_Q4_1][i], "mul_mat_vec_q4_1_f16_f32", arr_dmmv_q4_1_f16_f32_len[reduc], arr_dmmv_q4_1_f16_f32_data[reduc], "main", mul_mat_vec_num_bindings, sizeof(vk_mat_vec_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq, i+1}, 1, true, use_subgroups, force_subgroup_size); @@ -5458,6 +5468,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_F16 ], "mul_mat_vec_id_f16_f32", arr_dmmv_id_f16_f32_f32_len[reduc], arr_dmmv_id_f16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2, 1, 1}, {wg_size_subgroup, 2}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_BF16], "mul_mat_vec_id_bf16_f32", arr_dmmv_id_bf16_f32_f32_len[reduc], arr_dmmv_id_bf16_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2, 1, 1}, {wg_size_subgroup, 2}, 1, false, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q1_0], "mul_mat_vec_id_q1_0_f32", arr_dmmv_id_q1_0_f32_f32_len[reduc], arr_dmmv_id_q1_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); + ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_PTQ1_0], "mul_mat_vec_id_ptq1_0_f32", arr_dmmv_id_ptq1_0_f32_f32_len[reduc], arr_dmmv_id_ptq1_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q2_0], "mul_mat_vec_id_q2_0_f32", arr_dmmv_id_q2_0_f32_f32_len[reduc], arr_dmmv_id_q2_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_0], "mul_mat_vec_id_q4_0_f32", arr_dmmv_id_q4_0_f32_f32_len[reduc], arr_dmmv_id_q4_0_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); ggml_vk_create_pipeline(device, device->pipeline_dequant_mul_mat_vec_id_f32[w][GGML_TYPE_Q4_1], "mul_mat_vec_id_q4_1_f32", arr_dmmv_id_q4_1_f32_f32_len[reduc], arr_dmmv_id_q4_1_f32_f32_data[reduc], "main", mul_mat_vec_id_num_bindings, sizeof(vk_mat_vec_id_push_constants), {2*rm_stdq, 1, 1}, {wg_size_subgroup, 2*rm_stdq}, 1, true, use_subgroups, force_subgroup_size); @@ -5523,6 +5534,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { // dequant shaders ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_F32 ], "f32_to_f16", dequant_f32_len, dequant_f32_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q1_0], "dequant_q1_0", dequant_q1_0_len, dequant_q1_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 8, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_PTQ1_0], "dequant_ptq1_0", dequant_ptq1_0_len, dequant_ptq1_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 8, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q2_0], "dequant_q2_0", dequant_q2_0_len, dequant_q2_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_0], "dequant_q4_0", dequant_q4_0_len, dequant_q4_0_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_Q4_1], "dequant_q4_1", dequant_q4_1_len, dequant_q4_1_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); @@ -5553,6 +5565,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_F16 ], "get_rows_f16", get_rows_f16_len, get_rows_f16_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_BF16], "get_rows_bf16", get_rows_bf16_len, get_rows_bf16_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q1_0], "get_rows_q1_0", get_rows_q1_0_len, get_rows_q1_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_PTQ1_0], "get_rows_ptq1_0", get_rows_ptq1_0_len, get_rows_ptq1_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q2_0], "get_rows_q2_0", get_rows_q2_0_len, get_rows_q2_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_0], "get_rows_q4_0", get_rows_q4_0_len, get_rows_q4_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_Q4_1], "get_rows_q4_1", get_rows_q4_1_len, get_rows_q4_1_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); @@ -5582,6 +5595,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_F16 ], "get_rows_f16_f32", get_rows_f16_f32_len, get_rows_f16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_BF16], "get_rows_bf16_f32", get_rows_bf16_f32_len, get_rows_bf16_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q1_0], "get_rows_q1_0_f32", get_rows_q1_0_f32_len, get_rows_q1_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_PTQ1_0], "get_rows_ptq1_0_f32", get_rows_ptq1_0_f32_len, get_rows_ptq1_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q2_0], "get_rows_q2_0_f32", get_rows_q2_0_f32_len, get_rows_q2_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_0], "get_rows_q4_0_f32", get_rows_q4_0_f32_len, get_rows_q4_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_Q4_1], "get_rows_q4_1_f32", get_rows_q4_1_f32_len, get_rows_q4_1_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); @@ -7806,6 +7820,7 @@ static vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type switch (type) { case GGML_TYPE_F32: case GGML_TYPE_Q1_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -7881,6 +7896,7 @@ static vk_matmul_pipeline ggml_vk_get_mul_mat_mat_pipeline(ggml_backend_vk_conte switch (src0_type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -7951,6 +7967,7 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec(ggml_backend_vk_context * case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -8045,6 +8062,7 @@ static vk_matmul_pipeline ggml_vk_get_mul_mat_mat_id_pipeline(ggml_backend_vk_co switch (src0_type) { case GGML_TYPE_Q1_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -8118,6 +8136,7 @@ static vk_pipeline ggml_vk_get_dequantize_mul_mat_vec_id(ggml_backend_vk_context case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -18701,6 +18720,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -18807,6 +18827,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_Q1_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl index 627932bd3547..88be1360403a 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl @@ -126,6 +126,20 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { } #endif +#if defined(DATA_A_PTQ1_0) +#include "ptq1_0.glsl" + +vec2 dequantize(uint ib, uint iqs, uint a_offset) { + return vec2(ptq1_0_trit(ib, a_offset, iqs), ptq1_0_trit(ib, a_offset, iqs + 1u)); +} +vec4 dequantize4(uint ib, uint iqs, uint a_offset) { + return vec4(ptq1_0_trit(ib, a_offset, iqs), + ptq1_0_trit(ib, a_offset, iqs + 1u), + ptq1_0_trit(ib, a_offset, iqs + 2u), + ptq1_0_trit(ib, a_offset, iqs + 3u)); +} +#endif + #if defined(DATA_A_Q1_0) vec2 dequantize(uint ib, uint iqs, uint a_offset) { const uint bits = uint(data_a[a_offset + ib].qs[iqs / 8u]) >> (iqs % 8u); @@ -564,6 +578,13 @@ vec2 get_dm(uint ib, uint a_offset) { } #endif +#if defined(DATA_A_PTQ1_0) +vec2 get_dm(uint ib, uint a_offset) { + const float d = float(data_a[a_offset + ib].d); + return vec2(d, 0); +} +#endif + #if defined(DATA_A_Q1_0) vec2 get_dm(uint ib, uint a_offset) { const float d = float(data_a[a_offset + ib].d); diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp new file mode 100644 index 000000000000..53baf41f65b9 --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp @@ -0,0 +1,54 @@ +#version 450 + +#include "dequant_head.glsl" + +layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in; + +layout (binding = 0) readonly buffer A {block_ptq1_0 data_a[];}; +layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; + +// One invocation per 8 elements: 16 invocations cover a 128-weight block. Element +// order follows the CPU codec (16-byte qs chunk, 8-byte qs chunk, then qh), so the +// index maths lives in one place rather than being reproduced per shader. +float ptq1_0_trit_l(uint ib, uint e) { + uint b; + uint n; + if (e < 80u) { + b = uint(data_a[ib].qs[e & 15u]); + n = e >> 4u; + } else if (e < 120u) { + const uint t = e - 80u; + b = uint(data_a[ib].qs[16u + (t & 7u)]); + n = t >> 3u; + } else { + const uint t = e - 120u; + b = uint(data_a[ib].qh[t & 1u]); + n = t >> 1u; + } + + uint v = b; + for (uint i = 0u; i < n; ++i) { + v = (v * 3u) & 0xFFu; + } + return float(int((v * 3u) >> 8u) - 1); +} + +void main() { + const uint i = gl_WorkGroupID.x * 4 + gl_LocalInvocationID.x / 64; + + const uint tid = gl_LocalInvocationID.x % 64; + const uint il = tid / 4; + const uint ir = tid % 4; + const uint ib = 4*i + ir; + if (ib >= p.nel / 128) { + return; + } + + const uint b_idx = 512*i + 128*ir + 8*il; + + const float d = float(data_a[ib].d); + + [[unroll]] for (uint l = 0; l < 8; ++l) { + data_b[b_idx + l] = D_TYPE(ptq1_0_trit_l(ib, 8*il + l) * d); + } +} diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp index 63c4aaebcb1a..8a013b6c1fea 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp @@ -138,6 +138,9 @@ shared FLOAT_TYPEV2 buf_b[BN * SHMEM_STRIDE]; shared ACC_TYPE coopmat_stage[TM * TN * NUM_WARPS]; #endif +#if defined(DATA_A_PTQ1_0) +#include "ptq1_0.glsl" +#endif #include "mul_mm_id_funcs.glsl" #include "mul_mm_funcs.glsl" diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl index 7d852dced8ab..3125a3fcfa93 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl @@ -152,6 +152,21 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const uint k_pair = row * LOAD_VEC_A / 2; store_a(col, k_pair, FLOAT_TYPEV2(v.xy)); store_a(col, k_pair + 1, FLOAT_TYPEV2(v.zw)); +#elif defined(DATA_A_PTQ1_0) + const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; + + const uint ib = idx / 16; + const uint grp = idx & 0xfu; // which 8-element group inside the block + const uint e0 = grp * 8u; + + const float d = float(data_a[ib].d); + + const uint k_pair = row * LOAD_VEC_A / 2; + [[unroll]] for (uint l = 0; l < 4; ++l) { + store_a(col, k_pair + l, FLOAT_TYPEV2( + ptq1_0_trit(ib, 0u, e0 + 2u*l) * d, + ptq1_0_trit(ib, 0u, e0 + 2u*l + 1u) * d)); + } #elif defined(DATA_A_Q1_0) const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl new file mode 100644 index 000000000000..7eedd832b14f --- /dev/null +++ b/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl @@ -0,0 +1,38 @@ +#ifndef PTQ1_0_GLSL +#define PTQ1_0_GLSL + +// Shared PTQ1_0 trit accessor. Lives in its own header because two consumers need it +// from different include chains: dequant_funcs.glsl (mul_mat_vec, get_rows, +// copy_from_quant) and mul_mm.comp (via mul_mm_funcs.glsl), and mul_mm.comp does not +// include dequant_funcs.glsl. Duplicating it would leave two copies that must stay in +// step with the CPU codec in ggml-quants.c, where a divergence shows up as wrong +// matmul results rather than a build error. +// +// Element order is not positional: a 16-byte chunk of qs where byte j carries +// elements t*16+j, then an 8-byte chunk carrying 80 + t*8 + (j-16), then qh at four +// trits per byte carrying 120 + t*2 + h. Trits come out by the base-3 remainder +// recurrence t = (v*3)>>8, v = (v*3)&0xFF. +float ptq1_0_trit(uint ib, uint a_offset, uint e) { + uint b; + uint n; + if (e < 80u) { + b = uint(data_a[a_offset + ib].qs[e & 15u]); + n = e >> 4u; + } else if (e < 120u) { + const uint t = e - 80u; + b = uint(data_a[a_offset + ib].qs[16u + (t & 7u)]); + n = t >> 3u; + } else { + const uint t = e - 120u; + b = uint(data_a[a_offset + ib].qh[t & 1u]); + n = t >> 1u; + } + + uint v = b; + for (uint i = 0u; i < n; ++i) { + v = (v * 3u) & 0xFFu; + } + return float(int((v * 3u) >> 8u) - 1); +} + +#endif // PTQ1_0_GLSL diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index adb1bb8b32b5..469aad51ce89 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -211,6 +211,27 @@ struct block_q1_0 #define A_TYPE block_q1_0 #endif +// PTQ1_0: ternary at group 128, base-3 packed five trits per byte. +// Field order mirrors block_ptq1_0 in ggml-common.h EXACTLY -- qs, then qh, then d. +// Unlike q1_0 the scale is LAST, and getting that wrong silently misindexes every +// block rather than failing loudly. +#define QUANT_K_PTQ1_0 128 +#define QUANT_R_PTQ1_0 1 + +struct block_ptq1_0 +{ + uint8_t qs[24]; + uint8_t qh[2]; + float16_t d; +}; + +#if defined(DATA_A_PTQ1_0) +#define QUANT_K QUANT_K_PTQ1_0 +#define QUANT_R QUANT_R_PTQ1_0 +#define QUANT_AUXF 1 +#define A_TYPE block_ptq1_0 +#endif + #define QUANT_K_Q2_0 64 #define QUANT_R_Q2_0 1 diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 27ff68c10d5b..06510fcb61ff 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -50,6 +50,7 @@ const std::vector type_names = { "f32", "f16", "q1_0", + "ptq1_0", "q2_0", "q4_0", "q4_1", @@ -583,7 +584,7 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c for (const auto& tname : type_names) { std::string load_vec_quant = "2"; - if ((tname == "q1_0") || (tname == "q4_0") || (tname == "q4_1") || (tname == "q5_1") || (tname == "iq1_s") || (tname == "iq1_m") || (tname == "iq2_xxs") || (tname == "iq2_xs") || (tname == "iq2_s")) + if ((tname == "q1_0") || (tname == "ptq1_0") || (tname == "q4_0") || (tname == "q4_1") || (tname == "q5_1") || (tname == "iq1_s") || (tname == "iq1_m") || (tname == "iq2_xxs") || (tname == "iq2_xs") || (tname == "iq2_s")) load_vec_quant = "8"; else if ((tname == "q2_0") || (tname == "q5_0") || (tname == "q8_0") || (tname == "q2_k") || (tname == "q4_k") || (tname == "q5_k") || (tname == "iq3_xxs") || (tname == "iq3_s") || (tname == "iq4_xs") || (tname == "iq4_nl") || (tname == "mxfp4") || (tname == "nvfp4")) load_vec_quant = "4"; @@ -591,6 +592,13 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c if (tname == "bf16") { continue; } + // PTQ1_0 has no coopmat2 decoder: dequant_funcs_cm2.glsl carries no PTQ1_0 entry, + // so emitting mul_mm_cm2 for it fails shader compilation and takes the whole + // Vulkan build down, not just this type. Skip it; it falls back to the scalar and + // coopmat1 matmul paths, which are the ones implemented and tested. + if (coopmat2 && tname == "ptq1_0") { + continue; + } std::string data_a_key = "DATA_A_" + to_uppercase(tname); // For aligned matmul loads diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 6257cdbe5821..b7c48d7304fc 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -690,6 +690,14 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .to_float = (ggml_to_float_t) dequantize_row_q2_0, .from_float_ref = (ggml_from_float_t) quantize_row_q2_0_ref, }, + [GGML_TYPE_PTQ1_0] = { + .type_name = "ptq1_0", + .blck_size = QK_PTQ1_0, + .type_size = sizeof(block_ptq1_0), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_ptq1_0, + .from_float_ref = (ggml_from_float_t) quantize_row_ptq1_0_ref, + }, [GGML_TYPE_Q4_0] = { .type_name = "q4_0", .blck_size = QK4_0, diff --git a/include/llama.h b/include/llama.h index d0dda88ffd2e..aa4e2438d231 100644 --- a/include/llama.h +++ b/include/llama.h @@ -156,6 +156,7 @@ extern "C" { LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors + LLAMA_FTYPE_MOSTLY_PTQ1_0 = 143, // except 1d tensors (Prism group-128 ternary, 1.75 bpw) LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 89ae60d2d571..6215920fab67 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -39,6 +39,7 @@ const char * llama_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_BF16: name = LLAMA_FTYPE_PREFIX "BF16"; break; case LLAMA_FTYPE_MOSTLY_Q1_0: name = LLAMA_FTYPE_PREFIX "Q1_0"; break; case LLAMA_FTYPE_MOSTLY_Q2_0: name = LLAMA_FTYPE_PREFIX "Q2_0"; break; + case LLAMA_FTYPE_MOSTLY_PTQ1_0: name = LLAMA_FTYPE_PREFIX "PTQ1_0 - 1.75 bpw ternary (group 128)"; break; case LLAMA_FTYPE_MOSTLY_Q4_0: name = LLAMA_FTYPE_PREFIX "Q4_0"; break; case LLAMA_FTYPE_MOSTLY_Q4_1: name = LLAMA_FTYPE_PREFIX "Q4_1"; break; case LLAMA_FTYPE_MOSTLY_Q5_0: name = LLAMA_FTYPE_PREFIX "Q5_0"; break; @@ -775,6 +776,7 @@ llama_model_loader::llama_model_loader( case GGML_TYPE_NVFP4: ftype = LLAMA_FTYPE_MOSTLY_NVFP4; break; case GGML_TYPE_Q1_0: ftype = LLAMA_FTYPE_MOSTLY_Q1_0; break; case GGML_TYPE_Q2_0: ftype = LLAMA_FTYPE_MOSTLY_Q2_0; break; + case GGML_TYPE_PTQ1_0: ftype = LLAMA_FTYPE_MOSTLY_PTQ1_0; break; default: { LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max)); diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 34ff25db57e6..b3664bb7f21b 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -391,6 +391,7 @@ static ggml_type tensor_type_fallback(quantize_state_impl & qs, const ggml_tenso case GGML_TYPE_IQ3_S: // types on the right: block size 32 case GGML_TYPE_IQ4_XS: return_type = GGML_TYPE_IQ4_NL; break; case GGML_TYPE_Q2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_TQ1_0: @@ -502,7 +503,7 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) { new_type = GGML_TYPE_IQ3_S; } - else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0 || ftype == LLAMA_FTYPE_MOSTLY_Q2_0) { + else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0 || ftype == LLAMA_FTYPE_MOSTLY_Q2_0 || ftype == LLAMA_FTYPE_MOSTLY_PTQ1_0) { new_type = GGML_TYPE_Q4_K; } } @@ -856,6 +857,7 @@ ggml_type llama_ftype_get_default_type(llama_ftype ftype) { case LLAMA_FTYPE_ALL_F32: return GGML_TYPE_F32; case LLAMA_FTYPE_MOSTLY_Q1_0: return GGML_TYPE_Q1_0; case LLAMA_FTYPE_MOSTLY_Q2_0: return GGML_TYPE_Q2_0; + case LLAMA_FTYPE_MOSTLY_PTQ1_0: return GGML_TYPE_PTQ1_0; case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return GGML_TYPE_MXFP4; diff --git a/tools/quantize/quantize.cpp b/tools/quantize/quantize.cpp index 38950036cd82..d8161228bdfd 100644 --- a/tools/quantize/quantize.cpp +++ b/tools/quantize/quantize.cpp @@ -34,6 +34,7 @@ struct quant_option { static const std::vector QUANT_OPTIONS = { { "Q1_0", LLAMA_FTYPE_MOSTLY_Q1_0, " 1.125 bpw quantization", }, { "Q2_0", LLAMA_FTYPE_MOSTLY_Q2_0, " 2.25 bpw quantization (group 64)", }, + { "PTQ1_0", LLAMA_FTYPE_MOSTLY_PTQ1_0, " 1.75 bpw ternarization (group 128, Prism)", }, { "Q4_0", LLAMA_FTYPE_MOSTLY_Q4_0, " 4.34G, +0.4685 ppl @ Llama-3-8B", }, { "Q4_1", LLAMA_FTYPE_MOSTLY_Q4_1, " 4.78G, +0.4511 ppl @ Llama-3-8B", }, { "MXFP4_MOE",LLAMA_FTYPE_MOSTLY_MXFP4_MOE," MXFP4 MoE", }, From 52769a48e60e8e03cc57853728ff085856de1870 Mon Sep 17 00:00:00 2001 From: Laurent Zuijdwijk Date: Fri, 18 Sep 2026 11:15:36 +0100 Subject: [PATCH 2/4] ptq1_0 : port the prism.hadamard weight-rotation runtime Completes the Bonsai path. Bonsai weights are stored in a blockwise-Hadamard rotated basis (W x = (WQ)(Q^T x), Q folded into the weights offline), so the runtime must apply the matching transform to activations or the model loads cleanly and emits fluent garbage -- which is exactly what the previous commit did on its own. Ported from PrismML-Eng/llama.cpp: - llama-model.cpp: prism.hadamard.* metadata parsing and validation, plus construction of the per-(block_size, buffer type) rotation matrices and the per-width explicit sign vectors, after buffers are allocated. - llama-graph.{h,cpp}: llama_hadamard_transform, the transform hook in build_lora_mm and build_lora_mm_id (memoized per (activation, rotation) so weights sharing an input build it once), and the inverse applied to the token-embedding lookup, whose table stores rotated rows. - llama-context.{h,cpp}: llama_verify_hadamard_graph, run once on the first pristine graph. It throws if any folded weight reaches a matmul without its transform, so an unsupported graph fails loudly instead of computing wrong results. - llama-model-loader.cpp: the three string-key get_arr/get_key instantiations the metadata reads need. Deviations from the source fork: LLM_ARCH_DSPARK is Prism-private and its three special cases are dropped; the verified-architecture list keeps QWEN35 and the rest, so a Bonsai GGUF (arch qwen35) is accepted. No new kernels were needed -- llama_mul_mat_hadamard and fwht.comp already live here from the KV-rotation work, and the Vulkan backend picks the FWHT kernel off the GGML_HINT_SRC0_IS_HADAMARD hint. Verified on Ternary-Bonsai-2-27B-PTQ1_0 (Radeon 8060S, RADV STRIX_HALO): loads, reports 401 folded weights, and answers correctly on both CPU and Vulkan. llama-bench matches the source fork within noise -- pp512 210.13 vs 206.38, tg128 3.96 vs 4.00 -- so the port is faithful. Known performance gap, not introduced here: tg128 is ~22 GiB/s effective against ~193 GiB/s for IQ3_XXS on the same box and build. PTQ1_0 decode is bound by the serial dependent-multiply trit unpack in ptq1_0_trit (up to 4 iterations per element, trip count varying by element position, so it diverges within a subgroup), not by bandwidth. A LUT or position-unrolled decoder is the fix. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01K3NeoeVLYx9mNqMSEJHRvS --- src/llama-context.cpp | 81 ++++++++++ src/llama-context.h | 3 + src/llama-graph.cpp | 74 ++++++++- src/llama-graph.h | 23 +++ src/llama-model-loader.cpp | 4 + src/llama-model.cpp | 297 +++++++++++++++++++++++++++++++++++++ src/llama-model.h | 12 ++ 7 files changed, 492 insertions(+), 2 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 70628813cbc1..c7ea0d8f8a9f 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -24,6 +24,78 @@ // llama_context // +// Verify that every Hadamard-folded weight consumed by the graph receives its +// activation-side transform, and every latent lookup table gets the inverse. +// An architecture whose matmul path bypasses the transform helpers would +// otherwise load cleanly and silently compute wrong results. +static void llama_verify_hadamard_graph( + ggml_cgraph * gf, + const llama_hadamard_rotations & rotations, + const llama_hadamard_rotations & inverses) { + auto unwrap = [](const ggml_tensor * t) { + while (t && (t->op == GGML_OP_RESHAPE || t->op == GGML_OP_VIEW)) { + t = t->src[0]; + } + return t; + }; + + std::map lookups; // get_rows results of latent tables + + for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) { + const ggml_tensor * node = ggml_graph_node(gf, i); + + if (node->op == GGML_OP_GET_ROWS && inverses.count(node->src[0])) { + lookups.emplace(node, false); + continue; + } + + if (node->op != GGML_OP_MUL_MAT && node->op != GGML_OP_MUL_MAT_ID) { + continue; + } + + if (node->op == GGML_OP_MUL_MAT && ((const int32_t *) node->op_params)[1] == GGML_HINT_SRC0_IS_HADAMARD) { + const auto lk = lookups.find(unwrap(node->src[1])); + if (lk != lookups.end()) { + lk->second = true; + } + continue; + } + + const auto it = rotations.find(node->src[0]); + if (it == rotations.end()) { + continue; + } + const ggml_tensor * src = unwrap(node->src[1]); + const bool transformed = src && src->op == GGML_OP_MUL_MAT && + ((const int32_t *) src->op_params)[1] == GGML_HINT_SRC0_IS_HADAMARD && + src->src[0] == it->second.rot; + if (!transformed) { + throw std::runtime_error(format( + "Hadamard-folded weight '%s' is consumed without its activation transform; " + "this graph's matmul path does not support prism.hadamard folding", + node->src[0]->name)); + } + } + + for (const auto & [node, ok] : lookups) { + if (!ok) { + for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) { + const ggml_tensor * n2 = ggml_graph_node(gf, i); + for (int s = 0; s < GGML_MAX_SRC && n2->src[s]; ++s) { + if (unwrap(n2->src[s]) == node) { + LLAMA_LOG_WARN("%s: latent lookup '%s' consumed by op=%s name='%s' src%d hint=%d\n", + __func__, node->name, ggml_op_name(n2->op), n2->name, s, + ((const int32_t *) n2->op_params)[1]); + } + } + } + throw std::runtime_error(format( + "Hadamard-latent table '%s' is read without the inverse transform", + node->src[0]->name)); + } + } +} + static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) { switch (ctx_type) { case LLAMA_CONTEXT_TYPE_DEFAULT: return LLM_GRAPH_TYPE_DEFAULT; @@ -2465,6 +2537,13 @@ ggml_cgraph * llama_context::graph_reserve( auto * gf = model.build_graph(gparams); + // verify transform coverage on the pristine graph: after scheduling, + // cross-backend copies break the producer chain the check follows + if (!hadamard_verified && gf && (!model.hadamard_rotations.empty() || !model.hadamard_inverses.empty())) { + llama_verify_hadamard_graph(gf, model.hadamard_rotations, model.hadamard_inverses); + hadamard_verified = true; + } + this->n_outputs = save_n_outputs; // initialize scheduler with the specified graph @@ -2500,6 +2579,8 @@ llm_graph_params llama_context::graph_params( /*.loras =*/ loras.get(), /*.mctx =*/ mctx, /*.cross =*/ &cross, + /*.hadamard_rotations =*/&model.hadamard_rotations, + /*.hadamard_inverses =*/&model.hadamard_inverses, /*.samplers =*/ sampling.samplers, /*.n_outputs =*/ n_outputs, /*.cb =*/ graph_get_cb(), diff --git a/src/llama-context.h b/src/llama-context.h index b089e1267aaf..848fdc3f6857 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -376,6 +376,9 @@ struct llama_context { bool has_evaluated_once = false; + // prism.hadamard transform coverage is checked once, on the first built graph + bool hadamard_verified = false; + // env: LLAMA_GRAPH_REUSE_DISABLE bool graph_reuse_disable = false; diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index b137c6cfe8a0..cb30905aaa0e 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1490,6 +1490,8 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) : loras (params.loras), mctx (params.mctx), cross (params.cross), + hadamard_rotations(params.hadamard_rotations), + hadamard_inverses(params.hadamard_inverses), samplers (params.samplers), cb_func (params.cb), res (params.res), @@ -1516,7 +1518,35 @@ ggml_tensor * llm_graph_context::build_lora_mm( ggml_tensor * w, ggml_tensor * cur, ggml_tensor * w_s) const { - ggml_tensor * res = ggml_mul_mat(ctx0, w, cur); + ggml_tensor * cur_mm = cur; + if (hadamard_rotations) { + const auto it = hadamard_rotations->find(w); + if (it != hadamard_rotations->end()) { + const auto & t = it->second; + // another folded weight on this same activation already built the transform + const auto memo_key = std::make_pair((const ggml_tensor *) cur, (const ggml_tensor *) t.rot); + const auto memo_it = hadamard_memo.find(memo_key); + if (memo_it != hadamard_memo.end()) { + cur_mm = memo_it->second; + } else { + if (t.perm_rep > 1) { + // tiled [hd, nk, rep] -> grouped [hd, rep, nk] feature order + ggml_tensor * x = ggml_is_contiguous(cur_mm) ? cur_mm : ggml_cont(ctx0, cur_mm); + const int64_t ne1 = x->ne[1], ne2 = x->ne[2], ne3 = x->ne[3]; + x = ggml_reshape_4d(ctx0, x, t.perm_hd, t.perm_nk, t.perm_rep, ne1*ne2*ne3); + x = ggml_cont(ctx0, ggml_permute(ctx0, x, 0, 2, 1, 3)); + cur_mm = ggml_reshape_4d(ctx0, x, t.perm_hd*t.perm_nk*t.perm_rep, ne1, ne2, ne3); + } + if (t.signs) { + cur_mm = ggml_mul(ctx0, cur_mm, t.signs); + } + cur_mm = llama_mul_mat_hadamard(ctx0, cur_mm, t.rot); + hadamard_memo[memo_key] = cur_mm; + } + } + } + + ggml_tensor * res = ggml_mul_mat(ctx0, w, cur_mm); if (w_s) { res = ggml_mul(ctx0, res, w_s); @@ -1548,7 +1578,35 @@ ggml_tensor * llm_graph_context::build_lora_mm_id( ggml_tensor * cur, // ggml_tensor * b ggml_tensor * ids, ggml_tensor * w_s) const { - ggml_tensor * res = ggml_mul_mat_id(ctx0, w, cur, ids); + ggml_tensor * cur_mm = cur; + if (hadamard_rotations) { + const auto it = hadamard_rotations->find(w); + if (it != hadamard_rotations->end()) { + const auto & t = it->second; + // another folded weight on this same activation already built the transform + const auto memo_key = std::make_pair((const ggml_tensor *) cur, (const ggml_tensor *) t.rot); + const auto memo_it = hadamard_memo.find(memo_key); + if (memo_it != hadamard_memo.end()) { + cur_mm = memo_it->second; + } else { + if (t.perm_rep > 1) { + // tiled [hd, nk, rep] -> grouped [hd, rep, nk] feature order + ggml_tensor * x = ggml_is_contiguous(cur_mm) ? cur_mm : ggml_cont(ctx0, cur_mm); + const int64_t ne1 = x->ne[1], ne2 = x->ne[2], ne3 = x->ne[3]; + x = ggml_reshape_4d(ctx0, x, t.perm_hd, t.perm_nk, t.perm_rep, ne1*ne2*ne3); + x = ggml_cont(ctx0, ggml_permute(ctx0, x, 0, 2, 1, 3)); + cur_mm = ggml_reshape_4d(ctx0, x, t.perm_hd*t.perm_nk*t.perm_rep, ne1, ne2, ne3); + } + if (t.signs) { + cur_mm = ggml_mul(ctx0, cur_mm, t.signs); + } + cur_mm = llama_mul_mat_hadamard(ctx0, cur_mm, t.rot); + hadamard_memo[memo_key] = cur_mm; + } + } + } + + ggml_tensor * res = ggml_mul_mat_id(ctx0, w, cur_mm, ids); if (w_s) { const int64_t n_expert = w_s->ne[0]; @@ -2331,6 +2389,18 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd) const { cur = ggml_get_rows(ctx0, tok_embd, inp->tokens); + // a Hadamard-latent embedding table stores rotated rows; restore the + // primal basis right after the lookup: h = s * (H z) + if (hadamard_inverses) { + const auto it = hadamard_inverses->find(tok_embd); + if (it != hadamard_inverses->end()) { + cur = llama_mul_mat_hadamard(ctx0, cur, it->second.rot); + if (it->second.signs) { + cur = ggml_mul(ctx0, cur, it->second.signs); + } + } + } + // apply lora for embedding tokens if needed for (const auto & lora : *loras) { llama_adapter_lora_weight * lw = lora.first->get_weight(tok_embd); diff --git a/src/llama-graph.h b/src/llama-graph.h index dddfdac7b51e..a32de84d9499 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -16,6 +16,21 @@ struct ggml_cgraph; struct ggml_context; struct ggml_tensor; +// Maps a folded model weight to the activation-side transform applied +// immediately before the matmul: optional sign flip, then the normalized +// blockwise Hadamard rotation. +struct llama_hadamard_transform { + ggml_tensor * rot; + ggml_tensor * signs; // nullptr for identity sign mode + // when perm_rep > 1 the activation arrives with its feature axis in tiled + // head order [hd, nk, rep] and must be permuted to the grouped order + // the folded weight expects + int64_t perm_hd = 0; + int64_t perm_nk = 0; + int64_t perm_rep = 0; +}; +using llama_hadamard_rotations = std::unordered_map; + struct llama_cparams; struct llama_layer; @@ -785,6 +800,8 @@ struct llm_graph_params { const llama_adapter_loras * loras; const llama_memory_context_i * mctx; const llama_cross * cross; + const llama_hadamard_rotations * hadamard_rotations; + const llama_hadamard_rotations * hadamard_inverses; std::map samplers; @@ -1025,6 +1042,12 @@ struct llm_graph_context { const llama_adapter_loras * loras; const llama_memory_context_i * mctx; const llama_cross * cross; + const llama_hadamard_rotations * hadamard_rotations; + const llama_hadamard_rotations * hadamard_inverses; + + // Transforms shared by folded weights on the same activation. Key is (input, rotation); + // both must match. Valid for one graph build only. + mutable std::map, ggml_tensor *> hadamard_memo; std::map samplers; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 6215920fab67..6fd092525100 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -409,6 +409,9 @@ namespace GGUFMeta { return get_arr(llm_kv(kid), result, required); } + // string-key overloads, used by the prism.hadamard.* metadata + template bool llama_model_loader::get_arr(const std::string & key, std::vector & result, bool required); + template bool llama_model_loader::get_arr(const std::string & key, std::vector & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::vector & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::array & result, bool required); template bool llama_model_loader::get_arr>(enum llm_kv kid, std::vector & result, bool required); @@ -439,6 +442,7 @@ namespace GGUFMeta { } template bool llama_model_loader::get_key (enum llm_kv kid, bool & result, bool required); + template bool llama_model_loader::get_key (const std::string & key, bool & result, bool required); template bool llama_model_loader::get_key (enum llm_kv kid, float & result, bool required); template bool llama_model_loader::get_key (enum llm_kv kid, uint32_t & result, bool required); template bool llama_model_loader::get_key(enum llm_kv kid, std::string & result, bool required); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 734bd3edce98..35a715a61f2a 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1210,6 +1210,146 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { gguf_kv.emplace(name, value); } + uint32_t hadamard_version = 0; + if (ml.get_key("prism.hadamard.version", hadamard_version, false)) { + if (hadamard_version != 1) { + throw std::runtime_error(format("unsupported prism.hadamard.version: %u", hadamard_version)); + } + + uint32_t block_size = 0; + std::string transform; + std::string axis; + std::string sign_mode; + std::vector weight_names; + + ml.get_key("prism.hadamard.block_size", block_size); + ml.get_key("prism.hadamard.transform", transform); + ml.get_key("prism.hadamard.axis", axis); + ml.get_key("prism.hadamard.sign_mode", sign_mode); + ml.get_arr("prism.hadamard.weight_names", weight_names); + + if (block_size == 0 || (block_size & (block_size - 1)) != 0) { + throw std::runtime_error(format("invalid prism.hadamard.block_size: %u", block_size)); + } + if (transform != "normalized-sylvester-walsh-hadamard") { + throw std::runtime_error(format("unsupported prism.hadamard.transform: %s", transform.c_str())); + } + if (axis != "input-last-dimension") { + throw std::runtime_error(format("unsupported prism.hadamard.axis: %s", axis.c_str())); + } + if (sign_mode != "identity" && sign_mode != "explicit") { + throw std::runtime_error(format("unsupported prism.hadamard.sign_mode: %s", sign_mode.c_str())); + } + if (weight_names.empty()) { + throw std::runtime_error("prism.hadamard.weight_names is empty"); + } + + if (sign_mode == "explicit") { + std::vector sign_widths; + std::vector sign_values; + ml.get_arr("prism.hadamard.sign_widths", sign_widths); + ml.get_arr("prism.hadamard.sign_values", sign_values); + // explicit mode with no widths would leave the sign table empty, which reads + // as identity later and silently changes the model function + if (sign_widths.empty()) { + throw std::runtime_error("prism.hadamard.sign_mode is explicit but sign_widths is empty"); + } + size_t off = 0; + for (const int32_t width : sign_widths) { + if (width <= 0 || (uint32_t) width % block_size != 0 || off + width > sign_values.size()) { + throw std::runtime_error(format("invalid prism.hadamard sign width: %d", width)); + } + auto & vec = hadamard_sign_data[width]; + vec.assign(sign_values.begin() + off, sign_values.begin() + off + width); + for (const int32_t v : vec) { + if (v != 1 && v != -1) { + throw std::runtime_error("prism.hadamard sign values must be +/-1"); + } + } + off += width; + } + if (off != sign_values.size()) { + throw std::runtime_error("prism.hadamard.sign_values length mismatch"); + } + } + + ml.get_key("prism.hadamard.gdn_v_grouped", hadamard_gdn_v_grouped, false); + + // the activation-side transform is applied only by build_lora_mm/build_lora_mm_id; + // refuse to load folded weights for architectures or tensor kinds that are not + // verified to route every matmul through those helpers, rather than run wrong math + switch (arch) { + case LLM_ARCH_LLAMA: + case LLM_ARCH_QWEN3: + case LLM_ARCH_QWEN3MOE: + case LLM_ARCH_QWEN35: + case LLM_ARCH_QWEN35MOE: + case LLM_ARCH_QWEN3NEXT: + break; + default: + throw std::runtime_error(format( + "prism.hadamard: arch '%s' is not verified to apply the activation transform to all folded weights", + llm_arch_name(arch))); + } + + const auto is_foldable_weight = [](const std::string & name) { + static const char * kinds[] = { + "attn_q", "attn_k", "attn_v", "attn_qkv", "attn_gate", "attn_output", + "ffn_gate", "ffn_up", "ffn_down", + "ffn_gate_exps", "ffn_up_exps", "ffn_down_exps", "ffn_gate_up_exps", + "ffn_gate_shexp", "ffn_up_shexp", "ffn_down_shexp", + "ssm_out", + }; + if (name == "output.weight") { + return true; // the output head is built through build_lora_mm in every arch + } + if (name.compare(0, 4, "blk.") != 0) { + return false; + } + size_t pos = 4; + while (pos < name.size() && isdigit((unsigned char) name[pos])) { + pos++; + } + if (pos == 4 || pos >= name.size() || name[pos] != '.') { + return false; + } + pos++; + for (const char * kind : kinds) { + const std::string suffix = std::string(kind) + ".weight"; + if (name.compare(pos, std::string::npos, suffix) == 0) { + return true; + } + } + return false; + }; + + for (const auto & weight_name : weight_names) { + if (!is_foldable_weight(weight_name)) { + throw std::runtime_error(format( + "prism.hadamard: weight '%s' is not on a verified Hadamard-aware matmul path", weight_name.c_str())); + } + if (!hadamard_weight_blocks.emplace(weight_name, block_size).second) { + throw std::runtime_error(format("duplicate prism.hadamard weight: %s", weight_name.c_str())); + } + } + + // tensors consumed by row lookup store latent rows and need the + // inverse transform applied to the lookup result instead + std::vector inverse_names; + ml.get_arr("prism.hadamard.inverse_weight_names", inverse_names, false); + for (const auto & name : inverse_names) { + // the graph applies the inverse only to the token-embedding lookup; any + // other latent table would load and silently stay rotated + if (name != "token_embd.weight") { + throw std::runtime_error(format( + "prism.hadamard: weight '%s' is not a verified inverse-after-lookup table", name.c_str())); + } + if (hadamard_weight_blocks.count(name) || !hadamard_inverse_blocks.emplace(name, block_size).second) { + throw std::runtime_error(format("duplicate prism.hadamard inverse weight: %s", name.c_str())); + } + } + } + // get general kv ml.get_key(LLM_KV_GENERAL_NAME, name, false); @@ -1837,6 +1977,163 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { } } + if (!hadamard_weight_blocks.empty() || !hadamard_inverse_blocks.empty()) { + struct hadamard_rotation { + uint32_t block_size; + ggml_backend_buffer_type_t buft; + ggml_tensor * tensor; + }; + + std::vector rotations; + std::map, ggml_tensor *> sign_tensors; + + const std::pair *, llama_hadamard_rotations *> groups[] = { + { &hadamard_weight_blocks, &hadamard_rotations }, + { &hadamard_inverse_blocks, &hadamard_inverses }, + }; + // inverse (lookup-side) transforms must not inherit a host buffer + // type from a CPU-mapped table: the per-token transform would then + // ping-pong across the PCIe boundary. Prefer the buffer type the + // forward rotations live on (the GPU when layers are offloaded). + ggml_backend_buffer_type_t preferred_buft = nullptr; + + for (const auto & [blocks, target] : groups) + for (const auto & entry : *blocks) { + const std::string & weight_name = entry.first; + const uint32_t block_size = entry.second; + const ggml_tensor * weight = get_tensor(weight_name.c_str()); + if (weight == nullptr) { + throw std::runtime_error(format("prism.hadamard weight not found: %s", weight_name.c_str())); + } + if (weight->ne[0] % block_size != 0) { + throw std::runtime_error(format( + "prism.hadamard block size %u does not divide input dimension %lld for %s", + block_size, (long long) weight->ne[0], weight_name.c_str())); + } + if (weight->buffer == nullptr) { + throw std::runtime_error(format("prism.hadamard weight has no buffer: %s", weight_name.c_str())); + } + + ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(weight->buffer); + if (target == &hadamard_rotations) { + preferred_buft = buft; + } else if (preferred_buft) { + buft = preferred_buft; + } + auto it = std::find_if(rotations.begin(), rotations.end(), + [block_size, buft](const hadamard_rotation & rotation) { + return rotation.block_size == block_size && rotation.buft == buft; + }); + + if (it == rotations.end()) { + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx { ggml_init(params) }; + if (!ctx) { + throw std::runtime_error("failed to create Hadamard rotation context"); + } + + ggml_tensor * rotation = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_F32, block_size, block_size); + char rotation_name[GGML_MAX_NAME]; + snprintf(rotation_name, sizeof(rotation_name), "prism.hadamard.%u", block_size); + ggml_set_name(rotation, rotation_name); + + ggml_backend_buffer_ptr buffer { ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft) }; + if (!buffer) { + throw std::runtime_error(format("unable to allocate %s Hadamard rotation buffer", ggml_backend_buft_name(buft))); + } + ggml_backend_buffer_set_usage(buffer.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + std::vector data((size_t) block_size * block_size); + const float scale = 1.0f / sqrtf((float) block_size); + for (uint32_t row = 0; row < block_size; ++row) { + for (uint32_t col = 0; col < block_size; ++col) { + uint32_t parity = row & col; + parity ^= parity >> 16; + parity ^= parity >> 8; + parity ^= parity >> 4; + parity ^= parity >> 2; + parity ^= parity >> 1; + data[(size_t) row * block_size + col] = (parity & 1) ? -scale : scale; + } + } + ggml_backend_tensor_set(rotation, data.data(), 0, data.size() * sizeof(float)); + + std::vector buffers; + buffers.emplace_back(std::move(buffer)); + pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers)); + rotations.push_back({ block_size, buft, rotation }); + it = std::prev(rotations.end()); + } + + ggml_tensor * sign_tensor = nullptr; + if (!hadamard_sign_data.empty()) { + const uint32_t width = (uint32_t) weight->ne[0]; + const auto sd = hadamard_sign_data.find(width); + if (sd == hadamard_sign_data.end()) { + throw std::runtime_error(format( + "prism.hadamard has no sign vector for width %u (%s)", width, weight_name.c_str())); + } + const auto key = std::make_pair(width, buft); + auto st = sign_tensors.find(key); + if (st == sign_tensors.end()) { + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx { ggml_init(params) }; + if (!ctx) { + throw std::runtime_error("failed to create Hadamard sign context"); + } + + ggml_tensor * signs = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, width); + char sign_name[GGML_MAX_NAME]; + snprintf(sign_name, sizeof(sign_name), "prism.hadamard.signs.%u", width); + ggml_set_name(signs, sign_name); + + ggml_backend_buffer_ptr buffer { ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft) }; + if (!buffer) { + throw std::runtime_error(format("unable to allocate %s Hadamard sign buffer", ggml_backend_buft_name(buft))); + } + ggml_backend_buffer_set_usage(buffer.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + std::vector data(width); + for (uint32_t i = 0; i < width; ++i) { + data[i] = (float) sd->second[i]; + } + ggml_backend_tensor_set(signs, data.data(), 0, data.size() * sizeof(float)); + + std::vector buffers; + buffers.emplace_back(std::move(buffer)); + pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers)); + st = sign_tensors.emplace(key, signs).first; + } + sign_tensor = st->second; + } + + llama_hadamard_transform transform { it->tensor, sign_tensor }; + if (hadamard_gdn_v_grouped && weight_name.find(".ssm_out.") != std::string::npos) { + const int64_t n_v = hparams.ssm_dt_rank; + const int64_t n_k = hparams.ssm_n_group; + if (n_k <= 0 || n_v <= 0 || n_v % n_k != 0 || weight->ne[0] % n_v != 0) { + throw std::runtime_error(format("prism.hadamard: bad GDN head geometry for %s", weight_name.c_str())); + } + transform.perm_hd = weight->ne[0] / n_v; + transform.perm_nk = n_k; + transform.perm_rep = n_v / n_k; + } + target->emplace(weight, transform); + } + + LLAMA_LOG_INFO("%s: loaded %zu Hadamard-folded weight(s) (%zu inverse-lookup) using %zu rotation(s) and %zu sign vector(s)\n", + __func__, hadamard_rotations.size() + hadamard_inverses.size(), hadamard_inverses.size(), + rotations.size(), sign_tensors.size()); + } + return true; } diff --git a/src/llama-model.h b/src/llama-model.h index 4c4a30e018bc..891c4d2eb861 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -697,6 +697,18 @@ struct llama_model { // gguf metadata std::unordered_map gguf_kv; + // Hadamard-folded GGUF weights are matched with persistent model tensors + // containing the activation-side transform. The string map is populated + // from GGUF metadata while loading hparams; the pointer map is populated + // after model buffers have been allocated. In explicit sign mode the + // per-width sign vectors come from GGUF metadata as well. + std::unordered_map hadamard_weight_blocks; + std::unordered_map hadamard_inverse_blocks; + std::map> hadamard_sign_data; + bool hadamard_gdn_v_grouped = false; + llama_hadamard_rotations hadamard_rotations; + llama_hadamard_rotations hadamard_inverses; + // list of devices used in this model std::vector devices; From 8afb3f0533e47b2e0a03906717bb22736a146486 Mon Sep 17 00:00:00 2001 From: Laurent Zuijdwijk Date: Fri, 18 Sep 2026 11:29:57 +0100 Subject: [PATCH 3/4] ptq1_0 : collapse the serial trit unpack to a single multiply ptq1_0_trit walked to trit n with a loop of up to four dependent v = (v*3) & 0xFF steps, and the trip count is a function of element position, so lanes in a subgroup ran different iteration counts. Mod-256 multiplication is associative, so that walk is just b * 3^n mod 256. This is already how dequantize_row_ptq1_0 in ggml-quants.c decodes -- only the GLSL kept the serial form. 3^n is computed with selects rather than a const array, because a dynamically indexed local array can land in scratch on some drivers and this is the hot path of every ternary matmul. Equivalence was checked exhaustively over the whole input domain: all 256 byte values x 5 trit positions agree with the loop form, and all 3^5 = 243 encoder codes round-trip. Applied to both copies -- ptq1_0.glsl (mul_mat_vec, get_rows, mul_mm) and the standalone decoder in dequant_ptq1_0.comp. NOT YET VALIDATED ON GPU. The shaders compile and the arithmetic is proven equivalent offline, but nothing has been run on the Radeon 8060S since the change, so there is no measured speedup and no on-device correctness check yet. Baseline to beat, from the same box and build: pp512 210.13, tg128 3.96, which is ~22 GiB/s effective against ~193 GiB/s for IQ3_XXS on the same hardware. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01K3NeoeVLYx9mNqMSEJHRvS --- .../vulkan-shaders/dequant_ptq1_0.comp | 19 +++++++++++++++---- .../ggml-vulkan/vulkan-shaders/ptq1_0.glsl | 19 +++++++++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp index 53baf41f65b9..995235d041ce 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_ptq1_0.comp @@ -10,6 +10,14 @@ layout (binding = 1) writeonly buffer D {D_TYPE data_b[];}; // One invocation per 8 elements: 16 invocations cover a 128-weight block. Element // order follows the CPU codec (16-byte qs chunk, 8-byte qs chunk, then qh), so the // index maths lives in one place rather than being reproduced per shader. +// 3^n for n in [0,4]. Written as selects rather than a const array: a +// dynamically indexed local array can land in scratch memory on some drivers, +// and this sits on the hot path of every ternary matmul. +uint ptq1_0_pow3(uint n) { + return n < 2u ? (n == 0u ? 1u : 3u) + : (n == 2u ? 9u : (n == 3u ? 27u : 81u)); +} + float ptq1_0_trit_l(uint ib, uint e) { uint b; uint n; @@ -26,10 +34,13 @@ float ptq1_0_trit_l(uint ib, uint e) { n = t >> 1u; } - uint v = b; - for (uint i = 0u; i < n; ++i) { - v = (v * 3u) & 0xFFu; - } + // Mod-256 multiplication is associative, so the repeated v = (v*3)&0xFF that + // walks to trit n collapses to a single multiply by 3^n -- which is what + // dequantize_row_ptq1_0 in ggml-quants.c already does. Verified identical over + // all 256 byte values x 5 trit positions. The serial form cost up to 4 + // dependent multiplies per element with a trip count that varies by element + // position, so lanes in a subgroup diverged; this is branch-free and uniform. + const uint v = (b * ptq1_0_pow3(n)) & 0xFFu; return float(int((v * 3u) >> 8u) - 1); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl index 7eedd832b14f..1a2c17fd40a5 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl @@ -12,6 +12,14 @@ // elements t*16+j, then an 8-byte chunk carrying 80 + t*8 + (j-16), then qh at four // trits per byte carrying 120 + t*2 + h. Trits come out by the base-3 remainder // recurrence t = (v*3)>>8, v = (v*3)&0xFF. +// 3^n for n in [0,4]. Written as selects rather than a const array: a +// dynamically indexed local array can land in scratch memory on some drivers, +// and this sits on the hot path of every ternary matmul. +uint ptq1_0_pow3(uint n) { + return n < 2u ? (n == 0u ? 1u : 3u) + : (n == 2u ? 9u : (n == 3u ? 27u : 81u)); +} + float ptq1_0_trit(uint ib, uint a_offset, uint e) { uint b; uint n; @@ -28,10 +36,13 @@ float ptq1_0_trit(uint ib, uint a_offset, uint e) { n = t >> 1u; } - uint v = b; - for (uint i = 0u; i < n; ++i) { - v = (v * 3u) & 0xFFu; - } + // Mod-256 multiplication is associative, so the repeated v = (v*3)&0xFF that + // walks to trit n collapses to a single multiply by 3^n -- which is what + // dequantize_row_ptq1_0 in ggml-quants.c already does. Verified identical over + // all 256 byte values x 5 trit positions. The serial form cost up to 4 + // dependent multiplies per element with a trip count that varies by element + // position, so lanes in a subgroup diverged; this is branch-free and uniform. + const uint v = (b * ptq1_0_pow3(n)) & 0xFFu; return float(int((v * 3u) >> 8u) - 1); } From a15de7402994fb2c0f58e9b94d31d048d508c0b1 Mon Sep 17 00:00:00 2001 From: Laurent Zuijdwijk Date: Fri, 18 Sep 2026 14:10:25 +0100 Subject: [PATCH 4/4] vulkan: optimize PTQ1_0 decoding and matvec Assisted-by: Marshall --- .../vulkan-shaders/dequant_funcs.glsl | 7 +-- .../vulkan-shaders/mul_mat_vec.comp | 57 +++++++++++++++++++ .../vulkan-shaders/mul_mm_funcs.glsl | 13 +++-- .../ggml-vulkan/vulkan-shaders/ptq1_0.glsl | 53 +++++++++++++++++ .../src/ggml-vulkan/vulkan-shaders/types.glsl | 9 +++ ggml/src/ggml.c | 1 + tests/test-backend-ops.cpp | 15 +++++ tests/test-quantize-fns.cpp | 35 +++++++++++- 8 files changed, 179 insertions(+), 11 deletions(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl index 88be1360403a..3da9f858eb79 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/dequant_funcs.glsl @@ -130,13 +130,10 @@ vec4 dequantize4(uint ib, uint iqs, uint a_offset) { #include "ptq1_0.glsl" vec2 dequantize(uint ib, uint iqs, uint a_offset) { - return vec2(ptq1_0_trit(ib, a_offset, iqs), ptq1_0_trit(ib, a_offset, iqs + 1u)); + return ptq1_0_trits2(ib, a_offset, iqs); } vec4 dequantize4(uint ib, uint iqs, uint a_offset) { - return vec4(ptq1_0_trit(ib, a_offset, iqs), - ptq1_0_trit(ib, a_offset, iqs + 1u), - ptq1_0_trit(ib, a_offset, iqs + 2u), - ptq1_0_trit(ib, a_offset, iqs + 3u)); + return ptq1_0_trits4(ib, a_offset, iqs); } #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp index 5a9d0e778fdf..dec377c15ff1 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp @@ -74,8 +74,13 @@ void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const ibi += p.ncols; #if K_PER_ITER == 8 +#if defined(DATA_A_PTQ1_0) + vec4 v, v2; + ptq1_0_trits8(ib, a_offset, iqs, v, v2); +#else vec4 v = dequantize4(ib, iqs, a_offset); vec4 v2 = dequantize4(ib, iqs+(4/QUANT_R), a_offset); +#endif const vec2 dm = get_dm(ib, a_offset); if (dm.y != 0) { // quant has min component @@ -138,6 +143,54 @@ void iter_aligned_nonquant(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint } #endif +#if defined(DATA_A_PTQ1_0) +// Eight lanes own a whole block: six packed qs words, one qh tail, one +// inactive lane. Each packed byte is loaded once and supplies all its trits. +// Keep the common reduction and fusion path, including partial output tiles. +void accumulate_ptq1_0(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], + uint first_row, uint num_rows, uint tid) { + const uint lane = tid & 7u; + const uint blocks = p.ncols / 128u; + if (lane == 7u) { + return; + } + for (uint block = tid / 8u; block < blocks; block += BLOCK_SIZE / 8u) { + [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) { + const uint base = j*p.batch_stride_b + b_offset + block*128u; + if (lane < 6u) { + const uint start = lane < 4u ? lane*4u : 80u + (lane-4u)*4u; + const uint step = lane < 4u ? 16u : 8u; + vec4 activation[5]; + [[unroll]] for (uint t = 0; t < 5u; ++t) { + activation[t] = vec4(data_b_v4[(base + start + t*step)/4u]); + } + [[unroll]] for (uint row = 0; row < num_rows; ++row) { + const uint ib = a_offset + (first_row + row)*blocks + block; + const uint packed = data_a_packed32[ib].qs[lane]; + uvec4 bytes = (uvec4(packed) >> uvec4(0u, 8u, 16u, 24u)) & 255u; + float sum = 0.0f; + [[unroll]] for (uint t = 0; t < 5u; ++t) { + bytes *= 3u; + sum += dot(activation[t], vec4(bytes >> 8u) - 1.0f); + bytes &= 255u; + } + temp[j][row] += sum * get_dm(ib - a_offset, a_offset).x; + } + } else { + const vec4 b0 = vec4(data_b_v4[(base + 120u)/4u]); + const vec4 b1 = vec4(data_b_v4[(base + 124u)/4u]); + [[unroll]] for (uint row = 0; row < num_rows; ++row) { + const uint ib = (first_row + row)*blocks + block; + vec4 lo, hi; + ptq1_0_trits8(ib, a_offset, 120u, lo, hi); + temp[j][row] += (dot(b0, lo) + dot(b1, hi)) * get_dm(ib, a_offset).x; + } + } + } + } +} +#endif + void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { const uint tid = gl_LocalInvocationID.x; @@ -157,6 +210,9 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { } } +#if defined(DATA_A_PTQ1_0) + accumulate_ptq1_0(temp, first_row, num_rows, tid); +#else uint num_iters = p.ncols / (K_PER_ITER * BLOCK_SIZE); if (num_iters * K_PER_ITER * BLOCK_SIZE + K_PER_ITER*tid < p.ncols) { num_iters++; @@ -242,6 +298,7 @@ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) { } #endif +#endif // DATA_A_PTQ1_0 reduce_result(temp, d_offset, first_row, num_rows, tid); } diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl index 3125a3fcfa93..c15f7e1187d8 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl @@ -162,11 +162,14 @@ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uin const float d = float(data_a[ib].d); const uint k_pair = row * LOAD_VEC_A / 2; - [[unroll]] for (uint l = 0; l < 4; ++l) { - store_a(col, k_pair + l, FLOAT_TYPEV2( - ptq1_0_trit(ib, 0u, e0 + 2u*l) * d, - ptq1_0_trit(ib, 0u, e0 + 2u*l + 1u) * d)); - } + vec4 lo, hi; + ptq1_0_trits8(ib, 0u, e0, lo, hi); + lo *= d; + hi *= d; + store_a(col, k_pair, FLOAT_TYPEV2(lo.xy)); + store_a(col, k_pair + 1, FLOAT_TYPEV2(lo.zw)); + store_a(col, k_pair + 2, FLOAT_TYPEV2(hi.xy)); + store_a(col, k_pair + 3, FLOAT_TYPEV2(hi.zw)); #elif defined(DATA_A_Q1_0) const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl index 1a2c17fd40a5..0f4da53c3752 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/ptq1_0.glsl @@ -46,4 +46,57 @@ float ptq1_0_trit(uint ib, uint a_offset, uint e) { return float(int((v * 3u) >> 8u) - 1); } +// Decode four independent bytes at the same trit position. One aligned word +// load replaces four byte loads; mask before multiplying by 3, not afterwards. +vec4 ptq1_0_decode4(uint packed, uint power) { + const uvec4 bytes = (uvec4(packed) >> uvec4(0u, 8u, 16u, 24u)) & 255u; + return vec4((((bytes * power) & 255u) * 3u) >> 8u) - 1.0f; +} + +// e must be even. All qs region/position boundaries are multiples of eight. +vec2 ptq1_0_trits2(uint ib, uint a_offset, uint e) { + uint packed; + uint n; + if (e < 120u) { + const uint q = e < 80u ? (e & 15u) : (16u + (e & 7u)); + n = e < 80u ? (e >> 4u) : ((e - 80u) >> 3u); + packed = data_a_packed32[a_offset + ib].qs[q >> 2u] >> ((q & 2u) * 8u); + } else { + packed = uint(data_a_packed32[a_offset + ib].qh); + n = (e - 120u) >> 1u; + } + const uvec2 bytes = uvec2(packed, packed >> 8u) & 255u; + return vec2((((bytes * ptq1_0_pow3(n)) & 255u) * 3u) >> 8u) - 1.0f; +} + +// e must be four-aligned. The qh tail repeats two bytes at successive powers. +vec4 ptq1_0_trits4(uint ib, uint a_offset, uint e) { + if (e < 120u) { + const uint q = e < 80u ? (e & 15u) : (16u + (e & 7u)); + const uint n = e < 80u ? (e >> 4u) : ((e - 80u) >> 3u); + return ptq1_0_decode4(data_a_packed32[a_offset + ib].qs[q >> 2u], ptq1_0_pow3(n)); + } + const uint packed = uint(data_a_packed32[a_offset + ib].qh); + const uvec2 bytes = uvec2(packed, packed >> 8u) & 255u; + const uvec4 powers = e == 120u ? uvec4(1u, 1u, 3u, 3u) : uvec4(9u, 9u, 27u, 27u); + return vec4((((bytes.xyxy * powers) & 255u) * 3u) >> 8u) - 1.0f; +} + +// Shared hot path for matvec and matmul: one region decision and one power +// selection per eight outputs, with just two word loads (one halfword for qh). +void ptq1_0_trits8(uint ib, uint a_offset, uint e, out vec4 lo, out vec4 hi) { + if (e < 120u) { + const uint q = e < 80u ? (e & 15u) : 16u; + const uint n = e < 80u ? (e >> 4u) : ((e - 80u) >> 3u); + const uint power = ptq1_0_pow3(n); + lo = ptq1_0_decode4(data_a_packed32[a_offset + ib].qs[q >> 2u], power); + hi = ptq1_0_decode4(data_a_packed32[a_offset + ib].qs[(q >> 2u) + 1u], power); + } else { + const uint packed = uint(data_a_packed32[a_offset + ib].qh); + const uvec2 bytes = uvec2(packed, packed >> 8u) & 255u; + lo = vec4((((bytes.xyxy * uvec4(1u, 1u, 3u, 3u)) & 255u) * 3u) >> 8u) - 1.0f; + hi = vec4((((bytes.xyxy * uvec4(9u, 9u, 27u, 27u)) & 255u) * 3u) >> 8u) - 1.0f; + } +} + #endif // PTQ1_0_GLSL diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index 469aad51ce89..09239f512877 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -225,7 +225,16 @@ struct block_ptq1_0 float16_t d; }; +// Same 28-byte stride: qs at 0, qh at 24, fp16 scale at 26. +struct block_ptq1_0_packed32 +{ + uint32_t qs[6]; + uint16_t qh; + float16_t d; +}; + #if defined(DATA_A_PTQ1_0) +#define A_TYPE_PACKED32 block_ptq1_0_packed32 #define QUANT_K QUANT_K_PTQ1_0 #define QUANT_R QUANT_R_PTQ1_0 #define QUANT_AUXF 1 diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index b7c48d7304fc..e5f782be193a 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -8022,6 +8022,7 @@ size_t ggml_quantize_chunk( case GGML_TYPE_Q4_K: result = quantize_q4_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q5_K: result = quantize_q5_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q6_K: result = quantize_q6_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_PTQ1_0: result = quantize_ptq1_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_TQ1_0: result = quantize_tq1_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_TQ2_0: result = quantize_tq2_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_IQ2_XXS: result = quantize_iq2_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 9610b37c7281..d7736d601467 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8649,6 +8649,7 @@ static const ggml_type all_types[] = { GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, GGML_TYPE_Q1_0, + GGML_TYPE_PTQ1_0, GGML_TYPE_Q2_0, GGML_TYPE_MXFP4, GGML_TYPE_NVFP4, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, @@ -8665,6 +8666,7 @@ static const ggml_type base_types[] = { GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_Q8_0, // for I8MM tests GGML_TYPE_Q1_0, + GGML_TYPE_PTQ1_0, GGML_TYPE_Q2_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, // for I8MM tests @@ -8678,6 +8680,7 @@ static const ggml_type other_types[] = { GGML_TYPE_Q5_0, GGML_TYPE_Q5_1, GGML_TYPE_Q8_0, GGML_TYPE_Q1_0, + GGML_TYPE_PTQ1_0, GGML_TYPE_Q2_0, GGML_TYPE_Q2_K, GGML_TYPE_Q3_K, GGML_TYPE_Q5_K, @@ -9584,6 +9587,18 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q8_0, GGML_TYPE_F32, 8192, 512, 5120, {128, 1}, {1, 1})); #endif + // PTQ has independent scales every 128 elements and a two-byte qh tail. + // Exercise odd block counts, partial output tiles, and batched row offsets + // through both matvec and matmul, including non-contiguous weight views. + for (int64_t k : {128, 384}) { + for (int64_t n : {1, 3, 8, 32}) { + for (int64_t k_v : {int64_t(0), k + 128}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_PTQ1_0, GGML_TYPE_F32, + 17, n, k, {2, 1}, {2, 1}, {0, 1, 2, 3}, k_v)); + } + } + } + for (ggml_type type_a : all_types) { for (int i = 1; i < 10; ++i) { test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 16, i, 1*256, { 1, 1}, {1, 1})); diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index 9510ac14ce00..16a84b94c341 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -126,6 +126,37 @@ static int test_vec_dot_f32(bool verbose) { return num_failed; } +// Check arithmetic separately from the unavoidable loss of ternary quantization. +static int test_vec_dot_ptq1_0(bool verbose) { + const auto * q = ggml_get_type_traits(GGML_TYPE_PTQ1_0); + const auto * cpu = ggml_get_type_traits_cpu(GGML_TYPE_PTQ1_0); + const auto * v = ggml_get_type_traits(cpu->vec_dot_type); + const auto * vcpu = ggml_get_type_traits_cpu(cpu->vec_dot_type); + int num_failed = 0; + for (int n : {128, 256, 384, 4096, 5120, 17408}) { + std::vector a(n), b(n), da(n), db(n); + std::vector qa(ggml_row_size(GGML_TYPE_PTQ1_0, n)); + std::vector qb(ggml_row_size(cpu->vec_dot_type, n)); + generate_data(0.0f, n, a.data()); + generate_data(1.0f, n, b.data()); + cpu->from_float(a.data(), qa.data(), n); + vcpu->from_float(b.data(), qb.data(), n); + q->to_float(qa.data(), da.data(), n); + v->to_float(qb.data(), db.data(), n); + float result = INFINITY; + cpu->vec_dot(n, &result, 0, qa.data(), 0, qb.data(), 0, 1); + const float ref = dot_product(da.data(), db.data(), n); + const float error = fabsf(result - ref) / n; + const bool failed = !(error < MAX_QUANTIZATION_REFERENCE_ERROR); + num_failed += failed; + if (failed || verbose) { + printf("ptq1_0 dequantized dot n=%5d: %s (ref=%f got=%f err=%f)\n", + n, RESULT_STR[failed], ref, result, error); + } + } + return num_failed; +} + static int test_vec_dot_q(bool verbose) { int num_failed = 0; @@ -156,6 +187,7 @@ static int test_vec_dot_q(bool verbose) { const float total_error = total_quantization_error(qfns, qfns_cpu, test_size, test_data.data()); const float max_quantization_error = type == GGML_TYPE_Q1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_BINARY : + type == GGML_TYPE_PTQ1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY : type == GGML_TYPE_TQ1_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY : type == GGML_TYPE_TQ2_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY : type == GGML_TYPE_Q2_0 ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY : @@ -184,7 +216,7 @@ static int test_vec_dot_q(bool verbose) { ? MAX_DOT_PRODUCT_ERROR_LOWBIT : type == GGML_TYPE_Q1_0 ? MAX_DOT_PRODUCT_ERROR_BINARY - : type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0 || type == GGML_TYPE_Q2_0 + : type == GGML_TYPE_PTQ1_0 || type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0 || type == GGML_TYPE_Q2_0 ? MAX_DOT_PRODUCT_ERROR_TERNARY : type == GGML_TYPE_NVFP4 ? MAX_DOT_PRODUCT_ERROR_FP4 @@ -221,6 +253,7 @@ int main(int argc, char * argv[]) { num_failed += test_vec_dot_f32(verbose); num_failed += test_vec_dot_q(verbose); + num_failed += test_vec_dot_ptq1_0(verbose); if (num_failed || verbose) { printf("%d tests failed\n", num_failed);