From 1d9e1e03bb58b772059322aa4a606286052ebe5e Mon Sep 17 00:00:00 2001 From: Michael Verrilli Date: Sun, 31 May 2026 00:31:31 +0000 Subject: [PATCH] ggml-cpu : build TQ with SIMD Was compiled in ggml-base (no arch flags) so it ran scalar. Move to a ggml-cpu TU and use the standard vec op (ggml_vec_dot_f32). --- ggml/include/ggml-cpu.h | 1 + ggml/src/CMakeLists.txt | 2 +- ggml/src/ggml-cpu/CMakeLists.txt | 1 + ggml/src/ggml-cpu/ggml-cpu.c | 2 + ggml/src/ggml-cpu/ops.cpp | 18 +- ggml/src/ggml-cpu/quants.c | 12 - ggml/src/ggml-cpu/quants.h | 4 + ggml/src/ggml-cpu/turboq.c | 34 ++ ggml/src/ggml-turboq-impl.h | 460 ++++++++++++++++++++++ ggml/src/ggml-turboq.c | 632 +------------------------------ ggml/src/ggml-turboq.h | 21 - tests/test-quantize-fns.cpp | 1 - 12 files changed, 525 insertions(+), 663 deletions(-) create mode 100644 ggml/src/ggml-cpu/turboq.c create mode 100644 ggml/src/ggml-turboq-impl.h delete mode 100644 ggml/src/ggml-turboq.h diff --git a/ggml/include/ggml-cpu.h b/ggml/include/ggml-cpu.h index e3e067c916f1..a138f7657ca5 100644 --- a/ggml/include/ggml-cpu.h +++ b/ggml/include/ggml-cpu.h @@ -118,6 +118,7 @@ extern "C" { ggml_vec_dot_t vec_dot; enum ggml_type vec_dot_type; int64_t nrows; // number of rows to process simultaneously + ggml_to_float_t to_float; // optional SIMD override of the base to_float }; GGML_BACKEND_API const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type); diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index 46bdfa0d9305..36e38c39a2ae 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -206,7 +206,7 @@ add_library(ggml-base ggml-quants.c ggml-quants.h ggml-turboq.c - ggml-turboq.h + ggml-turboq-impl.h ggml-turboq-tables.h gguf.cpp) diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index beebc4760d2d..882742d2839e 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -35,6 +35,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name) ggml-cpu/hbm.h ggml-cpu/quants.c ggml-cpu/quants.h + ggml-cpu/turboq.c ggml-cpu/traits.cpp ggml-cpu/traits.h ggml-cpu/amx/amx.cpp diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index f1289d463cb7..04e0df2606d9 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -392,12 +392,14 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { }, [GGML_TYPE_TBQ3_0] = { .from_float = quantize_row_tbq3_0, + .to_float = (ggml_to_float_t) ggml_cpu_dequantize_row_tbq3_0, .vec_dot = ggml_vec_dot_tbq3_0_q8_K, .vec_dot_type = GGML_TYPE_Q8_K, .nrows = 1, }, [GGML_TYPE_TBQ4_0] = { .from_float = quantize_row_tbq4_0, + .to_float = (ggml_to_float_t) ggml_cpu_dequantize_row_tbq4_0, .vec_dot = ggml_vec_dot_tbq4_0_q8_K, .vec_dot_type = GGML_TYPE_Q8_K, .nrows = 1, diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index 8933acf800b8..46538d8117e5 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -14,6 +14,12 @@ #include #include +// prefer a SIMD dequant from the CPU traits, else fall back to the base to_float +static ggml_to_float_t ggml_cpu_to_float(enum ggml_type type) { + ggml_to_float_t const cpu = ggml_get_type_traits_cpu(type)->to_float; + return cpu ? cpu : ggml_get_type_traits(type)->to_float; +} + // ggml_compute_forward_dup static void ggml_compute_forward_dup_same_cont( @@ -511,7 +517,7 @@ static void ggml_compute_forward_dup_from_q( GGML_TENSOR_BINARY_OP_LOCALS const ggml_type type = src0->type; - ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float; + ggml_to_float_t const dequantize_row_q = ggml_cpu_to_float(type); size_t qk = ggml_blck_size(type); const int64_t nr = ggml_nelements(src1) / qk; @@ -642,7 +648,7 @@ static void ggml_compute_forward_add_q_f32( const ggml_type type = src0->type; const ggml_type dtype = dst->type; - ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float; + ggml_to_float_t const dequantize_row_q = ggml_cpu_to_float(type); ggml_from_float_t const quantize_row_q = ggml_get_type_traits_cpu(dtype)->from_float; // we don't support permuted src0 or src1 @@ -988,7 +994,7 @@ static void ggml_compute_forward_add1_q_f32( GGML_TENSOR_UNARY_OP_LOCALS const ggml_type type = src0->type; - ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float; + ggml_to_float_t const dequantize_row_q = ggml_cpu_to_float(type); ggml_from_float_t const quantize_row_q = ggml_get_type_traits_cpu(type)->from_float; // we don't support permuted src0 @@ -4303,7 +4309,7 @@ static void ggml_compute_forward_out_prod_q_f32( const int nth = params->nth; const ggml_type type = src0->type; - ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float; + ggml_to_float_t const dequantize_row_q = ggml_cpu_to_float(type); GGML_ASSERT(ne02 == ne12); GGML_ASSERT(ne03 == ne13); @@ -4727,7 +4733,7 @@ static void ggml_compute_forward_get_rows_q( const int64_t nr = ggml_nelements(src1); const ggml_type type = src0->type; - ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float; + ggml_to_float_t const dequantize_row_q = ggml_cpu_to_float(type); assert(ne0 == nc); assert(ne02 == ne11); @@ -8298,7 +8304,7 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk( ggml_type const k_vec_dot_type = ggml_get_type_traits_cpu(k->type)->vec_dot_type; ggml_from_float_t const q_to_vec_dot = ggml_get_type_traits_cpu(k_vec_dot_type)->from_float; ggml_vec_dot_t const kq_vec_dot = ggml_get_type_traits_cpu(k->type)->vec_dot; - ggml_to_float_t const v_to_float = ggml_get_type_traits(v->type)->to_float; + ggml_to_float_t const v_to_float = ggml_cpu_to_float(v->type); GGML_ASSERT(( q_to_vec_dot) && "fattn: unsupported K-type"); GGML_ASSERT((v->type == GGML_TYPE_F32 || v_to_float ) && "fattn: unsupported V-type"); diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index f5b068712231..9e8314acab00 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -108,18 +108,6 @@ void quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, quantize_row_tq2_0_ref(x, y, k); } -void quantize_row_tbq3_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { - assert(k % QK_K == 0); - block_tbq3_0 * GGML_RESTRICT y = vy; - quantize_row_tbq3_0_ref(x, y, k); -} - -void quantize_row_tbq4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) { - assert(k % QK_K == 0); - block_tbq4_0 * GGML_RESTRICT y = vy; - quantize_row_tbq4_0_ref(x, y, k); -} - //===================================== Q8_K ============================================== void quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h index c447fb4e4f18..d862e79a0fc7 100644 --- a/ggml/src/ggml-cpu/quants.h +++ b/ggml/src/ggml-cpu/quants.h @@ -35,6 +35,10 @@ void quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, i void quantize_row_tbq3_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_tbq4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); +// SIMD dequant for the CPU backend (arch-built); wired via cpu traits to_float +void ggml_cpu_dequantize_row_tbq3_0(const block_tbq3_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +void ggml_cpu_dequantize_row_tbq4_0(const block_tbq4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); diff --git a/ggml/src/ggml-cpu/turboq.c b/ggml/src/ggml-cpu/turboq.c new file mode 100644 index 000000000000..fc0d40cc3e1c --- /dev/null +++ b/ggml/src/ggml-cpu/turboq.c @@ -0,0 +1,34 @@ +// TurboQuant quantize/dequant. Built with arch flags, and the rotation matvec +// reuses ggml_vec_dot_f32 so it vectorizes on every supported CPU arch. + +#include "quants.h" +#include "vec.h" +#include "ggml-turboq-impl.h" + +static void matvec_row(float * y, const float * M, const float * x, int64_t d) { + for (int64_t i = 0; i < d; ++i) { + ggml_vec_dot_f32(d, &y[i], 0, M + i * d, 0, x, 0, 1); + } +} + +static void matvec_t(float * y, const float * M, const float * x, int64_t d) { + for (int64_t j = 0; j < d; ++j) { + ggml_vec_dot_f32(d, &y[j], 0, M + j * d, 0, x, 0, 1); + } +} + +void quantize_row_tbq3_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + turboq_quantize_tbq3_0(x, y, k); +} + +void quantize_row_tbq4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) { + turboq_quantize_tbq4_0(x, y, k); +} + +void ggml_cpu_dequantize_row_tbq3_0(const block_tbq3_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + turboq_dequantize_tbq3_0(x, y, k); +} + +void ggml_cpu_dequantize_row_tbq4_0(const block_tbq4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + turboq_dequantize_tbq4_0(x, y, k); +} diff --git a/ggml/src/ggml-turboq-impl.h b/ggml/src/ggml-turboq-impl.h new file mode 100644 index 000000000000..5490a3a8cc7d --- /dev/null +++ b/ggml/src/ggml-turboq-impl.h @@ -0,0 +1,460 @@ +#pragma once + +// Shared TurboQuant rotation + codebook routines, included by the base (scalar) +// and ggml-cpu (arch-built SIMD) translation units. + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "ggml-turboq-tables.h" +#include "ggml-impl.h" + +#include +#include +#include +#include + +#if defined(__GNUC__) || defined(__clang__) +#define TURBOQ_TLS __thread +#elif defined(_MSC_VER) +#define TURBOQ_TLS __declspec(thread) +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) +#define TURBOQ_TLS _Thread_local +#else +#define TURBOQ_TLS +#endif + +#define TURBOQ_KV_DIM 128 + +static inline uint64_t splitmix64_next(uint64_t * state) { + uint64_t z = (*state += 0x9e3779b97f4a7c15ULL); + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ULL; + z = (z ^ (z >> 27)) * 0x94d049bb133111ebULL; + return z ^ (z >> 31); +} + +static void turboq_generate_gaussian(float * out, int64_t n, uint64_t seed) { + uint64_t state = seed; + int64_t i = 0; + for (; i + 1 < n; i += 2) { + // Generate two uniform (0,1) variates + double u1 = ((double)(splitmix64_next(&state) >> 11) + 0.5) / (double)(1ULL << 53); + double u2 = ((double)(splitmix64_next(&state) >> 11) + 0.5) / (double)(1ULL << 53); + double r = sqrt(-2.0 * log(u1)); + double th = 2.0 * 3.14159265358979323846 * u2; + out[i] = (float)(r * cos(th)); + out[i + 1] = (float)(r * sin(th)); + } + if (i < n) { + double u1 = ((double)(splitmix64_next(&state) >> 11) + 0.5) / (double)(1ULL << 53); + double u2 = ((double)(splitmix64_next(&state) >> 11) + 0.5) / (double)(1ULL << 53); + double r = sqrt(-2.0 * log(u1)); + double th = 2.0 * 3.14159265358979323846 * u2; + out[i] = (float)(r * cos(th)); + } +} + +// --------------------------------------------------------------------------- +// Householder QR decomposition (in-place, no LAPACK dependency) +// +// Input: A[d*d] stored column-major (A[i + j*d] = A_{i,j}) +// Output: Q[d*d] column-major orthogonal matrix, with Haar sign correction +// +// Uses Householder reflections: Q = H_1 * H_2 * ... * H_d where +// H_k = I - 2 * v_k * v_k^T / (v_k^T * v_k) +// --------------------------------------------------------------------------- + +// Compute Q from Householder QR of column-major matrix A[d×d]. +// A is modified in-place (becomes R on upper triangle, v below diagonal). +// Q is written to Q_out[d×d] column-major. +// Applies Haar sign correction: Q[:,j] *= sign(R[j,j]) so that Q is +// uniformly distributed on O(d) (Haar measure). +static void turboq_householder_qr(float * A, float * Q_out, int64_t d) { + float * tau = (float *)malloc(d * sizeof(float)); + // Store sign(R[k,k]) = -sign(alpha_k) for Haar correction + float * r_sign = (float *)malloc(d * sizeof(float)); + + for (int64_t k = 0; k < d; k++) { + // Compute norm of A[k:d, k] + float norm_sq = 0.0f; + for (int64_t i = k; i < d; i++) { + float val = A[i + k * d]; + norm_sq += val * val; + } + float norm = sqrtf(norm_sq); + + // Choose sign to avoid cancellation + float alpha = A[k + k * d]; + float sign_alpha = (alpha >= 0.0f) ? 1.0f : -1.0f; + float u1 = alpha + sign_alpha * norm; + + // R[k,k] = -sign(alpha) * norm, so sign(R[k,k]) = -sign(alpha) + r_sign[k] = -sign_alpha; + + // Compute tau = 2 / (v^T v) + float vtv = u1 * u1 + (norm_sq - alpha * alpha); + if (vtv < 1e-30f) { + tau[k] = 0.0f; + continue; + } + tau[k] = 2.0f / vtv; + + // Store v in A[k:d, k] + A[k + k * d] = u1; + + // Apply H_k to remaining columns A[k:d, k+1:d] + for (int64_t j = k + 1; j < d; j++) { + float dot = 0.0f; + dot += u1 * A[k + j * d]; + for (int64_t i = k + 1; i < d; i++) { + dot += A[i + k * d] * A[i + j * d]; + } + dot *= tau[k]; + A[k + j * d] -= dot * u1; + for (int64_t i = k + 1; i < d; i++) { + A[i + j * d] -= dot * A[i + k * d]; + } + } + } + + // Build Q by back-accumulation: Q = H_1 * H_2 * ... * H_{d-1} + memset(Q_out, 0, d * d * sizeof(float)); + for (int64_t i = 0; i < d; i++) { + Q_out[i + i * d] = 1.0f; + } + + for (int64_t k = d - 1; k >= 0; k--) { + if (tau[k] == 0.0f) continue; + float u1 = A[k + k * d]; + for (int64_t j = 0; j < d; j++) { + float dot = 0.0f; + dot += u1 * Q_out[k + j * d]; + for (int64_t i = k + 1; i < d; i++) { + dot += A[i + k * d] * Q_out[i + j * d]; + } + dot *= tau[k]; + Q_out[k + j * d] -= dot * u1; + for (int64_t i = k + 1; i < d; i++) { + Q_out[i + j * d] -= dot * A[i + k * d]; + } + } + } + + // Haar sign correction: Q[:,j] *= sign(R[j,j]) + // This ensures Q is uniformly distributed on O(d), not just SO(d). + // Reference: Mezzadri (2007), "How to Generate Random Matrices from the Classical Compact Groups" + for (int64_t j = 0; j < d; j++) { + if (r_sign[j] < 0.0f) { + for (int64_t i = 0; i < d; i++) { + Q_out[i + j * d] = -Q_out[i + j * d]; + } + } + } + + free(tau); + free(r_sign); +} + +// --------------------------------------------------------------------------- +// Rotation matrix cache +// +// For a given (dimension, seed) pair, generate and cache the d×d orthogonal Q. +// The cache is thread-local to avoid locks. In practice, all rows of a weight +// matrix share the same dimension, so the cache hit rate is ~100%. +// --------------------------------------------------------------------------- + +static TURBOQ_TLS float * tl_Q = NULL; +static TURBOQ_TLS float * tl_Q_row = NULL; +static TURBOQ_TLS int64_t tl_Q_dim = 0; +static TURBOQ_TLS uint64_t tl_Q_seed = 0; + +static const float * turboq_get_rotation(int64_t d, uint64_t seed) { + if (tl_Q != NULL && tl_Q_dim == d && tl_Q_seed == seed) { + return tl_Q; + } + // Regenerate + free(tl_Q); + free(tl_Q_row); + tl_Q = (float *)malloc(d * d * sizeof(float)); + tl_Q_row = (float *)malloc(d * d * sizeof(float)); + tl_Q_dim = d; + tl_Q_seed = seed; + + // Generate d×d Gaussian random matrix (column-major) + float * A = (float *)malloc(d * d * sizeof(float)); + turboq_generate_gaussian(A, d * d, seed); + + // Compute QR, store Q in tl_Q + turboq_householder_qr(A, tl_Q, d); + + for (int64_t i = 0; i < d; ++i) { + for (int64_t j = 0; j < d; ++j) { + tl_Q_row[i * d + j] = tl_Q[i + j * d]; + } + } + + free(A); + return tl_Q; +} + +static const float * turboq_get_rotation_row(int64_t d, uint64_t seed) { + turboq_get_rotation(d, seed); + return tl_Q_row; +} + +// y[k] = dot(M + k*d, x). matvec_row takes M row-major, matvec_t takes M +// column-major (column k is contiguous) -- same contiguous dot product either +// way. Defined per-TU: scalar in ggml-base, ggml_vec_dot_f32 in ggml-cpu. +static void matvec_row(float * y, const float * M, const float * x, int64_t d); +static void matvec_t (float * y, const float * M, const float * x, int64_t d); + +// The rotation matrix is a global parameter (same for all vectors), per the paper. +// This seed is used to deterministically generate both Q and S matrices. +static inline uint64_t turboq_seed_from_row(int64_t row_idx) { + (void)row_idx; + return 0x517cc1b727220a95ULL; +} + +static inline float turboq_block_scale_up(void) { + return sqrtf((float) QK_K); +} + +static inline float turboq_block_scale_down(void) { + return 1.0f / turboq_block_scale_up(); +} + +static void turboq_rotate_block_forward(float * y, const float * x, uint64_t seed) { + const float * Q = turboq_get_rotation_row(TURBOQ_KV_DIM, seed); + + for (int64_t i = 0; i < QK_K; i += TURBOQ_KV_DIM) { + matvec_row(y + i, Q, x + i, TURBOQ_KV_DIM); + } +} + +static void turboq_rotate_block_inverse(float * x, const float * y, uint64_t seed) { + const float * Q = turboq_get_rotation(TURBOQ_KV_DIM, seed); + + for (int64_t i = 0; i < QK_K; i += TURBOQ_KV_DIM) { + matvec_t(x + i, Q, y + i, TURBOQ_KV_DIM); + } +} + +// --------------------------------------------------------------------------- +// Scratch buffer (thread-local, for temporary vectors) +// --------------------------------------------------------------------------- + +static TURBOQ_TLS float * tl_buf = NULL; +static TURBOQ_TLS int64_t tl_buf_size = 0; + +static float * turboq_get_scratch(int64_t n) { + if (n > tl_buf_size) { + free(tl_buf); + tl_buf = (float *)malloc(n * sizeof(float)); + tl_buf_size = n; + } + return tl_buf; +} + +// Second scratch buffer (needed when two temp vectors are required simultaneously, +// e.g. rotated-domain values + original-domain result in dequant) +static TURBOQ_TLS float * tl_buf2 = NULL; +static TURBOQ_TLS int64_t tl_buf2_size = 0; + +static float * turboq_get_scratch2(int64_t n) { + if (n > tl_buf2_size) { + free(tl_buf2); + tl_buf2 = (float *)malloc(n * sizeof(float)); + tl_buf2_size = n; + } + return tl_buf2; +} + +// --------------------------------------------------------------------------- +// Scalar codebook quantization +// --------------------------------------------------------------------------- + +static inline uint8_t quantize_scalar(float val, const float * boundaries, int n_boundaries) { + for (int i = 0; i < n_boundaries; i++) { + if (val < boundaries[i]) { + return (uint8_t)i; + } + } + return (uint8_t)n_boundaries; +} + +static inline uint8_t quantize_scalar_3bit(float val) { + return quantize_scalar(val, turboq_boundaries_3bit, 7); +} + +static inline uint8_t quantize_scalar_4bit(float val) { + return quantize_scalar(val, turboq_boundaries_4bit, 15); +} + +// --------------------------------------------------------------------------- +// 3-bit packing/unpacking +// --------------------------------------------------------------------------- + +static void pack_3bit(uint8_t * dst, const uint8_t * indices, int64_t n) { + int64_t full_groups = n / 8; + for (int64_t g = 0; g < full_groups; g++) { + const uint8_t * idx = indices + g * 8; + uint32_t bits = 0; + for (int j = 0; j < 8; j++) { + bits |= ((uint32_t)(idx[j] & 0x7)) << (j * 3); + } + dst[g * 3 + 0] = (uint8_t)(bits & 0xFF); + dst[g * 3 + 1] = (uint8_t)((bits >> 8) & 0xFF); + dst[g * 3 + 2] = (uint8_t)((bits >> 16) & 0xFF); + } +} + +static void unpack_3bit(uint8_t * indices, const uint8_t * src, int64_t n) { + int64_t full_groups = n / 8; + for (int64_t g = 0; g < full_groups; g++) { + uint32_t bits = (uint32_t)src[g * 3 + 0] + | ((uint32_t)src[g * 3 + 1] << 8) + | ((uint32_t)src[g * 3 + 2] << 16); + for (int j = 0; j < 8; j++) { + indices[g * 8 + j] = (uint8_t)((bits >> (j * 3)) & 0x7); + } + } +} + +// --------------------------------------------------------------------------- +// TBQ3_0: TurboQuant 3-bit +// --------------------------------------------------------------------------- + +static inline void turboq_quantize_tbq3_0(const float * GGML_RESTRICT x, block_tbq3_0 * GGML_RESTRICT y, int64_t k) { + assert(k % QK_K == 0); + const int64_t nb = k / QK_K; + float * unit = turboq_get_scratch(QK_K); + float * rotated = turboq_get_scratch2(QK_K); + const uint64_t seed = turboq_seed_from_row(0); + const float scale_up = turboq_block_scale_up(); + uint8_t indices[QK_K]; + + for (int64_t b = 0; b < nb; b++) { + const float * xb = x + b * QK_K; + + float norm_sq = 0.0f; + for (int64_t j = 0; j < QK_K; ++j) { + norm_sq += xb[j] * xb[j]; + } + + float norm = sqrtf(norm_sq); + if (norm < 1e-10f) { + norm = 1e-10f; + } + + for (int64_t j = 0; j < QK_K; ++j) { + unit[j] = xb[j] / norm; + } + + turboq_rotate_block_forward(rotated, unit, seed); + + for (int64_t j = 0; j < QK_K; j++) { + float val = rotated[j] * scale_up; + indices[j] = quantize_scalar_3bit(val); + } + pack_3bit(y[b].qs, indices, QK_K); + y[b].d = GGML_FP32_TO_FP16(norm); + } +} + +static inline void turboq_dequantize_tbq3_0(const block_tbq3_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_K == 0); + const int64_t nb = k / QK_K; + float * rotated = turboq_get_scratch(QK_K); + float * unit_approx = turboq_get_scratch2(QK_K); + const uint64_t seed = turboq_seed_from_row(0); + const float scale_down = turboq_block_scale_down(); + uint8_t indices[QK_K]; + + for (int64_t b = 0; b < nb; b++) { + const float norm = GGML_FP16_TO_FP32(x[b].d); + + unpack_3bit(indices, x[b].qs, QK_K); + for (int64_t j = 0; j < QK_K; j++) { + rotated[j] = turboq_codebook_3bit[indices[j]] * scale_down; + } + + turboq_rotate_block_inverse(unit_approx, rotated, seed); + + for (int64_t j = 0; j < QK_K; ++j) { + y[b * QK_K + j] = unit_approx[j] * norm; + } + } +} + +// --------------------------------------------------------------------------- +// TBQ4_0: TurboQuant 4-bit +// --------------------------------------------------------------------------- + +static inline void turboq_quantize_tbq4_0(const float * GGML_RESTRICT x, block_tbq4_0 * GGML_RESTRICT y, int64_t k) { + assert(k % QK_K == 0); + const int64_t nb = k / QK_K; + float * unit = turboq_get_scratch(QK_K); + float * rotated = turboq_get_scratch2(QK_K); + const uint64_t seed = turboq_seed_from_row(0); + const float scale_up = turboq_block_scale_up(); + + for (int64_t b = 0; b < nb; b++) { + const float * xb = x + b * QK_K; + + float norm_sq = 0.0f; + for (int64_t j = 0; j < QK_K; ++j) { + norm_sq += xb[j] * xb[j]; + } + + float norm = sqrtf(norm_sq); + if (norm < 1e-10f) { + norm = 1e-10f; + } + + for (int64_t j = 0; j < QK_K; ++j) { + unit[j] = xb[j] / norm; + } + + turboq_rotate_block_forward(rotated, unit, seed); + + memset(y[b].qs, 0, sizeof(y[b].qs)); + for (int64_t j = 0; j < QK_K; j++) { + float val = rotated[j] * scale_up; + uint8_t idx = quantize_scalar_4bit(val); + if (j % 2 == 0) { + y[b].qs[j / 2] = idx; + } else { + y[b].qs[j / 2] |= (idx << 4); + } + } + y[b].d = GGML_FP32_TO_FP16(norm); + } +} + +static inline void turboq_dequantize_tbq4_0(const block_tbq4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_K == 0); + const int64_t nb = k / QK_K; + float * rotated = turboq_get_scratch(QK_K); + float * unit_approx = turboq_get_scratch2(QK_K); + const uint64_t seed = turboq_seed_from_row(0); + const float scale_down = turboq_block_scale_down(); + + for (int64_t b = 0; b < nb; b++) { + const float norm = GGML_FP16_TO_FP32(x[b].d); + + for (int64_t j = 0; j < QK_K; j++) { + uint8_t idx; + if (j % 2 == 0) { + idx = x[b].qs[j / 2] & 0x0F; + } else { + idx = (x[b].qs[j / 2] >> 4) & 0x0F; + } + rotated[j] = turboq_codebook_4bit[idx] * scale_down; + } + + turboq_rotate_block_inverse(unit_approx, rotated, seed); + + for (int64_t j = 0; j < QK_K; ++j) { + y[b * QK_K + j] = unit_approx[j] * norm; + } + } +} diff --git a/ggml/src/ggml-turboq.c b/ggml/src/ggml-turboq.c index 58d260a214b7..d9ad1c463a3b 100644 --- a/ggml/src/ggml-turboq.c +++ b/ggml/src/ggml-turboq.c @@ -1,580 +1,34 @@ -// TurboQuant reference helpers for the CPU path. +// TurboQuant scalar reference. The arch-built SIMD dequant lives in ggml-cpu/turboq.c. -#define GGML_COMMON_IMPL_C -#include "ggml-common.h" - -#include "ggml-turboq.h" -#include "ggml-turboq-tables.h" #include "ggml-quants.h" -#include "ggml-impl.h" -#include "ggml.h" - -#include -#include -#include -#include - -#if defined(__AVX2__) -#include -#endif - -#if defined(__GNUC__) || defined(__clang__) -#define TURBOQ_TLS __thread -#elif defined(_MSC_VER) -#define TURBOQ_TLS __declspec(thread) -#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) -#define TURBOQ_TLS _Thread_local -#else -#define TURBOQ_TLS -#endif - -static inline uint64_t splitmix64_next(uint64_t * state) { - uint64_t z = (*state += 0x9e3779b97f4a7c15ULL); - z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9ULL; - z = (z ^ (z >> 27)) * 0x94d049bb133111ebULL; - return z ^ (z >> 31); -} - -static void turboq_generate_gaussian(float * out, int64_t n, uint64_t seed) { - uint64_t state = seed; - int64_t i = 0; - for (; i + 1 < n; i += 2) { - // Generate two uniform (0,1) variates - double u1 = ((double)(splitmix64_next(&state) >> 11) + 0.5) / (double)(1ULL << 53); - double u2 = ((double)(splitmix64_next(&state) >> 11) + 0.5) / (double)(1ULL << 53); - double r = sqrt(-2.0 * log(u1)); - double th = 2.0 * 3.14159265358979323846 * u2; - out[i] = (float)(r * cos(th)); - out[i + 1] = (float)(r * sin(th)); - } - if (i < n) { - double u1 = ((double)(splitmix64_next(&state) >> 11) + 0.5) / (double)(1ULL << 53); - double u2 = ((double)(splitmix64_next(&state) >> 11) + 0.5) / (double)(1ULL << 53); - double r = sqrt(-2.0 * log(u1)); - double th = 2.0 * 3.14159265358979323846 * u2; - out[i] = (float)(r * cos(th)); - } -} - -// --------------------------------------------------------------------------- -// Householder QR decomposition (in-place, no LAPACK dependency) -// -// Input: A[d*d] stored column-major (A[i + j*d] = A_{i,j}) -// Output: Q[d*d] column-major orthogonal matrix, with Haar sign correction -// -// Uses Householder reflections: Q = H_1 * H_2 * ... * H_d where -// H_k = I - 2 * v_k * v_k^T / (v_k^T * v_k) -// --------------------------------------------------------------------------- - -// Compute Q from Householder QR of column-major matrix A[d×d]. -// A is modified in-place (becomes R on upper triangle, v below diagonal). -// Q is written to Q_out[d×d] column-major. -// Applies Haar sign correction: Q[:,j] *= sign(R[j,j]) so that Q is -// uniformly distributed on O(d) (Haar measure). -static void turboq_householder_qr(float * A, float * Q_out, int64_t d) { - float * tau = (float *)malloc(d * sizeof(float)); - // Store sign(R[k,k]) = -sign(alpha_k) for Haar correction - float * r_sign = (float *)malloc(d * sizeof(float)); - - for (int64_t k = 0; k < d; k++) { - // Compute norm of A[k:d, k] - float norm_sq = 0.0f; - for (int64_t i = k; i < d; i++) { - float val = A[i + k * d]; - norm_sq += val * val; - } - float norm = sqrtf(norm_sq); - - // Choose sign to avoid cancellation - float alpha = A[k + k * d]; - float sign_alpha = (alpha >= 0.0f) ? 1.0f : -1.0f; - float u1 = alpha + sign_alpha * norm; - - // R[k,k] = -sign(alpha) * norm, so sign(R[k,k]) = -sign(alpha) - r_sign[k] = -sign_alpha; - - // Compute tau = 2 / (v^T v) - float vtv = u1 * u1 + (norm_sq - alpha * alpha); - if (vtv < 1e-30f) { - tau[k] = 0.0f; - continue; - } - tau[k] = 2.0f / vtv; - - // Store v in A[k:d, k] - A[k + k * d] = u1; - - // Apply H_k to remaining columns A[k:d, k+1:d] - for (int64_t j = k + 1; j < d; j++) { - float dot = 0.0f; - dot += u1 * A[k + j * d]; - for (int64_t i = k + 1; i < d; i++) { - dot += A[i + k * d] * A[i + j * d]; - } - dot *= tau[k]; - A[k + j * d] -= dot * u1; - for (int64_t i = k + 1; i < d; i++) { - A[i + j * d] -= dot * A[i + k * d]; - } - } - } - - // Build Q by back-accumulation: Q = H_1 * H_2 * ... * H_{d-1} - memset(Q_out, 0, d * d * sizeof(float)); - for (int64_t i = 0; i < d; i++) { - Q_out[i + i * d] = 1.0f; - } - - for (int64_t k = d - 1; k >= 0; k--) { - if (tau[k] == 0.0f) continue; - float u1 = A[k + k * d]; - for (int64_t j = 0; j < d; j++) { - float dot = 0.0f; - dot += u1 * Q_out[k + j * d]; - for (int64_t i = k + 1; i < d; i++) { - dot += A[i + k * d] * Q_out[i + j * d]; - } - dot *= tau[k]; - Q_out[k + j * d] -= dot * u1; - for (int64_t i = k + 1; i < d; i++) { - Q_out[i + j * d] -= dot * A[i + k * d]; - } - } - } - - // Haar sign correction: Q[:,j] *= sign(R[j,j]) - // This ensures Q is uniformly distributed on O(d), not just SO(d). - // Reference: Mezzadri (2007), "How to Generate Random Matrices from the Classical Compact Groups" - for (int64_t j = 0; j < d; j++) { - if (r_sign[j] < 0.0f) { - for (int64_t i = 0; i < d; i++) { - Q_out[i + j * d] = -Q_out[i + j * d]; - } - } - } - - free(tau); - free(r_sign); -} - -// --------------------------------------------------------------------------- -// Rotation matrix cache -// -// For a given (dimension, seed) pair, generate and cache the d×d orthogonal Q. -// The cache is thread-local to avoid locks. In practice, all rows of a weight -// matrix share the same dimension, so the cache hit rate is ~100%. -// --------------------------------------------------------------------------- - -static TURBOQ_TLS float * tl_Q = NULL; -static TURBOQ_TLS float * tl_Q_row = NULL; -static TURBOQ_TLS int64_t tl_Q_dim = 0; -static TURBOQ_TLS uint64_t tl_Q_seed = 0; - -static const float * turboq_get_rotation(int64_t d, uint64_t seed) { - if (tl_Q != NULL && tl_Q_dim == d && tl_Q_seed == seed) { - return tl_Q; - } - // Regenerate - free(tl_Q); - free(tl_Q_row); - tl_Q = (float *)malloc(d * d * sizeof(float)); - tl_Q_row = (float *)malloc(d * d * sizeof(float)); - tl_Q_dim = d; - tl_Q_seed = seed; - - // Generate d×d Gaussian random matrix (column-major) - float * A = (float *)malloc(d * d * sizeof(float)); - turboq_generate_gaussian(A, d * d, seed); - - // Compute QR, store Q in tl_Q - turboq_householder_qr(A, tl_Q, d); - - for (int64_t i = 0; i < d; ++i) { - for (int64_t j = 0; j < d; ++j) { - tl_Q_row[i * d + j] = tl_Q[i + j * d]; - } - } - - free(A); - return tl_Q; -} - -static const float * turboq_get_rotation_row(int64_t d, uint64_t seed) { - turboq_get_rotation(d, seed); - return tl_Q_row; -} - -// --------------------------------------------------------------------------- -// Projection matrix cache (for Q_prod QJL stage) -// -// S is a d×d random Gaussian matrix (NOT orthogonalized), used for QJL: -// qjl_signs = sign(S · residual) -// dequant: sqrt(pi/2)/d · gamma · S^T · signs -// Uses a different seed stream from the rotation matrix Q. -// --------------------------------------------------------------------------- - -static TURBOQ_TLS float * tl_S = NULL; -static TURBOQ_TLS float * tl_S_row = NULL; -static TURBOQ_TLS int64_t tl_S_dim = 0; -static TURBOQ_TLS uint64_t tl_S_seed = 0; - -static const float * turboq_get_projection(int64_t d, uint64_t seed) { - // Use a different seed stream for S vs Q - uint64_t s_seed = seed ^ 0x1234567890abcdefULL; - if (tl_S != NULL && tl_S_dim == d && tl_S_seed == s_seed) { - return tl_S; - } - free(tl_S); - free(tl_S_row); - tl_S = (float *)malloc(d * d * sizeof(float)); - tl_S_row = (float *)malloc(d * d * sizeof(float)); - tl_S_dim = d; - tl_S_seed = s_seed; - - // Generate d×d Gaussian random matrix (column-major), no QR - turboq_generate_gaussian(tl_S, d * d, s_seed); - - for (int64_t i = 0; i < d; ++i) { - for (int64_t j = 0; j < d; ++j) { - tl_S_row[i * d + j] = tl_S[i + j * d]; - } - } - - return tl_S; -} - -static const float * turboq_get_projection_row(int64_t d, uint64_t seed) { - turboq_get_projection(d, seed); - return tl_S_row; -} - -// --------------------------------------------------------------------------- -// Dense matrix-vector multiply: y = M * x (M is d×d column-major) -// --------------------------------------------------------------------------- - -static void matvec(float * y, const float * M, const float * x, int64_t d) { - for (int64_t i = 0; i < d; i++) { - float sum = 0.0f; - for (int64_t j = 0; j < d; j++) { - sum += M[i + j * d] * x[j]; // M[i,j] = M[i + j*d] (column-major) - } - y[i] = sum; - } -} - -#if defined(__AVX2__) -static inline float turboq_hsum_avx(__m256 v) { - __m128 lo = _mm256_castps256_ps128(v); - __m128 hi = _mm256_extractf128_ps(v, 1); - __m128 sum = _mm_add_ps(lo, hi); - sum = _mm_hadd_ps(sum, sum); - sum = _mm_hadd_ps(sum, sum); - return _mm_cvtss_f32(sum); -} -#endif +#include "ggml-turboq-impl.h" static void matvec_row(float * y, const float * M, const float * x, int64_t d) { for (int64_t i = 0; i < d; ++i) { - const float * row = M + i * d; float sum = 0.0f; - int64_t j = 0; -#if defined(__AVX2__) - __m256 acc = _mm256_setzero_ps(); - for (; j + 7 < d; j += 8) { - const __m256 mv = _mm256_loadu_ps(row + j); - const __m256 xv = _mm256_loadu_ps(x + j); -#if defined(__FMA__) - acc = _mm256_fmadd_ps(mv, xv, acc); -#else - acc = _mm256_add_ps(acc, _mm256_mul_ps(mv, xv)); -#endif - } - sum += turboq_hsum_avx(acc); -#endif - for (; j < d; ++j) { - sum += row[j] * x[j]; + for (int64_t j = 0; j < d; ++j) { + sum += M[i * d + j] * x[j]; } y[i] = sum; } } -// --------------------------------------------------------------------------- -// Dense matrix-transpose-vector multiply: y = M^T * x (M is d×d column-major) -// --------------------------------------------------------------------------- - static void matvec_t(float * y, const float * M, const float * x, int64_t d) { for (int64_t j = 0; j < d; j++) { - const float * col = M + j * d; float sum = 0.0f; - int64_t i = 0; -#if defined(__AVX2__) - __m256 acc = _mm256_setzero_ps(); - for (; i + 7 < d; i += 8) { - const __m256 mv = _mm256_loadu_ps(col + i); - const __m256 xv = _mm256_loadu_ps(x + i); -#if defined(__FMA__) - acc = _mm256_fmadd_ps(mv, xv, acc); -#else - acc = _mm256_add_ps(acc, _mm256_mul_ps(mv, xv)); -#endif - } - sum += turboq_hsum_avx(acc); -#endif - for (; i < d; ++i) { - sum += col[i] * x[i]; // M^T[j,i] = M[i,j] = M[i + j*d] + for (int64_t i = 0; i < d; ++i) { + sum += M[j * d + i] * x[i]; } y[j] = sum; } } -// --------------------------------------------------------------------------- -// Public API (kept for compatibility, now wraps dense rotation) -// --------------------------------------------------------------------------- - -// The rotation matrix is a global parameter (same for all vectors), per the paper. -// This seed is used to deterministically generate both Q and S matrices. -uint64_t turboq_seed_from_row(int64_t row_idx) { - (void)row_idx; - return 0x517cc1b727220a95ULL; -} - -// Forward rotation: y = Q · x (paper Algorithm 1, line 5: y <- Pi . x) -void turboq_rotate_forward(float * y, const float * x, int64_t d, uint64_t seed) { - const float * Q = turboq_get_rotation_row(d, seed); - matvec_row(y, Q, x, d); -} - -// Inverse rotation: x = Q^T · y (paper Algorithm 1, line 10: x_tilde <- Pi^T . y_tilde) -void turboq_rotate_inverse(float * x, const float * y, int64_t d, uint64_t seed) { - const float * Q = turboq_get_rotation(d, seed); - matvec_t(x, Q, y, d); -} - -// --------------------------------------------------------------------------- -// Scratch buffer (thread-local, for temporary vectors) -// --------------------------------------------------------------------------- - -static TURBOQ_TLS float * tl_buf = NULL; -static TURBOQ_TLS int64_t tl_buf_size = 0; - -static float * turboq_get_scratch(int64_t n) { - if (n > tl_buf_size) { - free(tl_buf); - tl_buf = (float *)malloc(n * sizeof(float)); - tl_buf_size = n; - } - return tl_buf; -} - -// Second scratch buffer (needed when two temp vectors are required simultaneously, -// e.g. rotated-domain values + original-domain result in dequant) -static TURBOQ_TLS float * tl_buf2 = NULL; -static TURBOQ_TLS int64_t tl_buf2_size = 0; - -static float * turboq_get_scratch2(int64_t n) { - if (n > tl_buf2_size) { - free(tl_buf2); - tl_buf2 = (float *)malloc(n * sizeof(float)); - tl_buf2_size = n; - } - return tl_buf2; -} - -// Third scratch buffer (needed by Q_prod dequant which requires three simultaneous vectors: -// mse_rot, signs_f, and mse_unit) -static TURBOQ_TLS float * tl_buf3 = NULL; -static TURBOQ_TLS int64_t tl_buf3_size = 0; - -static float * turboq_get_scratch3(int64_t n) { - if (n > tl_buf3_size) { - free(tl_buf3); - tl_buf3 = (float *)malloc(n * sizeof(float)); - tl_buf3_size = n; - } - return tl_buf3; -} - -#define TURBOQ_KV_DIM 128 - -static inline float turboq_block_scale_up(void) { - return sqrtf((float) QK_K); -} - -static inline float turboq_block_scale_down(void) { - return 1.0f / turboq_block_scale_up(); -} - -static void turboq_rotate_block_forward(float * y, const float * x, uint64_t seed) { - const float * Q = turboq_get_rotation_row(TURBOQ_KV_DIM, seed); - - for (int64_t i = 0; i < QK_K; i += TURBOQ_KV_DIM) { - matvec_row(y + i, Q, x + i, TURBOQ_KV_DIM); - } -} - -static void turboq_rotate_block_inverse(float * x, const float * y, uint64_t seed) { - const float * Q = turboq_get_rotation(TURBOQ_KV_DIM, seed); - - for (int64_t i = 0; i < QK_K; i += TURBOQ_KV_DIM) { - matvec_t(x + i, Q, y + i, TURBOQ_KV_DIM); - } -} - -static void turboq_project_block(float * y, const float * x, uint64_t seed) { - const float * S = turboq_get_projection_row(TURBOQ_KV_DIM, seed); - - for (int64_t i = 0; i < QK_K; i += TURBOQ_KV_DIM) { - matvec_row(y + i, S, x + i, TURBOQ_KV_DIM); - } -} - -static void turboq_project_block_inverse(float * x, const float * y, uint64_t seed) { - const float * S = turboq_get_projection(TURBOQ_KV_DIM, seed); - - for (int64_t i = 0; i < QK_K; i += TURBOQ_KV_DIM) { - matvec_t(x + i, S, y + i, TURBOQ_KV_DIM); - } -} - -static void turboq_rotate_qk_forward(float * y, const float * x, uint64_t seed) { - const float * Q = turboq_get_rotation_row(QK_K, seed); - matvec_row(y, Q, x, QK_K); -} - -static void turboq_rotate_qk_inverse(float * x, const float * y, uint64_t seed) { - const float * Q = turboq_get_rotation(QK_K, seed); - matvec_t(x, Q, y, QK_K); -} - -static void turboq_project_qk(float * y, const float * x, uint64_t seed) { - const float * S = turboq_get_projection_row(QK_K, seed); - matvec_row(y, S, x, QK_K); -} - -static void turboq_project_qk_inverse(float * x, const float * y, uint64_t seed) { - const float * S = turboq_get_projection(QK_K, seed); - matvec_t(x, S, y, QK_K); -} - -// --------------------------------------------------------------------------- -// Scalar codebook quantization -// --------------------------------------------------------------------------- - -static inline uint8_t quantize_scalar(float val, const float * boundaries, int n_boundaries) { - for (int i = 0; i < n_boundaries; i++) { - if (val < boundaries[i]) { - return (uint8_t)i; - } - } - return (uint8_t)n_boundaries; -} - -static inline uint8_t quantize_scalar_3bit(float val) { - return quantize_scalar(val, turboq_boundaries_3bit, 7); -} - -static inline uint8_t quantize_scalar_2bit(float val) { - return quantize_scalar(val, turboq_boundaries_2bit, 3); -} - -static inline uint8_t quantize_scalar_4bit(float val) { - return quantize_scalar(val, turboq_boundaries_4bit, 15); -} - -// --------------------------------------------------------------------------- -// 3-bit packing/unpacking -// --------------------------------------------------------------------------- - -static void pack_3bit(uint8_t * dst, const uint8_t * indices, int64_t n) { - int64_t full_groups = n / 8; - for (int64_t g = 0; g < full_groups; g++) { - const uint8_t * idx = indices + g * 8; - uint32_t bits = 0; - for (int j = 0; j < 8; j++) { - bits |= ((uint32_t)(idx[j] & 0x7)) << (j * 3); - } - dst[g * 3 + 0] = (uint8_t)(bits & 0xFF); - dst[g * 3 + 1] = (uint8_t)((bits >> 8) & 0xFF); - dst[g * 3 + 2] = (uint8_t)((bits >> 16) & 0xFF); - } -} - -static void unpack_3bit(uint8_t * indices, const uint8_t * src, int64_t n) { - int64_t full_groups = n / 8; - for (int64_t g = 0; g < full_groups; g++) { - uint32_t bits = (uint32_t)src[g * 3 + 0] - | ((uint32_t)src[g * 3 + 1] << 8) - | ((uint32_t)src[g * 3 + 2] << 16); - for (int j = 0; j < 8; j++) { - indices[g * 8 + j] = (uint8_t)((bits >> (j * 3)) & 0x7); - } - } -} - -// --------------------------------------------------------------------------- -// TBQ3_0: TurboQuant 3-bit -// --------------------------------------------------------------------------- - void quantize_row_tbq3_0_ref(const float * GGML_RESTRICT x, block_tbq3_0 * GGML_RESTRICT y, int64_t k) { - assert(k % QK_K == 0); - const int64_t nb = k / QK_K; - float * unit = turboq_get_scratch(QK_K); - float * rotated = turboq_get_scratch2(QK_K); - const uint64_t seed = turboq_seed_from_row(0); - const float scale_up = turboq_block_scale_up(); - uint8_t indices[QK_K]; - - for (int64_t b = 0; b < nb; b++) { - const float * xb = x + b * QK_K; - - float norm_sq = 0.0f; - for (int64_t j = 0; j < QK_K; ++j) { - norm_sq += xb[j] * xb[j]; - } - - float norm = sqrtf(norm_sq); - if (norm < 1e-10f) { - norm = 1e-10f; - } - - for (int64_t j = 0; j < QK_K; ++j) { - unit[j] = xb[j] / norm; - } - - turboq_rotate_block_forward(rotated, unit, seed); - - for (int64_t j = 0; j < QK_K; j++) { - float val = rotated[j] * scale_up; - indices[j] = quantize_scalar_3bit(val); - } - pack_3bit(y[b].qs, indices, QK_K); - y[b].d = GGML_FP32_TO_FP16(norm); - } + turboq_quantize_tbq3_0(x, y, k); } void dequantize_row_tbq3_0(const block_tbq3_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { - assert(k % QK_K == 0); - const int64_t nb = k / QK_K; - float * rotated = turboq_get_scratch(QK_K); - float * unit_approx = turboq_get_scratch2(QK_K); - const uint64_t seed = turboq_seed_from_row(0); - const float scale_down = turboq_block_scale_down(); - uint8_t indices[QK_K]; - - for (int64_t b = 0; b < nb; b++) { - const float norm = GGML_FP16_TO_FP32(x[b].d); - - unpack_3bit(indices, x[b].qs, QK_K); - for (int64_t j = 0; j < QK_K; j++) { - rotated[j] = turboq_codebook_3bit[indices[j]] * scale_down; - } - - turboq_rotate_block_inverse(unit_approx, rotated, seed); - - for (int64_t j = 0; j < QK_K; ++j) { - y[b * QK_K + j] = unit_approx[j] * norm; - } - } + turboq_dequantize_tbq3_0(x, y, k); } size_t quantize_tbq3_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) { @@ -592,78 +46,12 @@ size_t quantize_tbq3_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst return nrows * row_size; } -// --------------------------------------------------------------------------- -// TBQ4_0: TurboQuant 4-bit -// --------------------------------------------------------------------------- - void quantize_row_tbq4_0_ref(const float * GGML_RESTRICT x, block_tbq4_0 * GGML_RESTRICT y, int64_t k) { - assert(k % QK_K == 0); - const int64_t nb = k / QK_K; - float * unit = turboq_get_scratch(QK_K); - float * rotated = turboq_get_scratch2(QK_K); - const uint64_t seed = turboq_seed_from_row(0); - const float scale_up = turboq_block_scale_up(); - - for (int64_t b = 0; b < nb; b++) { - const float * xb = x + b * QK_K; - - float norm_sq = 0.0f; - for (int64_t j = 0; j < QK_K; ++j) { - norm_sq += xb[j] * xb[j]; - } - - float norm = sqrtf(norm_sq); - if (norm < 1e-10f) { - norm = 1e-10f; - } - - for (int64_t j = 0; j < QK_K; ++j) { - unit[j] = xb[j] / norm; - } - - turboq_rotate_block_forward(rotated, unit, seed); - - memset(y[b].qs, 0, sizeof(y[b].qs)); - for (int64_t j = 0; j < QK_K; j++) { - float val = rotated[j] * scale_up; - uint8_t idx = quantize_scalar_4bit(val); - if (j % 2 == 0) { - y[b].qs[j / 2] = idx; - } else { - y[b].qs[j / 2] |= (idx << 4); - } - } - y[b].d = GGML_FP32_TO_FP16(norm); - } + turboq_quantize_tbq4_0(x, y, k); } void dequantize_row_tbq4_0(const block_tbq4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { - assert(k % QK_K == 0); - const int64_t nb = k / QK_K; - float * rotated = turboq_get_scratch(QK_K); - float * unit_approx = turboq_get_scratch2(QK_K); - const uint64_t seed = turboq_seed_from_row(0); - const float scale_down = turboq_block_scale_down(); - - for (int64_t b = 0; b < nb; b++) { - const float norm = GGML_FP16_TO_FP32(x[b].d); - - for (int64_t j = 0; j < QK_K; j++) { - uint8_t idx; - if (j % 2 == 0) { - idx = x[b].qs[j / 2] & 0x0F; - } else { - idx = (x[b].qs[j / 2] >> 4) & 0x0F; - } - rotated[j] = turboq_codebook_4bit[idx] * scale_down; - } - - turboq_rotate_block_inverse(unit_approx, rotated, seed); - - for (int64_t j = 0; j < QK_K; ++j) { - y[b * QK_K + j] = unit_approx[j] * norm; - } - } + turboq_dequantize_tbq4_0(x, y, k); } size_t quantize_tbq4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix) { diff --git a/ggml/src/ggml-turboq.h b/ggml/src/ggml-turboq.h deleted file mode 100644 index e620e875e1fa..000000000000 --- a/ggml/src/ggml-turboq.h +++ /dev/null @@ -1,21 +0,0 @@ -#pragma once - -// TurboQuant helpers used by the CPU quantizers. - -#include "ggml.h" - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -void turboq_rotate_forward(float * y, const float * x, int64_t d, uint64_t seed); - -void turboq_rotate_inverse(float * x, const float * y, int64_t d, uint64_t seed); - -uint64_t turboq_seed_from_row(int64_t row_idx); - -#ifdef __cplusplus -} -#endif diff --git a/tests/test-quantize-fns.cpp b/tests/test-quantize-fns.cpp index cc50457bc123..458b6d352423 100644 --- a/tests/test-quantize-fns.cpp +++ b/tests/test-quantize-fns.cpp @@ -3,7 +3,6 @@ #include "ggml.h" #include "ggml-cpu.h" #include "../ggml/src/ggml-quants.h" -#include "../ggml/src/ggml-turboq.h" #include "../ggml/src/ggml-turboq-tables.h" #undef NDEBUG