elusznik · mverrilli · May 31, 2026
diff --git a/ggml/include/ggml-cpu.h b/ggml/include/ggml-cpu.h
@@ -118,6 +118,7 @@ extern "C" {
         ggml_vec_dot_t           vec_dot;
         enum ggml_type           vec_dot_type;
         int64_t                  nrows; // number of rows to process simultaneously
+        ggml_to_float_t          to_float; // optional SIMD override of the base to_float
     };
 
     GGML_BACKEND_API const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type);

diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt
@@ -206,7 +206,7 @@ add_library(ggml-base
             ggml-quants.c
             ggml-quants.h
             ggml-turboq.c
-            ggml-turboq.h
+            ggml-turboq-impl.h
             ggml-turboq-tables.h
             gguf.cpp)
 

diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt
@@ -35,6 +35,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
         ggml-cpu/hbm.h
         ggml-cpu/quants.c
         ggml-cpu/quants.h
+        ggml-cpu/turboq.c
         ggml-cpu/traits.cpp
         ggml-cpu/traits.h
         ggml-cpu/amx/amx.cpp

diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
@@ -392,12 +392,14 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
     },
     [GGML_TYPE_TBQ3_0] = {
         .from_float               = quantize_row_tbq3_0,
+        .to_float                 = (ggml_to_float_t) ggml_cpu_dequantize_row_tbq3_0,
         .vec_dot                  = ggml_vec_dot_tbq3_0_q8_K,
         .vec_dot_type             = GGML_TYPE_Q8_K,
         .nrows                    = 1,
     },
     [GGML_TYPE_TBQ4_0] = {
         .from_float               = quantize_row_tbq4_0,
+        .to_float                 = (ggml_to_float_t) ggml_cpu_dequantize_row_tbq4_0,
         .vec_dot                  = ggml_vec_dot_tbq4_0_q8_K,
         .vec_dot_type             = GGML_TYPE_Q8_K,
         .nrows                    = 1,

diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
@@ -14,6 +14,12 @@
 #include <type_traits>
 #include <vector>
 
+// prefer a SIMD dequant from the CPU traits, else fall back to the base to_float
+static ggml_to_float_t ggml_cpu_to_float(enum ggml_type type) {
+    ggml_to_float_t const cpu = ggml_get_type_traits_cpu(type)->to_float;
+    return cpu ? cpu : ggml_get_type_traits(type)->to_float;
+}
+
 // ggml_compute_forward_dup
 
 static void ggml_compute_forward_dup_same_cont(
@@ -511,7 +517,7 @@ static void ggml_compute_forward_dup_from_q(
     GGML_TENSOR_BINARY_OP_LOCALS
 
     const ggml_type type = src0->type;
-    ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;
+    ggml_to_float_t const dequantize_row_q = ggml_cpu_to_float(type);
 
     size_t qk = ggml_blck_size(type);
     const int64_t nr = ggml_nelements(src1) / qk;
@@ -642,7 +648,7 @@ static void ggml_compute_forward_add_q_f32(
 
     const ggml_type type = src0->type;
     const ggml_type dtype = dst->type;
-    ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;
+    ggml_to_float_t const dequantize_row_q = ggml_cpu_to_float(type);
     ggml_from_float_t const quantize_row_q = ggml_get_type_traits_cpu(dtype)->from_float;
 
     // we don't support permuted src0 or src1
@@ -988,7 +994,7 @@ static void ggml_compute_forward_add1_q_f32(
     GGML_TENSOR_UNARY_OP_LOCALS
 
     const ggml_type type = src0->type;
-    ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;
+    ggml_to_float_t const dequantize_row_q = ggml_cpu_to_float(type);
     ggml_from_float_t const quantize_row_q = ggml_get_type_traits_cpu(type)->from_float;
 
     // we don't support permuted src0
@@ -4303,7 +4309,7 @@ static void ggml_compute_forward_out_prod_q_f32(
     const int nth = params->nth;
 
     const ggml_type type = src0->type;
-    ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;
+    ggml_to_float_t const dequantize_row_q = ggml_cpu_to_float(type);
 
     GGML_ASSERT(ne02 == ne12);
     GGML_ASSERT(ne03 == ne13);
@@ -4727,7 +4733,7 @@ static void ggml_compute_forward_get_rows_q(
     const int64_t nr = ggml_nelements(src1);
 
     const ggml_type type = src0->type;
-    ggml_to_float_t const dequantize_row_q = ggml_get_type_traits(type)->to_float;
+    ggml_to_float_t const dequantize_row_q = ggml_cpu_to_float(type);
 
     assert(ne0  == nc);
     assert(ne02 == ne11);
@@ -8298,7 +8304,7 @@ static void ggml_compute_forward_flash_attn_ext_f16_one_chunk(
     ggml_type         const k_vec_dot_type = ggml_get_type_traits_cpu(k->type)->vec_dot_type;
     ggml_from_float_t const q_to_vec_dot   = ggml_get_type_traits_cpu(k_vec_dot_type)->from_float;
     ggml_vec_dot_t    const kq_vec_dot     = ggml_get_type_traits_cpu(k->type)->vec_dot;
-    ggml_to_float_t   const v_to_float     = ggml_get_type_traits(v->type)->to_float;
+    ggml_to_float_t   const v_to_float     = ggml_cpu_to_float(v->type);
 
     GGML_ASSERT((                            q_to_vec_dot) && "fattn: unsupported K-type");
     GGML_ASSERT((v->type == GGML_TYPE_F32 || v_to_float  ) && "fattn: unsupported V-type");

diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c
@@ -108,18 +108,6 @@ void quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy,
     quantize_row_tq2_0_ref(x, y, k);
 }
 
-void quantize_row_tbq3_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
-    assert(k % QK_K == 0);
-    block_tbq3_0 * GGML_RESTRICT y = vy;
-    quantize_row_tbq3_0_ref(x, y, k);
-}
-
-void quantize_row_tbq4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t k) {
-    assert(k % QK_K == 0);
-    block_tbq4_0 * GGML_RESTRICT y = vy;
-    quantize_row_tbq4_0_ref(x, y, k);
-}
-
 //===================================== Q8_K ==============================================
 
 void quantize_row_q8_K_generic(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {

diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h
@@ -35,6 +35,10 @@ void quantize_row_tq2_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, i
 void quantize_row_tbq3_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
 void quantize_row_tbq4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
 
+// SIMD dequant for the CPU backend (arch-built); wired via cpu traits to_float
+void ggml_cpu_dequantize_row_tbq3_0(const block_tbq3_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+void ggml_cpu_dequantize_row_tbq4_0(const block_tbq4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
+
 void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
 void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
 

diff --git a/ggml/src/ggml-cpu/turboq.c b/ggml/src/ggml-cpu/turboq.c
@@ -0,0 +1,34 @@
+// TurboQuant quantize/dequant. Built with arch flags, and the rotation matvec
+// reuses ggml_vec_dot_f32 so it vectorizes on every supported CPU arch.
+
+#include "quants.h"
+#include "vec.h"
+#include "ggml-turboq-impl.h"
+
+static void matvec_row(float * y, const float * M, const float * x, int64_t d) {
+    for (int64_t i = 0; i < d; ++i) {
+        ggml_vec_dot_f32(d, &y[i], 0, M + i * d, 0, x, 0, 1);
+    }
+}
+
+static void matvec_t(float * y, const float * M, const float * x, int64_t d) {
+    for (int64_t j = 0; j < d; ++j) {
+        ggml_vec_dot_f32(d, &y[j], 0, M + j * d, 0, x, 0, 1);
+    }
+}
+
+void quantize_row_tbq3_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+    turboq_quantize_tbq3_0(x, y, k);
+}
+
+void quantize_row_tbq4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k) {
+    turboq_quantize_tbq4_0(x, y, k);
+}
+
+void ggml_cpu_dequantize_row_tbq3_0(const block_tbq3_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+    turboq_dequantize_tbq3_0(x, y, k);
+}
+
+void ggml_cpu_dequantize_row_tbq4_0(const block_tbq4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
+    turboq_dequantize_tbq4_0(x, y, k);
+}