Skip to content

ggml: additive INT8/ternary types, fused CPU ops, and kernels for VibeASR (1/2) #447

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
0xShug0 merged 4 commits into from
Sep 4, 2026
Merged

ggml: additive INT8/ternary types, fused CPU ops, and kernels for VibeASR (1/2) #447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b22ebbd
ggml: add GGML_TYPE_I8_S / I2_S and five fused CPU INT8 ops
XsquirrelC Sep 4, 2026
91ed972
ggml-cpu: SIMD kernels for the I8_S fused ops
XsquirrelC Sep 4, 2026
abe1e7e
ggml-cpu: carry the I8_S in-band scale through dup/cont
XsquirrelC Sep 4, 2026
579c8f5
ggml-cpu: ternary I2_S matmul kernel
XsquirrelC Sep 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2279,6 +2279,20 @@ if (ENGINE_BUILD_TESTS)
COMMAND conv_lowering_matrix_test
)

add_engine_unittest(i8_s_fused_ops_test tests/unittests/test_i8_s_fused_ops.cpp)

add_test(
NAME i8_s_fused_ops_test
COMMAND i8_s_fused_ops_test
)

add_engine_unittest(i2_s_mul_mat_test tests/unittests/test_i2_s_mul_mat.cpp)

add_test(
NAME i2_s_mul_mat_test
COMMAND i2_s_mul_mat_test
)

add_engine_unittest(gguf_tensor_source_test tests/unittests/test_gguf_tensor_source.cpp)
target_include_directories(gguf_tensor_source_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests/unittests)

Expand Down
89 changes: 88 additions & 1 deletion external/ggml/include/ggml.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -429,7 +429,15 @@ extern "C" {
GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)
GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)
GGML_TYPE_Q1_0 = 41,
GGML_TYPE_COUNT = 42,
// INT8 / ternary with a single per-tensor scale, stored as one F32
// immediately after the payload rather than interleaved per block.
// Used by the VibeASR CPU pipeline; see ggml_mul_mat_add().
// NOTE: 36/37 are deliberately avoided even though VibeASR's own fork
// uses them -- they are retired IQ4_NL_4_4/4_8 slots and reusing an ID
// would silently misread GGUF files that still carry the old type.
GGML_TYPE_I8_S = 42,
GGML_TYPE_I2_S = 43,
GGML_TYPE_COUNT = 44,
};

// precision
Expand Down Expand Up @@ -588,6 +596,14 @@ extern "C" {

GGML_OP_GLU,
GGML_OP_CONVROT_LINEAR,

// VibeASR CPU INT8 pipeline. Appended at the tail so every existing
// op keeps its value -- GGML_OP_NAME and GGML_OP_SYMBOL are positional.
GGML_OP_ADD_SCALED,
GGML_OP_RMS_NORM_SCALED,
GGML_OP_MUL_MAT_ADD,
GGML_OP_MUL_MAT_ADD_RELU,
GGML_OP_IM2COL_ASYM,

GGML_OP_COUNT,
};
Expand Down Expand Up @@ -2472,6 +2488,77 @@ extern "C" {
struct ggml_tensor * bias,
int group_size);

// VibeASR CPU INT8 pipeline (GGML_TYPE_I8_S / GGML_TYPE_I2_S).
//
// Ported from https://github.com/microsoft/VibeASR.cpp, an end-to-end INT8
// ASR stack (INT8 VAE encoder, ternary-weight language model) built for CPU
// inference on edge devices.
//
// These are CPU-only, mirroring how ggml_convrot_linear and
// ggml_sage_attn2_i8 above are CUDA-only.
//
// Unlike those two, the scale is NOT a separate F32 src tensor. An I8_S
// activation's scale is recomputed from that activation at run time, and a
// ggml node has exactly one output, so the scale has to travel with the
// data: every I8_S/I2_S tensor stores one F32 immediately after its int8
// payload (see ggml_type_extra_bytes in ggml.c). Weight scales could have
// used the separate-tensor convention, but sharing one representation with
// activations keeps a single kernel per op instead of two.

// y = a*scale + b, fusing a ConvNeXt LayerScale into its residual add.
// a and b are I8_S and same-shape, scale is F32 per-channel broadcast on
// ne[0]. Output is I8_S and carries a freshly computed per-tensor scale.
GGML_API struct ggml_tensor * ggml_add_scaled(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * b,
struct ggml_tensor * scale);

// y = rms_norm(a) * scale, fused. a is I8_S, scale is F32 per-channel,
// output is I8_S. Equivalent to ggml_mul(ggml_rms_norm(a), scale) but
// avoids materializing the F32 intermediate.
GGML_API struct ggml_tensor * ggml_rms_norm_scaled(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * scale,
float eps);

// y = a*b + bias, with a I8_S weights and b I8_S activations. bias is F32
// and broadcasts on ne[0]. Output is I8_S. The ternary I2_S weights of the
// language model go through plain ggml_mul_mat, which has no bias to fuse.
//
// a with ne[1] == 1 and ne[2] > 1 selects a depthwise contraction: one
// length-ne[0] filter per channel, output indexed channel-major.
GGML_API struct ggml_tensor * ggml_mul_mat_add(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * b,
struct ggml_tensor * bias);

// As ggml_mul_mat_add, with ReLU folded into the epilogue.
GGML_API struct ggml_tensor * ggml_mul_mat_add_relu(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * b,
struct ggml_tensor * bias);

// im2col with independent left/right padding on the width axis. ggml_im2col
// only takes a single symmetric p0, so causal 1D convolutions otherwise need
// a separate ggml_pad_ext node and a full copy of the activation.
GGML_API struct ggml_tensor * ggml_im2col_asym(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * b,
int s0,
int s1,
int lp0,
int rp0,
int p1,
int d0,
int d1,
bool is_2D,
enum ggml_type dst_type);

// MINITTS_FLASH_BIAS_WRAPPER:
// Helper for models that already assemble a dense additive attention bias
// (for example relative-position scores). The helper expands an optional
Expand Down
99 changes: 79 additions & 20 deletions external/ggml/src/ggml-cpu/ggml-cpu.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1255,6 +1255,17 @@ void ggml_compute_forward_mul_mat(
return;
}

// Ternary weights own their activation quantization, so they cannot use the
// vec_dot_type path below: that quantizes src1 one row at a time into a
// fixed row_size and hands the kernel nothing but two row pointers, while
// I2_S needs the per-row activation scale and int8 row sum to survive into
// the epilogue. Branching here rather than adding an op keeps the language
// model graph on plain ggml_mul_mat.
if (src0->type == GGML_TYPE_I2_S) {
ggml_compute_forward_mul_mat_i2_s(params, dst);
return;
}

GGML_TENSOR_BINARY_OP_LOCALS

const int ith = params->ith;
Expand Down Expand Up @@ -1825,7 +1836,7 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
ggml_compute_forward_l2_norm(params, tensor);
} break;
case GGML_OP_MUL_MAT:
case GGML_OP_MUL_MAT_PACK4:
case GGML_OP_MUL_MAT_PACK4:
{
ggml_compute_forward_mul_mat(params, tensor);
} break;
Expand Down Expand Up @@ -1901,18 +1912,18 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
{
ggml_compute_forward_conv_transpose_1d(params, tensor);
} break;
case GGML_OP_IM2COL:
{
ggml_compute_forward_im2col(params, tensor);
} break;
case GGML_OP_IM2COL_FAST_1D:
{
ggml_compute_forward_im2col_fast_1d(params, tensor);
} break;
case GGML_OP_IM2COL_BACK:
{
ggml_compute_forward_im2col_back_f32(params, tensor);
} break;
case GGML_OP_IM2COL:
{
ggml_compute_forward_im2col(params, tensor);
} break;
case GGML_OP_IM2COL_FAST_1D:
{
ggml_compute_forward_im2col_fast_1d(params, tensor);
} break;
case GGML_OP_IM2COL_BACK:
{
ggml_compute_forward_im2col_back_f32(params, tensor);
} break;
case GGML_OP_IM2COL_3D:
{
ggml_compute_forward_im2col_3d(params, tensor);
Expand Down Expand Up @@ -2092,6 +2103,26 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
ggml_compute_forward_opt_step_sgd(params, tensor);
}
break;
case GGML_OP_ADD_SCALED:
{
ggml_compute_forward_add_scaled(params, tensor);
} break;
case GGML_OP_RMS_NORM_SCALED:
{
ggml_compute_forward_rms_norm_scaled(params, tensor);
} break;
case GGML_OP_MUL_MAT_ADD:
{
ggml_compute_forward_mul_mat_add(params, tensor);
} break;
case GGML_OP_MUL_MAT_ADD_RELU:
{
ggml_compute_forward_mul_mat_add_relu(params, tensor);
} break;
case GGML_OP_IM2COL_ASYM:
{
ggml_compute_forward_im2col_asym(params, tensor);
} break;
case GGML_OP_NONE:
{
// nop
Expand Down Expand Up @@ -2301,7 +2332,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
case GGML_OP_GROUP_NORM:
case GGML_OP_CONCAT:
case GGML_OP_MUL_MAT:
case GGML_OP_MUL_MAT_PACK4:
case GGML_OP_MUL_MAT_PACK4:
case GGML_OP_MUL_MAT_ID:
case GGML_OP_OUT_PROD:
{
Expand Down Expand Up @@ -2343,15 +2374,20 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
{
n_tasks = MIN(n_threads, ggml_nrows(node->src[0]));
} break;
case GGML_OP_IM2COL:
case GGML_OP_IM2COL_FAST_1D:
case GGML_OP_IM2COL_BACK:
case GGML_OP_IM2COL_3D:
case GGML_OP_CONV_2D:
case GGML_OP_IM2COL:
case GGML_OP_IM2COL_FAST_1D:
case GGML_OP_IM2COL_ASYM:
case GGML_OP_IM2COL_BACK:
case GGML_OP_IM2COL_3D:
case GGML_OP_CONV_2D:
case GGML_OP_CONV_3D:
case GGML_OP_CONV_2D_DW:
case GGML_OP_CONV_TRANSPOSE_1D:
case GGML_OP_CONV_TRANSPOSE_2D:
case GGML_OP_ADD_SCALED:
case GGML_OP_RMS_NORM_SCALED:
case GGML_OP_MUL_MAT_ADD:
case GGML_OP_MUL_MAT_ADD_RELU:
{
n_tasks = n_threads;
} break;
Expand Down Expand Up @@ -2819,8 +2855,19 @@ struct ggml_cplan ggml_graph_plan(
cur = ggml_type_size(node->type)*n_tasks;
} break;
case GGML_OP_MUL_MAT:
case GGML_OP_MUL_MAT_PACK4:
case GGML_OP_MUL_MAT_PACK4:
{
if (node->src[0]->type == GGML_TYPE_I2_S) {
// The int8 activation rows, then one scale and one
// int8 row sum each. I2_S has no vec_dot_type entry
// -- see ggml_compute_forward_mul_mat_i2_s.
const int64_t nrows_y = ggml_nrows(node->src[1]);

cur = GGML_PAD((size_t) ggml_nelements(node->src[1]), sizeof(float));
cur += nrows_y*(sizeof(float) + sizeof(int32_t));
break;
}

const enum ggml_type vec_dot_type = type_traits_cpu[node->src[0]->type].vec_dot_type;

if (node->src[1]->type != vec_dot_type) {
Expand Down Expand Up @@ -2948,6 +2995,18 @@ struct ggml_cplan ggml_graph_plan(
{
cur = ggml_type_size(node->type)*(n_tasks + node->src[0]->ne[0]*n_tasks);
} break;
case GGML_OP_ADD_SCALED:
case GGML_OP_RMS_NORM_SCALED:
case GGML_OP_MUL_MAT_ADD:
case GGML_OP_MUL_MAT_ADD_RELU:
{
// F32 staging for the whole output, plus one absmax per
// thread. The output cannot be quantized in place: its
// scale is only known once every element has been
// computed, so all of them have to be held somewhere
// first.
cur = sizeof(float)*ggml_nelements(node) + sizeof(float)*n_tasks;
} break;
case GGML_OP_GATED_DELTA_NET:
{
const int64_t S_v = node->src[2]->ne[0];
Expand Down
Loading
Loading