From 529fe55502f5faa26906e0557a58a06ffabad4f6 Mon Sep 17 00:00:00 2001 From: matrixfede Date: Thu, 20 Aug 2026 09:16:23 +0200 Subject: [PATCH 01/20] Add portable host code and C tokenizer for non-macOS builds --- h3_ffmpeg.c | 17 +- h3_host.c | 36 + h3_host.h | 6 +- h3_terminal.c | 6 +- h3_tokenizer.c | 938 ++++++++++++++++++++++++++ tests/test_host_portable.c | 24 + tests/test_tokenizer_portable.c | 47 ++ tests/tokenizer_portable_fixture.json | 13 + 8 files changed, 1075 insertions(+), 12 deletions(-) create mode 100644 h3_tokenizer.c create mode 100644 tests/test_host_portable.c create mode 100644 tests/test_tokenizer_portable.c create mode 100644 tests/tokenizer_portable_fixture.json diff --git a/h3_ffmpeg.c b/h3_ffmpeg.c index 66762425..171d3732 100644 --- a/h3_ffmpeg.c +++ b/h3_ffmpeg.c @@ -1,6 +1,8 @@ #include "h3_ffmpeg.h" #include +#include +#include #include #include #include @@ -14,6 +16,8 @@ extern char **environ; +#define H3_WRITE_MAX ((size_t)PTRDIFF_MAX) + static const char *ffmpeg_program(void) { const char *override = getenv("H3_FFMPEG"); return override && *override ? override : "ffmpeg"; @@ -57,8 +61,7 @@ static int write_all(int descriptor, const uint8_t *data, size_t bytes, char *error, size_t error_size) { while (bytes) { ssize_t written = write(descriptor, data, - bytes > (size_t)SSIZE_MAX ? - (size_t)SSIZE_MAX : bytes); + bytes > H3_WRITE_MAX ? H3_WRITE_MAX : bytes); if (written < 0 && errno == EINTR) continue; if (written <= 0) { fail(error, error_size, "cannot stream RGB frames to FFmpeg: %s", @@ -588,8 +591,7 @@ static void *stream_thread(void *opaque) { const uint8_t *data = writer->data; size_t remaining = writer->bytes; while (remaining) { - size_t request = remaining > (size_t)SSIZE_MAX ? - (size_t)SSIZE_MAX : remaining; + size_t request = remaining > H3_WRITE_MAX ? H3_WRITE_MAX : remaining; ssize_t written = write(writer->descriptor, data, request); if (written < 0 && errno == EINTR) continue; if (written <= 0) { @@ -640,7 +642,8 @@ int h3_ffmpeg_write_av_rgb24_f32(const char *path, const uint8_t *frames, for (int sample = 0; sample < samples; sample++) for (int channel = 0; channel < channels; channel++) interleaved[(size_t)sample * (size_t)channels + (size_t)channel] = - pcm[(size_t)channel * (size_t)samples + (size_t)sample]; + fmaxf(-0.5f, fminf(0.5f, + pcm[(size_t)channel * (size_t)samples + (size_t)sample])); int video_pipe[2] = {-1, -1}, audio_pipe[2] = {-1, -1}; if (pipe(video_pipe) != 0 || pipe(audio_pipe) != 0) { @@ -673,7 +676,9 @@ int h3_ffmpeg_write_av_rgb24_f32(const char *path, const uint8_t *frames, "-i", audio_input, "-map", "0:v:0", "-map", "1:a:0", "-c:v", "libx264", "-preset", "fast", "-crf", "18", - "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", + "-pix_fmt", "yuv420p", + "-af", "alimiter=limit=0.5:level=disabled", + "-c:a", "aac", "-b:a", "192k", "-movflags", "+faststart", (char *)path, NULL }; posix_spawn_file_actions_t actions; diff --git a/h3_host.c b/h3_host.c index a04a2a0a..ae3cd9da 100644 --- a/h3_host.c +++ b/h3_host.c @@ -1,6 +1,8 @@ #include "h3_host.h" +#ifdef __APPLE__ #include +#endif #include #include @@ -551,6 +553,7 @@ int h3_resize_rgb24_high_quality(const uint8_t *input, int frames, *output = pixels; return 1; } +#ifdef __APPLE__ if (input_area > SIZE_MAX / 4 || output_area > SIZE_MAX / 4) { free(pixels); return 0; @@ -594,6 +597,39 @@ int h3_resize_rgb24_high_quality(const uint8_t *input, int frames, } } free(source_argb); free(output_argb); +#else + size_t input_frame_bytes = input_area * 3; + size_t output_frame_bytes = output_area * 3; + for (int frame = 0; frame < frames; frame++) { + const uint8_t *source = input + (size_t)frame * input_frame_bytes; + uint8_t *destination = pixels + (size_t)frame * output_frame_bytes; + for (int y = 0; y < output_height; y++) { + double source_y = ((double)y + 0.5) * input_height / output_height - 0.5; + int y0 = (int)floor(source_y); + double fy = source_y - y0; + if (y0 < 0) { y0 = 0; fy = 0.0; } + int y1 = y0 + 1; + if (y1 >= input_height) { y1 = input_height - 1; fy = 0.0; } + for (int x = 0; x < output_width; x++) { + double source_x = ((double)x + 0.5) * input_width / output_width - 0.5; + int x0 = (int)floor(source_x); + double fx = source_x - x0; + if (x0 < 0) { x0 = 0; fx = 0.0; } + int x1 = x0 + 1; + if (x1 >= input_width) { x1 = input_width - 1; fx = 0.0; } + for (int channel = 0; channel < 3; channel++) { + double top = source[((size_t)y0 * input_width + x0) * 3 + channel] * (1.0 - fx) + + source[((size_t)y0 * input_width + x1) * 3 + channel] * fx; + double bottom = source[((size_t)y1 * input_width + x0) * 3 + channel] * (1.0 - fx) + + source[((size_t)y1 * input_width + x1) * 3 + channel] * fx; + double value = top * (1.0 - fy) + bottom * fy; + destination[((size_t)y * output_width + x) * 3 + channel] = + (uint8_t)lrint(fmin(255.0, fmax(0.0, value))); + } + } + } + } +#endif *output = pixels; return 1; } diff --git a/h3_host.h b/h3_host.h index f71eded7..f125d658 100644 --- a/h3_host.h +++ b/h3_host.h @@ -125,9 +125,9 @@ uint32_t h3_rng_u32(h3_rng *rng); float h3_rng_normal(h3_rng *rng); void h3_rng_fill_normal(h3_rng *rng, float *values, size_t count); -/* Resize interleaved RGB24 frames with Accelerate/vImage high-quality - * resampling. The caller owns *output. Identity geometry still returns an - * independent copy. */ +/* Resize interleaved RGB24 frames with the platform high-quality resampler. + * The caller owns *output. Identity geometry still returns an independent + * copy. */ int h3_resize_rgb24_high_quality(const uint8_t *input, int frames, int input_width, int input_height, int output_width, int output_height, diff --git a/h3_terminal.c b/h3_terminal.c index 5b216e2e..317f8cf8 100644 --- a/h3_terminal.c +++ b/h3_terminal.c @@ -192,7 +192,7 @@ static int encode_png(const uint8_t *pixels, size_t size, unlink(raw_path); return 0; } - int output = mkstemps(path, 4); + int output = mkstemp(path); if (output < 0) { fail(error, error_size, "cannot create terminal PNG temporary file: %s", strerror(errno)); @@ -205,7 +205,7 @@ static int encode_png(const uint8_t *pixels, size_t size, char *arguments[] = { "ffmpeg", "-v", "error", "-y", "-f", "rawvideo", "-pixel_format", "rgb24", "-video_size", dimensions, - "-i", raw_path, "-frames:v", "1", path, NULL + "-i", raw_path, "-frames:v", "1", "-f", "image2", path, NULL }; pid_t child = 0; int spawn_error = posix_spawnp(&child, "ffmpeg", NULL, NULL, @@ -261,7 +261,7 @@ static int iterm2_display(const uint8_t *pixels, size_t size, fail(error, error_size, "invalid iTerm2 display dimensions"); return 0; } - char path[] = "/tmp/h3-terminal-XXXXXX.png"; + char path[] = "/tmp/h3-terminal-XXXXXX"; if (!encode_png(pixels, size, width, height, path, error, error_size)) return 0; size_t png_size = 0; diff --git a/h3_tokenizer.c b/h3_tokenizer.c new file mode 100644 index 00000000..f20a0b33 --- /dev/null +++ b/h3_tokenizer.c @@ -0,0 +1,938 @@ +#include "h3_tokenizer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + char *key; + uint32_t value; +} h3_map_item; + +typedef struct { + h3_map_item *items; + size_t capacity; + size_t count; +} h3_map; + +struct h3_tokenizer { + h3_map vocab; + h3_map merges; + h3_map added; + char **inverse_vocab; + char **inverse_added; + size_t inverse_count; + char *byte_encoder[256]; + int16_t byte_decoder[324]; +}; + +typedef struct { + const char *cursor; + const char *end; + char message[160]; +} h3_json; + +typedef struct { + char **values; + size_t count; + size_t capacity; +} h3_strings; + +typedef struct { + uint32_t *values; + size_t count; + size_t capacity; +} h3_ids; + +typedef struct { + uint32_t value; + size_t offset; + size_t length; +} h3_codepoint; + +static void h3_error(char *error, size_t size, const char *message) { + if (error && size) snprintf(error, size, "%s", message ? message : "tokenizer failure"); +} + +static uint64_t h3_hash(const char *text) { + uint64_t value = UINT64_C(1469598103934665603); + for (const unsigned char *p = (const unsigned char *)text; *p; p++) { + value ^= *p; + value *= UINT64_C(1099511628211); + } + return value; +} + +static int h3_map_grow(h3_map *map) { + size_t capacity = map->capacity ? map->capacity * 2 : 1024; + h3_map_item *items = calloc(capacity, sizeof(*items)); + if (!items) return 0; + for (size_t index = 0; index < map->capacity; index++) { + h3_map_item item = map->items[index]; + if (!item.key) continue; + size_t slot = (size_t)h3_hash(item.key) & (capacity - 1); + while (items[slot].key) slot = (slot + 1) & (capacity - 1); + items[slot] = item; + } + free(map->items); + map->items = items; + map->capacity = capacity; + return 1; +} + +static int h3_map_put(h3_map *map, char *key, uint32_t value) { + if (!map->capacity || (map->count + 1) * 10 >= map->capacity * 7) + if (!h3_map_grow(map)) return 0; + size_t slot = (size_t)h3_hash(key) & (map->capacity - 1); + while (map->items[slot].key) { + if (!strcmp(map->items[slot].key, key)) { + free(key); + map->items[slot].value = value; + return 1; + } + slot = (slot + 1) & (map->capacity - 1); + } + map->items[slot] = (h3_map_item){key, value}; + map->count++; + return 1; +} + +static int h3_map_get(const h3_map *map, const char *key, uint32_t *value) { + if (!map->capacity) return 0; + size_t slot = (size_t)h3_hash(key) & (map->capacity - 1); + while (map->items[slot].key) { + if (!strcmp(map->items[slot].key, key)) { + if (value) *value = map->items[slot].value; + return 1; + } + slot = (slot + 1) & (map->capacity - 1); + } + return 0; +} + +static void h3_map_free(h3_map *map) { + for (size_t index = 0; index < map->capacity; index++) free(map->items[index].key); + free(map->items); + memset(map, 0, sizeof(*map)); +} + +static void h3_json_space(h3_json *json) { + while (json->cursor < json->end && isspace((unsigned char)*json->cursor)) json->cursor++; +} + +static int h3_json_fail(h3_json *json, const char *message) { + if (!json->message[0]) snprintf(json->message, sizeof(json->message), "%s", message); + return 0; +} + +static int h3_json_take(h3_json *json, char wanted) { + h3_json_space(json); + if (json->cursor >= json->end || *json->cursor != wanted) + return h3_json_fail(json, "malformed tokenizer JSON"); + json->cursor++; + return 1; +} + +static int h3_utf8_append(char **buffer, size_t *length, size_t *capacity, + uint32_t codepoint) { + unsigned char bytes[4]; + int count; + if (codepoint <= 0x7f) { bytes[0] = (unsigned char)codepoint; count = 1; } + else if (codepoint <= 0x7ff) { + bytes[0] = (unsigned char)(0xc0 | (codepoint >> 6)); + bytes[1] = (unsigned char)(0x80 | (codepoint & 0x3f)); count = 2; + } else if (codepoint <= 0xffff) { + bytes[0] = (unsigned char)(0xe0 | (codepoint >> 12)); + bytes[1] = (unsigned char)(0x80 | ((codepoint >> 6) & 0x3f)); + bytes[2] = (unsigned char)(0x80 | (codepoint & 0x3f)); count = 3; + } else if (codepoint <= 0x10ffff) { + bytes[0] = (unsigned char)(0xf0 | (codepoint >> 18)); + bytes[1] = (unsigned char)(0x80 | ((codepoint >> 12) & 0x3f)); + bytes[2] = (unsigned char)(0x80 | ((codepoint >> 6) & 0x3f)); + bytes[3] = (unsigned char)(0x80 | (codepoint & 0x3f)); count = 4; + } else return 0; + if (*length + (size_t)count + 1 > *capacity) { + size_t next = *capacity ? *capacity * 2 : 32; + while (next < *length + (size_t)count + 1) next *= 2; + char *grown = realloc(*buffer, next); + if (!grown) return 0; + *buffer = grown; *capacity = next; + } + memcpy(*buffer + *length, bytes, (size_t)count); + *length += (size_t)count; + (*buffer)[*length] = '\0'; + return 1; +} + +static int h3_hex(char value) { + if (value >= '0' && value <= '9') return value - '0'; + if (value >= 'a' && value <= 'f') return value - 'a' + 10; + if (value >= 'A' && value <= 'F') return value - 'A' + 10; + return -1; +} + +static int h3_json_u16(h3_json *json, uint32_t *value) { + if (json->end - json->cursor < 4) return h3_json_fail(json, "truncated JSON escape"); + uint32_t result = 0; + for (int index = 0; index < 4; index++) { + int digit = h3_hex(json->cursor[index]); + if (digit < 0) return h3_json_fail(json, "invalid JSON escape"); + result = result * 16 + (uint32_t)digit; + } + json->cursor += 4; + *value = result; + return 1; +} + +static char *h3_json_string(h3_json *json) { + if (!h3_json_take(json, '"')) return NULL; + char *result = NULL; + size_t length = 0, capacity = 0; + while (json->cursor < json->end && *json->cursor != '"') { + unsigned char value = (unsigned char)*json->cursor++; + uint32_t codepoint = value; + if (value == '\\') { + if (json->cursor >= json->end) goto malformed; + char escape = *json->cursor++; + if (escape == '"' || escape == '\\' || escape == '/') codepoint = (uint32_t)escape; + else if (escape == 'b') codepoint = '\b'; + else if (escape == 'f') codepoint = '\f'; + else if (escape == 'n') codepoint = '\n'; + else if (escape == 'r') codepoint = '\r'; + else if (escape == 't') codepoint = '\t'; + else if (escape == 'u') { + if (!h3_json_u16(json, &codepoint)) goto malformed; + if (codepoint >= 0xd800 && codepoint <= 0xdbff) { + if (json->end - json->cursor < 6 || json->cursor[0] != '\\' || json->cursor[1] != 'u') goto malformed; + json->cursor += 2; + uint32_t low; + if (!h3_json_u16(json, &low) || low < 0xdc00 || low > 0xdfff) goto malformed; + codepoint = 0x10000 + ((codepoint - 0xd800) << 10) + (low - 0xdc00); + } + } else goto malformed; + if (!h3_utf8_append(&result, &length, &capacity, codepoint)) goto memory; + } else { + if (value < 0x20) goto malformed; + if (length + 2 > capacity) { + size_t next = capacity ? capacity * 2 : 32; + char *grown = realloc(result, next); + if (!grown) goto memory; + result = grown; capacity = next; + } + result[length++] = (char)value; + result[length] = '\0'; + } + } + if (json->cursor >= json->end) goto malformed; + json->cursor++; + if (!result) result = calloc(1, 1); + return result; +memory: + h3_json_fail(json, "out of memory parsing tokenizer JSON"); + free(result); return NULL; +malformed: + h3_json_fail(json, "invalid JSON string"); + free(result); return NULL; +} + +static int h3_json_uint(h3_json *json, uint32_t *value) { + h3_json_space(json); + errno = 0; + char *stop = NULL; + unsigned long parsed = strtoul(json->cursor, &stop, 10); + if (stop == json->cursor || errno || parsed > UINT32_MAX || stop > json->end) + return h3_json_fail(json, "invalid tokenizer integer"); + json->cursor = stop; + *value = (uint32_t)parsed; + return 1; +} + +static int h3_json_literal(h3_json *json, const char *literal) { + h3_json_space(json); + size_t length = strlen(literal); + if ((size_t)(json->end - json->cursor) < length || + memcmp(json->cursor, literal, length)) return 0; + json->cursor += length; + return 1; +} + +static int h3_json_skip(h3_json *json); + +static int h3_json_skip_array(h3_json *json) { + if (!h3_json_take(json, '[')) return 0; + h3_json_space(json); + if (json->cursor < json->end && *json->cursor == ']') { json->cursor++; return 1; } + for (;;) { + if (!h3_json_skip(json)) return 0; + h3_json_space(json); + if (json->cursor < json->end && *json->cursor == ']') { json->cursor++; return 1; } + if (!h3_json_take(json, ',')) return 0; + } +} + +static int h3_json_skip_object(h3_json *json) { + if (!h3_json_take(json, '{')) return 0; + h3_json_space(json); + if (json->cursor < json->end && *json->cursor == '}') { json->cursor++; return 1; } + for (;;) { + char *key = h3_json_string(json); + if (!key) return 0; + free(key); + if (!h3_json_take(json, ':') || !h3_json_skip(json)) return 0; + h3_json_space(json); + if (json->cursor < json->end && *json->cursor == '}') { json->cursor++; return 1; } + if (!h3_json_take(json, ',')) return 0; + } +} + +static int h3_json_skip(h3_json *json) { + h3_json_space(json); + if (json->cursor >= json->end) return h3_json_fail(json, "truncated JSON value"); + if (*json->cursor == '"') { char *text = h3_json_string(json); free(text); return text != NULL; } + if (*json->cursor == '{') return h3_json_skip_object(json); + if (*json->cursor == '[') return h3_json_skip_array(json); + if (h3_json_literal(json, "true") || h3_json_literal(json, "false") || h3_json_literal(json, "null")) return 1; + char *stop = NULL; + (void)strtod(json->cursor, &stop); + if (stop == json->cursor) return h3_json_fail(json, "invalid JSON value"); + json->cursor = stop; + return 1; +} + +static char *h3_pair_key(const char *left, const char *right) { + size_t a = strlen(left), b = strlen(right); + if (a > SIZE_MAX - b - 2) return NULL; + char *key = malloc(a + b + 2); + if (!key) return NULL; + memcpy(key, left, a); key[a] = '\x1f'; + memcpy(key + a + 1, right, b + 1); + return key; +} + +static int h3_parse_vocab(h3_json *json, h3_tokenizer *tokenizer, + uint32_t *maximum_id) { + if (!h3_json_take(json, '{')) return 0; + h3_json_space(json); + if (json->cursor < json->end && *json->cursor == '}') { json->cursor++; return 1; } + for (;;) { + char *symbol = h3_json_string(json); + uint32_t identifier; + if (!symbol || !h3_json_take(json, ':') || !h3_json_uint(json, &identifier)) { + free(symbol); return 0; + } + if (!h3_map_put(&tokenizer->vocab, symbol, identifier)) + return h3_json_fail(json, "out of memory loading vocabulary"); + if (identifier > *maximum_id) *maximum_id = identifier; + h3_json_space(json); + if (json->cursor < json->end && *json->cursor == '}') { json->cursor++; return 1; } + if (!h3_json_take(json, ',')) return 0; + } +} + +static int h3_parse_merges(h3_json *json, h3_tokenizer *tokenizer) { + if (!h3_json_take(json, '[')) return 0; + h3_json_space(json); + if (json->cursor < json->end && *json->cursor == ']') { json->cursor++; return 1; } + uint32_t rank = 0; + for (;;) { + h3_json_space(json); + char *left = NULL, *right = NULL; + if (json->cursor < json->end && *json->cursor == '"') { + char *entry = h3_json_string(json); + if (!entry) return 0; + char *space = strchr(entry, ' '); + if (!space) { free(entry); return h3_json_fail(json, "invalid tokenizer merge"); } + *space = '\0'; + left = strdup(entry); right = strdup(space + 1); free(entry); + } else if (json->cursor < json->end && *json->cursor == '[') { + if (!h3_json_take(json, '[')) return 0; + left = h3_json_string(json); + if (!left || !h3_json_take(json, ',')) { free(left); return 0; } + right = h3_json_string(json); + if (!right || !h3_json_take(json, ']')) { free(left); free(right); return 0; } + } else return h3_json_fail(json, "invalid tokenizer merge"); + char *key = left && right ? h3_pair_key(left, right) : NULL; + free(left); free(right); + if (!key || !h3_map_put(&tokenizer->merges, key, rank++)) { + free(key); return h3_json_fail(json, "out of memory loading merges"); + } + h3_json_space(json); + if (json->cursor < json->end && *json->cursor == ']') { json->cursor++; return 1; } + if (!h3_json_take(json, ',')) return 0; + } +} + +static int h3_parse_model(h3_json *json, h3_tokenizer *tokenizer, + uint32_t *maximum_id) { + int type_ok = 0, vocab_ok = 0, merges_ok = 0, unk_null = 0; + if (!h3_json_take(json, '{')) return 0; + h3_json_space(json); + while (json->cursor < json->end && *json->cursor != '}') { + char *key = h3_json_string(json); + if (!key || !h3_json_take(json, ':')) { free(key); return 0; } + if (!strcmp(key, "type")) { + char *value = h3_json_string(json); + type_ok = value && !strcmp(value, "BPE"); free(value); + } else if (!strcmp(key, "unk_token")) { + unk_null = h3_json_literal(json, "null"); + if (!unk_null) { free(key); return h3_json_fail(json, "tokenizer unk_token must be null"); } + } else if (!strcmp(key, "vocab")) { + vocab_ok = h3_parse_vocab(json, tokenizer, maximum_id); + if (!vocab_ok) { free(key); return 0; } + } else if (!strcmp(key, "merges")) { + merges_ok = h3_parse_merges(json, tokenizer); + if (!merges_ok) { free(key); return 0; } + } else if (!h3_json_skip(json)) { free(key); return 0; } + free(key); + h3_json_space(json); + if (*json->cursor == ',') { json->cursor++; h3_json_space(json); } + else break; + } + if (!h3_json_take(json, '}')) return 0; + if (!type_ok || !vocab_ok || !merges_ok || !unk_null) + return h3_json_fail(json, "unexpected tokenizer model specification"); + return 1; +} + +static int h3_json_bool(h3_json *json, int *value) { + if (h3_json_literal(json, "true")) { *value = 1; return 1; } + if (h3_json_literal(json, "false")) { *value = 0; return 1; } + return h3_json_fail(json, "invalid tokenizer boolean"); +} + +static int h3_parse_added_item(h3_json *json, h3_tokenizer *tokenizer, + uint32_t *maximum_id) { + char *content = NULL; + uint32_t identifier = 0; + int has_id = 0, unsupported = 0; + if (!h3_json_take(json, '{')) return 0; + h3_json_space(json); + while (json->cursor < json->end && *json->cursor != '}') { + char *key = h3_json_string(json); + if (!key || !h3_json_take(json, ':')) { free(key); free(content); return 0; } + if (!strcmp(key, "content")) { + free(content); content = h3_json_string(json); + if (!content) { free(key); return 0; } + } else if (!strcmp(key, "id")) { + has_id = h3_json_uint(json, &identifier); + if (!has_id) { free(key); free(content); return 0; } + } else if (!strcmp(key, "single_word") || !strcmp(key, "lstrip") || + !strcmp(key, "rstrip") || !strcmp(key, "normalized")) { + int enabled; + if (!h3_json_bool(json, &enabled)) { free(key); free(content); return 0; } + unsupported |= enabled; + } else if (!h3_json_skip(json)) { free(key); free(content); return 0; } + free(key); + h3_json_space(json); + if (*json->cursor == ',') { json->cursor++; h3_json_space(json); } + else break; + } + if (!h3_json_take(json, '}')) { free(content); return 0; } + if (!content || !has_id || unsupported) { + free(content); return h3_json_fail(json, "unsupported added-token policy"); + } + if (!h3_map_put(&tokenizer->added, content, identifier)) + return h3_json_fail(json, "out of memory loading added tokens"); + if (identifier > *maximum_id) *maximum_id = identifier; + return 1; +} + +static int h3_parse_added(h3_json *json, h3_tokenizer *tokenizer, + uint32_t *maximum_id) { + if (!h3_json_take(json, '[')) return 0; + h3_json_space(json); + if (*json->cursor == ']') { json->cursor++; return 1; } + for (;;) { + if (!h3_parse_added_item(json, tokenizer, maximum_id)) return 0; + h3_json_space(json); + if (*json->cursor == ']') { json->cursor++; return 1; } + if (!h3_json_take(json, ',')) return 0; + } +} + +static int h3_parse_normalizer(h3_json *json) { + int nfc = 0; + if (!h3_json_take(json, '{')) return 0; + h3_json_space(json); + while (json->cursor < json->end && *json->cursor != '}') { + char *key = h3_json_string(json); + if (!key || !h3_json_take(json, ':')) { free(key); return 0; } + if (!strcmp(key, "type")) { + char *value = h3_json_string(json); + nfc = value && !strcmp(value, "NFC"); free(value); + } else if (!h3_json_skip(json)) { free(key); return 0; } + free(key); + h3_json_space(json); + if (*json->cursor == ',') { json->cursor++; h3_json_space(json); } + else break; + } + if (!h3_json_take(json, '}')) return 0; + return nfc ? 1 : h3_json_fail(json, "tokenizer normalizer is not NFC"); +} + +static int h3_parse_root(h3_json *json, h3_tokenizer *tokenizer, + uint32_t *maximum_id) { + int model = 0, normalizer = 0; + if (!h3_json_take(json, '{')) return 0; + h3_json_space(json); + while (json->cursor < json->end && *json->cursor != '}') { + char *key = h3_json_string(json); + if (!key || !h3_json_take(json, ':')) { free(key); return 0; } + if (!strcmp(key, "model")) model = h3_parse_model(json, tokenizer, maximum_id); + else if (!strcmp(key, "normalizer")) normalizer = h3_parse_normalizer(json); + else if (!strcmp(key, "added_tokens")) { + if (!h3_parse_added(json, tokenizer, maximum_id)) { free(key); return 0; } + } else if (!h3_json_skip(json)) { free(key); return 0; } + free(key); + if (json->message[0]) return 0; + h3_json_space(json); + if (*json->cursor == ',') { json->cursor++; h3_json_space(json); } + else break; + } + if (!h3_json_take(json, '}')) return 0; + h3_json_space(json); + if (json->cursor != json->end) return h3_json_fail(json, "trailing tokenizer JSON data"); + if (!model || !normalizer) return h3_json_fail(json, "incomplete tokenizer specification"); + return 1; +} + +static char *h3_read_all(const char *path, size_t *size) { + FILE *file = fopen(path, "rb"); + if (!file) return NULL; + if (fseek(file, 0, SEEK_END) || ftell(file) < 0) { fclose(file); return NULL; } + long length = ftell(file); + if (fseek(file, 0, SEEK_SET)) { fclose(file); return NULL; } + char *data = malloc((size_t)length + 1); + if (!data) { fclose(file); return NULL; } + size_t got = fread(data, 1, (size_t)length, file); + fclose(file); + if (got != (size_t)length) { free(data); return NULL; } + data[got] = '\0'; *size = got; + return data; +} + +static char *h3_codepoint_string(uint32_t codepoint) { + char *result = NULL; + size_t length = 0, capacity = 0; + if (!h3_utf8_append(&result, &length, &capacity, codepoint)) return NULL; + return result; +} + +h3_tokenizer *h3_tokenizer_load(const char *path, char *error, + size_t error_size) { + if (error && error_size) error[0] = '\0'; + if (!path) { h3_error(error, error_size, "tokenizer path is required"); return NULL; } + size_t size = 0; + char *data = h3_read_all(path, &size); + if (!data) { h3_error(error, error_size, "cannot read tokenizer JSON"); return NULL; } + h3_tokenizer *tokenizer = calloc(1, sizeof(*tokenizer)); + if (!tokenizer) { free(data); h3_error(error, error_size, "out of memory"); return NULL; } + h3_json json = {data, data + size, {0}}; + uint32_t maximum_id = 0; + if (!h3_parse_root(&json, tokenizer, &maximum_id)) { + h3_error(error, error_size, json.message); free(data); + h3_tokenizer_free(tokenizer); return NULL; + } + free(data); + tokenizer->inverse_count = (size_t)maximum_id + 1; + tokenizer->inverse_vocab = calloc(tokenizer->inverse_count, sizeof(char *)); + tokenizer->inverse_added = calloc(tokenizer->inverse_count, sizeof(char *)); + if (!tokenizer->inverse_vocab || !tokenizer->inverse_added) { + h3_error(error, error_size, "out of memory indexing vocabulary"); + h3_tokenizer_free(tokenizer); return NULL; + } + for (size_t index = 0; index < tokenizer->vocab.capacity; index++) { + h3_map_item item = tokenizer->vocab.items[index]; + if (item.key && item.value < tokenizer->inverse_count) + tokenizer->inverse_vocab[item.value] = item.key; + } + for (size_t index = 0; index < tokenizer->added.capacity; index++) { + h3_map_item item = tokenizer->added.items[index]; + if (item.key && item.value < tokenizer->inverse_count) + tokenizer->inverse_added[item.value] = item.key; + } + for (size_t index = 0; index < 324; index++) tokenizer->byte_decoder[index] = -1; + unsigned extra = 0; + for (unsigned byte = 0; byte < 256; byte++) { + int visible = (byte >= '!' && byte <= '~') || + (byte >= 0xa1 && byte <= 0xac) || + (byte >= 0xae && byte <= 0xff); + uint32_t codepoint = visible ? byte : 256 + extra++; + tokenizer->byte_encoder[byte] = h3_codepoint_string(codepoint); + if (!tokenizer->byte_encoder[byte]) { + h3_error(error, error_size, "out of memory building byte codec"); + h3_tokenizer_free(tokenizer); return NULL; + } + tokenizer->byte_decoder[codepoint] = (int16_t)byte; + } + return tokenizer; +} + +void h3_tokenizer_free(h3_tokenizer *tokenizer) { + if (!tokenizer) return; + for (size_t index = 0; index < 256; index++) free(tokenizer->byte_encoder[index]); + free(tokenizer->inverse_vocab); free(tokenizer->inverse_added); + h3_map_free(&tokenizer->vocab); h3_map_free(&tokenizer->merges); + h3_map_free(&tokenizer->added); free(tokenizer); +} + +static char *h3_nfc(const char *utf8) { + UErrorCode status = U_ZERO_ERROR; + int32_t utf16_length = 0; + u_strFromUTF8(NULL, 0, &utf16_length, utf8, -1, &status); + if (status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(status)) return NULL; + status = U_ZERO_ERROR; + UChar *utf16 = malloc(((size_t)utf16_length + 1) * sizeof(*utf16)); + if (!utf16) return NULL; + u_strFromUTF8(utf16, utf16_length + 1, NULL, utf8, -1, &status); + const UNormalizer2 *nfc = unorm2_getNFCInstance(&status); + int32_t normalized_length = unorm2_normalize(nfc, utf16, utf16_length, + NULL, 0, &status); + if (status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(status)) { free(utf16); return NULL; } + status = U_ZERO_ERROR; + UChar *normalized = malloc(((size_t)normalized_length + 1) * sizeof(*normalized)); + if (!normalized) { free(utf16); return NULL; } + unorm2_normalize(nfc, utf16, utf16_length, normalized, + normalized_length + 1, &status); + free(utf16); + if (U_FAILURE(status)) { free(normalized); return NULL; } + int32_t output_length = 0; + status = U_ZERO_ERROR; + u_strToUTF8(NULL, 0, &output_length, normalized, normalized_length, &status); + if (status != U_BUFFER_OVERFLOW_ERROR && U_FAILURE(status)) { free(normalized); return NULL; } + status = U_ZERO_ERROR; + char *output = malloc((size_t)output_length + 1); + if (!output) { free(normalized); return NULL; } + u_strToUTF8(output, output_length + 1, NULL, normalized, normalized_length, &status); + free(normalized); + if (U_FAILURE(status)) { free(output); return NULL; } + return output; +} + +static int h3_strings_push(h3_strings *strings, char *value) { + if (strings->count == strings->capacity) { + size_t capacity = strings->capacity ? strings->capacity * 2 : 16; + char **values = realloc(strings->values, capacity * sizeof(*values)); + if (!values) return 0; + strings->values = values; strings->capacity = capacity; + } + strings->values[strings->count++] = value; + return 1; +} + +static void h3_strings_free(h3_strings *strings) { + for (size_t index = 0; index < strings->count; index++) free(strings->values[index]); + free(strings->values); memset(strings, 0, sizeof(*strings)); +} + +static int h3_ids_push(h3_ids *ids, uint32_t value) { + if (ids->count == ids->capacity) { + size_t capacity = ids->capacity ? ids->capacity * 2 : 32; + uint32_t *values = realloc(ids->values, capacity * sizeof(*values)); + if (!values) return 0; + ids->values = values; ids->capacity = capacity; + } + ids->values[ids->count++] = value; + return 1; +} + +static int h3_codepoints(const char *text, h3_codepoint **output, + size_t *count) { + size_t bytes = strlen(text), used = 0; + h3_codepoint *points = malloc((bytes ? bytes : 1) * sizeof(*points)); + if (!points) return 0; + int32_t index = 0; + while ((size_t)index < bytes) { + int32_t start = index; + UChar32 value; + U8_NEXT((const uint8_t *)text, index, (int32_t)bytes, value); + if (value < 0) { free(points); return 0; } + points[used++] = (h3_codepoint){(uint32_t)value, (size_t)start, + (size_t)(index - start)}; + } + *output = points; *count = used; return 1; +} + +static int h3_letter(uint32_t value) { + int8_t category = u_charType((UChar32)value); + return category == U_UPPERCASE_LETTER || category == U_LOWERCASE_LETTER || + category == U_TITLECASE_LETTER || category == U_MODIFIER_LETTER || + category == U_OTHER_LETTER; +} + +static int h3_number(uint32_t value) { + int8_t category = u_charType((UChar32)value); + return category == U_DECIMAL_DIGIT_NUMBER || category == U_LETTER_NUMBER || + category == U_OTHER_NUMBER; +} + +static int h3_space(uint32_t value) { + return u_isUWhiteSpace((UChar32)value) || (value >= 0x1c && value <= 0x1f); +} + +static char *h3_slice(const char *text, const h3_codepoint *points, + size_t start, size_t stop) { + size_t offset = points[start].offset; + size_t end = points[stop - 1].offset + points[stop - 1].length; + char *result = malloc(end - offset + 1); + if (!result) return NULL; + memcpy(result, text + offset, end - offset); result[end - offset] = '\0'; + return result; +} + +static size_t h3_contraction(const h3_codepoint *points, size_t count, + size_t index) { + static const char *values[] = {"'s", "'t", "'re", "'ve", "'m", "'ll", "'d"}; + if (points[index].value != '\'') return 0; + for (size_t item = 0; item < sizeof(values) / sizeof(values[0]); item++) { + size_t length = strlen(values[item]); + if (index + length > count) continue; + int matches = 1; + for (size_t offset = 1; offset < length; offset++) { + uint32_t got = points[index + offset].value; + if (got >= 'A' && got <= 'Z') got += 'a' - 'A'; + if (got != (unsigned char)values[item][offset]) matches = 0; + } + if (matches) return length; + } + return 0; +} + +static int h3_pretokenize(const char *input, h3_strings *pieces) { + char *text = h3_nfc(input); + if (!text) return 0; + h3_codepoint *points = NULL; + size_t count = 0; + if (!h3_codepoints(text, &points, &count)) { free(text); return 0; } + size_t index = 0; + while (index < count) { + size_t contraction = h3_contraction(points, count, index); + size_t stop = index; + if (contraction) stop = index + contraction; + else { + uint32_t value = points[index].value; + ptrdiff_t letter_start = (ptrdiff_t)index; + if (!h3_letter(value)) { + if (value != '\r' && value != '\n' && !h3_number(value) && + index + 1 < count && h3_letter(points[index + 1].value)) + letter_start++; + else letter_start = -1; + } + if (letter_start >= 0) { + stop = (size_t)letter_start; + while (stop < count && h3_letter(points[stop].value)) stop++; + } else if (h3_number(value)) stop = index + 1; + else { + size_t punct_start = index + + (value == ' ' && index + 1 < count && + !h3_space(points[index + 1].value) && + !h3_letter(points[index + 1].value) && + !h3_number(points[index + 1].value)); + stop = punct_start; + while (stop < count && !h3_space(points[stop].value) && + !h3_letter(points[stop].value) && + !h3_number(points[stop].value)) stop++; + if (stop > punct_start) { + while (stop < count && (points[stop].value == '\r' || + points[stop].value == '\n')) stop++; + } else if (h3_space(value)) { + size_t whitespace_end = index + 1; + while (whitespace_end < count && h3_space(points[whitespace_end].value)) whitespace_end++; + ptrdiff_t newline_end = -1; + for (size_t cursor = index; cursor < whitespace_end; cursor++) + if (points[cursor].value == '\r' || points[cursor].value == '\n') + newline_end = (ptrdiff_t)cursor + 1; + if (newline_end >= 0) stop = (size_t)newline_end; + else if (whitespace_end == count) stop = whitespace_end; + else if (whitespace_end - index > 1) stop = whitespace_end - 1; + else stop = index + 1; + } else { free(points); free(text); return 0; } + } + } + char *piece = h3_slice(text, points, index, stop); + if (!piece || !h3_strings_push(pieces, piece)) { + free(piece); free(points); free(text); return 0; + } + index = stop; + } + free(points); free(text); return 1; +} + +static int h3_bpe(const h3_tokenizer *tokenizer, const char *piece, + h3_ids *output) { + h3_strings symbols = {0}; + for (const unsigned char *byte = (const unsigned char *)piece; *byte; byte++) { + char *symbol = strdup(tokenizer->byte_encoder[*byte]); + if (!symbol || !h3_strings_push(&symbols, symbol)) { + free(symbol); h3_strings_free(&symbols); return 0; + } + } + while (symbols.count > 1) { + uint32_t best_rank = UINT32_MAX; + size_t best = SIZE_MAX; + for (size_t index = 0; index + 1 < symbols.count; index++) { + char *key = h3_pair_key(symbols.values[index], symbols.values[index + 1]); + uint32_t rank; + int found = key && h3_map_get(&tokenizer->merges, key, &rank); + free(key); + if (found && rank < best_rank) { best_rank = rank; best = index; } + } + if (best == SIZE_MAX) break; + const char *left = symbols.values[best], *right = symbols.values[best + 1]; + h3_strings merged = {0}; + for (size_t index = 0; index < symbols.count;) { + if (index + 1 < symbols.count && !strcmp(symbols.values[index], left) && + !strcmp(symbols.values[index + 1], right)) { + size_t a = strlen(left), b = strlen(right); + char *value = malloc(a + b + 1); + if (value) { memcpy(value, left, a); memcpy(value + a, right, b + 1); } + if (!value || !h3_strings_push(&merged, value)) { + free(value); h3_strings_free(&merged); h3_strings_free(&symbols); return 0; + } + index += 2; + } else { + char *value = strdup(symbols.values[index++]); + if (!value || !h3_strings_push(&merged, value)) { + free(value); h3_strings_free(&merged); h3_strings_free(&symbols); return 0; + } + } + } + h3_strings_free(&symbols); symbols = merged; + } + for (size_t index = 0; index < symbols.count; index++) { + uint32_t identifier; + if (!h3_map_get(&tokenizer->vocab, symbols.values[index], &identifier) || + !h3_ids_push(output, identifier)) { + h3_strings_free(&symbols); return 0; + } + } + h3_strings_free(&symbols); return 1; +} + +static int h3_encode_plain(const h3_tokenizer *tokenizer, const char *text, + h3_ids *output) { + h3_strings pieces = {0}; + if (!h3_pretokenize(text, &pieces)) return 0; + for (size_t index = 0; index < pieces.count; index++) + if (!h3_bpe(tokenizer, pieces.values[index], output)) { + h3_strings_free(&pieces); return 0; + } + h3_strings_free(&pieces); return 1; +} + +static int h3_added_match(const h3_tokenizer *tokenizer, const char *text, + size_t start, size_t *offset, size_t *length, + uint32_t *identifier) { + int found = 0; + for (size_t index = 0; index < tokenizer->added.capacity; index++) { + h3_map_item item = tokenizer->added.items[index]; + if (!item.key) continue; + const char *match = strstr(text + start, item.key); + if (!match) continue; + size_t at = (size_t)(match - text), size = strlen(item.key); + if (!found || at < *offset || (at == *offset && size > *length)) { + found = 1; *offset = at; *length = size; *identifier = item.value; + } + } + return found; +} + +int h3_tokenizer_encode(const h3_tokenizer *tokenizer, const char *utf8, + int pad_empty, uint32_t **ids, size_t *count, + char *error, size_t error_size) { + if (error && error_size) error[0] = '\0'; + if (!tokenizer || !utf8 || !ids || !count) return 0; + *ids = NULL; *count = 0; + h3_codepoint *validation = NULL; size_t validation_count = 0; + if (!h3_codepoints(utf8, &validation, &validation_count)) { + h3_error(error, error_size, "prompt is not valid UTF-8"); return 0; + } + free(validation); + h3_ids output = {0}; + size_t start = 0, text_length = strlen(utf8); + while (start < text_length) { + size_t offset = 0, length = 0; uint32_t identifier = 0; + if (!h3_added_match(tokenizer, utf8, start, &offset, &length, &identifier)) break; + if (offset > start) { + char *plain = strndup(utf8 + start, offset - start); + int ok = plain && h3_encode_plain(tokenizer, plain, &output); + free(plain); + if (!ok) goto failure; + } + if (!h3_ids_push(&output, identifier)) goto failure; + start = offset + length; + } + if (start < text_length && !h3_encode_plain(tokenizer, utf8 + start, &output)) goto failure; + if (!output.count && pad_empty && !h3_ids_push(&output, H3_PAD_TOKEN_ID)) goto failure; + *ids = output.values; *count = output.count; return 1; +failure: + free(output.values); h3_error(error, error_size, "unable to encode prompt"); return 0; +} + +void h3_tokenizer_ids_free(uint32_t *ids) { free(ids); } + +static int h3_bytes_append(char **output, size_t *length, size_t *capacity, + const void *data, size_t bytes) { + if (*length + bytes + 1 > *capacity) { + size_t next = *capacity ? *capacity * 2 : 64; + while (next < *length + bytes + 1) next *= 2; + char *grown = realloc(*output, next); + if (!grown) return 0; + *output = grown; *capacity = next; + } + memcpy(*output + *length, data, bytes); *length += bytes; + (*output)[*length] = '\0'; return 1; +} + +char *h3_tokenizer_decode(const h3_tokenizer *tokenizer, + const uint32_t *ids, size_t count, + char *error, size_t error_size) { + if (error && error_size) error[0] = '\0'; + if (!tokenizer || (!ids && count)) return NULL; + char *result = NULL; + size_t length = 0, capacity = 0; + for (size_t index = 0; index < count; index++) { + uint32_t identifier = ids[index]; + if (identifier >= tokenizer->inverse_count) { + h3_error(error, error_size, "token ID is out of range"); free(result); return NULL; + } + const char *added = tokenizer->inverse_added[identifier]; + if (added) { + if (!h3_bytes_append(&result, &length, &capacity, added, strlen(added))) goto memory; + continue; + } + const char *symbol = tokenizer->inverse_vocab[identifier]; + if (!symbol) { + h3_error(error, error_size, "unknown token ID"); free(result); return NULL; + } + int32_t offset = 0, symbol_length = (int32_t)strlen(symbol); + while (offset < symbol_length) { + UChar32 codepoint; + U8_NEXT((const uint8_t *)symbol, offset, symbol_length, codepoint); + if (codepoint < 0 || codepoint >= 324 || + tokenizer->byte_decoder[codepoint] < 0) { + h3_error(error, error_size, "invalid byte-level token"); + free(result); return NULL; + } + unsigned char byte = (unsigned char)tokenizer->byte_decoder[codepoint]; + if (!h3_bytes_append(&result, &length, &capacity, &byte, 1)) goto memory; + } + } + if (!result) result = calloc(1, 1); + if (!result) goto memory; + h3_codepoint *validation = NULL; size_t validation_count = 0; + if (!h3_codepoints(result, &validation, &validation_count)) { + free(result); result = strdup("\xef\xbf\xbd"); + } + free(validation); + return result; +memory: + h3_error(error, error_size, "out of memory decoding tokens"); + free(result); return NULL; +} diff --git a/tests/test_host_portable.c b/tests/test_host_portable.c new file mode 100644 index 00000000..a9d4fd63 --- /dev/null +++ b/tests/test_host_portable.c @@ -0,0 +1,24 @@ +#include "h3_host.h" + +#include +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); return 1; \ +} } while (0) + +int main(void) { + const uint8_t constant[] = { + 17,33,201, 17,33,201, + 17,33,201, 17,33,201 + }; + uint8_t *output = NULL; + CHECK(h3_resize_rgb24_high_quality(constant, 1, 2, 2, 8, 8, &output)); + for (size_t pixel = 0; pixel < 64; pixel++) + CHECK(!memcmp(output + pixel * 3, constant, 3)); + free(output); + puts("ok: portable RGB resize"); + return 0; +} diff --git a/tests/test_tokenizer_portable.c b/tests/test_tokenizer_portable.c new file mode 100644 index 00000000..8457dfb0 --- /dev/null +++ b/tests/test_tokenizer_portable.c @@ -0,0 +1,47 @@ +#include "h3_tokenizer.h" + +#include +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); return 1; \ +} } while (0) + +static int check(h3_tokenizer *tokenizer, const char *text, + const uint32_t *expected, size_t expected_count, + const char *decoded_expected) { + char error[256]; + uint32_t *ids = NULL; + size_t count = 0; + CHECK(h3_tokenizer_encode(tokenizer, text, 0, &ids, &count, + error, sizeof(error))); + CHECK(count == expected_count); + CHECK(!memcmp(ids, expected, count * sizeof(*ids))); + char *decoded = h3_tokenizer_decode(tokenizer, ids, count, + error, sizeof(error)); + CHECK(decoded && !strcmp(decoded, decoded_expected)); + free(decoded); h3_tokenizer_ids_free(ids); return 0; +} + +int main(int argc, char **argv) { + const char *path = argc > 1 ? argv[1] : + "tests/tokenizer_portable_fixture.json"; + char error[256]; + h3_tokenizer *tokenizer = h3_tokenizer_load(path, error, sizeof(error)); + CHECK(tokenizer != NULL); + const uint32_t spaced[] = {1, 3}; + CHECK(!check(tokenizer, "A A", spaced, 2, "A A")); + const uint32_t normalized[] = {4}; + CHECK(!check(tokenizer, "e\xcc\x81", normalized, 1, "\xc3\xa9")); + const uint32_t added[] = {10, 1}; + CHECK(!check(tokenizer, "A", added, 2, "A")); + uint32_t *ids = NULL; size_t count = 99; + CHECK(h3_tokenizer_encode(tokenizer, "", 1, &ids, &count, + error, sizeof(error))); + CHECK(count == 1 && ids[0] == H3_PAD_TOKEN_ID); + h3_tokenizer_ids_free(ids); h3_tokenizer_free(tokenizer); + puts("ok: portable ICU byte-level BPE tokenizer"); + return 0; +} diff --git a/tests/tokenizer_portable_fixture.json b/tests/tokenizer_portable_fixture.json new file mode 100644 index 00000000..7eb5c7ea --- /dev/null +++ b/tests/tokenizer_portable_fixture.json @@ -0,0 +1,13 @@ +{ + "model": { + "type": "BPE", + "unk_token": null, + "vocab": {"A": 1, "Ġ": 2, "ĠA": 3, "é": 4, "Ã": 5, "©": 6}, + "merges": ["Ġ A", ["Ã", "©"]] + }, + "normalizer": {"type": "NFC"}, + "added_tokens": [ + {"id": 10, "content": "", "single_word": false, + "lstrip": false, "rstrip": false, "normalized": false} + ] +} From 27396a058e2d5b80425e9dbdc4584bf3c46366b4 Mon Sep 17 00:00:00 2001 From: matrixfede Date: Thu, 20 Aug 2026 09:16:23 +0200 Subject: [PATCH 02/20] Add CUDA backend implementing the h3_gpu API with Linux build selection --- Makefile | 163 +- h3_device.h | 16 + h3_device_cuda.cu | 50 + h3_gpu.h | 8 + h3_gpu_cuda.cu | 3514 ++++++++++++++++++++++++++++++++ tests/test_checkpoint_schema.c | 102 + tests/test_cuda_attention.c | 352 ++++ tests/test_cuda_linear.c | 310 +++ tests/test_cuda_ops.c | 229 +++ tests/test_cuda_primitives.c | 201 ++ tests/test_cuda_rope_tokens.c | 234 +++ tests/test_cuda_runtime.c | 107 + tests/test_device.c | 23 + 13 files changed, 5295 insertions(+), 14 deletions(-) create mode 100644 h3_device.h create mode 100644 h3_device_cuda.cu create mode 100644 h3_gpu_cuda.cu create mode 100644 tests/test_checkpoint_schema.c create mode 100644 tests/test_cuda_attention.c create mode 100644 tests/test_cuda_linear.c create mode 100644 tests/test_cuda_ops.c create mode 100644 tests/test_cuda_primitives.c create mode 100644 tests/test_cuda_rope_tokens.c create mode 100644 tests/test_cuda_runtime.c create mode 100644 tests/test_device.c diff --git a/Makefile b/Makefile index bb202379..d7bdd3fb 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,94 @@ -CC := clang -AR := ar +PLATFORM ?= $(shell uname -s) +.DEFAULT_GOAL := all +AR ?= ar CFLAGS := -std=c11 -O3 -MMD -MP -Wall -Wextra -Wpedantic -Wshadow \ - -Wconversion -Wno-sign-conversion -D_DARWIN_C_SOURCE -OBJCFLAGS := $(CFLAGS) -fobjc-arc + -Wconversion -Wno-sign-conversion + +ifeq ($(PLATFORM),Darwin) +CC ?= clang +OBJCFLAGS := $(CFLAGS) -D_DARWIN_C_SOURCE -fobjc-arc FRAMEWORKS := -framework Foundation -framework Metal \ -framework MetalPerformanceShaders -framework MetalPerformanceShadersGraph \ -framework Accelerate LDLIBS := $(FRAMEWORKS) -licucore -lm +DEVICE_LDLIBS := $(FRAMEWORKS) +DEVICE_SRC := h3_metal.m +GPU_SRC := h3_gpu.m +TOKENIZER_SRC := h3_tokenizer.m +TOKENIZER_OBJ := h3_tokenizer_metal.o +BACKEND := metal +PLATFORM_LD := $(CC) +else ifeq ($(PLATFORM),Linux) +CC ?= cc +NVCC ?= nvcc +CUDA_HOME ?= /usr/local/cuda +NVCC_ARCH ?= native +NVCCFLAGS ?= -O3 -std=c++17 -arch=$(NVCC_ARCH) -Xcompiler=-Wall,-Wextra,-Wshadow +CPPFLAGS += -D_POSIX_C_SOURCE=200809L -I$(CUDA_HOME)/include +CUDA_LDLIBS := -L$(CUDA_HOME)/lib64 -lcudart -lcublasLt +CUDNN_ROOT ?= +CUDNN_FRONTEND_ROOT ?= +ifneq ($(strip $(CUDNN_ROOT)$(CUDNN_FRONTEND_ROOT)),) +ifeq ($(strip $(CUDNN_ROOT)),) +$(error CUDNN_ROOT is required when enabling cuDNN attention) +endif +ifeq ($(strip $(CUDNN_FRONTEND_ROOT)),) +$(error CUDNN_FRONTEND_ROOT is required when enabling cuDNN attention) +endif +CPPFLAGS += -DH3_USE_CUDNN -isystem $(CUDNN_ROOT)/include \ + -isystem $(CUDNN_FRONTEND_ROOT)/include +CUDNN_LDLIBS := -L$(CUDNN_ROOT)/lib \ + -Xlinker -rpath -Xlinker $(CUDNN_ROOT)/lib -l:libcudnn.so.9 \ + -lnvrtc -lcuda +CUDA_LDLIBS += $(CUDNN_LDLIBS) +endif +LDLIBS := $(CUDA_LDLIBS) -lstdc++ -licui18n -licuuc -lm +DEVICE_LDLIBS := -L$(CUDA_HOME)/lib64 -lcudart +DEVICE_SRC := h3_device_cuda.cu +GPU_SRC := h3_gpu_cuda.cu +TOKENIZER_SRC := h3_tokenizer.c +TOKENIZER_OBJ := h3_tokenizer.o +BACKEND := cuda +PLATFORM_LD := $(NVCC) +else +$(error unsupported PLATFORM '$(PLATFORM)'; expected Darwin or Linux) +endif LIB_C := h3.c h3_host.c h3_safetensors.c h3_weights.c h3_text_encoder.c \ h3_dit_schedule.c h3_dit.c LIB_C += h3_video_vae.c h3_video_encoder.c h3_audio_vae.c h3_ffmpeg.c \ h3_terminal.c h3_vision_encoder.c h3_multimodal.c -LIB_M := h3_metal.m h3_gpu.m h3_tokenizer.m -LIB_OBJ := $(LIB_C:.c=.o) $(LIB_M:.m=.o) +LIB_PLATFORM := $(DEVICE_SRC) $(GPU_SRC) $(TOKENIZER_SRC) +DEVICE_OBJ := $(DEVICE_SRC:.m=.o) +DEVICE_OBJ := $(DEVICE_OBJ:.cu=.o) +GPU_OBJ := $(GPU_SRC:.m=.o) +GPU_OBJ := $(GPU_OBJ:.cu=.o) +LIB_OBJ := $(LIB_C:.c=.o) $(DEVICE_OBJ) $(GPU_OBJ) $(TOKENIZER_OBJ) CLI_OBJ := main.o h3_cli.o linenoise.o -.PHONY: all test parity real-parity clean +.PHONY: all test host-portable-test tokenizer-portable-test checkpoint-schema-test cuda-runtime-test cuda-primitives-test cuda-rope-tokens-test cuda-linear-test cuda-attention-test cuda-ops-test parity real-parity print-build-config clean + +print-build-config: + @echo "platform=$(PLATFORM) backend=$(BACKEND) cc=$(CC) sources=$(LIB_PLATFORM)" + +h3_host_portable_test: tests/test_host_portable.o h3_host.o + $(CC) -o $@ $^ -lm + +host-portable-test: h3_host_portable_test + ./h3_host_portable_test + +h3_tokenizer_portable_test: tests/test_tokenizer_portable.o h3_tokenizer.o + $(CC) -o $@ $^ -licui18n -licuuc + +tokenizer-portable-test: h3_tokenizer_portable_test + ./h3_tokenizer_portable_test + +h3_checkpoint_schema_test: tests/test_checkpoint_schema.o h3_safetensors.o h3_weights.o h3_gpu_cuda.o + $(NVCC) -o $@ $^ $(CUDA_LDLIBS) + +checkpoint-schema-test: h3_checkpoint_schema_test + ./h3_checkpoint_schema_test MiniMax-H3 all: h3 libh3.a @@ -30,14 +101,53 @@ libh3.a: $(LIB_OBJ) h3_tests: tests/test_h3.o $(LIB_OBJ) $(CC) -o $@ $^ $(LDLIBS) +h3_device_test: tests/test_device.o $(DEVICE_OBJ) + $(PLATFORM_LD) -o $@ $^ $(DEVICE_LDLIBS) + +h3_cuda_runtime_test: tests/test_cuda_runtime.o h3_gpu_cuda.o + $(NVCC) -o $@ $^ $(CUDA_LDLIBS) + +cuda-runtime-test: h3_cuda_runtime_test + ./h3_cuda_runtime_test + +h3_cuda_primitives_test: tests/test_cuda_primitives.o h3_gpu_cuda.o + $(NVCC) -o $@ $^ $(CUDA_LDLIBS) + +cuda-primitives-test: h3_cuda_primitives_test + ./h3_cuda_primitives_test + +h3_cuda_rope_tokens_test: tests/test_cuda_rope_tokens.o h3_gpu_cuda.o + $(NVCC) -o $@ $^ $(CUDA_LDLIBS) + +cuda-rope-tokens-test: h3_cuda_rope_tokens_test + ./h3_cuda_rope_tokens_test + +h3_cuda_linear_test: tests/test_cuda_linear.o h3_gpu_cuda.o + $(NVCC) -o $@ $^ $(CUDA_LDLIBS) + +cuda-linear-test: h3_cuda_linear_test + ./h3_cuda_linear_test + +h3_cuda_attention_test: tests/test_cuda_attention.o h3_gpu_cuda.o + $(NVCC) -o $@ $^ $(CUDA_LDLIBS) + +cuda-attention-test: h3_cuda_attention_test + ./h3_cuda_attention_test + +h3_cuda_ops_test: tests/test_cuda_ops.o h3_gpu_cuda.o + $(NVCC) -o $@ $^ $(CUDA_LDLIBS) + +cuda-ops-test: h3_cuda_ops_test + ./h3_cuda_ops_test + h3_metal_tests: tests/test_metal.o $(LIB_OBJ) $(CC) -o $@ $^ $(LDLIBS) h3_bf16_tests: tests/test_bf16.o $(LIB_OBJ) $(CC) -o $@ $^ $(LDLIBS) -h3_tokenizer_tests: tests/test_tokenizer.o $(LIB_OBJ) - $(CC) -o $@ $^ $(LDLIBS) +h3_tokenizer_tests: tests/test_tokenizer.o $(TOKENIZER_OBJ) + $(CC) -o $@ $^ $(if $(filter Darwin,$(PLATFORM)),-licucore,-licui18n -licuuc) h3_text_tests: tests/test_text_metal.o $(LIB_OBJ) $(CC) -o $@ $^ $(LDLIBS) @@ -91,9 +201,27 @@ tests/bench_dit_864.o: tests/bench_dit.c $(CC) $(CFLAGS) -I. -DH3_BENCH_LATENT_H=30 \ -DH3_BENCH_LATENT_W=54 -c $< -o $@ +h3_dit_bench_quality: tests/bench_dit_quality.o $(LIB_OBJ) + $(CC) -o $@ $^ $(LDLIBS) + +tests/bench_dit_quality.o: tests/bench_dit.c + $(CC) $(CPPFLAGS) $(CFLAGS) -I. -DH3_BENCH_LATENT_H=36 \ + -DH3_BENCH_LATENT_W=64 -DH3_BENCH_LATENT_T=32 \ + -DH3_BENCH_AUDIO_T=178 -c $< -o $@ + h3_real_video_vae_test: tests/test_real_video_vae.o $(LIB_OBJ) $(CC) -o $@ $^ $(LDLIBS) +h3_vae_bench_quality: tests/bench_video_vae.o $(LIB_OBJ) + $(CC) -o $@ $^ $(LDLIBS) + +h3_vae_bench_smoke: tests/bench_video_vae_smoke.o $(LIB_OBJ) + $(CC) -o $@ $^ $(LDLIBS) + +tests/bench_video_vae_smoke.o: tests/bench_video_vae.c + $(CC) $(CPPFLAGS) $(CFLAGS) -I. -DH3_BENCH_VAE_LATENT_T=7 \ + -DH3_BENCH_VAE_LATENT_H=4 -DH3_BENCH_VAE_LATENT_W=4 -c $< -o $@ + h3_semantic_vae_test: tests/test_semantic_vae.o $(LIB_OBJ) $(CC) -o $@ $^ $(LDLIBS) @@ -188,13 +316,19 @@ real-parity: h3_real_prompt_test h3_real_dit_block_test ./h3_real_dit_block_test MiniMax-H3 misc/fixtures/h3_real_dit_block0_bf16.safetensors %.o: %.c - $(CC) $(CFLAGS) -I. -c $< -o $@ + $(CC) $(CPPFLAGS) $(CFLAGS) -I. -c $< -o $@ %.o: %.m - $(CC) $(OBJCFLAGS) -I. -c $< -o $@ + $(CC) $(CPPFLAGS) $(OBJCFLAGS) -I. -c $< -o $@ + +h3_tokenizer_metal.o: h3_tokenizer.m + $(CC) $(CPPFLAGS) $(OBJCFLAGS) -I. -c $< -o $@ + +%.o: %.cu + $(NVCC) $(CPPFLAGS) $(NVCCFLAGS) -I. -c $< -o $@ tests/%.o: tests/%.c - $(CC) $(CFLAGS) -I. -c $< -o $@ + $(CC) $(CPPFLAGS) $(CFLAGS) -I. -c $< -o $@ # Vendored from Iris. Keep the main project strict without rewriting this small # terminal editor for conversion diagnostics unrelated to H3. @@ -203,7 +337,7 @@ linenoise.o: CFLAGS += -Wno-conversion -Wno-variadic-macro-arguments-omitted -include $(wildcard *.d tests/*.d) clean: - rm -f h3 h3_tests h3_metal_tests h3_bf16_tests h3_tokenizer_tests \ + rm -f h3 h3_tests h3_device_test h3_host_portable_test h3_tokenizer_portable_test h3_checkpoint_schema_test h3_cuda_runtime_test h3_cuda_primitives_test h3_cuda_rope_tokens_test h3_cuda_linear_test h3_cuda_attention_test h3_cuda_ops_test h3_metal_tests h3_bf16_tests h3_tokenizer_tests \ h3_text_tests h3_real_prompt_test h3_real_dit_block_test \ h3_audio_gpu_tests h3_real_audio_vae_test h3_real_audio_encoder_test \ h3_av_mux_test \ @@ -211,5 +345,6 @@ clean: h3_real_multimodal_text_test h3_real_ref_video_text_test \ h3_real_dit_schedule_test h3_real_dit_test h3_semantic_dit_test \ h3_real_video_vae_test h3_semantic_vae_test \ - h3_dit_bench h3_dit_bench_864 \ + h3_dit_bench h3_dit_bench_864 h3_dit_bench_quality h3_vae_bench_quality \ + h3_vae_bench_smoke \ libh3.a *.o *.d tests/*.o tests/*.d diff --git a/h3_device.h b/h3_device.h new file mode 100644 index 00000000..e3be12ab --- /dev/null +++ b/h3_device.h @@ -0,0 +1,16 @@ +#ifndef H3_DEVICE_H +#define H3_DEVICE_H + +#include "h3.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int h3_device_probe(h3_device_info *info, char *error, size_t error_size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/h3_device_cuda.cu b/h3_device_cuda.cu new file mode 100644 index 00000000..f78bce6c --- /dev/null +++ b/h3_device_cuda.cu @@ -0,0 +1,50 @@ +#include "h3_device.h" + +#include + +#include +#include +#include + +static void h3_cuda_error(char *error, size_t error_size, const char *operation, + cudaError_t status) { + if (error && error_size) { + snprintf(error, error_size, "%s: %s", operation, + cudaGetErrorString(status)); + } +} + +extern "C" int h3_device_probe(h3_device_info *info, char *error, + size_t error_size) { + if (!info) { + if (error && error_size) snprintf(error, error_size, "device info is required"); + return 0; + } + memset(info, 0, sizeof(*info)); + + int device = 0; + cudaError_t status = cudaGetDevice(&device); + if (status != cudaSuccess) { + h3_cuda_error(error, error_size, "cannot select CUDA device", status); + return 0; + } + cudaDeviceProp properties; + status = cudaGetDeviceProperties(&properties, device); + if (status != cudaSuccess) { + h3_cuda_error(error, error_size, "cannot inspect CUDA device", status); + return 0; + } + + snprintf(info->name, sizeof(info->name), "%.127s", properties.name); + snprintf(info->architecture, sizeof(info->architecture), "CUDA sm_%d%d", + properties.major, properties.minor); + struct sysinfo system; + if (sysinfo(&system) == 0) { + info->physical_memory = + (uint64_t)system.totalram * (uint64_t)system.mem_unit; + } + info->recommended_working_set = (uint64_t)properties.totalGlobalMem; + info->max_buffer_length = (uint64_t)properties.totalGlobalMem; + info->unified_memory = properties.unifiedAddressing ? 1 : 0; + return 1; +} diff --git a/h3_gpu.h b/h3_gpu.h index 3a47cc35..7fb2871f 100644 --- a/h3_gpu.h +++ b/h3_gpu.h @@ -4,6 +4,10 @@ #include #include +#ifdef __cplusplus +extern "C" { +#endif + typedef struct h3_gpu h3_gpu; typedef struct h3_gpu_tensor h3_gpu_tensor; @@ -610,4 +614,8 @@ int h3_gpu_silu_mul_bf16(h3_gpu *gpu, h3_gpu_tensor *output, const h3_gpu_tensor *gate, const h3_gpu_tensor *up, uint32_t elements); +#ifdef __cplusplus +} +#endif + #endif diff --git a/h3_gpu_cuda.cu b/h3_gpu_cuda.cu new file mode 100644 index 00000000..a5a74573 --- /dev/null +++ b/h3_gpu_cuda.cu @@ -0,0 +1,3514 @@ +#include "h3_gpu.h" + +#include +#include +#include + +#ifdef H3_USE_CUDNN +#include +#include +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef H3_USE_CUDNN +namespace h3_fe = cudnn_frontend; + +struct h3_cudnn_sdpa { + uint32_t sequence; + uint32_t heads; + float scale; + int head_major_output; + int ready; + cudnnHandle_t handle; + std::shared_ptr graph; + void *workspace; + h3_cudnn_sdpa *next; +}; +#endif + +struct h3_gpu { + cudaStream_t stream; + cublasLtHandle_t blas; + cudaEvent_t begin_event; + cudaEvent_t end_event; + cudaEvent_t continue_event; + h3_gpu_stats stats; + char error[512]; + char profile_label[128]; + double profile_mark_time; + double encode_start_time; + int recording; +#ifdef H3_USE_CUDNN + h3_cudnn_sdpa *cudnn_sdpa; +#endif +}; + +struct h3_gpu_tensor { + h3_gpu *gpu; + void *data; + size_t elements = 0; + size_t bytes; + h3_gpu_dtype dtype; +}; + +static double h3_wall_time(void) { + struct timespec value; + if (clock_gettime(CLOCK_MONOTONIC, &value) != 0) return 0.0; + return (double)value.tv_sec + (double)value.tv_nsec * 1e-9; +} + +static int h3_set_error(h3_gpu *gpu, const char *format, ...) { + if (gpu) { + va_list arguments; + va_start(arguments, format); + vsnprintf(gpu->error, sizeof(gpu->error), format, arguments); + va_end(arguments); + } + return 0; +} + +static int h3_cuda_ok(h3_gpu *gpu, cudaError_t status, + const char *operation) { + if (status == cudaSuccess) return 1; + return h3_set_error(gpu, "%s: %s", operation, cudaGetErrorString(status)); +} + +#ifdef H3_USE_CUDNN +static void h3_cudnn_sdpa_free(h3_cudnn_sdpa *entry) { + while (entry) { + h3_cudnn_sdpa *next = entry->next; + if (entry->workspace) (void)cudaFree(entry->workspace); + entry->graph.reset(); + if (entry->handle) (void)cudnnDestroy(entry->handle); + delete entry; + entry = next; + } +} + +static h3_cudnn_sdpa *h3_cudnn_sdpa_get( + h3_gpu *gpu, uint32_t sequence, uint32_t heads, float scale, + int head_major_output, char *reason, size_t reason_size) { + for (h3_cudnn_sdpa *entry = gpu->cudnn_sdpa; entry; entry = entry->next) + if (entry->sequence == sequence && entry->heads == heads && + entry->scale == scale && + entry->head_major_output == head_major_output) + return entry; + + h3_cudnn_sdpa *entry = new (std::nothrow) h3_cudnn_sdpa{}; + if (!entry) { + snprintf(reason, reason_size, "out of memory creating cuDNN SDPA cache"); + return NULL; + } + entry->sequence = sequence; + entry->heads = heads; + entry->scale = scale; + entry->head_major_output = head_major_output; + entry->next = gpu->cudnn_sdpa; + gpu->cudnn_sdpa = entry; + + try { + cudnnStatus_t cudnn_status = cudnnCreate(&entry->handle); + if (cudnn_status != CUDNN_STATUS_SUCCESS) { + snprintf(reason, reason_size, "cudnnCreate: %s", + cudnnGetErrorString(cudnn_status)); + return entry; + } + cudnn_status = cudnnSetStream(entry->handle, gpu->stream); + if (cudnn_status != CUDNN_STATUS_SUCCESS) { + snprintf(reason, reason_size, "cudnnSetStream: %s", + cudnnGetErrorString(cudnn_status)); + return entry; + } + + enum { Q_UID = 1, K_UID = 2, V_UID = 3, O_UID = 4 }; + int64_t b = 1, h = heads, s = sequence, d = 128; + entry->graph = std::make_shared(); + entry->graph->set_io_data_type(h3_fe::DataType_t::BFLOAT16) + .set_intermediate_data_type(h3_fe::DataType_t::FLOAT) + .set_compute_data_type(h3_fe::DataType_t::FLOAT); + auto q = entry->graph->tensor( + h3_fe::graph::Tensor_attributes() + .set_name("Q").set_uid(Q_UID) + .set_dim({b, h, s, d}) + .set_stride({h * s * d, s * d, d, 1})); + auto k = entry->graph->tensor( + h3_fe::graph::Tensor_attributes() + .set_name("K").set_uid(K_UID) + .set_dim({b, h, s, d}) + .set_stride({h * s * d, s * d, d, 1})); + auto v = entry->graph->tensor( + h3_fe::graph::Tensor_attributes() + .set_name("V").set_uid(V_UID) + .set_dim({b, h, s, d}) + .set_stride({h * s * d, s * d, d, 1})); + auto options = h3_fe::graph::SDPA_attributes() + .set_name("h3_sdpa") + .set_generate_stats(false) + .set_attn_scale(scale); + auto result = entry->graph->sdpa(q, k, v, options); + auto output = result[0]; + output->set_output(true).set_uid(O_UID).set_dim({b, h, s, d}); + if (head_major_output) + output->set_stride({h * s * d, s * d, d, 1}); + else + output->set_stride({h * s * d, d, h * d, 1}); + + auto status = entry->graph->build(entry->handle, + {h3_fe::HeurMode_t::A}); + if (!status.is_good()) { + snprintf(reason, reason_size, "cuDNN graph build: %s", + status.get_message().c_str()); + return entry; + } + int64_t workspace_size = 0; + auto workspace_status = + entry->graph->get_workspace_size(workspace_size); + if (!workspace_status.is_good() || workspace_size < 0) { + snprintf(reason, reason_size, "cuDNN workspace query: %s", + workspace_status.get_message().c_str()); + return entry; + } + if (workspace_size > 0) { + cudaError_t cuda_status = cudaMalloc(&entry->workspace, + (size_t)workspace_size); + if (cuda_status != cudaSuccess) { + snprintf(reason, reason_size, "cuDNN workspace: %s", + cudaGetErrorString(cuda_status)); + return entry; + } + } + entry->ready = 1; + return entry; + } catch (const std::exception &error) { + snprintf(reason, reason_size, "cuDNN frontend: %s", error.what()); + return entry; + } +} + +static int h3_cudnn_sdpa_execute( + h3_gpu *gpu, h3_cudnn_sdpa *entry, void *output, + const void *query, const void *key, const void *value, + char *reason, size_t reason_size) { + enum { Q_UID = 1, K_UID = 2, V_UID = 3, O_UID = 4 }; + if (!entry || !entry->ready) return 0; + std::unordered_map pointers = { + {Q_UID, const_cast(query)}, + {K_UID, const_cast(key)}, + {V_UID, const_cast(value)}, + {O_UID, output}, + }; + cudnnStatus_t cudnn_status = cudnnSetStream(entry->handle, gpu->stream); + if (cudnn_status != CUDNN_STATUS_SUCCESS) { + snprintf(reason, reason_size, "cudnnSetStream: %s", + cudnnGetErrorString(cudnn_status)); + return 0; + } + auto status = entry->graph->execute(entry->handle, pointers, + entry->workspace); + if (!status.is_good()) { + snprintf(reason, reason_size, "cuDNN SDPA execute: %s", + status.get_message().c_str()); + return 0; + } + return 1; +} +#endif + +static size_t h3_dtype_size(h3_gpu_dtype dtype) { + switch (dtype) { + case H3_GPU_F32: return sizeof(float); + case H3_GPU_BF16: return sizeof(uint16_t); + case H3_GPU_I8: return sizeof(int8_t); + case H3_GPU_U32: return sizeof(uint32_t); + } + return 0; +} + +static h3_gpu_tensor *h3_tensor_new(h3_gpu *gpu, size_t elements, + h3_gpu_dtype dtype) { + if (!gpu) return NULL; + size_t item_size = h3_dtype_size(dtype); + if (!item_size || elements > SIZE_MAX / item_size) { + h3_set_error(gpu, "invalid or overflowing tensor size"); + return NULL; + } + h3_gpu_tensor *tensor = (h3_gpu_tensor *)calloc(1, sizeof(*tensor)); + if (!tensor) { + h3_set_error(gpu, "out of memory allocating tensor metadata"); + return NULL; + } + tensor->gpu = gpu; + tensor->elements = elements; + tensor->bytes = elements * item_size; + tensor->dtype = dtype; + if (tensor->bytes && !h3_cuda_ok(gpu, cudaMalloc(&tensor->data, tensor->bytes), + "cudaMalloc")) { + free(tensor); + return NULL; + } + gpu->stats.allocated_bytes += tensor->bytes; + gpu->stats.live_bytes += tensor->bytes; + if (gpu->stats.live_bytes > gpu->stats.peak_live_bytes) + gpu->stats.peak_live_bytes = gpu->stats.live_bytes; + gpu->stats.tensor_allocations++; + return tensor; +} + +static int h3_require_tensor(const h3_gpu_tensor *tensor, + h3_gpu_dtype dtype, size_t elements) { + return tensor && tensor->dtype == dtype && tensor->elements >= elements; +} + +h3_gpu *h3_gpu_create(const char *shader_source_path, + char *error, size_t error_size) { + (void)shader_source_path; + h3_gpu *gpu = (h3_gpu *)calloc(1, sizeof(*gpu)); + if (!gpu) { + if (error && error_size) snprintf(error, error_size, "out of memory"); + return NULL; + } + cudaError_t status = cudaStreamCreateWithFlags(&gpu->stream, + cudaStreamNonBlocking); + cublasStatus_t blas_status = CUBLAS_STATUS_SUCCESS; + if (status == cudaSuccess) blas_status = cublasLtCreate(&gpu->blas); + if (status == cudaSuccess && blas_status != CUBLAS_STATUS_SUCCESS) + status = cudaErrorInitializationError; + if (status == cudaSuccess) status = cudaEventCreate(&gpu->begin_event); + if (status == cudaSuccess) status = cudaEventCreate(&gpu->end_event); + if (status == cudaSuccess) status = cudaEventCreate(&gpu->continue_event); + if (status != cudaSuccess) { + if (error && error_size) + snprintf(error, error_size, "CUDA initialization: %s", + cudaGetErrorString(status)); + if (gpu->continue_event) cudaEventDestroy(gpu->continue_event); + if (gpu->end_event) cudaEventDestroy(gpu->end_event); + if (gpu->begin_event) cudaEventDestroy(gpu->begin_event); + if (gpu->blas) cublasLtDestroy(gpu->blas); + if (gpu->stream) cudaStreamDestroy(gpu->stream); + free(gpu); + return NULL; + } + snprintf(gpu->profile_label, sizeof(gpu->profile_label), "CUDA context"); + gpu->profile_mark_time = h3_wall_time(); + if (error && error_size) error[0] = '\0'; + return gpu; +} + +void h3_gpu_free(h3_gpu *gpu) { + if (!gpu) return; + cudaError_t status = cudaStreamSynchronize(gpu->stream); + if (getenv("H3_PROFILE")) { + fprintf(stderr, "%s: %.6fs GPU, %.6fs encode, %.6fs wait, " + "%llu bytes peak, %llu submissions%s%s\n", + gpu->profile_label, gpu->stats.gpu_seconds, + gpu->stats.command_encode_seconds, + gpu->stats.command_wait_seconds, + (unsigned long long)gpu->stats.peak_live_bytes, + (unsigned long long)gpu->stats.submissions, + status == cudaSuccess ? "" : ", teardown error: ", + status == cudaSuccess ? "" : cudaGetErrorString(status)); + } + (void)cudaEventDestroy(gpu->continue_event); + (void)cudaEventDestroy(gpu->end_event); + (void)cudaEventDestroy(gpu->begin_event); +#ifdef H3_USE_CUDNN + h3_cudnn_sdpa_free(gpu->cudnn_sdpa); +#endif + (void)cublasLtDestroy(gpu->blas); + (void)cudaStreamDestroy(gpu->stream); + free(gpu); +} + +int h3_gpu_is_m5(const h3_gpu *gpu) { (void)gpu; return 0; } +int h3_gpu_has_nax_mlp(const h3_gpu *gpu) { (void)gpu; return 0; } +int h3_gpu_has_int8_mlp(const h3_gpu *gpu) { (void)gpu; return 0; } + +h3_gpu_tensor *h3_gpu_tensor_new_f32(h3_gpu *gpu, size_t elements) { + return h3_tensor_new(gpu, elements, H3_GPU_F32); +} +h3_gpu_tensor *h3_gpu_tensor_new_bf16(h3_gpu *gpu, size_t elements) { + return h3_tensor_new(gpu, elements, H3_GPU_BF16); +} +h3_gpu_tensor *h3_gpu_tensor_new_i8(h3_gpu *gpu, size_t elements) { + return h3_tensor_new(gpu, elements, H3_GPU_I8); +} + +static h3_gpu_tensor *h3_tensor_from(h3_gpu *gpu, const void *values, + size_t elements, h3_gpu_dtype dtype) { + if (elements && !values) { + h3_set_error(gpu, "tensor source is required"); + return NULL; + } + h3_gpu_tensor *tensor = h3_tensor_new(gpu, elements, dtype); + if (!tensor) return NULL; + if (tensor->bytes && !h3_cuda_ok(gpu, cudaMemcpy(tensor->data, values, + tensor->bytes, cudaMemcpyHostToDevice), "tensor upload")) { + h3_gpu_tensor_free(tensor); + return NULL; + } + return tensor; +} + +h3_gpu_tensor *h3_gpu_tensor_from_f32(h3_gpu *gpu, const float *values, + size_t elements) { + return h3_tensor_from(gpu, values, elements, H3_GPU_F32); +} +h3_gpu_tensor *h3_gpu_tensor_from_bf16(h3_gpu *gpu, const uint16_t *values, + size_t elements) { + return h3_tensor_from(gpu, values, elements, H3_GPU_BF16); +} +h3_gpu_tensor *h3_gpu_tensor_from_u32(h3_gpu *gpu, const uint32_t *values, + size_t elements) { + return h3_tensor_from(gpu, values, elements, H3_GPU_U32); +} + +static int h3_read_file(h3_gpu_tensor *tensor, const char *path, + uint64_t file_offset, size_t elements, int streaming, + char *error, size_t error_size) { + if (!tensor || !path || tensor->dtype != H3_GPU_BF16 || + elements > tensor->elements || file_offset > (uint64_t)INT64_MAX) { + if (error && error_size) snprintf(error, error_size, "invalid BF16 file read"); + return 0; + } + size_t bytes = elements * sizeof(uint16_t); + if ((uint64_t)bytes > (uint64_t)INT64_MAX - file_offset) { + if (error && error_size) snprintf(error, error_size, "BF16 file range overflows off_t"); + return 0; + } + void *staging = bytes ? malloc(bytes) : NULL; + if (bytes && !staging) { + if (error && error_size) snprintf(error, error_size, "cannot allocate pinned staging buffer"); + return 0; + } + int fd = open(path, O_RDONLY); + if (fd < 0) { + if (error && error_size) snprintf(error, error_size, "cannot open %s: %s", path, strerror(errno)); + free(staging); + return 0; + } + size_t done = 0; + while (done < bytes) { + ssize_t got = pread(fd, (char *)staging + done, bytes - done, + (off_t)(file_offset + done)); + if (got <= 0) { + if (error && error_size) snprintf(error, error_size, "short read from %s", path); + close(fd); + free(staging); + return 0; + } + done += (size_t)got; + } + if (streaming && bytes) + (void)posix_fadvise(fd, (off_t)file_offset, (off_t)bytes, + POSIX_FADV_DONTNEED); + close(fd); + cudaError_t status = bytes ? cudaMemcpy(tensor->data, staging, bytes, + cudaMemcpyHostToDevice) : cudaSuccess; + free(staging); + if (status != cudaSuccess) { + if (error && error_size) snprintf(error, error_size, "CUDA file upload: %s", cudaGetErrorString(status)); + return 0; + } + if (error && error_size) error[0] = '\0'; + return 1; +} + +h3_gpu_tensor *h3_gpu_tensor_load_bf16(h3_gpu *gpu, const char *path, + uint64_t file_offset, size_t elements) { + h3_gpu_tensor *tensor = h3_gpu_tensor_new_bf16(gpu, elements); + char error[256]; + if (tensor && !h3_read_file(tensor, path, file_offset, elements, 0, + error, sizeof(error))) { + h3_set_error(gpu, "%s", error); + h3_gpu_tensor_free(tensor); + return NULL; + } + return tensor; +} + +h3_gpu_tensor *h3_gpu_tensor_load_f32(h3_gpu *gpu, const char *path, + uint64_t file_offset, size_t elements) { + if (!gpu || !path || elements > SIZE_MAX / sizeof(float) || + file_offset > (uint64_t)INT64_MAX) { + h3_set_error(gpu, "invalid F32 file read"); + return NULL; + } + h3_gpu_tensor *tensor = h3_gpu_tensor_new_f32(gpu, elements); + if (!tensor) return NULL; + size_t bytes = elements * sizeof(float); + if ((uint64_t)bytes > (uint64_t)INT64_MAX - file_offset) { + h3_set_error(gpu, "F32 file range overflows off_t"); + h3_gpu_tensor_free(tensor); + return NULL; + } + void *staging = bytes ? malloc(bytes) : NULL; + if (bytes && !staging) { + h3_set_error(gpu, "cannot allocate pinned staging buffer"); + h3_gpu_tensor_free(tensor); + return NULL; + } + int fd = open(path, O_RDONLY); + if (fd < 0) { + h3_set_error(gpu, "cannot open %s: %s", path, strerror(errno)); + free(staging); + h3_gpu_tensor_free(tensor); + return NULL; + } + size_t done = 0; + while (done < bytes) { + ssize_t got = pread(fd, (char *)staging + done, bytes - done, + (off_t)(file_offset + done)); + if (got <= 0) break; + done += (size_t)got; + } + close(fd); + cudaError_t status = cudaSuccess; + if (done != bytes) status = cudaErrorInvalidValue; + else if (bytes) status = cudaMemcpy(tensor->data, staging, bytes, + cudaMemcpyHostToDevice); + free(staging); + if (done != bytes || status != cudaSuccess) { + h3_set_error(gpu, "cannot load F32 tensor from %s", path); + h3_gpu_tensor_free(tensor); + return NULL; + } + return tensor; +} + +int h3_gpu_tensor_read_file_bf16(h3_gpu_tensor *tensor, const char *path, + uint64_t file_offset, size_t elements, + char *error, size_t error_size) { + return h3_read_file(tensor, path, file_offset, elements, 0, error, error_size); +} +int h3_gpu_tensor_stream_file_bf16(h3_gpu_tensor *tensor, const char *path, + uint64_t file_offset, size_t elements, + char *error, size_t error_size) { + return h3_read_file(tensor, path, file_offset, elements, 1, error, error_size); +} + +void h3_gpu_tensor_free(h3_gpu_tensor *tensor) { + if (!tensor) return; + if (tensor->data) cudaFree(tensor->data); + if (tensor->gpu && tensor->gpu->stats.live_bytes >= tensor->bytes) + tensor->gpu->stats.live_bytes -= tensor->bytes; + free(tensor); +} +size_t h3_gpu_tensor_elements(const h3_gpu_tensor *tensor) { + return tensor ? tensor->elements : 0; +} +h3_gpu_dtype h3_gpu_tensor_dtype(const h3_gpu_tensor *tensor) { + return tensor ? tensor->dtype : H3_GPU_F32; +} + +__global__ static void h3_bf16_to_f32_kernel(float *output, + const __nv_bfloat16 *input, + size_t count) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index < count) output[index] = __bfloat162float(input[index]); +} +__global__ static void h3_f32_to_bf16_kernel(__nv_bfloat16 *output, + const float *input, + size_t count) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index < count) output[index] = __float2bfloat16(input[index]); +} + +int h3_gpu_tensor_read_f32_range(const h3_gpu_tensor *tensor, + size_t source_offset, float *values, + size_t elements) { + if (!tensor || !values || source_offset > tensor->elements || + elements > tensor->elements - source_offset) + return h3_set_error(tensor ? tensor->gpu : NULL, "invalid F32 tensor read range"); + if (tensor->dtype == H3_GPU_F32) { + return h3_cuda_ok(tensor->gpu, cudaMemcpy(values, + (const float *)tensor->data + source_offset, + elements * sizeof(float), cudaMemcpyDeviceToHost), "read F32 tensor"); + } + if (tensor->dtype != H3_GPU_BF16) + return h3_set_error(tensor->gpu, "tensor is neither F32 nor BF16"); + float *temporary = NULL; + if (elements && cudaMalloc(&temporary, elements * sizeof(float)) != cudaSuccess) + return h3_set_error(tensor->gpu, "cannot allocate BF16 conversion buffer"); + if (elements) h3_bf16_to_f32_kernel<<<(elements + 255) / 256, 256, 0, + tensor->gpu->stream>>>(temporary, + (const __nv_bfloat16 *)tensor->data + source_offset, elements); + cudaError_t status = cudaGetLastError(); + if (status == cudaSuccess && elements) status = cudaMemcpyAsync(values, temporary, + elements * sizeof(float), cudaMemcpyDeviceToHost, tensor->gpu->stream); + if (status == cudaSuccess) status = cudaStreamSynchronize(tensor->gpu->stream); + if (temporary) cudaFree(temporary); + return h3_cuda_ok(tensor->gpu, status, "read BF16 tensor as F32"); +} +int h3_gpu_tensor_read_f32(const h3_gpu_tensor *tensor, float *values, + size_t elements) { + return h3_gpu_tensor_read_f32_range(tensor, 0, values, elements); +} +int h3_gpu_tensor_read_bf16(const h3_gpu_tensor *tensor, uint16_t *values, + size_t elements) { + if (!h3_require_tensor(tensor, H3_GPU_BF16, elements) || !values) + return h3_set_error(tensor ? tensor->gpu : NULL, "invalid BF16 tensor read"); + return h3_cuda_ok(tensor->gpu, cudaMemcpy(values, tensor->data, + elements * sizeof(uint16_t), cudaMemcpyDeviceToHost), "read BF16 tensor"); +} + +int h3_gpu_tensor_write_f32_range(h3_gpu_tensor *tensor, + size_t destination_offset, + const float *values, size_t elements) { + if (!tensor || !values || destination_offset > tensor->elements || + elements > tensor->elements - destination_offset) + return h3_set_error(tensor ? tensor->gpu : NULL, "invalid F32 tensor write range"); + if (tensor->dtype == H3_GPU_F32) { + return h3_cuda_ok(tensor->gpu, cudaMemcpy( + (float *)tensor->data + destination_offset, values, + elements * sizeof(float), cudaMemcpyHostToDevice), "write F32 tensor"); + } + if (tensor->dtype != H3_GPU_BF16) + return h3_set_error(tensor->gpu, "tensor is neither F32 nor BF16"); + float *temporary = NULL; + if (elements && cudaMalloc(&temporary, elements * sizeof(float)) != cudaSuccess) + return h3_set_error(tensor->gpu, "cannot allocate BF16 conversion buffer"); + cudaError_t status = elements ? cudaMemcpyAsync(temporary, values, + elements * sizeof(float), cudaMemcpyHostToDevice, tensor->gpu->stream) : cudaSuccess; + if (status == cudaSuccess && elements) h3_f32_to_bf16_kernel<<< + (elements + 255) / 256, 256, 0, tensor->gpu->stream>>>( + (__nv_bfloat16 *)tensor->data + destination_offset, temporary, elements); + if (status == cudaSuccess) status = cudaGetLastError(); + if (status == cudaSuccess) status = cudaStreamSynchronize(tensor->gpu->stream); + if (temporary) cudaFree(temporary); + return h3_cuda_ok(tensor->gpu, status, "write F32 as BF16"); +} +int h3_gpu_tensor_write_f32(h3_gpu_tensor *tensor, const float *values, + size_t elements) { + return h3_gpu_tensor_write_f32_range(tensor, 0, values, elements); +} +int h3_gpu_tensor_write_bf16_range(h3_gpu_tensor *tensor, + size_t destination_offset, + const uint16_t *values, size_t elements) { + if (!tensor || tensor->dtype != H3_GPU_BF16 || !values || + destination_offset > tensor->elements || + elements > tensor->elements - destination_offset) + return h3_set_error(tensor ? tensor->gpu : NULL, "invalid BF16 tensor write range"); + return h3_cuda_ok(tensor->gpu, cudaMemcpy( + (uint16_t *)tensor->data + destination_offset, values, + elements * sizeof(uint16_t), cudaMemcpyHostToDevice), "write BF16 tensor"); +} +int h3_gpu_tensor_write_bf16(h3_gpu_tensor *tensor, const uint16_t *values, + size_t elements) { + return h3_gpu_tensor_write_bf16_range(tensor, 0, values, elements); +} + +int h3_gpu_begin(h3_gpu *gpu) { + if (!gpu || gpu->recording) return h3_set_error(gpu, "command stream is already active"); + gpu->error[0] = '\0'; + cudaError_t status = cudaEventRecord(gpu->begin_event, gpu->stream); + if (status != cudaSuccess) + return h3_cuda_ok(gpu, status, "record begin event"); + gpu->recording = 1; + gpu->encode_start_time = h3_wall_time(); + return 1; +} +int h3_gpu_continue(h3_gpu *gpu) { + if (!gpu || !gpu->recording) return h3_set_error(gpu, "no active command stream"); + cudaError_t status = cudaEventRecord(gpu->continue_event, gpu->stream); + if (status == cudaSuccess) + status = cudaStreamWaitEvent(gpu->stream, gpu->continue_event, 0); + if (status == cudaSuccess) gpu->stats.submissions++; + return h3_cuda_ok(gpu, status, "record CUDA continuation boundary"); +} +int h3_gpu_submit(h3_gpu *gpu) { + if (!gpu || !gpu->recording) return h3_set_error(gpu, "no active command stream"); + double wait_start = h3_wall_time(); + gpu->stats.command_encode_seconds += wait_start - gpu->encode_start_time; + cudaError_t status = cudaEventRecord(gpu->end_event, gpu->stream); + if (status == cudaSuccess) status = cudaEventSynchronize(gpu->end_event); + gpu->stats.command_wait_seconds += h3_wall_time() - wait_start; + if (status == cudaSuccess) { + float milliseconds = 0.0f; + status = cudaEventElapsedTime(&milliseconds, gpu->begin_event, + gpu->end_event); + gpu->stats.gpu_seconds += (double)milliseconds / 1000.0; + } + gpu->recording = 0; + if (status == cudaSuccess) gpu->stats.submissions++; + return h3_cuda_ok(gpu, status, "submit CUDA stream"); +} +const char *h3_gpu_error(const h3_gpu *gpu) { + return gpu ? gpu->error : "CUDA context is null"; +} +int h3_gpu_get_stats(const h3_gpu *gpu, h3_gpu_stats *stats) { + if (!gpu || !stats) return 0; + *stats = gpu->stats; + return 1; +} +void h3_gpu_profile_set_label(h3_gpu *gpu, const char *label) { + if (!gpu) return; + snprintf(gpu->profile_label, sizeof(gpu->profile_label), "%.127s", + label ? label : "CUDA context"); +} +void h3_gpu_profile_mark(h3_gpu *gpu, const char *phase) { + if (!gpu || !getenv("H3_PROFILE")) return; + cudaError_t status = cudaStreamSynchronize(gpu->stream); + if (status != cudaSuccess) { + h3_cuda_ok(gpu, status, "profile stream synchronization"); + return; + } + double now = h3_wall_time(); + fprintf(stderr, "%s: %s %.6fs\n", gpu->profile_label, + phase ? phase : "mark", now - gpu->profile_mark_time); + gpu->profile_mark_time = now; +} + +static int h3_copy(h3_gpu *gpu, h3_gpu_tensor *destination, + size_t destination_offset, const h3_gpu_tensor *source, + size_t source_offset, size_t elements, + h3_gpu_dtype dtype) { + if (!gpu || !destination || !source || destination->gpu != gpu || + source->gpu != gpu || destination->dtype != dtype || + source->dtype != dtype || destination_offset > destination->elements || + source_offset > source->elements || + elements > destination->elements - destination_offset || + elements > source->elements - source_offset) + return h3_set_error(gpu, "invalid tensor copy"); + size_t item_size = h3_dtype_size(dtype); + cudaError_t status = cudaMemcpyAsync( + (char *)destination->data + destination_offset * item_size, + (const char *)source->data + source_offset * item_size, + elements * item_size, cudaMemcpyDeviceToDevice, gpu->stream); + if (status == cudaSuccess) gpu->stats.blit_copies++; + return h3_cuda_ok(gpu, status, "CUDA tensor copy"); +} +int h3_gpu_copy_bf16(h3_gpu *gpu, h3_gpu_tensor *destination, + size_t destination_offset, + const h3_gpu_tensor *source, size_t source_offset, + size_t elements) { + return h3_copy(gpu, destination, destination_offset, source, source_offset, + elements, H3_GPU_BF16); +} +int h3_gpu_copy_f32(h3_gpu *gpu, h3_gpu_tensor *destination, + size_t destination_offset, + const h3_gpu_tensor *source, size_t source_offset, + size_t elements) { + return h3_copy(gpu, destination, destination_offset, source, source_offset, + elements, H3_GPU_F32); +} + +static int h3_launch_ok(h3_gpu *gpu, const char *operation) { + cudaError_t status = cudaGetLastError(); + if (status == cudaSuccess) gpu->stats.direct_dispatches++; + return h3_cuda_ok(gpu, status, operation); +} + +static int h3_tensor_is(const h3_gpu_tensor *tensor, const h3_gpu *gpu, + h3_gpu_dtype dtype, size_t elements) { + return tensor && tensor->gpu == gpu && tensor->dtype == dtype && + tensor->elements >= elements; +} + +static int h3_mul_size(size_t left, size_t right, size_t *result) { + if (left && right > SIZE_MAX / left) return 0; + *result = left * right; + return 1; +} + +static int h3_blas_ok(h3_gpu *gpu, cublasStatus_t status, + const char *operation) { + if (status == CUBLAS_STATUS_SUCCESS) return 1; + return h3_set_error(gpu, "%s: cuBLASLt status %d", operation, + (int)status); +} + +static int h3_linear_lt(h3_gpu *gpu, void *output, const void *input, + const void *weight, uint32_t rows, + uint32_t input_dim, uint32_t output_dim, + cudaDataType_t input_type, cudaDataType_t weight_type, + cudaDataType_t output_type) { + cublasLtMatmulDesc_t operation = NULL; + cublasLtMatrixLayout_t input_layout = NULL; + cublasLtMatrixLayout_t weight_layout = NULL; + cublasLtMatrixLayout_t output_layout = NULL; + cublasOperation_t transpose = CUBLAS_OP_T; + cublasLtOrder_t row_major = CUBLASLT_ORDER_ROW; + float alpha = 1.0f; + float beta = 0.0f; + cublasStatus_t status = cublasLtMatmulDescCreate( + &operation, CUBLAS_COMPUTE_32F, CUDA_R_32F); + if (status == CUBLAS_STATUS_SUCCESS) + status = cublasLtMatmulDescSetAttribute( + operation, CUBLASLT_MATMUL_DESC_TRANSB, &transpose, + sizeof(transpose)); + if (status == CUBLAS_STATUS_SUCCESS) + status = cublasLtMatrixLayoutCreate( + &input_layout, input_type, rows, input_dim, input_dim); + if (status == CUBLAS_STATUS_SUCCESS) + status = cublasLtMatrixLayoutCreate( + &weight_layout, weight_type, output_dim, input_dim, input_dim); + if (status == CUBLAS_STATUS_SUCCESS) + status = cublasLtMatrixLayoutCreate( + &output_layout, output_type, rows, output_dim, output_dim); + if (status == CUBLAS_STATUS_SUCCESS) + status = cublasLtMatrixLayoutSetAttribute( + input_layout, CUBLASLT_MATRIX_LAYOUT_ORDER, &row_major, + sizeof(row_major)); + if (status == CUBLAS_STATUS_SUCCESS) + status = cublasLtMatrixLayoutSetAttribute( + weight_layout, CUBLASLT_MATRIX_LAYOUT_ORDER, &row_major, + sizeof(row_major)); + if (status == CUBLAS_STATUS_SUCCESS) + status = cublasLtMatrixLayoutSetAttribute( + output_layout, CUBLASLT_MATRIX_LAYOUT_ORDER, &row_major, + sizeof(row_major)); + if (status == CUBLAS_STATUS_SUCCESS) + status = cublasLtMatmul(gpu->blas, operation, &alpha, input, + input_layout, weight, weight_layout, &beta, output, output_layout, + output, output_layout, NULL, NULL, 0, gpu->stream); + if (output_layout) cublasLtMatrixLayoutDestroy(output_layout); + if (weight_layout) cublasLtMatrixLayoutDestroy(weight_layout); + if (input_layout) cublasLtMatrixLayoutDestroy(input_layout); + if (operation) cublasLtMatmulDescDestroy(operation); + if (status == CUBLAS_STATUS_SUCCESS) gpu->stats.mps_linear_dispatches++; + return h3_blas_ok(gpu, status, "cuBLASLt linear"); +} + +template +__global__ static void h3_linear_bias_kernel(T *output, const T *bias, + size_t elements, + uint32_t columns) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index >= elements) return; + float value = (float)output[index] + (float)bias[index % columns]; + output[index] = (T)value; +} + +static int h3_linear_bias(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *bias, size_t elements, + uint32_t columns) { + if (!bias) return 1; + unsigned blocks = (unsigned)((elements + 255) / 256); + if (output->dtype == H3_GPU_F32) + h3_linear_bias_kernel<<stream>>>( + (float *)output->data, (const float *)bias->data, elements, + columns); + else + h3_linear_bias_kernel<<stream>>>( + (__nv_bfloat16 *)output->data, + (const __nv_bfloat16 *)bias->data, elements, columns); + return h3_launch_ok(gpu, "linear bias"); +} + +__global__ static void h3_patch_convert_kernel(__nv_bfloat16 *output, + const float *input, + const float *bias, + size_t elements, + uint32_t columns, + int has_bias) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index >= elements) return; + float value = input[index]; + if (has_bias) value += bias[index % columns]; + output[index] = __float2bfloat16(value); +} + +static int h3_patch_convert(h3_gpu *gpu, __nv_bfloat16 *output, + const float *input, + const h3_gpu_tensor *bias, size_t elements, + uint32_t columns) { + unsigned blocks = (unsigned)((elements + 255) / 256); + h3_patch_convert_kernel<<stream>>>( + output, input, bias ? (const float *)bias->data : input, elements, + columns, bias != NULL); + return h3_launch_ok(gpu, "patch linear conversion"); +} + +int h3_gpu_linear_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *weight, + const h3_gpu_tensor *bias, uint32_t rows, uint32_t input_dim, + uint32_t output_dim) { + size_t inputs, weights, outputs; + if (!rows || !input_dim || !output_dim || + !h3_mul_size(rows, input_dim, &inputs) || + !h3_mul_size(output_dim, input_dim, &weights) || + !h3_mul_size(rows, output_dim, &outputs) || + !h3_tensor_is(input, gpu, H3_GPU_F32, inputs) || + !h3_tensor_is(weight, gpu, H3_GPU_F32, weights) || + !h3_tensor_is(output, gpu, H3_GPU_F32, outputs) || + (bias && !h3_tensor_is(bias, gpu, H3_GPU_F32, output_dim))) + return h3_set_error(gpu, "invalid F32 linear tensors or shape"); + return h3_linear_lt(gpu, output->data, input->data, weight->data, rows, + input_dim, output_dim, CUDA_R_32F, CUDA_R_32F, + CUDA_R_32F) && + h3_linear_bias(gpu, output, bias, outputs, output_dim); +} + +int h3_gpu_linear_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *weight, + const h3_gpu_tensor *bias, uint32_t rows, uint32_t input_dim, + uint32_t output_dim) { + size_t inputs, weights, outputs; + if (!rows || !input_dim || !output_dim || + !h3_mul_size(rows, input_dim, &inputs) || + !h3_mul_size(output_dim, input_dim, &weights) || + !h3_mul_size(rows, output_dim, &outputs) || + !h3_tensor_is(input, gpu, H3_GPU_BF16, inputs) || + !h3_tensor_is(weight, gpu, H3_GPU_BF16, weights) || + !h3_tensor_is(output, gpu, H3_GPU_BF16, outputs) || + (bias && !h3_tensor_is(bias, gpu, H3_GPU_BF16, output_dim))) + return h3_set_error(gpu, "invalid BF16 linear tensors or shape"); + return h3_linear_lt(gpu, output->data, input->data, weight->data, rows, + input_dim, output_dim, CUDA_R_16BF, CUDA_R_16BF, + CUDA_R_16BF) && + h3_linear_bias(gpu, output, bias, outputs, output_dim); +} + +int h3_gpu_patch_linear_bf16_offset(h3_gpu *gpu, h3_gpu_tensor *output, + size_t output_offset, const h3_gpu_tensor *input, + size_t input_offset, const h3_gpu_tensor *weight, + const h3_gpu_tensor *bias, uint32_t rows, uint32_t input_dim, + uint32_t output_dim) { + size_t inputs, weights, outputs; + if (!rows || output_dim != 5376 || + (input_dim != 32 && input_dim != 96) || + !h3_mul_size(rows, input_dim, &inputs) || + !h3_mul_size(output_dim, input_dim, &weights) || + !h3_mul_size(rows, output_dim, &outputs) || + input_offset > SIZE_MAX - inputs || + output_offset > SIZE_MAX - outputs || + !h3_tensor_is(input, gpu, H3_GPU_F32, input_offset + inputs) || + !h3_tensor_is(weight, gpu, H3_GPU_F32, weights) || + !h3_tensor_is(output, gpu, H3_GPU_BF16, output_offset + outputs) || + (bias && !h3_tensor_is(bias, gpu, H3_GPU_F32, output_dim))) + return h3_set_error(gpu, "invalid patch linear tensors or shape"); + const float *input_data = (const float *)input->data + input_offset; + __nv_bfloat16 *output_data = + (__nv_bfloat16 *)output->data + output_offset; + float *temporary = NULL; + cudaError_t status = cudaMallocAsync((void **)&temporary, + outputs * sizeof(*temporary), gpu->stream); + if (!h3_cuda_ok(gpu, status, "patch linear temporary allocation")) + return 0; + int ok = h3_linear_lt(gpu, temporary, input_data, weight->data, rows, + input_dim, output_dim, CUDA_R_32F, CUDA_R_32F, + CUDA_R_32F) && + h3_patch_convert(gpu, output_data, temporary, bias, outputs, + output_dim); + status = cudaFreeAsync(temporary, gpu->stream); + if (ok) ok = h3_cuda_ok(gpu, status, "patch linear temporary free"); + return ok; +} + +int h3_gpu_patch_linear_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *weight, + const h3_gpu_tensor *bias, uint32_t rows, uint32_t input_dim, + uint32_t output_dim) { + return h3_gpu_patch_linear_bf16_offset(gpu, output, 0, input, 0, weight, + bias, rows, input_dim, + output_dim); +} + +__global__ static void h3_patch_scatter_kernel( + __nv_bfloat16 *output, const float *input, const float *bias, + const uint32_t *row_map, uint32_t output_rows, uint32_t rows, + uint32_t columns, int has_bias) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + size_t elements = (size_t)rows * columns; + if (index >= elements) return; + uint32_t source_row = (uint32_t)(index / columns); + uint32_t destination_row = row_map[source_row]; + if (destination_row < output_rows) + output[(size_t)destination_row * columns + index % columns] = + __float2bfloat16(input[index] + + (has_bias ? bias[index % columns] : 0.0f)); +} + +int h3_gpu_patch_linear_bf16_map(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *weight, + const h3_gpu_tensor *bias, const h3_gpu_tensor *row_map, + uint32_t output_rows, uint32_t rows, uint32_t input_dim, + uint32_t output_dim) { + size_t inputs, weights, outputs, mapped_outputs; + if (!rows || !output_rows || output_dim != 5376 || + (input_dim != 32 && input_dim != 96) || + !h3_mul_size(rows, input_dim, &inputs) || + !h3_mul_size(output_dim, input_dim, &weights) || + !h3_mul_size(output_rows, output_dim, &outputs) || + !h3_mul_size(rows, output_dim, &mapped_outputs) || + !h3_tensor_is(input, gpu, H3_GPU_F32, inputs) || + !h3_tensor_is(weight, gpu, H3_GPU_F32, weights) || + !h3_tensor_is(output, gpu, H3_GPU_BF16, outputs) || + !h3_tensor_is(row_map, gpu, H3_GPU_U32, rows) || + (bias && !h3_tensor_is(bias, gpu, H3_GPU_F32, output_dim))) + return h3_set_error(gpu, "invalid mapped patch linear tensors or shape"); + float *temporary = NULL; + cudaError_t status = cudaMallocAsync((void **)&temporary, + mapped_outputs * sizeof(*temporary), gpu->stream); + if (!h3_cuda_ok(gpu, status, "mapped patch temporary allocation")) + return 0; + int ok = h3_linear_lt(gpu, temporary, input->data, weight->data, rows, + input_dim, output_dim, CUDA_R_32F, CUDA_R_32F, + CUDA_R_32F); + if (ok) { + unsigned blocks = (unsigned)((mapped_outputs + 255) / 256); + h3_patch_scatter_kernel<<stream>>>( + (__nv_bfloat16 *)output->data, temporary, + bias ? (const float *)bias->data : temporary, + (const uint32_t *)row_map->data, output_rows, rows, output_dim, + bias != NULL); + ok = h3_launch_ok(gpu, "mapped patch scatter"); + } + status = cudaFreeAsync(temporary, gpu->stream); + if (ok) ok = h3_cuda_ok(gpu, status, "mapped patch temporary free"); + return ok; +} + +__global__ static void h3_quantize_rows_kernel( + int8_t *output, float *scales, const __nv_bfloat16 *input, + uint32_t rows, uint32_t columns) { + uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= rows) return; + size_t base = (size_t)row * columns; + float maximum = 0.0f; + for (uint32_t column = 0; column < columns; column++) + maximum = fmaxf(maximum, fabsf(__bfloat162float(input[base + column]))); + float scale = maximum > 0.0f ? maximum / 127.0f : 1.0f / 127.0f; + float inverse = 1.0f / scale; + scales[row] = scale; + for (uint32_t column = 0; column < columns; column++) { + int value = (int)nearbyintf( + __bfloat162float(input[base + column]) * inverse); + output[base + column] = (int8_t)max(-127, min(127, value)); + } +} + +static int h3_quantize_rows(h3_gpu *gpu, h3_gpu_tensor *output, + h3_gpu_tensor *scales, + const h3_gpu_tensor *input, uint32_t rows, + uint32_t columns) { + size_t elements; + if (!rows || !columns || !h3_mul_size(rows, columns, &elements) || + !h3_tensor_is(input, gpu, H3_GPU_BF16, elements) || + !h3_tensor_is(output, gpu, H3_GPU_I8, elements) || + !h3_tensor_is(scales, gpu, H3_GPU_F32, rows)) + return h3_set_error(gpu, "invalid INT8 quantization tensors or shape"); + h3_quantize_rows_kernel<<<(rows + 127) / 128, 128, 0, gpu->stream>>>( + (int8_t *)output->data, (float *)scales->data, + (const __nv_bfloat16 *)input->data, rows, columns); + return h3_launch_ok(gpu, "BF16 row quantization"); +} + +int h3_gpu_quantize_weight_int8(h3_gpu *gpu, h3_gpu_tensor *output, + h3_gpu_tensor *scales, const h3_gpu_tensor *input, uint32_t rows, + uint32_t columns) { + return h3_quantize_rows(gpu, output, scales, input, rows, columns); +} + +__global__ static void h3_linear_int8_kernel( + __nv_bfloat16 *output, const int8_t *input, const int8_t *weight, + const float *input_scales, const float *weight_scales, + uint32_t rows, uint32_t input_dim, uint32_t output_dim) { + uint32_t column = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t row = blockIdx.y; + if (column >= output_dim || row >= rows) return; + int32_t sum = 0; + size_t input_base = (size_t)row * input_dim; + size_t weight_base = (size_t)column * input_dim; + for (uint32_t inner = 0; inner < input_dim; inner++) + sum += (int32_t)input[input_base + inner] * + (int32_t)weight[weight_base + inner]; + output[(size_t)row * output_dim + column] = __float2bfloat16( + (float)sum * input_scales[row] * weight_scales[column]); +} + +static int h3_linear_int8_quantized(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *quantized_input, + const h3_gpu_tensor *input_scales, const h3_gpu_tensor *weight, + const h3_gpu_tensor *weight_scales, uint32_t rows, + uint32_t input_dim, uint32_t output_dim) { + size_t inputs, weights, outputs; + if (!h3_mul_size(rows, input_dim, &inputs) || + !h3_mul_size(output_dim, input_dim, &weights) || + !h3_mul_size(rows, output_dim, &outputs) || + !h3_tensor_is(quantized_input, gpu, H3_GPU_I8, inputs) || + !h3_tensor_is(input_scales, gpu, H3_GPU_F32, rows) || + !h3_tensor_is(weight, gpu, H3_GPU_I8, weights) || + !h3_tensor_is(weight_scales, gpu, H3_GPU_F32, output_dim) || + !h3_tensor_is(output, gpu, H3_GPU_BF16, outputs)) + return h3_set_error(gpu, "invalid quantized INT8 linear tensors"); + dim3 grid((output_dim + 127) / 128, rows); + h3_linear_int8_kernel<<stream>>>( + (__nv_bfloat16 *)output->data, (const int8_t *)quantized_input->data, + (const int8_t *)weight->data, (const float *)input_scales->data, + (const float *)weight_scales->data, rows, input_dim, output_dim); + return h3_launch_ok(gpu, "INT8 linear"); +} + +int h3_gpu_linear_int8_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + h3_gpu_tensor *quantized_input, h3_gpu_tensor *input_scales, + const h3_gpu_tensor *input, const h3_gpu_tensor *weight, + const h3_gpu_tensor *weight_scales, uint32_t rows, + uint32_t input_dim, uint32_t output_dim, + int use_slower_uncached_int8_scales) { + (void)use_slower_uncached_int8_scales; + size_t inputs, weights, outputs; + if (!rows || !input_dim || !output_dim || + !h3_mul_size(rows, input_dim, &inputs) || + !h3_mul_size(output_dim, input_dim, &weights) || + !h3_mul_size(rows, output_dim, &outputs) || + !h3_tensor_is(input, gpu, H3_GPU_BF16, inputs) || + !h3_tensor_is(quantized_input, gpu, H3_GPU_I8, inputs) || + !h3_tensor_is(input_scales, gpu, H3_GPU_F32, rows) || + !h3_tensor_is(weight, gpu, H3_GPU_I8, weights) || + !h3_tensor_is(weight_scales, gpu, H3_GPU_F32, output_dim) || + !h3_tensor_is(output, gpu, H3_GPU_BF16, outputs)) + return h3_set_error(gpu, "invalid INT8 linear tensors or shape"); + if (!h3_quantize_rows(gpu, quantized_input, input_scales, input, rows, + input_dim)) return 0; + return h3_linear_int8_quantized(gpu, output, quantized_input, + input_scales, weight, weight_scales, rows, input_dim, output_dim); +} + +__global__ static void h3_quantize_head_major_kernel( + int8_t *output, float *scales, const __nv_bfloat16 *input, + uint32_t rows, uint32_t heads, uint32_t head_dim) { + uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= rows) return; + uint32_t columns = heads * head_dim; + float maximum = 0.0f; + for (uint32_t head = 0; head < heads; head++) + for (uint32_t dimension = 0; dimension < head_dim; dimension++) { + size_t source = ((size_t)head * rows + row) * head_dim + dimension; + maximum = fmaxf(maximum, fabsf(__bfloat162float(input[source]))); + } + float scale = maximum > 0.0f ? maximum / 127.0f : 1.0f / 127.0f; + float inverse = 1.0f / scale; + scales[row] = scale; + for (uint32_t head = 0; head < heads; head++) + for (uint32_t dimension = 0; dimension < head_dim; dimension++) { + size_t source = ((size_t)head * rows + row) * head_dim + dimension; + int value = (int)nearbyintf( + __bfloat162float(input[source]) * inverse); + output[(size_t)row * columns + (size_t)head * head_dim + dimension] = + (int8_t)max(-127, min(127, value)); + } +} + +int h3_gpu_linear_int8_head_major_bf16(h3_gpu *gpu, + h3_gpu_tensor *output, h3_gpu_tensor *quantized_input, + h3_gpu_tensor *input_scales, const h3_gpu_tensor *input, + const h3_gpu_tensor *weight, const h3_gpu_tensor *weight_scales, + uint32_t rows, uint32_t heads, uint32_t head_dim, + uint32_t output_dim) { + size_t columns, inputs; + if (!rows || !heads || !head_dim || !output_dim || + !h3_mul_size(heads, head_dim, &columns) || columns > UINT32_MAX || + !h3_mul_size(rows, columns, &inputs) || + !h3_tensor_is(input, gpu, H3_GPU_BF16, inputs) || + !h3_tensor_is(quantized_input, gpu, H3_GPU_I8, inputs) || + !h3_tensor_is(input_scales, gpu, H3_GPU_F32, rows)) + return h3_set_error(gpu, "invalid head-major INT8 linear tensors"); + h3_quantize_head_major_kernel<<<(rows + 127) / 128, 128, 0, + gpu->stream>>>((int8_t *)quantized_input->data, + (float *)input_scales->data, (const __nv_bfloat16 *)input->data, + rows, heads, head_dim); + if (!h3_launch_ok(gpu, "head-major INT8 quantization")) return 0; + return h3_linear_int8_quantized(gpu, output, quantized_input, + input_scales, weight, weight_scales, rows, (uint32_t)columns, + output_dim); +} + +int h3_gpu_mlp_int8_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + h3_gpu_tensor *activated, h3_gpu_tensor *quantized_activation, + h3_gpu_tensor *activation_scales, const h3_gpu_tensor *input, + const h3_gpu_tensor *fc1_weight, const h3_gpu_tensor *fc1_scales, + const h3_gpu_tensor *fc2_weight, const h3_gpu_tensor *fc2_scales, + const h3_gpu_tensor *fc1_bf16, const h3_gpu_tensor *fc2_bf16, + uint32_t rows, uint32_t input_dim, uint32_t hidden_dim, + uint32_t output_dim, int use_slower_grouped_quantizer, + int use_slower_dynamic_fc1_k, int use_int8_row_fc2, + int input_is_quantized) { + (void)fc1_bf16; + (void)fc2_bf16; + (void)use_slower_grouped_quantizer; + (void)use_slower_dynamic_fc1_k; + (void)use_int8_row_fc2; + size_t fused_elements, activation_elements, input_elements; + if (!rows || !input_dim || !hidden_dim || !output_dim || + hidden_dim > UINT32_MAX / 2 || + !h3_mul_size(rows, (size_t)hidden_dim * 2, &fused_elements) || + !h3_mul_size(rows, hidden_dim, &activation_elements) || + !h3_mul_size(rows, input_dim, &input_elements) || + !h3_tensor_is(input, gpu, H3_GPU_BF16, input_elements) || + !h3_tensor_is(activated, gpu, H3_GPU_BF16, activation_elements) || + !h3_tensor_is(quantized_activation, gpu, H3_GPU_I8, + input_elements > activation_elements ? + input_elements : activation_elements) || + !h3_tensor_is(activation_scales, gpu, H3_GPU_F32, rows)) + return h3_set_error(gpu, "invalid INT8 MLP activation tensors"); + __nv_bfloat16 *fused_data = NULL; + cudaError_t status = cudaMallocAsync((void **)&fused_data, + fused_elements * sizeof(*fused_data), gpu->stream); + if (!h3_cuda_ok(gpu, status, "INT8 MLP temporary allocation")) return 0; + h3_gpu_tensor fused = {gpu, fused_data, fused_elements, + fused_elements * sizeof(*fused_data), H3_GPU_BF16}; + int ok; + if (input_is_quantized) + ok = h3_linear_int8_quantized(gpu, &fused, quantized_activation, + activation_scales, fc1_weight, fc1_scales, rows, input_dim, + hidden_dim * 2); + else + ok = h3_gpu_linear_int8_bf16(gpu, &fused, quantized_activation, + activation_scales, input, fc1_weight, fc1_scales, rows, input_dim, + hidden_dim * 2, 0); + if (ok) + ok = h3_gpu_swiglu_bf16(gpu, activated, &fused, rows, hidden_dim); + if (ok) + ok = h3_gpu_linear_int8_bf16(gpu, output, quantized_activation, + activation_scales, activated, fc2_weight, fc2_scales, rows, + hidden_dim, output_dim, 0); + status = cudaFreeAsync(fused_data, gpu->stream); + if (ok) ok = h3_cuda_ok(gpu, status, "INT8 MLP temporary free"); + return ok; +} + +__global__ static void h3_rms_inverse_kernel( + float *inverse, const __nv_bfloat16 *input, size_t input_offset, + uint32_t rows, uint32_t width, float epsilon) { + uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= rows) return; + float sum = 0.0f; + size_t base = input_offset + (size_t)row * width; + for (uint32_t column = 0; column < width; column++) { + float value = __bfloat162float(input[base + column]); + sum = fmaf(value, value, sum); + } + inverse[row] = rsqrtf(sum / (float)width + epsilon); +} + +int h3_gpu_adaln_linear_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + h3_gpu_tensor *inverse, const h3_gpu_tensor *input, + size_t input_offset, const h3_gpu_tensor *norm_weight, + const h3_gpu_tensor *modulation, const h3_gpu_tensor *row_map, + const h3_gpu_tensor *weight, const h3_gpu_tensor *bias, + uint32_t rows, uint32_t width, uint32_t output_dim, uint32_t slots, + uint32_t shift_slot, uint32_t scale_slot, float epsilon) { + size_t elements; + if (!rows || !width || !output_dim || epsilon < 0.0f || + !h3_mul_size(rows, width, &elements) || + !h3_tensor_is(inverse, gpu, H3_GPU_F32, rows)) + return h3_set_error(gpu, "invalid AdaLN linear inverse or shape"); + __nv_bfloat16 *normalized_data = NULL; + cudaError_t status = cudaMallocAsync((void **)&normalized_data, + elements * sizeof(*normalized_data), gpu->stream); + if (!h3_cuda_ok(gpu, status, "AdaLN linear temporary allocation")) + return 0; + h3_gpu_tensor normalized = {gpu, normalized_data, elements, + elements * sizeof(*normalized_data), H3_GPU_BF16}; + h3_rms_inverse_kernel<<<(rows + 127) / 128, 128, 0, gpu->stream>>>( + (float *)inverse->data, (const __nv_bfloat16 *)input->data, + input_offset, rows, width, epsilon); + int ok = h3_launch_ok(gpu, "AdaLN inverse RMS") && + h3_gpu_adaln_bf16_offset(gpu, &normalized, input, input_offset, + norm_weight, modulation, row_map, rows, width, slots, shift_slot, + scale_slot, epsilon) && + h3_gpu_linear_bf16(gpu, output, &normalized, weight, bias, rows, + width, output_dim); + status = cudaFreeAsync(normalized_data, gpu->stream); + if (ok) ok = h3_cuda_ok(gpu, status, "AdaLN linear temporary free"); + return ok; +} + +__global__ static void h3_fill_scales_kernel(float *scales, uint32_t begin, + uint32_t end) { + uint32_t row = begin + blockIdx.x * blockDim.x + threadIdx.x; + if (row < end) scales[row] = 1.0f; +} + +int h3_gpu_gate_adaln_quantize_int8(h3_gpu *gpu, + h3_gpu_tensor *gated_residual, h3_gpu_tensor *quantized_output, + h3_gpu_tensor *quantized_scales, const h3_gpu_tensor *residual, + const h3_gpu_tensor *branch, const h3_gpu_tensor *norm_weight, + const h3_gpu_tensor *gate_modulation, + const h3_gpu_tensor *norm_modulation, const h3_gpu_tensor *row_map, + uint32_t rows, uint32_t padded_rows, uint32_t width, uint32_t slots, + uint32_t gate_slot, uint32_t shift_slot, uint32_t scale_slot, + float epsilon) { + size_t elements, padded_elements; + if (!rows || padded_rows < rows || !width || + !h3_mul_size(rows, width, &elements) || + !h3_mul_size(padded_rows, width, &padded_elements) || + !h3_tensor_is(quantized_output, gpu, H3_GPU_I8, padded_elements) || + !h3_tensor_is(quantized_scales, gpu, H3_GPU_F32, padded_rows)) + return h3_set_error(gpu, "invalid fused gate/AdaLN INT8 tensors"); + __nv_bfloat16 *normalized_data = NULL; + cudaError_t status = cudaMallocAsync((void **)&normalized_data, + elements * sizeof(*normalized_data), gpu->stream); + if (status == cudaSuccess && padded_rows > rows) + status = cudaMemsetAsync((int8_t *)quantized_output->data + elements, + 0, (padded_elements - elements) * sizeof(int8_t), gpu->stream); + if (!h3_cuda_ok(gpu, status, "fused gate/AdaLN temporary setup")) { + if (normalized_data) (void)cudaFreeAsync(normalized_data, gpu->stream); + return 0; + } + h3_gpu_tensor normalized = {gpu, normalized_data, elements, + elements * sizeof(*normalized_data), H3_GPU_BF16}; + int ok = h3_gpu_gate_adaln_bf16(gpu, gated_residual, &normalized, + residual, branch, norm_weight, gate_modulation, norm_modulation, + row_map, rows, width, slots, gate_slot, shift_slot, scale_slot, + epsilon) && + h3_quantize_rows(gpu, quantized_output, quantized_scales, &normalized, + rows, width); + if (ok && padded_rows > rows) { + h3_fill_scales_kernel<<<(padded_rows - rows + 127) / 128, 128, 0, + gpu->stream>>>((float *)quantized_scales->data, rows, padded_rows); + ok = h3_launch_ok(gpu, "INT8 padding scales"); + } + status = cudaFreeAsync(normalized_data, gpu->stream); + if (ok) ok = h3_cuda_ok(gpu, status, "fused gate/AdaLN temporary free"); + return ok; +} + +int h3_gpu_grouped_qkv_linear_rope_bf16(h3_gpu *gpu, + h3_gpu_tensor *query, h3_gpu_tensor *key, h3_gpu_tensor *value, + h3_gpu_tensor *qkv, const h3_gpu_tensor *input, + const h3_gpu_tensor *weight, const h3_gpu_tensor *q_norm, + const h3_gpu_tensor *k_norm, const h3_gpu_tensor *rope_cos, + const h3_gpu_tensor *rope_sin, uint32_t rows, uint32_t input_dim, + uint32_t heads, uint32_t head_dim, uint32_t rope_half, + float epsilon) { + size_t inner; + if (!h3_mul_size(heads, head_dim, &inner) || inner > UINT32_MAX / 3) + return h3_set_error(gpu, "grouped QKV projection shape overflows"); + return h3_gpu_linear_bf16(gpu, qkv, input, weight, NULL, rows, input_dim, + (uint32_t)inner * 3) && + h3_gpu_grouped_qkv_rope_bf16(gpu, query, key, value, qkv, q_norm, + k_norm, rope_cos, rope_sin, rows, heads, head_dim, rope_half, + epsilon); +} + +int h3_gpu_grouped_qkv_linear_rope_int8(h3_gpu *gpu, + h3_gpu_tensor *query, h3_gpu_tensor *key, h3_gpu_tensor *value, + h3_gpu_tensor *quantized_input, h3_gpu_tensor *input_scales, + const h3_gpu_tensor *input, const h3_gpu_tensor *weight, + const h3_gpu_tensor *weight_scales, const h3_gpu_tensor *q_norm, + const h3_gpu_tensor *k_norm, const h3_gpu_tensor *rope_cos, + const h3_gpu_tensor *rope_sin, uint32_t rows, uint32_t input_dim, + uint32_t heads, uint32_t head_dim, uint32_t rope_half, float epsilon, + int input_is_quantized, int use_slower_unfused_qkv_rope, + int use_slower_scalar_qkv_rms, + int use_slower_uncached_int8_scales) { + (void)use_slower_unfused_qkv_rope; + (void)use_slower_scalar_qkv_rms; + size_t inner, qkv_elements; + if (!h3_mul_size(heads, head_dim, &inner) || inner > UINT32_MAX / 3 || + !h3_mul_size(rows, inner * 3, &qkv_elements)) + return h3_set_error(gpu, "INT8 grouped QKV shape overflows"); + __nv_bfloat16 *qkv_data = NULL; + cudaError_t status = cudaMallocAsync((void **)&qkv_data, + qkv_elements * sizeof(*qkv_data), gpu->stream); + if (!h3_cuda_ok(gpu, status, "INT8 QKV temporary allocation")) return 0; + h3_gpu_tensor qkv = {gpu, qkv_data, qkv_elements, + qkv_elements * sizeof(*qkv_data), H3_GPU_BF16}; + int ok; + if (input_is_quantized) + ok = h3_linear_int8_quantized(gpu, &qkv, quantized_input, + input_scales, weight, weight_scales, rows, input_dim, + (uint32_t)inner * 3); + else + ok = h3_gpu_linear_int8_bf16(gpu, &qkv, quantized_input, + input_scales, input, weight, weight_scales, rows, input_dim, + (uint32_t)inner * 3, use_slower_uncached_int8_scales); + if (ok) + ok = h3_gpu_grouped_qkv_rope_bf16(gpu, query, key, value, &qkv, + q_norm, k_norm, rope_cos, rope_sin, rows, heads, head_dim, + rope_half, epsilon); + status = cudaFreeAsync(qkv_data, gpu->stream); + if (ok) ok = h3_cuda_ok(gpu, status, "INT8 QKV temporary free"); + return ok; +} + +template +__global__ static void h3_attention_kernel( + T *output, const T *query, const T *key, const T *value, + uint32_t batch, uint32_t sequence, uint32_t query_heads, + uint32_t kv_heads, uint32_t head_dim, float scale, int causal, + int head_major_output, int head_major_input) { + __shared__ float partial[1024]; + __shared__ float rescale; + __shared__ float weight; + __shared__ float denominator; + uint32_t query_head = blockIdx.x; + uint32_t row = blockIdx.y; + uint32_t batch_index = blockIdx.z; + uint32_t kv_head = query_head / (query_heads / kv_heads); + uint32_t keys = causal ? row + 1 : sequence; + size_t query_base = head_major_input ? + (((size_t)batch_index * query_heads + query_head) * sequence + row) * + head_dim : + (((size_t)batch_index * sequence + row) * query_heads + query_head) * + head_dim; + uint32_t dimension = threadIdx.x; + float result = 0.0f; + float maximum = -INFINITY; + if (threadIdx.x == 0) denominator = 0.0f; + __syncthreads(); + for (uint32_t key_row = 0; key_row < keys; key_row++) { + size_t key_base = head_major_input ? + (((size_t)batch_index * kv_heads + kv_head) * sequence + key_row) * + head_dim : + (((size_t)batch_index * sequence + key_row) * kv_heads + kv_head) * + head_dim; + float product = 0.0f; + if (dimension < head_dim) { + if constexpr (std::is_same::value) + product = query[query_base + dimension] * + key[key_base + dimension]; + else + product = __bfloat162float(query[query_base + dimension]) * + __bfloat162float(key[key_base + dimension]); + } + partial[threadIdx.x] = product; + __syncthreads(); + for (uint32_t offset = blockDim.x / 2; offset; offset >>= 1) { + if (threadIdx.x < offset) + partial[threadIdx.x] += partial[threadIdx.x + offset]; + __syncthreads(); + } + if (threadIdx.x == 0) { + float score = partial[0] * scale; + float next_maximum = fmaxf(maximum, score); + rescale = expf(maximum - next_maximum); + weight = expf(score - next_maximum); + denominator = denominator * rescale + weight; + maximum = next_maximum; + } + __syncthreads(); + if (dimension < head_dim) { + size_t value_index = key_base + dimension; + float value_element; + if constexpr (std::is_same::value) + value_element = value[value_index]; + else + value_element = __bfloat162float(value[value_index]); + result = result * rescale + weight * value_element; + } + __syncthreads(); + } + if (dimension < head_dim) { + result /= denominator; + size_t output_index = head_major_output ? + (((size_t)batch_index * query_heads + query_head) * sequence + row) * + head_dim + dimension : + (((size_t)batch_index * sequence + row) * query_heads + query_head) * + head_dim + dimension; + if constexpr (std::is_same::value) + output[output_index] = result; + else + output[output_index] = __float2bfloat16(result); + } +} + +__global__ static void h3_attention_tiled_bf16_kernel( + __nv_bfloat16 *output, const __nv_bfloat16 *query, + const __nv_bfloat16 *key, const __nv_bfloat16 *value, + uint32_t sequence, uint32_t heads, float scale, + int head_major_output) { + enum { HEAD_DIM = 128, QUERIES = 8 }; + __shared__ float shared_key[HEAD_DIM]; + __shared__ float shared_value[HEAD_DIM]; + uint32_t warp = threadIdx.x / 32; + uint32_t lane = threadIdx.x % 32; + uint32_t head = blockIdx.x; + uint32_t row = blockIdx.y * QUERIES + warp; + int active = row < sequence; + size_t query_base = ((size_t)head * sequence + row) * HEAD_DIM; + float result[4] = {0.0f, 0.0f, 0.0f, 0.0f}; + float maximum = -INFINITY; + float denominator = 0.0f; + for (uint32_t key_row = 0; key_row < sequence; key_row++) { + if (threadIdx.x < HEAD_DIM) { + size_t index = ((size_t)head * sequence + key_row) * HEAD_DIM + + threadIdx.x; + shared_key[threadIdx.x] = __bfloat162float(key[index]); + shared_value[threadIdx.x] = __bfloat162float(value[index]); + } + __syncthreads(); + float product = 0.0f; + if (active) { +#pragma unroll + for (uint32_t item = 0; item < 4; item++) { + uint32_t dimension = lane + item * 32; + product += __bfloat162float(query[query_base + dimension]) * + shared_key[dimension]; + } +#pragma unroll + for (uint32_t offset = 16; offset; offset >>= 1) + product += __shfl_down_sync(0xffffffffu, product, offset); + } + float rescale = 1.0f; + float weight = 0.0f; + if (active && lane == 0) { + float score = product * scale; + float next_maximum = fmaxf(maximum, score); + rescale = expf(maximum - next_maximum); + weight = expf(score - next_maximum); + denominator = denominator * rescale + weight; + maximum = next_maximum; + } + rescale = __shfl_sync(0xffffffffu, rescale, 0); + weight = __shfl_sync(0xffffffffu, weight, 0); + if (active) { +#pragma unroll + for (uint32_t item = 0; item < 4; item++) { + uint32_t dimension = lane + item * 32; + result[item] = result[item] * rescale + + weight * shared_value[dimension]; + } + } + __syncthreads(); + } + denominator = __shfl_sync(0xffffffffu, denominator, 0); + if (!active) return; +#pragma unroll + for (uint32_t item = 0; item < 4; item++) { + uint32_t dimension = lane + item * 32; + size_t index = head_major_output ? + ((size_t)head * sequence + row) * HEAD_DIM + dimension : + ((size_t)row * heads + head) * HEAD_DIM + dimension; + output[index] = __float2bfloat16(result[item] / denominator); + } +} + +template +__global__ static void h3_attention_tiled_f32_kernel( + float *output, const float *query, const float *key, + const float *value, uint32_t sequence, uint32_t heads, float scale, + int head_major_output) { + enum { QUERIES = 8, ITEMS = HEAD_DIM / 32 }; + __shared__ float shared_key[HEAD_DIM]; + __shared__ float shared_value[HEAD_DIM]; + uint32_t warp = threadIdx.x / 32; + uint32_t lane = threadIdx.x % 32; + uint32_t head = blockIdx.x; + uint32_t row = blockIdx.y * QUERIES + warp; + int active = row < sequence; + size_t query_base = ((size_t)head * sequence + row) * HEAD_DIM; + float result[ITEMS]; +#pragma unroll + for (uint32_t item = 0; item < ITEMS; item++) result[item] = 0.0f; + float maximum = -INFINITY; + float denominator = 0.0f; + for (uint32_t key_row = 0; key_row < sequence; key_row++) { + if (threadIdx.x < HEAD_DIM) { + size_t index = ((size_t)head * sequence + key_row) * HEAD_DIM + + threadIdx.x; + shared_key[threadIdx.x] = key[index]; + shared_value[threadIdx.x] = value[index]; + } + __syncthreads(); + float product = 0.0f; + if (active) { +#pragma unroll + for (uint32_t item = 0; item < ITEMS; item++) { + uint32_t dimension = lane + item * 32; + product += query[query_base + dimension] * + shared_key[dimension]; + } +#pragma unroll + for (uint32_t offset = 16; offset; offset >>= 1) + product += __shfl_down_sync(0xffffffffu, product, offset); + } + float rescale = 1.0f; + float weight = 0.0f; + if (active && lane == 0) { + float score = product * scale; + float next_maximum = fmaxf(maximum, score); + rescale = expf(maximum - next_maximum); + weight = expf(score - next_maximum); + denominator = denominator * rescale + weight; + maximum = next_maximum; + } + rescale = __shfl_sync(0xffffffffu, rescale, 0); + weight = __shfl_sync(0xffffffffu, weight, 0); + if (active) { +#pragma unroll + for (uint32_t item = 0; item < ITEMS; item++) { + uint32_t dimension = lane + item * 32; + result[item] = result[item] * rescale + + weight * shared_value[dimension]; + } + } + __syncthreads(); + } + denominator = __shfl_sync(0xffffffffu, denominator, 0); + if (!active) return; +#pragma unroll + for (uint32_t item = 0; item < ITEMS; item++) { + uint32_t dimension = lane + item * 32; + size_t index = head_major_output ? + ((size_t)head * sequence + row) * HEAD_DIM + dimension : + ((size_t)row * heads + head) * HEAD_DIM + dimension; + output[index] = result[item] / denominator; + } +} + +static int h3_attention_dispatch(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *query, const h3_gpu_tensor *key, + const h3_gpu_tensor *value, uint32_t batch, uint32_t sequence, + uint32_t query_heads, uint32_t kv_heads, uint32_t head_dim, + float scale, h3_gpu_dtype dtype, int causal, int head_major_output) { + int head_major_input = causal != 2; + if (causal == 2) causal = 1; + size_t query_elements, kv_elements; + size_t batch_sequence, query_rows, kv_rows; + if (!batch || !sequence || !query_heads || !kv_heads || !head_dim || + query_heads % kv_heads || head_dim > 1024 || + !h3_mul_size(batch, sequence, &batch_sequence) || + !h3_mul_size(batch_sequence, query_heads, &query_rows) || + !h3_mul_size(batch_sequence, kv_heads, &kv_rows) || + !h3_mul_size(query_rows, head_dim, &query_elements) || + !h3_mul_size(kv_rows, head_dim, &kv_elements) || + !h3_tensor_is(query, gpu, dtype, query_elements) || + !h3_tensor_is(key, gpu, dtype, kv_elements) || + !h3_tensor_is(value, gpu, dtype, kv_elements) || + !h3_tensor_is(output, gpu, dtype, query_elements)) + return h3_set_error(gpu, "invalid attention tensors or shape"); + uint32_t threads = 128; + while (threads < head_dim) threads <<= 1; + dim3 grid(query_heads, sequence, batch); + int used_cudnn = 0; +#ifdef H3_USE_CUDNN + if (!getenv("H3_DISABLE_CUDNN_ATTENTION") && dtype == H3_GPU_BF16 && + batch == 1 && !causal && head_dim == 128 && + query_heads == kv_heads && head_major_input) { + char reason[512] = {0}; + h3_cudnn_sdpa *entry = h3_cudnn_sdpa_get( + gpu, sequence, query_heads, scale, head_major_output, + reason, sizeof(reason)); + if (entry && entry->ready) { + if (!h3_cudnn_sdpa_execute( + gpu, entry, output->data, query->data, key->data, + value->data, reason, sizeof(reason))) + return h3_set_error(gpu, "%s", reason); + used_cudnn = 1; + } else if (getenv("H3_REQUIRE_CUDNN_ATTENTION")) { + return h3_set_error(gpu, "%s", reason[0] ? reason : + "cuDNN SDPA is unavailable"); + } + } +#else + if (getenv("H3_REQUIRE_CUDNN_ATTENTION")) + return h3_set_error(gpu, "cuDNN SDPA was not enabled at build time"); +#endif + if (used_cudnn) { + /* The graph is enqueued on the same nonblocking stream. */ + } else if (!getenv("H3_DISABLE_TILED_ATTENTION") && + dtype == H3_GPU_BF16 && + batch == 1 && !causal && head_dim == 128 && + query_heads == kv_heads && head_major_input) { + dim3 tiled_grid(query_heads, (sequence + 7) / 8, batch); + h3_attention_tiled_bf16_kernel<<stream>>>( + (__nv_bfloat16 *)output->data, + (const __nv_bfloat16 *)query->data, + (const __nv_bfloat16 *)key->data, + (const __nv_bfloat16 *)value->data, sequence, query_heads, scale, + head_major_output); + } else if (!getenv("H3_DISABLE_TILED_ATTENTION") && + dtype == H3_GPU_F32 && + batch == 1 && !causal && head_dim == 64 && + query_heads == kv_heads && head_major_input) { + dim3 tiled_grid(query_heads, (sequence + 7) / 8, batch); + h3_attention_tiled_f32_kernel<64><<stream>>>( + (float *)output->data, (const float *)query->data, + (const float *)key->data, (const float *)value->data, sequence, + query_heads, scale, head_major_output); + } else if (dtype == H3_GPU_F32) + h3_attention_kernel<<stream>>>( + (float *)output->data, (const float *)query->data, + (const float *)key->data, (const float *)value->data, batch, + sequence, query_heads, kv_heads, head_dim, scale, causal, + head_major_output, head_major_input); + else + h3_attention_kernel<__nv_bfloat16><<stream>>>((__nv_bfloat16 *)output->data, + (const __nv_bfloat16 *)query->data, + (const __nv_bfloat16 *)key->data, + (const __nv_bfloat16 *)value->data, batch, sequence, query_heads, + kv_heads, head_dim, scale, causal, head_major_output, + head_major_input); + int ok = h3_launch_ok(gpu, "scaled dot-product attention"); + if (ok) gpu->stats.mps_sdpa_dispatches++; + return ok; +} + +int h3_gpu_sdpa_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *query, const h3_gpu_tensor *key, + const h3_gpu_tensor *value, uint32_t sequence, uint32_t heads, + uint32_t head_dim, float scale) { + return h3_attention_dispatch(gpu, output, query, key, value, 1, sequence, + heads, heads, head_dim, scale, H3_GPU_F32, 0, 0); +} + +int h3_gpu_sdpa_causal_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *query, const h3_gpu_tensor *key, + const h3_gpu_tensor *value, uint32_t batch, uint32_t sequence, + uint32_t heads, uint32_t head_dim, float scale) { + return h3_attention_dispatch(gpu, output, query, key, value, batch, + sequence, heads, heads, head_dim, scale, H3_GPU_F32, 1, 0); +} + +int h3_gpu_sdpa_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *query, const h3_gpu_tensor *key, + const h3_gpu_tensor *value, uint32_t sequence, uint32_t heads, + uint32_t head_dim, float scale) { + return h3_attention_dispatch(gpu, output, query, key, value, 1, sequence, + heads, heads, head_dim, scale, H3_GPU_BF16, 0, 0); +} + +int h3_gpu_sdpa_bf16_head_major_output(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *query, const h3_gpu_tensor *key, + const h3_gpu_tensor *value, uint32_t sequence, uint32_t heads, + uint32_t head_dim, float scale) { + return h3_attention_dispatch(gpu, output, query, key, value, 1, sequence, + heads, heads, head_dim, scale, H3_GPU_BF16, 0, 1); +} + +int h3_gpu_gqa_causal_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *query, const h3_gpu_tensor *key, + const h3_gpu_tensor *value, uint32_t sequence, uint32_t query_heads, + uint32_t kv_heads, uint32_t head_dim, float scale) { + return h3_attention_dispatch(gpu, output, query, key, value, 1, sequence, + query_heads, kv_heads, head_dim, scale, H3_GPU_BF16, 2, 0); +} + +__global__ static void h3_conv1d_kernel(float *output, const float *input, + const float *weight, const float *bias, uint32_t batch, + uint32_t length, uint32_t input_channels, uint32_t output_channels, + uint32_t kernel, uint32_t stride, uint32_t padding, + uint32_t dilation, uint32_t output_length, size_t output_elements, + int has_bias) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index >= output_elements) return; + uint32_t output_channel = (uint32_t)(index % output_channels); + size_t row = index / output_channels; + uint32_t output_time = (uint32_t)(row % output_length); + uint32_t batch_index = (uint32_t)(row / output_length); + float sum = has_bias ? bias[output_channel] : 0.0f; + for (uint32_t input_channel = 0; input_channel < input_channels; + input_channel++) + for (uint32_t tap = 0; tap < kernel; tap++) { + int64_t input_time = (int64_t)output_time * stride - padding + + (int64_t)tap * dilation; + if (input_time < 0 || input_time >= length) continue; + size_t input_index = ((size_t)batch_index * length + + (size_t)input_time) * input_channels + + input_channel; + size_t weight_index = ((size_t)output_channel * input_channels + + input_channel) * kernel + tap; + sum = fmaf(input[input_index], weight[weight_index], sum); + } + output[((size_t)batch_index * output_length + output_time) * + output_channels + output_channel] = sum; +} + +int h3_gpu_conv1d_stride_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *weight, + const h3_gpu_tensor *bias, uint32_t batch, uint32_t length, + uint32_t input_channels, uint32_t output_channels, uint32_t kernel, + uint32_t stride, uint32_t padding, uint32_t dilation) { + uint64_t effective = (uint64_t)dilation * (kernel ? kernel - 1 : 0) + 1; + if (!batch || !length || !input_channels || !output_channels || !kernel || + !stride || !dilation || (uint64_t)length + 2ull * padding < effective) + return h3_set_error(gpu, "invalid Conv1d shape"); + uint64_t output_length64 = ((uint64_t)length + 2ull * padding - effective) / + stride + 1; + if (output_length64 > UINT32_MAX) + return h3_set_error(gpu, "Conv1d output length overflows"); + uint32_t output_length = (uint32_t)output_length64; + size_t input_elements, weight_elements, output_elements; + if (!h3_mul_size((size_t)batch * length, input_channels, &input_elements) || + !h3_mul_size((size_t)output_channels * input_channels, kernel, + &weight_elements) || + !h3_mul_size((size_t)batch * output_length, output_channels, + &output_elements) || + !h3_tensor_is(input, gpu, H3_GPU_F32, input_elements) || + !h3_tensor_is(weight, gpu, H3_GPU_F32, weight_elements) || + !h3_tensor_is(output, gpu, H3_GPU_F32, output_elements) || + (bias && !h3_tensor_is(bias, gpu, H3_GPU_F32, output_channels))) + return h3_set_error(gpu, "invalid Conv1d tensors"); + h3_conv1d_kernel<<<(output_elements + 255) / 256, 256, 0, gpu->stream>>>( + (float *)output->data, (const float *)input->data, + (const float *)weight->data, + bias ? (const float *)bias->data : (const float *)input->data, + batch, length, input_channels, output_channels, kernel, stride, + padding, dilation, output_length, output_elements, bias != NULL); + int ok = h3_launch_ok(gpu, "Conv1d"); + if (ok) gpu->stats.mps_conv_dispatches++; + return ok; +} + +int h3_gpu_conv1d_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *weight, + const h3_gpu_tensor *bias, uint32_t batch, uint32_t length, + uint32_t input_channels, uint32_t output_channels, uint32_t kernel, + uint32_t padding, uint32_t dilation) { + return h3_gpu_conv1d_stride_f32(gpu, output, input, weight, bias, batch, + length, input_channels, output_channels, kernel, 1, padding, dilation); +} + +__global__ static void h3_conv_transpose1d_kernel(float *output, + const float *input, const float *weight, const float *bias, + uint32_t batch, uint32_t length, uint32_t input_channels, + uint32_t output_channels, uint32_t kernel, uint32_t stride, + uint32_t padding, uint32_t output_length, size_t output_elements, + int has_bias) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index >= output_elements) return; + uint32_t output_channel = (uint32_t)(index % output_channels); + size_t row = index / output_channels; + uint32_t output_time = (uint32_t)(row % output_length); + uint32_t batch_index = (uint32_t)(row / output_length); + float sum = has_bias ? bias[output_channel] : 0.0f; + for (uint32_t input_channel = 0; input_channel < input_channels; + input_channel++) + for (uint32_t tap = 0; tap < kernel; tap++) { + int64_t numerator = (int64_t)output_time + padding - tap; + if (numerator < 0 || numerator % stride) continue; + uint64_t input_time = (uint64_t)numerator / stride; + if (input_time >= length) continue; + size_t input_index = ((size_t)batch_index * length + input_time) * + input_channels + input_channel; + size_t weight_index = ((size_t)input_channel * output_channels + + output_channel) * kernel + tap; + sum = fmaf(input[input_index], weight[weight_index], sum); + } + output[((size_t)batch_index * output_length + output_time) * + output_channels + output_channel] = sum; +} + +int h3_gpu_conv_transpose1d_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *weight, + const h3_gpu_tensor *bias, uint32_t batch, uint32_t length, + uint32_t input_channels, uint32_t output_channels, uint32_t kernel, + uint32_t stride, uint32_t padding) { + uint64_t full = length ? (uint64_t)(length - 1) * stride + kernel : 0; + if (!batch || !length || !input_channels || !output_channels || !kernel || + !stride || full < 2ull * padding || full - 2ull * padding > UINT32_MAX) + return h3_set_error(gpu, "invalid ConvTranspose1d shape"); + uint32_t output_length = (uint32_t)(full - 2ull * padding); + size_t input_elements, weight_elements, output_elements; + if (!h3_mul_size((size_t)batch * length, input_channels, &input_elements) || + !h3_mul_size((size_t)input_channels * output_channels, kernel, + &weight_elements) || + !h3_mul_size((size_t)batch * output_length, output_channels, + &output_elements) || + !h3_tensor_is(input, gpu, H3_GPU_F32, input_elements) || + !h3_tensor_is(weight, gpu, H3_GPU_F32, weight_elements) || + !h3_tensor_is(output, gpu, H3_GPU_F32, output_elements) || + (bias && !h3_tensor_is(bias, gpu, H3_GPU_F32, output_channels))) + return h3_set_error(gpu, "invalid ConvTranspose1d tensors"); + h3_conv_transpose1d_kernel<<<(output_elements + 255) / 256, 256, 0, + gpu->stream>>>( + (float *)output->data, (const float *)input->data, + (const float *)weight->data, + bias ? (const float *)bias->data : (const float *)input->data, + batch, length, input_channels, output_channels, kernel, stride, + padding, output_length, output_elements, bias != NULL); + int ok = h3_launch_ok(gpu, "ConvTranspose1d"); + if (ok) gpu->stats.mps_conv_dispatches++; + return ok; +} + +__global__ static void h3_audio_qkv_split_kernel(float *query, float *key, + float *value, const float *qkv, const float *q_bias, + const float *k_bias, const float *v_bias, size_t count, + uint32_t length, uint32_t heads, uint32_t head_dim) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index >= count) return; + uint32_t width = heads * head_dim; + uint32_t column = (uint32_t)(index % width); + size_t row = index / width; + uint32_t time = (uint32_t)(row % length); + uint32_t batch_index = (uint32_t)(row / length); + uint32_t head = column / head_dim; + uint32_t dimension = column % head_dim; + size_t output = (((size_t)batch_index * heads + head) * length + time) * + head_dim + dimension; + size_t base = row * width * 3; + query[output] = qkv[base + column] + q_bias[column]; + key[output] = qkv[base + width + column] + k_bias[column]; + value[output] = qkv[base + width * 2 + column] + v_bias[column]; +} + +int h3_gpu_audio_qkv_split_f32(h3_gpu *gpu, h3_gpu_tensor *query, + h3_gpu_tensor *key, h3_gpu_tensor *value, const h3_gpu_tensor *qkv, + const h3_gpu_tensor *q_bias, const h3_gpu_tensor *k_bias, + const h3_gpu_tensor *v_bias, uint32_t batch, uint32_t length, + uint32_t heads, uint32_t head_dim) { + size_t width, count; + if (!batch || !length || !h3_mul_size(heads, head_dim, &width) || !width || + !h3_mul_size((size_t)batch * length, width, &count) || + count > SIZE_MAX / 3 || !h3_tensor_is(qkv, gpu, H3_GPU_F32, count * 3) || + !h3_tensor_is(q_bias, gpu, H3_GPU_F32, width) || + !h3_tensor_is(k_bias, gpu, H3_GPU_F32, width) || + !h3_tensor_is(v_bias, gpu, H3_GPU_F32, width) || + !h3_tensor_is(query, gpu, H3_GPU_F32, count) || + !h3_tensor_is(key, gpu, H3_GPU_F32, count) || + !h3_tensor_is(value, gpu, H3_GPU_F32, count)) + return h3_set_error(gpu, "invalid audio QKV tensors or shape"); + h3_audio_qkv_split_kernel<<<(count + 255) / 256, 256, 0, gpu->stream>>>( + (float *)query->data, (float *)key->data, (float *)value->data, + (const float *)qkv->data, (const float *)q_bias->data, + (const float *)k_bias->data, (const float *)v_bias->data, count, + length, heads, head_dim); + return h3_launch_ok(gpu, "audio QKV split"); +} + +__global__ static void h3_audio_pool_kernel(float *output, + const float *attended, size_t count, uint32_t heads, + uint32_t head_dim, uint32_t output_dim) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index >= count) return; + uint32_t column = (uint32_t)(index % output_dim); + size_t row = index / output_dim; + uint32_t pool = head_dim / output_dim; + float sum = 0.0f; + for (uint32_t head = 0; head < heads; head++) { + size_t base = (row * heads + head) * head_dim + column * pool; + for (uint32_t item = 0; item < pool; item++) sum += attended[base + item]; + } + output[index] = sum / (float)(heads * pool); +} + +int h3_gpu_audio_attention_pool_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *attended, uint32_t batch, uint32_t length, + uint32_t heads, uint32_t head_dim, uint32_t output_dim) { + size_t input_elements, output_elements; + if (!batch || !length || !heads || !head_dim || !output_dim || + head_dim % output_dim || + !h3_mul_size((size_t)batch * length * heads, head_dim, + &input_elements) || + !h3_mul_size((size_t)batch * length, output_dim, &output_elements) || + !h3_tensor_is(attended, gpu, H3_GPU_F32, input_elements) || + !h3_tensor_is(output, gpu, H3_GPU_F32, output_elements)) + return h3_set_error(gpu, "invalid audio attention pool tensors"); + h3_audio_pool_kernel<<<(output_elements + 255) / 256, 256, 0, + gpu->stream>>>((float *)output->data, (const float *)attended->data, + output_elements, heads, head_dim, output_dim); + return h3_launch_ok(gpu, "audio attention pool"); +} + +__global__ static void h3_alias_free_snake_kernel(float *output, + const float *input, const float *alpha_log, const float *beta_log, + const float *upsample_filter, const float *downsample_filter, + uint32_t batch, uint32_t length, uint32_t channels, size_t elements) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index >= elements) return; + uint32_t channel = (uint32_t)(index % channels); + size_t row = index / channels; + uint32_t time = (uint32_t)(row % length); + uint32_t batch_index = (uint32_t)(row / length); + float alpha = expf(alpha_log[channel]); + float beta = expf(beta_log[channel]); + float result = 0.0f; + for (int down_tap = 0; down_tap < 12; down_tap++) { + int up_time = max(0, min((int)length * 2 - 1, + (int)time * 2 + down_tap - 5)); + int raw_time = up_time + 15; + float upsampled = 0.0f; + for (int up_tap = 0; up_tap < 12; up_tap++) { + int numerator = raw_time - up_tap; + if (numerator < 0 || (numerator & 1)) continue; + int source_time = max(0, min((int)length - 1, + numerator / 2 - 5)); + size_t source = ((size_t)batch_index * length + source_time) * + channels + channel; + upsampled = fmaf(input[source], 2.0f * upsample_filter[up_tap], + upsampled); + } + float sine = sinf(alpha * upsampled); + float activated = upsampled + sine * sine / (beta + 1e-9f); + result = fmaf(activated, downsample_filter[down_tap], result); + } + output[((size_t)batch_index * length + time) * channels + channel] = result; +} + +int h3_gpu_alias_free_snake_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *alpha_log, + const h3_gpu_tensor *beta_log, + const h3_gpu_tensor *upsample_filter, + const h3_gpu_tensor *downsample_filter, uint32_t batch, + uint32_t length, uint32_t channels) { + size_t elements; + if (!batch || !length || !channels || + !h3_mul_size((size_t)batch * length, channels, &elements) || + !h3_tensor_is(input, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(output, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(alpha_log, gpu, H3_GPU_F32, channels) || + !h3_tensor_is(beta_log, gpu, H3_GPU_F32, channels) || + !h3_tensor_is(upsample_filter, gpu, H3_GPU_F32, 12) || + !h3_tensor_is(downsample_filter, gpu, H3_GPU_F32, 12)) + return h3_set_error(gpu, "invalid alias-free Snake tensors"); + h3_alias_free_snake_kernel<<<(elements + 255) / 256, 256, 0, gpu->stream>>>( + (float *)output->data, (const float *)input->data, + (const float *)alpha_log->data, (const float *)beta_log->data, + (const float *)upsample_filter->data, + (const float *)downsample_filter->data, batch, length, channels, + elements); + return h3_launch_ok(gpu, "alias-free Snake"); +} + +__device__ static int h3_reflect(int coordinate, int length) { + if (coordinate < 0) return -coordinate; + if (coordinate >= length) return 2 * length - coordinate - 2; + return coordinate; +} + +__global__ static void h3_vae_pad_kernel(float *output, const float *input, + uint32_t batch, uint32_t depth, uint32_t height, uint32_t width, + uint32_t channels, uint32_t depth_front, uint32_t height_before, + uint32_t height_after, uint32_t width_before, uint32_t width_after) { + uint32_t channel = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t output_x = blockIdx.y; + uint32_t plane = blockIdx.z; + uint32_t output_depth = depth + depth_front; + uint32_t output_height = height + height_before + height_after; + uint32_t output_width = width + width_before + width_after; + if (channel >= channels || output_x >= output_width || + plane >= batch * output_depth * output_height) return; + uint32_t output_y = plane % output_height; + uint32_t temporal = plane / output_height; + uint32_t output_t = temporal % output_depth; + uint32_t batch_index = temporal / output_depth; + size_t destination = ((((size_t)batch_index * output_depth + output_t) * + output_height + output_y) * output_width + output_x) * channels + + channel; + if (output_t < depth_front) { output[destination] = 0.0f; return; } + int source_y = h3_reflect((int)output_y - (int)height_before, (int)height); + int source_x = h3_reflect((int)output_x - (int)width_before, (int)width); + uint32_t source_t = output_t - depth_front; + size_t source = ((((size_t)batch_index * depth + source_t) * height + + (uint32_t)source_y) * width + (uint32_t)source_x) * channels + channel; + output[destination] = input[source]; +} + +int h3_gpu_vae_encoder_pad_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, uint32_t batch, uint32_t depth, + uint32_t height, uint32_t width, uint32_t channels, + uint32_t depth_front, uint32_t height_before, uint32_t height_after, + uint32_t width_before, uint32_t width_after) { + if (!batch || !depth || height < 2 || width < 2 || !channels || + height_before >= height || height_after >= height || + width_before >= width || width_after >= width) + return h3_set_error(gpu, "invalid VAE padding shape"); + size_t input_elements, output_elements; + uint64_t output_depth = (uint64_t)depth + depth_front; + uint64_t output_height = (uint64_t)height + height_before + height_after; + uint64_t output_width = (uint64_t)width + width_before + width_after; + if (output_depth > UINT32_MAX || output_height > UINT32_MAX || + output_width > UINT32_MAX || + !h3_mul_size((size_t)batch * depth * height * width, channels, + &input_elements) || + !h3_mul_size((size_t)batch * (size_t)output_depth * + (size_t)output_height * (size_t)output_width, channels, + &output_elements) || + !h3_tensor_is(input, gpu, H3_GPU_F32, input_elements) || + !h3_tensor_is(output, gpu, H3_GPU_F32, output_elements)) + return h3_set_error(gpu, "invalid VAE padding tensors"); + dim3 grid((channels + 127) / 128, (uint32_t)output_width, + (uint32_t)((uint64_t)batch * output_depth * output_height)); + h3_vae_pad_kernel<<stream>>>( + (float *)output->data, (const float *)input->data, batch, depth, + height, width, channels, depth_front, height_before, height_after, + width_before, width_after); + return h3_launch_ok(gpu, "VAE encoder padding"); +} + +__global__ static void h3_conv3d_kernel(float *output, const float *input, + const float *weight, const float *bias, uint32_t depth, + uint32_t height, uint32_t width, uint32_t input_channels, + uint32_t output_channels, uint32_t kernel_depth, + uint32_t kernel_height, uint32_t kernel_width, uint32_t stride_depth, + uint32_t stride_height, uint32_t stride_width, uint32_t output_depth, + uint32_t output_height, uint32_t output_width, size_t output_elements, + int has_bias) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index >= output_elements) return; + uint32_t output_channel = (uint32_t)(index % output_channels); + size_t row = index / output_channels; + uint32_t spatial_count = output_depth * output_height * output_width; + uint32_t spatial = (uint32_t)(row % spatial_count); + uint32_t batch_index = (uint32_t)(row / spatial_count); + uint32_t output_x = spatial % output_width; + uint32_t output_y = (spatial / output_width) % output_height; + uint32_t output_t = spatial / (output_width * output_height); + if (output_t >= output_depth) return; + float sum = has_bias ? bias[output_channel] : 0.0f; + for (uint32_t input_channel = 0; input_channel < input_channels; + input_channel++) + for (uint32_t kt = 0; kt < kernel_depth; kt++) + for (uint32_t ky = 0; ky < kernel_height; ky++) + for (uint32_t kx = 0; kx < kernel_width; kx++) { + uint32_t it = output_t * stride_depth + kt; + uint32_t iy = output_y * stride_height + ky; + uint32_t ix = output_x * stride_width + kx; + size_t input_index = ((((size_t)batch_index * depth + it) * + height + iy) * width + ix) * input_channels + input_channel; + size_t weight_index = (((((size_t)output_channel * + input_channels + input_channel) * kernel_depth + kt) * + kernel_height + ky) * kernel_width + kx); + sum = fmaf(input[input_index], weight[weight_index], sum); + } + size_t destination = ((((size_t)batch_index * output_depth + output_t) * + output_height + output_y) * output_width + output_x) * output_channels + + output_channel; + output[destination] = sum; +} + +int h3_gpu_conv3d_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *weight, + const h3_gpu_tensor *bias, uint32_t batch, uint32_t depth, + uint32_t height, uint32_t width, uint32_t input_channels, + uint32_t output_channels, uint32_t kernel_depth, + uint32_t kernel_height, uint32_t kernel_width, uint32_t stride_depth, + uint32_t stride_height, uint32_t stride_width) { + if (!batch || !depth || !height || !width || !input_channels || + !output_channels || !kernel_depth || !kernel_height || !kernel_width || + !stride_depth || !stride_height || !stride_width || depth < kernel_depth || + height < kernel_height || width < kernel_width) + return h3_set_error(gpu, "invalid Conv3d shape"); + uint32_t od = (depth - kernel_depth) / stride_depth + 1; + uint32_t oh = (height - kernel_height) / stride_height + 1; + uint32_t ow = (width - kernel_width) / stride_width + 1; + size_t input_elements, weight_elements, output_elements; + if (!h3_mul_size((size_t)batch * depth * height * width, input_channels, + &input_elements) || + !h3_mul_size((size_t)output_channels * input_channels * kernel_depth * + kernel_height, kernel_width, &weight_elements) || + !h3_mul_size((size_t)batch * od * oh * ow, output_channels, + &output_elements) || + !h3_tensor_is(input, gpu, H3_GPU_F32, input_elements) || + !h3_tensor_is(weight, gpu, H3_GPU_F32, weight_elements) || + !h3_tensor_is(output, gpu, H3_GPU_F32, output_elements) || + (bias && !h3_tensor_is(bias, gpu, H3_GPU_F32, output_channels))) + return h3_set_error(gpu, "invalid Conv3d tensors"); + h3_conv3d_kernel<<<(output_elements + 255) / 256, 256, 0, gpu->stream>>>( + (float *)output->data, (const float *)input->data, + (const float *)weight->data, + bias ? (const float *)bias->data : (const float *)input->data, + depth, height, width, input_channels, output_channels, kernel_depth, + kernel_height, kernel_width, stride_depth, stride_height, stride_width, + od, oh, ow, output_elements, bias != NULL); + int ok = h3_launch_ok(gpu, "Conv3d"); + if (ok) gpu->stats.mps_conv_dispatches++; + return ok; +} + +__global__ static void h3_group_norm_silu_kernel(float *output, + const float *input, const float *weight, const float *bias, + uint32_t depth, uint32_t height, uint32_t width, uint32_t channels, + uint32_t groups, uint32_t rows, float epsilon) { + uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= rows) return; + uint32_t channels_per_group = channels / groups; + uint32_t group = row % groups; + uint32_t temporal = row / groups; + uint32_t elements = height * width * channels_per_group; + float mean = 0.0f; + for (uint32_t index = 0; index < elements; index++) { + uint32_t spatial = index / channels_per_group; + uint32_t channel = group * channels_per_group + index % channels_per_group; + mean += input[((size_t)temporal * height * width + spatial) * channels + + channel]; + } + mean /= (float)elements; + float variance = 0.0f; + for (uint32_t index = 0; index < elements; index++) { + uint32_t spatial = index / channels_per_group; + uint32_t channel = group * channels_per_group + index % channels_per_group; + float centered = input[((size_t)temporal * height * width + spatial) * + channels + channel] - mean; + variance = fmaf(centered, centered, variance); + } + float inverse = rsqrtf(variance / (float)elements + epsilon); + for (uint32_t index = 0; index < elements; index++) { + uint32_t spatial = index / channels_per_group; + uint32_t channel = group * channels_per_group + index % channels_per_group; + size_t destination = ((size_t)temporal * height * width + spatial) * + channels + channel; + float value = (input[destination] - mean) * inverse * weight[channel] + + bias[channel]; + output[destination] = value / (1.0f + expf(-value)); + } + (void)depth; +} + +int h3_gpu_vae_encoder_group_norm_silu_f32(h3_gpu *gpu, + h3_gpu_tensor *output, const h3_gpu_tensor *input, + const h3_gpu_tensor *weight, const h3_gpu_tensor *bias, + uint32_t batch, uint32_t depth, uint32_t height, uint32_t width, + uint32_t channels, uint32_t groups, float epsilon) { + size_t elements; + uint64_t rows64 = (uint64_t)batch * depth * groups; + if (!batch || !depth || !height || !width || !channels || !groups || + channels % groups || !(epsilon > 0.0f) || rows64 > UINT32_MAX || + !h3_mul_size((size_t)batch * depth * height * width, channels, + &elements) || + !h3_tensor_is(input, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(output, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(weight, gpu, H3_GPU_F32, channels) || + !h3_tensor_is(bias, gpu, H3_GPU_F32, channels)) + return h3_set_error(gpu, "invalid VAE group norm tensors or shape"); + uint32_t rows = (uint32_t)rows64; + h3_group_norm_silu_kernel<<<(rows + 127) / 128, 128, 0, gpu->stream>>>( + (float *)output->data, (const float *)input->data, + (const float *)weight->data, (const float *)bias->data, depth, height, + width, channels, groups, rows, epsilon); + return h3_launch_ok(gpu, "VAE group norm SiLU"); +} + +static int h3_mlp_bf16_impl(h3_gpu *gpu, h3_gpu_tensor *output, + h3_gpu_tensor *activated, const h3_gpu_tensor *input, + const h3_gpu_tensor *fc1_weight, const h3_gpu_tensor *fc2_weight, + uint32_t rows, uint32_t input_dim, uint32_t hidden_dim, + uint32_t output_dim) { + size_t fused_elements, activated_elements, output_elements; + if (!rows || !input_dim || !hidden_dim || !output_dim || + hidden_dim > UINT32_MAX / 2 || + !h3_mul_size(rows, (size_t)hidden_dim * 2, &fused_elements) || + !h3_mul_size(rows, hidden_dim, &activated_elements) || + !h3_mul_size(rows, output_dim, &output_elements) || + !h3_tensor_is(input, gpu, H3_GPU_BF16, + (size_t)rows * input_dim) || + !h3_tensor_is(fc1_weight, gpu, H3_GPU_BF16, + (size_t)hidden_dim * 2 * input_dim) || + !h3_tensor_is(fc2_weight, gpu, H3_GPU_BF16, + (size_t)output_dim * hidden_dim) || + !h3_tensor_is(output, gpu, H3_GPU_BF16, output_elements) || + (activated && !h3_tensor_is(activated, gpu, H3_GPU_BF16, + activated_elements))) + return h3_set_error(gpu, "invalid BF16 MLP tensors or shape"); + __nv_bfloat16 *fused_data = NULL; + __nv_bfloat16 *activated_data = activated ? + (__nv_bfloat16 *)activated->data : NULL; + cudaError_t status = cudaMallocAsync((void **)&fused_data, + fused_elements * sizeof(*fused_data), gpu->stream); + if (status == cudaSuccess && !activated_data) + status = cudaMallocAsync((void **)&activated_data, + activated_elements * sizeof(*activated_data), gpu->stream); + if (!h3_cuda_ok(gpu, status, "BF16 MLP temporary allocation")) { + if (fused_data) (void)cudaFreeAsync(fused_data, gpu->stream); + return 0; + } + h3_gpu_tensor fused = {gpu, fused_data, fused_elements, + fused_elements * sizeof(*fused_data), H3_GPU_BF16}; + h3_gpu_tensor activation = {gpu, activated_data, activated_elements, + activated_elements * sizeof(*activated_data), H3_GPU_BF16}; + int ok = h3_gpu_linear_bf16(gpu, &fused, input, fc1_weight, NULL, rows, + input_dim, hidden_dim * 2) && + h3_gpu_swiglu_bf16(gpu, &activation, &fused, rows, hidden_dim) && + h3_gpu_linear_bf16(gpu, output, &activation, fc2_weight, NULL, + rows, hidden_dim, output_dim); + status = cudaFreeAsync(fused_data, gpu->stream); + if (ok && !activated) + status = cudaFreeAsync(activated_data, gpu->stream); + if (ok) ok = h3_cuda_ok(gpu, status, "BF16 MLP temporary free"); + return ok; +} + +int h3_gpu_mlp_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *fc1_weight, + const h3_gpu_tensor *fc2_weight, uint32_t rows, uint32_t input_dim, + uint32_t hidden_dim, uint32_t output_dim) { + return h3_mlp_bf16_impl(gpu, output, NULL, input, fc1_weight, fc2_weight, + rows, input_dim, hidden_dim, output_dim); +} + +int h3_gpu_mlp_nax_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + h3_gpu_tensor *activated, const h3_gpu_tensor *input, + const h3_gpu_tensor *fc1_weight, const h3_gpu_tensor *fc2_weight, + uint32_t rows, uint32_t input_dim, uint32_t hidden_dim, + uint32_t output_dim) { + return h3_mlp_bf16_impl(gpu, output, activated, input, fc1_weight, + fc2_weight, rows, input_dim, hidden_dim, + output_dim); +} + +enum h3_unary_kind { H3_SILU, H3_GELU_EXACT, H3_GELU_APPROX, H3_CLIP }; + +__global__ static void h3_unary_f32_kernel(float *output, const float *input, + size_t count, int kind, + float minimum, float maximum) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index >= count) return; + float value = input[index]; + if (kind == H3_SILU) value /= 1.0f + expf(-value); + else if (kind == H3_CLIP) value = fminf(maximum, fmaxf(minimum, value)); + output[index] = value; +} + +__global__ static void h3_unary_bf16_kernel(__nv_bfloat16 *output, + const __nv_bfloat16 *input, + size_t count, int kind) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index >= count) return; + float value = __bfloat162float(input[index]); + if (kind == H3_SILU) value /= 1.0f + expf(-value); + else if (kind == H3_GELU_APPROX) { + float inner = 0.7978845608028654f * + (value + 0.044715f * value * value * value); + value = inner <= -10.0f ? 0.0f : inner >= 10.0f ? value : + 0.5f * value * (1.0f + tanhf(inner)); + } else { + value = value <= -10.0f ? 0.0f : value >= 10.0f ? value : + 0.5f * value * (1.0f + erff(value * 0.7071067811865475f)); + } + output[index] = __float2bfloat16(value); +} + +int h3_gpu_silu_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, uint32_t elements) { + if (!h3_tensor_is(output, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(input, gpu, H3_GPU_F32, elements)) + return h3_set_error(gpu, "invalid F32 SiLU tensors"); + if (elements) h3_unary_f32_kernel<<<(elements + 255) / 256, 256, 0, + gpu->stream>>>((float *)output->data, (const float *)input->data, + elements, H3_SILU, 0.0f, 0.0f); + return h3_launch_ok(gpu, "F32 SiLU"); +} + +int h3_gpu_cast_f32_to_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, uint32_t elements) { + if (!h3_tensor_is(output, gpu, H3_GPU_BF16, elements) || + !h3_tensor_is(input, gpu, H3_GPU_F32, elements)) + return h3_set_error(gpu, "invalid F32-to-BF16 tensors"); + if (elements) h3_f32_to_bf16_kernel<<<(elements + 255) / 256, 256, 0, + gpu->stream>>>((__nv_bfloat16 *)output->data, + (const float *)input->data, elements); + return h3_launch_ok(gpu, "F32-to-BF16 cast"); +} + +int h3_gpu_cast_bf16_to_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, uint32_t elements) { + if (!h3_tensor_is(output, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(input, gpu, H3_GPU_BF16, elements)) + return h3_set_error(gpu, "invalid BF16-to-F32 tensors"); + if (elements) h3_bf16_to_f32_kernel<<<(elements + 255) / 256, 256, 0, + gpu->stream>>>((float *)output->data, + (const __nv_bfloat16 *)input->data, elements); + return h3_launch_ok(gpu, "BF16-to-F32 cast"); +} + +int h3_gpu_clip_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, uint32_t elements, + float minimum, float maximum) { + if (!h3_tensor_is(output, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(input, gpu, H3_GPU_F32, elements) || minimum > maximum) + return h3_set_error(gpu, "invalid F32 clip arguments"); + if (elements) h3_unary_f32_kernel<<<(elements + 255) / 256, 256, 0, + gpu->stream>>>((float *)output->data, (const float *)input->data, + elements, H3_CLIP, minimum, maximum); + return h3_launch_ok(gpu, "F32 clip"); +} + +int h3_gpu_silu_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, uint32_t elements) { + if (!h3_tensor_is(output, gpu, H3_GPU_BF16, elements) || + !h3_tensor_is(input, gpu, H3_GPU_BF16, elements)) + return h3_set_error(gpu, "invalid BF16 SiLU tensors"); + if (elements) h3_unary_bf16_kernel<<<(elements + 255) / 256, 256, 0, + gpu->stream>>>((__nv_bfloat16 *)output->data, + (const __nv_bfloat16 *)input->data, elements, H3_SILU); + return h3_launch_ok(gpu, "BF16 SiLU"); +} + +int h3_gpu_gelu_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, uint32_t elements, + int approximate) { + if (!h3_tensor_is(output, gpu, H3_GPU_BF16, elements) || + !h3_tensor_is(input, gpu, H3_GPU_BF16, elements)) + return h3_set_error(gpu, "invalid BF16 GELU tensors"); + if (elements) h3_unary_bf16_kernel<<<(elements + 255) / 256, 256, 0, + gpu->stream>>>((__nv_bfloat16 *)output->data, + (const __nv_bfloat16 *)input->data, elements, + approximate ? H3_GELU_APPROX : H3_GELU_EXACT); + return h3_launch_ok(gpu, "BF16 GELU"); +} + +enum h3_binary_kind { H3_ADD, H3_SUB, H3_SILU_MUL }; + +__global__ static void h3_binary_bf16_kernel(__nv_bfloat16 *output, + const __nv_bfloat16 *left, const __nv_bfloat16 *right, + size_t count, int kind) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index >= count) return; + float a = __bfloat162float(left[index]); + float b = __bfloat162float(right[index]); + float value = kind == H3_ADD ? a + b : kind == H3_SUB ? a - b : + a / (1.0f + expf(-a)) * b; + output[index] = __float2bfloat16(value); +} + +static int h3_binary_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *left, const h3_gpu_tensor *right, + uint32_t elements, int kind, const char *label) { + if (!h3_tensor_is(output, gpu, H3_GPU_BF16, elements) || + !h3_tensor_is(left, gpu, H3_GPU_BF16, elements) || + !h3_tensor_is(right, gpu, H3_GPU_BF16, elements)) + return h3_set_error(gpu, "invalid %s tensors", label); + if (elements) h3_binary_bf16_kernel<<<(elements + 255) / 256, 256, 0, + gpu->stream>>>((__nv_bfloat16 *)output->data, + (const __nv_bfloat16 *)left->data, + (const __nv_bfloat16 *)right->data, elements, kind); + return h3_launch_ok(gpu, label); +} + +int h3_gpu_add_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *left, const h3_gpu_tensor *right, + uint32_t elements) { + return h3_binary_bf16(gpu, output, left, right, elements, H3_ADD, "BF16 add"); +} +int h3_gpu_sub_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *left, const h3_gpu_tensor *right, + uint32_t elements) { + return h3_binary_bf16(gpu, output, left, right, elements, H3_SUB, "BF16 subtract"); +} +int h3_gpu_silu_mul_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *gate, + const h3_gpu_tensor *up, uint32_t elements) { + return h3_binary_bf16(gpu, output, gate, up, elements, H3_SILU_MUL, "BF16 SiLU multiply"); +} + +__global__ static void h3_add_scaled_kernel(float *output, const float *left, + const float *right, size_t count, float left_scale, + float right_scale) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index < count) + output[index] = left[index] * left_scale + right[index] * right_scale; +} + +int h3_gpu_add_scaled_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *left, + const h3_gpu_tensor *right, float left_scale, + float right_scale, uint32_t elements) { + if (!h3_tensor_is(output, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(left, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(right, gpu, H3_GPU_F32, elements)) + return h3_set_error(gpu, "invalid F32 scaled-add tensors"); + if (elements) h3_add_scaled_kernel<<<(elements + 255) / 256, 256, 0, + gpu->stream>>>((float *)output->data, (const float *)left->data, + (const float *)right->data, elements, left_scale, right_scale); + return h3_launch_ok(gpu, "F32 scaled add"); +} + +__global__ static void h3_euler_kernel(float *sample, size_t sample_offset, + const __nv_bfloat16 *last, const __nv_bfloat16 *previous, + size_t count, float delta, float ratio) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index >= count) return; + float last_value = __bfloat162float(last[index]); + float velocity = fmaf(ratio, + last_value - __bfloat162float(previous[index]), last_value); + sample[sample_offset + index] = + fmaf(delta, velocity, sample[sample_offset + index]); +} + +int h3_gpu_euler_bf16(h3_gpu *gpu, h3_gpu_tensor *sample, + size_t sample_offset, const h3_gpu_tensor *last, + const h3_gpu_tensor *previous, uint32_t elements, + float delta, float ratio) { + if (!h3_tensor_is(sample, gpu, H3_GPU_F32, sample_offset + elements) || + !h3_tensor_is(last, gpu, H3_GPU_BF16, elements) || + !h3_tensor_is(previous, gpu, H3_GPU_BF16, elements)) + return h3_set_error(gpu, "invalid Euler tensors"); + if (elements) h3_euler_kernel<<<(elements + 255) / 256, 256, 0, + gpu->stream>>>((float *)sample->data, sample_offset, + (const __nv_bfloat16 *)last->data, + (const __nv_bfloat16 *)previous->data, elements, delta, ratio); + return h3_launch_ok(gpu, "BF16 Euler update"); +} + +__global__ static void h3_rms_norm_f32_kernel(float *output, + const float *input, const float *weight, uint32_t rows, + uint32_t width, float epsilon) { + uint32_t row = blockIdx.x; + if (row >= rows) return; + __shared__ float sums[256]; + float sum = 0.0f; + for (uint32_t column = threadIdx.x; column < width; column += blockDim.x) { + float value = input[(size_t)row * width + column]; + sum = fmaf(value, value, sum); + } + sums[threadIdx.x] = sum; + __syncthreads(); + for (uint32_t stride = blockDim.x / 2; stride; stride /= 2) { + if (threadIdx.x < stride) sums[threadIdx.x] += sums[threadIdx.x + stride]; + __syncthreads(); + } + float inverse = rsqrtf(sums[0] / (float)width + epsilon); + for (uint32_t column = threadIdx.x; column < width; column += blockDim.x) { + size_t index = (size_t)row * width + column; + output[index] = input[index] * inverse * weight[column]; + } +} + +__global__ static void h3_rms_norm_bf16_kernel(__nv_bfloat16 *output, + const __nv_bfloat16 *input, const __nv_bfloat16 *weight, + uint32_t rows, uint32_t width, float epsilon) { + uint32_t row = blockIdx.x; + if (row >= rows) return; + __shared__ float sums[256]; + float sum = 0.0f; + for (uint32_t column = threadIdx.x; column < width; column += blockDim.x) { + float value = __bfloat162float(input[(size_t)row * width + column]); + sum = fmaf(value, value, sum); + } + sums[threadIdx.x] = sum; + __syncthreads(); + for (uint32_t stride = blockDim.x / 2; stride; stride /= 2) { + if (threadIdx.x < stride) sums[threadIdx.x] += sums[threadIdx.x + stride]; + __syncthreads(); + } + float inverse = rsqrtf(sums[0] / (float)width + epsilon); + for (uint32_t column = threadIdx.x; column < width; column += blockDim.x) { + size_t index = (size_t)row * width + column; + float value = __bfloat162float(input[index]) * inverse * + __bfloat162float(weight[column]); + output[index] = __float2bfloat16(value); + } +} + +static int h3_matrix_elements(h3_gpu *gpu, uint32_t rows, uint32_t width, + size_t *elements) { + if (!rows || !width || (size_t)rows > SIZE_MAX / width) + return h3_set_error(gpu, "invalid matrix shape"); + *elements = (size_t)rows * width; + return 1; +} + +int h3_gpu_rms_norm_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, + const h3_gpu_tensor *weight, uint32_t rows, + uint32_t width, float epsilon) { + size_t elements = 0; + if (!h3_matrix_elements(gpu, rows, width, &elements) || epsilon < 0.0f || + !h3_tensor_is(output, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(input, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(weight, gpu, H3_GPU_F32, width)) + return h3_set_error(gpu, "invalid F32 RMS norm arguments"); + h3_rms_norm_f32_kernel<<stream>>>( + (float *)output->data, (const float *)input->data, + (const float *)weight->data, rows, width, epsilon); + return h3_launch_ok(gpu, "F32 RMS norm"); +} + +int h3_gpu_rms_norm_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, + const h3_gpu_tensor *weight, uint32_t rows, + uint32_t width, float epsilon) { + size_t elements = 0; + if (!h3_matrix_elements(gpu, rows, width, &elements) || epsilon < 0.0f || + !h3_tensor_is(output, gpu, H3_GPU_BF16, elements) || + !h3_tensor_is(input, gpu, H3_GPU_BF16, elements) || + !h3_tensor_is(weight, gpu, H3_GPU_BF16, width)) + return h3_set_error(gpu, "invalid BF16 RMS norm arguments"); + h3_rms_norm_bf16_kernel<<stream>>>( + (__nv_bfloat16 *)output->data, + (const __nv_bfloat16 *)input->data, + (const __nv_bfloat16 *)weight->data, rows, width, epsilon); + return h3_launch_ok(gpu, "BF16 RMS norm"); +} + +__global__ static void h3_layer_norm_f32_kernel(float *output, + const float *input, const float *weight, const float *bias, + uint32_t rows, uint32_t width, float epsilon) { + uint32_t row = blockIdx.x; + if (row >= rows) return; + __shared__ float sums[256]; + float sum = 0.0f; + for (uint32_t column = threadIdx.x; column < width; column += blockDim.x) + sum += input[(size_t)row * width + column]; + sums[threadIdx.x] = sum; + __syncthreads(); + for (uint32_t stride = blockDim.x / 2; stride; stride /= 2) { + if (threadIdx.x < stride) sums[threadIdx.x] += sums[threadIdx.x + stride]; + __syncthreads(); + } + float mean = sums[0] / (float)width; + sum = 0.0f; + for (uint32_t column = threadIdx.x; column < width; column += blockDim.x) { + float centered = input[(size_t)row * width + column] - mean; + sum = fmaf(centered, centered, sum); + } + sums[threadIdx.x] = sum; + __syncthreads(); + for (uint32_t stride = blockDim.x / 2; stride; stride /= 2) { + if (threadIdx.x < stride) sums[threadIdx.x] += sums[threadIdx.x + stride]; + __syncthreads(); + } + float inverse = rsqrtf(sums[0] / (float)width + epsilon); + for (uint32_t column = threadIdx.x; column < width; column += blockDim.x) { + size_t index = (size_t)row * width + column; + output[index] = (input[index] - mean) * inverse * weight[column] + bias[column]; + } +} + +__global__ static void h3_layer_norm_bf16_kernel(__nv_bfloat16 *output, + const __nv_bfloat16 *input, const __nv_bfloat16 *weight, + const __nv_bfloat16 *bias, uint32_t rows, uint32_t width, + float epsilon) { + uint32_t row = blockIdx.x; + if (row >= rows) return; + __shared__ float sums[256]; + float sum = 0.0f; + for (uint32_t column = threadIdx.x; column < width; column += blockDim.x) + sum += __bfloat162float(input[(size_t)row * width + column]); + sums[threadIdx.x] = sum; + __syncthreads(); + for (uint32_t stride = blockDim.x / 2; stride; stride /= 2) { + if (threadIdx.x < stride) sums[threadIdx.x] += sums[threadIdx.x + stride]; + __syncthreads(); + } + float mean = sums[0] / (float)width; + sum = 0.0f; + for (uint32_t column = threadIdx.x; column < width; column += blockDim.x) { + float centered = __bfloat162float(input[(size_t)row * width + column]) - mean; + sum = fmaf(centered, centered, sum); + } + sums[threadIdx.x] = sum; + __syncthreads(); + for (uint32_t stride = blockDim.x / 2; stride; stride /= 2) { + if (threadIdx.x < stride) sums[threadIdx.x] += sums[threadIdx.x + stride]; + __syncthreads(); + } + float inverse = rsqrtf(sums[0] / (float)width + epsilon); + for (uint32_t column = threadIdx.x; column < width; column += blockDim.x) { + size_t index = (size_t)row * width + column; + float value = (__bfloat162float(input[index]) - mean) * inverse * + __bfloat162float(weight[column]) + + __bfloat162float(bias[column]); + output[index] = __float2bfloat16(value); + } +} + +int h3_gpu_layer_norm_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *weight, + const h3_gpu_tensor *bias, uint32_t rows, uint32_t width, + float epsilon) { + size_t elements = 0; + if (!h3_matrix_elements(gpu, rows, width, &elements) || epsilon < 0.0f || + !h3_tensor_is(output, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(input, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(weight, gpu, H3_GPU_F32, width) || + !h3_tensor_is(bias, gpu, H3_GPU_F32, width)) + return h3_set_error(gpu, "invalid F32 layer norm arguments"); + h3_layer_norm_f32_kernel<<stream>>>( + (float *)output->data, (const float *)input->data, + (const float *)weight->data, (const float *)bias->data, + rows, width, epsilon); + return h3_launch_ok(gpu, "F32 layer norm"); +} + +int h3_gpu_layer_norm_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *weight, + const h3_gpu_tensor *bias, uint32_t rows, uint32_t width, + float epsilon) { + size_t elements = 0; + if (!h3_matrix_elements(gpu, rows, width, &elements) || epsilon < 0.0f || + !h3_tensor_is(output, gpu, H3_GPU_BF16, elements) || + !h3_tensor_is(input, gpu, H3_GPU_BF16, elements) || + !h3_tensor_is(weight, gpu, H3_GPU_BF16, width) || + !h3_tensor_is(bias, gpu, H3_GPU_BF16, width)) + return h3_set_error(gpu, "invalid BF16 layer norm arguments"); + h3_layer_norm_bf16_kernel<<stream>>>( + (__nv_bfloat16 *)output->data, + (const __nv_bfloat16 *)input->data, + (const __nv_bfloat16 *)weight->data, + (const __nv_bfloat16 *)bias->data, rows, width, epsilon); + return h3_launch_ok(gpu, "BF16 layer norm"); +} + +template +__device__ static float h3_value(T value); +template <> +__device__ float h3_value(float value) { return value; } +template <> +__device__ float h3_value<__nv_bfloat16>(__nv_bfloat16 value) { + return __bfloat162float(value); +} +template +__device__ static T h3_store(float value); +template <> +__device__ float h3_store(float value) { return value; } +template <> +__device__ __nv_bfloat16 h3_store<__nv_bfloat16>(float value) { + return __float2bfloat16(value); +} + +template +__global__ static void h3_adaln_kernel(T *output, const T *input, + const T *weight, const T *modulation, const uint32_t *row_map, + uint32_t rows, uint32_t width, uint32_t slots, + uint32_t shift_slot, uint32_t scale_slot, float epsilon, + size_t input_offset) { + __shared__ float inverse; + uint32_t row = blockIdx.x; + if (row >= rows) return; + const T *source = input + input_offset + (size_t)row * width; + if (threadIdx.x == 0) { + float square_sum = 0.0f; + for (uint32_t k = 0; k < width; ++k) { + float value = h3_value(source[k]); + square_sum = fmaf(value, value, square_sum); + } + inverse = rsqrtf(square_sum / (float)width + epsilon); + } + __syncthreads(); + size_t base = (size_t)row_map[row] * slots * width; + for (uint32_t column = threadIdx.x; column < width; + column += blockDim.x) { + float normalized = h3_value(source[column]) * inverse * + h3_value(weight[column]); + float shift = h3_value( + modulation[base + (size_t)shift_slot * width + column]); + float scale = h3_value( + modulation[base + (size_t)scale_slot * width + column]); + output[(size_t)row * width + column] = + h3_store(normalized * (1.0f + scale) + shift); + } +} + +template +__global__ static void h3_gate_kernel(T *output, const T *residual, + const T *branch, const T *modulation, const uint32_t *row_map, + uint32_t rows, uint32_t width, uint32_t slots, uint32_t gate_slot) { + uint32_t column = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t row = blockIdx.y; + if (row >= rows || column >= width) return; + size_t index = (size_t)row * width + column; + size_t base = (size_t)row_map[row] * slots * width; + float gate = h3_value(modulation[base + (size_t)gate_slot * width + column]); + output[index] = h3_store(h3_value(residual[index]) + + h3_value(branch[index]) * gate); +} + +static int h3_adaln_validate(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, size_t input_offset, + const h3_gpu_tensor *weight, const h3_gpu_tensor *modulation, + const h3_gpu_tensor *row_map, uint32_t rows, uint32_t width, + uint32_t slots, uint32_t shift_slot, uint32_t scale_slot, + h3_gpu_dtype dtype, size_t *elements) { + if (!h3_matrix_elements(gpu, rows, width, elements) || !slots || + shift_slot >= slots || scale_slot >= slots || + input_offset > SIZE_MAX - *elements || + !h3_tensor_is(output, gpu, dtype, *elements) || + !h3_tensor_is(input, gpu, dtype, input_offset + *elements) || + !h3_tensor_is(weight, gpu, dtype, width) || + !modulation || modulation->gpu != gpu || modulation->dtype != dtype || + !h3_tensor_is(row_map, gpu, H3_GPU_U32, rows)) + return h3_set_error(gpu, "invalid AdaLN arguments"); + return 1; +} + +int h3_gpu_adaln_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *norm_weight, + const h3_gpu_tensor *modulation, const h3_gpu_tensor *row_map, + uint32_t rows, uint32_t width, uint32_t slots, uint32_t shift_slot, + uint32_t scale_slot, float epsilon) { + size_t elements = 0; + if (epsilon < 0.0f || !h3_adaln_validate(gpu, output, input, 0, + norm_weight, modulation, row_map, rows, width, slots, shift_slot, + scale_slot, H3_GPU_F32, &elements)) return 0; + h3_adaln_kernel<<stream>>>( + (float *)output->data, (const float *)input->data, + (const float *)norm_weight->data, (const float *)modulation->data, + (const uint32_t *)row_map->data, rows, width, slots, shift_slot, + scale_slot, epsilon, 0); + return h3_launch_ok(gpu, "F32 AdaLN"); +} + +int h3_gpu_adaln_bf16_offset(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, size_t input_offset, + const h3_gpu_tensor *norm_weight, const h3_gpu_tensor *modulation, + const h3_gpu_tensor *row_map, uint32_t rows, uint32_t width, + uint32_t slots, uint32_t shift_slot, uint32_t scale_slot, + float epsilon) { + size_t elements = 0; + if (epsilon < 0.0f || !h3_adaln_validate(gpu, output, input, input_offset, + norm_weight, modulation, row_map, rows, width, slots, shift_slot, + scale_slot, H3_GPU_BF16, &elements)) return 0; + h3_adaln_kernel<__nv_bfloat16><<stream>>>( + (__nv_bfloat16 *)output->data, + (const __nv_bfloat16 *)input->data, + (const __nv_bfloat16 *)norm_weight->data, + (const __nv_bfloat16 *)modulation->data, + (const uint32_t *)row_map->data, rows, width, slots, shift_slot, + scale_slot, epsilon, input_offset); + return h3_launch_ok(gpu, "BF16 AdaLN"); +} + +int h3_gpu_adaln_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *norm_weight, + const h3_gpu_tensor *modulation, const h3_gpu_tensor *row_map, + uint32_t rows, uint32_t width, uint32_t slots, uint32_t shift_slot, + uint32_t scale_slot, float epsilon) { + return h3_gpu_adaln_bf16_offset(gpu, output, input, 0, norm_weight, + modulation, row_map, rows, width, slots, shift_slot, scale_slot, + epsilon); +} + +static int h3_gate_dispatch(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *residual, const h3_gpu_tensor *branch, + const h3_gpu_tensor *modulation, const h3_gpu_tensor *row_map, + uint32_t rows, uint32_t width, uint32_t slots, uint32_t gate_slot, + h3_gpu_dtype dtype) { + size_t elements = 0; + if (!h3_matrix_elements(gpu, rows, width, &elements) || !slots || + gate_slot >= slots || !h3_tensor_is(output, gpu, dtype, elements) || + !h3_tensor_is(residual, gpu, dtype, elements) || + !h3_tensor_is(branch, gpu, dtype, elements) || + !modulation || modulation->gpu != gpu || modulation->dtype != dtype || + !h3_tensor_is(row_map, gpu, H3_GPU_U32, rows)) + return h3_set_error(gpu, "invalid gate arguments"); + dim3 grid((width + 255) / 256, rows); + if (dtype == H3_GPU_F32) + h3_gate_kernel<<stream>>>( + (float *)output->data, (const float *)residual->data, + (const float *)branch->data, (const float *)modulation->data, + (const uint32_t *)row_map->data, rows, width, slots, gate_slot); + else + h3_gate_kernel<__nv_bfloat16><<stream>>>( + (__nv_bfloat16 *)output->data, + (const __nv_bfloat16 *)residual->data, + (const __nv_bfloat16 *)branch->data, + (const __nv_bfloat16 *)modulation->data, + (const uint32_t *)row_map->data, rows, width, slots, gate_slot); + return h3_launch_ok(gpu, dtype == H3_GPU_F32 ? "F32 gate" : "BF16 gate"); +} + +int h3_gpu_gate_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *residual, const h3_gpu_tensor *branch, + const h3_gpu_tensor *modulation, const h3_gpu_tensor *row_map, + uint32_t rows, uint32_t width, uint32_t slots, uint32_t gate_slot) { + return h3_gate_dispatch(gpu, output, residual, branch, modulation, row_map, + rows, width, slots, gate_slot, H3_GPU_F32); +} +int h3_gpu_gate_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *residual, const h3_gpu_tensor *branch, + const h3_gpu_tensor *modulation, const h3_gpu_tensor *row_map, + uint32_t rows, uint32_t width, uint32_t slots, uint32_t gate_slot) { + return h3_gate_dispatch(gpu, output, residual, branch, modulation, row_map, + rows, width, slots, gate_slot, H3_GPU_BF16); +} + +__global__ static void h3_embedding_kernel(__nv_bfloat16 *output, + const __nv_bfloat16 *weight, const uint32_t *token_ids, + uint32_t tokens, uint32_t vocab_size, uint32_t width) { + uint32_t column = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t token = blockIdx.y; + if (token >= tokens || column >= width) return; + uint32_t id = token_ids[token]; + output[(size_t)token * width + column] = id < vocab_size ? + weight[(size_t)id * width + column] : __float2bfloat16(0.0f); +} + +int h3_gpu_embedding_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *weight, const h3_gpu_tensor *token_ids, + uint32_t tokens, uint32_t vocab_size, uint32_t width) { + size_t output_elements = 0; + size_t weight_elements = 0; + if (!h3_matrix_elements(gpu, tokens, width, &output_elements) || + !h3_matrix_elements(gpu, vocab_size, width, &weight_elements) || + !h3_tensor_is(output, gpu, H3_GPU_BF16, output_elements) || + !h3_tensor_is(weight, gpu, H3_GPU_BF16, weight_elements) || + !h3_tensor_is(token_ids, gpu, H3_GPU_U32, tokens)) + return h3_set_error(gpu, "invalid embedding arguments"); + dim3 grid((width + 255) / 256, tokens); + h3_embedding_kernel<<stream>>>( + (__nv_bfloat16 *)output->data, + (const __nv_bfloat16 *)weight->data, + (const uint32_t *)token_ids->data, tokens, vocab_size, width); + return h3_launch_ok(gpu, "BF16 embedding"); +} + +template +__global__ static void h3_swiglu_kernel(T *output, const T *fused, + uint32_t rows, uint32_t width) { + uint32_t column = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t row = blockIdx.y; + if (row >= rows || column >= width) return; + size_t base = (size_t)row * width * 2; + float gate = h3_value(fused[base + column]); + float up = h3_value(fused[base + width + column]); + output[(size_t)row * width + column] = + h3_store(gate / (1.0f + expf(-gate)) * up); +} + +static int h3_swiglu_dispatch(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *fused, uint32_t rows, uint32_t width, + h3_gpu_dtype dtype) { + size_t output_elements = 0; + if (!h3_matrix_elements(gpu, rows, width, &output_elements) || + output_elements > SIZE_MAX / 2 || + !h3_tensor_is(output, gpu, dtype, output_elements) || + !h3_tensor_is(fused, gpu, dtype, output_elements * 2)) + return h3_set_error(gpu, "invalid SwiGLU arguments"); + dim3 grid((width + 255) / 256, rows); + if (dtype == H3_GPU_F32) + h3_swiglu_kernel<<stream>>>( + (float *)output->data, (const float *)fused->data, rows, width); + else + h3_swiglu_kernel<__nv_bfloat16><<stream>>>( + (__nv_bfloat16 *)output->data, + (const __nv_bfloat16 *)fused->data, rows, width); + return h3_launch_ok(gpu, dtype == H3_GPU_F32 ? "F32 SwiGLU" : "BF16 SwiGLU"); +} + +int h3_gpu_swiglu_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *fused, uint32_t rows, + uint32_t width) { + return h3_swiglu_dispatch(gpu, output, fused, rows, width, H3_GPU_F32); +} +int h3_gpu_swiglu_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *fused, uint32_t rows, + uint32_t width) { + return h3_swiglu_dispatch(gpu, output, fused, rows, width, H3_GPU_BF16); +} + +__global__ static void h3_scale_add_kernel(float *output, + const float *residual, const float *branch, const float *scale, + uint32_t rows, uint32_t width) { + uint32_t column = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t row = blockIdx.y; + if (row >= rows || column >= width) return; + size_t index = (size_t)row * width + column; + output[index] = residual[index] + branch[index] * scale[column]; +} + +int h3_gpu_scale_add_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *residual, const h3_gpu_tensor *branch, + const h3_gpu_tensor *scale, uint32_t rows, uint32_t width) { + size_t elements = 0; + if (!h3_matrix_elements(gpu, rows, width, &elements) || + !h3_tensor_is(output, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(residual, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(branch, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(scale, gpu, H3_GPU_F32, width)) + return h3_set_error(gpu, "invalid scale-add arguments"); + dim3 grid((width + 255) / 256, rows); + h3_scale_add_kernel<<stream>>>( + (float *)output->data, (const float *)residual->data, + (const float *)branch->data, (const float *)scale->data, rows, width); + return h3_launch_ok(gpu, "F32 scale add"); +} + +__global__ static void h3_geglu_kernel(float *output, const float *gate, + const float *linear, uint32_t count) { + uint32_t index = blockIdx.x * blockDim.x + threadIdx.x; + if (index >= count) return; + float value = gate[index]; + float gelu = 0.5f * value * (1.0f + tanhf(0.7978845608028654f * + (value + 0.044715f * value * value * value))); + output[index] = gelu * linear[index]; +} + +int h3_gpu_geglu_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *gate, const h3_gpu_tensor *linear, + uint32_t elements) { + if (!h3_tensor_is(output, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(gate, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(linear, gpu, H3_GPU_F32, elements)) + return h3_set_error(gpu, "invalid GEGLU arguments"); + if (elements) h3_geglu_kernel<<<(elements + 255) / 256, 256, 0, + gpu->stream>>>((float *)output->data, (const float *)gate->data, + (const float *)linear->data, elements); + return h3_launch_ok(gpu, "F32 GEGLU"); +} + +__global__ static void h3_snake_kernel(float *output, const float *input, + const float *alpha, size_t count, uint32_t channels) { + size_t index = (size_t)blockIdx.x * blockDim.x + threadIdx.x; + if (index >= count) return; + float a = alpha[index % channels]; + float value = input[index]; + float wave = sinf(a * value); + output[index] = value + wave * wave / (a + 1e-9f); +} + +int h3_gpu_snake1d_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, const h3_gpu_tensor *alpha, + uint32_t batch, uint32_t length, uint32_t channels) { + size_t count = (size_t)batch * length; + if (!batch || !length || !channels || count > SIZE_MAX / channels) + return h3_set_error(gpu, "invalid Snake shape"); + count *= channels; + if (!h3_tensor_is(output, gpu, H3_GPU_F32, count) || + !h3_tensor_is(input, gpu, H3_GPU_F32, count) || + !h3_tensor_is(alpha, gpu, H3_GPU_F32, channels)) + return h3_set_error(gpu, "invalid Snake tensors"); + h3_snake_kernel<<<(count + 255) / 256, 256, 0, gpu->stream>>>( + (float *)output->data, (const float *)input->data, + (const float *)alpha->data, count, channels); + return h3_launch_ok(gpu, "F32 Snake1d"); +} + +__global__ static void h3_weight_norm_kernel(float *output, + const float *vector, const float *magnitude, uint32_t outer, + uint32_t inner) { + uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; + if (row >= outer) return; + size_t base = (size_t)row * inner; + float square_sum = 0.0f; + for (uint32_t column = 0; column < inner; column++) + square_sum = fmaf(vector[base + column], vector[base + column], square_sum); + float scale = magnitude[row] * rsqrtf(square_sum); + for (uint32_t column = 0; column < inner; column++) + output[base + column] = vector[base + column] * scale; +} + +int h3_gpu_weight_norm_f32(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *vector, const h3_gpu_tensor *magnitude, + uint32_t outer, uint32_t inner) { + size_t elements = 0; + if (!h3_matrix_elements(gpu, outer, inner, &elements) || + !h3_tensor_is(output, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(vector, gpu, H3_GPU_F32, elements) || + !h3_tensor_is(magnitude, gpu, H3_GPU_F32, outer)) + return h3_set_error(gpu, "invalid weight norm arguments"); + h3_weight_norm_kernel<<<(outer + 255) / 256, 256, 0, gpu->stream>>>( + (float *)output->data, (const float *)vector->data, + (const float *)magnitude->data, outer, inner); + return h3_launch_ok(gpu, "F32 weight norm"); +} + +__global__ static void h3_head_rms_kernel(__nv_bfloat16 *tensor, + const __nv_bfloat16 *weight, uint32_t sequence, uint32_t heads, + uint32_t head_dim, float epsilon) { + uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t head = blockIdx.y; + if (row >= sequence || head >= heads) return; + size_t base = ((size_t)row * heads + head) * head_dim; + float sum = 0.0f; + for (uint32_t d = 0; d < head_dim; d++) { + float value = __bfloat162float(tensor[base + d]); + sum = fmaf(value, value, sum); + } + float inverse = rsqrtf(sum / (float)head_dim + epsilon); + for (uint32_t d = 0; d < head_dim; d++) + tensor[base + d] = __float2bfloat16( + __bfloat162float(tensor[base + d]) * inverse * + __bfloat162float(weight[d])); +} + +int h3_gpu_head_rms_norm_bf16(h3_gpu *gpu, h3_gpu_tensor *tensor, + const h3_gpu_tensor *weight, uint32_t sequence, uint32_t heads, + uint32_t head_dim, float epsilon) { + size_t elements = (size_t)sequence * heads; + if (!sequence || !heads || !head_dim || elements > SIZE_MAX / head_dim) + return h3_set_error(gpu, "invalid head RMS shape"); + elements *= head_dim; + if (!h3_tensor_is(tensor, gpu, H3_GPU_BF16, elements) || + !h3_tensor_is(weight, gpu, H3_GPU_BF16, head_dim) || epsilon < 0.0f) + return h3_set_error(gpu, "invalid head RMS arguments"); + dim3 grid((sequence + 127) / 128, heads); + h3_head_rms_kernel<<stream>>>( + (__nv_bfloat16 *)tensor->data, + (const __nv_bfloat16 *)weight->data, sequence, heads, head_dim, epsilon); + return h3_launch_ok(gpu, "BF16 head RMS norm"); +} + +__global__ static void h3_rope_text_kernel(__nv_bfloat16 *query, + __nv_bfloat16 *key, const float *rope_cos, const float *rope_sin, + uint32_t sequence, uint32_t query_heads, uint32_t kv_heads, + uint32_t head_dim) { + uint32_t row = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t head = blockIdx.y; + if (row >= sequence) return; + uint32_t half = head_dim / 2; + for (uint32_t d = 0; d < half; d++) { + float c = rope_cos[(size_t)row * half + d]; + float s = rope_sin[(size_t)row * half + d]; + if (head < query_heads) { + size_t base = ((size_t)row * query_heads + head) * head_dim; + float first = __bfloat162float(query[base + d]); + float second = __bfloat162float(query[base + half + d]); + query[base + d] = __float2bfloat16(first * c - second * s); + query[base + half + d] = __float2bfloat16(second * c + first * s); + } + if (head < kv_heads) { + size_t base = ((size_t)row * kv_heads + head) * head_dim; + float first = __bfloat162float(key[base + d]); + float second = __bfloat162float(key[base + half + d]); + key[base + d] = __float2bfloat16(first * c - second * s); + key[base + half + d] = __float2bfloat16(second * c + first * s); + } + } +} + +int h3_gpu_rope_text_bf16(h3_gpu *gpu, h3_gpu_tensor *query, + h3_gpu_tensor *key, const h3_gpu_tensor *rope_cos_f32, + const h3_gpu_tensor *rope_sin_f32, uint32_t sequence, + uint32_t query_heads, uint32_t kv_heads, uint32_t head_dim) { + if (!sequence || !query_heads || !kv_heads || !head_dim || head_dim % 2) + return h3_set_error(gpu, "invalid text RoPE shape"); + size_t query_elements = (size_t)sequence * query_heads * head_dim; + size_t key_elements = (size_t)sequence * kv_heads * head_dim; + size_t rope_elements = (size_t)sequence * (head_dim / 2); + if (!h3_tensor_is(query, gpu, H3_GPU_BF16, query_elements) || + !h3_tensor_is(key, gpu, H3_GPU_BF16, key_elements) || + !h3_tensor_is(rope_cos_f32, gpu, H3_GPU_F32, rope_elements) || + !h3_tensor_is(rope_sin_f32, gpu, H3_GPU_F32, rope_elements)) + return h3_set_error(gpu, "invalid text RoPE tensors"); + uint32_t maximum_heads = query_heads > kv_heads ? query_heads : kv_heads; + dim3 grid((sequence + 127) / 128, maximum_heads); + h3_rope_text_kernel<<stream>>>( + (__nv_bfloat16 *)query->data, (__nv_bfloat16 *)key->data, + (const float *)rope_cos_f32->data, (const float *)rope_sin_f32->data, + sequence, query_heads, kv_heads, head_dim); + return h3_launch_ok(gpu, "BF16 text RoPE"); +} + +template +__global__ static void h3_qkv_rope_kernel(T *query, T *key, T *value, + const T *qkv, const T *q_weight, const T *k_weight, + const T *rope_cos, const T *rope_sin, uint32_t sequence, + uint32_t heads, uint32_t head_dim, uint32_t rope_half, + float epsilon, int grouped, int normalize, int weighted) { + uint32_t dimension = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t head = blockIdx.y; + uint32_t row = blockIdx.z; + if (row >= sequence || head >= heads || dimension >= head_dim) return; + size_t inner = (size_t)heads * head_dim; + size_t row_base = (size_t)row * inner * 3; + size_t q_base = row_base + (size_t)head * head_dim; + size_t k_base = q_base + inner; + size_t v_base = q_base + inner * 2; + if (grouped) { + q_base = row_base + (size_t)head * head_dim * 3; + k_base = q_base + head_dim; + v_base = k_base + head_dim; + } + float q_inverse = 1.0f; + float k_inverse = 1.0f; + if (normalize) { + float q_sum = 0.0f, k_sum = 0.0f; + for (uint32_t d = 0; d < head_dim; d++) { + float q = h3_value(qkv[q_base + d]); + float k = h3_value(qkv[k_base + d]); + q_sum = fmaf(q, q, q_sum); + k_sum = fmaf(k, k, k_sum); + } + q_inverse = rsqrtf(q_sum / (float)head_dim + epsilon); + k_inverse = rsqrtf(k_sum / (float)head_dim + epsilon); + } + float qw = weighted ? h3_value(q_weight[dimension]) : 1.0f; + float kw = weighted ? h3_value(k_weight[dimension]) : 1.0f; + float q0 = h3_value(qkv[q_base + dimension]) * q_inverse * qw; + float k0 = h3_value(qkv[k_base + dimension]) * k_inverse * kw; + if (dimension < rope_half * 2) { + uint32_t rope_index = dimension % rope_half; + uint32_t pair = dimension < rope_half ? dimension + rope_half : + dimension - rope_half; + float q1 = h3_value(qkv[q_base + pair]) * q_inverse * + (weighted ? h3_value(q_weight[pair]) : 1.0f); + float k1 = h3_value(qkv[k_base + pair]) * k_inverse * + (weighted ? h3_value(k_weight[pair]) : 1.0f); + float c = h3_value(rope_cos[(size_t)row * rope_half + rope_index]); + float s = h3_value(rope_sin[(size_t)row * rope_half + rope_index]); + if (dimension < rope_half) { + q0 = q0 * c - q1 * s; + k0 = k0 * c - k1 * s; + } else { + q0 = q0 * c + q1 * s; + k0 = k0 * c + k1 * s; + } + } + size_t output_index = ((size_t)head * sequence + row) * head_dim + dimension; + query[output_index] = h3_store(q0); + key[output_index] = h3_store(k0); + value[output_index] = qkv[v_base + dimension]; +} + +static int h3_qkv_shape(h3_gpu *gpu, uint32_t sequence, uint32_t heads, + uint32_t head_dim, uint32_t rope_half, size_t *elements) { + if (!sequence || !heads || !head_dim || rope_half > head_dim / 2) + return h3_set_error(gpu, "invalid QKV/RoPE shape"); + size_t count = (size_t)sequence * heads; + if (count > SIZE_MAX / head_dim) return h3_set_error(gpu, "QKV shape overflow"); + *elements = count * head_dim; + return 1; +} + +static int h3_qkv_validate(h3_gpu *gpu, h3_gpu_tensor *query, + h3_gpu_tensor *key, h3_gpu_tensor *value, const h3_gpu_tensor *qkv, + const h3_gpu_tensor *q_weight, const h3_gpu_tensor *k_weight, + const h3_gpu_tensor *rope_cos, const h3_gpu_tensor *rope_sin, + uint32_t sequence, uint32_t heads, uint32_t head_dim, + uint32_t rope_half, h3_gpu_dtype dtype, int weighted, + size_t *elements) { + if (!h3_qkv_shape(gpu, sequence, heads, head_dim, rope_half, elements) || + *elements > SIZE_MAX / 3 || + !h3_tensor_is(query, gpu, dtype, *elements) || + !h3_tensor_is(key, gpu, dtype, *elements) || + !h3_tensor_is(value, gpu, dtype, *elements) || + !h3_tensor_is(qkv, gpu, dtype, *elements * 3) || + !h3_tensor_is(rope_cos, gpu, dtype, (size_t)sequence * rope_half) || + !h3_tensor_is(rope_sin, gpu, dtype, (size_t)sequence * rope_half) || + (weighted && (!h3_tensor_is(q_weight, gpu, dtype, head_dim) || + !h3_tensor_is(k_weight, gpu, dtype, head_dim)))) + return h3_set_error(gpu, "invalid QKV/RoPE tensors"); + return 1; +} + +int h3_gpu_qkv_rope_f32(h3_gpu *gpu, h3_gpu_tensor *query, + h3_gpu_tensor *key, h3_gpu_tensor *value, const h3_gpu_tensor *qkv, + const h3_gpu_tensor *q_norm, const h3_gpu_tensor *k_norm, + const h3_gpu_tensor *rope_cos, const h3_gpu_tensor *rope_sin, + uint32_t sequence, uint32_t heads, uint32_t head_dim, + uint32_t rope_half, float epsilon) { + size_t elements = 0; + if (!h3_qkv_validate(gpu, query, key, value, qkv, q_norm, k_norm, + rope_cos, rope_sin, sequence, heads, head_dim, rope_half, + H3_GPU_F32, 1, &elements) || epsilon < 0.0f) return 0; + dim3 grid((head_dim + 127) / 128, heads, sequence); + h3_qkv_rope_kernel<<stream>>>( + (float *)query->data, (float *)key->data, (float *)value->data, + (const float *)qkv->data, (const float *)q_norm->data, + (const float *)k_norm->data, (const float *)rope_cos->data, + (const float *)rope_sin->data, sequence, heads, head_dim, + rope_half, epsilon, 0, 1, 1); + return h3_launch_ok(gpu, "F32 QKV RoPE"); +} + +int h3_gpu_video_qkv_rope_f32(h3_gpu *gpu, h3_gpu_tensor *query, + h3_gpu_tensor *key, h3_gpu_tensor *value, const h3_gpu_tensor *qkv, + const h3_gpu_tensor *rope_cos, const h3_gpu_tensor *rope_sin, + uint32_t sequence, uint32_t heads, uint32_t head_dim, + uint32_t rope_half, float epsilon) { + size_t elements = 0; + if (!h3_qkv_validate(gpu, query, key, value, qkv, NULL, NULL, + rope_cos, rope_sin, sequence, heads, head_dim, rope_half, + H3_GPU_F32, 0, &elements) || epsilon < 0.0f) return 0; + dim3 grid((head_dim + 127) / 128, heads, sequence); + h3_qkv_rope_kernel<<stream>>>( + (float *)query->data, (float *)key->data, (float *)value->data, + (const float *)qkv->data, NULL, NULL, (const float *)rope_cos->data, + (const float *)rope_sin->data, sequence, heads, head_dim, + rope_half, epsilon, 1, 1, 0); + return h3_launch_ok(gpu, "F32 video QKV RoPE"); +} + +static int h3_qkv_rope_bf16_dispatch(h3_gpu *gpu, h3_gpu_tensor *query, + h3_gpu_tensor *key, h3_gpu_tensor *value, const h3_gpu_tensor *qkv, + const h3_gpu_tensor *q_norm, const h3_gpu_tensor *k_norm, + const h3_gpu_tensor *rope_cos, const h3_gpu_tensor *rope_sin, + uint32_t sequence, uint32_t heads, uint32_t head_dim, + uint32_t rope_half, float epsilon, int grouped) { + size_t elements = 0; + if (!h3_qkv_validate(gpu, query, key, value, qkv, q_norm, k_norm, + rope_cos, rope_sin, sequence, heads, head_dim, rope_half, + H3_GPU_BF16, 1, &elements) || epsilon < 0.0f) return 0; + dim3 grid((head_dim + 127) / 128, heads, sequence); + h3_qkv_rope_kernel<__nv_bfloat16><<stream>>>( + (__nv_bfloat16 *)query->data, (__nv_bfloat16 *)key->data, + (__nv_bfloat16 *)value->data, (const __nv_bfloat16 *)qkv->data, + (const __nv_bfloat16 *)q_norm->data, + (const __nv_bfloat16 *)k_norm->data, + (const __nv_bfloat16 *)rope_cos->data, + (const __nv_bfloat16 *)rope_sin->data, sequence, heads, head_dim, + rope_half, epsilon, grouped, 1, 1); + return h3_launch_ok(gpu, grouped ? "BF16 grouped QKV RoPE" : "BF16 QKV RoPE"); +} + +int h3_gpu_qkv_rope_bf16(h3_gpu *gpu, h3_gpu_tensor *query, + h3_gpu_tensor *key, h3_gpu_tensor *value, const h3_gpu_tensor *qkv, + const h3_gpu_tensor *q_norm, const h3_gpu_tensor *k_norm, + const h3_gpu_tensor *rope_cos, const h3_gpu_tensor *rope_sin, + uint32_t sequence, uint32_t heads, uint32_t head_dim, + uint32_t rope_half, float epsilon) { + return h3_qkv_rope_bf16_dispatch(gpu, query, key, value, qkv, q_norm, + k_norm, rope_cos, rope_sin, sequence, heads, head_dim, rope_half, + epsilon, 0); +} + +int h3_gpu_grouped_qkv_rope_bf16(h3_gpu *gpu, h3_gpu_tensor *query, + h3_gpu_tensor *key, h3_gpu_tensor *value, const h3_gpu_tensor *qkv, + const h3_gpu_tensor *q_norm, const h3_gpu_tensor *k_norm, + const h3_gpu_tensor *rope_cos, const h3_gpu_tensor *rope_sin, + uint32_t sequence, uint32_t heads, uint32_t head_dim, + uint32_t rope_half, float epsilon) { + return h3_qkv_rope_bf16_dispatch(gpu, query, key, value, qkv, q_norm, + k_norm, rope_cos, rope_sin, sequence, heads, head_dim, rope_half, + epsilon, 1); +} + +int h3_gpu_vision_qkv_rope_bf16(h3_gpu *gpu, h3_gpu_tensor *query, + h3_gpu_tensor *key, h3_gpu_tensor *value, const h3_gpu_tensor *qkv, + const h3_gpu_tensor *rope_cos, const h3_gpu_tensor *rope_sin, + uint32_t sequence, uint32_t heads, uint32_t head_dim, + uint32_t rope_half) { + size_t elements = 0; + if (!h3_qkv_validate(gpu, query, key, value, qkv, NULL, NULL, + rope_cos, rope_sin, sequence, heads, head_dim, rope_half, + H3_GPU_BF16, 0, &elements)) return 0; + dim3 grid((head_dim + 127) / 128, heads, sequence); + h3_qkv_rope_kernel<__nv_bfloat16><<stream>>>( + (__nv_bfloat16 *)query->data, (__nv_bfloat16 *)key->data, + (__nv_bfloat16 *)value->data, (const __nv_bfloat16 *)qkv->data, + NULL, NULL, (const __nv_bfloat16 *)rope_cos->data, + (const __nv_bfloat16 *)rope_sin->data, sequence, heads, head_dim, + rope_half, 0.0f, 0, 0, 0); + return h3_launch_ok(gpu, "BF16 vision QKV RoPE"); +} + +__global__ static void h3_token_pool_kernel(__nv_bfloat16 *output, + const __nv_bfloat16 *input, size_t input_offset, + __nv_bfloat16 *original, size_t original_offset, + __nv_bfloat16 *baseline, size_t baseline_offset, + const uint32_t *baseline_indices, const uint32_t *pairs, + uint32_t rows, uint32_t width) { + uint32_t column = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t row = blockIdx.y; + if (row >= rows || column >= width) return; + uint32_t first_row = pairs[(size_t)row * 2]; + uint32_t second_row = pairs[(size_t)row * 2 + 1]; + __nv_bfloat16 first = input[input_offset + (size_t)first_row * width + column]; + original[original_offset + (size_t)first_row * width + column] = first; + __nv_bfloat16 pooled = first; + if (first_row != second_row) { + __nv_bfloat16 second = input[input_offset + (size_t)second_row * width + column]; + original[original_offset + (size_t)second_row * width + column] = second; + pooled = __float2bfloat16((__bfloat162float(first) + + __bfloat162float(second)) * 0.5f); + } + output[(size_t)row * width + column] = pooled; + uint32_t baseline_row = baseline_indices[row]; + if (baseline_row != UINT32_MAX) + baseline[baseline_offset + (size_t)baseline_row * width + column] = pooled; +} + +int h3_gpu_token_pool_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *input, size_t input_offset, + h3_gpu_tensor *original, size_t original_offset, + h3_gpu_tensor *baseline, size_t baseline_offset, + const h3_gpu_tensor *baseline_indices, const h3_gpu_tensor *pairs, + uint32_t input_rows, uint32_t rows, uint32_t baseline_rows, + uint32_t width) { + size_t input_elements = 0, output_elements = 0, baseline_elements = 0; + if (!h3_matrix_elements(gpu, input_rows, width, &input_elements) || + !h3_matrix_elements(gpu, rows, width, &output_elements) || + !h3_matrix_elements(gpu, baseline_rows, width, &baseline_elements) || + input_offset > SIZE_MAX - input_elements || + original_offset > SIZE_MAX - input_elements || + baseline_offset > SIZE_MAX - baseline_elements || + !h3_tensor_is(output, gpu, H3_GPU_BF16, output_elements) || + !h3_tensor_is(input, gpu, H3_GPU_BF16, input_offset + input_elements) || + !h3_tensor_is(original, gpu, H3_GPU_BF16, original_offset + input_elements) || + !h3_tensor_is(baseline, gpu, H3_GPU_BF16, baseline_offset + baseline_elements) || + !h3_tensor_is(baseline_indices, gpu, H3_GPU_U32, rows) || + rows > SIZE_MAX / 2 || !h3_tensor_is(pairs, gpu, H3_GPU_U32, (size_t)rows * 2)) + return h3_set_error(gpu, "invalid token pool arguments"); + dim3 grid((width + 255) / 256, rows); + h3_token_pool_kernel<<stream>>>( + (__nv_bfloat16 *)output->data, (const __nv_bfloat16 *)input->data, + input_offset, (__nv_bfloat16 *)original->data, original_offset, + (__nv_bfloat16 *)baseline->data, baseline_offset, + (const uint32_t *)baseline_indices->data, + (const uint32_t *)pairs->data, rows, width); + return h3_launch_ok(gpu, "BF16 token pool"); +} + +__global__ static void h3_token_expand_kernel(__nv_bfloat16 *output, + const __nv_bfloat16 *original, size_t original_offset, + const __nv_bfloat16 *reduced, const __nv_bfloat16 *baseline, + size_t baseline_offset, const uint32_t *baseline_indices, + const uint32_t *parents, uint32_t rows, uint32_t width, + uint32_t exact_prefix_rows, float update_scale) { + uint32_t column = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t row = blockIdx.y; + if (row >= rows || column >= width) return; + uint32_t parent = parents[row]; + size_t destination = (size_t)row * width + column; + size_t reduced_index = (size_t)parent * width + column; + uint32_t baseline_row = baseline_indices[parent]; + if (row < exact_prefix_rows || baseline_row == UINT32_MAX) { + output[destination] = reduced[reduced_index]; + return; + } + float update = __bfloat162float(reduced[reduced_index]) - + __bfloat162float(baseline[baseline_offset + + (size_t)baseline_row * width + column]); + output[destination] = __float2bfloat16( + __bfloat162float(original[original_offset + destination]) + + update_scale * update); +} + +int h3_gpu_token_expand_delta_bf16(h3_gpu *gpu, h3_gpu_tensor *output, + const h3_gpu_tensor *original, size_t original_offset, + const h3_gpu_tensor *reduced, const h3_gpu_tensor *baseline, + size_t baseline_offset, const h3_gpu_tensor *baseline_indices, + const h3_gpu_tensor *parents, uint32_t rows, uint32_t reduced_rows, + uint32_t baseline_rows, uint32_t width, uint32_t exact_prefix_rows, + float update_scale) { + size_t output_elements = 0, reduced_elements = 0, baseline_elements = 0; + if (!h3_matrix_elements(gpu, rows, width, &output_elements) || + !h3_matrix_elements(gpu, reduced_rows, width, &reduced_elements) || + !h3_matrix_elements(gpu, baseline_rows, width, &baseline_elements) || + exact_prefix_rows > rows || original_offset > SIZE_MAX - output_elements || + baseline_offset > SIZE_MAX - baseline_elements || + !h3_tensor_is(output, gpu, H3_GPU_BF16, output_elements) || + !h3_tensor_is(original, gpu, H3_GPU_BF16, original_offset + output_elements) || + !h3_tensor_is(reduced, gpu, H3_GPU_BF16, reduced_elements) || + !h3_tensor_is(baseline, gpu, H3_GPU_BF16, baseline_offset + baseline_elements) || + !h3_tensor_is(baseline_indices, gpu, H3_GPU_U32, reduced_rows) || + !h3_tensor_is(parents, gpu, H3_GPU_U32, rows)) + return h3_set_error(gpu, "invalid token expand arguments"); + dim3 grid((width + 255) / 256, rows); + h3_token_expand_kernel<<stream>>>( + (__nv_bfloat16 *)output->data, + (const __nv_bfloat16 *)original->data, original_offset, + (const __nv_bfloat16 *)reduced->data, + (const __nv_bfloat16 *)baseline->data, baseline_offset, + (const uint32_t *)baseline_indices->data, + (const uint32_t *)parents->data, rows, width, exact_prefix_rows, + update_scale); + return h3_launch_ok(gpu, "BF16 token expand"); +} + +int h3_gpu_gate_adaln_bf16(h3_gpu *gpu, h3_gpu_tensor *gated_residual, + h3_gpu_tensor *output, const h3_gpu_tensor *residual, + const h3_gpu_tensor *branch, const h3_gpu_tensor *norm_weight, + const h3_gpu_tensor *gate_modulation, + const h3_gpu_tensor *norm_modulation, const h3_gpu_tensor *row_map, + uint32_t rows, uint32_t width, uint32_t slots, uint32_t gate_slot, + uint32_t shift_slot, uint32_t scale_slot, float epsilon) { + if (!h3_gpu_gate_bf16(gpu, gated_residual, residual, branch, + gate_modulation, row_map, rows, width, slots, + gate_slot)) return 0; + return h3_gpu_adaln_bf16(gpu, output, gated_residual, norm_weight, + norm_modulation, row_map, rows, width, slots, + shift_slot, scale_slot, epsilon); +} + +int h3_gpu_token_pool_adaln_bf16(h3_gpu *gpu, h3_gpu_tensor *residual, + h3_gpu_tensor *output, const h3_gpu_tensor *input, size_t input_offset, + h3_gpu_tensor *original, size_t original_offset, + h3_gpu_tensor *baseline, size_t baseline_offset, + const h3_gpu_tensor *baseline_indices, const h3_gpu_tensor *pairs, + const h3_gpu_tensor *norm_weight, const h3_gpu_tensor *modulation, + const h3_gpu_tensor *row_map, uint32_t input_rows, uint32_t rows, + uint32_t baseline_rows, uint32_t width, uint32_t slots, + uint32_t shift_slot, uint32_t scale_slot, float epsilon) { + if (!h3_gpu_token_pool_bf16(gpu, residual, input, input_offset, original, + original_offset, baseline, baseline_offset, baseline_indices, pairs, + input_rows, rows, baseline_rows, width)) return 0; + return h3_gpu_adaln_bf16(gpu, output, residual, norm_weight, modulation, + row_map, rows, width, slots, shift_slot, scale_slot, epsilon); +} + +int h3_gpu_token_expand_adaln_bf16(h3_gpu *gpu, h3_gpu_tensor *residual, + h3_gpu_tensor *output, const h3_gpu_tensor *original, + size_t original_offset, const h3_gpu_tensor *reduced, + const h3_gpu_tensor *baseline, size_t baseline_offset, + const h3_gpu_tensor *baseline_indices, const h3_gpu_tensor *parents, + const h3_gpu_tensor *norm_weight, const h3_gpu_tensor *modulation, + const h3_gpu_tensor *row_map, uint32_t rows, uint32_t reduced_rows, + uint32_t baseline_rows, uint32_t width, uint32_t exact_prefix_rows, + float update_scale, uint32_t slots, uint32_t shift_slot, + uint32_t scale_slot, float epsilon) { + if (!h3_gpu_token_expand_delta_bf16(gpu, residual, original, + original_offset, reduced, baseline, baseline_offset, baseline_indices, + parents, rows, reduced_rows, baseline_rows, width, exact_prefix_rows, + update_scale)) return 0; + return h3_gpu_adaln_bf16(gpu, output, residual, norm_weight, modulation, + row_map, rows, width, slots, shift_slot, scale_slot, epsilon); +} + +__global__ static void h3_text_qk_rope_kernel(__nv_bfloat16 *query_output, + __nv_bfloat16 *key_output, const __nv_bfloat16 *query_input, + const __nv_bfloat16 *key_input, const __nv_bfloat16 *q_weight, + const __nv_bfloat16 *k_weight, const __nv_bfloat16 *rope_cos, + const __nv_bfloat16 *rope_sin, uint32_t sequence, + uint32_t query_heads, uint32_t kv_heads, uint32_t head_dim, + float epsilon) { + uint32_t dimension = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t head = blockIdx.y; + uint32_t row = blockIdx.z; + if (dimension >= head_dim || head >= query_heads || row >= sequence) return; + uint32_t half = head_dim / 2; + uint32_t pair = dimension < half ? dimension + half : dimension - half; + uint32_t rope_index = dimension % half; + float c = __bfloat162float(rope_cos[(size_t)row * half + rope_index]); + float s = __bfloat162float(rope_sin[(size_t)row * half + rope_index]); + size_t q_base = ((size_t)row * query_heads + head) * head_dim; + float q_sum = 0.0f; + for (uint32_t d = 0; d < head_dim; d++) { + float value = __bfloat162float(query_input[q_base + d]); + q_sum = fmaf(value, value, q_sum); + } + float q_inverse = rsqrtf(q_sum / (float)head_dim + epsilon); + float q0 = __bfloat162float(query_input[q_base + dimension]) * q_inverse * + __bfloat162float(q_weight[dimension]); + float q1 = __bfloat162float(query_input[q_base + pair]) * q_inverse * + __bfloat162float(q_weight[pair]); + query_output[q_base + dimension] = __float2bfloat16( + dimension < half ? q0 * c - q1 * s : q0 * c + q1 * s); + if (head < kv_heads) { + size_t k_base = ((size_t)row * kv_heads + head) * head_dim; + float k_sum = 0.0f; + for (uint32_t d = 0; d < head_dim; d++) { + float value = __bfloat162float(key_input[k_base + d]); + k_sum = fmaf(value, value, k_sum); + } + float k_inverse = rsqrtf(k_sum / (float)head_dim + epsilon); + float k0 = __bfloat162float(key_input[k_base + dimension]) * k_inverse * + __bfloat162float(k_weight[dimension]); + float k1 = __bfloat162float(key_input[k_base + pair]) * k_inverse * + __bfloat162float(k_weight[pair]); + key_output[k_base + dimension] = __float2bfloat16( + dimension < half ? k0 * c - k1 * s : k0 * c + k1 * s); + } +} + +int h3_gpu_text_qk_rope_bf16(h3_gpu *gpu, + h3_gpu_tensor *query_output, h3_gpu_tensor *key_output, + const h3_gpu_tensor *query_input, const h3_gpu_tensor *key_input, + const h3_gpu_tensor *q_norm, const h3_gpu_tensor *k_norm, + const h3_gpu_tensor *rope_cos, const h3_gpu_tensor *rope_sin, + uint32_t sequence, uint32_t query_heads, uint32_t kv_heads, + uint32_t head_dim, float epsilon) { + if (!sequence || !query_heads || !kv_heads || !head_dim || head_dim % 2 || + query_heads < kv_heads || epsilon < 0.0f) + return h3_set_error(gpu, "invalid text QK/RoPE shape"); + size_t query_elements = (size_t)sequence * query_heads * head_dim; + size_t key_elements = (size_t)sequence * kv_heads * head_dim; + size_t rope_elements = (size_t)sequence * (head_dim / 2); + if (!h3_tensor_is(query_output, gpu, H3_GPU_BF16, query_elements) || + !h3_tensor_is(key_output, gpu, H3_GPU_BF16, key_elements) || + !h3_tensor_is(query_input, gpu, H3_GPU_BF16, query_elements) || + !h3_tensor_is(key_input, gpu, H3_GPU_BF16, key_elements) || + !h3_tensor_is(q_norm, gpu, H3_GPU_BF16, head_dim) || + !h3_tensor_is(k_norm, gpu, H3_GPU_BF16, head_dim) || + !h3_tensor_is(rope_cos, gpu, H3_GPU_BF16, rope_elements) || + !h3_tensor_is(rope_sin, gpu, H3_GPU_BF16, rope_elements)) + return h3_set_error(gpu, "invalid text QK/RoPE tensors"); + dim3 grid((head_dim + 127) / 128, query_heads, sequence); + h3_text_qk_rope_kernel<<stream>>>( + (__nv_bfloat16 *)query_output->data, + (__nv_bfloat16 *)key_output->data, + (const __nv_bfloat16 *)query_input->data, + (const __nv_bfloat16 *)key_input->data, + (const __nv_bfloat16 *)q_norm->data, + (const __nv_bfloat16 *)k_norm->data, + (const __nv_bfloat16 *)rope_cos->data, + (const __nv_bfloat16 *)rope_sin->data, sequence, query_heads, + kv_heads, head_dim, epsilon); + return h3_launch_ok(gpu, "BF16 text QK RoPE"); +} diff --git a/tests/test_checkpoint_schema.c b/tests/test_checkpoint_schema.c new file mode 100644 index 00000000..caad620c --- /dev/null +++ b/tests/test_checkpoint_schema.c @@ -0,0 +1,102 @@ +#include "h3_safetensors.h" +#include "h3_weights.h" + +#include +#include +#include +#include + +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #condition); \ + return 1; \ +} } while (0) + +static const h3_st_tensor *require_tensor(const h3_st_header *header, + const char *name, h3_dtype dtype, int dimensions, + const uint64_t *shape) { + const h3_st_tensor *tensor = h3_st_find(header, name); + if (!tensor || tensor->dtype != dtype || tensor->ndim != dimensions) + return NULL; + for (int dimension = 0; dimension < dimensions; dimension++) + if (tensor->shape[dimension] != shape[dimension]) return NULL; + return tensor; +} + +static int check_partition(const char *root, const char *partition, h3_gpu *gpu) { + char audio_path[512]; + char audio_directory[512]; + char video_path[512]; + CHECK(snprintf(audio_path, sizeof(audio_path), "%s/%s/audio_vae/model.safetensors", + root, partition) > 0); + CHECK(snprintf(audio_directory, sizeof(audio_directory), "%s/%s/audio_vae", + root, partition) > 0); + CHECK(snprintf(video_path, sizeof(video_path), + "%s/%s/video_vae/source/model.safetensors", + root, partition) > 0); + char error[512]; + h3_st_header audio; + h3_st_header video; + CHECK(h3_st_read_header(audio_path, &audio, error, sizeof(error))); + CHECK(h3_st_read_header(video_path, &video, error, sizeof(error))); + + const uint64_t audio_bias_shape[] = {2048}; + const uint64_t filter_shape[] = {1, 1, 12}; + const uint64_t video_post_shape[] = {24, 24, 1, 1, 1}; + const uint64_t video_norm_shape[] = {2048}; + const h3_st_tensor *audio_bias = require_tensor( + &audio, "dec_in_proj.bias", H3_DTYPE_F32, 1, audio_bias_shape); + const h3_st_tensor *filter = require_tensor( + &audio, "decoder.activation_post.downsample.lowpass.filter", + H3_DTYPE_F32, 3, filter_shape); + const h3_st_tensor *video_post = require_tensor( + &video, "post_quant_conv.weight", H3_DTYPE_F32, 5, + video_post_shape); + const h3_st_tensor *video_norm = require_tensor( + &video, "decoder.norm_out.weight", H3_DTYPE_F32, 1, + video_norm_shape); + CHECK(audio_bias && filter && video_post && video_norm); + + float filter_values[12]; + float bias_values[2048]; + CHECK(h3_st_read_data(&audio, filter, filter_values, + sizeof(filter_values), error, sizeof(error))); + CHECK(h3_st_read_data(&audio, audio_bias, bias_values, + sizeof(bias_values), error, sizeof(error))); + CHECK(isfinite(bias_values[0])); + float magnitude = 0.0f; + for (size_t index = 0; index < 12; index++) { + CHECK(isfinite(filter_values[index])); + magnitude += fabsf(filter_values[index]); + } + CHECK(magnitude > 0.0f); + CHECK(video_post->data_end > video_post->data_begin); + + h3_weight_store *store = h3_weight_store_open( + audio_directory, error, sizeof(error)); + CHECK(store != NULL && h3_weight_store_shards(store) == 1); + h3_gpu_tensor *loaded_filter = h3_weight_load_f32( + store, gpu, "decoder.activation_post.downsample.lowpass.filter", + 3, filter_shape, error, sizeof(error)); + CHECK(loaded_filter != NULL); + float loaded_values[12]; + CHECK(h3_gpu_tensor_read_f32(loaded_filter, loaded_values, 12)); + CHECK(memcmp(filter_values, loaded_values, sizeof(filter_values)) == 0); + h3_gpu_tensor_free(loaded_filter); + h3_weight_store_free(store); + + h3_st_free_header(&video); + h3_st_free_header(&audio); + return 0; +} + +int main(int argc, char **argv) { + const char *root = argc > 1 ? argv[1] : "MiniMax-H3"; + char error[512]; + h3_gpu *gpu = h3_gpu_create(NULL, error, sizeof(error)); + CHECK(gpu != NULL); + CHECK(check_partition(root, "FL2VA", gpu) == 0); + CHECK(check_partition(root, "Ref2VA", gpu) == 0); + h3_gpu_free(gpu); + puts("ok: official FL2VA/Ref2VA audio and video VAE schemas/payloads"); + return 0; +} diff --git a/tests/test_cuda_attention.c b/tests/test_cuda_attention.c new file mode 100644 index 00000000..4d243ac3 --- /dev/null +++ b/tests/test_cuda_attention.c @@ -0,0 +1,352 @@ +#include "h3_gpu.h" + +#include +#include +#include +#include + +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #condition); \ + return 1; \ +} } while (0) + +static int close_values(const float *actual, const float *expected, + size_t count, float tolerance) { + for (size_t index = 0; index < count; index++) + if (fabsf(actual[index] - expected[index]) > tolerance) return 0; + return 1; +} + +static double seconds_now(void) { + struct timespec time; + clock_gettime(CLOCK_MONOTONIC, &time); + return (double)time.tv_sec + (double)time.tv_nsec * 1e-9; +} + +static int benchmark_attention(uint32_t sequence) { + enum { HEADS = 56, HEAD_DIM = 128, RUNS = 3 }; + size_t elements = (size_t)sequence * HEADS * HEAD_DIM; + if (!sequence || sequence > 20000 || elements > SIZE_MAX / sizeof(float)) + return 1; + float *zeros = calloc(elements, sizeof(*zeros)); + char error[256]; + h3_gpu *gpu = h3_gpu_create(NULL, error, sizeof(error)); + h3_gpu_tensor *query = h3_gpu_tensor_new_bf16(gpu, elements); + h3_gpu_tensor *key = h3_gpu_tensor_new_bf16(gpu, elements); + h3_gpu_tensor *value = h3_gpu_tensor_new_bf16(gpu, elements); + h3_gpu_tensor *output = h3_gpu_tensor_new_bf16(gpu, elements); + CHECK(zeros && gpu && query && key && value && output); + CHECK(h3_gpu_tensor_write_f32(query, zeros, elements)); + CHECK(h3_gpu_tensor_write_f32(key, zeros, elements)); + CHECK(h3_gpu_tensor_write_f32(value, zeros, elements)); + for (int run = -1; run < RUNS; run++) { + double started = seconds_now(); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_sdpa_bf16(gpu, output, query, key, value, sequence, + HEADS, HEAD_DIM, 1.0f / sqrtf(HEAD_DIM))); + CHECK(h3_gpu_submit(gpu)); + double elapsed = seconds_now() - started; + if (run >= 0) + printf("attention sequence=%u run=%d seconds=%.6f\n", + sequence, run + 1, elapsed); + } + h3_gpu_tensor_free(output); h3_gpu_tensor_free(value); + h3_gpu_tensor_free(key); h3_gpu_tensor_free(query); + h3_gpu_free(gpu); free(zeros); + return 0; +} + +int main(int argc, char **argv) { + if (argc == 2) return benchmark_attention((uint32_t)strtoul(argv[1], NULL, 10)); + char error[256]; + h3_gpu *gpu = h3_gpu_create(NULL, error, sizeof(error)); + CHECK(gpu); + const float zeros[] = {0,0,0,0}; + const float values[] = {1,2,3,4}; + const float noncausal_expected[] = {2,3,2,3}; + const float causal_expected[] = {1,2,2,3}; + float actual[16]; + h3_gpu_tensor *query = h3_gpu_tensor_from_f32(gpu, zeros, 4); + h3_gpu_tensor *key = h3_gpu_tensor_from_f32(gpu, zeros, 4); + h3_gpu_tensor *value = h3_gpu_tensor_from_f32(gpu, values, 4); + h3_gpu_tensor *output = h3_gpu_tensor_new_f32(gpu, 4); + CHECK(query && key && value && output); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_sdpa_f32(gpu, output, query, key, value, 2, 1, 2, 1.0f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(output, actual, 4)); + CHECK(close_values(actual, noncausal_expected, 4, 1e-6f)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_sdpa_causal_f32(gpu, output, query, key, value, + 1, 2, 1, 2, 1.0f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(output, actual, 4)); + CHECK(close_values(actual, causal_expected, 4, 1e-6f)); + + float wide_values[1024] = {0}; + float wide_actual[1024]; + for (size_t index = 0; index < 1024; index++) + wide_values[index] = (float)index; + h3_gpu_tensor *wide_query = h3_gpu_tensor_new_f32(gpu, 1024); + h3_gpu_tensor *wide_key = h3_gpu_tensor_new_f32(gpu, 1024); + h3_gpu_tensor *wide_value = h3_gpu_tensor_from_f32(gpu, wide_values, 1024); + h3_gpu_tensor *wide_output = h3_gpu_tensor_new_f32(gpu, 1024); + CHECK(wide_query && wide_key && wide_value && wide_output); + CHECK(h3_gpu_tensor_write_f32(wide_query, (const float[1024]){0}, 1024)); + CHECK(h3_gpu_tensor_write_f32(wide_key, (const float[1024]){0}, 1024)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_sdpa_causal_f32(gpu, wide_output, wide_query, wide_key, + wide_value, 2, 2, 1, 256, 0.0625f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(wide_output, wide_actual, 1024)); + CHECK(close_values(wide_actual, wide_values, 256, 1e-6f)); + CHECK(close_values(wide_actual + 512, wide_values + 512, 256, 1e-6f)); + for (size_t index = 256; index < 512; index++) + CHECK(fabsf(wide_actual[index] - + 0.5f * (wide_values[index - 256] + wide_values[index])) < + 1e-6f); + for (size_t index = 768; index < 1024; index++) + CHECK(fabsf(wide_actual[index] - + 0.5f * (wide_values[index - 256] + wide_values[index])) < + 1e-6f); + + const float two_head_values[] = {1,2,3,4, 10,20,30,40}; + const float row_major_expected[] = {2,3,20,30, 2,3,20,30}; + const float head_major_expected[] = {2,3,2,3, 20,30,20,30}; + h3_gpu_tensor *bquery = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *bkey = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *bvalue = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *boutput = h3_gpu_tensor_new_bf16(gpu, 8); + CHECK(bquery && bkey && bvalue && boutput); + CHECK(h3_gpu_tensor_write_f32(bquery, two_head_values, 8)); + CHECK(h3_gpu_tensor_write_f32(bkey, (const float[8]){0}, 8)); + CHECK(h3_gpu_tensor_write_f32(bvalue, two_head_values, 8)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_sdpa_bf16(gpu, boutput, bquery, bkey, bvalue, + 2, 2, 2, 0.5f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(boutput, actual, 8)); + CHECK(close_values(actual, row_major_expected, 8, 0.02f)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_sdpa_bf16_head_major_output( + gpu, boutput, bquery, bkey, bvalue, 2, 2, 2, 0.5f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(boutput, actual, 8)); + CHECK(close_values(actual, head_major_expected, 8, 0.02f)); + + enum { TSEQ = 3, THEADS = 2, TDIM = 128, TELEMS = TSEQ * THEADS * TDIM }; + float tiled_query[TELEMS], tiled_key[TELEMS], tiled_value[TELEMS]; + float tiled_expected[TELEMS], tiled_head_expected[TELEMS]; + float tiled_actual[TELEMS]; + for (int index = 0; index < TELEMS; index++) { + tiled_query[index] = (float)(index % 17 - 8) * 0.01f; + tiled_key[index] = (float)(index % 13 - 6) * 0.0125f; + tiled_value[index] = (float)(index % 29 - 14) * 0.025f; + } + for (int head = 0; head < THEADS; head++) + for (int row = 0; row < TSEQ; row++) { + float scores[TSEQ], maximum = -INFINITY, denominator = 0.0f; + for (int key_row = 0; key_row < TSEQ; key_row++) { + float score = 0.0f; + for (int dimension = 0; dimension < TDIM; dimension++) + score += tiled_query[(head * TSEQ + row) * TDIM + dimension] * + tiled_key[(head * TSEQ + key_row) * TDIM + dimension]; + scores[key_row] = score / sqrtf((float)TDIM); + maximum = fmaxf(maximum, scores[key_row]); + } + for (int key_row = 0; key_row < TSEQ; key_row++) + denominator += expf(scores[key_row] - maximum); + for (int dimension = 0; dimension < TDIM; dimension++) { + float sum = 0.0f; + for (int key_row = 0; key_row < TSEQ; key_row++) + sum += expf(scores[key_row] - maximum) * + tiled_value[(head * TSEQ + key_row) * TDIM + dimension]; + tiled_expected[(row * THEADS + head) * TDIM + dimension] = + sum / denominator; + tiled_head_expected[(head * TSEQ + row) * TDIM + dimension] = + sum / denominator; + } + } + h3_gpu_tensor *tiled_q = h3_gpu_tensor_new_bf16(gpu, TELEMS); + h3_gpu_tensor *tiled_k = h3_gpu_tensor_new_bf16(gpu, TELEMS); + h3_gpu_tensor *tiled_v = h3_gpu_tensor_new_bf16(gpu, TELEMS); + h3_gpu_tensor *tiled_o = h3_gpu_tensor_new_bf16(gpu, TELEMS); + CHECK(tiled_q && tiled_k && tiled_v && tiled_o); + CHECK(h3_gpu_tensor_write_f32(tiled_q, tiled_query, TELEMS)); + CHECK(h3_gpu_tensor_write_f32(tiled_k, tiled_key, TELEMS)); + CHECK(h3_gpu_tensor_write_f32(tiled_v, tiled_value, TELEMS)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_sdpa_bf16(gpu, tiled_o, tiled_q, tiled_k, tiled_v, + TSEQ, THEADS, TDIM, 1.0f / sqrtf((float)TDIM))); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(tiled_o, tiled_actual, TELEMS)); + CHECK(close_values(tiled_actual, tiled_expected, TELEMS, 0.002f)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_sdpa_bf16_head_major_output( + gpu, tiled_o, tiled_q, tiled_k, tiled_v, TSEQ, THEADS, TDIM, + 1.0f / sqrtf((float)TDIM))); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(tiled_o, tiled_actual, TELEMS)); + CHECK(close_values(tiled_actual, tiled_head_expected, TELEMS, 0.002f)); + + enum { FSEQ = 13, FHEADS = 2, FDIM = 64, FELEMS = FSEQ * FHEADS * FDIM }; + float tiled_f32_query[FELEMS], tiled_f32_key[FELEMS]; + float tiled_f32_value[FELEMS], tiled_f32_expected[FELEMS]; + float tiled_f32_actual[FELEMS], tiled_f32_scalar[FELEMS]; + for (int index = 0; index < FELEMS; index++) { + tiled_f32_query[index] = (float)(index % 17 - 8) * 0.01f; + tiled_f32_key[index] = (float)(index % 13 - 6) * 0.0125f; + tiled_f32_value[index] = (float)(index % 29 - 14) * 0.025f; + } + for (int head = 0; head < FHEADS; head++) + for (int row = 0; row < FSEQ; row++) { + float scores[FSEQ], maximum = -INFINITY, denominator = 0.0f; + for (int key_row = 0; key_row < FSEQ; key_row++) { + float score = 0.0f; + for (int dimension = 0; dimension < FDIM; dimension++) + score += tiled_f32_query[(head * FSEQ + row) * FDIM + + dimension] * + tiled_f32_key[(head * FSEQ + key_row) * FDIM + + dimension]; + scores[key_row] = score / sqrtf((float)FDIM); + maximum = fmaxf(maximum, scores[key_row]); + } + for (int key_row = 0; key_row < FSEQ; key_row++) + denominator += expf(scores[key_row] - maximum); + for (int dimension = 0; dimension < FDIM; dimension++) { + float sum = 0.0f; + for (int key_row = 0; key_row < FSEQ; key_row++) + sum += expf(scores[key_row] - maximum) * + tiled_f32_value[(head * FSEQ + key_row) * FDIM + + dimension]; + tiled_f32_expected[(row * FHEADS + head) * FDIM + dimension] = + sum / denominator; + } + } + h3_gpu_tensor *f32_q = h3_gpu_tensor_new_f32(gpu, FELEMS); + h3_gpu_tensor *f32_k = h3_gpu_tensor_new_f32(gpu, FELEMS); + h3_gpu_tensor *f32_v = h3_gpu_tensor_new_f32(gpu, FELEMS); + h3_gpu_tensor *f32_o = h3_gpu_tensor_new_f32(gpu, FELEMS); + CHECK(f32_q && f32_k && f32_v && f32_o); + CHECK(h3_gpu_tensor_write_f32(f32_q, tiled_f32_query, FELEMS)); + CHECK(h3_gpu_tensor_write_f32(f32_k, tiled_f32_key, FELEMS)); + CHECK(h3_gpu_tensor_write_f32(f32_v, tiled_f32_value, FELEMS)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_sdpa_f32(gpu, f32_o, f32_q, f32_k, f32_v, + FSEQ, FHEADS, FDIM, 1.0f / sqrtf((float)FDIM))); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(f32_o, tiled_f32_actual, FELEMS)); + CHECK(close_values(tiled_f32_actual, tiled_f32_expected, FELEMS, 2e-5f)); + CHECK(setenv("H3_DISABLE_TILED_ATTENTION", "1", 1) == 0); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_sdpa_f32(gpu, f32_o, f32_q, f32_k, f32_v, + FSEQ, FHEADS, FDIM, 1.0f / sqrtf((float)FDIM))); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(f32_o, tiled_f32_scalar, FELEMS)); + CHECK(unsetenv("H3_DISABLE_TILED_ATTENTION") == 0); + CHECK(close_values(tiled_f32_scalar, tiled_f32_expected, FELEMS, 2e-5f)); + CHECK(close_values(tiled_f32_scalar, tiled_f32_actual, FELEMS, 2e-5f)); + + /* Video VAE limit shape: sequence 2805, head_dim 64. */ + enum { LSEQ = 2805, LHEADS = 4, LDIM = 64, LELEMS = LSEQ * LHEADS * LDIM }; + float *long_query = malloc(LELEMS * sizeof(float)); + float *long_key = malloc(LELEMS * sizeof(float)); + float *long_value = malloc(LELEMS * sizeof(float)); + float *long_expected = malloc(LELEMS * sizeof(float)); + float *long_actual = malloc(LELEMS * sizeof(float)); + float *long_scores = malloc(LSEQ * sizeof(float)); + CHECK(long_query && long_key && long_value && long_expected && + long_actual && long_scores); + for (int index = 0; index < LELEMS; index++) { + long_query[index] = (float)(index % 31 - 15) * 0.005f; + long_key[index] = (float)(index % 19 - 9) * 0.0075f; + long_value[index] = (float)(index % 47 - 23) * 0.0125f; + } + for (int head = 0; head < LHEADS; head++) + for (int row = 0; row < LSEQ; row++) { + float maximum = -INFINITY, denominator = 0.0f; + for (int key_row = 0; key_row < LSEQ; key_row++) { + float score = 0.0f; + for (int dimension = 0; dimension < LDIM; dimension++) + score += long_query[(head * LSEQ + row) * LDIM + + dimension] * + long_key[(head * LSEQ + key_row) * LDIM + + dimension]; + long_scores[key_row] = score / sqrtf((float)LDIM); + maximum = fmaxf(maximum, long_scores[key_row]); + } + for (int key_row = 0; key_row < LSEQ; key_row++) + denominator += expf(long_scores[key_row] - maximum); + for (int dimension = 0; dimension < LDIM; dimension++) { + float sum = 0.0f; + for (int key_row = 0; key_row < LSEQ; key_row++) + sum += expf(long_scores[key_row] - maximum) * + long_value[(head * LSEQ + key_row) * LDIM + + dimension]; + long_expected[(row * LHEADS + head) * LDIM + dimension] = + sum / denominator; + } + } + h3_gpu_tensor *long_q = h3_gpu_tensor_new_f32(gpu, LELEMS); + h3_gpu_tensor *long_k = h3_gpu_tensor_new_f32(gpu, LELEMS); + h3_gpu_tensor *long_v = h3_gpu_tensor_new_f32(gpu, LELEMS); + h3_gpu_tensor *long_o = h3_gpu_tensor_new_f32(gpu, LELEMS); + CHECK(long_q && long_k && long_v && long_o); + CHECK(h3_gpu_tensor_write_f32(long_q, long_query, LELEMS)); + CHECK(h3_gpu_tensor_write_f32(long_k, long_key, LELEMS)); + CHECK(h3_gpu_tensor_write_f32(long_v, long_value, LELEMS)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_sdpa_f32(gpu, long_o, long_q, long_k, long_v, + LSEQ, LHEADS, LDIM, 1.0f / sqrtf((float)LDIM))); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(long_o, long_actual, LELEMS)); + CHECK(close_values(long_actual, long_expected, LELEMS, 2e-5f)); + h3_gpu_tensor_free(long_o); h3_gpu_tensor_free(long_v); + h3_gpu_tensor_free(long_k); h3_gpu_tensor_free(long_q); + free(long_scores); free(long_actual); + free(long_expected); free(long_value); free(long_key); free(long_query); + h3_gpu_tensor_free(f32_o); h3_gpu_tensor_free(f32_v); + h3_gpu_tensor_free(f32_k); h3_gpu_tensor_free(f32_q); + + h3_gpu_tensor *gqa_query = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *gqa_key = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *gqa_value = h3_gpu_tensor_new_bf16(gpu, 8); + CHECK(gqa_query && gqa_key && gqa_value); + CHECK(h3_gpu_tensor_write_f32(gqa_query, (const float[8]){0}, 8)); + CHECK(h3_gpu_tensor_write_f32(gqa_key, (const float[8]){0}, 8)); + CHECK(h3_gpu_tensor_write_f32(gqa_value, + (const float[]){1,2,10,20, 3,4,30,40}, 8)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_gqa_causal_bf16(gpu, boutput, gqa_query, gqa_key, + gqa_value, 2, 2, 2, 2, 1.0f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(boutput, actual, 8)); + const float gqa_expected[] = {1,2,10,20, 2,3,20,30}; + CHECK(close_values(actual, gqa_expected, 8, 0.02f)); + CHECK(h3_gpu_begin(gpu)); + CHECK(!h3_gpu_gqa_causal_bf16(gpu, boutput, gqa_query, gqa_key, + gqa_value, 2, 3, 2, 2, 1.0f)); + CHECK(h3_gpu_error(gpu)[0] != '\0'); + CHECK(h3_gpu_submit(gpu)); + + h3_gpu_tensor_free(gqa_value); + h3_gpu_tensor_free(gqa_key); + h3_gpu_tensor_free(gqa_query); + h3_gpu_tensor_free(tiled_o); h3_gpu_tensor_free(tiled_v); + h3_gpu_tensor_free(tiled_k); h3_gpu_tensor_free(tiled_q); + h3_gpu_tensor_free(wide_output); + h3_gpu_tensor_free(wide_value); + h3_gpu_tensor_free(wide_key); + h3_gpu_tensor_free(wide_query); + h3_gpu_tensor_free(boutput); + h3_gpu_tensor_free(bvalue); + h3_gpu_tensor_free(bkey); + h3_gpu_tensor_free(bquery); + h3_gpu_tensor_free(output); + h3_gpu_tensor_free(value); + h3_gpu_tensor_free(key); + h3_gpu_tensor_free(query); + h3_gpu_free(gpu); + puts("ok: CUDA SDPA causal, GQA and head-major layouts"); + return 0; +} diff --git a/tests/test_cuda_linear.c b/tests/test_cuda_linear.c new file mode 100644 index 00000000..18d8f573 --- /dev/null +++ b/tests/test_cuda_linear.c @@ -0,0 +1,310 @@ +#include "h3_gpu.h" + +#include +#include +#include +#include +#include + +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #condition); \ + return 1; \ +} } while (0) + +static int close_values(const float *actual, const float *expected, + size_t count, float tolerance) { + for (size_t index = 0; index < count; index++) + if (fabsf(actual[index] - expected[index]) > tolerance) return 0; + return 1; +} + +static double wall_time(void) { + struct timespec value; + CHECK(clock_gettime(CLOCK_MONOTONIC, &value) == 0); + return (double)value.tv_sec + (double)value.tv_nsec * 1e-9; +} + +static int benchmark_shape(h3_gpu *gpu, const char *name, uint32_t rows, + uint32_t input_dim, uint32_t output_dim) { + h3_gpu_tensor *input = h3_gpu_tensor_new_bf16( + gpu, (size_t)rows * input_dim); + h3_gpu_tensor *weight = h3_gpu_tensor_new_bf16( + gpu, (size_t)output_dim * input_dim); + h3_gpu_tensor *output = h3_gpu_tensor_new_bf16( + gpu, (size_t)rows * output_dim); + CHECK(input && weight && output); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_linear_bf16(gpu, output, input, weight, NULL, rows, + input_dim, output_dim)); + CHECK(h3_gpu_submit(gpu)); + for (int run = 0; run < 3; run++) { + double start = wall_time(); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_linear_bf16(gpu, output, input, weight, NULL, rows, + input_dim, output_dim)); + CHECK(h3_gpu_submit(gpu)); + printf("benchmark %s run=%d seconds=%.6f\n", name, run + 1, + wall_time() - start); + } + h3_gpu_tensor_free(output); + h3_gpu_tensor_free(weight); + h3_gpu_tensor_free(input); + return 0; +} + +static int run_benchmark(void) { + char error[256]; + h3_gpu *gpu = h3_gpu_create(NULL, error, sizeof(error)); + CHECK(gpu); + CHECK(!benchmark_shape(gpu, "qkv", 2048, 5376, 21504)); + CHECK(!benchmark_shape(gpu, "mlp", 2048, 5376, 14336)); + CHECK(!benchmark_shape(gpu, "output", 2048, 5376, 5376)); + h3_gpu_free(gpu); + return 0; +} + +int main(int argc, char **argv) { + if (argc == 2 && strcmp(argv[1], "benchmark") == 0) + return run_benchmark(); + char error[256]; + h3_gpu *gpu = h3_gpu_create(NULL, error, sizeof(error)); + CHECK(gpu); + const float input_values[] = {1, 2, 3, -1, 0.5f, 2}; + const float weight_values[] = {2, -1, 0.5f, -3, 4, 1}; + const float bias_values[] = {0.25f, -0.5f}; + const float expected[] = {1.75f, 7.5f, -1.25f, 6.5f}; + float actual[4]; + + h3_gpu_tensor *input = h3_gpu_tensor_from_f32(gpu, input_values, 6); + h3_gpu_tensor *weight = h3_gpu_tensor_from_f32(gpu, weight_values, 6); + h3_gpu_tensor *bias = h3_gpu_tensor_from_f32(gpu, bias_values, 2); + h3_gpu_tensor *output = h3_gpu_tensor_new_f32(gpu, 4); + CHECK(input && weight && bias && output); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_linear_f32(gpu, output, input, weight, bias, 2, 3, 2)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(output, actual, 4)); + CHECK(close_values(actual, expected, 4, 1e-6f)); + + h3_gpu_tensor *bf_input = h3_gpu_tensor_new_bf16(gpu, 6); + h3_gpu_tensor *bf_weight = h3_gpu_tensor_new_bf16(gpu, 6); + h3_gpu_tensor *bf_bias = h3_gpu_tensor_new_bf16(gpu, 2); + h3_gpu_tensor *bf_output = h3_gpu_tensor_new_bf16(gpu, 4); + CHECK(bf_input && bf_weight && bf_bias && bf_output); + CHECK(h3_gpu_tensor_write_f32(bf_input, input_values, 6)); + CHECK(h3_gpu_tensor_write_f32(bf_weight, weight_values, 6)); + CHECK(h3_gpu_tensor_write_f32(bf_bias, bias_values, 2)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_linear_bf16(gpu, bf_output, bf_input, bf_weight, bf_bias, + 2, 3, 2)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(bf_output, actual, 4)); + CHECK(close_values(actual, expected, 4, 0.04f)); + + const float int8_input_values[] = {1, -2, 0.5f, 1}; + const float int8_weight_values[] = {2, -1, -1, 3}; + const float int8_expected[] = {4, -7, 0, 2.5f}; + h3_gpu_tensor *int8_input = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *int8_weight_source = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *int8_weight = h3_gpu_tensor_new_i8(gpu, 4); + h3_gpu_tensor *int8_weight_scales = h3_gpu_tensor_new_f32(gpu, 2); + h3_gpu_tensor *int8_quantized_input = h3_gpu_tensor_new_i8(gpu, 4); + h3_gpu_tensor *int8_input_scales = h3_gpu_tensor_new_f32(gpu, 2); + h3_gpu_tensor *int8_output = h3_gpu_tensor_new_bf16(gpu, 4); + CHECK(int8_input && int8_weight_source && int8_weight && + int8_weight_scales && int8_quantized_input && int8_input_scales && + int8_output); + CHECK(h3_gpu_tensor_write_f32(int8_input, int8_input_values, 4)); + CHECK(h3_gpu_tensor_write_f32(int8_weight_source, int8_weight_values, 4)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_quantize_weight_int8(gpu, int8_weight, int8_weight_scales, + int8_weight_source, 2, 2)); + CHECK(h3_gpu_linear_int8_bf16( + gpu, int8_output, int8_quantized_input, int8_input_scales, int8_input, + int8_weight, int8_weight_scales, 2, 2, 2, 0)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(int8_output, actual, 4)); + CHECK(close_values(actual, int8_expected, 4, 0.06f)); + + const float mlp_input_values[] = {1, 2}; + const float mlp_fc1_values[] = {1,0, 0,1, 2,0, 0,3}; + const float mlp_fc2_values[] = {1,0, 0,1}; + const float mlp_expected[] = { + 2.0f / (1.0f + expf(-1.0f)), + 12.0f / (1.0f + expf(-2.0f)) + }; + h3_gpu_tensor *mlp_input = h3_gpu_tensor_new_bf16(gpu, 2); + h3_gpu_tensor *mlp_fc1 = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *mlp_fc2 = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *mlp_output = h3_gpu_tensor_new_bf16(gpu, 2); + h3_gpu_tensor *mlp_activated = h3_gpu_tensor_new_bf16(gpu, 2); + CHECK(mlp_input && mlp_fc1 && mlp_fc2 && mlp_output && mlp_activated); + CHECK(h3_gpu_tensor_write_f32(mlp_input, mlp_input_values, 2)); + CHECK(h3_gpu_tensor_write_f32(mlp_fc1, mlp_fc1_values, 8)); + CHECK(h3_gpu_tensor_write_f32(mlp_fc2, mlp_fc2_values, 4)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_mlp_bf16(gpu, mlp_output, mlp_input, mlp_fc1, mlp_fc2, + 1, 2, 2, 2)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(mlp_output, actual, 2)); + CHECK(close_values(actual, mlp_expected, 2, 0.08f)); + + h3_gpu_tensor *mlp_fc1_i8 = h3_gpu_tensor_new_i8(gpu, 8); + h3_gpu_tensor *mlp_fc1_scales = h3_gpu_tensor_new_f32(gpu, 4); + h3_gpu_tensor *mlp_fc2_i8 = h3_gpu_tensor_new_i8(gpu, 4); + h3_gpu_tensor *mlp_fc2_scales = h3_gpu_tensor_new_f32(gpu, 2); + h3_gpu_tensor *mlp_quantized = h3_gpu_tensor_new_i8(gpu, 2); + h3_gpu_tensor *mlp_scales = h3_gpu_tensor_new_f32(gpu, 1); + CHECK(mlp_fc1_i8 && mlp_fc1_scales && mlp_fc2_i8 && mlp_fc2_scales && + mlp_quantized && mlp_scales); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_quantize_weight_int8(gpu, mlp_fc1_i8, mlp_fc1_scales, + mlp_fc1, 4, 2)); + CHECK(h3_gpu_quantize_weight_int8(gpu, mlp_fc2_i8, mlp_fc2_scales, + mlp_fc2, 2, 2)); + CHECK(h3_gpu_mlp_int8_bf16( + gpu, mlp_output, mlp_activated, mlp_quantized, mlp_scales, mlp_input, + mlp_fc1_i8, mlp_fc1_scales, mlp_fc2_i8, mlp_fc2_scales, mlp_fc1, + mlp_fc2, 1, 2, 2, 2, 0, 0, 0, 0)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(mlp_output, actual, 2)); + CHECK(close_values(actual, mlp_expected, 2, 0.12f)); + + const float head_values[] = {1,2, 3,4, 5,6, 7,8}; + const float identity_values[] = { + 1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1 + }; + const float head_expected[] = {1,2,5,6, 3,4,7,8}; + h3_gpu_tensor *head_input = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *head_weight_source = h3_gpu_tensor_new_bf16(gpu, 16); + h3_gpu_tensor *head_weight = h3_gpu_tensor_new_i8(gpu, 16); + h3_gpu_tensor *head_weight_scales = h3_gpu_tensor_new_f32(gpu, 4); + h3_gpu_tensor *head_quantized = h3_gpu_tensor_new_i8(gpu, 8); + h3_gpu_tensor *head_scales = h3_gpu_tensor_new_f32(gpu, 2); + h3_gpu_tensor *head_output = h3_gpu_tensor_new_bf16(gpu, 8); + float head_actual[8]; + CHECK(head_input && head_weight_source && head_weight && + head_weight_scales && head_quantized && head_scales && head_output); + CHECK(h3_gpu_tensor_write_f32(head_input, head_values, 8)); + CHECK(h3_gpu_tensor_write_f32(head_weight_source, identity_values, 16)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_quantize_weight_int8(gpu, head_weight, head_weight_scales, + head_weight_source, 4, 4)); + CHECK(h3_gpu_linear_int8_head_major_bf16( + gpu, head_output, head_quantized, head_scales, head_input, head_weight, + head_weight_scales, 2, 2, 2, 4)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(head_output, head_actual, 8)); + CHECK(close_values(head_actual, head_expected, 8, 0.08f)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_mlp_nax_bf16(gpu, mlp_output, mlp_activated, mlp_input, + mlp_fc1, mlp_fc2, 1, 2, 2, 2)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(mlp_activated, actual, 2)); + CHECK(close_values(actual, mlp_expected, 2, 0.08f)); + + CHECK(h3_gpu_begin(gpu)); + CHECK(!h3_gpu_linear_bf16(gpu, bf_output, input, bf_weight, bf_bias, + 2, 3, 2)); + CHECK(h3_gpu_error(gpu)[0] != '\0'); + CHECK(h3_gpu_submit(gpu)); + + const uint32_t patch_input_dim = 32; + const uint32_t patch_output_dim = 5376; + float *patch_input_values = calloc(patch_input_dim + 1, sizeof(float)); + float *patch_weight_values = calloc( + (size_t)patch_output_dim * patch_input_dim, sizeof(float)); + float *patch_bias_values = malloc((size_t)patch_output_dim * sizeof(float)); + float *patch_actual = malloc((size_t)(patch_output_dim * 2) * sizeof(float)); + CHECK(patch_input_values && patch_weight_values && patch_bias_values && + patch_actual); + for (uint32_t column = 0; column < patch_input_dim; column++) + patch_input_values[column + 1] = (float)(column + 1); + for (uint32_t row = 0; row < patch_output_dim; row++) + patch_bias_values[row] = 1.0f; + patch_weight_values[0] = 2.0f; + patch_weight_values[patch_input_dim + 1] = -1.0f; + h3_gpu_tensor *patch_input = h3_gpu_tensor_from_f32( + gpu, patch_input_values, patch_input_dim + 1); + h3_gpu_tensor *patch_weight = h3_gpu_tensor_from_f32( + gpu, patch_weight_values, (size_t)patch_output_dim * patch_input_dim); + h3_gpu_tensor *patch_bias = h3_gpu_tensor_from_f32( + gpu, patch_bias_values, patch_output_dim); + h3_gpu_tensor *patch_output = h3_gpu_tensor_new_bf16( + gpu, patch_output_dim + 2); + CHECK(patch_input && patch_weight && patch_bias && patch_output); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_patch_linear_bf16_offset( + gpu, patch_output, 2, patch_input, 1, patch_weight, patch_bias, 1, + patch_input_dim, patch_output_dim)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32_range( + patch_output, 2, patch_actual, patch_output_dim)); + CHECK(fabsf(patch_actual[0] - 3.0f) < 0.04f); + CHECK(fabsf(patch_actual[1] + 1.0f) < 0.04f); + CHECK(fabsf(patch_actual[patch_output_dim - 1] - 1.0f) < 0.04f); + + const uint32_t map_value[] = {1}; + h3_gpu_tensor *row_map = h3_gpu_tensor_from_u32(gpu, map_value, 1); + h3_gpu_tensor *mapped_output = h3_gpu_tensor_new_bf16( + gpu, (size_t)patch_output_dim * 2); + CHECK(row_map && mapped_output); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_patch_linear_bf16_map( + gpu, mapped_output, patch_input, patch_weight, patch_bias, row_map, + 2, 1, patch_input_dim, patch_output_dim)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(mapped_output, patch_actual, + (size_t)patch_output_dim * 2)); + CHECK(fabsf(patch_actual[patch_output_dim] - 1.0f) < 0.04f); + CHECK(fabsf(patch_actual[patch_output_dim + 1]) < 0.04f); + + h3_gpu_tensor_free(mapped_output); + h3_gpu_tensor_free(row_map); + h3_gpu_tensor_free(patch_output); + h3_gpu_tensor_free(patch_bias); + h3_gpu_tensor_free(patch_weight); + h3_gpu_tensor_free(patch_input); + free(patch_actual); + free(patch_bias_values); + free(patch_weight_values); + free(patch_input_values); + + h3_gpu_tensor_free(int8_output); + h3_gpu_tensor_free(int8_input_scales); + h3_gpu_tensor_free(int8_quantized_input); + h3_gpu_tensor_free(int8_weight_scales); + h3_gpu_tensor_free(int8_weight); + h3_gpu_tensor_free(int8_weight_source); + h3_gpu_tensor_free(int8_input); + h3_gpu_tensor_free(mlp_activated); + h3_gpu_tensor_free(mlp_output); + h3_gpu_tensor_free(mlp_fc2); + h3_gpu_tensor_free(mlp_fc1); + h3_gpu_tensor_free(mlp_input); + h3_gpu_tensor_free(mlp_scales); + h3_gpu_tensor_free(mlp_quantized); + h3_gpu_tensor_free(mlp_fc2_scales); + h3_gpu_tensor_free(mlp_fc2_i8); + h3_gpu_tensor_free(mlp_fc1_scales); + h3_gpu_tensor_free(mlp_fc1_i8); + h3_gpu_tensor_free(head_output); + h3_gpu_tensor_free(head_scales); + h3_gpu_tensor_free(head_quantized); + h3_gpu_tensor_free(head_weight_scales); + h3_gpu_tensor_free(head_weight); + h3_gpu_tensor_free(head_weight_source); + h3_gpu_tensor_free(head_input); + + h3_gpu_tensor_free(bf_output); + h3_gpu_tensor_free(bf_bias); + h3_gpu_tensor_free(bf_weight); + h3_gpu_tensor_free(bf_input); + h3_gpu_tensor_free(output); + h3_gpu_tensor_free(bias); + h3_gpu_tensor_free(weight); + h3_gpu_tensor_free(input); + h3_gpu_free(gpu); + puts("ok: CUDA cuBLASLt F32/BF16 linear"); + return 0; +} diff --git a/tests/test_cuda_ops.c b/tests/test_cuda_ops.c new file mode 100644 index 00000000..eddc4da7 --- /dev/null +++ b/tests/test_cuda_ops.c @@ -0,0 +1,229 @@ +#include "h3_gpu.h" + +#include +#include + +#define CHECK(condition) do { if (!(condition)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #condition); \ + return 1; \ +} } while (0) + +static int close_values(const float *actual, const float *expected, + size_t count, float tolerance) { + for (size_t index = 0; index < count; index++) + if (fabsf(actual[index] - expected[index]) > tolerance) return 0; + return 1; +} + +static float snake_oracle(const float *input, const float *up, + const float *down, unsigned length, unsigned time) { + float result = 0.0f; + for (int down_tap = 0; down_tap < 12; down_tap++) { + int up_time = (int)time * 2 + down_tap - 5; + if (up_time < 0) up_time = 0; + if (up_time >= (int)length * 2) up_time = (int)length * 2 - 1; + int raw_time = up_time + 15; + float upsampled = 0.0f; + for (int up_tap = 0; up_tap < 12; up_tap++) { + int numerator = raw_time - up_tap; + if (numerator < 0 || (numerator & 1)) continue; + int source = numerator / 2 - 5; + if (source < 0) source = 0; + if (source >= (int)length) source = (int)length - 1; + upsampled += input[source] * 2.0f * up[up_tap]; + } + float sine = sinf(upsampled); + result += (upsampled + sine * sine) * down[down_tap]; + } + return result; +} + +int main(void) { + char error[256]; + h3_gpu *gpu = h3_gpu_create(NULL, error, sizeof(error)); + CHECK(gpu); + float actual[64]; + + const float input1d_values[] = {1,2,3,4}; + const float weight1d_values[] = {1,0,-1}; + const float bias1d_values[] = {0.5f}; + h3_gpu_tensor *input1d = h3_gpu_tensor_from_f32(gpu, input1d_values, 4); + h3_gpu_tensor *weight1d = h3_gpu_tensor_from_f32(gpu, weight1d_values, 3); + h3_gpu_tensor *bias1d = h3_gpu_tensor_from_f32(gpu, bias1d_values, 1); + h3_gpu_tensor *output1d = h3_gpu_tensor_new_f32(gpu, 4); + CHECK(input1d && weight1d && bias1d && output1d); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_conv1d_f32(gpu, output1d, input1d, weight1d, bias1d, + 1, 4, 1, 1, 3, 1, 1)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(output1d, actual, 4)); + CHECK(close_values(actual, (const float[]){-1.5f,-1.5f,-1.5f,3.5f}, + 4, 1e-6f)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_conv1d_stride_f32(gpu, output1d, input1d, weight1d, bias1d, + 1, 4, 1, 1, 3, 2, 1, 1)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(output1d, actual, 2)); + CHECK(close_values(actual, (const float[]){-1.5f,-1.5f}, 2, 1e-6f)); + + const float transpose_input_values[] = {1,2}; + const float transpose_weight_values[] = {1,2}; + h3_gpu_tensor *transpose_input = h3_gpu_tensor_from_f32( + gpu, transpose_input_values, 2); + h3_gpu_tensor *transpose_weight = h3_gpu_tensor_from_f32( + gpu, transpose_weight_values, 2); + CHECK(transpose_input && transpose_weight); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_conv_transpose1d_f32(gpu, output1d, transpose_input, + transpose_weight, NULL, 1, 2, 1, 1, 2, 2, 0)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(output1d, actual, 4)); + CHECK(close_values(actual, (const float[]){1,2,2,4}, 4, 1e-6f)); + + h3_gpu_tensor *long_input = h3_gpu_tensor_new_f32(gpu, 65536); + h3_gpu_tensor *long_output = h3_gpu_tensor_new_f32(gpu, 65536); + CHECK(long_input && long_output); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_conv1d_f32(gpu, long_output, long_input, weight1d, NULL, + 1, 65536, 1, 1, 1, 0, 1)); + CHECK(h3_gpu_conv_transpose1d_f32(gpu, long_output, long_input, + transpose_weight, NULL, 1, 65536, 1, 1, 1, 1, 0)); + CHECK(h3_gpu_conv3d_f32(gpu, long_output, long_input, weight1d, NULL, + 1, 1, 256, 256, 1, 1, 1, 1, 1, 1, 1, 1)); + CHECK(h3_gpu_submit(gpu)); + + const float volume[] = {1,2,3,4,5,6,7,8}; + const float volume_weight[] = {1,1,1,1,1,1,1,1}; + h3_gpu_tensor *volume_input = h3_gpu_tensor_from_f32(gpu, volume, 8); + h3_gpu_tensor *volume_weights = h3_gpu_tensor_from_f32( + gpu, volume_weight, 8); + h3_gpu_tensor *volume_output = h3_gpu_tensor_new_f32(gpu, 1); + CHECK(volume_input && volume_weights && volume_output); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_conv3d_f32(gpu, volume_output, volume_input, volume_weights, + bias1d, 1, 2, 2, 2, 1, 1, 2, 2, 2, 1, 1, 1)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(volume_output, actual, 1)); + CHECK(fabsf(actual[0] - 36.5f) < 1e-6f); + + const float image[] = {1,2,3,4}; + h3_gpu_tensor *image_input = h3_gpu_tensor_from_f32(gpu, image, 4); + h3_gpu_tensor *image_output = h3_gpu_tensor_new_f32(gpu, 32); + CHECK(image_input && image_output); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_vae_encoder_pad_f32(gpu, image_output, image_input, + 1, 1, 2, 2, 1, 1, 1, 1, 1, 1)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(image_output, actual, 32)); + for (size_t index = 0; index < 16; index++) CHECK(actual[index] == 0.0f); + const float reflected[] = {4,3,4,3, 2,1,2,1, 4,3,4,3, 2,1,2,1}; + CHECK(close_values(actual + 16, reflected, 16, 0.0f)); + + const float norm_input_values[] = {1,3,5,7}; + const float norm_weight_values[] = {1,1}; + const float norm_bias_values[] = {0,0}; + h3_gpu_tensor *norm_input = h3_gpu_tensor_from_f32(gpu, norm_input_values, 4); + h3_gpu_tensor *norm_weight = h3_gpu_tensor_from_f32(gpu, norm_weight_values, 2); + h3_gpu_tensor *norm_bias = h3_gpu_tensor_from_f32(gpu, norm_bias_values, 2); + h3_gpu_tensor *norm_output = h3_gpu_tensor_new_f32(gpu, 4); + CHECK(norm_input && norm_weight && norm_bias && norm_output); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_vae_encoder_group_norm_silu_f32(gpu, norm_output, norm_input, + norm_weight, norm_bias, 1, 1, 1, 2, 2, 1, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(norm_output, actual, 4)); + for (size_t index = 0; index < 4; index++) { + float normalized = (norm_input_values[index] - 4.0f) / + sqrtf(5.0f + 1e-5f); + float expected = normalized / (1.0f + expf(-normalized)); + CHECK(fabsf(actual[index] - expected) < 2e-6f); + } + + const float qkv_values[] = {1,2, 3,4, 5,6}; + const float q_bias_values[] = {10,20}; + const float k_bias_values[] = {30,40}; + const float v_bias_values[] = {50,60}; + h3_gpu_tensor *qkv = h3_gpu_tensor_from_f32(gpu, qkv_values, 6); + h3_gpu_tensor *q_bias = h3_gpu_tensor_from_f32(gpu, q_bias_values, 2); + h3_gpu_tensor *k_bias = h3_gpu_tensor_from_f32(gpu, k_bias_values, 2); + h3_gpu_tensor *v_bias = h3_gpu_tensor_from_f32(gpu, v_bias_values, 2); + h3_gpu_tensor *q = h3_gpu_tensor_new_f32(gpu, 2); + h3_gpu_tensor *k = h3_gpu_tensor_new_f32(gpu, 2); + h3_gpu_tensor *v = h3_gpu_tensor_new_f32(gpu, 2); + CHECK(qkv && q_bias && k_bias && v_bias && q && k && v); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_audio_qkv_split_f32(gpu, q, k, v, qkv, q_bias, k_bias, + v_bias, 1, 1, 1, 2)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(q, actual, 2)); + CHECK(close_values(actual, (const float[]){11,22}, 2, 0.0f)); + + const float audio_multi_values[] = { + 1,2,3,4, 0,0,0,0, 0,0,0,0, + 5,6,7,8, 0,0,0,0, 0,0,0,0}; + const float audio_zero_bias[] = {0,0,0,0}; + h3_gpu_tensor *audio_multi = h3_gpu_tensor_from_f32( + gpu, audio_multi_values, 24); + h3_gpu_tensor *audio_bias = h3_gpu_tensor_from_f32( + gpu, audio_zero_bias, 4); + h3_gpu_tensor *audio_q = h3_gpu_tensor_new_f32(gpu, 8); + h3_gpu_tensor *audio_k = h3_gpu_tensor_new_f32(gpu, 8); + h3_gpu_tensor *audio_v = h3_gpu_tensor_new_f32(gpu, 8); + CHECK(audio_multi && audio_bias && audio_q && audio_k && audio_v); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_audio_qkv_split_f32(gpu, audio_q, audio_k, audio_v, + audio_multi, audio_bias, audio_bias, audio_bias, 1, 2, 2, 2)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(audio_q, actual, 8)); + CHECK(close_values(actual, + (const float[]){1,2,5,6,3,4,7,8}, 8, 0.0f)); + + const float attended_values[] = {1,2,3,4,5,6,7,8}; + h3_gpu_tensor *attended = h3_gpu_tensor_from_f32(gpu, attended_values, 8); + h3_gpu_tensor *pooled = h3_gpu_tensor_new_f32(gpu, 2); + CHECK(attended && pooled); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_audio_attention_pool_f32(gpu, pooled, attended, + 1, 1, 2, 4, 2)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(pooled, actual, 2)); + CHECK(close_values(actual, (const float[]){3.5f,5.5f}, 2, 0.0f)); + + const float filter[] = {0.02f,0.04f,0.06f,0.08f,0.1f,0.2f, + 0.2f,0.1f,0.08f,0.06f,0.04f,0.02f}; + const float logs[] = {0}; + h3_gpu_tensor *filter_tensor = h3_gpu_tensor_from_f32(gpu, filter, 12); + h3_gpu_tensor *log_tensor = h3_gpu_tensor_from_f32(gpu, logs, 1); + h3_gpu_tensor *snake_output = h3_gpu_tensor_new_f32(gpu, 4); + CHECK(filter_tensor && log_tensor && snake_output); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_alias_free_snake_f32(gpu, long_output, long_input, + log_tensor, log_tensor, filter_tensor, filter_tensor, 1, 65536, 1)); + CHECK(h3_gpu_alias_free_snake_f32(gpu, snake_output, input1d, log_tensor, + log_tensor, filter_tensor, filter_tensor, 1, 4, 1)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(snake_output, actual, 4)); + for (unsigned time = 0; time < 4; time++) + CHECK(fabsf(actual[time] - snake_oracle(input1d_values, filter, filter, + 4, time)) < 2e-5f); + + h3_gpu_tensor_free(snake_output); h3_gpu_tensor_free(log_tensor); + h3_gpu_tensor_free(filter_tensor); h3_gpu_tensor_free(pooled); + h3_gpu_tensor_free(attended); h3_gpu_tensor_free(audio_v); + h3_gpu_tensor_free(audio_k); h3_gpu_tensor_free(audio_q); + h3_gpu_tensor_free(audio_bias); h3_gpu_tensor_free(audio_multi); + h3_gpu_tensor_free(v); h3_gpu_tensor_free(k); + h3_gpu_tensor_free(q); h3_gpu_tensor_free(v_bias); h3_gpu_tensor_free(k_bias); + h3_gpu_tensor_free(q_bias); h3_gpu_tensor_free(qkv); + h3_gpu_tensor_free(norm_output); h3_gpu_tensor_free(norm_bias); + h3_gpu_tensor_free(norm_weight); h3_gpu_tensor_free(norm_input); + h3_gpu_tensor_free(image_output); h3_gpu_tensor_free(image_input); + h3_gpu_tensor_free(volume_output); h3_gpu_tensor_free(volume_weights); + h3_gpu_tensor_free(volume_input); h3_gpu_tensor_free(transpose_weight); + h3_gpu_tensor_free(transpose_input); h3_gpu_tensor_free(long_output); + h3_gpu_tensor_free(long_input); h3_gpu_tensor_free(output1d); + h3_gpu_tensor_free(bias1d); h3_gpu_tensor_free(weight1d); + h3_gpu_tensor_free(input1d); h3_gpu_free(gpu); + puts("ok: CUDA convolution, audio and VAE operators"); + return 0; +} diff --git a/tests/test_cuda_primitives.c b/tests/test_cuda_primitives.c new file mode 100644 index 00000000..4c50f419 --- /dev/null +++ b/tests/test_cuda_primitives.c @@ -0,0 +1,201 @@ +#include "h3_gpu.h" + +#include +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); return 1; \ +} } while (0) + +static int close_array(const float *got, const float *want, size_t count, + float tolerance) { + for (size_t index = 0; index < count; index++) + if (fabsf(got[index] - want[index]) > tolerance) return 0; + return 1; +} + +int main(void) { + char error[256]; + h3_gpu *gpu = h3_gpu_create(NULL, error, sizeof(error)); + CHECK(gpu); + const float input[] = {-2.0f, -0.5f, 0.5f, 2.0f}; + const float other[] = {1.0f, 2.0f, 3.0f, 4.0f}; + h3_gpu_tensor *x = h3_gpu_tensor_from_f32(gpu, input, 4); + h3_gpu_tensor *y = h3_gpu_tensor_from_f32(gpu, other, 4); + h3_gpu_tensor *out = h3_gpu_tensor_new_f32(gpu, 4); + h3_gpu_tensor *bx = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *by = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *bout = h3_gpu_tensor_new_bf16(gpu, 4); + CHECK(x && y && out && bx && by && bout); + CHECK(h3_gpu_tensor_write_f32(bx, input, 4)); + CHECK(h3_gpu_tensor_write_f32(by, other, 4)); + + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_silu_f32(gpu, out, x, 4)); + CHECK(h3_gpu_submit(gpu)); + float got[4]; + CHECK(h3_gpu_tensor_read_f32(out, got, 4)); + float want_silu[4]; + for (size_t i = 0; i < 4; i++) + want_silu[i] = input[i] / (1.0f + expf(-input[i])); + CHECK(close_array(got, want_silu, 4, 1e-6f)); + + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_add_scaled_f32(gpu, out, x, y, 0.5f, -2.0f, 4)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(out, got, 4)); + const float want_scaled[] = {-3.0f, -4.25f, -5.75f, -7.0f}; + CHECK(close_array(got, want_scaled, 4, 1e-7f)); + + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_add_bf16(gpu, bout, bx, by, 4)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(bout, got, 4)); + const float want_add[] = {-1.0f, 1.5f, 3.5f, 6.0f}; + CHECK(close_array(got, want_add, 4, 0.01f)); + + const float fused_values[] = {-2, -0.5f, 0.5f, 2, 1, 2, 3, 4}; + const float unit_values[] = {1,1,1,1}; + const float magnitude_values[] = {2}; + h3_gpu_tensor *fused = h3_gpu_tensor_from_f32(gpu, fused_values, 8); + h3_gpu_tensor *bfused = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *units = h3_gpu_tensor_from_f32(gpu, unit_values, 4); + h3_gpu_tensor *bunits = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *magnitude = h3_gpu_tensor_from_f32(gpu, magnitude_values, 1); + CHECK(fused && bfused && units && bunits && magnitude); + CHECK(h3_gpu_tensor_write_f32(bfused, fused_values, 8)); + CHECK(h3_gpu_tensor_write_f32(bunits, unit_values, 4)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_clip_f32(gpu, out, x, 4, -1.0f, 1.0f)); + CHECK(h3_gpu_scale_add_f32(gpu, out, x, y, units, 1, 4)); + CHECK(h3_gpu_swiglu_f32(gpu, out, fused, 1, 4)); + CHECK(h3_gpu_swiglu_bf16(gpu, bout, bfused, 1, 4)); + CHECK(h3_gpu_geglu_f32(gpu, out, x, y, 4)); + CHECK(h3_gpu_snake1d_f32(gpu, out, x, units, 1, 1, 4)); + CHECK(h3_gpu_weight_norm_f32(gpu, out, x, magnitude, 1, 4)); + CHECK(h3_gpu_sub_bf16(gpu, bout, by, bx, 4)); + CHECK(h3_gpu_silu_bf16(gpu, bout, bx, 4)); + CHECK(h3_gpu_gelu_bf16(gpu, bout, bx, 4, 0)); + CHECK(h3_gpu_gelu_bf16(gpu, bout, bx, 4, 1)); + CHECK(h3_gpu_silu_mul_bf16(gpu, bout, bx, by, 4)); + CHECK(h3_gpu_cast_f32_to_bf16(gpu, bout, x, 4)); + CHECK(h3_gpu_cast_bf16_to_f32(gpu, out, bout, 4)); + CHECK(h3_gpu_euler_bf16(gpu, out, 0, bx, by, 4, 0.1f, 0.5f)); + CHECK(h3_gpu_rms_norm_bf16(gpu, bout, bx, bunits, 1, 4, 1e-5f)); + CHECK(h3_gpu_layer_norm_bf16(gpu, bout, bx, bunits, bunits, + 1, 4, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + + const float norm_input[] = {1, 2, 3, 4, -1, -2, -3, -4}; + const float weights[] = {1, 1, 1, 1}; + const float biases[] = {0, 0, 0, 0}; + h3_gpu_tensor *nx = h3_gpu_tensor_from_f32(gpu, norm_input, 8); + h3_gpu_tensor *nw = h3_gpu_tensor_from_f32(gpu, weights, 4); + h3_gpu_tensor *nb = h3_gpu_tensor_from_f32(gpu, biases, 4); + h3_gpu_tensor *no = h3_gpu_tensor_new_f32(gpu, 8); + CHECK(nx && nw && nb && no); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_rms_norm_f32(gpu, no, nx, nw, 2, 4, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + float norm_got[8]; + CHECK(h3_gpu_tensor_read_f32(no, norm_got, 8)); + float inverse = 1.0f / sqrtf(7.5f + 1e-5f); + for (size_t i = 0; i < 8; i++) CHECK(fabsf(norm_got[i] - norm_input[i] * inverse) < 2e-6f); + + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_layer_norm_f32(gpu, no, nx, nw, nb, 2, 4, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(no, norm_got, 8)); + CHECK(fabsf(norm_got[0] + 1.341635f) < 2e-5f); + CHECK(fabsf(norm_got[3] - 1.341635f) < 2e-5f); + + const uint32_t row_map_values[] = {1, 0}; + const float modulation_values[] = { + 0,0,0,0, 0.1f,0.2f,0.3f,0.4f, 0.5f,0.5f,0.5f,0.5f, + 0,0,0,0, -0.1f,-0.2f,-0.3f,-0.4f, 2,2,2,2 + }; + h3_gpu_tensor *row_map = h3_gpu_tensor_from_u32(gpu, row_map_values, 2); + h3_gpu_tensor *mod = h3_gpu_tensor_from_f32(gpu, modulation_values, 24); + h3_gpu_tensor *bnx = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *bnw = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *bmod = h3_gpu_tensor_new_bf16(gpu, 24); + h3_gpu_tensor *bgated = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *bnout = h3_gpu_tensor_new_bf16(gpu, 8); + CHECK(row_map && mod && bnx && bnw && bmod && bgated && bnout); + CHECK(h3_gpu_tensor_write_f32(bnx, norm_input, 8)); + CHECK(h3_gpu_tensor_write_f32(bnw, weights, 4)); + CHECK(h3_gpu_tensor_write_f32(bmod, modulation_values, 24)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_adaln_f32(gpu, no, nx, nw, mod, row_map, + 2, 4, 3, 1, 2, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(no, norm_got, 8)); + CHECK(fabsf(norm_got[0] - (norm_input[0] * inverse * 3.0f - 0.1f)) < 2e-5f); + CHECK(fabsf(norm_got[4] - (norm_input[4] * inverse * 1.5f + 0.1f)) < 2e-5f); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_gate_f32(gpu, no, nx, nx, mod, row_map, 2, 4, 3, 2)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(no, norm_got, 8)); + CHECK(norm_got[0] == 3.0f && norm_got[4] == -1.5f); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_adaln_bf16_offset(gpu, bnout, bnx, 0, bnw, bmod, row_map, + 2, 4, 3, 1, 2, 1e-5f)); + CHECK(h3_gpu_gate_adaln_bf16(gpu, bgated, bnout, bnx, bnx, bnw, + bmod, bmod, row_map, 2, 4, 3, 2, 1, 2, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + const float identity4[] = { + 1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1 + }; + h3_gpu_tensor *bidentity = h3_gpu_tensor_new_bf16(gpu, 16); + h3_gpu_tensor *binverse = h3_gpu_tensor_new_f32(gpu, 2); + h3_gpu_tensor *blinear = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *bquantized = h3_gpu_tensor_new_i8(gpu, 12); + h3_gpu_tensor *bscales = h3_gpu_tensor_new_f32(gpu, 3); + CHECK(bidentity && binverse && blinear && bquantized && bscales); + CHECK(h3_gpu_tensor_write_f32(bidentity, identity4, 16)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_adaln_linear_bf16(gpu, blinear, binverse, bnx, 0, bnw, + bmod, row_map, bidentity, NULL, 2, 4, 4, 3, 1, 2, 1e-5f)); + CHECK(h3_gpu_gate_adaln_quantize_int8(gpu, bgated, bquantized, bscales, + bnx, bnx, bnw, bmod, bmod, row_map, 2, 3, 4, 3, 2, 1, 2, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + float inverse_values[2]; + CHECK(h3_gpu_tensor_read_f32(binverse, inverse_values, 2)); + CHECK(fabsf(inverse_values[0] - inverse) < 2e-5f); + + const float embedding_values[] = {1,2, 3,4, 5,6}; + const uint32_t ids[] = {2, 0, 9}; + h3_gpu_tensor *embedding = h3_gpu_tensor_new_bf16(gpu, 6); + h3_gpu_tensor *embedding_out = h3_gpu_tensor_new_bf16(gpu, 6); + h3_gpu_tensor *token_ids = h3_gpu_tensor_from_u32(gpu, ids, 3); + CHECK(embedding && embedding_out && token_ids); + CHECK(h3_gpu_tensor_write_f32(embedding, embedding_values, 6)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_embedding_bf16(gpu, embedding_out, embedding, token_ids, + 3, 3, 2)); + CHECK(h3_gpu_submit(gpu)); + float embedding_got[6]; + CHECK(h3_gpu_tensor_read_f32(embedding_out, embedding_got, 6)); + const float embedding_want[] = {5,6, 1,2, 0,0}; + CHECK(close_array(embedding_got, embedding_want, 6, 0.0f)); + + h3_gpu_tensor_free(bscales); h3_gpu_tensor_free(bquantized); + h3_gpu_tensor_free(blinear); h3_gpu_tensor_free(binverse); + h3_gpu_tensor_free(bidentity); + h3_gpu_tensor_free(token_ids); h3_gpu_tensor_free(embedding_out); + h3_gpu_tensor_free(embedding); h3_gpu_tensor_free(bnout); + h3_gpu_tensor_free(bgated); h3_gpu_tensor_free(bmod); + h3_gpu_tensor_free(bnw); h3_gpu_tensor_free(bnx); h3_gpu_tensor_free(mod); + h3_gpu_tensor_free(row_map); h3_gpu_tensor_free(magnitude); + h3_gpu_tensor_free(bunits); h3_gpu_tensor_free(units); + h3_gpu_tensor_free(bfused); h3_gpu_tensor_free(fused); + h3_gpu_tensor_free(no); + h3_gpu_tensor_free(nb); h3_gpu_tensor_free(nw); + h3_gpu_tensor_free(nx); h3_gpu_tensor_free(bout); h3_gpu_tensor_free(by); + h3_gpu_tensor_free(bx); h3_gpu_tensor_free(out); h3_gpu_tensor_free(y); + h3_gpu_tensor_free(x); h3_gpu_free(gpu); + puts("ok: CUDA elementwise and normalization primitives"); + return 0; +} diff --git a/tests/test_cuda_rope_tokens.c b/tests/test_cuda_rope_tokens.c new file mode 100644 index 00000000..b9eab73b --- /dev/null +++ b/tests/test_cuda_rope_tokens.c @@ -0,0 +1,234 @@ +#include "h3_gpu.h" + +#include +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); return 1; \ +} } while (0) + +static int close_values(const float *got, const float *want, size_t count, + float tolerance) { + for (size_t i = 0; i < count; i++) + if (fabsf(got[i] - want[i]) > tolerance) return 0; + return 1; +} + +int main(void) { + char error[256]; + h3_gpu *gpu = h3_gpu_create(NULL, error, sizeof(error)); + CHECK(gpu); + const float qkv_values[] = {1,2,3,4, 5,6,7,8, 9,10,11,12}; + const float ones[] = {1,1,1,1}; + const float cos_values[] = {1,1}; + const float sin_values[] = {0,0}; + h3_gpu_tensor *qkv = h3_gpu_tensor_from_f32(gpu, qkv_values, 12); + h3_gpu_tensor *weight = h3_gpu_tensor_from_f32(gpu, ones, 4); + h3_gpu_tensor *cosine = h3_gpu_tensor_from_f32(gpu, cos_values, 2); + h3_gpu_tensor *sine = h3_gpu_tensor_from_f32(gpu, sin_values, 2); + h3_gpu_tensor *q = h3_gpu_tensor_new_f32(gpu, 4); + h3_gpu_tensor *k = h3_gpu_tensor_new_f32(gpu, 4); + h3_gpu_tensor *v = h3_gpu_tensor_new_f32(gpu, 4); + CHECK(qkv && weight && cosine && sine && q && k && v); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_qkv_rope_f32(gpu, q, k, v, qkv, weight, weight, + cosine, sine, 1, 1, 4, 2, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + float got[12]; + CHECK(h3_gpu_tensor_read_f32(q, got, 4)); + float q_inverse = 1.0f / sqrtf(7.5f + 1e-5f); + const float q_want[] = {1*q_inverse,2*q_inverse,3*q_inverse,4*q_inverse}; + CHECK(close_values(got, q_want, 4, 2e-6f)); + CHECK(h3_gpu_tensor_read_f32(v, got, 4)); + CHECK(close_values(got, qkv_values + 8, 4, 0.0f)); + + const float multi_qkv_values[] = { + 1,2,3,4, 5,6,7,8, 9,10,11,12, + 13,14,15,16, 17,18,19,20, 21,22,23,24}; + h3_gpu_tensor *multi_qkv = h3_gpu_tensor_from_f32( + gpu, multi_qkv_values, 24); + h3_gpu_tensor *multi_q = h3_gpu_tensor_new_f32(gpu, 8); + h3_gpu_tensor *multi_k = h3_gpu_tensor_new_f32(gpu, 8); + h3_gpu_tensor *multi_v = h3_gpu_tensor_new_f32(gpu, 8); + CHECK(multi_qkv && multi_q && multi_k && multi_v); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_qkv_rope_f32(gpu, multi_q, multi_k, multi_v, multi_qkv, + weight, weight, weight, weight, 2, 2, 2, 0, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(multi_v, got, 8)); + CHECK(close_values(got, + (const float[]){9,10,21,22,11,12,23,24}, 8, 0.0f)); + + h3_gpu_tensor *bqkv = h3_gpu_tensor_new_bf16(gpu, 12); + h3_gpu_tensor *bweight = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *bcos = h3_gpu_tensor_new_bf16(gpu, 2); + h3_gpu_tensor *bsin = h3_gpu_tensor_new_bf16(gpu, 2); + h3_gpu_tensor *bq = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *bk = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *bv = h3_gpu_tensor_new_bf16(gpu, 4); + CHECK(bqkv && bweight && bcos && bsin && bq && bk && bv); + CHECK(h3_gpu_tensor_write_f32(bqkv, qkv_values, 12)); + CHECK(h3_gpu_tensor_write_f32(bweight, ones, 4)); + CHECK(h3_gpu_tensor_write_f32(bcos, cos_values, 2)); + CHECK(h3_gpu_tensor_write_f32(bsin, sin_values, 2)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_qkv_rope_bf16(gpu, bq, bk, bv, bqkv, bweight, bweight, + bcos, bsin, 1, 1, 4, 2, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(bq, got, 4)); + CHECK(close_values(got, q_want, 4, 0.01f)); + const float projection_input_values[] = {1,1,1,1}; + float projection_weight_values[48] = {0}; + for (size_t row = 0; row < 12; row++) + projection_weight_values[row * 4] = qkv_values[row]; + h3_gpu_tensor *projection_input = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *projection_weight = h3_gpu_tensor_new_bf16(gpu, 48); + CHECK(projection_input && projection_weight); + CHECK(h3_gpu_tensor_write_f32(projection_input, projection_input_values, 4)); + CHECK(h3_gpu_tensor_write_f32(projection_weight, projection_weight_values, 48)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_grouped_qkv_linear_rope_bf16( + gpu, bq, bk, bv, bqkv, projection_input, projection_weight, bweight, + bweight, bcos, bsin, 1, 4, 1, 4, 2, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(bq, got, 4)); + CHECK(close_values(got, q_want, 4, 0.01f)); + h3_gpu_tensor *projection_weight_i8 = h3_gpu_tensor_new_i8(gpu, 48); + h3_gpu_tensor *projection_weight_scales = h3_gpu_tensor_new_f32(gpu, 12); + h3_gpu_tensor *projection_quantized = h3_gpu_tensor_new_i8(gpu, 4); + h3_gpu_tensor *projection_scales = h3_gpu_tensor_new_f32(gpu, 1); + CHECK(projection_weight_i8 && projection_weight_scales && + projection_quantized && projection_scales); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_quantize_weight_int8(gpu, projection_weight_i8, + projection_weight_scales, projection_weight, 12, 4)); + CHECK(h3_gpu_grouped_qkv_linear_rope_int8( + gpu, bq, bk, bv, projection_quantized, projection_scales, + projection_input, projection_weight_i8, projection_weight_scales, + bweight, bweight, bcos, bsin, 1, 4, 1, 4, 2, 1e-5f, + 0, 0, 0, 0)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(bq, got, 4)); + CHECK(close_values(got, q_want, 4, 0.02f)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_grouped_qkv_rope_bf16(gpu, bq, bk, bv, bqkv, bweight, + bweight, bcos, bsin, 1, 1, 4, 2, 1e-5f)); + CHECK(h3_gpu_grouped_qkv_rope_bf16(gpu, bq, bk, bv, bqkv, bweight, + bweight, bweight, bweight, 1, 1, 4, 0, 1e-5f)); + CHECK(h3_gpu_vision_qkv_rope_bf16(gpu, bq, bk, bv, bqkv, bcos, bsin, + 1, 1, 4, 2)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(bq, got, 4)); + CHECK(close_values(got, qkv_values, 4, 0.0f)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_video_qkv_rope_f32(gpu, q, k, v, qkv, cosine, sine, + 1, 1, 4, 2, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(q, got, 4)); + CHECK(close_values(got, q_want, 4, 2e-6f)); + + const float text_values[] = {1,2,3,4}; + const float zero_cos[] = {0,0}; + const float one_sin[] = {1,1}; + h3_gpu_tensor *text_q = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *text_k = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *text_cos = h3_gpu_tensor_from_f32(gpu, zero_cos, 2); + h3_gpu_tensor *text_sin = h3_gpu_tensor_from_f32(gpu, one_sin, 2); + CHECK(text_q && text_k && text_cos && text_sin); + CHECK(h3_gpu_tensor_write_f32(text_q, text_values, 4)); + CHECK(h3_gpu_tensor_write_f32(text_k, text_values, 4)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_rope_text_bf16(gpu, text_q, text_k, text_cos, text_sin, + 1, 1, 1, 4)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(text_q, got, 4)); + const float text_want[] = {-3,-4,1,2}; + CHECK(close_values(got, text_want, 4, 0.0f)); + CHECK(h3_gpu_tensor_write_f32(text_q, text_values, 4)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_head_rms_norm_bf16(gpu, text_q, bweight, 1, 1, 4, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(text_q, got, 4)); + CHECK(close_values(got, q_want, 4, 0.01f)); + h3_gpu_tensor *text_q_out = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *text_k_out = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *bzero_cos = h3_gpu_tensor_new_bf16(gpu, 2); + h3_gpu_tensor *bone_sin = h3_gpu_tensor_new_bf16(gpu, 2); + CHECK(text_q_out && text_k_out && bzero_cos && bone_sin); + CHECK(h3_gpu_tensor_write_f32(text_q, text_values, 4)); + CHECK(h3_gpu_tensor_write_f32(text_k, text_values, 4)); + CHECK(h3_gpu_tensor_write_f32(bzero_cos, zero_cos, 2)); + CHECK(h3_gpu_tensor_write_f32(bone_sin, one_sin, 2)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_text_qk_rope_bf16(gpu, text_q_out, text_k_out, text_q, + text_k, bweight, bweight, bzero_cos, bone_sin, 1, 1, 1, 4, 1e-5f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(text_q_out, got, 4)); + const float text_norm_want[] = {-3*q_inverse,-4*q_inverse, + 1*q_inverse, 2*q_inverse}; + CHECK(close_values(got, text_norm_want, 4, 0.01f)); + + const float pool_input[] = { + 1,2, 3,4, 5,6, 7,8 + }; + const uint32_t pairs_values[] = {0,0, 1,2, 3,3}; + const uint32_t baseline_index_values[] = {UINT32_MAX, 0, UINT32_MAX}; + h3_gpu_tensor *pool_source = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *pool_output = h3_gpu_tensor_new_bf16(gpu, 6); + h3_gpu_tensor *original = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *baseline = h3_gpu_tensor_new_bf16(gpu, 2); + h3_gpu_tensor *pairs = h3_gpu_tensor_from_u32(gpu, pairs_values, 6); + h3_gpu_tensor *baseline_indices = h3_gpu_tensor_from_u32( + gpu, baseline_index_values, 3); + CHECK(pool_source && pool_output && original && baseline && pairs && baseline_indices); + CHECK(h3_gpu_tensor_write_f32(pool_source, pool_input, 8)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_token_pool_bf16(gpu, pool_output, pool_source, 0, original, + 0, baseline, 0, baseline_indices, pairs, 4, 3, 1, 2)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(pool_output, got, 6)); + const float pooled_want[] = {1,2, 4,5, 7,8}; + CHECK(close_values(got, pooled_want, 6, 0.0f)); + + const float reduced_values[] = {10,20, 6,8, 70,80}; + const uint32_t parents_values[] = {0,1,1,2}; + h3_gpu_tensor *reduced = h3_gpu_tensor_new_bf16(gpu, 6); + h3_gpu_tensor *expanded = h3_gpu_tensor_new_bf16(gpu, 8); + h3_gpu_tensor *parents = h3_gpu_tensor_from_u32(gpu, parents_values, 4); + CHECK(reduced && expanded && parents); + CHECK(h3_gpu_tensor_write_f32(reduced, reduced_values, 6)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_token_expand_delta_bf16(gpu, expanded, original, 0, reduced, + baseline, 0, baseline_indices, parents, 4, 3, 1, 2, 1, 1.0f)); + CHECK(h3_gpu_submit(gpu)); + CHECK(h3_gpu_tensor_read_f32(expanded, got, 8)); + const float expanded_want[] = {10,20, 5,7, 7,9, 70,80}; + CHECK(close_values(got, expanded_want, 8, 0.0f)); + + h3_gpu_tensor_free(parents); h3_gpu_tensor_free(expanded); + h3_gpu_tensor_free(reduced); h3_gpu_tensor_free(baseline_indices); + h3_gpu_tensor_free(pairs); h3_gpu_tensor_free(baseline); + h3_gpu_tensor_free(original); h3_gpu_tensor_free(pool_output); + h3_gpu_tensor_free(pool_source); h3_gpu_tensor_free(bone_sin); + h3_gpu_tensor_free(bzero_cos); h3_gpu_tensor_free(text_k_out); + h3_gpu_tensor_free(text_q_out); h3_gpu_tensor_free(text_sin); + h3_gpu_tensor_free(text_cos); h3_gpu_tensor_free(text_k); + h3_gpu_tensor_free(projection_scales); + h3_gpu_tensor_free(projection_quantized); + h3_gpu_tensor_free(projection_weight_scales); + h3_gpu_tensor_free(projection_weight_i8); + h3_gpu_tensor_free(projection_weight); + h3_gpu_tensor_free(projection_input); + h3_gpu_tensor_free(text_q); h3_gpu_tensor_free(multi_v); + h3_gpu_tensor_free(multi_k); h3_gpu_tensor_free(multi_q); + h3_gpu_tensor_free(multi_qkv); h3_gpu_tensor_free(v); h3_gpu_tensor_free(k); + h3_gpu_tensor_free(q); h3_gpu_tensor_free(bv); h3_gpu_tensor_free(bk); + h3_gpu_tensor_free(bq); h3_gpu_tensor_free(bsin); h3_gpu_tensor_free(bcos); + h3_gpu_tensor_free(bweight); h3_gpu_tensor_free(bqkv); + h3_gpu_tensor_free(sine); h3_gpu_tensor_free(cosine); + h3_gpu_tensor_free(weight); h3_gpu_tensor_free(qkv); h3_gpu_free(gpu); + puts("ok: CUDA RoPE and token transforms"); + return 0; +} diff --git a/tests/test_cuda_runtime.c b/tests/test_cuda_runtime.c new file mode 100644 index 00000000..1b30a948 --- /dev/null +++ b/tests/test_cuda_runtime.c @@ -0,0 +1,107 @@ +#include "h3_gpu.h" + +#include +#include +#include +#include +#include +#include + +#define CHECK(expression) do { \ + if (!(expression)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #expression); \ + return 1; \ + } \ +} while (0) + +int main(void) { + char error[256]; + h3_gpu *gpu = h3_gpu_create(NULL, error, sizeof(error)); + CHECK(gpu != NULL); + CHECK(!h3_gpu_is_m5(gpu)); + CHECK(!h3_gpu_has_nax_mlp(gpu)); + + const float input[] = {1.0f, -2.5f, 0.125f, 65504.0f}; + h3_gpu_tensor *f32 = h3_gpu_tensor_from_f32(gpu, input, 4); + h3_gpu_tensor *copy = h3_gpu_tensor_new_f32(gpu, 6); + h3_gpu_tensor *bf16 = h3_gpu_tensor_new_bf16(gpu, 4); + h3_gpu_tensor *bf16_copy = h3_gpu_tensor_new_bf16(gpu, 6); + CHECK(f32 && copy && bf16 && bf16_copy); + CHECK(h3_gpu_tensor_elements(f32) == 4); + CHECK(h3_gpu_tensor_dtype(f32) == H3_GPU_F32); + CHECK(h3_gpu_tensor_write_f32(bf16, input, 4)); + + float roundtrip[4] = {0}; + CHECK(h3_gpu_tensor_read_f32(bf16, roundtrip, 4)); + for (size_t index = 0; index < 4; index++) + CHECK(fabsf(roundtrip[index] - input[index]) <= + fmaxf(0.01f, fabsf(input[index]) * 0.008f)); + + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_copy_f32(gpu, copy, 1, f32, 0, 4)); + CHECK(h3_gpu_continue(gpu)); + CHECK(h3_gpu_submit(gpu)); + float copied[4] = {0}; + CHECK(h3_gpu_tensor_read_f32_range(copy, 1, copied, 4)); + CHECK(memcmp(input, copied, sizeof(input)) == 0); + CHECK(!h3_gpu_tensor_read_f32_range(copy, 5, copied, 4)); + CHECK(strstr(h3_gpu_error(gpu), "range") != NULL); + + const uint16_t range_values[] = {0x3f80, 0xc020, 0x3e00, 0x4780}; + CHECK(h3_gpu_tensor_write_bf16_range(bf16_copy, 1, range_values, 4)); + CHECK(h3_gpu_begin(gpu)); + CHECK(h3_gpu_copy_bf16(gpu, bf16, 0, bf16_copy, 1, 4)); + CHECK(h3_gpu_submit(gpu)); + uint16_t range_roundtrip[4] = {0}; + CHECK(h3_gpu_tensor_read_bf16(bf16, range_roundtrip, 4)); + CHECK(memcmp(range_values, range_roundtrip, sizeof(range_values)) == 0); + + char path[] = "/tmp/h3-cuda-runtime-XXXXXX"; + int fd = mkstemp(path); + CHECK(fd >= 0); + const uint16_t file_values[] = {0x3f80, 0xc020, 0x3e00, 0x4780}; + CHECK(write(fd, file_values, sizeof(file_values)) == + (ssize_t)sizeof(file_values)); + const float file_f32[] = {3.5f, -7.0f}; + CHECK(write(fd, file_f32, sizeof(file_f32)) == (ssize_t)sizeof(file_f32)); + CHECK(close(fd) == 0); + h3_gpu_tensor *loaded = h3_gpu_tensor_load_bf16(gpu, path, 0, 4); + CHECK(loaded != NULL); + uint16_t loaded_values[4] = {0}; + CHECK(h3_gpu_tensor_read_bf16(loaded, loaded_values, 4)); + CHECK(memcmp(file_values, loaded_values, sizeof(file_values)) == 0); + h3_gpu_tensor *loaded_f32 = h3_gpu_tensor_load_f32( + gpu, path, sizeof(file_values), 2); + CHECK(loaded_f32 != NULL); + float loaded_f32_values[2] = {0}; + CHECK(h3_gpu_tensor_read_f32(loaded_f32, loaded_f32_values, 2)); + CHECK(memcmp(file_f32, loaded_f32_values, sizeof(file_f32)) == 0); + CHECK(h3_gpu_tensor_stream_file_bf16(bf16, path, 0, 4, + error, sizeof(error))); + CHECK(!h3_gpu_tensor_stream_file_bf16( + bf16, path, (uint64_t)INT64_MAX, 4, error, sizeof(error))); + CHECK(strstr(error, "overflows") != NULL); + CHECK(unlink(path) == 0); + + h3_gpu_stats stats; + CHECK(h3_gpu_get_stats(gpu, &stats)); + CHECK(stats.tensor_allocations == 6); + CHECK(stats.live_bytes > 0 && stats.peak_live_bytes >= stats.live_bytes); + CHECK(stats.blit_copies == 2 && stats.submissions == 3); + CHECK(stats.gpu_seconds >= 0.0 && stats.command_encode_seconds >= 0.0); + + h3_gpu_tensor_free(loaded_f32); + h3_gpu_tensor_free(loaded); + h3_gpu_tensor_free(bf16_copy); + h3_gpu_tensor_free(bf16); + h3_gpu_tensor_free(copy); + h3_gpu_tensor_free(f32); + CHECK(h3_gpu_get_stats(gpu, &stats)); + CHECK(stats.live_bytes == 0); + CHECK(setenv("H3_PROFILE", "1", 1) == 0); + h3_gpu_profile_set_label(gpu, "runtime test"); + h3_gpu_profile_mark(gpu, "complete"); + h3_gpu_free(gpu); + puts("ok: CUDA runtime allocation, conversion, copy and file I/O"); + return 0; +} diff --git a/tests/test_device.c b/tests/test_device.c new file mode 100644 index 00000000..f80dcc73 --- /dev/null +++ b/tests/test_device.c @@ -0,0 +1,23 @@ +#include "h3_device.h" + +#include +#include + +int main(void) { + h3_device_info info; + char error[256] = {0}; + if (!h3_device_probe(&info, error, sizeof(error))) { + fprintf(stderr, "FAIL device probe: %s\n", error); + return 1; + } + if (!info.name[0] || !info.architecture[0] || + !info.recommended_working_set || !info.max_buffer_length) { + fprintf(stderr, "FAIL device probe: incomplete device information\n"); + return 1; + } + printf("ok: %s (%s), %.1f GiB GPU memory, unified=%s\n", info.name, + info.architecture, + (double)info.recommended_working_set / (1024.0 * 1024.0 * 1024.0), + info.unified_memory ? "yes" : "no"); + return 0; +} From 0c95fb49744b3b32cfedb6aac817acceb3a65348 Mon Sep 17 00:00:00 2001 From: matrixfede Date: Thu, 20 Aug 2026 09:16:23 +0200 Subject: [PATCH 03/20] Integrate CUDA backend into CLI and modules, add attention optimizations and benchmarks --- README.md | 70 ++++++++++-- docs/CUDA_GPU_API_INVENTORY.md | 189 +++++++++++++++++++++++++++++++++ docs/GB10_PROFILE.md | 67 ++++++++++++ h3.c | 18 ++-- h3.h | 2 +- h3_cli.c | 11 +- h3_metal.h | 8 -- h3_metal.m | 4 +- main.c | 8 +- tests/bench_dit.c | 14 ++- tests/bench_video_vae.c | 95 +++++++++++++++++ tests/test_av_mux.c | 6 +- tests/test_h3.c | 10 +- 13 files changed, 461 insertions(+), 41 deletions(-) create mode 100644 docs/CUDA_GPU_API_INVENTORY.md create mode 100644 docs/GB10_PROFILE.md delete mode 100644 h3_metal.h create mode 100644 tests/bench_video_vae.c diff --git a/README.md b/README.md index 4750ac49..c8ca60ab 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,65 @@ -# h3-metal +# h3.c -Native MiniMax-H3 inference for Apple Silicon. The project is being built as a -sequence of working vertical slices: deterministic host/model metadata first, -then portable Metal block parity, prompt encoding, prompt-to-video/audio, and -first/last-frame conditioning and then ordered references. +Native MiniMax-H3 inference in C for NVIDIA CUDA on Linux and Metal on Apple +Silicon. Prompt-to-video/audio, first/last-frame conditioning, and ordered +Ref2VA image/video/audio references work end to end on both backends. -Prompt-to-video/audio, first/last-frame conditioning, and ordered Ref2VA -image/video/audio references work end to end. The current work is incremental -H3-specific Metal performance and memory optimization on M3 Max and M5 Max. +## Platforms and prerequisites + +The validated Linux configuration is Ubuntu ARM64, NVIDIA GB10, driver 595.84, +and CUDA Toolkit 13.0. The supported Linux baseline is a C11 compiler, GNU +Make, CUDA Toolkit 13.0 or newer, ICU 72 or newer, and FFmpeg/FFprobe 6.1 or +newer. The build links `cudart`, `cublasLt`, ICU, pthreads, and libm. NVIDIA +drivers must support the installed toolkit and GPU. macOS continues to use the +Metal backend and the Objective-C/Foundation tokenizer; its existing Apple +Silicon optimization switches remain available there. + +Install the ordinary Ubuntu build dependencies with your package manager and +install CUDA from NVIDIA's repository for the target architecture. Verify the +toolchain before building: + +```sh +cc --version +nvcc --version +ffmpeg -version +pkg-config --modversion icu-uc +``` + +The released checkpoint is about 465 GB. If `./MiniMax-H3` already contains +the complete snapshot, reuse it: no second download is needed. Otherwise the +current Hugging Face CLI resumes and reuses cached blobs: + +```sh +hf auth login +hf download MiniMaxAI/MiniMax-H3 --local-dir ./MiniMax-H3 +hf cache verify MiniMaxAI/MiniMax-H3 --local-dir ./MiniMax-H3 \ + --fail-on-missing-files +``` + +Build and run the complete local gate: + +```sh +make -j"$(nproc)" +./scripts/verify.sh all +./h3 --info -d ./MiniMax-H3 +``` + +The GB10 end-to-end command validated with the official checkpoint is: + +```sh +mkdir -p outputs +./h3 --profile -d ./MiniMax-H3 \ + -p "A bright red cube rotates smoothly on a white background." \ + --width 512 --height 512 --frames 22 --steps 20 --layers 50 \ + --token-reduction -o outputs/gb10-cube.mp4 +``` + +Linux/CUDA limitations: `--use-int8-row-fc2` is currently a Metal/M5 +specialization and a measured CUDA no-op; there is no verified FP8 execution +path. `--show` depends on terminal graphics support. `--ssd-streaming` is exact +and cuts the measured GB10 DiT peak from 27.06 GB to 1.63 GB, but increased +load-plus-denoise time by 37.6% in the three-run benchmark. See +[`docs/GB10_PROFILE.md`](docs/GB10_PROFILE.md) for the reproducible measurements. ## Tutorial @@ -22,7 +74,7 @@ mkdir -p outputs ./h3 --info -d ./MiniMax-H3 ``` -`--info` checks the model layout and prints the selected Metal device without +`--info` checks the model layout and prints the selected GPU device without mapping all weights or generating media. Run `./h3 --help` for the complete CLI reference. diff --git a/docs/CUDA_GPU_API_INVENTORY.md b/docs/CUDA_GPU_API_INVENTORY.md new file mode 100644 index 00000000..988f16ed --- /dev/null +++ b/docs/CUDA_GPU_API_INVENTORY.md @@ -0,0 +1,189 @@ +# Inventario API GPU per il port CUDA + +Fonte di verità: `h3_gpu.h` al commit `8974cc0`. Le shape sono espresse in +ordine di memoria row-major; `N` indica il numero di elementi. `custom+BLAS` +indica un wrapper che usa cuBLASLt per GEMM e kernel CUDA per layout/fusione. + +## Runtime, tensori e profiling + +| API | Backend CUDA | Dtype e shape | Verifica CUDA prevista | +|---|---|---|---| +| `h3_gpu_create` | host + CUDA runtime/NVRTC | contesto; sorgente shader ignorata su CUDA | create su device 0, errore leggibile senza device | +| `h3_gpu_free` | host + CUDA runtime | contesto | teardown senza leak (Compute Sanitizer) | +| `h3_gpu_is_m5` | host | N/A; sempre 0 su CUDA | capability test | +| `h3_gpu_has_nax_mlp` | host | N/A; sempre 0 su CUDA | capability test | +| `h3_gpu_has_int8_mlp` | host | N/A; 1 se cuBLASLt INT8 disponibile | capability test | +| `h3_gpu_tensor_new_f32` | CUDA runtime | F32 `[N]` | allocazione, dtype, contatori | +| `h3_gpu_tensor_new_bf16` | CUDA runtime | BF16 `[N]` | allocazione, dtype, contatori | +| `h3_gpu_tensor_new_i8` | CUDA runtime | I8 `[N]` | allocazione, dtype, contatori | +| `h3_gpu_tensor_from_f32` | host + CUDA runtime | F32 host `[N]` -> device `[N]` | round-trip esatto | +| `h3_gpu_tensor_from_bf16` | host + CUDA runtime | BF16 host `[N]` -> device `[N]` | round-trip bit-esatto | +| `h3_gpu_tensor_from_u32` | host + CUDA runtime | U32 host `[N]` -> device `[N]` | round-trip bit-esatto | +| `h3_gpu_tensor_load_bf16` | host + CUDA runtime | file BF16 `[N]` -> device `[N]` | fixture safetensors, offset e short read | +| `h3_gpu_tensor_load_f32` | host + CUDA runtime | file F32 `[N]` -> device `[N]` | fixture safetensors, offset e short read | +| `h3_gpu_tensor_read_file_bf16` | host + CUDA runtime | file BF16 `[N]` -> tensor BF16 `[N]` | reload in-place e bounds | +| `h3_gpu_tensor_stream_file_bf16` | host + CUDA runtime | file BF16 `[N]` -> tensor BF16 `[N]` | stesso golden del read; hint cache best-effort | +| `h3_gpu_tensor_free` | CUDA runtime | tensor | double ownership escluso; contatori e leak | +| `h3_gpu_tensor_elements` | host | metadata `N` | tutti i costruttori | +| `h3_gpu_tensor_dtype` | host | metadata F32/BF16/I8/U32 | tutti i costruttori | +| `h3_gpu_tensor_read_f32` | CUDA runtime + custom | tensor F32/BF16 `[N]` -> host F32 `[N]` | F32 esatto; BF16 conversione esatta | +| `h3_gpu_tensor_read_f32_range` | CUDA runtime + custom | tensor F32/BF16 `[N]`, slice `[offset, n]` -> host F32 | offset/bounds e conversione | +| `h3_gpu_tensor_read_bf16` | CUDA runtime | BF16 `[N]` -> host BF16 `[N]` | bit-esatto | +| `h3_gpu_tensor_write_f32` | CUDA runtime + custom | host F32 `[N]` -> tensor F32/BF16 `[N]` | F32 esatto; BF16 round-to-nearest-even | +| `h3_gpu_tensor_write_f32_range` | CUDA runtime + custom | host F32 `[n]` -> slice tensor `[offset,n]` | offset/bounds, sentinelle adiacenti | +| `h3_gpu_tensor_write_bf16` | CUDA runtime | host BF16 `[N]` -> tensor BF16 `[N]` | bit-esatto | +| `h3_gpu_tensor_write_bf16_range` | CUDA runtime | host BF16 `[n]` -> slice `[offset,n]` | bit-esatto e bounds | +| `h3_gpu_begin` | CUDA runtime | stream/graph corrente | state-machine test | +| `h3_gpu_continue` | CUDA runtime | event + stream ordinato | due tranche, risultato e ordine | +| `h3_gpu_submit` | CUDA runtime | sincronizzazione stream | error propagation e contatori | +| `h3_gpu_error` | host | stringa | errore sintetico non vuoto | +| `h3_gpu_get_stats` | host + CUDA events | `h3_gpu_stats` | allocazioni, dispatch, submission, tempi non negativi | +| `h3_gpu_profile_set_label` | host | stringa | smoke con `H3_PROFILE` | +| `h3_gpu_profile_mark` | host + CUDA events | fase | smoke con `H3_PROFILE`, stream ordinato | + +## Algebra lineare, quantizzazione e fusioni + +| API | Backend CUDA | Dtype e shape | Verifica CUDA prevista | +|---|---|---|---| +| `h3_gpu_linear_f32` | cuBLASLt | X F32 `[R,K]`, W F32 `[O,K]`, bias `[O]?` -> Y `[R,O]` | fixture `test_metal`; CPU piccoli | +| `h3_gpu_patch_linear_bf16` | cuBLASLt | X/W/B/Y BF16, `[R,K]·[O,K] -> [R,O]` | BF16 fixture + CPU piccoli | +| `h3_gpu_patch_linear_bf16_offset` | cuBLASLt | come sopra con offset elemento input/output | sentinelle e parità patch-linear | +| `h3_gpu_patch_linear_bf16_map` | custom+BLAS | X `[R,K]`, map U32 `[R]`, Y `[output_rows,O]` | map sparsa, duplicati, bounds | +| `h3_gpu_linear_bf16` | cuBLASLt | X/W/B/Y BF16, `[R,K]·[O,K] -> [R,O]`, accumulo F32 | `test_bf16`, `test_text_metal`, fixture reali | +| `h3_gpu_mlp_bf16` | custom+BLAS | X `[R,K]`, W1 `[2H,K]`, W2 `[O,H]` -> Y `[R,O]` | `test_bf16`, confronto pipeline non fusa | +| `h3_gpu_mlp_nax_bf16` | host fallback | stesso contratto MLP; alias della pipeline portabile su CUDA | risultato identico a `mlp_bf16`, capability false | +| `h3_gpu_quantize_weight_int8` | CUDA custom | BF16/F32 `[R,C]` -> I8 `[R,C]` + scale F32 `[R]` | dequant CPU, errore per-canale | +| `h3_gpu_linear_int8_bf16` | custom+cuBLASLt | X BF16 `[R,K]`, W I8 `[O,K]`, scale X `[R]`, W `[O]` -> BF16 `[R,O]` | CPU INT8 e confronto BF16 | +| `h3_gpu_linear_int8_head_major_bf16` | custom+cuBLASLt | X BF16 `[heads,R,D]`, W I8 `[O,heads*D]` -> BF16 `[R,O]` | confronto transpose+linear | +| `h3_gpu_mlp_int8_bf16` | custom+cuBLASLt | X `[R,K]`, W1 I8 `[2H,K]`, W2 I8 `[O,H]` -> BF16 `[R,O]` | confronto MLP BF16 e flag-paths | +| `h3_gpu_adaln_linear_bf16` | custom+cuBLASLt | AdaLN `[R,W]` + Wgt `[O,W]` -> BF16 `[R,O]`, inverse `[R]` | `test_bf16`, confronto due chiamate | +| `h3_gpu_grouped_qkv_linear_rope_bf16` | custom+cuBLASLt | X `[R,K]`, W `[3*H*D,K]` -> Q/K/V `[H,R,D]` | `test_bf16`, confronto linear+grouped QKV | +| `h3_gpu_grouped_qkv_linear_rope_int8` | custom+cuBLASLt | X BF16/I8 `[R,K]`, W I8 `[3HD,K]` -> Q/K/V BF16 `[H,R,D]` | confronto variante BF16 e flag-paths | + +## Elementwise, norm, embedding, RoPE e token transforms + +| API | Backend CUDA | Dtype e shape | Verifica CUDA prevista | +|---|---|---|---| +| `h3_gpu_silu_f32` | CUDA custom | F32 `[N] -> [N]` | CPU edge values | +| `h3_gpu_cast_f32_to_bf16` | CUDA custom | F32 `[N] -> BF16 [N]` | bit oracle inclusi NaN/Inf | +| `h3_gpu_cast_bf16_to_f32` | CUDA custom | BF16 `[N] -> F32 [N]` | bit oracle | +| `h3_gpu_copy_bf16` | CUDA runtime | BF16 slice `[src,n] -> [dst,n]` | sentinelle, overlap policy | +| `h3_gpu_copy_f32` | CUDA runtime | F32 slice `[src,n] -> [dst,n]` | sentinelle, overlap policy | +| `h3_gpu_rms_norm_f32` | CUDA custom | X F32 `[R,W]`, weight `[W]` -> F32 `[R,W]` | `test_metal`/CPU | +| `h3_gpu_adaln_f32` | CUDA custom | X `[R,W]`, norm `[W]`, mod `[M,S,W]`, map U32 `[R]` -> `[R,W]` | `test_metal`/CPU | +| `h3_gpu_gate_f32` | CUDA custom | residual/branch `[R,W]`, mod `[M,S,W]`, map `[R]` -> `[R,W]` | `test_metal`/CPU | +| `h3_gpu_swiglu_f32` | CUDA custom | fused F32 `[R,2W] -> [R,W]` | `test_metal`/CPU | +| `h3_gpu_scale_add_f32` | CUDA custom | residual/branch `[R,W]`, scale `[R]` o `[1]` -> `[R,W]` | CPU broadcast cases | +| `h3_gpu_layer_norm_f32` | CUDA custom | X `[R,W]`, weight/bias `[W]` -> `[R,W]` | CPU, constant row | +| `h3_gpu_weight_norm_f32` | CUDA custom | vector `[outer,inner]`, magnitude `[outer]` -> `[outer,inner]` | `test_audio_gpu`, abs `2e-6` | +| `h3_gpu_add_scaled_f32` | CUDA custom | left/right F32 `[N] -> [N]` | `test_audio_gpu`, abs `1e-7` | +| `h3_gpu_alias_free_snake_f32` | CUDA custom | X F32 `[B,L,C]`, params `[C]`, filters `[12]` -> `[B,L,C]` | `test_audio_gpu`, abs `2e-5` | +| `h3_gpu_snake1d_f32` | CUDA custom | X F32 `[B,L,C]`, alpha `[C]` -> `[B,L,C]` | CPU edge values | +| `h3_gpu_geglu_f32` | CUDA custom | gate/linear F32 `[N] -> [N]` | CPU GELU oracle | +| `h3_gpu_clip_f32` | CUDA custom | F32 `[N] -> [N]` | `test_audio_gpu`, abs `1e-7` | +| `h3_gpu_silu_bf16` | CUDA custom | BF16 `[N] -> [N]`, accumulo F32 | `test_bf16`, BF16 boundary | +| `h3_gpu_rms_norm_bf16` | CUDA custom | X BF16 `[R,W]`, weight `[W]` -> BF16 `[R,W]` | `test_bf16`, fixture reali | +| `h3_gpu_layer_norm_bf16` | CUDA custom | X BF16 `[R,W]`, weight/bias `[W]` -> BF16 `[R,W]` | vision fixture + CPU | +| `h3_gpu_gelu_bf16` | CUDA custom | BF16 `[N] -> [N]`, exact/approx flag | entrambe le modalità contro CPU | +| `h3_gpu_vision_qkv_rope_bf16` | CUDA custom | QKV `[S,3,H,D]` -> Q/K/V `[H,S,D]` BF16 | vision fixture | +| `h3_gpu_adaln_bf16` | CUDA custom | BF16 equivalente AdaLN F32 `[R,W]` | `test_bf16` | +| `h3_gpu_adaln_bf16_offset` | CUDA custom | come AdaLN, input slice da offset | `test_bf16`, sentinelle | +| `h3_gpu_gate_bf16` | CUDA custom | BF16 equivalente gate `[R,W]` | `test_bf16` | +| `h3_gpu_gate_adaln_bf16` | CUDA custom | gate residual + AdaLN, due output `[R,W]` | `test_bf16`, confronto non fuso | +| `h3_gpu_gate_adaln_quantize_int8` | CUDA custom | gate/AdaLN BF16 `[R,W]` -> residual BF16 + I8 `[padded_R,W]`, scale `[padded_R]` | confronto non fuso + dequant | +| `h3_gpu_qkv_rope_bf16` | CUDA custom | QKV BF16 `[S,3,H,D]` -> Q/K/V `[H,S,D]` | `test_bf16` | +| `h3_gpu_grouped_qkv_rope_bf16` | CUDA custom | QKV BF16 `[S,H,3,D]` -> Q/K/V `[H,S,D]` | `test_bf16`, bit-identico alla permutazione | +| `h3_gpu_swiglu_bf16` | CUDA custom | fused BF16 `[R,2W] -> [R,W]` | `test_bf16`, fixture MLX | +| `h3_gpu_embedding_bf16` | CUDA custom | weight BF16 `[V,W]`, ids U32 `[T]` -> BF16 `[T,W]` | `test_text_metal`, ids limite/invalidi | +| `h3_gpu_text_qk_rope_bf16` | CUDA custom | Q `[S,QH,D]`, K `[S,KH,D]` -> head-major BF16 | confronto head-norm + rope separati | +| `h3_gpu_head_rms_norm_bf16` | CUDA custom | tensor BF16 `[S,H,D]` in-place, weight `[D]` | `test_text_metal` | +| `h3_gpu_rope_text_bf16` | CUDA custom | Q `[S,QH,D]`, K `[S,KH,D]`, cos/sin F32 `[S,D/2]` | `test_text_metal` | +| `h3_gpu_add_bf16` | CUDA custom | BF16 `[N] + [N] -> [N]` | `test_bf16`, `test_text_metal` | +| `h3_gpu_sub_bf16` | CUDA custom | BF16 `[N] - [N] -> [N]` | CPU e round boundary | +| `h3_gpu_token_pool_bf16` | CUDA custom | BF16 input `[input_R,W]` -> pooled `[R,W]`; indices U32 | `test_bf16` synthetic exact | +| `h3_gpu_token_pool_adaln_bf16` | CUDA custom | pool + AdaLN, residual/output `[R,W]` | `test_bf16`, confronto non fuso | +| `h3_gpu_token_expand_delta_bf16` | CUDA custom | reduced `[reduced_R,W]` + maps U32 -> output `[R,W]` | `test_bf16` synthetic exact | +| `h3_gpu_token_expand_adaln_bf16` | CUDA custom | expand + AdaLN, residual/output `[R,W]` | `test_bf16`, confronto non fuso | +| `h3_gpu_euler_bf16` | CUDA custom | sample F32 slice `[N]`, last/previous BF16 `[N]` -> F32 | `test_bf16`, CPU formula | +| `h3_gpu_silu_mul_bf16` | CUDA custom | gate/up BF16 `[N] -> [N]` | `test_text_metal`, CPU | + +## Attention + +| API | Backend CUDA | Dtype e shape | Verifica CUDA prevista | +|---|---|---|---| +| `h3_gpu_qkv_rope_f32` | CUDA custom | QKV F32 `[S,3,H,D]` -> Q/K/V `[H,S,D]`, norm/RoPE | `test_metal` fixture | +| `h3_gpu_sdpa_f32` | CUDA custom + cuBLASLt | Q/K/V F32 `[H,S,D]` -> `[S,H,D]`, non causale | `test_metal`, CPU piccoli | +| `h3_gpu_video_qkv_rope_f32` | CUDA custom | QKV F32 `[S,3,H,D]` -> `[H,S,D]`, video RoPE | video encoder fixture | +| `h3_gpu_audio_qkv_split_f32` | CUDA custom | QKV F32 `[B,L,3,H,D]` + bias -> Q/K/V `[B,H,L,D]` | audio encoder fixture + CPU | +| `h3_gpu_sdpa_causal_f32` | CUDA custom + cuBLASLt | Q/K/V F32 `[B,H,S,D]` -> `[B,S,H,D]`, causal | CPU mask test | +| `h3_gpu_audio_attention_pool_f32` | CUDA custom | attended `[B,L,H,D]` -> `[B,output_dim]` | audio encoder fixture | +| `h3_gpu_sdpa_bf16` | CUDA custom + cuBLASLt | Q/K/V BF16 `[H,S,D]` -> `[S,H,D]` | `test_bf16`, real DiT fixture | +| `h3_gpu_sdpa_bf16_head_major_output` | CUDA custom + cuBLASLt | Q/K/V BF16 `[H,S,D]` -> `[H,S,D]` | confronto `sdpa_bf16` + transpose | +| `h3_gpu_gqa_causal_bf16` | CUDA custom + cuBLASLt | Q `[QH,S,D]`, K/V `[KH,S,D]` -> `[S,QH,D]`, causal | `test_text_metal`, CPU mask/GQA | + +## Convoluzioni e VAE + +| API | Backend CUDA | Dtype e shape | Verifica CUDA prevista | +|---|---|---|---| +| `h3_gpu_conv1d_f32` | CUDA custom + cuBLASLt | X `[B,L,Cin]`, W `[Cout,Cin,K]` -> `[B,Lout,Cout]` | `test_audio_gpu`, abs `2e-5` | +| `h3_gpu_conv1d_stride_f32` | CUDA custom + cuBLASLt | come Conv1d con stride, `Lout=floor((L+2P-D(K-1)-1)/S)+1` | CPU stride/dilation | +| `h3_gpu_conv_transpose1d_f32` | CUDA custom + cuBLASLt | X `[B,L,Cin]`, W `[Cin,Cout,K]` -> `[(L-1)S+K-2P]` | `test_audio_gpu`, abs `2e-5` | +| `h3_gpu_vae_encoder_pad_f32` | CUDA custom | X `[B,T,H,W,C]` -> padded channels-last tensor | semantic VAE encoder, border oracle | +| `h3_gpu_conv3d_f32` | CUDA custom + cuBLASLt | X `[B,T,H,W,Cin]`, W `[Cout,Cin,Kt,Kh,Kw]` -> channels-last output | semantic/real video encoder fixtures | +| `h3_gpu_vae_encoder_group_norm_silu_f32` | CUDA custom | X F32 `[B,T,H,W,C]`, weight/bias `[C]` -> stessa shape | semantic VAE encoder + CPU | + +## Soglie di parità + +- Copie, indici, metadata, layout puri e round-trip BF16/U32/I8: confronto + bit-esatto. Le fusioni devono coincidere bit-per-bit con la pipeline CUDA non + fusa quando condividono gli stessi confini di arrotondamento BF16. +- Primitive F32 con oracle CPU: `max_abs <= 2e-5`, salvo weight norm + `2e-6` e add/clip `1e-7`, mantenendo le soglie già usate da + `tests/test_audio_gpu.c`. +- Blocchi F32 contro fixture MLX: `max_rel < 5e-3`, come + `tests/test_metal.c`. +- Primitive e blocchi BF16 contro fixture MLX: `max_rel < 1e-2`, come + `tests/test_bf16.c` e `tests/test_text_metal.c`. Per una singola primitiva si + registra anche `max_abs`; NaN/Inf o mismatch di shape sono sempre FAIL. +- INT8: oltre alla soglia finale BF16 `max_rel < 2e-2`, la quantizzazione deve + rispettare `max_abs(dequant-input) <= scale/2 + 1e-6` per riga/canale. + +## Equivalenza della suite Metal + +Ogni eseguibile GPU esistente viene compilato una seconda volta contro il +backend CUDA, senza cambiare fixture né assertions: + +| Suite esistente | Equivalente CUDA | Copertura principale | +|---|---|---| +| `tests/test_metal.c` | `h3_cuda_tests` | blocco DiT F32, statistiche | +| `tests/test_bf16.c` | `h3_cuda_bf16_tests` | DiT BF16, fusioni, token reduction, Euler, INT8 fallback | +| `tests/test_text_metal.c` | `h3_cuda_text_tests` | embedding, Qwen norm/RoPE/GQA/MLP | +| `tests/test_audio_gpu.c` | `h3_cuda_audio_gpu_tests` | Conv1d, transpose, weight norm, Snake, elementwise | +| `tests/test_real_dit_block.c` | stesso target con backend CUDA | primitive reali e confini BF16 | +| `tests/test_real_dit.c`, `test_real_dit_schedule.c`, `test_semantic_dit.c` | stessi target con backend CUDA | integrazione DiT e scheduling | +| test real/semantic audio e video già elencati nel `Makefile` | stessi target con backend CUDA | operatori encoder/VAE e fixture checkpoint | + +Le API di lifecycle/I/O non coperte direttamente dalle suite Metal ricevono un +test CUDA dedicato. Il test genera un file temporaneo controllato per coprire +load, reload, streaming, range, errori e contatori. + +## Rischi specifici GB10 + +1. La memoria unificata CPU/GPU non rende automaticamente conveniente + `cudaMallocManaged`: page migration e fault su stream di pesi possono + serializzare I/O e compute. Il default resta device memory con staging + pinned; managed+prefetch richiede benchmark del Task 11. +2. cuBLASLt su ARM64/Blackwell può scegliere workspace e algoritmi diversi tra + shape; ogni matmul deve avere fallback deterministico e workspace limitato. +3. BF16 Tensor Core cambia l’ordine delle riduzioni rispetto a MPSGraph. Le + soglie sopra verificano il risultato, mentre i test bit-esatti sono limitati + a layout, copie e fusioni con uguali confini BF16. +4. Gli operatori conv/attention senza cuDNN devono evitare materializzazioni + `im2col` o score `[S,S]` non limitate: sul modello reale possono consumare + decine di GiB nonostante i 121 GiB disponibili. +5. Lo streaming SSD deve mantenere vivi staging buffer e CUDA event fino alla + copia completata; riuso anticipato produce corruzioni intermittenti che i + soli test piccoli non rilevano. +6. Le opzioni Metal 4/NAX non hanno equivalente diretto. Su CUDA usano la + pipeline portabile; INT8 è dichiarato disponibile solo dopo un probe reale + cuBLASLt, mai in base al solo compute capability. diff --git a/docs/GB10_PROFILE.md b/docs/GB10_PROFILE.md new file mode 100644 index 00000000..8397b115 --- /dev/null +++ b/docs/GB10_PROFILE.md @@ -0,0 +1,67 @@ +# NVIDIA GB10 profile + +Measured on NVIDIA GB10 with CUDA 13.0 and driver 595.84. Each result is the +median of three complete runs using the released `MiniMax-H3` checkpoint: + +```sh +./h3 -d ./MiniMax-H3 \ + -p "A bright red cube rotates on a white background." \ + --width 256 --height 256 --frames 22 --steps 2 --layers 35 \ + --token-reduction --seed 42 --profile -o OUTPUT.mp4 +``` + +| Mode | DiT load | Denoise | Load + denoise | DiT peak | +| --- | ---: | ---: | ---: | ---: | +| Resident BF16 baseline | 96.563 s | 2.715 s | 99.278 s | 27.063 GB | +| `--ssd-streaming` | 51.829 s | 84.798 s | 136.627 s | 1.630 GB | +| `--use-int8-row-fc2` | 96.491 s | 2.714 s | 99.205 s | 27.063 GB | + +All three repetitions within each mode produced the same SHA-256, and all +nine files share SHA-256 +`dbae1b441cface55bfe86aaabe78d44c0c05746909f7874908dd2cb298d8c5c8`. +Baseline versus SSD also measures SSIM 1.0. + +`--ssd-streaming` reduces the DiT peak by 94.0%, at a 37.6% increase in +load-plus-denoise time for this two-step workload. Its prefetch read 50.962 +GiB per run at a median 0.592 GiB/s, leaving a median 79.488 s of unhidden I/O. +It is therefore useful as an exact low-memory mode, not as the GB10 speed +default. + +`--use-int8-row-fc2` is a Metal/M5 specialization and is intentionally a no-op +in the CUDA backend. The measured 0.001 s denoise difference is noise. No +FP8 path is exposed by the current backend, and adding one without an +independent numeric oracle would violate the correctness gate. CUDA device +allocation already uses the GB10 unified physical memory; managed-memory +prefetch would add migration policy without reducing the measured resident +footprint. + +No new performance switch is enabled by default: none demonstrated a speedup +while preserving the verified output. Use resident BF16 for speed and +`--ssd-streaming` only when the 27 GB resident DiT peak is unacceptable. + +## Video VAE tiled F32 attention (T29-T32) + +Nsight profiling of the max-quality decode (1024x576, 107 frames) attributed +90.5% of the video VAE phase to the scalar F32 attention kernel (1007.06 s of +1112.96 s median; 1728 calls x 582.8 ms at sequence 2805, head_dim 64). A +tiled F32 kernel (`h3_attention_tiled_f32_kernel<64>`, 8 query rows per block, +online softmax, identical F32 recurrence) now serves non-causal batch-1 F32 +attention with head_dim 64; the scalar kernel remains the fallback and is +selectable again with `H3_DISABLE_TILED_ATTENTION=1`. + +Median of three `h3_vae_bench_quality` runs (latent 32x36x64x24, 107 frames): + +| Metric | Scalar F32 | Tiled F32 | +| --- | ---: | ---: | +| Video VAE decode | 1112.960 s | 247.737 s (4.49x) | +| Max-quality render wall | 33:36.80 | 18:56.36 (1.78x) | +| DiT denoise (unchanged) | 830.999 s | 814.958 s | +| Video VAE peak | 10.26 GB | 10.26 GB | + +Quality: CPU oracle parity within max_abs 2e-5 on shapes 13x2x64 and +2805x4x64, Compute Sanitizer clean, `VERIFY: PASS all`. Matched short render +(512x288/22 frames, same binary, only the VAE path toggled) measures SSIM +0.999303 / PSNR 55.45 dB. The max-quality MP4 measures SSIM 0.984985 / +PSNR 43.94 dB against the pre-optimization T28 artifact; two identical +rerenders are bit-identical (SSIM 1.0), so the delta is the VAE reduction +order amplified by the high-frequency content and H.264, not run noise. diff --git a/h3.c b/h3.c index d5dca259..c6e18929 100644 --- a/h3.c +++ b/h3.c @@ -3,7 +3,7 @@ #include "h3_host.h" #include "h3_dit.h" #include "h3_ffmpeg.h" -#include "h3_metal.h" +#include "h3_device.h" #include "h3_multimodal.h" #include "h3_safetensors.h" #include "h3_text_encoder.h" @@ -130,10 +130,14 @@ static int h3_key_file(h3_key *key, const char *role, const char *path) { if (stat(path, &status) != 0) return h3_key_append(key, "|%s=%zu:%s:missing", role, strlen(path), path); +#if defined(__APPLE__) + const struct timespec modified = status.st_mtimespec; +#else + const struct timespec modified = status.st_mtim; +#endif return h3_key_append(key, "|%s=%zu:%s:%lld:%lld:%ld", role, strlen(path), path, (long long)status.st_size, - (long long)status.st_mtimespec.tv_sec, - status.st_mtimespec.tv_nsec); + (long long)modified.tv_sec, modified.tv_nsec); } static char *h3_conditioning_key(const char *prompt, const h3_params *params, @@ -449,9 +453,9 @@ h3_ctx *h3_load_dir(const char *model_dir) { h3_free(ctx); return NULL; } - char metal_error[256]; - if (!h3_metal_probe(&ctx->device, metal_error, sizeof(metal_error))) { - h3_set_error(ctx, "%s", metal_error); + char device_error[256]; + if (!h3_device_probe(&ctx->device, device_error, sizeof(device_error))) { + h3_set_error(ctx, "%s", device_error); snprintf(h3_global_error, sizeof(h3_global_error), "%s", ctx->error); h3_free(ctx); return NULL; @@ -559,10 +563,12 @@ static int h3_valid_params(h3_ctx *ctx, const h3_params *params) { h3_set_error(ctx, "int8 row FC2 cannot be combined with the BF16 MLP"); return 0; } +#if defined(__APPLE__) if (params->use_int8_row_fc2 && !h3_device(ctx)->metal4) { h3_set_error(ctx, "int8 row FC2 requires an M5-class Metal 4 GPU"); return 0; } +#endif if (params->preview_denoise != 0 && params->preview_denoise != 1) { h3_set_error(ctx, "denoising preview must be zero or one"); return 0; diff --git a/h3.h b/h3.h index 29640b37..c0f4f236 100644 --- a/h3.h +++ b/h3.h @@ -169,7 +169,7 @@ struct h3_result { uint64_t seed; }; -/* Load model metadata and initialize the Metal device. Weights remain unmapped. */ +/* Load model metadata and initialize the selected GPU. Weights remain unmapped. */ h3_ctx *h3_load_dir(const char *model_dir); void h3_free(h3_ctx *ctx); diff --git a/h3_cli.c b/h3_cli.c index 79339c82..dd6d9538 100644 --- a/h3_cli.c +++ b/h3_cli.c @@ -102,9 +102,14 @@ static int set_directory(char destination[H3_CLI_PATH], const char *path) { } static uint64_t random_seed(void) { - uint64_t value; - arc4random_buf(&value, sizeof(value)); - return value; + uint64_t value = 0; + FILE *random = fopen("/dev/urandom", "rb"); + if (random) { + size_t count = fread(&value, 1, sizeof(value), random); + fclose(random); + if (count == sizeof(value)) return value; + } + return (uint64_t)time(NULL) ^ ((uint64_t)(unsigned)getpid() << 32); } static int cli_progress(const char *phase, int completed, int total, diff --git a/h3_metal.h b/h3_metal.h deleted file mode 100644 index beaf628d..00000000 --- a/h3_metal.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef H3_METAL_H -#define H3_METAL_H - -#include "h3.h" - -int h3_metal_probe(h3_device_info *info, char *error, size_t error_size); - -#endif diff --git a/h3_metal.m b/h3_metal.m index 85e1d4c4..137c9ba4 100644 --- a/h3_metal.m +++ b/h3_metal.m @@ -1,7 +1,7 @@ #import #import -#include "h3_metal.h" +#include "h3_device.h" #include #include @@ -12,7 +12,7 @@ static void h3_copy_string(char *destination, size_t size, NSString *value) { snprintf(destination, size, "%s", source ? source : "unknown"); } -int h3_metal_probe(h3_device_info *info, char *error, size_t error_size) { +int h3_device_probe(h3_device_info *info, char *error, size_t error_size) { if (!info) return 0; memset(info, 0, sizeof(*info)); @autoreleasepool { diff --git a/main.c b/main.c index 7f11e470..a15c5e47 100644 --- a/main.c +++ b/main.c @@ -130,9 +130,11 @@ static void print_info(const h3_ctx *ctx) { printf("Device: %s (%s)\n", device->name, device->architecture); printf(" physical memory %.1f GiB\n", gib(device->physical_memory)); printf(" recommended GPU set %.1f GiB\n", gib(device->recommended_working_set)); - printf(" max Metal buffer %.1f GiB\n", gib(device->max_buffer_length)); - printf(" Apple GPU family %d\n", device->apple_gpu_family); - printf(" Metal 4 %s\n", device->metal4 ? "yes" : "no"); + printf(" max GPU buffer %.1f GiB\n", gib(device->max_buffer_length)); + if (device->apple_gpu_family > 0) { + printf(" Apple GPU family %d\n", device->apple_gpu_family); + printf(" Metal 4 %s\n", device->metal4 ? "yes" : "no"); + } printf(" unified memory %s\n", device->unified_memory ? "yes" : "no"); printf("Native checkpoint inventory (header-only):\n"); print_component("Qwen3-VL encoder", &model->text_encoder); diff --git a/tests/bench_dit.c b/tests/bench_dit.c index 7a1d4440..6cd19859 100644 --- a/tests/bench_dit.c +++ b/tests/bench_dit.c @@ -14,16 +14,22 @@ #ifndef H3_BENCH_LATENT_W #define H3_BENCH_LATENT_W 32 #endif +#ifndef H3_BENCH_LATENT_T +#define H3_BENCH_LATENT_T 7 +#endif +#ifndef H3_BENCH_AUDIO_T +#define H3_BENCH_AUDIO_T 37 +#endif enum { TEXT_ROWS = 6, TEXT_WIDTH = 5120, - LATENT_T = 7, + LATENT_T = H3_BENCH_LATENT_T, LATENT_H = H3_BENCH_LATENT_H, LATENT_W = H3_BENCH_LATENT_W, CANVAS_H = LATENT_H * 16, CANVAS_W = LATENT_W * 16, - AUDIO_T = 37, + AUDIO_T = H3_BENCH_AUDIO_T, VIDEO_ELEMENTS = 24 * LATENT_T * LATENT_H * LATENT_W, AUDIO_ELEMENTS = 32 * 2 * AUDIO_T }; @@ -1514,7 +1520,9 @@ int main(int argc, char **argv) { const char *model_root = argc > 1 ? argv[1] : "MiniMax-H3"; const char *prompt_fixture = argc > 2 ? argv[2] : "misc/fixtures/h3_real_prompt_bf16.safetensors"; - uint16_t *text_values = load_text(prompt_fixture); + uint16_t *text_values = getenv("H3_BENCH_SYNTHETIC_TEXT") ? + calloc(TEXT_ROWS * TEXT_WIDTH, sizeof(*text_values)) : + load_text(prompt_fixture); float *video = calloc(VIDEO_ELEMENTS, sizeof(*video)); float *audio = calloc(AUDIO_ELEMENTS, sizeof(*audio)); float *video_velocity = malloc(VIDEO_ELEMENTS * sizeof(*video_velocity)); diff --git a/tests/bench_video_vae.c b/tests/bench_video_vae.c new file mode 100644 index 00000000..968d3918 --- /dev/null +++ b/tests/bench_video_vae.c @@ -0,0 +1,95 @@ +#include "h3_video_vae.h" + +#include +#include +#include +#include +#include + +/* Quality preset latent shape: 1024x576 canvas, 107 frames. + * latent_time 32 -> 6 temporal chunks -> 107 decoded frames. */ +#ifndef H3_BENCH_VAE_LATENT_T +#define H3_BENCH_VAE_LATENT_T 32 +#endif +#ifndef H3_BENCH_VAE_LATENT_H +#define H3_BENCH_VAE_LATENT_H 36 +#endif +#ifndef H3_BENCH_VAE_LATENT_W +#define H3_BENCH_VAE_LATENT_W 64 +#endif + +enum { + LATENT_CHANNELS = 24, + LATENT_TIME = H3_BENCH_VAE_LATENT_T, + LATENT_H = H3_BENCH_VAE_LATENT_H, + LATENT_W = H3_BENCH_VAE_LATENT_W, + EXPECTED_FRAMES = (LATENT_TIME - 2) / 5 * 17 + 5, + MAX_RUNS = 16 +}; + +static void die(const char *message) { + fprintf(stderr, "h3_vae_bench: %s\n", message); + exit(1); +} + +static double seconds(void) { + struct timespec value; + if (clock_gettime(CLOCK_MONOTONIC, &value) != 0) return 0.0; + return (double)value.tv_sec + (double)value.tv_nsec * 1e-9; +} + +static int compare_double(const void *left, const void *right) { + double a = *(const double *)left, b = *(const double *)right; + return (a > b) - (a < b); +} + +int main(int argc, char **argv) { + const char *model_dir = argc > 1 ? argv[1] : "./MiniMax-H3"; + int runs = argc > 2 ? atoi(argv[2]) : 3; + if (runs < 1) runs = 1; + if (runs > MAX_RUNS) runs = MAX_RUNS; + char weights[4096]; + snprintf(weights, sizeof(weights), "%s/FL2VA/video_vae/source", model_dir); + size_t latent_elements = + (size_t)LATENT_TIME * LATENT_H * LATENT_W * LATENT_CHANNELS; + float *latent = malloc(latent_elements * sizeof(*latent)); + if (!latent) die("out of memory for latents"); + /* Deterministic pseudo-random latents: decode timing is value + * independent, so synthetic latents profile the real compute path. */ + uint64_t state = UINT64_C(0x9E3779B97F4A7C15); + for (size_t index = 0; index < latent_elements; index++) { + state = state * UINT64_C(6364136223846793005) + + UINT64_C(1442695040888963407); + latent[index] = (float)((double)(state >> 40) / 8388608.0 - 1.0); + } + double wall[MAX_RUNS], gpu[MAX_RUNS]; + char error[512]; + for (int run = 0; run < runs; run++) { + h3_video_frames frames; + double start = seconds(); + int ok = h3_video_vae_decode(weights, "h3_shaders.metal", latent, + LATENT_TIME, LATENT_H, LATENT_W, NULL, NULL, &frames, error, + sizeof(error)); + double elapsed = seconds() - start; + if (!ok) die(error); + if (frames.frames != EXPECTED_FRAMES || frames.height != LATENT_H * 16 || + frames.width != LATENT_W * 16) { + h3_video_frames_free(&frames); + die("decoded frame shape does not match the quality preset"); + } + wall[run] = elapsed; + gpu[run] = frames.gpu_stats.gpu_seconds; + printf("run %d: wall %.3fs, gpu %.3fs, submissions %llu, peak %.3f GB\n", + run + 1, elapsed, frames.gpu_stats.gpu_seconds, + (unsigned long long)frames.gpu_stats.submissions, + (double)frames.gpu_stats.peak_live_bytes / 1e9); + fflush(stdout); + h3_video_frames_free(&frames); + } + free(latent); + qsort(wall, (size_t)runs, sizeof(*wall), compare_double); + qsort(gpu, (size_t)runs, sizeof(*gpu), compare_double); + printf("median: wall %.3fs, gpu %.3fs over %d run(s)\n", + wall[runs / 2], gpu[runs / 2], runs); + return 0; +} diff --git a/tests/test_av_mux.c b/tests/test_av_mux.c index eefc1f70..6f8a5071 100644 --- a/tests/test_av_mux.c +++ b/tests/test_av_mux.c @@ -29,7 +29,7 @@ int main(int argc, char **argv) { for (int channel = 0; channel < 2; channel++) for (int sample = 0; sample < SAMPLES; sample++) pcm[(size_t)channel * SAMPLES + (size_t)sample] = - 0.05f * sinf(2.0f * 3.14159265358979323846f * + 1.5f * sinf(2.0f * 3.14159265358979323846f * (float)(220 + channel * 110) * (float)sample / 32000.0f); char error[512]; @@ -89,6 +89,7 @@ int main(int argc, char **argv) { if (decoded_samples != SAMPLES) die("FFmpeg audio input returned an unexpected sample count"); double left_energy = 0.0, right_energy = 0.0; + float peak = 0.0f; for (int sample = 0; sample < decoded_samples; sample++) { float left = decoded_pcm[sample]; float right = decoded_pcm[decoded_samples + sample]; @@ -96,9 +97,12 @@ int main(int argc, char **argv) { die("decoded FFmpeg audio contains non-finite PCM"); left_energy += (double)left * left; right_energy += (double)right * right; + peak = fmaxf(peak, fmaxf(fabsf(left), fabsf(right))); } if (left_energy < 1.0 || right_energy < 1.0) die("decoded FFmpeg audio has no stereo signal"); + if (peak > 1.0f) + die("decoded FFmpeg audio exceeds the anti-clipping ceiling"); free(decoded_pcm); printf("ok: concurrent FFmpeg video/PCM pipes created %s (%lld bytes)\n", path, (long long)status.st_size); diff --git a/tests/test_h3.c b/tests/test_h3.c index 3f66fa32..52a5308f 100644 --- a/tests/test_h3.c +++ b/tests/test_h3.c @@ -1,6 +1,6 @@ #include "h3_host.h" #include "h3_dit.h" -#include "h3_metal.h" +#include "h3_device.h" #include "h3_safetensors.h" #include "h3_terminal.h" @@ -373,14 +373,14 @@ static void test_dit_row_conversions(void) { CHECK(memcmp(audio, unpacked, sizeof(audio)) == 0); } -static void test_metal_probe(void) { +static void test_device_probe(void) { h3_device_info info; char error[256]; - CHECK(h3_metal_probe(&info, error, sizeof(error))); + CHECK(h3_device_probe(&info, error, sizeof(error))); CHECK(info.name[0] != '\0'); CHECK(info.physical_memory >= UINT64_C(8) * 1024 * 1024 * 1024); CHECK(info.max_buffer_length > 0); - CHECK(info.apple_gpu_family > 0); + CHECK(info.recommended_working_set > 0); } static void test_terminal_zoom(void) { @@ -407,7 +407,7 @@ int main(void) { test_rng_and_solver(); test_rgb_resize(); test_dit_row_conversions(); - test_metal_probe(); + test_device_probe(); test_terminal_zoom(); printf("ok: %d checks\n", tests_run); return 0; From 10a7dc1cb05513eb24da6d9362186add2583222d Mon Sep 17 00:00:00 2001 From: matrixfede Date: Wed, 26 Aug 2026 21:06:01 +0200 Subject: [PATCH 04/20] Add h3.c Studio: web UI, Docker deployment and public-repo material MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A FastAPI backend and a React front end drive the same h3 binary the CLI does, so a video can be made without the command line. Every one of the 42 flags is reachable; Create shows only what a first-time user needs, and each choice that changes the wait carries its own estimate, learnt from the jobs this machine has actually finished. The backend runs one job at a time — a single GPU, 27 GB at the DiT peak — streams progress and live denoising previews over SSE, and keeps uploads in a library that later jobs can reuse. A generation no longer occupies the page: a monitoring strip keeps it visible while the next one is composed. main.c gains --preview-dir, the only change to the C: it writes the previews preview_denoise already produces to files, and leaves the mp4 byte-identical. Docker builds h3 for CUDA and ships the binary with the service; the 465 GB checkpoint is bind-mounted read-only and never enters an image. The service has no authentication and binds to the loopback address by default. An optional post-processing stage can hand the finished video to an external program. No model, no runtime and no download URL is included. Co-Authored-By: Claude Opus 5 (1M context) --- .env.example | 24 + .github/workflows/ci.yml | 69 + .gitignore | 15 +- CONTRIBUTING.md | 73 + README.md | 30 + THIRD_PARTY_NOTICES.md | 33 + docker-compose.faceswap.yml | 19 + docker-compose.yml | 54 + docker/Dockerfile.backend | 61 + docker/Dockerfile.frontend | 13 + docker/nginx.conf | 23 + docs/POSTPROCESSING.md | 81 + docs/WEBUI.md | 142 ++ docs/mockup/WIREFRAME.md | 99 + docs/mockup/index.html | 453 ++++ docs/mockup/v2.html | 625 +++++ eslint.config.js | 55 + main.c | 53 +- package-lock.json | 2113 +++++++++++++++++ package.json | 14 + scripts/benchmark_sdpa.py | 84 + scripts/snapshot_ui.mjs | 53 + scripts/verify.sh | 111 + webui/backend/app/__init__.py | 1 + webui/backend/app/argv.py | 78 + webui/backend/app/assets.py | 188 ++ webui/backend/app/capabilities.py | 11 + webui/backend/app/config.py | 40 + webui/backend/app/db.py | 99 + webui/backend/app/events.py | 51 + webui/backend/app/jobspec.py | 237 ++ webui/backend/app/main.py | 218 ++ webui/backend/app/media.py | 70 + webui/backend/app/postprocess.py | 123 + webui/backend/app/progress.py | 247 ++ webui/backend/app/runner.py | 399 ++++ webui/backend/app/system.py | 88 + webui/backend/pyproject.toml | 22 + webui/backend/tests/conftest.py | 6 + webui/backend/tests/test_api_basics.py | 116 + webui/backend/tests/test_assets.py | 149 ++ webui/backend/tests/test_copy.py | 95 + webui/backend/tests/test_estimate.py | 150 ++ webui/backend/tests/test_events_and_media.py | 137 ++ .../tests/test_frontend_covers_schema.py | 120 + .../backend/tests/test_jobspec_validation.py | 251 ++ .../tests/test_mockup_covers_schema.py | 82 + webui/backend/tests/test_postprocess.py | 132 + webui/backend/tests/test_preview.py | 115 + webui/backend/tests/test_progress.py | 226 ++ webui/backend/tests/test_runner.py | 214 ++ .../backend/tests/test_schema_matches_cli.py | 125 + webui/backend/tools/calibrate_progress.py | 125 + webui/frontend/index.html | 12 + webui/frontend/package-lock.json | 1879 +++++++++++++++ webui/frontend/package.json | 26 + webui/frontend/scripts/generate-options.mjs | 63 + webui/frontend/src/App.tsx | 307 +++ webui/frontend/src/api.ts | 81 + webui/frontend/src/components/Create.tsx | 218 ++ webui/frontend/src/components/Expert.tsx | 230 ++ webui/frontend/src/components/FineTune.tsx | 182 ++ webui/frontend/src/components/LiveStrip.tsx | 48 + webui/frontend/src/components/PhotoSlot.tsx | 115 + webui/frontend/src/components/References.tsx | 164 ++ webui/frontend/src/components/RenderStage.tsx | 134 ++ webui/frontend/src/components/Takes.tsx | 66 + webui/frontend/src/copy.ts | 130 + webui/frontend/src/generated/options.ts | 615 +++++ webui/frontend/src/main.tsx | 20 + webui/frontend/src/spec.ts | 99 + webui/frontend/src/styles.css | 393 +++ webui/frontend/src/types.ts | 119 + webui/frontend/src/useEstimates.ts | 54 + webui/frontend/tsconfig.json | 19 + webui/frontend/vite.config.ts | 21 + webui/shared/copy.json | 289 +++ webui/shared/options.schema.json | 696 ++++++ webui/shared/progress_weights.json | 88 + 79 files changed, 14277 insertions(+), 3 deletions(-) create mode 100644 .env.example create mode 100644 .github/workflows/ci.yml create mode 100644 CONTRIBUTING.md create mode 100644 docker-compose.faceswap.yml create mode 100644 docker-compose.yml create mode 100644 docker/Dockerfile.backend create mode 100644 docker/Dockerfile.frontend create mode 100644 docker/nginx.conf create mode 100644 docs/POSTPROCESSING.md create mode 100644 docs/WEBUI.md create mode 100644 docs/mockup/WIREFRAME.md create mode 100644 docs/mockup/index.html create mode 100644 docs/mockup/v2.html create mode 100644 eslint.config.js create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 scripts/benchmark_sdpa.py create mode 100755 scripts/snapshot_ui.mjs create mode 100755 scripts/verify.sh create mode 100644 webui/backend/app/__init__.py create mode 100644 webui/backend/app/argv.py create mode 100644 webui/backend/app/assets.py create mode 100644 webui/backend/app/capabilities.py create mode 100644 webui/backend/app/config.py create mode 100644 webui/backend/app/db.py create mode 100644 webui/backend/app/events.py create mode 100644 webui/backend/app/jobspec.py create mode 100644 webui/backend/app/main.py create mode 100644 webui/backend/app/media.py create mode 100644 webui/backend/app/postprocess.py create mode 100644 webui/backend/app/progress.py create mode 100644 webui/backend/app/runner.py create mode 100644 webui/backend/app/system.py create mode 100644 webui/backend/pyproject.toml create mode 100644 webui/backend/tests/conftest.py create mode 100644 webui/backend/tests/test_api_basics.py create mode 100644 webui/backend/tests/test_assets.py create mode 100644 webui/backend/tests/test_copy.py create mode 100644 webui/backend/tests/test_estimate.py create mode 100644 webui/backend/tests/test_events_and_media.py create mode 100644 webui/backend/tests/test_frontend_covers_schema.py create mode 100644 webui/backend/tests/test_jobspec_validation.py create mode 100644 webui/backend/tests/test_mockup_covers_schema.py create mode 100644 webui/backend/tests/test_postprocess.py create mode 100644 webui/backend/tests/test_preview.py create mode 100644 webui/backend/tests/test_progress.py create mode 100644 webui/backend/tests/test_runner.py create mode 100644 webui/backend/tests/test_schema_matches_cli.py create mode 100644 webui/backend/tools/calibrate_progress.py create mode 100644 webui/frontend/index.html create mode 100644 webui/frontend/package-lock.json create mode 100644 webui/frontend/package.json create mode 100644 webui/frontend/scripts/generate-options.mjs create mode 100644 webui/frontend/src/App.tsx create mode 100644 webui/frontend/src/api.ts create mode 100644 webui/frontend/src/components/Create.tsx create mode 100644 webui/frontend/src/components/Expert.tsx create mode 100644 webui/frontend/src/components/FineTune.tsx create mode 100644 webui/frontend/src/components/LiveStrip.tsx create mode 100644 webui/frontend/src/components/PhotoSlot.tsx create mode 100644 webui/frontend/src/components/References.tsx create mode 100644 webui/frontend/src/components/RenderStage.tsx create mode 100644 webui/frontend/src/components/Takes.tsx create mode 100644 webui/frontend/src/copy.ts create mode 100644 webui/frontend/src/generated/options.ts create mode 100644 webui/frontend/src/main.tsx create mode 100644 webui/frontend/src/spec.ts create mode 100644 webui/frontend/src/styles.css create mode 100644 webui/frontend/src/types.ts create mode 100644 webui/frontend/src/useEstimates.ts create mode 100644 webui/frontend/tsconfig.json create mode 100644 webui/frontend/vite.config.ts create mode 100644 webui/shared/copy.json create mode 100644 webui/shared/options.schema.json create mode 100644 webui/shared/progress_weights.json diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..2b7cba3a --- /dev/null +++ b/.env.example @@ -0,0 +1,24 @@ +# Copy to .env and adjust. Only H3_MODEL_DIR is required. + +# The MiniMax-H3 checkpoint directory, mounted read-only (about 465 GB). +H3_MODEL_DIR=./MiniMax-H3 + +# Where jobs, uploads and generated videos are written. Keep this separate +# from the local development directory: the container writes as root. +H3_DATA_DIR=./data + +# CUDA architecture the h3 binary is compiled for. sm_121 is the NVIDIA GB10. +NVCC_ARCH=sm_121 + +# Interface the UI is published on. 127.0.0.1 keeps it on this machine only. +# To reach it from another machine, use a private-network address rather than +# 0.0.0.0: this service has no authentication. With Tailscale, `tailscale ip -4` +# prints the address to use, and only your tailnet can then connect. +H3_BIND=127.0.0.1 + +# Largest accepted upload, in bytes (512 MB). +H3_MAX_UPLOAD_BYTES=536870912 + +# Optional post-processing runtime. Left empty, the faceswap plugin stays +# unavailable and nothing is downloaded. See docs/POSTPROCESSING.md. +H3_FACESWAP_CMD= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..b90bf196 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,69 @@ +# CI without a GPU: everything that does not need CUDA or the 465 GB +# checkpoint. The end-to-end render stays a local gate (see PLAN.md task 79). +name: CI + +on: + push: + branches: ["**"] + pull_request: + +jobs: + backend: + name: Backend (lint + tests) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install FFmpeg + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends ffmpeg + - name: Install dependencies + run: | + python -m venv webui/backend/.venv + webui/backend/.venv/bin/pip install --upgrade pip + webui/backend/.venv/bin/pip install \ + "fastapi>=0.115" "uvicorn[standard]>=0.34" "pydantic-settings>=2.6" \ + python-multipart pytest httpx ruff + - name: Lint + run: webui/backend/.venv/bin/ruff check webui + - name: Tests + run: webui/backend/.venv/bin/pytest webui/backend/tests -q + + frontend: + name: Frontend (build + lint) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: Install repo tooling + run: npm install --no-audit --no-fund + - name: Install frontend dependencies + run: npm install --no-audit --no-fund + working-directory: webui/frontend + - name: Regenerate the options module and check it is committed + run: | + node scripts/generate-options.mjs + git diff --exit-code src/generated/options.ts + working-directory: webui/frontend + - name: Build + run: npm run build + working-directory: webui/frontend + - name: Lint + run: npx eslint . + + compose: + name: Docker compose files + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Validate compose configuration + env: + H3_MODEL_DIR: ./MiniMax-H3 + run: | + docker compose config >/dev/null + H3_FACESWAP_CMD=/opt/faceswap/run H3_FACESWAP_DIR=/opt/faceswap \ + docker compose -f docker-compose.yml -f docker-compose.faceswap.yml \ + config >/dev/null diff --git a/.gitignore b/.gitignore index 6e4c2c90..1bb5ed87 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ outputs/ h3 h3_*test h3_*tests -h3_*bench +h3_*bench* h3_dit_bench_864 h3_tests h3_metal_tests @@ -28,3 +28,16 @@ h3_real_video_vae_test h3_semantic_vae_test libh3.a .DS_Store +._* + +# Node dev tooling and web UI build output. +node_modules/ +webui/frontend/dist/ +webui/backend/.venv/ +webui/backend/data/ +data/ +__pycache__/ +.pytest_cache/ + +# Local docker env +.env diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..dde44f52 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,73 @@ +# Contributing + +Thanks for looking. This is a small project with a strict habit: **nothing is +marked done until a command says so.** + +## Before you start + +Work is planned in [`PLAN.md`](PLAN.md), which is versioned on purpose — its +git history is the log of how the requirements evolved. Concluded phases are +archived in `PLAN_ARCHIVE.md`. If you are adding something substantial, open an +issue first and say what you intend to change; a plan beats a surprise pull +request. + +## The gate + +One command decides whether a change is acceptable: + +```sh +./scripts/verify.sh all +``` + +Its last line is `VERIFY: PASS` or `VERIFY: FAIL`. It runs the C test suite, +the CUDA primitive tests when `nvcc` is present, `ruff` and `pytest` over +`webui/`, and ESLint over the repository. A pull request whose gate fails is +not ready, and "it works on my machine" is not an exit code. + +The GPU end-to-end render is not in CI — no runner has a GB10 and the +checkpoint is 465 GB. If your change can affect generation, run one yourself +and say so in the pull request: + +```sh +./h3 -d ./MiniMax-H3 -p "A bright red cube on a white background." \ + --width 256 --height 256 --frames 22 --steps 2 -o outputs/smoke.mp4 +``` + +## Working on the C engine + +- Keep the CLI and the public API in `h3.h` backward compatible. New behaviour + is an additive flag that changes nothing when it is absent. +- Prove non-invasiveness where it matters. `--preview-dir` was accepted because + the SHA-256 of the generated mp4 is identical with and without the flag. +- Both backends stay alive: Metal on Apple Silicon, CUDA on Linux. Selection + happens at build time in the `Makefile`, never with a runtime branch. + +## Working on the web UI + +- `webui/shared/options.schema.json` is the single source of truth for the + generation options. Add the flag to `main.c`, then to the schema; a test + reads `main.c` and fails if the two drift. +- The frontend's `src/generated/options.ts` is generated by + `node scripts/generate-options.mjs`. Never edit it by hand, and commit the + regenerated file: CI checks that it is current. +- Validation messages are copied verbatim from `h3.c`, so the browser and the + engine say the same thing. +- Tests use a stand-in for the `h3` binary: a shell script that prints progress + lines and writes a file. No GPU, no checkpoint, no waiting. + +## Style + +Match the surrounding code. C follows the existing project conventions; Python +is formatted for `ruff` with the settings in `webui/backend/pyproject.toml`; +TypeScript follows the ESLint configuration at the repository root. Comments +explain why, not what. + +Keep diffs surgical: a change that also reformats a neighbouring function is +two changes, and reviewers can only review one of them. + +## Licensing + +By contributing you agree that your work is released under the MIT license in +[`LICENSE`](LICENSE). Do not add model weights, checkpoints or download URLs to +this repository, and do not add dependencies whose license is more restrictive +than MIT without saying so explicitly in the pull request. diff --git a/README.md b/README.md index c8ca60ab..7409fac7 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,11 @@ Native MiniMax-H3 inference in C for NVIDIA CUDA on Linux and Metal on Apple Silicon. Prompt-to-video/audio, first/last-frame conditioning, and ordered Ref2VA image/video/audio references work end to end on both backends. +A browser front end is included: **h3.c Studio** exposes every generation +option of the CLI, with a live preview of the denoising and a weighted +progress bar. If you would rather click than type, jump to +[Web UI](#web-ui) — the rest of this document is the CLI. + ## Platforms and prerequisites The validated Linux configuration is Ubuntu ARM64, NVIDIA GB10, driver 595.84, @@ -455,6 +460,31 @@ Standalone audio must accompany an image or video reference. Audio references must be 2–15 seconds; at most three audio inputs are accepted and their total decoded duration is capped at 15 seconds. +## Web UI + +`h3.c Studio` is a small FastAPI backend and a React front end that drive the +same `h3` binary this document describes. Every CLI option is reachable from +the browser, generation progress streams live, and uploaded photos and clips +stay in a reusable library. + +```sh +cp .env.example .env # set H3_MODEL_DIR to your checkpoint +docker compose up --build # then open http://127.0.0.1:8080 +``` + +Docker needs the NVIDIA Container Toolkit; the checkpoint is bind-mounted +read-only and never enters an image. A local, non-Docker setup and the design +notes are in [`docs/WEBUI.md`](docs/WEBUI.md). + +The service has **no authentication** and binds to `127.0.0.1`. To reach it +from another machine, set `H3_BIND` to a private address — a Tailscale address, +or an SSH tunnel — rather than publishing it on the LAN. See +[`docs/WEBUI.md`](docs/WEBUI.md#security). + +An optional post-processing stage can hand the finished video to an external +program. This repository ships no such program and no models: see +[`docs/POSTPROCESSING.md`](docs/POSTPROCESSING.md). + ## Tests and runtime requirements ```sh diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 460e0a0f..94d9b0d0 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -30,3 +30,36 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +## Upstream project + +`h3.c` began as [antirez/h3.c](https://github.com/antirez/h3.c) by Salvatore +Sanfilippo, released under the MIT license reproduced in `LICENSE`. This fork +keeps that license and adds the CUDA backend and the web UI under the same +terms. + +## Web UI dependencies + +The web UI does not vendor any third-party source. Its dependencies are +installed from their own registries at build time and each keeps its own +license: + +| Component | Where | License | +| --- | --- | --- | +| FastAPI, Starlette | backend | MIT | +| Uvicorn | backend | BSD-3-Clause | +| Pydantic, pydantic-settings | backend | MIT | +| python-multipart | backend | Apache-2.0 | +| React, React DOM | frontend | MIT | +| Vite, @vitejs/plugin-react | frontend | MIT | +| TypeScript | frontend | Apache-2.0 | +| ESLint, typescript-eslint | dev tooling | MIT | +| Playwright | dev tooling | Apache-2.0 | +| nginx | container image | BSD-2-Clause | +| FFmpeg | runtime dependency | LGPL-2.1-or-later or GPL-2.0-or-later, depending on the build | + +The MiniMax-H3 checkpoint is **not** part of this repository and is covered by +its own license from MiniMax. No model weights of any kind are distributed +here, including for the optional post-processing stage. diff --git a/docker-compose.faceswap.yml b/docker-compose.faceswap.yml new file mode 100644 index 00000000..f9c9882c --- /dev/null +++ b/docker-compose.faceswap.yml @@ -0,0 +1,19 @@ +# Optional override that wires an already installed face-swapping runtime into +# the post-processing stage. It is NOT active by default: +# +# docker compose -f docker-compose.yml -f docker-compose.faceswap.yml up +# +# This repository ships no models, no weights and no download URLs. You install +# the runtime yourself, you check its licence, and you are responsible for the +# consent of anyone whose face you process. See docs/POSTPROCESSING.md for the +# contract the executable must satisfy. + +services: + api: + environment: + # Path *inside the container* of the executable implementing the + # --input/--output contract. + H3_FACESWAP_CMD: ${H3_FACESWAP_CMD:?set H3_FACESWAP_CMD to your runtime} + volumes: + # Mount your own runtime and its models read-only. + - ${H3_FACESWAP_DIR:?set H3_FACESWAP_DIR to the runtime directory}:/opt/faceswap:ro diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..6555a280 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,54 @@ +# h3.c Studio: the CUDA backend and the static UI. +# +# Prerequisites: an NVIDIA driver matching CUDA 13, the NVIDIA Container +# Toolkit, and the MiniMax-H3 checkpoint on disk (about 465 GB). The checkpoint +# is never copied into an image: it is bind-mounted read-only. +# +# cp .env.example .env # then set H3_MODEL_DIR +# docker compose up --build +# +# The UI listens on 127.0.0.1 only. It has no authentication: do not publish +# these ports on an untrusted network. To reach it from another machine, set +# H3_BIND to a private-network address — a Tailscale address is the safe +# choice, because the tailnet does the authenticating that this service does +# not. Binding to 0.0.0.0 hands the GPU and every generated file to anyone on +# the LAN. + +services: + api: + build: + context: . + dockerfile: docker/Dockerfile.backend + args: + NVCC_ARCH: ${NVCC_ARCH:-sm_121} + environment: + H3_MODEL_DIR: /models + H3_DATA_DIR: /data + H3_MAX_UPLOAD_BYTES: ${H3_MAX_UPLOAD_BYTES:-536870912} + # Post-processing stays off unless you install a runtime yourself. + H3_FACESWAP_CMD: ${H3_FACESWAP_CMD:-} + volumes: + - ${H3_MODEL_DIR:?set H3_MODEL_DIR in .env}:/models:ro + - ${H3_DATA_DIR:-./data}:/data + # The API is only published for local debugging: the web container reaches + # it over the compose network, and nginx proxies /api for the browser. + ports: + - "127.0.0.1:8000:8000" + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + restart: unless-stopped + + web: + build: + context: . + dockerfile: docker/Dockerfile.frontend + depends_on: + - api + ports: + - "${H3_BIND:-127.0.0.1}:8080:80" + restart: unless-stopped diff --git a/docker/Dockerfile.backend b/docker/Dockerfile.backend new file mode 100644 index 00000000..7179e703 --- /dev/null +++ b/docker/Dockerfile.backend @@ -0,0 +1,61 @@ +# Builds h3 for CUDA, then ships only the binary and the Python service. +# The 465 GB checkpoint is never copied in: it is bind-mounted at runtime. +ARG CUDA_VERSION=13.0.0 +ARG UBUNTU_VERSION=24.04 + +FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} AS build +# nvcc is already in the image: this stage only needs Ubuntu packages. NVIDIA's +# apt repository is dropped because it is a frequent source of hash-sum +# mismatches behind caching proxies, and apt is told to retry without +# pipelining, which is what makes those mismatches transient. +RUN set -eux; \ + rm -f /etc/apt/sources.list.d/cuda*.list \ + /etc/apt/sources.list.d/nvidia*.list \ + /etc/apt/sources.list.d/*cuda*.sources; \ + printf 'Acquire::Retries "5";\nAcquire::http::Pipeline-Depth "0";\nAcquire::http::No-Cache "true";\nAcquire::BrokenProxy "true";\n' \ + > /etc/apt/apt.conf.d/99-h3-resilient; \ + apt-get clean; rm -rf /var/lib/apt/lists/*; \ + apt-get update; \ + apt-get install -y --no-install-recommends build-essential libicu-dev pkg-config; \ + rm -rf /var/lib/apt/lists/* +WORKDIR /src +# Only what the C build needs, so editing the web UI does not rebuild h3. +COPY Makefile *.c *.h *.cu *.metal ./ +COPY tests ./tests +# A concrete architecture, because `native` needs a visible GPU and the build +# stage has none. sm_121 is the GB10; override with --build-arg NVCC_ARCH=... +ARG NVCC_ARCH=sm_121 +RUN make -j"$(nproc)" NVCC_ARCH=${NVCC_ARCH} h3 + +FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION} AS runtime +# Same treatment: only Ubuntu packages are needed here. +RUN set -eux; \ + rm -f /etc/apt/sources.list.d/cuda*.list \ + /etc/apt/sources.list.d/nvidia*.list \ + /etc/apt/sources.list.d/*cuda*.sources; \ + printf 'Acquire::Retries "5";\nAcquire::http::Pipeline-Depth "0";\nAcquire::http::No-Cache "true";\nAcquire::BrokenProxy "true";\n' \ + > /etc/apt/apt.conf.d/99-h3-resilient; \ + apt-get clean; rm -rf /var/lib/apt/lists/*; \ + apt-get update; \ + apt-get install -y --no-install-recommends ffmpeg libicu74 python3 python3-venv; \ + rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY --from=build /src/h3 /app/h3 +COPY webui/backend/pyproject.toml /app/webui/backend/pyproject.toml +RUN python3 -m venv /opt/venv \ + && /opt/venv/bin/pip install --no-cache-dir -q --upgrade pip \ + && /opt/venv/bin/pip install --no-cache-dir -q \ + "fastapi>=0.115" "uvicorn[standard]>=0.34" "pydantic-settings>=2.6" \ + python-multipart +COPY webui/backend/app /app/webui/backend/app +COPY webui/shared /app/webui/shared + +ENV PYTHONUNBUFFERED=1 \ + H3_BINARY=/app/h3 \ + H3_MODEL_DIR=/models \ + H3_DATA_DIR=/data \ + H3_SCHEMA_PATH=/app/webui/shared/options.schema.json \ + H3_PROGRESS_WEIGHTS_PATH=/app/webui/shared/progress_weights.json +EXPOSE 8000 +WORKDIR /app/webui/backend +CMD ["/opt/venv/bin/uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/docker/Dockerfile.frontend b/docker/Dockerfile.frontend new file mode 100644 index 00000000..359d77ad --- /dev/null +++ b/docker/Dockerfile.frontend @@ -0,0 +1,13 @@ +# Builds the static UI and serves it behind nginx, which also proxies /api. +FROM node:22-slim AS build +WORKDIR /src +COPY webui/frontend/package.json webui/frontend/package-lock.json* ./ +RUN npm install --no-audit --no-fund +COPY webui/shared /shared +COPY webui/frontend ./ +RUN node scripts/generate-options.mjs && npm run build + +FROM nginx:1.27-alpine AS runtime +COPY --from=build /src/dist /usr/share/nginx/html +COPY docker/nginx.conf /etc/nginx/conf.d/default.conf +EXPOSE 80 diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 00000000..cfd37c4c --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,23 @@ +server { + listen 80; + server_name _; + root /usr/share/nginx/html; + + # Generated videos are large; uploads are capped by the backend too. + client_max_body_size 1024m; + + location / { + try_files $uri /index.html; + } + + location /api/ { + proxy_pass http://api:8000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # Server-sent events: no buffering, no timeout during a long phase. + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 24h; + } +} diff --git a/docs/POSTPROCESSING.md b/docs/POSTPROCESSING.md new file mode 100644 index 00000000..6c425b02 --- /dev/null +++ b/docs/POSTPROCESSING.md @@ -0,0 +1,81 @@ +# Post-processing plugins + +After a video is generated, h3.c Studio can hand it to an external program +before publishing it. That is the whole extension point: a contract, not an +integration. + +**This repository contains no models, no weights, and no download URLs.** +Nothing is fetched at build time or at run time. Every plugin is unavailable +until you install a runtime yourself and point an environment variable at it. + +## The contract + +A plugin is an executable. The backend calls it with an argument list — never +through a shell — and waits: + +``` +$H3__CMD --input /path/to/in.mp4 --output /path/to/out.mp4 +``` + +| Outcome | What the backend does | +| --- | --- | +| exit code `0` and the output file exists | the job's video is replaced by the output | +| exit code `0` but no output file | the job fails; the generated video is kept | +| any non-zero exit code | the job fails with the last line of stderr; the generated video is kept | +| the process does not finish within an hour | the job fails with a timeout | + +The generated video is never deleted: a failed post-processing step costs you +the stage, not the render. + +Anything the plugin writes to stderr ends up in the job log, so make errors +readable in one line. + +## Registered plugins + +| Name | Environment variable | Status in this repository | +| --- | --- | --- | +| `faceswap` | `H3_FACESWAP_CMD` | unavailable — no model, no runtime | + +`GET /api/capabilities` reports the same thing at run time, with the reason, +and the UI shows it disabled. Requesting an unavailable plugin fails the job +instead of silently ignoring it. + +## Enabling one + +Installing a runtime is one configuration step, not a code change: + +```sh +# 1. Install the runtime yourself, in its own environment. +# 2. Point the variable at the executable and restart the backend. +export H3_FACESWAP_CMD=/opt/faceswap/run +``` + +With Docker, use the override file, which mounts your runtime read-only: + +```sh +export H3_FACESWAP_DIR=/opt/faceswap +export H3_FACESWAP_CMD=/opt/faceswap/run +docker compose -f docker-compose.yml -f docker-compose.faceswap.yml up +``` + +## Adding another plugin + +Add an entry to `registry()` in `webui/backend/app/postprocess.py` with a name, +a label, a description and an environment variable. Nothing else changes: the +API lists it, and the UI renders it from the API — there is no plugin name +hard-coded in the frontend. + +The stage is not specific to faces. Upscaling, frame interpolation or +watermarking fit the same `--input`/`--output` contract. + +## Licences and consent + +The face-swapping models that were evaluated for this project are licensed for +**non-commercial or research use only**. That is why they are not shipped here +and why no URL is given: checking the licence of what you install is your +responsibility, not this repository's. + +**Do not use face replacement on images or videos of real people without their +informed consent.** Depicting someone saying or doing something they did not is +harmful regardless of how good the result looks, and in many jurisdictions it +is illegal. If you cannot obtain consent, do not run the stage. diff --git a/docs/WEBUI.md b/docs/WEBUI.md new file mode 100644 index 00000000..1a917f3a --- /dev/null +++ b/docs/WEBUI.md @@ -0,0 +1,142 @@ +# h3.c Studio — the web UI + +A browser front end for `h3`. Everything the CLI accepts is here: duration, +canvas, sampler, first/last frame anchors, ordered Ref2VA references, the +memory and parity switches, and a live preview of the denoising. + +A clickable mockup of the interface is in [`docs/mockup/index.html`](mockup/index.html). + +## What you need + +- The same prerequisites as `h3` itself: an NVIDIA driver matching CUDA 13, the + CUDA toolkit, ICU, and FFmpeg/FFprobe 6.1 or newer on `PATH`. +- The MiniMax-H3 checkpoint on disk (about 465 GB). It is never copied into a + container image; it is mounted read-only. +- Python 3.12 and Node 22 for the local (non-Docker) path. +- For Docker: the NVIDIA Container Toolkit, so the container can see the GPU. + +## Docker + +```sh +cp .env.example .env +# Set H3_MODEL_DIR to your checkpoint directory, then: +docker compose up --build +``` + +To check that the container sees the GPU and the checkpoint before waiting on +a first render — it prints the device and a tensor inventory, and nothing else: + +```sh +docker compose run --rm --no-deps api /app/h3 --info -d /models +``` + +The UI is then on and the API on +. Both bind to the loopback address on purpose — see +*Security* below. + +`NVCC_ARCH` in `.env` selects the CUDA architecture `h3` is compiled for; +`sm_121` is the NVIDIA GB10. The image is built for the architecture of the +machine that builds it. + +## Running it without Docker + +```sh +# 1. Build h3 as usual. +make -j"$(nproc)" + +# 2. Backend. +python3 -m venv webui/backend/.venv +webui/backend/.venv/bin/pip install \ + "fastapi>=0.115" "uvicorn[standard]>=0.34" "pydantic-settings>=2.6" \ + python-multipart +H3_MODEL_DIR=./MiniMax-H3 \ + webui/backend/.venv/bin/uvicorn app.main:app \ + --app-dir webui/backend --host 127.0.0.1 --port 8000 + +# 3. Frontend, in another terminal. +cd webui/frontend +npm install +node scripts/generate-options.mjs +npm run dev +``` + +Open . The dev server proxies `/api` to the backend. + +## How it works + +``` +browser ── /api ──▶ FastAPI ──▶ serial queue ──▶ ./h3 (one process per job) + ▲ │ │ + └── SSE progress ────┘ └── mp4, log, previews +``` + +- **One job at a time.** A single GPU with a 27 GB DiT peak: the queue is + serial by design, and a queued job can be cancelled before it starts. +- **One `./h3` process per job.** Every CLI flag is therefore reachable, and + cancelling is a signal to the process group. The cost is about 96 seconds of + DiT load per job on a GB10. +- **Validation mirrors the engine.** The messages the browser shows before you + submit are copied verbatim from `h3.c`, so a job that the UI accepts is a job + the engine accepts. +- **Progress is weighted.** Phases are not equal: on the calibration run the + transformer load took 40.9 s and the denoising 5.1 s. Weights live in + `webui/shared/progress_weights.json`; regenerate them for your hardware with + `webui/backend/tools/calibrate_progress.py`. +- **The option inventory has one source.** `webui/shared/options.schema.json` + feeds both the backend validator and the generated TypeScript module, and a + test fails if it drifts from `main.c`. + +## Options + +The Simple tab covers prompt, duration, format, a quality preset, first/last +frame and seed. The Advanced tab exposes everything else, including the ten +`--use-slower-*` parity flags. Two CLI options are deliberately absent: +`--show` and `--zoom` are terminal graphics protocols with no meaning in a +browser — the live preview uses `--preview-dir` instead. + +Uploaded images, clips and soundtracks stay in a library and can be reused in +later jobs, as anchors or as ordered references. + +## Security + +**There is no authentication.** The backend runs `h3` with parameters supplied +by whoever can reach it, accepts file uploads, and serves generated media. +Everything binds to `127.0.0.1` by default. + +Do not expose these ports on an untrusted network. Reaching the UI from +another machine means putting something that authenticates in front of it. + +**A private overlay network.** With [Tailscale](https://tailscale.com), publish +the UI on the tailnet address instead of the loopback: only your own devices +can then connect, and the tailnet does the authenticating. + +```sh +tailscale ip -4 # e.g. 100.117.213.82 +# in .env: +H3_BIND=100.117.213.82 +docker compose up -d +``` + +Bear in mind that the container can only bind that address while Tailscale is +up; `restart: unless-stopped` retries if it is not. + +**An SSH tunnel.** No configuration at all, from the other machine: + +```sh +ssh -N -L 8080:127.0.0.1:8080 you@the-machine +``` + +Then open there. + +**Publishing on the LAN** (`H3_BIND=0.0.0.0`) gives everyone on the network the +GPU, the uploads and every generated file, with no password. If you do it +anyway, two things to know: only port 8080 needs publishing, because nginx +proxies `/api`; and a host firewall will not save you — ports published by +Docker are DNAT-ed in the `DOCKER-USER` chain, which `ufw` does not filter, so +a `ufw deny` rule has no effect on them. + +## Post-processing + +An optional stage can hand the finished video to an external program. No such +program is included, and no model is downloaded: see +[`docs/POSTPROCESSING.md`](POSTPROCESSING.md). diff --git a/docs/mockup/WIREFRAME.md b/docs/mockup/WIREFRAME.md new file mode 100644 index 00000000..8d306b03 --- /dev/null +++ b/docs/mockup/WIREFRAME.md @@ -0,0 +1,99 @@ +# Wireframe di riferimento — web UI h3.c + +Materiale approvato con R20/R21 e input del task 67 in `PLAN.md`. + +Schermata 1 — Simple (default): + +``` ++------------------------------------------------------------------------------+ +| h3.c Studio GPU: NVIDIA GB10 - 121 GiB - CUDA 13.0 * | ++------------------------------------------------------------------------------+ +| [ Simple ] [ Advanced ] Queue: 1 running | ++------------------------------------------+-----------------------------------+ +| PROMPT | QUEUE | +| +--------------------------------------+ | +-------------------------------+ | +| | A red fox walks through fresh snow. | | | > #12 "fox in snow" running | | +| +--------------------------------------+ | | denoise 7/20 [####...] | | +| | | 03:41 elapsed [x] | | +| DURATION o-------*---------- 4.46 s | +-------------------------------+ | +| 107 frames (aligned 5+17n) | | #11 "surfer" queued [x] | | +| | +-------------------------------+ | +| FORMAT [16:9] [9:16] [1:1] [4:3] [3:4] | | #10 "cube" done 00:41 > v | | +| SIZE ( ) 256 (*) 512 ( ) 768 | +-------------------------------+ | +| 512 x 512 - 0.26 / 1.03 MP | | +| | PREVIEW | +| QUALITY o-------*----------o | +-------------------------------+ | +| Draft Balanced Reference | | | | +| steps 20 - layers 45 - reuse 2 | | [ video player ] | | +| | | | | +| FIRST FRAME LAST FRAME | +-------------------------------+ | +| +-------------+ +-------------+ | #10 - 512x512 - 22f - seed 42 | +| | drop image | | drop image | | [ Reuse settings ] [ Download ] | +| +-------------+ +-------------+ | | +| | | +| SEED [ 42 ] [ random ] | | +| | | +| [ Generate video ] | | ++------------------------------------------+-----------------------------------+ +``` + +Schermata 2 — Advanced (stessa colonna destra, pannello sinistro a sezioni): + +``` ++------------------------------------------+ +| [ Simple ] [ Advanced ] | ++------------------------------------------+ +| v OUTPUT | +| width [ 512 ] height [ 512 ] (x32) | +| internal canvas [x] custom | +| render-width [384] render-height[384]| +| output file [ outputs/fox.mp4 ] | +| [ ] no mp4 (-o '') | +| [ ] write frames frames-dir [ ... ] | ++------------------------------------------+ +| v DURATION | +| (*) seconds [ 4.5 ] ( ) frames [107] | +| -> aligned 107 frames = 4.458 s @24fps | ++------------------------------------------+ +| v SAMPLER | +| steps [ 20 ] (2..1000) | +| layers [ 45 ] (35..50) | +| (*) reuse [ 2 ] (1..3) | +| ( ) core-reuse [ 4 ] (1..6) | +| ^ mutually exclusive | +| [ ] token-reduction | ++------------------------------------------+ +| v MEMORY / BACKEND | +| [ ] ssd-streaming 27.1 GB -> 1.6 GB, | +| slower | +| [ ] use-int8-row-fc2 (no-op on CUDA) | +| [ ] use-reference-rope | ++------------------------------------------+ +| > PARITY / DEBUG FLAGS (10) | +| collapsed: --use-slower-* | +| [ ] profile | ++------------------------------------------+ +| > REFERENCES (3) | ++------------------------------------------+ +| [ Generate video ] | ++------------------------------------------+ +``` + +Schermata 3 — References (sezione espansa, lista ordinata, max 12): + +``` ++--------------------------------------------------------------+ +| REFERENCES (Ref2VA) order is significant | +| ref-image-size: (*) match ( ) max | ++--------------------------------------------------------------+ +| 1 [img] fox.png 720x720 [^][v][x] | +| 2 [vid] clip.mp4 audio: keep (--ref-video) [^][v][x] | +| 3 [vid] silent.mp4 audio: drop (--ref-silent) [^][v][x] | +| 4 [vid+a] scene.mp4 + music.wav [^][v][x] | +| 5 [aud] music.wav 6.2 s [^][v][x] | ++--------------------------------------------------------------+ +| [ + image ] [ + video ] [ + video+audio ] [ + audio ] | +| rules: audio 2-15 s, max 3 audio, total <= 15 s, audio only | +| alongside an image or a video reference | ++--------------------------------------------------------------+ +``` diff --git a/docs/mockup/index.html b/docs/mockup/index.html new file mode 100644 index 00000000..f58c0aea --- /dev/null +++ b/docs/mockup/index.html @@ -0,0 +1,453 @@ + + + + + +h3.c Studio — UI mockup + + + +
+

h3.c Studio

+ GPU NVIDIA GB10 · 121.7 GiB · CUDA 13.0 + checkpoint ready · FL2VA + Ref2VA + + queue 1 running · 1 queued +
+ +
+
+
+ + + +
+ + +
+
+ + +
+ +
+ + +
4.5 s requested → rounded up to 124 frames = 5.167 s at 24 fps + (legal shapes are 5 + 17n, 22…362 frames). The two flags are mutually exclusive.
+
+ +
+ +
+ 16:91:1 + 9:164:33:4 +
+
+ 256²512² + 768²1344×768768×1344 +
+
512 × 512 = 0.26 MP of the 1.03 MP limit (768 × 1344). + Both sides must be multiples of 32 and at least 32.
+
+ +
+ +
+ DraftBalanced + Reference +
+
Balanced = --steps 20 --layers 45 --reuse 2 --token-reduction. + Estimated 6 min on this GPU. Change any of it in Advanced.
+
+ +
+ +
+ Decode one frame after every denoising step — adds a preview VAE load + phase and one decode per step.
+
+ +
+
+ +
drop an image
choose from library
+
+
+ +
drop an image
choose from library
+
+
+
Frame anchors select the FL2VA path and + cannot be combined with ordered references. Clear the 5 references to enable them.
+ +
+ +
random
+
+ + +
Command this will run +
./h3 -d $H3_MODEL_DIR -p "A red fox walks through fresh snow…" --width 512 --height 512 --frames 124 --steps 20 --layers 45 --reuse 2 --token-reduction --seed 42 --preview-dir data/jobs/13/preview -o data/jobs/13/out.mp4
+
+ + + + + + +
+ + +
+
+

Queue

+
+
running + #13 fox in snowcancel
+ +
+
41 % · denoise 7/20 · elapsed 03:41 · remaining ~05:20
+
+
+
queued + #14 surfer inside a waveremove
+
512×512 · 124f · steps 20 · position 1 of 1
+
+
+
failed + #12 portrait 1400×800dismiss
+
h3: canvas exceeds the released 768*1344 pixel limit
+
+
+
+
done + #11 red cube on whitedelete
+
512×512 · 22f · 0.917 s · seed 42 · 00:41
+
+
+
One job runs at a time: a single GPU, 27 GB DiT peak.
+
+ +
+

Library

+
+
+
fox.png · 720×720
+
portrait.jpg · 1024×1536
+
clip.mp4 · 4.2 s
+
music.wav · 6.2 s
+
+
Uploaded assets stay selectable for later jobs as + first/last frame or as ordered references.
+
+
+ +
+

Gallery

+
+
+
#11 cube · 00:41
+
#09 surfer · 06:12
+
#08 hummingbird · 05:48
+
#07 fox 256² · 01:03
+
+
+
+
+
+ + + + diff --git a/docs/mockup/v2.html b/docs/mockup/v2.html new file mode 100644 index 00000000..e87b1d01 --- /dev/null +++ b/docs/mockup/v2.html @@ -0,0 +1,625 @@ + + + + + +h3.c studio + + + + + + + +
+ h3.c studio + light / dark + show a render + NVIDIA GB10 · ready +
+ +
+ +
+

What should the video show?

+ +
+ Try: + + + +
+ +
+
How long --seconds
+
+ 00:04.5 + seconds · 24 fps +
+ +

Videos come in fixed lengths. The nearest to 4.5 s is + 4.5 s — 107 frames. ≈ 6 min to make

+
+ +
+
Shape --width --height
+
+ + + + +
+

Bigger pictures take longer — the times above are for the + quality picked below. The largest this model was released for is + 1344 × 768.

+
+ +
+
Quality --steps --layers --reuse
+
+ + + +
+
+ +
+
Start and end --first-frame --last-frame
+
+ + +
+

Optional. Add a photo and the video begins — or ends — there.

+
+ +
+
Variation --seed
+
+ 42 + + Same settings and same variation give + the same video, every time. +
+
+ +
+ That size is larger than the model can make. +

The largest is 1344 × 768. Pick a smaller shape, or use More sizes + to set one by hand.

+
+ what h3 reported + 
h3: canvas exceeds the released 768*1344 pixel limit
+
+
+ +
+ + about 6 min on this machine · nothing else is queued +
+ +
+ Fine-tune quality, length and size, one control at a time +
+
+
+ Detail passes --steps +

How many times the picture is refined. More passes, more detail.

+ + each pass adds ≈ 12 s +
+
+ Model depth --layers +

How much of the model runs. Less is faster and slightly looser.

+ + 45 instead of 50 saves ≈ 40 s +
+
+
+
+ How often it redraws --reuse +

Redrawing at every pass is closest to the reference. Less often is faster, and the framing can shift.

+ + saves ≈ 2 min at this length +
+
+ Work smaller, then enlarge --render-width +

Draw at a smaller size and scale up. Much faster, less fine detail.

+ + saves ≈ 3 min +
+
+
+ + Watch it being made --preview-dir + Shows a picture after every pass. Adds about 15 s. +
+
+ + Pair up detail while drawing --token-reduction + Faster, but the composition can drift. Leave off at small sizes. +
+
+
+ +
+ Expert every h3 flag, named as it is on the command line +
+
+
Text and variation --prompt --seed +

The prompt is sent as written. The same prompt, settings and seed + produce the same video on the same build.

+
References --ref-image --ref-video --ref-silent-video --ref-video-audio --ref-audio --ref-image-size +

Ordered Ref2VA references: at most 12 — 9 images, 3 videos, 3 audio + inputs. Audio needs a picture or a clip beside it, lasts 2–15 s, and + cannot be combined with a start or end frame.

+
Duration and canvas --frames --seconds --width --height --render-width --render-height +

Frames round up to 5 + 17n, 22…362. Sides are multiples of 32 and the + area stays under 768 × 1344.

+
+
Sampler --steps --layers --reuse --core-reuse --token-reduction +

Steps 2…1000, layers 35…50, reuse 1…3, core reuse 1…6. Reuse and core + reuse cannot both exceed 1.

+
+
SSD streaming --ssd-streaming + 27.06 GB → 1.63 GB of GPU memory, about 38 % slower.
+
int8 row FC2 --use-int8-row-fc2 + Metal/M5 only — measured as a no-op on CUDA.
+
Reference RoPE --use-reference-rope + Restores the released 256 × 256 grid for parity checks.
+
Write frames --frames-dir + Every final frame as a PPM file.
+
Profile phases --profile + Per-phase wall time and peak memory in the log.
+
Face replacement faceswap + Unavailable: no model and no runtime installed.
+
+
Parity flags 10 +

Force close-reference implementations, slower by design: + --use-slower-bf16-mlp --use-slower-bf16-qkv + --use-slower-bf16-attention-output --use-slower-row-major-attention-output + --use-slower-unfused-int8-inputs --use-slower-unfused-qkv-rope + --use-slower-scalar-qkv-rms --use-slower-uncached-int8-scales + --use-slower-dynamic-fc1-k --use-slower-grouped-quantizer

+
Set by the server --model-dir --output --info +

Not exposed: --show and + --zoom draw in a terminal, which a browser has none of.

+
+
+
+ + +
+
+ “A red fox walks through fresh snow in a pine forest” + 4.5 s · widescreen · variation 42 + +
+ +
+ +
+ + + + + + + + + + + + + + + + + pass 7 of 20 +
+
+ +
+
Painting the picture — pass 7 of 20
+
+ elapsed 00:03:41 + about 5 min left + 41 % done +
+
denoise 7/20 · 864×480 · seed 42
+
+
+
+ +
+

Takes

+
+ + + + +
+
+ + + + diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 00000000..9108afa1 --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,55 @@ +import js from "@eslint/js"; +import reactHooks from "eslint-plugin-react-hooks"; +import tseslint from "typescript-eslint"; + +export default [ + { + ignores: [ + "node_modules/**", + "**/node_modules/**", + "MiniMax-H3/**", + "outputs/**", + "logs/**", + "misc/**", + "webui/frontend/dist/**", + "webui/frontend/src/generated/**", + "**/.venv/**", + "**/__pycache__/**", + "**/._*", + ], + }, + js.configs.recommended, + ...tseslint.configs.recommended, + { + files: ["**/*.mjs", "**/*.js"], + languageOptions: { + ecmaVersion: 2023, + sourceType: "module", + globals: { console: "readonly", process: "readonly" }, + }, + }, + { + files: ["webui/frontend/**/*.{ts,tsx}"], + plugins: { "react-hooks": reactHooks }, + languageOptions: { + ecmaVersion: 2023, + sourceType: "module", + globals: { + window: "readonly", + document: "readonly", + fetch: "readonly", + FormData: "readonly", + File: "readonly", + EventSource: "readonly", + MessageEvent: "readonly", + HTMLInputElement: "readonly", + setTimeout: "readonly", + clearTimeout: "readonly", + console: "readonly", + }, + }, + rules: { + ...reactHooks.configs.recommended.rules, + }, + }, +]; diff --git a/main.c b/main.c index a15c5e47..da07287e 100644 --- a/main.c +++ b/main.c @@ -55,6 +55,7 @@ static void usage(const char *program) { " --ref-video-audio VIDEO AUDIO Append video + soundtrack\n" " --ref-audio PATH Append an ordered standalone audio clip\n" " --frames-dir PATH Write generated frames as PPM files\n" + " --preview-dir PATH Write a PPM preview after every denoising step\n" " --show Display a frame after every denoising step (M5)\n" " --zoom N Terminal image zoom (default: 2 for Retina)\n" " --profile Print per-phase Metal timing and allocation data\n" @@ -153,8 +154,25 @@ typedef struct { int display_failed; const char *frames_dir; int frame_write_failed; + const char *preview_dir; + int preview_write_failed; } cli_state; +/* Write one RGB24 frame as a binary PPM. */ +static int cli_write_ppm(const char *path, const h3_frame *frame) { + FILE *output = fopen(path, "wb"); + if (!output) return 0; + int ok = fprintf(output, "P6\n%d %d\n255\n", frame->width, + frame->height) >= 0; + size_t row_bytes = (size_t)frame->width * 3; + for (int row = 0; ok && row < frame->height; row++) { + if (fwrite(frame->rgb + (size_t)row * frame->stride, 1, row_bytes, + output) != row_bytes) ok = 0; + } + if (fclose(output) != 0) ok = 0; + return ok; +} + static int cli_progress(const char *phase, int completed, int total, void *opaque) { cli_state *state = opaque; @@ -176,6 +194,23 @@ static int cli_progress(const char *phase, int completed, int total, static int cli_frame(const h3_frame *frame, void *opaque) { cli_state *state = opaque; int preview = frame->denoise_step >= 0; + if (preview && state->preview_dir && !state->preview_write_failed) { + /* Write to a scratch name and rename, so a reader never sees a + * partial preview. */ + char staging[1024]; + char path[1024]; + int staged = snprintf(staging, sizeof(staging), "%s/.step.ppm", + state->preview_dir); + int length = snprintf(path, sizeof(path), "%s/step-%04d.ppm", + state->preview_dir, frame->denoise_step); + if (staged <= 0 || (size_t)staged >= sizeof(staging) || + length <= 0 || (size_t)length >= sizeof(path) || + !cli_write_ppm(staging, frame) || rename(staging, path) != 0) { + fprintf(stderr, "h3: cannot write preview %d to %s\n", + frame->denoise_step, state->preview_dir); + state->preview_write_failed = 1; + } + } if (!preview && state->frames_dir && !state->frame_write_failed) { char path[1024]; int length = snprintf(path, sizeof(path), "%s/frame-%04d.ppm", @@ -252,7 +287,7 @@ int main(int argc, char **argv) { OPT_SEED, OPT_FIRST, OPT_LAST, OPT_REF_IMAGE, OPT_REF_IMAGE_SIZE, OPT_REF_VIDEO, OPT_REF_SILENT_VIDEO, OPT_REF_VIDEO_AUDIO, - OPT_REF_AUDIO, OPT_FRAMES_DIR, OPT_SHOW, OPT_ZOOM, + OPT_REF_AUDIO, OPT_FRAMES_DIR, OPT_PREVIEW_DIR, OPT_SHOW, OPT_ZOOM, OPT_PROFILE, OPT_INFO }; static const struct option options[] = { {"model-dir", required_argument, NULL, 'd'}, @@ -302,6 +337,7 @@ int main(int argc, char **argv) { {"ref-video-audio", required_argument, NULL, OPT_REF_VIDEO_AUDIO}, {"ref-audio", required_argument, NULL, OPT_REF_AUDIO}, {"frames-dir", required_argument, NULL, OPT_FRAMES_DIR}, + {"preview-dir", required_argument, NULL, OPT_PREVIEW_DIR}, {"show", no_argument, NULL, OPT_SHOW}, {"zoom", required_argument, NULL, OPT_ZOOM}, {"profile", no_argument, NULL, OPT_PROFILE}, @@ -315,7 +351,7 @@ int main(int argc, char **argv) { h3_params params = H3_PARAMS_DEFAULT; h3_reference references[12]; size_t reference_count = 0; - cli_state cli = {{0}, 0, -1, -1, H3_TERM_NONE, 0, NULL, 0}; + cli_state cli = {{0}, 0, -1, -1, H3_TERM_NONE, 0, NULL, 0, NULL, 0}; int show = 0; int profile = 0; int info = 0; @@ -454,6 +490,7 @@ int main(int argc, char **argv) { break; } case OPT_FRAMES_DIR: cli.frames_dir = optarg; break; + case OPT_PREVIEW_DIR: cli.preview_dir = optarg; break; case OPT_SHOW: show = 1; break; case OPT_ZOOM: if (!h3_terminal_set_zoom(parse_int(optarg, "zoom"))) { @@ -488,6 +525,12 @@ int main(int argc, char **argv) { cli.frames_dir, strerror(errno)); return 1; } + if (cli.preview_dir && mkdir(cli.preview_dir, 0755) != 0 && + errno != EEXIST) { + fprintf(stderr, "h3: cannot create preview directory %s: %s\n", + cli.preview_dir, strerror(errno)); + return 1; + } if (profile) setenv("H3_PROFILE", "1", 1); h3_ctx *ctx = h3_load_dir(model_dir); if (!ctx) { @@ -500,6 +543,10 @@ int main(int argc, char **argv) { params.on_progress = cli_progress; params.callback_opaque = &cli; if (cli.frames_dir) params.on_frame = cli_frame; + if (cli.preview_dir) { + params.on_frame = cli_frame; + params.preview_denoise = 1; + } if (show) { cli.terminal = h3_terminal_detect(); if (cli.terminal == H3_TERM_NONE) { @@ -523,6 +570,8 @@ int main(int argc, char **argv) { if (output && *output) fprintf(stderr, "h3: wrote %s\n", output); if (cli.frames_dir) fprintf(stderr, "h3: wrote frames to %s\n", cli.frames_dir); + if (cli.preview_dir) + fprintf(stderr, "h3: wrote previews to %s\n", cli.preview_dir); } else if (!info) { int cli_status = h3_cli_run(ctx, model_dir, ¶ms, show, seed_given); h3_free(ctx); diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..d8970cdb --- /dev/null +++ b/package-lock.json @@ -0,0 +1,2113 @@ +{ + "name": "h3c-dev-tools", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "h3c-dev-tools", + "devDependencies": { + "eslint": "^9.39.0", + "eslint-formatter-compact": "^9.0.1", + "eslint-plugin-react-hooks": "^7.1.1", + "playwright": "^1.57.0", + "typescript": "^6.0.3", + "typescript-eslint": "^8.68.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.8.tgz", + "integrity": "sha512-gZbepsdh3WDtgZKWL+vTPh71LSBrm/Y4/QDZBVCcYfmeTEEuoOYwlSy+G1StfJg+/Zy550u/3TATbm7qDbbMtg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.8.tgz", + "integrity": "sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.8" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/template": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.8.tgz", + "integrity": "sha512-I5z7H3bf/41ktsNVLtpN0wAa336HkqIHQ5BuPLEhTkt1jVSyZpeNKIzTgEWmlxjdg81R0IgUCcaE+Ok3NvrfZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.8", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.8", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.8", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", + "integrity": "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", + "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.21.2.tgz", + "integrity": "sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^2.1.7", + "debug": "^4.3.1", + "minimatch": "^3.1.5" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.4.2", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.4.2.tgz", + "integrity": "sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/core": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-0.17.0.tgz", + "integrity": "sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-3.3.6.tgz", + "integrity": "sha512-l2Ul9PrHsPCKcEY/ac7VgFj9D80C7S68sOKc618SyHDPK36s1XcFebXY0iTzUVn4Yq+YbwvSnDmCz9yxjX+QrA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.14.0", + "debug": "^4.3.2", + "espree": "^10.0.1", + "globals": "^14.0.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.3.0", + "minimatch": "^3.1.5", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/js": { + "version": "9.39.5", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.5.tgz", + "integrity": "sha512-QywQuszQh77pIXCsq998c8hbhSTI/azTty1Z6N53dmAudKHhy573j3yvRLsX2BSp8YpLtoCEG8E9DJe+8zUh4A==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + } + }, + "node_modules/@eslint/object-schema": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-2.1.7.tgz", + "integrity": "sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.4.1.tgz", + "integrity": "sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^0.17.0", + "levn": "^0.4.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "8.68.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.68.0.tgz", + "integrity": "sha512-WASHDpCm6qO5jj9g1a+8NiW5+GCkAyLReR56/4VruYmNgfUmqpxOfZ2Yfb8xGfJPWv5Qi6LSD8sXdces3vbp/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/regexpp": "^4.12.2", + "@typescript-eslint/scope-manager": "8.68.0", + "@typescript-eslint/type-utils": "8.68.0", + "@typescript-eslint/utils": "8.68.0", + "@typescript-eslint/visitor-keys": "8.68.0", + "ignore": "^7.0.5", + "natural-compare": "^1.4.0", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^8.68.0", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.6.tgz", + "integrity": "sha512-BAg6QkE8W+TuQLrrw0Ugr7HegXduRuuj8/ti2kSOc+jz1dmx8/WNcjr6XGnq5YpDWxFwwaavqD0+jIUOKelTsw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "8.68.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.68.0.tgz", + "integrity": "sha512-fHq2VC1kpyYfvEcbiMjOpySY4WS7voEp89yAThrHRX5sm9j2lzYppCb2umFMEed4fWcyeLjHxrz0mpjNBaBxMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/scope-manager": "8.68.0", + "@typescript-eslint/types": "8.68.0", + "@typescript-eslint/typescript-estree": "8.68.0", + "@typescript-eslint/visitor-keys": "8.68.0", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/project-service": { + "version": "8.68.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.68.0.tgz", + "integrity": "sha512-5GQtWZCXFcFYux955pvoS02WLc49pXNlvIxocKjS0clvwo3in1RdlzVKyiqQH9vE5AKWFLTaUgeQkOrTS+0Qxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.68.0", + "@typescript-eslint/types": "^8.68.0", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.68.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.68.0.tgz", + "integrity": "sha512-T5eXpcaJNg8bhjHJ8Rjp68Vq/QBteYtTKY8TZqVNPaUbuz0f6jI9t6aDkylwvalpAB9XTTFeFOjrjXAZ3YvmVA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.68.0", + "@typescript-eslint/visitor-keys": "8.68.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.68.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.68.0.tgz", + "integrity": "sha512-F7zrGQfiJHojPwi8vhxZQC1tWtJzvL74cK/nqri2lk8YUXvYaYwl263xOJ69jDWPUk1hmcdoayFwk9lX09npVw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "8.68.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.68.0.tgz", + "integrity": "sha512-X77zqoY1EjeWGs/0JNxeaMfp5C5lIz4Tw8y66F1Ne8Faq6g424sBNYM6xBAqElfGZPLpWS+CZAp0DXyKDzWiHg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.68.0", + "@typescript-eslint/typescript-estree": "8.68.0", + "@typescript-eslint/utils": "8.68.0", + "debug": "^4.4.3", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/types": { + "version": "8.68.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.68.0.tgz", + "integrity": "sha512-9RnpsGJjrAllCMefGVVsImJM24YurhC0Q1h4UbvivtvOqXmR/vEJge2OoE++z9m6hyg8T1Q8t5SNT6tHSbrxcg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "8.68.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.68.0.tgz", + "integrity": "sha512-OKKsD0tYmoNiU5PW2zehO1yO56jYOm1ShYlxon/Z0SJNidAkdVg86eg9ruRuoXf8xfnuWZGbwDsStkoXbZtIIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/project-service": "8.68.0", + "@typescript-eslint/tsconfig-utils": "8.68.0", + "@typescript-eslint/types": "8.68.0", + "@typescript-eslint/visitor-keys": "8.68.0", + "debug": "^4.4.3", + "minimatch": "^10.2.2", + "semver": "^7.7.3", + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "8.68.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.68.0.tgz", + "integrity": "sha512-PB5gJMMOg0Q5P1tsgWtEAqQacJXq0qEqRHDX/YJ4FaTMLfZPpHB3gjl2EJuiZyPABxmj4ZQYiY9m1bdAJ5y7tQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.9.1", + "@typescript-eslint/scope-manager": "8.68.0", + "@typescript-eslint/types": "8.68.0", + "@typescript-eslint/typescript-estree": "8.68.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.68.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.68.0.tgz", + "integrity": "sha512-YR65gGdGvTUAWLldC3xLOvOzamdGzB4A5/N8rehEaHs3Zvoe39BhgY+u0SPch1OvrVTfLcc55wsSgK2NcnTS/A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.68.0", + "eslint-visitor-keys": "^5.0.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/acorn": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.19", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.19.tgz", + "integrity": "sha512-Grytf1xOxOEMTGRwx6rLGKkTabd4vMg3VrKdj/7joCmV0qgh4QwMMO6xh34YEXQqirAuUdgQGa5orJQQ+69RBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.18.tgz", + "integrity": "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/browserslist": { + "version": "4.28.8", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.8.tgz", + "integrity": "sha512-V2NpofLblG64mfOtSgDhOJESZEGogzDMBv/q+W6oc4LXWP/q75eOXoOaaOu1EOadB9U4Bwx/e0yzbvwKH8zalA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.11.12", + "caniuse-lite": "^1.0.30001809", + "electron-to-chromium": "^1.5.402", + "node-releases": "^2.0.53", + "update-browserslist-db": "^1.3.0" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001810", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz", + "integrity": "sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.414", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.414.tgz", + "integrity": "sha512-aYlviXiaXBbzvKgyALpcMmqa3Np3sDr0XnZbEG62n2UpZFbEcjQ4EEMOLGzVPhwVnwTz0lvKY+GcARbunuHekw==", + "dev": true, + "license": "ISC" + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "9.39.5", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.5.tgz", + "integrity": "sha512-DgZS62aPLXKlnxILS/AYCoRvHaZeXceIzlXPkkGGzJWSow1aEk0lbTlxUSlyjC8jcaKxAdOnTDz+o1JFSBsyjw==", + "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.1", + "@eslint/config-array": "^0.21.2", + "@eslint/config-helpers": "^0.4.2", + "@eslint/core": "^0.17.0", + "@eslint/eslintrc": "^3.3.6", + "@eslint/js": "9.39.5", + "@eslint/plugin-kit": "^0.4.1", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^8.4.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "esquery": "^1.5.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^8.0.0", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.5", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-formatter-compact": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/eslint-formatter-compact/-/eslint-formatter-compact-9.0.1.tgz", + "integrity": "sha512-mBAti2tb403dQGMyilQTYHU80stem3N7jdtKW+tmn5gj3JNF7ki0rgCZtJFw4iMayTH862FTUIqCdp70ug0S0Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + } + }, + "node_modules/eslint-plugin-react-hooks": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-7.1.1.tgz", + "integrity": "sha512-f2I7Gw6JbvCexzIInuSbZpfdQ44D7iqdWX01FKLvrPgqxoE7oMj8clOfto8U6vYiz4yd5oKu39rRSVOe1zRu0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.24.4", + "@babel/parser": "^7.24.4", + "hermes-parser": "^0.25.1", + "zod": "^3.25.0 || ^4.0.0", + "zod-validation-error": "^3.5.0 || ^4.0.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0 || ^10.0.0" + } + }, + "node_modules/eslint-scope": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-8.4.0.tgz", + "integrity": "sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/file-entry-cache": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", + "integrity": "sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^4.0.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-4.0.1.tgz", + "integrity": "sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/flatted": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", + "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", + "dev": true, + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", + "integrity": "sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/hermes-estree": { + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.25.1.tgz", + "integrity": "sha512-0wUoCcLp+5Ev5pDW2OriHC2MJCbwLwuRx+gAqMTOkGKJJiBCLjtrvy4PWUGn6MIVefecRpzoOZ/UV6iGdOr+Cw==", + "dev": true, + "license": "MIT" + }, + "node_modules/hermes-parser": { + "version": "0.25.1", + "resolved": "https://registry.npmjs.org/hermes-parser/-/hermes-parser-0.25.1.tgz", + "integrity": "sha512-6pEjquH3rqaI6cYAXYPcz9MS4rY6R4ngRgrgfDshRptUZIc3lw0MCIJIGDj9++mfySOuPTHB4nrSW99BCvOPIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hermes-estree": "0.25.1" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", + "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-releases": { + "version": "2.0.53", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.53.tgz", + "integrity": "sha512-D9UOmYG3UH1V+ENW56t5QXBwJw1YEY18ruVeus89Rw+SyIgjPkCO84bRzO3uNIYosJbNwiabWVn48o3uJLjxFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/ts-api-utils": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", + "integrity": "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12" + }, + "peerDependencies": { + "typescript": ">=4.8.4" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/typescript-eslint": { + "version": "8.68.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.68.0.tgz", + "integrity": "sha512-MHy0Y0ynqeEbx/S45+i/bBssdy3X6KNBfmJAP35GrgtNxu2TQ5K5xsFDhAnmsq1jvpdoZOPG1LGtJo0HWqYCrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/eslint-plugin": "8.68.0", + "@typescript-eslint/parser": "8.68.0", + "@typescript-eslint/typescript-estree": "8.68.0", + "@typescript-eslint/utils": "8.68.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.1.tgz", + "integrity": "sha512-ZZ61DsRsOnakl74HAmp3oSN4aXUmEWXf+i/yv0h7tIBfICc3VdrFErQKUUKPgu3AMsTUMbcongALEN4l6GSUrQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-validation-error": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/zod-validation-error/-/zod-validation-error-4.0.2.tgz", + "integrity": "sha512-Q6/nZLe6jxuU80qb/4uJ4t5v2VEZ44lzQjPDhYJNztRQ4wyWc6VF3D3Kb/fAuPetZQnhS3hnajCf9CsWesghLQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 00000000..7777f9a3 --- /dev/null +++ b/package.json @@ -0,0 +1,14 @@ +{ + "name": "h3c-dev-tools", + "private": true, + "type": "module", + "description": "Repo-level dev tooling: Playwright for scripts/snapshot_ui.mjs and ESLint for the verify.sh static gate.", + "devDependencies": { + "eslint": "^9.39.0", + "eslint-formatter-compact": "^9.0.1", + "eslint-plugin-react-hooks": "^7.1.1", + "playwright": "^1.57.0", + "typescript": "^6.0.3", + "typescript-eslint": "^8.68.0" + } +} diff --git a/scripts/benchmark_sdpa.py b/scripts/benchmark_sdpa.py new file mode 100644 index 00000000..33a0bee1 --- /dev/null +++ b/scripts/benchmark_sdpa.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Benchmark CUDA SDPA backends on the MiniMax-H3 attention shape.""" + +import argparse +import statistics + +import torch +from torch.nn.attention import SDPBackend, sdpa_kernel + + +BACKENDS = ( + ("flash", SDPBackend.FLASH_ATTENTION), + ("cudnn", SDPBackend.CUDNN_ATTENTION), + ("efficient", SDPBackend.EFFICIENT_ATTENTION), +) + + +def run_backend(name, backend, query, key, value, runs): + torch.cuda.reset_peak_memory_stats() + with sdpa_kernel(backend): + output = torch.nn.functional.scaled_dot_product_attention( + query, key, value + ) + torch.cuda.synchronize() + times = [] + for _ in range(runs): + begin = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + begin.record() + output = torch.nn.functional.scaled_dot_product_attention( + query, key, value + ) + end.record() + end.synchronize() + times.append(begin.elapsed_time(end) / 1000.0) + peak = torch.cuda.max_memory_allocated() / (1024.0 ** 3) + print( + f"backend={name} median_seconds={statistics.median(times):.6f} " + f"runs={','.join(f'{value:.6f}' for value in times)} " + f"peak_gib={peak:.3f}" + ) + return output + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--sequence", type=int, default=18816) + parser.add_argument("--heads", type=int, default=56) + parser.add_argument("--head-dim", type=int, default=128) + parser.add_argument("--runs", type=int, default=3) + args = parser.parse_args() + + torch.manual_seed(42) + shape = (1, args.heads, args.sequence, args.head_dim) + query = torch.randn(shape, device="cuda", dtype=torch.bfloat16) * 0.1 + key = torch.randn_like(query) * 0.1 + value = torch.randn_like(query) * 0.1 + print( + f"torch={torch.__version__} cuda={torch.version.cuda} " + f"device={torch.cuda.get_device_name()} capability=" + f"{torch.cuda.get_device_capability()} shape={shape} dtype=bf16" + ) + + outputs = {} + for name, backend in BACKENDS: + try: + outputs[name] = run_backend( + name, backend, query, key, value, args.runs + ) + except RuntimeError as error: + print(f"backend={name} unavailable={error}") + + if "flash" in outputs and "cudnn" in outputs: + difference = ( + outputs["flash"].float() - outputs["cudnn"].float() + ).abs() + print( + f"flash_vs_cudnn max_abs={difference.max().item():.9g} " + f"mean_abs={difference.mean().item():.9g}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/snapshot_ui.mjs b/scripts/snapshot_ui.mjs new file mode 100755 index 00000000..b728ad34 --- /dev/null +++ b/scripts/snapshot_ui.mjs @@ -0,0 +1,53 @@ +#!/usr/bin/env node +// snapshot_ui.mjs — screenshot multi-viewport + cattura errori console. +// Uso: node scripts/snapshot_ui.mjs [url] [nome] +// Prerequisito: npm i -D playwright && npx playwright install chromium +// Esito: ultima riga "UI: PASS" oppure "UI: FAIL" + exit code. +// +// Lo screenshot cattura rotture invisibili nei log (layout, overflow, contrasto). +// Gli errori console catturano rotture invisibili nello screenshot. +// Servono entrambi: nessuno dei due da solo è una verifica sufficiente. + +import { chromium } from "playwright"; +import { mkdirSync } from "node:fs"; + +const [url = "http://localhost:5173", name = "snapshot"] = process.argv.slice(2); +const OUT = "logs/agent"; +mkdirSync(OUT, { recursive: true }); + +const VIEWPORTS = { + desktop: { width: 1440, height: 900 }, + mobile: { width: 390, height: 844 }, +}; + +const problems = []; +const browser = await chromium.launch(); + +for (const [label, viewport] of Object.entries(VIEWPORTS)) { + const page = await browser.newPage({ viewport }); + page.on("console", (m) => { + if (m.type() === "error") problems.push(`[${label}] CONSOLE ${m.text()}`); + }); + page.on("pageerror", (e) => problems.push(`[${label}] PAGEERROR ${e.message}`)); + page.on("response", (r) => { + if (r.status() >= 400) problems.push(`[${label}] HTTP ${r.status()} ${r.url()}`); + }); + + try { + await page.goto(url, { waitUntil: "networkidle", timeout: 30_000 }); + await page.screenshot({ path: `${OUT}/${name}_${label}.png`, fullPage: true }); + console.log(` screenshot: ${OUT}/${name}_${label}.png`); + } catch (e) { + problems.push(`[${label}] NAVIGATION ${e.message}`); + } + await page.close(); +} + +await browser.close(); + +if (problems.length) { + console.log(problems.slice(0, 15).join("\n")); + console.log("UI: FAIL"); + process.exit(1); +} +console.log("UI: PASS"); diff --git a/scripts/verify.sh b/scripts/verify.sh new file mode 100755 index 00000000..87f20355 --- /dev/null +++ b/scripts/verify.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# verify.sh — gate unico di validazione per agenti di codice. +# Uso: ./scripts/verify.sh [all|static|unit|integration] +# Esito: ultima riga "VERIFY: PASS" oppure "VERIFY: FAIL" + exit code. +# +# Rileva automaticamente lo stack presente. Output volutamente compatto: +# il consumatore è un agente, non un umano — ogni riga superflua è token bruciati. + +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 + +MODE="${1:-all}" +FAIL=0 +mkdir -p logs/agent + +step() { printf '\n── %s\n' "$1"; } +has() { command -v "$1" >/dev/null 2>&1; } +run() { echo " \$ $*"; "$@" || FAIL=1; } + +# Web UI: stack Python isolato nel proprio venv, non su PATH. +WEBUI_VENV="webui/backend/.venv/bin" +[[ -x "$WEBUI_VENV/pytest" ]] && WEBUI=1 || WEBUI=0 + +# Rilevamento stack +PY=0; JS=0 +[[ -f pyproject.toml || -f setup.py || -f requirements.txt || -n "$(ls -1 ./*.py 2>/dev/null)" ]] && PY=1 +[[ -f package.json ]] && JS=1 + +# ─────────────────────────────── ANALISI STATICA ─────────────────────────────── +if [[ "$MODE" == "static" || "$MODE" == "all" ]]; then + step "Analisi statica" + if [[ $PY -eq 1 ]]; then + has ruff && run ruff check . --output-format=concise + has mypy && run mypy . --no-error-summary --pretty=False + fi + if [[ $JS -eq 1 ]]; then + has npx && [[ -f tsconfig.json ]] && run npx --no-install tsc --noEmit --pretty false + has npx && run npx --no-install eslint . --format=compact + fi + if [[ $WEBUI -eq 1 ]]; then + run "$WEBUI_VENV/ruff" check webui + fi + # Debug print dimenticati. + # Il pattern è spezzato ("AGENT""DBG") così questo script non matcha se stesso; + # esclusi anche .md e scripts/ per non segnalare la documentazione del pack. + DBG=$(grep -rn --exclude-dir={.git,node_modules,logs,.venv,scripts,MiniMax-H3} \ + --exclude="*.md" "AGENT""DBG|" . 2>/dev/null | head -5) + if [[ -n "$DBG" ]]; then + echo " ! debug print temporanei ancora presenti — rimuovere prima di chiudere il task" + echo "$DBG" | sed 's/^/ /' + FAIL=1 + fi +fi + +# ──────────────────────────────── TEST UNITARI ──────────────────────────────── +if [[ "$MODE" == "unit" || "$MODE" == "all" ]]; then + step "Test unitari" + if [[ "$(uname -s)" == "Linux" && -f Makefile ]] && + grep -q '^cuda-runtime-test:' Makefile; then + run make PLATFORM=Linux host-portable-test + run make PLATFORM=Linux tokenizer-portable-test + if [[ -f MiniMax-H3/FL2VA/audio_vae/model.safetensors && + -f MiniMax-H3/Ref2VA/video_vae/source/model.safetensors ]]; then + run make PLATFORM=Linux checkpoint-schema-test + else + echo " · checkpoint MiniMax-H3 assente: smoke schema non applicabile" + fi + if has nvcc; then + run make PLATFORM=Linux cuda-runtime-test + run make PLATFORM=Linux cuda-primitives-test + run make PLATFORM=Linux cuda-rope-tokens-test + run make PLATFORM=Linux cuda-linear-test + run make PLATFORM=Linux cuda-attention-test + run make PLATFORM=Linux cuda-ops-test + run make PLATFORM=Linux test + else + echo " ! nvcc assente: impossibile eseguire il gate CUDA" + FAIL=1 + fi + fi + if [[ $WEBUI -eq 1 ]]; then + run "$WEBUI_VENV/pytest" -q -x --tb=short -m "not integration" webui/backend/tests + fi + if [[ $PY -eq 1 ]] && has pytest; then + run pytest -q -x --tb=short -m "not integration" + fi + if [[ $JS -eq 1 ]] && has npx; then + if grep -q '"vitest"' package.json 2>/dev/null; then + run npx --no-install vitest run --reporter=dot + elif grep -q '"jest"' package.json 2>/dev/null; then + run npx --no-install jest --silent + fi + fi +fi + +# ───────────────────────────── TEST DI INTEGRAZIONE ──────────────────────────── +if [[ "$MODE" == "integration" || "$MODE" == "all" ]]; then + step "Test di integrazione" + if [[ $PY -eq 1 ]] && has pytest; then + run pytest -q --tb=short -m integration || true # nessun test marcato = non è un errore + fi +fi + +# ──────────────────────────────────── ESITO ─────────────────────────────────── +if [[ $FAIL -eq 0 ]]; then + RESULT="VERIFY: PASS" +else + RESULT="VERIFY: FAIL" +fi +echo "$RESULT ($(date +%H:%M:%S), mode=$MODE)" | tee logs/agent/last_verify.txt +exit $FAIL diff --git a/webui/backend/app/__init__.py b/webui/backend/app/__init__.py new file mode 100644 index 00000000..ec4695fe --- /dev/null +++ b/webui/backend/app/__init__.py @@ -0,0 +1 @@ +"""h3.c web UI backend.""" diff --git a/webui/backend/app/argv.py b/webui/backend/app/argv.py new file mode 100644 index 00000000..75d97359 --- /dev/null +++ b/webui/backend/app/argv.py @@ -0,0 +1,78 @@ +"""Build the exact argv for one `./h3` run. + +An argv list, never a shell string: prompts and file names come from the +browser and must never be parsed by a shell. Durations are resolved to an +explicit frame count here, so the number shown in the UI is the number h3 runs. +""" + +from pathlib import Path + +from .jobspec import JobSpec, Reference + +_REFERENCE_FLAG = { + "image": "--ref-image", + "video": "--ref-video", + "silent_video": "--ref-silent-video", + "video_audio": "--ref-video-audio", + "audio": "--ref-audio", +} + + +def build_argv( + spec: JobSpec, + binary: Path, + model_dir: Path, + output: Path | None, + frames_dir: Path | None = None, + preview_dir: Path | None = None, +) -> list[str]: + argv = [str(binary), "-d", str(model_dir), "-p", spec.prompt] + argv += ["-o", str(output) if output else ""] + argv += ["--width", str(spec.width), "--height", str(spec.height)] + if spec.render_width and spec.render_height: + argv += [ + "--render-width", + str(spec.render_width), + "--render-height", + str(spec.render_height), + ] + argv += ["--frames", str(spec.resolved_frames())] + argv += ["--steps", str(spec.steps)] + argv += ["--layers", str(spec.dit_layers)] + if spec.core_reuse > 1: + argv += ["--core-reuse", str(spec.core_reuse)] + else: + argv += ["--reuse", str(spec.denoise_reuse)] + if spec.token_reduction: + argv.append("--token-reduction") + if spec.ssd_streaming: + argv.append("--ssd-streaming") + if spec.use_int8_row_fc2: + argv.append("--use-int8-row-fc2") + if spec.use_reference_rope: + argv.append("--use-reference-rope") + for flag in spec.slower: + argv.append(f"--{flag}") + argv += ["--seed", str(spec.seed)] + if spec.first_frame: + argv += ["--first-frame", spec.first_frame] + if spec.last_frame: + argv += ["--last-frame", spec.last_frame] + if any(reference.kind == "image" for reference in spec.references): + argv += ["--ref-image-size", spec.reference_image_size] + for reference in spec.references: + argv += _reference_argv(reference) + if frames_dir is not None: + argv += ["--frames-dir", str(frames_dir)] + if preview_dir is not None: + argv += ["--preview-dir", str(preview_dir)] + if spec.profile: + argv.append("--profile") + return argv + + +def _reference_argv(reference: Reference) -> list[str]: + flag = _REFERENCE_FLAG[reference.kind] + if reference.kind == "video_audio": + return [flag, reference.path, reference.audio_path or ""] + return [flag, reference.path] diff --git a/webui/backend/app/assets.py b/webui/backend/app/assets.py new file mode 100644 index 00000000..9bcd0705 --- /dev/null +++ b/webui/backend/app/assets.py @@ -0,0 +1,188 @@ +"""Uploaded images, clips and soundtracks. + +Files are stored by content hash, so re-uploading the same photo reuses the +existing entry and the library stays free of duplicates. Extensions are +whitelisted and every file is probed with ffprobe: what the browser calls a +PNG is only accepted if ffprobe agrees. +""" + +import hashlib +import json +import shutil +import sqlite3 +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .db import Database + +IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".webp", ".bmp"} +VIDEO_SUFFIXES = {".mp4", ".mov", ".mkv", ".webm", ".avi"} +AUDIO_SUFFIXES = {".wav", ".mp3", ".flac", ".m4a", ".aac", ".ogg"} +ALLOWED_SUFFIXES = IMAGE_SUFFIXES | VIDEO_SUFFIXES | AUDIO_SUFFIXES + +# h3 accepts reference audio between 2 and 15 seconds. +MIN_AUDIO_SECONDS = 2.0 +MAX_AUDIO_SECONDS = 15.0 + + +class AssetError(ValueError): + """The upload cannot be stored, with a reason meant for the user.""" + + +@dataclass +class Probe: + kind: str + seconds: float | None + width: int | None + height: int | None + has_audio: bool + + +def kind_from_suffix(suffix: str) -> str: + lowered = suffix.lower() + if lowered in IMAGE_SUFFIXES: + return "image" + if lowered in VIDEO_SUFFIXES: + return "video" + if lowered in AUDIO_SUFFIXES: + return "audio" + raise AssetError(f"unsupported file type: {suffix or 'no extension'}") + + +def probe(path: Path, ffprobe: str = "ffprobe") -> Probe: + """Ask ffprobe what this file really is.""" + try: + done = subprocess.run( # noqa: S603 - fixed argv, no shell + [ + ffprobe, + "-v", + "error", + "-show_entries", + "stream=codec_type,width,height:format=duration", + "-of", + "json", + str(path), + ], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as error: + raise AssetError(f"cannot probe the upload: {error}") from error + if done.returncode != 0: + raise AssetError("the file is not a readable image, video or audio track") + report = json.loads(done.stdout or "{}") + streams = report.get("streams", []) + if not streams: + raise AssetError("the file has no decodable stream") + video = next((s for s in streams if s.get("codec_type") == "video"), None) + has_audio = any(s.get("codec_type") == "audio" for s in streams) + duration = report.get("format", {}).get("duration") + seconds = float(duration) if duration not in (None, "N/A") else None + if video is None: + kind = "audio" + elif seconds is None or seconds == 0 or _is_still(seconds, video): + kind = "image" + else: + kind = "video" + return Probe( + kind=kind, + seconds=seconds, + width=video.get("width") if video else None, + height=video.get("height") if video else None, + has_audio=has_audio, + ) + + +def _is_still(seconds: float, video: dict[str, Any]) -> bool: + # ffprobe reports a tiny synthetic duration for single-frame images. + return seconds < 0.1 and not video.get("nb_frames", "").isdigit() + + +def store( + database: Database, + source: Path, + filename: str, + root: Path, + max_bytes: int, + ffprobe: str = "ffprobe", +) -> dict[str, Any]: + """Validate, deduplicate and record one upload.""" + declared = kind_from_suffix(Path(filename).suffix) + size = source.stat().st_size + if size == 0: + raise AssetError("the upload is empty") + if size > max_bytes: + raise AssetError( + f"the upload is {size / 1e6:.1f} MB, over the " + f"{max_bytes / 1e6:.0f} MB limit" + ) + detected = probe(source, ffprobe) + if detected.kind != declared: + raise AssetError( + f"the extension says {declared} but the file is {detected.kind}" + ) + + digest = _sha256(source) + existing = database.query_one("SELECT * FROM assets WHERE sha256 = ?", (digest,)) + if existing: + return _row_to_dict(existing) | {"duplicate": True} + + target = root / digest[:2] / f"{digest}{Path(filename).suffix.lower()}" + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + metadata = { + "seconds": detected.seconds, + "width": detected.width, + "height": detected.height, + "has_audio": detected.has_audio, + "notes": _notes(detected), + } + asset_id = database.run( + "INSERT INTO assets (sha256, kind, filename, path, bytes, metadata) " + "VALUES (?, ?, ?, ?, ?, ?)", + (digest, detected.kind, filename, str(target), size, json.dumps(metadata)), + ) + row = database.query_one("SELECT * FROM assets WHERE id = ?", (asset_id,)) + return _row_to_dict(row) | {"duplicate": False} + + +def listing(database: Database) -> list[dict[str, Any]]: + return [ + _row_to_dict(row) + for row in database.query_all("SELECT * FROM assets ORDER BY id DESC") + ] + + +def _notes(detected: Probe) -> list[str]: + """Usage limits worth showing next to the file, not reasons to reject it.""" + notes: list[str] = [] + if detected.seconds is None: + return notes + if detected.kind == "audio" and detected.seconds < MIN_AUDIO_SECONDS: + notes.append("shorter than the 2 s minimum for a reference audio track") + elif detected.kind == "audio" and detected.seconds > MAX_AUDIO_SECONDS: + notes.append( + "longer than the 15 s total budget for reference audio; " + "h3 will use the first 15 s" + ) + elif detected.kind == "video" and detected.seconds < MIN_AUDIO_SECONDS: + notes.append("shorter than 2 s: usable only as a silent video reference") + return notes + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _row_to_dict(row: sqlite3.Row) -> dict[str, Any]: + item = dict(row) + item["metadata"] = json.loads(item["metadata"] or "{}") + return item diff --git a/webui/backend/app/capabilities.py b/webui/backend/app/capabilities.py new file mode 100644 index 00000000..b62a46cd --- /dev/null +++ b/webui/backend/app/capabilities.py @@ -0,0 +1,11 @@ +"""Serve the canonical option inventory to the frontend.""" + +import json +from functools import lru_cache +from pathlib import Path +from typing import Any + + +@lru_cache +def load_schema(path: Path) -> dict[str, Any]: + return json.loads(path.read_text()) diff --git a/webui/backend/app/config.py b/webui/backend/app/config.py new file mode 100644 index 00000000..78af155a --- /dev/null +++ b/webui/backend/app/config.py @@ -0,0 +1,40 @@ +"""Runtime configuration, read once from the environment.""" + +from functools import lru_cache +from pathlib import Path + +from pydantic_settings import BaseSettings, SettingsConfigDict + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +class Settings(BaseSettings): + """Every value can be overridden with an H3_ prefixed environment variable.""" + + model_config = SettingsConfigDict(env_prefix="H3_", extra="ignore") + + # Path to the h3 binary and to the MiniMax-H3 checkpoint directory. + binary: Path = REPO_ROOT / "h3" + model_dir: Path = REPO_ROOT / "MiniMax-H3" + # Where jobs, uploads and generated media are written. + data_dir: Path = REPO_ROOT / "webui/backend/data" + # Canonical option inventory shared with the frontend. + schema_path: Path = REPO_ROOT / "webui/shared/options.schema.json" + # Measured phase durations behind the weighted progress bar. + progress_weights_path: Path = REPO_ROOT / "webui/shared/progress_weights.json" + # Seconds allowed for `h3 --info`, which only reads checkpoint headers. + info_timeout: float = 120.0 + # Largest accepted upload, in bytes. + max_upload_bytes: int = 512 * 1024 * 1024 + ffprobe: str = "ffprobe" + ffmpeg: str = "ffmpeg" + # Seconds between SIGTERM and SIGKILL when a job is cancelled. + kill_grace: float = 10.0 + # Post-processing plugins: an executable path enables the plugin. + # Nothing is installed or downloaded by this repository. + faceswap_cmd: str = "" + + +@lru_cache +def settings() -> Settings: + return Settings() diff --git a/webui/backend/app/db.py b/webui/backend/app/db.py new file mode 100644 index 00000000..54ce04c3 --- /dev/null +++ b/webui/backend/app/db.py @@ -0,0 +1,99 @@ +"""SQLite storage for jobs and uploaded assets. + +The worker thread and the request handlers share one connection, so every +statement goes through a lock: sqlite3 allows cross-thread use but not +concurrent use. One writer, one job at a time — no server is needed. +""" + +import sqlite3 +import threading +from pathlib import Path +from typing import Any + +SCHEMA = """ +CREATE TABLE IF NOT EXISTS jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + state TEXT NOT NULL DEFAULT 'queued', + prompt TEXT NOT NULL DEFAULT '', + params TEXT NOT NULL, + argv TEXT, + phase TEXT, + completed INTEGER NOT NULL DEFAULT 0, + total INTEGER NOT NULL DEFAULT 0, + progress REAL NOT NULL DEFAULT 0.0, + error TEXT, + output_path TEXT, + log_path TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + started_at TEXT, + finished_at TEXT +); + +CREATE TABLE IF NOT EXISTS assets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + sha256 TEXT NOT NULL UNIQUE, + kind TEXT NOT NULL, + filename TEXT NOT NULL, + path TEXT NOT NULL, + bytes INTEGER NOT NULL, + metadata TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); +""" + + +class Closed(RuntimeError): + """The database was closed while a background thread was still writing.""" + + +class Database: + """Every access is serialized and returns plain rows, never live cursors.""" + + def __init__(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + self._lock = threading.RLock() + self._open = True + self._connection = sqlite3.connect(path, check_same_thread=False) + self._connection.row_factory = sqlite3.Row + self._connection.execute("PRAGMA journal_mode=WAL") + self._connection.execute("PRAGMA foreign_keys=ON") + with self._lock: + self._connection.executescript(SCHEMA) + self._connection.commit() + + def query_one(self, sql: str, params: tuple[Any, ...] = ()) -> sqlite3.Row | None: + with self._guard() as connection: + return connection.execute(sql, params).fetchone() + + def query_all(self, sql: str, params: tuple[Any, ...] = ()) -> list[sqlite3.Row]: + with self._guard() as connection: + return connection.execute(sql, params).fetchall() + + def run(self, sql: str, params: tuple[Any, ...] = ()) -> int: + with self._guard() as connection: + cursor = connection.execute(sql, params) + connection.commit() + return int(cursor.lastrowid or 0) + + def close(self) -> None: + with self._lock: + if self._open: + self._open = False + self._connection.close() + + def _guard(self): + database = self + + class _Guard: + def __enter__(self) -> sqlite3.Connection: + database._lock.acquire() + if not database._open: + database._lock.release() + raise Closed("the database is closed") + return database._connection + + def __exit__(self, *_: object) -> bool: + database._lock.release() + return False + + return _Guard() diff --git a/webui/backend/app/events.py b/webui/backend/app/events.py new file mode 100644 index 00000000..3d11e1ba --- /dev/null +++ b/webui/backend/app/events.py @@ -0,0 +1,51 @@ +"""Server-sent events for one job. + +The runner emits from its worker thread; each subscriber owns an asyncio queue +fed through the event loop. The stream ends when the job reaches a terminal +state, so the browser does not need to poll or to guess when to stop. +""" + +import asyncio +import json +from collections.abc import AsyncIterator +from typing import Any + +from .runner import TERMINAL_STATES, JobRunner + +HEARTBEAT_SECONDS = 15.0 + + +async def job_events(runner: JobRunner, job_id: int) -> AsyncIterator[str]: + loop = asyncio.get_running_loop() + queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue() + + def listener(job: dict[str, Any]) -> None: + if job["id"] == job_id: + loop.call_soon_threadsafe(queue.put_nowait, job) + + snapshot = runner.get(job_id) + if snapshot is None: + yield _event("error", {"detail": "unknown job"}) + return + + runner.add_listener(listener) + try: + yield _event("job", snapshot) + if snapshot["state"] in TERMINAL_STATES: + return + while True: + try: + job = await asyncio.wait_for(queue.get(), timeout=HEARTBEAT_SECONDS) + except TimeoutError: + # Keeps proxies from closing an idle stream during a long phase. + yield ": keep-alive\n\n" + continue + yield _event("job", job) + if job["state"] in TERMINAL_STATES: + return + finally: + runner.remove_listener(listener) + + +def _event(name: str, payload: dict[str, Any]) -> str: + return f"event: {name}\ndata: {json.dumps(payload)}\n\n" diff --git a/webui/backend/app/jobspec.py b/webui/backend/app/jobspec.py new file mode 100644 index 00000000..d7071cdc --- /dev/null +++ b/webui/backend/app/jobspec.py @@ -0,0 +1,237 @@ +"""Job specification and the validation h3 would otherwise refuse at runtime. + +Every message here is copied verbatim from h3.c or main.c, so what the browser +shows before submitting is what the engine would have said afterwards. +""" + +from typing import Any, Literal + +from pydantic import BaseModel, Field + +FPS = 24 +CANVAS_MULTIPLE = 32 +MAX_PIXELS = 768 * 1344 +MIN_FRAMES_GENERATION = 22 +MAX_FRAMES = 362 +# A truncated video soundtrack needs 2 s, and 39 frames is only 1.625 s. +MIN_FRAMES_FOR_SOUNDTRACK = 56 +MAX_REFERENCE_AUDIO_SECONDS = 15.0 +MIN_REFERENCE_AUDIO_SECONDS = 2.0 + +ReferenceKind = Literal["image", "video", "silent_video", "video_audio", "audio"] + + +def align_frames(requested: int) -> int: + """Mirror of h3_align_frame_count: legal shapes are 5 + 17n.""" + value = max(requested, 5) + remainder = (value - 5) % 17 + return value if remainder == 0 else value + 17 - remainder + + +def frames_from_seconds(seconds: float) -> int: + return round(seconds * FPS) + + +class Reference(BaseModel): + kind: ReferenceKind + path: str + # Only for video_audio: the replacement soundtrack. + audio_path: str | None = None + # Filled from the asset store; used for the audio duration rules. + seconds: float | None = None + + +class JobSpec(BaseModel): + prompt: str = "" + width: int = 864 + height: int = 480 + render_width: int = 0 + render_height: int = 0 + frames: int | None = None + seconds: float | None = None + steps: int = 20 + denoise_reuse: int = 1 + dit_layers: int = 50 + core_reuse: int = 1 + token_reduction: bool = False + ssd_streaming: bool = False + use_int8_row_fc2: bool = False + use_reference_rope: bool = False + seed: int = 42 + first_frame: str | None = None + last_frame: str | None = None + references: list[Reference] = Field(default_factory=list) + reference_image_size: Literal["match", "max"] = "match" + write_frames: bool = False + profile: bool = False + preview: bool = False + slower: list[str] = Field(default_factory=list) + postprocess: list[str] = Field(default_factory=list) + + def resolved_frames(self) -> int: + """The frame count h3 will actually generate.""" + requested = ( + self.frames + if self.frames is not None + else frames_from_seconds(self.seconds) + if self.seconds is not None + else 56 + ) + return align_frames(max(requested, 1)) + + +class EstimateRequest(BaseModel): + """A job, plus the alternatives the interface wants labelled with a time.""" + + spec: JobSpec + variants: list[dict[str, Any]] = Field(default_factory=list) + + +def validate(spec: JobSpec, backend: str = "cuda") -> tuple[list[str], list[str]]: + """Return (errors, warnings). Errors mean h3 would refuse the job.""" + errors: list[str] = [] + warnings: list[str] = [] + + if not spec.prompt.strip(): + errors.append("a prompt is required") + + _check_canvas(spec, errors) + _check_duration(spec, errors) + _check_sampler(spec, errors) + _check_backend_flags(spec, backend, errors, warnings) + _check_references(spec, errors) + return errors, warnings + + +def _check_canvas(spec: JobSpec, errors: list[str]) -> None: + if ( + spec.width < CANVAS_MULTIPLE + or spec.height < CANVAS_MULTIPLE + or spec.width % CANVAS_MULTIPLE + or spec.height % CANVAS_MULTIPLE + ): + errors.append("width and height must be multiples of 32 and at least 32") + if spec.width * spec.height > MAX_PIXELS: + errors.append("canvas exceeds the released 768*1344 pixel limit") + if (spec.render_width == 0) != (spec.render_height == 0): + errors.append("render width and height must be set together") + elif spec.render_width and ( + spec.render_width < CANVAS_MULTIPLE + or spec.render_height < CANVAS_MULTIPLE + or spec.render_width % CANVAS_MULTIPLE + or spec.render_height % CANVAS_MULTIPLE + or spec.render_width > spec.width + or spec.render_height > spec.height + or spec.render_width * spec.height != spec.render_height * spec.width + ): + errors.append( + "internal render canvas must be same-aspect multiples of 32 " + "no larger than the output canvas" + ) + + +def _check_duration(spec: JobSpec, errors: list[str]) -> None: + if spec.frames is not None and spec.seconds is not None: + errors.append("--seconds and --frames are mutually exclusive") + if spec.seconds is not None and spec.seconds <= 0: + errors.append("invalid seconds") + return + requested = spec.frames if spec.frames is not None else None + if requested is not None and requested < 5: + errors.append("frames must align within the released 5..362 range") + return + aligned = spec.resolved_frames() + if aligned > MAX_FRAMES: + errors.append("frames must align within the released 5..362 range") + elif aligned < MIN_FRAMES_GENERATION: + errors.append("generation requires at least one trained 22-frame decoder chunk") + + +def _check_sampler(spec: JobSpec, errors: list[str]) -> None: + if not 2 <= spec.steps <= 1000: + errors.append("denoising steps must be in [2, 1000]") + if not 1 <= spec.denoise_reuse <= 3: + errors.append("denoise reuse must be in [1, 3]") + if not 35 <= spec.dit_layers <= 50: + errors.append("DiT layers must be in [35, 50]") + if not 1 <= spec.core_reuse <= 6: + errors.append("core reuse must be in [1, 6]") + if spec.core_reuse > 1 and spec.denoise_reuse > 1: + errors.append("core reuse and denoiser reuse cannot be combined") + + +def _check_backend_flags( + spec: JobSpec, backend: str, errors: list[str], warnings: list[str] +) -> None: + if spec.ssd_streaming and spec.use_int8_row_fc2: + errors.append( + "SSD streaming uses original BF16 weights and cannot be combined " + "with int8 row FC2" + ) + if spec.use_int8_row_fc2 and "use-slower-bf16-mlp" in spec.slower: + errors.append("int8 row FC2 cannot be combined with the BF16 MLP") + if spec.use_int8_row_fc2 and backend == "cuda": + warnings.append( + "--use-int8-row-fc2 is a Metal/M5 specialization and a measured " + "no-op on this CUDA backend" + ) + + +def _check_references(spec: JobSpec, errors: list[str]) -> None: + references = spec.references + if not references: + return + if spec.first_frame or spec.last_frame: + errors.append("full references cannot be combined with frame anchors") + if len(references) > 12: + errors.append("Ref2VA supports at most 12 references") + + video_kinds = ("video", "silent_video", "video_audio") + images = sum(1 for r in references if r.kind == "image") + videos = sum(1 for r in references if r.kind in video_kinds) + # A plain --ref-video keeps its embedded audio, so it counts as an input; + # --ref-silent-video does not. + audio_inputs = sum( + 1 for r in references if r.kind in ("audio", "video", "video_audio") + ) + if images > 9 or videos > 3 or audio_inputs > 3: + errors.append("Ref2VA limits are 9 images, 3 videos, and 3 audio inputs") + if not any(r.kind != "audio" for r in references): + errors.append("reference audio requires an image or video reference") + + for index, reference in enumerate(references, start=1): + if reference.kind == "video_audio" and not reference.audio_path: + errors.append(f"video+audio reference {index} has no soundtrack path") + + # A video soundtrack is truncated to min(clip length, output length), and + # h3 refuses anything shorter than two seconds. + output_seconds = spec.resolved_frames() / FPS + has_soundtrack = any(r.kind in ("video", "video_audio") for r in references) + if has_soundtrack and output_seconds < MIN_REFERENCE_AUDIO_SECONDS: + errors.append( + "a video soundtrack requires at least 2 seconds; " + "request at least 56 output frames" + ) + for index, reference in enumerate(references, start=1): + if reference.kind not in ("video", "video_audio"): + continue + if ( + reference.seconds is not None + and reference.seconds < MIN_REFERENCE_AUDIO_SECONDS + ): + errors.append( + f"video soundtrack {index} requires at least 2 seconds: " + f"the clip is only {reference.seconds:g} s" + ) + + total = 0.0 + for reference in references: + if reference.kind != "audio": + continue + if reference.seconds is None: + continue + if reference.seconds < MIN_REFERENCE_AUDIO_SECONDS: + errors.append("reference audio requires at least 2 seconds at 32 kHz") + total += reference.seconds + if total > MAX_REFERENCE_AUDIO_SECONDS: + errors.append("ordered reference audio exceeds 15 seconds in total") diff --git a/webui/backend/app/main.py b/webui/backend/app/main.py new file mode 100644 index 00000000..e7e4c8cc --- /dev/null +++ b/webui/backend/app/main.py @@ -0,0 +1,218 @@ +"""FastAPI application: health, capabilities and system inventory. + +Binds to 127.0.0.1 by default and has no authentication: see the security note +in the README before exposing it anywhere. +""" + +import shutil +import tempfile +from contextlib import asynccontextmanager +from pathlib import Path +from typing import Any + +from fastapi import FastAPI, HTTPException, UploadFile +from fastapi.responses import FileResponse, PlainTextResponse, StreamingResponse +from pydantic import ValidationError + +from . import assets, media +from .capabilities import load_schema +from .config import Settings, settings +from .db import Database +from .events import job_events +from .jobspec import EstimateRequest, JobSpec, validate +from .postprocess import registry +from .progress import observed_correction +from .runner import JobRunner +from .system import read_system + + +def create_app(config: Settings | None = None) -> FastAPI: + config = config or settings() + + @asynccontextmanager + async def lifespan(app: FastAPI): + app.state.config = config + app.state.db = Database(config.data_dir / "h3.sqlite3") + app.state.runner = JobRunner(app.state.db, config) + app.state.runner.start() + yield + app.state.runner.shutdown() + app.state.db.close() + + app = FastAPI(title="h3.c Studio", version="0.1.0", lifespan=lifespan) + + @app.get("/api/health") + def health() -> dict[str, Any]: + return {"status": "ok", "version": app.version} + + @app.get("/api/capabilities") + def capabilities() -> dict[str, Any]: + return load_schema(config.schema_path) | { + "plugins": [plugin.as_dict() for plugin in registry(config)] + } + + @app.get("/api/system") + def system() -> dict[str, Any]: + return read_system(config.binary, config.model_dir, config.info_timeout) + + @app.post("/api/jobs", status_code=201) + def create_job(spec: JobSpec) -> dict[str, Any]: + errors, warnings = validate(spec) + if errors: + raise HTTPException(status_code=422, detail={"errors": errors}) + job = app.state.runner.submit(spec) + return job | {"warnings": warnings} + + @app.post("/api/jobs/validate") + def validate_job(spec: JobSpec) -> dict[str, Any]: + errors, warnings = validate(spec) + model = app.state.runner.model + correction, learned_from = observed_correction(app.state.db, model) + return { + "errors": errors, + "warnings": warnings, + "frames": spec.resolved_frames(), + "seconds": round(spec.resolved_frames() / 24, 3), + "estimate_seconds": round( + sum(seconds for _, seconds in model.plan(spec)) * correction, 1 + ), + "learned_from": learned_from, + } + + @app.post("/api/jobs/estimate") + def estimate(request: EstimateRequest) -> dict[str, Any]: + """How long this job would take, and how long each alternative would. + + One request labels every choice on screen, so a card can say what it + costs before it is picked. + """ + model = app.state.runner.model + correction, learned_from = observed_correction(app.state.db, model) + + def total(candidate: JobSpec) -> float: + plan = model.plan(candidate) + return round(sum(seconds for _, seconds in plan) * correction, 1) + + spec = request.spec + answered = [] + for override in request.variants: + try: + candidate = JobSpec.model_validate(spec.model_dump() | override) + except ValidationError as error: + answered.append({"override": override, "error": error.error_count()}) + continue + answered.append({"override": override, "seconds": total(candidate)}) + return { + "seconds": total(spec), + "variants": answered, + "learned_from": learned_from, + } + + @app.get("/api/jobs") + def list_jobs(limit: int = 100) -> list[dict[str, Any]]: + return app.state.runner.listing(limit) + + @app.get("/api/jobs/{job_id}") + def read_job(job_id: int) -> dict[str, Any]: + job = app.state.runner.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail="unknown job") + return job + + @app.post("/api/jobs/{job_id}/cancel") + def cancel_job(job_id: int) -> dict[str, Any]: + job = app.state.runner.cancel(job_id) + if job is None: + raise HTTPException(status_code=404, detail="unknown job") + return job + + @app.get("/api/jobs/{job_id}/events") + async def job_stream(job_id: int) -> StreamingResponse: + return StreamingResponse( + job_events(app.state.runner, job_id), + media_type="text/event-stream", + headers={"Cache-Control": "no-store", "X-Accel-Buffering": "no"}, + ) + + @app.get("/api/jobs/{job_id}/video") + def job_video(job_id: int) -> FileResponse: + job = _job_or_404(app, job_id) + path = Path(job["output_path"] or "") + if not path.is_file(): + raise HTTPException(status_code=404, detail="this job has no video") + return FileResponse(path, media_type="video/mp4", filename=f"h3-{job_id}.mp4") + + @app.get("/api/jobs/{job_id}/poster") + def job_poster(job_id: int) -> FileResponse: + job = _job_or_404(app, job_id) + video = Path(job["output_path"] or "") + if not video.is_file(): + raise HTTPException(status_code=404, detail="this job has no video") + poster = video.with_name("poster.jpg") + if not poster.is_file() and not media.extract_poster(video, poster, config): + raise HTTPException(status_code=404, detail="cannot build a poster") + return FileResponse(poster, media_type="image/jpeg") + + @app.get("/api/jobs/{job_id}/preview") + def job_preview(job_id: int) -> FileResponse: + _job_or_404(app, job_id) + jpeg = media.preview_jpeg(app.state.runner.preview_dir(job_id), config) + if jpeg is None: + raise HTTPException(status_code=404, detail="no preview yet") + return FileResponse( + jpeg, media_type="image/jpeg", headers={"Cache-Control": "no-store"} + ) + + @app.get("/api/jobs/{job_id}/log", response_class=PlainTextResponse) + def job_log(job_id: int) -> str: + job = _job_or_404(app, job_id) + path = Path(job["log_path"] or "") + if not path.is_file(): + raise HTTPException(status_code=404, detail="this job has no log") + return path.read_text(errors="replace") + + @app.get("/api/assets") + def list_assets() -> list[dict[str, Any]]: + return assets.listing(app.state.db) + + @app.post("/api/assets", status_code=201) + async def upload_asset(file: UploadFile) -> dict[str, Any]: + filename = Path(file.filename or "").name + if not filename: + raise HTTPException(status_code=400, detail="the upload has no file name") + with tempfile.TemporaryDirectory() as staging: + staged = Path(staging) / filename + with staged.open("wb") as handle: + shutil.copyfileobj(file.file, handle) + try: + return assets.store( + app.state.db, + staged, + filename, + config.data_dir / "assets", + config.max_upload_bytes, + config.ffprobe, + ) + except assets.AssetError as error: + raise HTTPException(status_code=400, detail=str(error)) from error + + @app.get("/api/assets/{asset_id}/file") + def asset_file(asset_id: int) -> FileResponse: + row = app.state.db.query_one( + "SELECT * FROM assets WHERE id = ?", (asset_id,) + ) + if row is None: + raise HTTPException(status_code=404, detail="unknown asset") + return FileResponse(row["path"], filename=row["filename"]) + + return app + + +def _job_or_404(app: FastAPI, job_id: int) -> dict[str, Any]: + job = app.state.runner.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail="unknown job") + return job + + +app = create_app() diff --git a/webui/backend/app/media.py b/webui/backend/app/media.py new file mode 100644 index 00000000..05184b44 --- /dev/null +++ b/webui/backend/app/media.py @@ -0,0 +1,70 @@ +"""Small FFmpeg helpers for the gallery.""" + +import re +import subprocess +from pathlib import Path + +from .config import Settings + +_STEP = re.compile(r"^step-(\d+)\.ppm$") + + +def extract_poster(video: Path, poster: Path, config: Settings) -> bool: + """Grab the first frame as a JPEG thumbnail. Best effort.""" + try: + done = subprocess.run( # noqa: S603 - fixed argv, no shell + [ + config.ffmpeg, + "-y", + "-loglevel", + "error", + "-i", + str(video), + "-frames:v", + "1", + str(poster), + ], + capture_output=True, + timeout=120, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return False + return done.returncode == 0 and poster.is_file() + + +def latest_preview(directory: Path) -> tuple[int, Path] | None: + """Newest complete denoising preview, or None if there is not one yet. + + h3 writes to a staging name and renames, so every step-*.ppm here is whole. + """ + if not directory.is_dir(): + return None + best: tuple[int, Path] | None = None + for entry in directory.iterdir(): + match = _STEP.match(entry.name) + if match and (best is None or int(match.group(1)) > best[0]): + best = (int(match.group(1)), entry) + return best + + +def preview_jpeg(directory: Path, config: Settings) -> Path | None: + """Convert the newest preview to JPEG once, then reuse it.""" + newest = latest_preview(directory) + if newest is None: + return None + step, source = newest + target = directory / f"step-{step:04d}.jpg" + if target.is_file(): + return target + try: + done = subprocess.run( # noqa: S603 - fixed argv, no shell + [config.ffmpeg, "-y", "-loglevel", "error", "-i", str(source), + "-q:v", "3", str(target)], + capture_output=True, + timeout=60, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return None + return target if done.returncode == 0 and target.is_file() else None diff --git a/webui/backend/app/postprocess.py b/webui/backend/app/postprocess.py new file mode 100644 index 00000000..169a53bb --- /dev/null +++ b/webui/backend/app/postprocess.py @@ -0,0 +1,123 @@ +"""Optional post-processing stage: an extension point, not an integration. + +A plugin is an external executable, not a Python import: FaceFusion and its +kin live in their own virtualenv with pinned onnxruntime builds, and importing +them here would inherit those constraints. + +Contract, so a third party can implement one without reading this file: + + $H3__CMD --input IN.mp4 --output OUT.mp4 [--param value ...] + + exit 0 and OUT.mp4 written -> the job's video is replaced + exit != 0 -> the job fails, IN.mp4 is kept + +This repository ships no models, no weights and no download URLs. Plugins are +unavailable until the operator points the environment variable at an +executable they installed themselves. +""" + +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .config import Settings + + +@dataclass +class Plugin: + name: str + label: str + description: str + env_var: str + command: str + notice: str | None = None + + @property + def available(self) -> bool: + return bool(self.command) and ( + Path(self.command).is_file() or shutil.which(self.command) is not None + ) + + @property + def reason(self) -> str | None: + if self.available: + return None + if not self.command: + return ( + f"no model and no runtime installed: set {self.env_var} to an " + "executable to enable it" + ) + return f"{self.env_var} points at {self.command}, which is not executable" + + def as_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "label": self.label, + "description": self.description, + "env_var": self.env_var, + "available": self.available, + "reason": self.reason, + "notice": self.notice, + } + + +def registry(config: Settings) -> list[Plugin]: + return [ + Plugin( + name="faceswap", + label="Face replacement", + description=( + "Replaces faces in the generated video using an external " + "face-swapping runtime." + ), + env_var="H3_FACESWAP_CMD", + command=config.faceswap_cmd, + notice=( + "This repository ships neither models nor download URLs. Known " + "model licences are non-commercial/research only, and the " + "operator is responsible for checking them. Do not use it on " + "images of real people without their consent." + ), + ) + ] + + +def by_name(config: Settings, name: str) -> Plugin | None: + return next((plugin for plugin in registry(config) if plugin.name == name), None) + + +class PluginError(RuntimeError): + pass + + +def run_stage( + config: Settings, video: Path, requested: list[str], timeout: float = 3600.0 +) -> Path: + """Run each requested plugin in order, replacing the video each time.""" + for name in requested: + plugin = by_name(config, name) + if plugin is None: + raise PluginError(f"unknown post-processing plugin: {name}") + if not plugin.available: + raise PluginError(f"post-processing plugin {name} is unavailable") + produced = video.with_name(f"{video.stem}-{name}{video.suffix}") + try: + done = subprocess.run( # noqa: S603 - argv list, no shell + [plugin.command, "--input", str(video), "--output", str(produced)], + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as error: + raise PluginError( + f"post-processing {name} could not run: {error}" + ) from error + if done.returncode != 0 or not produced.is_file(): + detail = (done.stderr or done.stdout or "").strip().splitlines() + last = detail[-1] if detail else "no output" + raise PluginError(f"post-processing {name} failed: {last}") + produced.replace(video) + return video diff --git a/webui/backend/app/progress.py b/webui/backend/app/progress.py new file mode 100644 index 00000000..583a2b94 --- /dev/null +++ b/webui/backend/app/progress.py @@ -0,0 +1,247 @@ +"""Weighted progress and ETA. + +A naive bar would jump: on the calibration run `load transformer core` took +40.9 s and `denoise` 5.1 s, so counting phases equally is misleading. Weights +come from a real measured run (webui/shared/progress_weights.json, produced by +tools/calibrate_progress.py) and are scaled to the job's own settings. The ETA +then corrects itself with what the current run has actually taken so far. +""" + +import json +from functools import lru_cache +from pathlib import Path +from statistics import median +from typing import Any + +from .db import Database +from .jobspec import JobSpec + +# What each phase's cost is proportional to. A phase not listed here is a fixed +# cost: loading the text encoder takes the same time whatever the video is. +SCALES: dict[str, tuple[str, ...]] = { + "denoise": ("steps", "pixels", "frames"), + "denoise enqueue": ("steps", "pixels", "frames"), + # The decode and the mux happen inside this phase's wall time. + "video VAE load": ("pixels", "frames"), + "FFmpeg": ("frames",), +} +UNKNOWN_PHASE_SECONDS = 1.0 + + +@lru_cache +def load_weights(path: Path) -> dict[str, Any]: + return json.loads(path.read_text()) + + +def _work(phase: str, spec_like: dict[str, Any]) -> float: + """Absolute amount of work a phase does for one configuration. + + Pixels use the internal canvas when there is one, because that is what the + model and the VAE actually run at. + """ + width = spec_like.get("render_width") or spec_like["width"] + height = spec_like.get("render_height") or spec_like["height"] + amount = { + "steps": spec_like["steps"], + "pixels": width * height / 1e6, + "frames": spec_like["frames"], + } + work = 1.0 + for dimension in SCALES.get(phase, ()): + work *= amount[dimension] + return work + + +class ProgressModel: + def __init__(self, weights: dict[str, Any]) -> None: + self.reference = weights["reference"] + self.phase_seconds: dict[str, float] = weights["phase_seconds"] + self.order = list(self.phase_seconds) + self.factors: dict[str, Any] = weights.get("factors", {}) + self.fit = _fit(weights.get("samples", [])) + + def plan(self, spec: JobSpec) -> list[tuple[str, float]]: + """Expected seconds per phase for this job, in order.""" + ratios = self._ratios(spec) + shape = { + "width": spec.width, + "height": spec.height, + "render_width": spec.render_width, + "render_height": spec.render_height, + "steps": spec.steps, + "frames": spec.resolved_frames(), + } + plan = [] + for phase in self.order: + if phase in self.fit: + fixed, variable = self.fit[phase] + seconds = fixed + variable * _work(phase, shape) + else: + seconds = self.phase_seconds[phase] + for dimension in SCALES.get(phase, ()): + seconds *= ratios[dimension] + if phase.startswith("denoise"): + seconds *= self._denoise_factor(spec) + seconds += self._streaming_seconds(spec) + elif phase == "load transformer core" and spec.ssd_streaming: + seconds *= self._factor("ssd_streaming", "load_factor", 1.0) + plan.append((phase, seconds)) + if spec.preview and "preview VAE load" not in self.phase_seconds: + # Enabling the preview adds a VAE load; charge it like the decoder. + index = next( + (i for i, (name, _) in enumerate(plan) if name == "denoise"), len(plan) + ) + plan.insert(index, ("preview VAE load", self._preview_seconds(spec))) + return plan + + def fraction(self, spec: JobSpec, phase: str | None, completed: int, total: int + ) -> float: + plan = self.plan(spec) + budget = sum(seconds for _, seconds in plan) + if budget <= 0: + return 0.0 + done = 0.0 + for name, seconds in plan: + if name == phase: + share = completed / total if total else 0.0 + return min(1.0, (done + seconds * min(max(share, 0.0), 1.0)) / budget) + done += seconds + # An unknown phase carries no information about position: report + # nothing and let the caller keep the highest value seen so far. + return 0.0 + + def remaining_seconds( + self, spec: JobSpec, phase: str | None, completed: int, total: int, + elapsed: float + ) -> float | None: + """Estimate what is left, corrected by how this run is actually going.""" + share = self.fraction(spec, phase, completed, total) + if share <= 0.02 or elapsed <= 0: + budget = sum(seconds for _, seconds in self.plan(spec)) + return max(budget - elapsed, 0.0) if budget else None + return max(elapsed / share - elapsed, 0.0) + + def _denoise_factor(self, spec: JobSpec) -> float: + """What the sampler settings do to the cost of one pass. + + Steps, pixels and frames are already in the ratios; this is everything + else the quality presets change, which is most of what they change. + """ + reference_layers = self._factor("dit_layers", "reference", 50) or 50 + factor = spec.dit_layers / reference_layers + if spec.core_reuse > 1: + heads = self._factor("core_reuse", "head_share", 0.3) + factor *= heads + (1 - heads) / spec.core_reuse + else: + reuse = self.factors.get("denoise_reuse", {}) + factor *= float(reuse.get(str(spec.denoise_reuse), 1.0)) + if spec.token_reduction: + factor *= self._factor("token_reduction", "factor", 1.0) + return factor + + def _streaming_seconds(self, spec: JobSpec) -> float: + """Streaming the weights from disk costs the same on every step.""" + if not spec.ssd_streaming: + return 0.0 + return self._factor("ssd_streaming", "added_seconds_per_step", 0.0) * spec.steps + + def _factor(self, group: str, key: str, fallback: float) -> float: + return float(self.factors.get(group, {}).get(key, fallback)) + + def _preview_seconds(self, spec: JobSpec) -> float: + return self.phase_seconds.get("video VAE load", 10.0) * 0.5 * self._ratios( + spec + )["pixels"] + + def _ratios(self, spec: JobSpec) -> dict[str, float]: + reference = self.reference + width = spec.render_width or spec.width + height = spec.render_height or spec.height + return { + "steps": spec.steps / max(reference["steps"], 1), + "pixels": (width * height) + / max(reference["width"] * reference["height"], 1), + "frames": spec.resolved_frames() / max(reference["frames"], 1), + } + + +def _fit(samples: list[dict[str, Any]]) -> dict[str, tuple[float, float]]: + """Split each phase into a fixed and a per-unit-of-work cost. + + One calibration run cannot tell the two apart: with a single sample every + phase looks purely proportional, and the quality presets then all cost the + same. Two runs at different sizes separate them. + """ + if len(samples) < 2: + return {} + first, last = samples[0], samples[-1] + fitted: dict[str, tuple[float, float]] = {} + for phase in first["phase_seconds"]: + if phase not in last["phase_seconds"]: + continue + w1 = _work(phase, first["reference"]) + w2 = _work(phase, last["reference"]) + s1 = first["phase_seconds"][phase] + s2 = last["phase_seconds"][phase] + if abs(w2 - w1) < 1e-9: + fitted[phase] = (min(s1, s2), 0.0) + continue + variable = (s2 - s1) / (w2 - w1) + fixed = s1 - variable * w1 + if variable < 0: + # Noise, not a real saving: treat the phase as a fixed cost. + fitted[phase] = (min(s1, s2), 0.0) + elif fixed < 0: + # All of it scales; anchor on the larger, more reliable sample. + fitted[phase] = (0.0, s2 / w2) + else: + fitted[phase] = (fixed, variable) + return fitted + + +# How far a learned correction may pull the estimate. Beyond this the history +# is telling us something the model cannot express, and a wrong number with +# confidence is worse than a rough one. +CORRECTION_RANGE = (0.25, 4.0) +CORRECTION_SAMPLE = 12 + + +def observed_correction( + database: Database, model: "ProgressModel" +) -> tuple[float, int]: + """How wrong the estimate has been lately, as a single factor. + + Two calibration runs fix the shape of the model, not its accuracy across + every size: the cost of drawing does not grow linearly forever. Rather than + pretend otherwise, the estimate is scaled by what recent jobs on this + machine actually took. + """ + rows = database.query_all( + "SELECT params, started_at, finished_at FROM jobs " + "WHERE state = 'completed' AND started_at IS NOT NULL " + "AND finished_at IS NOT NULL ORDER BY id DESC LIMIT ?", + (CORRECTION_SAMPLE,), + ) + ratios: list[float] = [] + for row in rows: + try: + spec = JobSpec.model_validate(json.loads(row["params"])) + except (ValueError, TypeError): + continue + predicted = sum(seconds for _, seconds in model.plan(spec)) + actual = _seconds_between(row["started_at"], row["finished_at"]) + if predicted > 1 and actual > 1: + ratios.append(actual / predicted) + if len(ratios) < 2: + return 1.0, len(ratios) + low, high = CORRECTION_RANGE + return min(max(median(ratios), low), high), len(ratios) + + +def _seconds_between(started: str, finished: str) -> float: + from datetime import UTC, datetime + + fmt = "%Y-%m-%d %H:%M:%S" + begin = datetime.strptime(started, fmt).replace(tzinfo=UTC) + end = datetime.strptime(finished, fmt).replace(tzinfo=UTC) + return (end - begin).total_seconds() diff --git a/webui/backend/app/runner.py b/webui/backend/app/runner.py new file mode 100644 index 00000000..c49bf3d8 --- /dev/null +++ b/webui/backend/app/runner.py @@ -0,0 +1,399 @@ +"""Serial job queue over the h3 CLI. + +One GPU, one job at a time. A worker thread pulls queued jobs, runs `./h3` as +a subprocess and mirrors its stderr progress into SQLite. h3 rewrites the +current progress line with a carriage return, so the reader splits on both +CR and LF instead of iterating over lines. +""" + +import contextlib +import json +import os +import re +import signal +import sqlite3 +import subprocess +import threading +from collections.abc import Callable +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from .argv import build_argv +from .config import Settings +from .db import Closed, Database +from .jobspec import JobSpec +from .media import latest_preview +from .postprocess import PluginError, run_stage +from .progress import ProgressModel, load_weights + +# "denoise 7/20 " +PROGRESS = re.compile(r"^(?P\S.*?)\s{2,}(?P\d+)/(?P\d+)\s*$") + +TERMINAL_STATES = {"completed", "failed", "cancelled"} +Listener = Callable[[dict[str, Any]], None] + + +class JobRunner: + """Owns the worker thread, the current process and the event listeners.""" + + def __init__(self, database: Database, config: Settings) -> None: + self.db = database + self.config = config + self.model = ProgressModel(load_weights(config.progress_weights_path)) + self._lock = threading.Lock() + self._wake = threading.Event() + self._stop = threading.Event() + self._process: subprocess.Popen[str] | None = None + # start_new_session makes the child its own group leader, so the group + # id equals its pid and stays valid even after the leader is reaped. + self._pgid: int | None = None + self._current: int | None = None + # Cancellation is recorded, not inferred: a killed child may still + # exit with a normal status if it traps the signal. + self._cancelled: set[int] = set() + self._listeners: list[Listener] = [] + self._thread = threading.Thread(target=self._loop, daemon=True) + + # ── lifecycle ──────────────────────────────────────────────────────── + def start(self) -> None: + # A previous backend may have died mid-job: nothing is running now. + self.db.run( + "UPDATE jobs SET state='failed', finished_at=datetime('now'), " + "error='interrupted by a backend restart' WHERE state='running'" + ) + self._thread.start() + self._wake.set() + + def shutdown(self) -> None: + self._stop.set() + self._wake.set() + with self._lock: + if self._current is not None: + self._cancelled.add(self._current) + self.cancel_current() + self._thread.join(timeout=30) + # If the worker is still winding down, record the outcome here: the + # database is about to close and a job must never stay 'running'. + with self._lock: + pending = self._current + if pending is not None: + with contextlib.suppress(Closed): + self._finish(pending, "cancelled", error="backend shutting down") + + # ── public API ─────────────────────────────────────────────────────── + def submit(self, spec: JobSpec) -> dict[str, Any]: + job_id = self.db.run( + "INSERT INTO jobs (state, prompt, params) VALUES ('queued', ?, ?)", + (spec.prompt, spec.model_dump_json()), + ) + self._wake.set() + job = self.get(job_id) + self._emit(job) + return job + + def get(self, job_id: int) -> dict[str, Any] | None: + row = self.db.query_one("SELECT * FROM jobs WHERE id = ?", (job_id,)) + return self._decorate(_row(row)) if row else None + + def preview_dir(self, job_id: int) -> Path: + return self.config.data_dir / "jobs" / str(job_id) / "preview" + + def _decorate(self, job: dict[str, Any]) -> dict[str, Any]: + """Attach the newest preview and the weighted progress estimate.""" + newest = ( + latest_preview(self.preview_dir(job["id"])) + if job["params"].get("preview") + else None + ) + job["preview_step"] = newest[0] if newest else None + job["elapsed"] = _elapsed(job) + job["remaining"] = ( + _remaining(job["progress"], job["elapsed"]) + if job["state"] == "running" + else None + ) + return job + + def listing(self, limit: int = 100) -> list[dict[str, Any]]: + rows = self.db.query_all( + "SELECT * FROM jobs ORDER BY id DESC LIMIT ?", (limit,) + ) + return [self._decorate(_row(row)) for row in rows] + + def cancel(self, job_id: int) -> dict[str, Any] | None: + job = self.get(job_id) + if job is None or job["state"] in TERMINAL_STATES: + return job + with self._lock: + # Recorded first: a job claimed but not yet spawned would otherwise + # slip through and leave an orphan process behind. + self._cancelled.add(job_id) + is_current = self._current == job_id and self._process is not None + if is_current: + self.cancel_current() + elif job["state"] == "queued": + # The request stays recorded: the worker may already be claiming + # this job, and it checks the set right after spawning. + self._finish(job_id, "cancelled", error="cancelled before it started") + return self.get(job_id) + + def cancel_current(self) -> None: + """Signal the child and return: the worker thread owns its lifecycle.""" + with self._lock: + process = self._process + if process is None or process.poll() is not None: + return + with self._lock: + pgid = self._pgid + if pgid is None: + return + _signal_group(pgid, signal.SIGTERM) + killer = threading.Timer( + self.config.kill_grace, _signal_group, args=(pgid, signal.SIGKILL) + ) + killer.daemon = True + killer.start() + + def add_listener(self, listener: Listener) -> None: + self._listeners.append(listener) + + def remove_listener(self, listener: Listener) -> None: + if listener in self._listeners: + self._listeners.remove(listener) + + # ── worker ─────────────────────────────────────────────────────────── + def _loop(self) -> None: + while not self._stop.is_set(): + self._wake.wait(timeout=1.0) + self._wake.clear() + while not self._stop.is_set(): + try: + row = self.db.query_one( + "SELECT * FROM jobs WHERE state = 'queued' ORDER BY id LIMIT 1" + ) + except Closed: + return + if row is None: + break + try: + self._run(_row(row)) + except Closed: + return + + def _run(self, job: dict[str, Any]) -> None: + job_id = job["id"] + spec = JobSpec.model_validate(job["params"]) + directory = self.config.data_dir / "jobs" / str(job_id) + directory.mkdir(parents=True, exist_ok=True) + output = directory / "out.mp4" + preview_dir = directory / "preview" if spec.preview else None + frames_dir = directory / "frames" if spec.write_frames else None + for extra in (preview_dir, frames_dir): + if extra is not None: + extra.mkdir(parents=True, exist_ok=True) + argv = build_argv( + spec, + self.config.binary, + self.config.model_dir, + output, + frames_dir=frames_dir, + preview_dir=preview_dir, + ) + log_path = directory / "job.log" + self.db.run( + "UPDATE jobs SET state='running', started_at=datetime('now'), " + "argv=?, output_path=?, log_path=?, phase=NULL, completed=0, total=0, " + "progress=0.0 WHERE id=?", + (json.dumps(argv), str(output), str(log_path), job_id), + ) + self._emit(self.get(job_id)) + + try: + process = subprocess.Popen( # noqa: S603 - argv list, no shell + argv, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + errors="replace", + start_new_session=True, + ) + except OSError as error: + self._finish(job_id, "failed", error=f"cannot start h3: {error}") + return + + with self._lock: + self._process = process + self._pgid = process.pid + self._current = job_id + # A shutdown that lands between the claim and this point must not + # leave the child running: treat it as a cancellation. + if self._stop.is_set(): + self._cancelled.add(job_id) + cancel_pending = job_id in self._cancelled + if cancel_pending: + self.cancel_current() + + # h3 spawns FFmpeg, which inherits stderr: a lingering grandchild would + # keep the pipe open forever, so the reader lives in its own thread and + # the pipe is closed from this side if it outlives the process. + tail: list[str] = [] + reader = threading.Thread( + target=self._pump, + args=(process, job_id, log_path, tail, spec), + daemon=True, + ) + reader.start() + code = process.wait() + # Reap anything h3 left behind (FFmpeg, in practice) before waiting on + # the reader: an orphan would otherwise hold the stderr pipe open. + _signal_group(process.pid, signal.SIGKILL) + reader.join(timeout=5) + if reader.is_alive() and process.stderr is not None: + with contextlib.suppress(OSError, ValueError): + process.stderr.close() + reader.join(timeout=5) + with self._lock: + self._process = None + self._pgid = None + self._current = None + + with self._lock: + was_cancelled = job_id in self._cancelled + self._cancelled.discard(job_id) + + if code == 0 and not was_cancelled: + try: + if spec.postprocess and output.is_file(): + run_stage(self.config, output, spec.postprocess) + except PluginError as error: + # The raw video stays where it is: the generation succeeded. + self._finish(job_id, "failed", error=str(error)) + return + self._finish(job_id, "completed") + elif was_cancelled: + self._finish(job_id, "cancelled", error="cancelled") + else: + self._finish(job_id, "failed", error=_reason(tail, code)) + + def _pump( + self, + process: subprocess.Popen[str], + job_id: int, + log_path: Path, + tail: list[str], + spec: JobSpec, + ) -> None: + """Mirror stderr into the log and turn progress lines into updates.""" + buffer = "" + if process.stderr is None: + return + with log_path.open("w", encoding="utf-8") as log: + while True: + try: + chunk = process.stderr.read(1) + except (OSError, ValueError): + break + if not chunk: + break + log.write(chunk) + if chunk in "\r\n": + line, buffer = buffer, "" + if line.strip(): + self._consume(job_id, line, tail, spec) + log.flush() + else: + buffer += chunk + if buffer.strip(): + self._consume(job_id, buffer, tail, spec) + + def _consume( + self, job_id: int, line: str, tail: list[str], spec: JobSpec + ) -> None: + match = PROGRESS.match(line.strip("\r\n")) + if match: + phase = match["phase"].strip() + completed = int(match["completed"]) + total = int(match["total"]) + # max(): an unknown phase reports 0, and the bar must never regress. + self.db.run( + "UPDATE jobs SET phase=?, completed=?, total=?, " + "progress=max(progress, ?) WHERE id=?", + ( + phase, + completed, + total, + self.model.fraction(spec, phase, completed, total), + job_id, + ), + ) + self._emit(self.get(job_id)) + return + tail.append(line.strip()) + del tail[:-20] + + def _finish(self, job_id: int, state: str, error: str | None = None) -> None: + # progress is NOT NULL: a cancelled or failed job keeps what it reached. + if state == "completed": + self.db.run( + "UPDATE jobs SET state=?, error=?, finished_at=datetime('now'), " + "progress=1.0 WHERE id=?", + (state, error, job_id), + ) + else: + self.db.run( + "UPDATE jobs SET state=?, error=?, finished_at=datetime('now') " + "WHERE id=?", + (state, error, job_id), + ) + self._emit(self.get(job_id)) + + def _emit(self, job: dict[str, Any] | None) -> None: + if job is None: + return + for listener in list(self._listeners): + try: + listener(job) + except Exception: # noqa: BLE001 - a broken listener must not stop a job + self.remove_listener(listener) + + +def _signal_group(pgid: int, number: int) -> None: + """Signal the whole group, including children the leader left behind.""" + with contextlib.suppress(ProcessLookupError, PermissionError, OSError): + os.killpg(pgid, number) + + +def _reason(tail: list[str], code: int) -> str: + for line in reversed(tail): + if line.startswith("h3:"): + return line + return tail[-1] if tail else f"h3 exited with code {code}" + + +def _remaining(progress: float, elapsed: float | None) -> float | None: + """Correct the estimate with the pace this run is actually keeping.""" + if not elapsed or progress <= 0.02: + return None + return max(elapsed / progress - elapsed, 0.0) + + +def _elapsed(job: dict[str, Any]) -> float | None: + started = job.get("started_at") + if not started: + return None + ended = job.get("finished_at") + start = datetime.strptime(started, "%Y-%m-%d %H:%M:%S").replace(tzinfo=UTC) + stop = ( + datetime.strptime(ended, "%Y-%m-%d %H:%M:%S").replace(tzinfo=UTC) + if ended + else datetime.now(UTC) + ) + return max((stop - start).total_seconds(), 0.0) + + +def _row(row: sqlite3.Row) -> dict[str, Any]: + job = dict(row) + job["params"] = json.loads(job["params"]) if job["params"] else {} + job["argv"] = json.loads(job["argv"]) if job["argv"] else None + return job diff --git a/webui/backend/app/system.py b/webui/backend/app/system.py new file mode 100644 index 00000000..6f23226a --- /dev/null +++ b/webui/backend/app/system.py @@ -0,0 +1,88 @@ +"""Device and checkpoint inventory, read from `h3 --info`. + +`h3 --info` only reads checkpoint headers, so it is cheap enough to call on +demand and cache. Everything degrades to `available: false` with a reason +instead of raising, so the UI can explain what is missing. +""" + +import re +import subprocess +from pathlib import Path +from typing import Any + +_COMPONENT = re.compile( + r"^\s{2}(?P