Skip to content

cli, core: add --no-audio option to skip audio VAE decode and write video-only MP4 #58

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
lorenzoperrone wants to merge 1 commit into
base: main
Choose a base branch
from
Open

cli, core: add --no-audio option to skip audio VAE decode and write video-only MP4 #58

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 20 additions & 8 deletions h3.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,10 @@ static int h3_valid_params(h3_ctx *ctx, const h3_params *params) {
h3_set_error(ctx, "denoising preview requires a frame callback");
return 0;
}
if (params->disable_audio != 0 && params->disable_audio != 1) {
h3_set_error(ctx, "audio disable must be zero or one");
return 0;
}
if (params->core_reuse > 1 && params->denoise_reuse > 1) {
h3_set_error(ctx, "core reuse and denoiser reuse cannot be combined");
return 0;
Expand Down Expand Up @@ -1591,12 +1595,15 @@ h3_result *h3_generate(h3_ctx *ctx, const char *prompt,
if (!dit_is_cached) h3_dit_free(dit);
dit = NULL;
if (progress.cancelled) goto cleanup;
h3_progress_emit(&progress, "audio VAE", 0, 7);
if (!h3_audio_vae_decode(audio_vae_path, "h3_shaders.metal", audio,
temporal.audio_t, h3_audio_vae_progress_bridge,
&progress, &waveform, detail, sizeof(detail))) {
h3_set_error(ctx, "%s", detail);
goto cleanup;
if (!params->disable_audio) {
h3_progress_emit(&progress, "audio VAE", 0, 7);
if (!h3_audio_vae_decode(audio_vae_path, "h3_shaders.metal", audio,
temporal.audio_t, h3_audio_vae_progress_bridge,
&progress, &waveform, detail,
sizeof(detail))) {
h3_set_error(ctx, "%s", detail);
goto cleanup;
}
}
free(audio);
audio = NULL;
Expand Down Expand Up @@ -1667,11 +1674,16 @@ h3_result *h3_generate(h3_ctx *ctx, const char *prompt,
}
if (params->output_path && *params->output_path) {
h3_progress_emit(&progress, "FFmpeg", 0, frames.frames);
if (!h3_ffmpeg_write_av_rgb24_f32(
int written = params->disable_audio ?
h3_ffmpeg_write_rgb24(params->output_path, rgb8, frames.frames,
output_width, output_height, H3_FPS,
detail, sizeof(detail)) :
h3_ffmpeg_write_av_rgb24_f32(
params->output_path, rgb8, frames.frames, output_width,
output_height, H3_FPS, waveform.pcm, waveform.samples,
waveform.channels, waveform.sample_rate,
detail, sizeof(detail))) {
detail, sizeof(detail));
if (!written) {
h3_set_error(ctx, "%s", detail);
goto cleanup;
}
Expand Down
5 changes: 4 additions & 1 deletion h3.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,8 @@ typedef struct {
int use_slower_grouped_quantizer;
/* Decode and deliver one representative frame after every Euler step. */
int preview_denoise;
/* Skip the audio VAE decode and write a video-only MP4. */
int disable_audio;
h3_frame_callback on_frame;
h3_progress_callback on_progress;
void *callback_opaque;
Expand All @@ -131,7 +133,8 @@ typedef struct {
#define H3_PARAMS_DEFAULT { \
H3_DEFAULT_WIDTH, H3_DEFAULT_HEIGHT, H3_DEFAULT_FRAMES, H3_DEFAULT_STEPS, \
UINT64_C(42), NULL, NULL, NULL, NULL, 0, H3_REFERENCE_IMAGE_MATCH, \
1, H3_DEFAULT_DIT_LAYERS, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, NULL, NULL, NULL \
1, H3_DEFAULT_DIT_LAYERS, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
0, 0, 0, NULL, NULL, NULL \
}

typedef struct {
Expand Down
10 changes: 10 additions & 0 deletions h3_cli.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ static void print_help(void) {
puts(" !layers [N] Set or show active DiT blocks");
puts(" !core-reuse [N] Set or show core reuse");
puts(" !token-reduction [on|off] Toggle token reduction");
puts(" !audio [on|off] Toggle generated audio");
puts(" !ssd-streaming [on|off] Toggle original-BF16 SSD streaming");
puts(" !int8-row-fc2 [on|off] Toggle faster one-scale FC2");
puts(" !reference-rope [on|off] Toggle released spatial RoPE");
Expand Down Expand Up @@ -198,6 +199,7 @@ static void print_status(const h3_cli_state *state) {
state->params.use_int8_row_fc2 ? "int8 row" : "int8 grouped");
printf("Spatial RoPE: %s\n", state->params.use_reference_rope ?
"released reference" : "native 256 adapted");
printf("Audio: %s\n", state->params.disable_audio ? "off" : "on");
if (state->random_seed) puts("Seed: random");
else printf("Seed: %" PRIu64 "\n", state->params.seed);
printf("First: %s\n", state->first_frame ? state->first_frame : "none");
Expand Down Expand Up @@ -585,6 +587,14 @@ static int process_command(h3_cli_state *state, char *line, int *repeat) {
state->params.token_reduction = value;
printf("Token reduction: %s\n", value ? "on" : "off");
}
} else if (!strcasecmp(command, "audio")) {
int value;
if (!parse_toggle(argument, !state->params.disable_audio, &value))
fprintf(stderr, "h3: use on or off\n");
else {
state->params.disable_audio = !value;
printf("Audio: %s\n", value ? "on" : "off");
}
} else if (!strcasecmp(command, "ssd-streaming")) {
int value;
if (!parse_toggle(argument, state->params.ssd_streaming, &value))
Expand Down
7 changes: 5 additions & 2 deletions main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ static void usage(const char *program) {
" -d, --model-dir PATH MiniMax-H3 local directory\n"
" -p, --prompt TEXT Raw H3 prompt\n"
" -o, --output PATH Output MP4 (default: outputs/h3.mp4)\n"
" --no-audio Skip the audio VAE and write a video-only MP4\n"
" --width N Output width (default: 864)\n"
" --height N Output height (default: 480)\n"
" --render-width N Lower internal model width (optional)\n"
Expand Down Expand Up @@ -250,12 +251,13 @@ int main(int argc, char **argv) {
OPT_SEED,
OPT_FIRST, OPT_LAST, OPT_REF_IMAGE, OPT_REF_IMAGE_SIZE,
OPT_REF_VIDEO, OPT_REF_SILENT_VIDEO, OPT_REF_VIDEO_AUDIO,
OPT_REF_AUDIO, OPT_FRAMES_DIR, OPT_SHOW, OPT_ZOOM,
OPT_PROFILE, OPT_INFO };
OPT_REF_AUDIO, OPT_FRAMES_DIR, OPT_SHOW, OPT_ZOOM,
OPT_PROFILE, OPT_INFO, OPT_NO_AUDIO };
static const struct option options[] = {
{"model-dir", required_argument, NULL, 'd'},
{"prompt", required_argument, NULL, 'p'},
{"output", required_argument, NULL, 'o'},
{"no-audio", no_argument, NULL, OPT_NO_AUDIO},
{"width", required_argument, NULL, OPT_WIDTH},
{"height", required_argument, NULL, OPT_HEIGHT},
{"render-width", required_argument, NULL, OPT_RENDER_WIDTH},
Expand Down Expand Up @@ -327,6 +329,7 @@ int main(int argc, char **argv) {
case 'p': prompt = optarg; break;
case 'o': output = optarg; break;
case 'h': usage(argv[0]); return 0;
case OPT_NO_AUDIO: params.disable_audio = 1; break;
case OPT_WIDTH: params.width = parse_int(optarg, "width"); break;
case OPT_HEIGHT: params.height = parse_int(optarg, "height"); break;
case OPT_RENDER_WIDTH:
Expand Down