diff --git a/README.md b/README.md index c24e0951a..446d157fb 100644 --- a/README.md +++ b/README.md @@ -653,6 +653,7 @@ The server exposes: - `GET /v1/models` - `POST /v1/audio/speech` - `POST /v1/audio/transcriptions` +- `POST /v1/audio/alignments` - `POST /v1/tasks/run` More server examples are in [app/server/README.md](app/server/README.md). diff --git a/app/server/README.md b/app/server/README.md index 5e6acd0e0..b37030c3c 100644 --- a/app/server/README.md +++ b/app/server/README.md @@ -380,6 +380,20 @@ The stream emits `transcript.text.delta` events, one final `transcript.text.done Note that `stream=true` streams the *output* of an already-uploaded file: the whole recording is sent first, and the deltas describe decoding it. It shortens time-to-first-token on long audio, but nothing can appear while the speaker is still talking. For that, use the live endpoint below. +### `POST /v1/audio/alignments` + +Multipart forced-alignment request using uploaded audio bytes and a known transcript. Use this when the server cannot see the client's local audio path, for example when the server is remote or running in Docker. + +```bash +curl http://127.0.0.1:8080/v1/audio/alignments \ + -F model=qwen3-align \ + -F language=en \ + -F text='The task has completed successfully.' \ + -F file=@/path/to/input.wav +``` + +`file`, `model`, and `text` are required; `language` is optional. The selected model must be configured with `task: "align"` and `mode: "offline"`. Uploaded WAV bytes are decoded in memory and are not written to a temporary file. The response includes word timestamps in seconds plus sample offsets. + ### `POST /v1/audio/transcriptions/live` Streams raw PCM **as it is captured** and returns transcript deltas on the same connection, so partial text can appear while the user is still speaking. diff --git a/app/server/main.cpp b/app/server/main.cpp index 8fe29f694..1fec357ef 100644 --- a/app/server/main.cpp +++ b/app/server/main.cpp @@ -111,6 +111,8 @@ void print_help() { << " raw PCM in a chunked body, speech audio deltas as SSE on the same connection\n" << " POST /v1/audio/transcriptions\n" << " fields: file, model, language, prompt, stream\n" + << " POST /v1/audio/alignments\n" + << " fields: file, model, text, language\n" << " OpenAI-style streaming: speech stream_format=sse|audio, transcription stream=true\n" << " POST /v1/audio/transcriptions/live?model=\n" << " raw PCM in a chunked body, transcript deltas as SSE on the same connection\n" diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index 74d1e3807..2f2cfbd2d 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -795,6 +795,53 @@ std::string task_result_json(const engine::runtime::TaskResult & result, double return task_result_json_with_timing(result, timing_json(wall_ms)); } +std::string alignment_result_json( + const engine::runtime::TaskResult & result, + const engine::runtime::AudioBuffer & audio, + double wall_ms) { + if (audio.sample_rate <= 0) { + throw std::runtime_error("alignment timing requires a positive input sample rate"); + } + + std::ostringstream out; + out << "{"; + bool first = true; + auto field = [&](const std::string & name) { + if (!first) { + out << ","; + } + first = false; + out << json_quote(name) << ":"; + }; + if (result.text_output.has_value()) { + field("text"); + out << json_quote(result.text_output->text); + if (!result.text_output->language.empty()) { + field("language"); + out << json_quote(result.text_output->language); + } + } + field("words"); + out << "["; + for (size_t i = 0; i < result.word_timestamps.size(); ++i) { + if (i != 0) { + out << ","; + } + const auto & word = result.word_timestamps[i]; + out << "{\"word\":" << json_quote(word.word) + << ",\"start\":" << (static_cast(word.span.start_sample) / audio.sample_rate) + << ",\"end\":" << (static_cast(word.span.end_sample) / audio.sample_rate) + << ",\"start_sample\":" << word.span.start_sample + << ",\"end_sample\":" << word.span.end_sample + << ",\"confidence\":" << word.confidence << "}"; + } + out << "]"; + field("timing"); + out << timing_json(wall_ms, audio); + out << "}"; + return out.str(); +} + std::string streaming_task_result_json( const engine::runtime::TaskResult & result, const std::optional & ttft_ms) { @@ -1107,6 +1154,9 @@ HttpResponse ServerState::handle(const HttpRequest & request) { else if (request.method == "POST" && request.path == "/v1/audio/transcriptions") { response = handle_transcription(request); } + else if (request.method == "POST" && request.path == "/v1/audio/alignments") { + response = handle_alignment(request); + } // Separate path rather than a flag on the endpoint above: the input transport // differs (raw chunked PCM vs a complete upload), so keeping it distinct leaves // every existing transcription client untouched. @@ -2588,6 +2638,130 @@ HttpResponse ServerState::run_transcription_stream( }); } +HttpResponse ServerState::handle_alignment(const HttpRequest & request) { + std::string content_type; + if (const auto it = request.headers.find("content-type"); it != request.headers.end()) { + content_type = it->second; + } + if (const auto boundary = extract_multipart_boundary(content_type)) { + return handle_alignment_multipart(request.body, *boundary); + } + return error_response( + 400, + "audio alignment requests must use multipart/form-data", + "invalid_request_error"); +} + +HttpResponse ServerState::handle_alignment_multipart(const std::string & body_text, const std::string & boundary) { + const auto parts = parse_multipart_body(body_text, boundary); + log_multipart_request_summary_if_enabled(config_, parts); + + const MultipartPart * file_part = nullptr; + std::string model_id; + std::string text; + std::string language; + std::optional busy_timeout_ms; + for (const auto & part : parts) { + if (part.name == "file") { + file_part = ∂ + } else if (part.name == "model") { + model_id = part.data; + } else if (part.name == "text") { + text = part.data; + } else if (part.name == "language") { + language = part.data; + } else if (part.name == "busy_timeout_ms") { + try { + busy_timeout_ms = std::stoi(part.data); + } catch (const std::exception &) { + return error_response( + 400, + "multipart busy_timeout_ms field must be an integer", + "invalid_request_error"); + } + if (*busy_timeout_ms < 0) { + return error_response( + 400, + "busy_timeout_ms must be >= 0 (0 means no client-side bound)", + "invalid_request_error"); + } + } + } + if (file_part == nullptr || file_part->data.empty()) { + return error_response( + 400, + "multipart alignment request requires a non-empty 'file' field", + "invalid_request_error"); + } + if (model_id.empty()) { + return error_response( + 400, + "multipart alignment request requires a 'model' field", + "invalid_request_error"); + } + if (text.empty()) { + return error_response( + 400, + "multipart alignment request requires a non-empty 'text' field", + "invalid_request_error"); + } + if (!is_wav_upload_filename(file_part->filename)) { + return error_response( + 400, + "only WAV audio uploads are currently supported for alignment", + "invalid_request_error"); + } + + LoadedModel * model_ptr = nullptr; + { + std::lock_guard state_lock(models_mutex_); + const auto it = model_index_.find(model_id); + if (it == model_index_.end()) { + return error_response( + 400, + "unknown model id: " + model_id, + "invalid_request_error"); + } + model_ptr = models_.at(it->second).get(); + } + auto & model = *model_ptr; + if (model.task.task != engine::runtime::VoiceTaskKind::Alignment) { + return error_response( + 400, + "audio alignment requires a model configured with task=align", + "invalid_request_error"); + } + if (model.task.mode != engine::runtime::RunMode::Offline) { + return error_response( + 400, + "audio alignment requires a model configured with mode=offline", + "invalid_request_error"); + } + + engine::runtime::TaskRequest task_request; + task_request.audio_input = minitts::cli::read_audio_buffer(std::string_view(file_part->data)); + task_request.text_input = engine::runtime::Transcript{std::move(text), std::move(language)}; + task_request = apply_default_request_options(model, std::move(task_request)); + return run_alignment(model, task_request, busy_timeout_ms); +} + +HttpResponse ServerState::run_alignment( + LoadedModel & model, + const engine::runtime::TaskRequest & request, + std::optional busy_timeout_ms) { + if (model_run_mode(model) != engine::runtime::RunMode::Offline) { + throw std::runtime_error("audio alignment requires a model configured with mode=offline"); + } + const auto timed_result = run_model(model, request, busy_timeout_ms); + if (timed_result.result.word_timestamps.empty()) { + throw std::runtime_error("alignment model produced no word timestamps"); + } + if (!request.audio_input.has_value()) { + throw std::runtime_error("alignment timing requires audio_input"); + } + return json_response(alignment_result_json(timed_result.result, *request.audio_input, timed_result.wall_ms)); +} + // Live PCM ingest. The client streams raw interleaved samples in a chunked request // body while transcript deltas stream back as SSE on the same connection, so // partials track capture instead of waiting for a finished upload. Same event shape diff --git a/app/server/runtime.h b/app/server/runtime.h index 07c5a52b9..0b7241a4d 100644 --- a/app/server/runtime.h +++ b/app/server/runtime.h @@ -178,6 +178,12 @@ class ServerState final : public IHttpHandler { LoadedModel & model, const engine::runtime::TaskRequest & request, std::optional busy_timeout_ms = std::nullopt); + HttpResponse handle_alignment(const HttpRequest & request); + HttpResponse handle_alignment_multipart(const std::string & body_text, const std::string & boundary); + HttpResponse run_alignment( + LoadedModel & model, + const engine::runtime::TaskRequest & request, + std::optional busy_timeout_ms = std::nullopt); HttpResponse handle_transcription_live(const HttpRequest & request); HttpResponse handle_generic_run(const std::string & body_text); HttpResponse handle_generic_stream(const std::string & body_text);