From df50919da631acd6497cb1d6cb874142be671de7 Mon Sep 17 00:00:00 2001 From: Dave Taylor Date: Fri, 28 Aug 2026 08:19:09 -0400 Subject: [PATCH 1/2] server: cache the model contract option instead of re-reading the GGUF per request build_speech_request called model_accepts_request_option() on every POST /v1/audio/speech. That resolves the model contract through model_contract(family), which reads the embedded spec with gguf_init_from_file() on the full GGUF. The resolution cache is thread_local and serve_http handles each request on a fresh detached thread, so the cache is cold every time and each request re-opens and re-parses the whole model file. With a 298 MB GGUF this adds a fixed ~900 ms to every request, independent of backend, text length and audio duration. Resolve the option once per model instead: - LoadedModel gains accepts_reference_text, defaulting to true (the existing permissive behaviour for models with no contract). - refresh_model_option_flags() sets it in make_model and again in handle_model_load, so reconfiguration stays correct. - build_speech_request reads the cached flag. A contract that fails to resolve is caught at registration, reported on stderr, and falls back to the permissive default rather than throwing per request. Measured on Windows x64 / Vulkan with supertonic-3-f16.gguf, same server.json and request: ttfb 0.956-1.153 s before, 0.045-0.049 s after. Response bytes are unchanged (Content-Length 114090, identical MD5 before and after). --- app/server/runtime.cpp | 36 +++++++++++++++++++++++++++--------- app/server/runtime.h | 9 +++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index 03f1fcf5..309221b5 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -1180,9 +1180,31 @@ std::unique_ptr ServerState::make_model(ServerModelCon engine::runtime::parse_run_mode(loaded->config.mode), }; load_voice_presets(*loaded); + refresh_model_option_flags(*loaded); return loaded; } +void ServerState::refresh_model_option_flags(LoadedModel & model) { + const auto effective_override = model.config.model_spec_override.has_value() + ? model.config.model_spec_override + : config_.model_spec_override; + try { + model.accepts_reference_text = model_accepts_request_option( + model.config.family, + "reference_text", + effective_override, + model.config.path); + } catch (const std::exception & ex) { + // Keep registration robust: a model whose contract cannot be resolved gets + // the same permissive behavior as a model with no contract; the failure is + // reported so the misconfiguration stays visible. + std::cerr << "[server] model '" << model.config.id + << "': reference_text option check failed (" << ex.what() + << "); assuming the option is accepted\n"; + model.accepts_reference_text = true; + } +} + HttpResponse ServerState::handle_model_load(const std::string & body_text) { if (!config_.ui_management) { return error_response(403, "dynamic model management is disabled", "forbidden"); @@ -1221,6 +1243,7 @@ HttpResponse ServerState::handle_model_load(const std::string & body_text) { engine::runtime::parse_run_mode(existing->config.mode), }; load_voice_presets(*existing); + refresh_model_option_flags(*existing); } ensure_model_loaded_locked(*existing); return json_response( @@ -1869,15 +1892,10 @@ engine::runtime::TaskRequest ServerState::build_speech_request(const LoadedModel bool voice_field_is_preset = false; const auto * preset = select_voice_preset(model, body, voice_field_is_preset); - const auto effective_model_spec_override = model.config.model_spec_override.has_value() - ? model.config.model_spec_override - : config_.model_spec_override; - const bool can_inject_reference_text = - model_accepts_request_option( - model.config.family, - "reference_text", - effective_model_spec_override, - model.config.path); + // Resolved once at registration (refresh_model_option_flags): calling + // model_accepts_request_option per request re-reads the model file's embedded + // spec on the request thread, which cost ~0.9 s per request for large GGUFs. + const bool can_inject_reference_text = model.accepts_reference_text; engine::runtime::VoiceCondition voice; bool has_voice = false; diff --git a/app/server/runtime.h b/app/server/runtime.h index d4658885..07c5a52b 100644 --- a/app/server/runtime.h +++ b/app/server/runtime.h @@ -65,6 +65,12 @@ class ServerState final : public IHttpHandler { mutable std::shared_mutex metadata_mutex; std::unordered_map voice_presets; std::optional default_voice_preset; + // Whether this model's contract accepts the `reference_text` request + // option, resolved once at registration (refresh_model_option_flags). + // Resolving it per request re-reads the model file's embedded spec on + // the request thread, which costs ~0.9 s per request for large GGUFs. + // `true` mirrors model_accepts_request_option's no-contract behavior. + bool accepts_reference_text = true; // Serializes runs on this model and bounds how long a caller waits for its // turn; see BusyGuard. BusyGuard busy; @@ -85,6 +91,9 @@ class ServerState final : public IHttpHandler { void load_models(); std::unique_ptr make_model(ServerModelConfig config); + // Recompute the per-model, config-derived request-option flags (currently + // accepts_reference_text). Called at registration and on reconfiguration. + void refresh_model_option_flags(LoadedModel & model); std::filesystem::path resolve_ui_model_path(const std::filesystem::path & path) const; HttpResponse handle_model_load(const std::string & body_text); HttpResponse handle_model_unload(const std::string & body_text); From 2fe62cfd97a1897d49568dbb0d724b4ef8c93d45 Mon Sep 17 00:00:00 2001 From: Dave Taylor Date: Fri, 28 Aug 2026 17:58:29 -0400 Subject: [PATCH 2/2] server: let an unresolvable model contract fail at registration Review feedback: the catch added in the previous commit swallowed real model-spec errors. An invalid spec override logged the real reason, started anyway, and then failed the request with an unrelated "model path does not exist". model_accepts_request_option already returns true for a model with no contract (it swallows only the missing-contract errors and rethrows the rest), so the catch could only ever extend permissiveness to genuine misconfigurations. It is removed rather than narrowed: nothing replaces it. An invalid spec now fails once, at registration, with the real error, instead of 500ing every request as it did before this branch. --- app/server/runtime.cpp | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index 309221b5..74d1e380 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -1188,21 +1188,16 @@ void ServerState::refresh_model_option_flags(LoadedModel & model) { const auto effective_override = model.config.model_spec_override.has_value() ? model.config.model_spec_override : config_.model_spec_override; - try { - model.accepts_reference_text = model_accepts_request_option( - model.config.family, - "reference_text", - effective_override, - model.config.path); - } catch (const std::exception & ex) { - // Keep registration robust: a model whose contract cannot be resolved gets - // the same permissive behavior as a model with no contract; the failure is - // reported so the misconfiguration stays visible. - std::cerr << "[server] model '" << model.config.id - << "': reference_text option check failed (" << ex.what() - << "); assuming the option is accepted\n"; - model.accepts_reference_text = true; - } + // Deliberately uncaught: model_accepts_request_option already returns true for a + // model with no contract, swallowing only the missing-contract errors and + // rethrowing the rest. Anything that propagates here is therefore a real + // misconfiguration (invalid spec, missing override file, family mismatch) and + // must fail at registration rather than be assumed away. + model.accepts_reference_text = model_accepts_request_option( + model.config.family, + "reference_text", + effective_override, + model.config.path); } HttpResponse ServerState::handle_model_load(const std::string & body_text) {