From 1f4de9c8091b7f5926c4ae71b26a5e6451a1d23f Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Thu, 3 Sep 2026 13:01:32 +0200 Subject: [PATCH 1/5] chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `caf7eae5282d840d77e9f91a56df7d2ef28fa612` (#11842) :arrow_up: Update ikawrakow/ik_llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/ik-llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/ik-llama-cpp/Makefile b/backend/cpp/ik-llama-cpp/Makefile index c659e6d35e32..ca4c45114431 100644 --- a/backend/cpp/ik-llama-cpp/Makefile +++ b/backend/cpp/ik-llama-cpp/Makefile @@ -1,5 +1,5 @@ -IK_LLAMA_VERSION?=3c58ae373a0081c884099f435fb16ca720852bf7 +IK_LLAMA_VERSION?=caf7eae5282d840d77e9f91a56df7d2ef28fa612 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp CMAKE_ARGS?= From 09f42db913d287570b65b5198c0c214633ded1a7 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Thu, 3 Sep 2026 13:01:45 +0200 Subject: [PATCH 2/5] chore: :arrow_up: Update NVIDIA/NeMo-Speech.cpp to `56b60d432f1731d6d5b28a4c5a31cbaf871daba1` (#11846) :arrow_up: Update NVIDIA/NeMo-Speech.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/nemo-speech-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/nemo-speech-cpp/Makefile b/backend/go/nemo-speech-cpp/Makefile index 1120267f14f1..24b944e11ab7 100644 --- a/backend/go/nemo-speech-cpp/Makefile +++ b/backend/go/nemo-speech-cpp/Makefile @@ -12,7 +12,7 @@ # runs 'make -C backend/go/$(BACKEND) build' and then copies package/), so it # has to produce the binary and the package, not just the shared libraries. -NEMO_SPEECH_VERSION?=4f9676226f667d14608487df744f375db87127f8 +NEMO_SPEECH_VERSION?=56b60d432f1731d6d5b28a4c5a31cbaf871daba1 NEMO_SPEECH_REPO?=https://github.com/NVIDIA/NeMo-Speech.cpp GOCMD?=go From e9ba60ba57610670401bd369e33dfa2c385ded24 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Thu, 3 Sep 2026 13:02:00 +0200 Subject: [PATCH 3/5] chore: :arrow_up: Update CrispStrobe/CrispASR to `ff3945c94cab9191199a5d531a32c4e9535c094b` (#11829) :arrow_up: Update CrispStrobe/CrispASR Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/crispasr/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/crispasr/Makefile b/backend/go/crispasr/Makefile index da4d19480eb1..50293f642448 100644 --- a/backend/go/crispasr/Makefile +++ b/backend/go/crispasr/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # CrispASR version (release tag) CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR -CRISPASR_VERSION?=78c545eb80409b91291642ddb23b3a6dc044fd34 +CRISPASR_VERSION?=ff3945c94cab9191199a5d531a32c4e9535c094b SO_TARGET?=libgocrispasr.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From 335acce21fadd818c0d71271331c513e0b522395 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Thu, 3 Sep 2026 13:02:33 +0200 Subject: [PATCH 4/5] fix(ds4): cancel abandoned inference (#11822) Propagate gRPC cancellation into DS4 prompt synchronization and poll it at decode boundaries. Stop on failed stream writes and skip parser finalization and KV persistence for abandoned partial requests. Assisted-by: Codex:gpt-5.6-sol Signed-off-by: Claudio Maradonna --- backend/cpp/ds4/grpc-server.cpp | 206 +++++++--- backend/cpp/ds4/request_lifecycle.h | 111 ++++++ backend/cpp/ds4/request_lifecycle_test.cpp | 414 +++++++++++++++++++++ docs/content/features/backends.md | 9 + 4 files changed, 692 insertions(+), 48 deletions(-) create mode 100644 backend/cpp/ds4/request_lifecycle.h create mode 100644 backend/cpp/ds4/request_lifecycle_test.cpp diff --git a/backend/cpp/ds4/grpc-server.cpp b/backend/cpp/ds4/grpc-server.cpp index 68ebdd3e3551..d00f6ba3f61e 100644 --- a/backend/cpp/ds4/grpc-server.cpp +++ b/backend/cpp/ds4/grpc-server.cpp @@ -12,6 +12,7 @@ #include "dsml_renderer.h" // populated in Task 16 #include "generation_limits.h" #include "kv_cache.h" // populated in Task 17 +#include "request_lifecycle.h" extern "C" { #include "ds4.h" @@ -36,6 +37,7 @@ extern "C" { #include #include #include +#include #include using grpc::Server; @@ -70,6 +72,21 @@ int g_route_timeout_sec = 60; std::atomic g_server{nullptr}; +static bool server_context_cancelled(void *ud) { + return static_cast(ud)->IsCancelled(); +} + +static void set_session_cancel(void *target, ds4cpp::CancelCallback callback, + void *userdata) noexcept { + ds4_session_set_cancel(static_cast(target), callback, userdata); +} + +static bool request_should_continue(ds4cpp::RequestLifecycle *request, + ServerContext *context) { + request->ObserveContextCancellation(context->IsCancelled()); + return request->ShouldContinue(); +} + // Parse a "key:value" option string. Returns empty when no colon. static std::pair split_option(const std::string &opt) { auto colon = opt.find(':'); @@ -239,37 +256,58 @@ static bool apply_engine_option(ds4_engine_options *opt, const std::string &key, // When acting as a distributed coordinator, block until the worker route // covers all layers (ds4_session_distributed_route_ready == 1) or the timeout -// elapses. Returns an empty string on success, or an error message to return -// to the client. No-op when not distributed. +// elapses. No-op when not distributed. // // Takes the g_engine_mu lock by reference and RELEASES it during each poll // sleep. The wait can span up to g_route_timeout_sec seconds while workers // connect; holding g_engine_mu the whole time would block the Status/Health // readiness probes (they also lock g_engine_mu), making LocalAI's loader treat // a still-starting worker as hung. -static std::string wait_route_ready(std::unique_lock &lock) { - if (!g_distributed) return ""; +struct RouteWaitResult { + ds4cpp::RouteWaitDecision decision; + std::string error; +}; + +static RouteWaitResult wait_route_ready(std::unique_lock &lock, + ServerContext *context) { + if (!g_distributed) return {ds4cpp::RouteWaitDecision::Ready, ""}; char err[256] = {0}; const int deadline_polls = g_route_timeout_sec * 10; // 100ms per poll for (int i = 0; i <= deadline_polls; ++i) { int ready = ds4_session_distributed_route_ready(g_session, err, sizeof(err)); - if (ready == 1) return ""; - if (ready < 0) { - return std::string("ds4 distributed route error: ") + - (err[0] ? err : "unknown"); + switch (ds4cpp::DecideRouteWait(ready, context->IsCancelled())) { + case ds4cpp::RouteWaitDecision::Ready: + return {ds4cpp::RouteWaitDecision::Ready, ""}; + case ds4cpp::RouteWaitDecision::Error: + return {ds4cpp::RouteWaitDecision::Error, + std::string("ds4 distributed route error: ") + + (err[0] ? err : "unknown")}; + case ds4cpp::RouteWaitDecision::Cancelled: + return {ds4cpp::RouteWaitDecision::Cancelled, ""}; + case ds4cpp::RouteWaitDecision::Pending: + break; } + if (i == deadline_polls) break; // Release the lock while sleeping so Status/Health and other RPCs can // interleave during worker startup. lock.unlock(); struct timespec ts = {0, 100L * 1000L * 1000L}; // 100ms nanosleep(&ts, nullptr); lock.lock(); + if (context->IsCancelled()) { + return {ds4cpp::RouteWaitDecision::Cancelled, ""}; + } // A concurrent Free() may have torn down the engine while we slept. if (!g_engine || !g_session) { - return "ds4: model unloaded while waiting for distributed route"; + return {ds4cpp::RouteWaitDecision::Error, + "ds4: model unloaded while waiting for distributed route"}; } } - return "ds4 distributed route incomplete: workers not connected (layers uncovered)"; + if (context->IsCancelled()) { + return {ds4cpp::RouteWaitDecision::Cancelled, ""}; + } + return {ds4cpp::RouteWaitDecision::Error, + "ds4 distributed route incomplete: workers not connected (layers uncovered)"}; } static void append_token_text(ds4_engine *engine, int token, std::string &out) { @@ -342,9 +380,9 @@ static void collect_done(void *) {} struct StreamCtx { ds4_engine *engine; ServerWriter *writer; + ds4cpp::RequestLifecycle *request; ds4cpp::DsmlParser parser; int tokens; - bool aborted; // Track which tool indices we've seen TOOL_START for, so subsequent // ARGS deltas can elide the redundant id/name fields. std::vector tool_started; @@ -352,7 +390,7 @@ struct StreamCtx { static void stream_emit(void *ud, int token) { auto *s = static_cast(ud); - if (s->aborted) return; + if (!s->request->ShouldContinue()) return; if (token == ds4_token_eos(s->engine)) return; size_t len = 0; const char *text = ds4_token_text(s->engine, token, &len); @@ -402,7 +440,7 @@ static void stream_emit(void *ud, int token) { reply.set_message(chunk); reply.set_tokens(1); if (any_field) { - if (!s->writer->Write(reply)) s->aborted = true; + s->request->ObserveStreamWrite(s->writer->Write(reply)); } s->tokens++; } @@ -758,15 +796,19 @@ class DS4Backend final : public backend::Backend::Service { return GStatus::OK; } - GStatus Predict(ServerContext *, const backend::PredictOptions *request, + GStatus Predict(ServerContext *context, const backend::PredictOptions *request, backend::Reply *reply) override { std::unique_lock lock(g_engine_mu); if (!g_engine || !g_session) { return GStatus(StatusCode::FAILED_PRECONDITION, "ds4: model not loaded"); } if (GStatus id = check_model_identity(request); !id.ok()) return id; - if (std::string route_err = wait_route_ready(lock); !route_err.empty()) { - return GStatus(StatusCode::UNAVAILABLE, route_err); + RouteWaitResult route = wait_route_ready(lock, context); + if (route.decision == ds4cpp::RouteWaitDecision::Cancelled) { + return GStatus(StatusCode::CANCELLED, "ds4 request cancelled"); + } + if (route.decision == ds4cpp::RouteWaitDecision::Error) { + return GStatus(StatusCode::UNAVAILABLE, route.error); } ds4_tokens prompt = {}; build_prompt(g_engine, request, &prompt); @@ -777,6 +819,7 @@ class DS4Backend final : public backend::Backend::Service { CollectCtx collect = { g_engine, "", ds4cpp::DsmlParser(starts_in_thinking), reply, 0, {}, "", ""}; + ds4cpp::RequestLifecycle lifecycle; std::string cache_key = render_prompt_text(request); size_t cache_hit = maybe_load_cache(cache_key); (void)cache_hit; // future: skip prompt prefix if hit covers full prompt @@ -788,10 +831,19 @@ class DS4Backend final : public backend::Backend::Service { // Either way g_session advances so the disk KV cache picks up a // real checkpoint after the call (see maybe_save_cache below). char err[256] = {0}; - int rc = ds4_session_sync(g_session, &prompt, err, sizeof(err)); + int rc; + { + ds4cpp::CancelCallbackScope cancel_scope( + g_session, set_session_cancel, server_context_cancelled, context); + rc = ds4_session_sync(g_session, &prompt, err, sizeof(err)); + } int prompt_len = prompt.len; ds4_tokens_free(&prompt); - if (rc == 0) { + if (rc == DS4_SESSION_SYNC_INTERRUPTED) { + lifecycle.ObserveContextCancellation(true); + } + const bool generation_started = rc == 0; + if (generation_started) { const int n_predict = ds4cpp::EffectiveGenerationLimit( request->tokens(), ds4_session_ctx(g_session), ds4_session_pos(g_session)); @@ -799,6 +851,7 @@ class DS4Backend final : public backend::Backend::Service { const int draft_max = ds4_engine_mtp_draft_tokens(g_engine); int produced = 0; while (produced < n_predict) { + if (!request_should_continue(&lifecycle, context)) break; SampleParams sp = compute_sample_params(request, collect.parser, think_enabled); int first; if (sp.temperature <= 0.0f) { @@ -823,6 +876,10 @@ class DS4Backend final : public backend::Backend::Service { if (n < 0) { rc = -1; break; } bool stop = false; for (int j = 0; j < n; ++j) { + if (!request_should_continue(&lifecycle, context)) { + stop = true; + break; + } if (accepted[j] == eos) { stop = true; break; } collect_emit(&collect, accepted[j]); if (++produced >= n_predict) { stop = true; break; } @@ -831,12 +888,26 @@ class DS4Backend final : public backend::Backend::Service { } else { collect_emit(&collect, first); if (++produced >= n_predict) break; + if (!request_should_continue(&lifecycle, context)) break; rc = ds4_session_eval(g_session, first, err, sizeof(err)); if (rc != 0) break; } } - collect_done(&collect); } + + request_should_continue(&lifecycle, context); + ds4cpp::TerminalDecision terminal = ds4cpp::ResolveTerminalDecision( + rc == DS4_SESSION_SYNC_INTERRUPTED, rc != 0, + !lifecycle.ShouldFinalize()); + if (!terminal.should_finalize) { + if (terminal.cause == ds4cpp::TerminalCause::EngineError) { + return GStatus(StatusCode::INTERNAL, + std::string("ds4 generation failed: ") + err); + } + return GStatus(StatusCode::CANCELLED, + "ds4 request cancelled"); + } + if (generation_started) collect_done(&collect); maybe_save_cache(cache_key); // Flush any buffered parser state. @@ -844,7 +915,7 @@ class DS4Backend final : public backend::Backend::Service { collect.parser.Flush(events); apply_events(&collect, events); - if (rc != 0) { + if (terminal.cause == ds4cpp::TerminalCause::EngineError) { return GStatus(StatusCode::INTERNAL, std::string("ds4 generation failed: ") + err); } @@ -867,15 +938,19 @@ class DS4Backend final : public backend::Backend::Service { return GStatus::OK; } - GStatus PredictStream(ServerContext *, const backend::PredictOptions *request, + GStatus PredictStream(ServerContext *context, const backend::PredictOptions *request, ServerWriter *writer) override { std::unique_lock lock(g_engine_mu); if (!g_engine || !g_session) { return GStatus(StatusCode::FAILED_PRECONDITION, "ds4: model not loaded"); } if (GStatus id = check_model_identity(request); !id.ok()) return id; - if (std::string route_err = wait_route_ready(lock); !route_err.empty()) { - return GStatus(StatusCode::UNAVAILABLE, route_err); + RouteWaitResult route = wait_route_ready(lock, context); + if (route.decision == ds4cpp::RouteWaitDecision::Cancelled) { + return GStatus(StatusCode::CANCELLED, "ds4 request cancelled"); + } + if (route.decision == ds4cpp::RouteWaitDecision::Error) { + return GStatus(StatusCode::UNAVAILABLE, route.error); } ds4_tokens prompt = {}; build_prompt(g_engine, request, &prompt); @@ -883,9 +958,10 @@ class DS4Backend final : public backend::Backend::Service { const bool think_enabled = ds4_think_mode_enabled(parse_think_mode(request)); const bool starts_in_thinking = think_enabled && request->usetokenizertemplate() && request->messages_size() > 0; + ds4cpp::RequestLifecycle lifecycle; StreamCtx s = { - g_engine, writer, ds4cpp::DsmlParser(starts_in_thinking), - 0, false, {}}; + g_engine, writer, &lifecycle, + ds4cpp::DsmlParser(starts_in_thinking), 0, {}}; std::string cache_key = render_prompt_text(request); size_t cache_hit = maybe_load_cache(cache_key); (void)cache_hit; @@ -893,16 +969,26 @@ class DS4Backend final : public backend::Backend::Service { // Manual loop on g_session - see Predict() above for the rationale. // MTP speculative path used when ds4_engine_mtp_draft_tokens > 0. char err[256] = {0}; - int rc = ds4_session_sync(g_session, &prompt, err, sizeof(err)); + int rc; + { + ds4cpp::CancelCallbackScope cancel_scope( + g_session, set_session_cancel, server_context_cancelled, context); + rc = ds4_session_sync(g_session, &prompt, err, sizeof(err)); + } ds4_tokens_free(&prompt); - if (rc == 0) { + if (rc == DS4_SESSION_SYNC_INTERRUPTED) { + lifecycle.ObserveContextCancellation(true); + } + const bool generation_started = rc == 0; + if (generation_started) { const int n_predict = ds4cpp::EffectiveGenerationLimit( request->tokens(), ds4_session_ctx(g_session), ds4_session_pos(g_session)); const int eos = ds4_token_eos(g_engine); const int draft_max = ds4_engine_mtp_draft_tokens(g_engine); int produced = 0; - while (produced < n_predict && !s.aborted) { + while (produced < n_predict) { + if (!request_should_continue(&lifecycle, context)) break; SampleParams sp = compute_sample_params(request, s.parser, think_enabled); int first; if (sp.temperature <= 0.0f) { @@ -926,43 +1012,67 @@ class DS4Backend final : public backend::Backend::Service { if (n < 0) { rc = -1; break; } bool stop = false; for (int j = 0; j < n; ++j) { + if (!request_should_continue(&lifecycle, context)) { + stop = true; + break; + } if (accepted[j] == eos) { stop = true; break; } stream_emit(&s, accepted[j]); - if (s.aborted) { stop = true; break; } + if (!lifecycle.ShouldContinue()) { stop = true; break; } if (++produced >= n_predict) { stop = true; break; } } if (stop) break; } else { stream_emit(&s, first); - if (s.aborted || ++produced >= n_predict) break; + if (!lifecycle.ShouldContinue() || ++produced >= n_predict) break; + if (!request_should_continue(&lifecycle, context)) break; rc = ds4_session_eval(g_session, first, err, sizeof(err)); if (rc != 0) break; } } - stream_done(&s); } - maybe_save_cache(cache_key); - // Flush parser state. - std::vector events; - s.parser.Flush(events); - if (!events.empty() && !s.aborted) { - backend::Reply reply; - auto *delta = reply.add_chat_deltas(); - for (const auto &e : events) { - if (e.type == ds4cpp::ParserEvent::CONTENT) { - delta->set_content(delta->content() + e.text); - } else if (e.type == ds4cpp::ParserEvent::REASONING) { - delta->set_reasoning_content(delta->reasoning_content() + e.text); + request_should_continue(&lifecycle, context); + ds4cpp::TerminalDecision terminal = ds4cpp::ResolveTerminalDecision( + rc == DS4_SESSION_SYNC_INTERRUPTED, rc != 0, + !lifecycle.ShouldFinalize()); + terminal = ds4cpp::RunPostlude( + terminal, + [&]() { + ds4cpp::DsmlParser staged_parser = s.parser; + std::vector events; + staged_parser.Flush(events); + bool write_succeeded = true; + if (!events.empty()) { + backend::Reply reply; + auto *delta = reply.add_chat_deltas(); + for (const auto &e : events) { + if (e.type == ds4cpp::ParserEvent::CONTENT) { + delta->set_content(delta->content() + e.text); + } else if (e.type == ds4cpp::ParserEvent::REASONING) { + delta->set_reasoning_content( + delta->reasoning_content() + e.text); + } + } + write_succeeded = s.writer->Write(reply); } - } - s.writer->Write(reply); - } - - if (rc != 0 && !s.aborted) { + lifecycle.ObserveStreamWrite(write_succeeded); + request_should_continue(&lifecycle, context); + if (!lifecycle.ShouldFinalize()) return false; + s.parser = std::move(staged_parser); + if (generation_started) stream_done(&s); + return true; + }, + [&]() { maybe_save_cache(cache_key); }); + + if (terminal.cause == ds4cpp::TerminalCause::EngineError) { return GStatus(StatusCode::INTERNAL, std::string("ds4 generation failed: ") + err); } + if (terminal.cause == ds4cpp::TerminalCause::Cancelled) { + return GStatus(StatusCode::CANCELLED, + "ds4 request cancelled"); + } return GStatus::OK; } diff --git a/backend/cpp/ds4/request_lifecycle.h b/backend/cpp/ds4/request_lifecycle.h new file mode 100644 index 000000000000..c3bc88f3c87c --- /dev/null +++ b/backend/cpp/ds4/request_lifecycle.h @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: MIT +#pragma once + +namespace ds4cpp { + +using CancelCallback = bool (*)(void *); +using CancelSetter = void (*)(void *, CancelCallback, void *) noexcept; + +class CancelCallbackScope { +public: + CancelCallbackScope(void *target, CancelSetter setter, + CancelCallback callback, void *userdata) noexcept + : target_(target), setter_(setter) { + setter_(target_, callback, userdata); + } + + ~CancelCallbackScope() noexcept { + setter_(target_, nullptr, nullptr); + } + + CancelCallbackScope(const CancelCallbackScope &) = delete; + CancelCallbackScope &operator=(const CancelCallbackScope &) = delete; + +private: + void *target_; + CancelSetter setter_; +}; + +enum class RouteWaitDecision { + Pending, + Ready, + Error, + Cancelled, +}; + +inline RouteWaitDecision DecideRouteWait(int route_status, bool cancelled) { + if (cancelled) return RouteWaitDecision::Cancelled; + if (route_status > 0) return RouteWaitDecision::Ready; + if (route_status < 0) return RouteWaitDecision::Error; + return RouteWaitDecision::Pending; +} + +enum class TerminalCause { + Success, + Cancelled, + EngineError, +}; + +inline TerminalCause DecideTerminalCause(bool sync_interrupted, + bool engine_error, + bool abandoned) { + if (sync_interrupted) return TerminalCause::Cancelled; + if (engine_error) return TerminalCause::EngineError; + if (abandoned) return TerminalCause::Cancelled; + return TerminalCause::Success; +} + +struct TerminalDecision { + TerminalCause cause; + bool should_finalize; +}; + +inline TerminalDecision ResolveTerminalDecision(bool sync_interrupted, + bool engine_error, + bool abandoned) { + return { + DecideTerminalCause(sync_interrupted, engine_error, abandoned), + !sync_interrupted && !abandoned, + }; +} + +template +TerminalDecision RunPostlude(TerminalDecision terminal, + Finalize transactional_finalize, + Persist persist) { + if (!terminal.should_finalize) return terminal; + if (!transactional_finalize()) { + terminal.should_finalize = false; + if (terminal.cause != TerminalCause::EngineError) { + terminal.cause = TerminalCause::Cancelled; + } + return terminal; + } + persist(); + return terminal; +} + +class RequestLifecycle { +public: + void ObserveContextCancellation(bool cancelled) { + context_cancelled_ = context_cancelled_ || cancelled; + } + + void ObserveStreamWrite(bool succeeded) { + stream_write_aborted_ = stream_write_aborted_ || !succeeded; + } + + bool ShouldContinue() const { + return !context_cancelled_ && !stream_write_aborted_; + } + + bool ShouldFinalize() const { + return ShouldContinue(); + } + +private: + bool context_cancelled_ = false; + bool stream_write_aborted_ = false; +}; + +} // namespace ds4cpp diff --git a/backend/cpp/ds4/request_lifecycle_test.cpp b/backend/cpp/ds4/request_lifecycle_test.cpp new file mode 100644 index 000000000000..d6b968604d04 --- /dev/null +++ b/backend/cpp/ds4/request_lifecycle_test.cpp @@ -0,0 +1,414 @@ +// SPDX-License-Identifier: MIT +// Standalone regression tests for DS4 request cancellation policy. + +#include "request_lifecycle.h" + +#include + +namespace { + +int failures = 0; + +struct FakeCancelTarget { + ds4cpp::CancelCallback callback = nullptr; + void *userdata = nullptr; + int installs = 0; + int clears = 0; +}; + +struct PostludeCounts { + int finalize_attempts = 0; + int finalize_commits = 0; + int cache_persists = 0; + bool cache_followed_commit = true; +}; + +ds4cpp::TerminalDecision run_fake_postlude( + ds4cpp::TerminalDecision terminal, bool finalize_succeeds, + PostludeCounts *counts) { + return ds4cpp::RunPostlude( + terminal, + [=]() { + counts->finalize_attempts++; + if (!finalize_succeeds) return false; + counts->finalize_commits++; + return true; + }, + [=]() { + counts->cache_followed_commit = counts->finalize_commits == 1; + counts->cache_persists++; + }); +} + +bool fake_cancel(void *) { + return false; +} + +void fake_set_cancel(void *target, ds4cpp::CancelCallback callback, + void *userdata) noexcept { + auto *fake = static_cast(target); + fake->callback = callback; + fake->userdata = userdata; + if (callback) { + fake->installs++; + } else { + fake->clears++; + } +} + +void check(bool condition, const char *name) { + if (condition) return; + std::fprintf(stderr, "FAIL %s\n", name); + failures++; +} + +// Production mutation caught: treating an active request as abandoned would +// skip its parser finalization and cache save. +void test_active_request_continues_and_finalizes() { + ds4cpp::RequestLifecycle request; + + check(request.ShouldContinue(), "active:continue"); + check(request.ShouldFinalize(), "active:finalize"); +} + +// Production mutation caught: omitting the ServerContext cancellation branch +// would continue decoding and finalize a partial response. +void test_context_cancellation_stops_without_finalizing() { + ds4cpp::RequestLifecycle request; + + request.ObserveContextCancellation(true); + + check(!request.ShouldContinue(), "context_cancelled:stop"); + check(!request.ShouldFinalize(), "context_cancelled:no_finalize"); +} + +// Production mutation caught: ignoring ServerWriter::Write failure would keep +// streaming and finalize a response whose client has gone away. +void test_stream_write_abort_stops_without_finalizing() { + ds4cpp::RequestLifecycle request; + + request.ObserveStreamWrite(false); + + check(!request.ShouldContinue(), "write_abort:stop"); + check(!request.ShouldFinalize(), "write_abort:no_finalize"); +} + +// Production mutation caught: combining cancellation and write failure with +// AND would fail to stop when either signal occurs on its own. +void test_cancellation_and_write_abort_are_independent_or_conditions() { + ds4cpp::RequestLifecycle cancelled; + cancelled.ObserveContextCancellation(true); + cancelled.ObserveStreamWrite(true); + + ds4cpp::RequestLifecycle write_aborted; + write_aborted.ObserveContextCancellation(false); + write_aborted.ObserveStreamWrite(false); + + check(!cancelled.ShouldContinue(), "or:context_only"); + check(!write_aborted.ShouldContinue(), "or:write_only"); +} + +// Production mutation caught: treating an incomplete distributed route as an +// error would return before workers have time to connect. +void test_route_wait_pending() { + check(ds4cpp::DecideRouteWait(0, false) == + ds4cpp::RouteWaitDecision::Pending, + "route_wait:pending"); +} + +// Production mutation caught: failing to recognize a complete route would +// keep a ready inference request in the polling loop. +void test_route_wait_ready() { + check(ds4cpp::DecideRouteWait(1, false) == + ds4cpp::RouteWaitDecision::Ready, + "route_wait:ready"); +} + +// Production mutation caught: ignoring a route probe error would poll until a +// misleading timeout instead of returning UNAVAILABLE promptly. +void test_route_wait_error() { + check(ds4cpp::DecideRouteWait(-1, false) == + ds4cpp::RouteWaitDecision::Error, + "route_wait:error"); +} + +// Production mutation caught: omitting cancellation from route waiting would +// leave an abandoned request blocked until the distributed timeout. +void test_route_wait_cancellation() { + check(ds4cpp::DecideRouteWait(0, true) == + ds4cpp::RouteWaitDecision::Cancelled, + "route_wait:cancelled"); +} + +// Production mutation caught: checking route errors before cancellation would +// report UNAVAILABLE for a request the client already abandoned. +void test_route_wait_cancellation_precedes_error() { + check(ds4cpp::DecideRouteWait(-1, true) == + ds4cpp::RouteWaitDecision::Cancelled, + "route_wait:cancellation_precedence"); +} + +// Production mutation caught: classifying a successful active request as a +// terminal failure would suppress its normal response finalization. +void test_terminal_success() { + check(ds4cpp::DecideTerminalCause(false, false, false) == + ds4cpp::TerminalCause::Success, + "terminal:success"); +} + +// Production mutation caught: treating DS4's cooperative sync interruption +// as an ordinary engine error would return INTERNAL instead of CANCELLED. +void test_terminal_sync_interruption_is_cancelled() { + check(ds4cpp::DecideTerminalCause(true, true, true) == + ds4cpp::TerminalCause::Cancelled, + "terminal:sync_interrupted"); +} + +// Production mutation caught: treating every nonzero engine result as client +// abandonment would hide genuine DS4 failures behind CANCELLED. +void test_terminal_engine_error() { + check(ds4cpp::DecideTerminalCause(false, true, false) == + ds4cpp::TerminalCause::EngineError, + "terminal:engine_error"); +} + +// Production mutation caught: ignoring an rc==0 context cancellation would +// finalize and cache an abandoned request. +void test_terminal_context_abandonment() { + ds4cpp::RequestLifecycle request; + request.ObserveContextCancellation(true); + + check(ds4cpp::DecideTerminalCause( + false, false, !request.ShouldFinalize()) == + ds4cpp::TerminalCause::Cancelled, + "terminal:context_abandonment"); +} + +// Production mutation caught: ignoring an rc==0 stream write failure would +// finalize and cache an abandoned streaming request. +void test_terminal_write_abandonment() { + ds4cpp::RequestLifecycle request; + request.ObserveStreamWrite(false); + + check(ds4cpp::DecideTerminalCause( + false, false, !request.ShouldFinalize()) == + ds4cpp::TerminalCause::Cancelled, + "terminal:write_abandonment"); +} + +// Production mutation caught: checking late cancellation or write failure +// before a determined ordinary DS4 error would replace INTERNAL with CANCELLED. +void test_terminal_engine_error_precedes_late_abandonment() { + ds4cpp::RequestLifecycle cancelled; + cancelled.ObserveContextCancellation(true); + ds4cpp::RequestLifecycle write_aborted; + write_aborted.ObserveStreamWrite(false); + + check(ds4cpp::DecideTerminalCause( + false, true, !cancelled.ShouldFinalize()) == + ds4cpp::TerminalCause::EngineError, + "terminal:engine_error_precedes_cancellation"); + check(ds4cpp::DecideTerminalCause( + false, true, !write_aborted.ShouldFinalize()) == + ds4cpp::TerminalCause::EngineError, + "terminal:engine_error_precedes_write_abort"); +} + +// Production mutation caught: using status precedence alone to gate side +// effects would finalize and persist an engine-error request abandoned later. +void test_abandoned_engine_error_keeps_internal_without_finalizing() { + ds4cpp::RequestLifecycle request; + request.ObserveContextCancellation(true); + + ds4cpp::TerminalDecision terminal = ds4cpp::ResolveTerminalDecision( + false, true, !request.ShouldFinalize()); + + check(terminal.cause == ds4cpp::TerminalCause::EngineError, + "terminal_decision:abandoned_engine_error_status"); + check(!terminal.should_finalize, + "terminal_decision:abandoned_engine_error_no_finalize"); +} + +// Production mutation caught: suppressing side effects for every engine error +// would change the existing finalization and cache behavior of active failures. +void test_active_engine_error_still_finalizes() { + ds4cpp::RequestLifecycle request; + + ds4cpp::TerminalDecision terminal = ds4cpp::ResolveTerminalDecision( + false, true, !request.ShouldFinalize()); + + check(terminal.cause == ds4cpp::TerminalCause::EngineError, + "terminal_decision:active_engine_error_status"); + check(terminal.should_finalize, + "terminal_decision:active_engine_error_finalize"); +} + +// Production mutation caught: persisting before committed finalization would +// cache a state whose final buffered stream reply was never completed. +void test_postlude_active_success_commits_then_persists() { + PostludeCounts counts; + + ds4cpp::TerminalDecision terminal = run_fake_postlude( + {ds4cpp::TerminalCause::Success, true}, true, &counts); + + check(terminal.cause == ds4cpp::TerminalCause::Success, + "postlude:success_outcome"); + check(terminal.should_finalize, "postlude:success_committed"); + check(counts.finalize_attempts == 1, "postlude:success_attempts"); + check(counts.finalize_commits == 1, "postlude:success_commits"); + check(counts.cache_persists == 1, "postlude:success_cache"); + check(counts.cache_followed_commit, "postlude:success_cache_order"); +} + +// Production mutation caught: starting the postlude for an already-cancelled +// request would flush buffered parser state or persist an abandoned session. +void test_postlude_cancellation_skips_all_side_effects() { + PostludeCounts counts; + + ds4cpp::TerminalDecision terminal = run_fake_postlude( + {ds4cpp::TerminalCause::Cancelled, false}, true, &counts); + + check(terminal.cause == ds4cpp::TerminalCause::Cancelled, + "postlude:cancelled_outcome"); + check(counts.finalize_attempts == 0, "postlude:cancelled_attempts"); + check(counts.finalize_commits == 0, "postlude:cancelled_commits"); + check(counts.cache_persists == 0, "postlude:cancelled_cache"); +} + +// Production mutation caught: committing the live parser or cache after a +// failed final Write would publish an abandoned streaming postlude. +void test_postlude_finalize_failure_cancels_without_commit_or_cache() { + PostludeCounts counts; + + ds4cpp::TerminalDecision terminal = run_fake_postlude( + {ds4cpp::TerminalCause::Success, true}, false, &counts); + + check(terminal.cause == ds4cpp::TerminalCause::Cancelled, + "postlude:write_failure_outcome"); + check(!terminal.should_finalize, "postlude:write_failure_not_committed"); + check(counts.finalize_attempts == 1, "postlude:write_failure_attempts"); + check(counts.finalize_commits == 0, "postlude:write_failure_commits"); + check(counts.cache_persists == 0, "postlude:write_failure_cache"); +} + +// Production mutation caught: skipping the postlude for every engine error +// would change active internal-error finalization and cache behavior. +void test_postlude_active_engine_error_finalizes_and_persists() { + PostludeCounts counts; + + ds4cpp::TerminalDecision terminal = run_fake_postlude( + {ds4cpp::TerminalCause::EngineError, true}, true, &counts); + + check(terminal.cause == ds4cpp::TerminalCause::EngineError, + "postlude:engine_error_outcome"); + check(counts.finalize_attempts == 1, "postlude:engine_error_attempts"); + check(counts.finalize_commits == 1, "postlude:engine_error_commits"); + check(counts.cache_persists == 1, "postlude:engine_error_cache"); + check(counts.cache_followed_commit, "postlude:engine_error_cache_order"); +} + +// Production mutation caught: replacing every failed transactional finalize +// with cancellation would hide an already-determined engine error. +void test_postlude_engine_error_finalize_failure_preserves_internal() { + PostludeCounts counts; + + ds4cpp::TerminalDecision terminal = run_fake_postlude( + {ds4cpp::TerminalCause::EngineError, true}, false, &counts); + + check(terminal.cause == ds4cpp::TerminalCause::EngineError, + "postlude:engine_error_write_failure_outcome"); + check(!terminal.should_finalize, + "postlude:engine_error_write_failure_not_committed"); + check(counts.finalize_attempts == 1, + "postlude:engine_error_write_failure_attempts"); + check(counts.finalize_commits == 0, + "postlude:engine_error_write_failure_commits"); + check(counts.cache_persists == 0, + "postlude:engine_error_write_failure_cache"); +} + +// Production mutation caught: status precedence must not grant side-effect +// permission to an engine-error request that was also abandoned. +void test_postlude_abandoned_engine_error_skips_all_side_effects() { + PostludeCounts counts; + + ds4cpp::TerminalDecision terminal = run_fake_postlude( + {ds4cpp::TerminalCause::EngineError, false}, true, &counts); + + check(terminal.cause == ds4cpp::TerminalCause::EngineError, + "postlude:abandoned_engine_error_outcome"); + check(counts.finalize_attempts == 0, + "postlude:abandoned_engine_error_attempts"); + check(counts.finalize_commits == 0, + "postlude:abandoned_engine_error_commits"); + check(counts.cache_persists == 0, + "postlude:abandoned_engine_error_cache"); +} + +// Production mutation caught: failing to install the request callback would +// make DS4 prompt synchronization unable to observe client cancellation. +void test_cancel_callback_scope_installs_callback() { + FakeCancelTarget target; + int request_context = 42; + + { + ds4cpp::CancelCallbackScope scope( + &target, fake_set_cancel, fake_cancel, &request_context); + check(target.callback == fake_cancel, "cancel_scope:callback_installed"); + check(target.userdata == &request_context, "cancel_scope:userdata_installed"); + check(target.installs == 1, "cancel_scope:installed_once"); + } +} + +// Production mutation caught: failing to clear the callback at every scope +// exit would leave DS4 pointing at a destroyed stack-owned ServerContext. +void test_cancel_callback_scope_clears_callback() { + FakeCancelTarget target; + int request_context = 42; + + { + ds4cpp::CancelCallbackScope scope( + &target, fake_set_cancel, fake_cancel, &request_context); + } + + check(target.callback == nullptr, "cancel_scope:callback_cleared"); + check(target.userdata == nullptr, "cancel_scope:userdata_cleared"); + check(target.clears == 1, "cancel_scope:cleared_once"); +} + +} // namespace + +int main() { + test_active_request_continues_and_finalizes(); + test_context_cancellation_stops_without_finalizing(); + test_stream_write_abort_stops_without_finalizing(); + test_cancellation_and_write_abort_are_independent_or_conditions(); + test_route_wait_pending(); + test_route_wait_ready(); + test_route_wait_error(); + test_route_wait_cancellation(); + test_route_wait_cancellation_precedes_error(); + test_terminal_success(); + test_terminal_sync_interruption_is_cancelled(); + test_terminal_engine_error(); + test_terminal_context_abandonment(); + test_terminal_write_abandonment(); + test_terminal_engine_error_precedes_late_abandonment(); + test_abandoned_engine_error_keeps_internal_without_finalizing(); + test_active_engine_error_still_finalizes(); + test_postlude_active_success_commits_then_persists(); + test_postlude_cancellation_skips_all_side_effects(); + test_postlude_finalize_failure_cancels_without_commit_or_cache(); + test_postlude_active_engine_error_finalizes_and_persists(); + test_postlude_engine_error_finalize_failure_preserves_internal(); + test_postlude_abandoned_engine_error_skips_all_side_effects(); + test_cancel_callback_scope_installs_callback(); + test_cancel_callback_scope_clears_callback(); + + if (failures == 0) { + std::fprintf(stderr, "all request_lifecycle checks passed\n"); + return 0; + } + std::fprintf(stderr, "%d check(s) failed\n", failures); + return 1; +} diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index 7d2ec66a4f4a..2d04b987c592 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -186,3 +186,12 @@ LocalAI supports various types of backends: - **Utility Backends**: For reranking, PII/NER token classification, fine-tuning, quantization, and vector storage (e.g., rerankers, privacy-filter.cpp, TRL, local-store, valkey-store) See the [Backend & Model Compatibility Table]({{%relref "reference/compatibility-table" %}}) for the full catalog. + +### DS4 request cancellation + +The DS4 backend stops inference when a client cancels or disconnects, including +when a streaming response can no longer be written. Already-streamed chunks +cannot be retracted; DS4 does not flush incomplete buffered parser state or +persist an abandoned request to the disk KV cache. Cancellation is cooperative: +DS4 checks it at safe prompt-prefill and decode-loop boundaries, so a GPU kernel +already in flight may finish before the request stops. From 9e831d7709c55ff97a340963d911b8b79b890cea Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Thu, 3 Sep 2026 13:03:44 +0200 Subject: [PATCH 5/5] fix(ds4): build CUDA kernels for the target architecture (#11840) * fix(ds4): build CUDA kernels for the target architecture The ds4 backend compiled its CUDA objects with no -arch. Upstream's Makefile leaves CUDA_ARCH empty and its `cuda` target refuses to build without one, offering `cuda-spark` (sm_121) and `cuda-generic` (native) instead. We invoke its object targets directly, which bypasses that guard, so nvcc fell back to its default architecture and the kernels ran as JIT'd PTX on the real GPU. On GB10 (sm_121) that silently corrupted inference: any prompt over roughly 128 tokens produced text unrelated to the input and never closed its thinking block, so content came back empty and the chat showed only reasoning; longer prompts failed with "cuda decode failed". It also cost close to two orders of magnitude of prefill throughput. Measured on one box, same model, same prompt, same GPU, upstream ds4 at the pinned commit, differing only in the nvcc flags: make -B ds4 (archless, as we build it) garbage output 4.21 t/s make cuda-spark (compute_121a/sm_121a) correct output 325.70 t/s Select an architecture list from CUDA_MAJOR_VERSION, which the backend matrix already declares for both ds4 cublas entries but Dockerfile.ds4 never forwarded. Upstream's CUDA_ARCH takes a single value, so it cannot express the fat binary these images need; NVCC_ARCH_FLAGS is overridden instead, since a command-line assignment wins over its `:=`. The lists are copied from vllm-cpp rather than invented so the two CUDA images cover the same GPUs, with l4t/arm64 covering Orin, Thor and GB10. An empty CUDA_MAJOR_VERSION keeps upstream's `native` behaviour for local developer builds, and no CI runner has a GPU to enumerate. DS4_CUDA_HAVE_MXF4 is deliberately left unset: upstream defines it only for single-arch sm_120/sm_121 builds and guards it with a plain #ifdef rather than __CUDA_ARCH__, so it cannot be combined with older archs. It gates an optional MXFP4 indexer fast path whose #ifndef branch returns 0 and falls back cleanly, so omitting it costs speed on GB10, not correctness. Assisted-by: Claude Code:claude-opus-5 Signed-off-by: Claudio Maradonna * test(ds4): cover the multi-batch prefill regression The architecture fix has no automated guard: every existing e2e spec uses a short prompt, and the miscompiled backend answered short prompts correctly. The corruption only appears once a prompt spans more than one prefill batch, so the whole suite passed against a backend that produced garbage in normal use. Add an opt-in "long_prefill" capability to the backend e2e suite that sends a prompt well past one batch with a known needle and asserts the answer still reflects it, and document in the ds4 guide why the build must never omit an nvcc architecture, how to check which flags a configuration resolves to without compiling, and how to run the new spec. Assisted-by: Claude Code:claude-opus-5 Signed-off-by: Claudio Maradonna --------- Signed-off-by: Claudio Maradonna --- .agents/ds4-backend.md | 50 ++++++++++++++++++++++++ backend/Dockerfile.ds4 | 4 +- backend/cpp/ds4/Makefile | 63 +++++++++++++++++++++++++++++- tests/e2e-backends/backend_test.go | 47 ++++++++++++++++++++++ 4 files changed, 162 insertions(+), 2 deletions(-) diff --git a/.agents/ds4-backend.md b/.agents/ds4-backend.md index c1b649857c85..8bf1eb45f23c 100644 --- a/.agents/ds4-backend.md +++ b/.agents/ds4-backend.md @@ -77,6 +77,56 @@ spectrum. **Metal (Darwin) only** - it is a no-op on CUDA/CPU. Enable with budget). Gallery entries built on this: `deepseek-v4-flash-q4-ssd` (153 GB Flash on a 128 GB Mac) and `deepseek-v4-pro-q2-ssd` (433 GB Pro, experimental). +## CUDA architecture (do not build without one) + +`backend/cpp/ds4/Makefile` drives upstream's **object targets** directly +(`$(MAKE) -C ds4 ds4.o ds4_cuda.o ...`), which bypasses upstream's own guard: +its `cuda` target refuses to build unless `CUDA_ARCH` is set, and offers +`cuda-spark` (sm_121, DGX Spark / GB10) and `cuda-generic` (native) instead. +Built with no `-arch`, nvcc targets its default architecture and the kernels run +as JIT'd PTX. On GB10 that silently corrupted every prefill batch of >=128 +tokens - the model emitted text unrelated to the prompt and never closed its +thinking block, so `content` came back empty - and cost close to two orders of +magnitude of prefill throughput (4.21 t/s vs 325.70 t/s, same box, same model). +Short prompts stayed correct, which is why it went unnoticed. + +The Makefile therefore picks a gencode list from `CUDA_MAJOR_VERSION` (a build +arg the backend matrix already declares, forwarded by `Dockerfile.ds4`) and +`uname -m`, and passes it as `NVCC_ARCH_FLAGS` to the sub-make. Upstream's +`CUDA_ARCH` accepts a single value, so it cannot express the fat binary the +shipped images need; a command-line assignment beats its `:=`. An empty +`CUDA_MAJOR_VERSION` falls back to upstream's `native` for local developer +builds, and an unrecognised one is a hard error - no CI runner has a GPU, so a +silent `native` there is exactly the failure mode this guards against. + +`DS4_CUDA_HAVE_MXF4` is deliberately unset: upstream defines it only for +single-arch sm_120/sm_121 builds and guards it with a plain `#ifdef` rather than +`__CUDA_ARCH__`, so it cannot be combined with older archs. It gates an optional +MXFP4 indexer fast path whose `#ifndef` branch returns 0, so omitting it costs +speed, not correctness. + +### Verifying a build + +Check which flags a configuration resolves to, without compiling anything: + +``` +make -C backend/cpp/ds4 BUILD_TYPE=cublas CUDA_MAJOR_VERSION=13 NATIVE=false \ + --eval='show: ; @echo [$(DS4_ARCH_MAKEVARS)]' show +``` + +Do not use `make -n` for this: the recipe is `+$(MAKE) ...`, and the `+` prefix +makes it run even under `-n`. + +Then exercise the failure mode itself against a built backend. It only appears +above one prefill batch, so the ordinary `predict` spec cannot catch it: + +``` +BACKEND_BINARY=$(pwd)/backend/cpp/ds4/package/run.sh \ +BACKEND_TEST_MODEL_FILE=/path/to/ds4flash.gguf \ +BACKEND_TEST_CAPS=health,load,predict,long_prefill \ +go test -count=1 -timeout=30m -v ./tests/e2e-backends/... +``` + ## Build matrix | Build | Where | Notes | diff --git a/backend/Dockerfile.ds4 b/backend/Dockerfile.ds4 index 370d1eaae3a1..5d4f02a8db9e 100644 --- a/backend/Dockerfile.ds4 +++ b/backend/Dockerfile.ds4 @@ -10,6 +10,7 @@ FROM ${BASE_IMAGE} AS builder ARG BUILD_TYPE ARG TARGETARCH ARG TARGETVARIANT +ARG CUDA_MAJOR_VERSION ENV BUILD_TYPE=${BUILD_TYPE} \ DEBIAN_FRONTEND=noninteractive \ @@ -35,7 +36,8 @@ RUN apt-get update && \ COPY . /LocalAI RUN --mount=type=cache,target=/root/.ccache,id=ds4-ccache-${TARGETARCH}-${BUILD_TYPE},sharing=locked \ - make -C /LocalAI/backend/cpp/ds4 BUILD_TYPE=${BUILD_TYPE} NATIVE=false grpc-server package + make -C /LocalAI/backend/cpp/ds4 BUILD_TYPE=${BUILD_TYPE} \ + CUDA_MAJOR_VERSION=${CUDA_MAJOR_VERSION} NATIVE=false grpc-server package FROM scratch COPY --from=builder /LocalAI/backend/cpp/ds4/package/. ./ diff --git a/backend/cpp/ds4/Makefile b/backend/cpp/ds4/Makefile index b171fa391d76..807a01a09ec0 100644 --- a/backend/cpp/ds4/Makefile +++ b/backend/cpp/ds4/Makefile @@ -18,6 +18,67 @@ UNAME_S := $(shell uname -s) CMAKE_ARGS ?= -DCMAKE_BUILD_TYPE=Release +# nvcc must be told the target architecture explicitly for a cublas build, and +# this is not a tuning knob. Upstream's Makefile leaves CUDA_ARCH empty and its +# `cuda` target REFUSES to build without one, offering `cuda-spark` +# (CUDA_ARCH=sm_121) and `cuda-generic` (CUDA_ARCH=native) instead. We drive its +# object targets directly, which bypasses that guard: nvcc then compiles with no +# -arch at all, and the kernels run as JIT'd PTX for its default architecture. +# On GB10 (sm_121) that silently produced corrupt inference output above a +# ~128-token prefill batch and ~77x slower prefill (4.21 t/s vs 325.70 t/s, +# measured on the same box with the same model). No CI runner has a GPU, so +# `native` has nothing to enumerate there. +# +# Upstream's CUDA_ARCH takes a SINGLE value (see its sm_120/sm_121 special cases +# and the `-arch=$(CUDA_ARCH)` fallback), so it cannot express the fat binary +# these images need. NVCC_ARCH_FLAGS is overridden instead: a command-line +# assignment wins over the `:=` in upstream's Makefile, and its NVCCFLAGS +# expands whatever we pass. +# +# The architecture lists are copied from backend/go/vllm-cpp/Makefile rather +# than invented, so the two CUDA images cover the same GPUs: amd64 datacenter + +# consumer, and l4t/arm64 covering Orin (87), Thor (110) and GB10 (121a). +# +# -DDS4_CUDA_HAVE_MXF4=1 is deliberately NOT set. Upstream only defines it for +# single-arch sm_120/sm_121 builds and guards the code with a plain #ifdef +# rather than __CUDA_ARCH__, so it cannot be combined with older archs in one +# fat binary. It gates an optional MXFP4 indexer fast path whose #ifndef branch +# returns 0 and falls back to the generic path, so omitting it costs some speed +# on GB10, not correctness. Revisit if upstream adds __CUDA_ARCH__ guards. +# +# An EMPTY CUDA_MAJOR_VERSION means a local developer build, not CI: fall back +# to upstream's own `native` handling, which needs a GPU present but is what a +# developer building on their own machine wants. Both variables are `?=` so an +# explicit value on the command line always wins. +UNAME_M := $(shell uname -m) +CUDA_MAJOR_VERSION ?= +ifeq ($(BUILD_TYPE),cublas) +ifeq ($(CUDA_MAJOR_VERSION),13) +ifeq ($(UNAME_M),aarch64) + DS4_NVCC_ARCH_FLAGS ?= -gencode arch=compute_87,code=sm_87 \ + -gencode arch=compute_90a,code=sm_90a \ + -gencode arch=compute_100a,code=sm_100a \ + -gencode arch=compute_110,code=sm_110 \ + -gencode arch=compute_121a,code=sm_121a +else + DS4_NVCC_ARCH_FLAGS ?= -gencode arch=compute_80,code=sm_80 \ + -gencode arch=compute_86,code=sm_86 \ + -gencode arch=compute_89,code=sm_89 \ + -gencode arch=compute_90a,code=sm_90a \ + -gencode arch=compute_100a,code=sm_100a \ + -gencode arch=compute_103a,code=sm_103a \ + -gencode arch=compute_120a,code=sm_120a \ + -gencode arch=compute_121a,code=sm_121a +endif + DS4_ARCH_MAKEVARS := NVCC_ARCH_FLAGS="$(DS4_NVCC_ARCH_FLAGS)" +else ifeq ($(CUDA_MAJOR_VERSION),) + # Local build: let upstream resolve the host GPU. + DS4_ARCH_MAKEVARS := CUDA_ARCH=native +else + $(error CUDA_MAJOR_VERSION=$(CUDA_MAJOR_VERSION) has no architecture list here (13 does). Leave it empty for a native build, or pass DS4_NVCC_ARCH_FLAGS explicitly.) +endif +endif + # Upstream splits distributed inference, tensor-parallel transport, the SSD # expert cache, and layer placement into GPU-agnostic translation units. They # are shared by every GPU mode, so append them unconditionally below. @@ -57,7 +118,7 @@ ds4: # the right per-platform compile flags (Objective-C/Metal on Darwin, nvcc on Linux+CUDA). ds4/ds4.o: ds4 ifeq ($(BUILD_TYPE),cublas) - +$(MAKE) -C ds4 $(DS4_OBJ_TARGET) + +$(MAKE) -C ds4 $(DS4_ARCH_MAKEVARS) $(DS4_OBJ_TARGET) else ifeq ($(UNAME_S),Darwin) +$(MAKE) -C ds4 ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o else diff --git a/tests/e2e-backends/backend_test.go b/tests/e2e-backends/backend_test.go index a86a2a08896a..73b30f9909c2 100644 --- a/tests/e2e-backends/backend_test.go +++ b/tests/e2e-backends/backend_test.go @@ -62,6 +62,11 @@ import ( // model output into ChatDelta.tool_calls. // "image" exercises the GenerateImage RPC and asserts a // non-empty file is written to the requested dst path. +// "long_prefill" sends a prompt long enough to span more +// than one prefill batch and asserts the answer still +// reflects the prompt. Catches GPU backends whose kernels +// were built for the wrong architecture, which corrupt +// batched prefill while short prompts stay correct. // BACKEND_TEST_IMAGE_PROMPT Override the positive prompt for the image spec // (default: "a photograph of an astronaut riding a horse"). // BACKEND_TEST_IMAGE_STEPS Override the diffusion step count for the image spec @@ -108,6 +113,7 @@ const ( capVoiceAnalyze = "voice_analyze" capAudioTransform = "audio_transform" capLogprobs = "logprobs" + capLongPrefill = "long_prefill" capLogitBias = "logit_bias" capTokenize = "tokenize" capTokenClassify = "token_classify" @@ -433,6 +439,47 @@ var _ = Describe("Backend container", Ordered, func() { res.GetMessage(), res.GetTokens(), res.GetPromptTokens()) }) + // Regression guard for GPU backends compiled without an explicit device + // architecture. LocalAI built ds4's CUDA objects with no -arch/-gencode, so + // on a GB10 (sm_121) the kernels ran as JIT'd PTX for nvcc's default + // architecture and silently corrupted any prefill batch of 128 tokens or + // more: the model produced text unrelated to the prompt. Short prompts stayed + // correct, so every other spec here passed. Only a prompt long enough to need + // a multi-batch prefill exposes it. + It("answers a prompt long enough to span multiple prefill batches", func() { + if !caps[capLongPrefill] { + Skip("long_prefill capability not enabled") + } + const needle = "PLATYPUS" + filler := strings.Repeat("The merchants kept careful ledgers of every voyage they financed. ", 20) + longPrompt := "Read this passage.\n\n" + filler + + "\nThe secret word is " + needle + ".\n" + filler + + "\nQuestion: what is the secret word?\nAnswer: The secret word is" + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + res, err := client.Predict(ctx, &pb.PredictOptions{ + Prompt: longPrompt, + Tokens: 300, + Temperature: 0.1, + TopK: 40, + TopP: 0.9, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(res.GetMessage()).NotTo(BeEmpty(), "long prompt produced empty output") + // Only meaningful if the prompt really did exceed one batch. Backends that + // do not report prompt tokens still run the substring assertion below. + if res.GetPromptTokens() > 0 { + Expect(res.GetPromptTokens()).To(BeNumerically(">", 128), + "prompt is too short to span multiple prefill batches; this spec would not prove anything") + } + Expect(strings.ToUpper(res.GetMessage())).To(ContainSubstring(needle), + "a long prompt lost information the model repeats correctly from a short one - "+ + "batched prefill is corrupting state (check the backend's device architecture flags)") + GinkgoWriter.Printf("LongPrefill: prompt_tokens=%d tokens=%d msg=%q\n", + res.GetPromptTokens(), res.GetTokens(), res.GetMessage()) + }) + // Regression guard for the raw-prompt tokenize RPC. The llama.cpp handler // read the prompt from the wrong JSON key ("content" instead of "prompt"), // so any non-empty prompt threw and the RPC returned "Unexpected error in