From 0cdc31dcb30cf1464a7402b2065e4c9bc3fe3646 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:06:35 +0200 Subject: [PATCH] chore: :arrow_up: Update ggml-org/llama.cpp to `e70802a01f03f0ed31a26338a5664796f3824371` (#11755) * :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(llama-cpp): follow upstream MTMD APIs The dependency update adds MTMD initialization options to prompt and bitmap helpers. The gRPC adapter now passes the server options through each affected path. The update also replaces the per-layer MoE regex helper. Preparation probes both APIs because older forks still reuse this adapter. Assisted-by: Codex:gpt-5 --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- backend/cpp/llama-cpp/Makefile | 2 +- backend/cpp/llama-cpp/grpc-server.cpp | 32 ++++++++++++------- .../0002-add-server-task-type-tts.patch | 2 +- backend/cpp/llama-cpp/prepare.sh | 12 +++++++ 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index bf3e20aab12d..196023dbf4f7 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=925e1179947ea0c0ebfb0032df18af3a729822be +LLAMA_VERSION?=e70802a01f03f0ed31a26338a5664796f3824371 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index 7c1a3d320c3a..95345aaf1203 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -88,6 +88,12 @@ using grpc::ServerBuilder; using grpc::ServerContext; using grpc::Status; +#if LOCALAI_HAS_MTMD_INIT_OPT +#define LOCALAI_MTMD_INIT_OPT_ARG(value) , value +#else +#define LOCALAI_MTMD_INIT_OPT_ARG(value) +#endif + // gRPC bearer token auth for distributed mode. // Reads LOCALAI_GRPC_AUTH_TOKEN from the environment. When set, rejects // requests without a matching "authorization: Bearer " metadata header. @@ -1119,14 +1125,16 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt try { int n = std::stoi(optval_str); if (n < 0) n = 0; - // Keep override-name storage alive for the lifetime of the params struct - // (mirrors upstream arg.cpp behavior with a function-local static). +#if LOCALAI_HAS_N_CPU_FFN_HELPER + llm_add_n_cpu_ffn_overrides(n, LLM_FFN_EXPS_REGEX, params.speculative.draft.tensor_buft_overrides); +#else static std::list buft_overrides_draft; for (int i = 0; i < n; ++i) { buft_overrides_draft.push_back(llm_ffn_exps_block_regex(i)); params.speculative.draft.tensor_buft_overrides.push_back( {buft_overrides_draft.back().c_str(), ggml_backend_cpu_buffer_type()}); } +#endif } catch (...) {} } @@ -1144,14 +1152,16 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt try { int n = std::stoi(optval_str); if (n < 0) n = 0; - // Keep override-name storage alive for the lifetime of the - // params struct (mirrors upstream arg.cpp's function-local static). +#if LOCALAI_HAS_N_CPU_FFN_HELPER + llm_add_n_cpu_ffn_overrides(n, LLM_FFN_EXPS_REGEX, params.tensor_buft_overrides); +#else static std::list buft_overrides_main; for (int i = 0; i < n; ++i) { buft_overrides_main.push_back(llm_ffn_exps_block_regex(i)); params.tensor_buft_overrides.push_back( {buft_overrides_main.back().c_str(), ggml_backend_cpu_buffer_type()}); } +#endif } catch (...) {} } @@ -2111,10 +2121,10 @@ class BackendServiceImpl final : public backend::Backend::Service { std::vector inputs; if (has_mtmd) { // multimodal - inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files)); + inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt))); } else { // Everything else, including multimodal completions. - inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true); + inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)); } tasks.reserve(inputs.size()); @@ -2686,10 +2696,10 @@ class BackendServiceImpl final : public backend::Backend::Service { std::vector inputs; if (has_mtmd) { // multimodal - inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files)); + inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt))); } else { // Everything else, including multimodal completions. - inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true); + inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)); } tasks.reserve(inputs.size()); @@ -2876,7 +2886,7 @@ class BackendServiceImpl final : public backend::Backend::Service { json prompt = body.at("embeddings"); - auto tokenized_prompts = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt, true, true); + auto tokenized_prompts = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt, true, true LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)); for (const auto & tokens : tokenized_prompts) { // this check is necessary for models that do not add BOS token to the input if (tokens.empty()) { @@ -2981,7 +2991,7 @@ class BackendServiceImpl final : public backend::Backend::Service { tasks.reserve(documents.size()); for (size_t i = 0; i < documents.size(); i++) { - auto tmp = format_prompt_rerank(ctx_server.impl->model_tgt, ctx_server.impl->vocab, ctx_server.impl->mctx, request->query(), documents[i]); + auto tmp = format_prompt_rerank(ctx_server.impl->model_tgt, ctx_server.impl->vocab, ctx_server.impl->mctx, request->query(), documents[i] LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)); server_task task = server_task(SERVER_TASK_TYPE_RERANK); task.id = rd.queue_tasks.get_new_id(); task.index = i; @@ -3062,7 +3072,7 @@ class BackendServiceImpl final : public backend::Backend::Service { return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, opts.error); } - auto wrapper = mtmd_helper_bitmap_init_from_file(ctx_server.impl->mctx, opts.voice_path.c_str(), false); + auto wrapper = mtmd_helper_bitmap_init_from_file(ctx_server.impl->mctx, opts.voice_path.c_str(), false LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)); if (!wrapper.bitmap) { return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "failed to read speaker reference audio: " + opts.voice_path); diff --git a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch b/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch index 6681b04e86bf..6789ffc0df5a 100644 --- a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch +++ b/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch @@ -659,7 +659,7 @@ index 9069463fe..b7fa1e534 100644 + } + + if (speaker_ref_len > 0) { -+ auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false); ++ auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt); + if (!wrapper.bitmap) { + res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST)); + return res; diff --git a/backend/cpp/llama-cpp/prepare.sh b/backend/cpp/llama-cpp/prepare.sh index e658a940ae86..e00b139cb1c6 100644 --- a/backend/cpp/llama-cpp/prepare.sh +++ b/backend/cpp/llama-cpp/prepare.sh @@ -61,11 +61,23 @@ if grep -q "server_metrics metrics;" llama.cpp/tools/server/server-task.h; then else HAS_SERVER_METRICS=0 fi +if grep -q "mtmd_helper_init_opt" llama.cpp/tools/mtmd/mtmd-helper.h; then + HAS_MTMD_INIT_OPT=1 +else + HAS_MTMD_INIT_OPT=0 +fi +if grep -q "llm_add_n_cpu_ffn_overrides" llama.cpp/common/common.h; then + HAS_N_CPU_FFN_HELPER=1 +else + HAS_N_CPU_FFN_HELPER=0 +fi cat > llama.cpp/tools/grpc-server/llama_compat.h <