From 964be3bceb4c15f09505a82abdb1695692fb4660 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 25 Aug 2026 08:54:29 +0200 Subject: [PATCH 1/5] chore: :arrow_up: Update ikawrakow/ik_llama.cpp to `0ed847d3140baead542abe3e5e6fe841013e7340` (#11708) :arrow_up: Update ikawrakow/ik_llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/ik-llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/ik-llama-cpp/Makefile b/backend/cpp/ik-llama-cpp/Makefile index 847ddbb20a38..f3ff79a7fca1 100644 --- a/backend/cpp/ik-llama-cpp/Makefile +++ b/backend/cpp/ik-llama-cpp/Makefile @@ -1,5 +1,5 @@ -IK_LLAMA_VERSION?=8337e4cd3861406fc04e0854b1409cd1b027fbc9 +IK_LLAMA_VERSION?=0ed847d3140baead542abe3e5e6fe841013e7340 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp CMAKE_ARGS?= From a760a7ab4bd16e3e6c41ba8f543a4c9188a2760e Mon Sep 17 00:00:00 2001 From: lei_lei <96427312+leilei3167@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:52:57 +0800 Subject: [PATCH 2/5] fix(backends): honor enable_thinking=false in sglang and vllm (#11715) Those backends only forwarded the flag when it was "true", so "false" never reached apply_chat_template and Qwen3 kept thinking on. Signed-off-by: lei_lei <96427312+leilei3167@users.noreply.github.com> --- backend/python/sglang/backend.py | 5 +++-- backend/python/sglang/test.py | 32 ++++++++++++++++++++++++++++++++ backend/python/vllm/backend.py | 6 +++--- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/backend/python/sglang/backend.py b/backend/python/sglang/backend.py index c28b59a0edde..ad6c6ca10197 100644 --- a/backend/python/sglang/backend.py +++ b/backend/python/sglang/backend.py @@ -363,8 +363,9 @@ def _build_prompt(self, request) -> str: template_kwargs["tools"] = json.loads(request.Tools) except json.JSONDecodeError: pass - if request.Metadata.get("enable_thinking", "").lower() == "true": - template_kwargs["enable_thinking"] = True + _thinking = request.Metadata.get("enable_thinking", "").lower() + if _thinking in ("true", "false"): + template_kwargs["enable_thinking"] = (_thinking == "true") try: return self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) diff --git a/backend/python/sglang/test.py b/backend/python/sglang/test.py index 92688f4440a5..deb615883b5e 100644 --- a/backend/python/sglang/test.py +++ b/backend/python/sglang/test.py @@ -96,6 +96,38 @@ def test_apply_engine_args_non_object_raises(self): servicer._apply_engine_args({}, "[1,2,3]") self.assertIn("must be a JSON object", str(ctx.exception)) + def test_build_prompt_forwards_enable_thinking(self): + from types import SimpleNamespace + + class Tok: + def __init__(self): + self.kwargs = None + + def apply_chat_template(self, messages, **kwargs): + self.kwargs = kwargs + return "PROMPT" + + def kwargs_for(metadata): + servicer = self._servicer() + tok = Tok() + servicer.tokenizer = tok + msg = SimpleNamespace( + role="user", content="hi", name="", + tool_call_id="", reasoning_content="", tool_calls="", + ) + req = SimpleNamespace( + Prompt="", UseTokenizerTemplate=True, + Messages=[msg], Tools="", Metadata=metadata, + ) + self.assertEqual(servicer._build_prompt(req), "PROMPT") + return tok.kwargs + + self.assertIs(kwargs_for({"enable_thinking": "true"})["enable_thinking"], True) + # "false" used to be dropped, so Qwen3 kept thinking on + self.assertIs(kwargs_for({"enable_thinking": "false"})["enable_thinking"], False) + self.assertNotIn("enable_thinking", kwargs_for({})) + self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False) + if __name__ == "__main__": unittest.main() diff --git a/backend/python/vllm/backend.py b/backend/python/vllm/backend.py index 8fd3c2dc1c1f..f3f01ec45214 100644 --- a/backend/python/vllm/backend.py +++ b/backend/python/vllm/backend.py @@ -587,9 +587,9 @@ async def _predict(self, request, context, streaming=False): except json.JSONDecodeError: pass - # Enable thinking mode if requested - if request.Metadata.get("enable_thinking", "").lower() == "true": - template_kwargs["enable_thinking"] = True + _thinking = request.Metadata.get("enable_thinking", "").lower() + if _thinking in ("true", "false"): + template_kwargs["enable_thinking"] = (_thinking == "true") try: prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) From f7c55788c7fdab3ba6de234fa3f8788f91bcd4ed Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 25 Aug 2026 12:54:00 +0200 Subject: [PATCH 3/5] feat(gallery): add Ornith 1.5 35B variants (#11714) Add the official Q4_K_M and Q8_0 GGUF builds with their shared BF16 vision projector. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 97 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index 1c67d799bdbb..35392e405c94 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -597,6 +597,103 @@ - filename: llama-cpp/mmproj/ornith-1.5-9b/mmproj-BF16.gguf uri: huggingface://ornith-ai/Ornith-1.5-9B-GGUF/mmproj-Ornith-1.5-9B-BF16.gguf sha256: 626f9f90627402a6bf4a999111d0fbd69b5fcca7aa8ba089d69e5f10e8858e1d +- &ornith-1-5-35b-a3b + name: "ornith-1.5-35b-a3b-q4" + variants: + - model: ornith-1.5-35b-a3b-q8 + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B + - https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B-GGUF + description: | + Ornith-1.5-35B-A3B is an MIT-licensed Qwen3.5 mixture-of-experts model + from Ornith AI for agentic coding, reasoning, repository-level software + tasks, and tool use. It activates about 3B parameters per token and + supports text and image input with a context window of 262K tokens. + + This default entry uses the Q4_K_M GGUF and BF16 vision projector. A + higher-quality Q8_0 model is available as a variant. + license: "mit" + tags: + - llm + - gguf + - cpu + - gpu + - qwen + - moe + - reasoning + - thinking + - coding + - agent + - tools + - vision + - multimodal + - long-context + last_checked: "2026-08-25" + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/ornith-1.5-35b-a3b/Ornith-1.5-35B-Q4_K_M.gguf + repeat_penalty: 1 + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/ornith-1.5-35b-a3b/Ornith-1.5-35B-Q4_K_M.gguf + uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/Ornith-1.5-35B-Q4_K_M.gguf + sha256: 42739874cc2ccfdb8523b23fbe52e29b2a7555c8176737ca9ca0b5d59859d41f + - filename: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf + uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/mmproj-Ornith-1.5-35B-BF16.gguf + sha256: 1921a36a85aee56cd2abd27f46701802c9d85a33474792e600df6c3b282a135d +- !!merge <<: *ornith-1-5-35b-a3b + name: "ornith-1.5-35b-a3b-q8" + variants: [] + description: | + Ornith-1.5-35B-A3B in the higher-quality Q8_0 GGUF format, with the shared + BF16 vision projector for multimodal prompts. + overrides: + backend: llama-cpp + context_size: 262144 + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + - vision + mmproj: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf + options: + - use_jinja:true + parameters: + min_p: 0 + model: llama-cpp/models/ornith-1.5-35b-a3b/Ornith-1.5-35B-Q8_0.gguf + repeat_penalty: 1 + temperature: 0.6 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/ornith-1.5-35b-a3b/Ornith-1.5-35B-Q8_0.gguf + uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/Ornith-1.5-35B-Q8_0.gguf + sha256: de46c4baf4b4dd85ea438bb0f757f21c38841a353506579979bba114311658c3 + - filename: llama-cpp/mmproj/ornith-1.5-35b-a3b/mmproj-BF16.gguf + uri: huggingface://ornith-ai/Ornith-1.5-35B-A3B-GGUF/mmproj-Ornith-1.5-35B-BF16.gguf + sha256: 1921a36a85aee56cd2abd27f46701802c9d85a33474792e600df6c3b282a135d - &qwen3-8-27b-obliterated name: "qwen3.8-27b-obliterated-q4" variants: From ccb9a0a088fd5630ffcd94d88c5ec7321728e7ed Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:55:43 +0200 Subject: [PATCH 4/5] chore: :arrow_up: Update 0xShug0/audio.cpp to `d25ffac094a9d5a240940b4955ea79ad9b7b4c78` (#11710) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index bee144fbcba4..2462a4f4018c 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=288a2712316470847a730e55db9ac9e5062a2b03 +AUDIO_CPP_VERSION?=d25ffac094a9d5a240940b4955ea79ad9b7b4c78 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From fa9ffc181ce2b0e7682e9b600061cb603335b3d0 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 25 Aug 2026 12:57:12 +0200 Subject: [PATCH 5/5] chore: :arrow_up: Update ggml-org/llama.cpp to `f280b26983ad0fdb705a0d9ebf0503e76f2899b0` (#11646) * :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(llama-cpp): adapt to the common JSON API The llama.cpp bump replaces its nlohmann JSON alias with common_json. Update the gRPC adapter for the new exception, iterator, conversion, and container APIs. Assisted-by: Codex:gpt-5.6 [systematic-debugging] * fix(turboquant): adapt the JSON exception type The shared gRPC source now follows the upstream common_json API. The TurboQuant fork still exposes nlohmann JSON and cannot compile the new exception type. Translate that exception in the fork-specific source patch so both llama.cpp variants compile from the shared adapter. Assisted-by: Codex:gpt-5.6 [systematic-debugging] * fix(bonsai): adapt the JSON exception type The shared gRPC source uses upstream's common_json wrapper. The Bonsai fork still exposes nlohmann JSON and cannot compile that exception type.\n\nTranslate the exception in the fork-specific preparation step and verify that repeated preparation stays idempotent.\n\nAssisted-by: Codex:gpt-5.6 [systematic-debugging] * fix(llama-cpp): let prepare register gRPC The score patch duplicated the gRPC CMake registration that prepare.sh already owns. Its stale context rejects the current upstream tools file on Darwin before compilation starts. Assisted-by: Codex:gpt-5 --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- backend/cpp/bonsai/Makefile | 2 + backend/cpp/bonsai/patch-grpc-server.sh | 24 ++++++++ backend/cpp/llama-cpp/Makefile | 2 +- backend/cpp/llama-cpp/grpc-server.cpp | 55 +++++++++---------- backend/cpp/llama-cpp/message_content.h | 7 ++- .../0001-add-server-task-type-score.patch | 12 +--- backend/cpp/turboquant/patch-grpc-server.sh | 14 +++++ scripts/build/bonsai-json-compat_test.sh | 25 +++++++++ 8 files changed, 97 insertions(+), 44 deletions(-) create mode 100644 backend/cpp/bonsai/patch-grpc-server.sh create mode 100644 scripts/build/bonsai-json-compat_test.sh diff --git a/backend/cpp/bonsai/Makefile b/backend/cpp/bonsai/Makefile index 96171e6a9cc8..a3cdf980fd32 100644 --- a/backend/cpp/bonsai/Makefile +++ b/backend/cpp/bonsai/Makefile @@ -41,6 +41,7 @@ define bonsai-build # and are applied by apply-patches.sh below. rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/patches $(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build purge + bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp $(info $(GREEN)I bonsai build info:$(1)$(RESET)) @@ -79,6 +80,7 @@ bonsai-cpu-all: # and are applied by apply-patches.sh below. rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/patches $(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build purge + bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp $(info $(GREEN)I bonsai build info:cpu-all-variants$(RESET)) diff --git a/backend/cpp/bonsai/patch-grpc-server.sh b/backend/cpp/bonsai/patch-grpc-server.sh new file mode 100644 index 000000000000..aa9b23a50539 --- /dev/null +++ b/backend/cpp/bonsai/patch-grpc-server.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Adapt the shared llama.cpp gRPC source to the older JSON API in Bonsai. + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +SRC=$1 +if [[ ! -f "$SRC" ]]; then + echo "grpc-server.cpp not found at $SRC" >&2 + exit 2 +fi + +if grep -q 'common_json_error' "$SRC"; then + echo "==> patching $SRC to use the Bonsai JSON exception type" + awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp" + mv "$SRC.tmp" "$SRC" + echo "==> Bonsai JSON exception patch OK" +else + echo "==> $SRC already uses a Bonsai-compatible JSON exception type, skipping" +fi diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index 41861f5f21d0..d9e248adfa45 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=d59d455fd8ea09e5a2e87ce2a9d668267ffb5ccd +LLAMA_VERSION?=f280b26983ad0fdb705a0d9ebf0503e76f2899b0 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index 171ae0483e70..7c1a3d320c3a 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -294,7 +294,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const } else { SRV_WRN("[TOOLS DEBUG] parse_options: Parsed tools JSON is not an array: %s\n", tools_json.dump().c_str()); } - } catch (const json::parse_error& e) { + } catch (const common_json_error& e) { SRV_WRN("Failed to parse tools JSON from proto: %s\n", e.what()); SRV_WRN("[TOOLS DEBUG] parse_options: Tools string that failed to parse: %s\n", predict->tools().c_str()); } @@ -324,7 +324,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const SRV_DBG("[TOOLS DEBUG] Received tool_choice object from Go layer: %s\n", tool_choice_json.dump().c_str()); } SRV_INF("Extracted tool_choice from proto: %s\n", predict->toolchoice().c_str()); - } catch (const json::parse_error& e) { + } catch (const common_json_error& e) { // If parsing fails, treat as string data["tool_choice"] = predict->toolchoice(); SRV_INF("Extracted tool_choice as string: %s\n", predict->toolchoice().c_str()); @@ -353,7 +353,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const // Add to data - llama.cpp server expects it as an object (map) data["logit_bias"] = logit_bias_json; SRV_INF("Using logit_bias: %s\n", predict->logitbias().c_str()); - } catch (const json::parse_error& e) { + } catch (const common_json_error& e) { SRV_ERR("Failed to parse logit_bias JSON from proto: %s\n", e.what()); } } @@ -398,7 +398,10 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const }); } - data["stop"] = predict->stopprompts(); + data["stop"] = json::array(); + for (const auto & stop : predict->stopprompts()) { + data["stop"].push_back(stop); + } // data["n_probs"] = predict->nprobs(); //TODO: images, @@ -1795,7 +1798,7 @@ class BackendServiceImpl final : public backend::Backend::Service { for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j)); for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j)); } - messages_json.push_back(llama_grpc::build_reconstructed_message(rin)); + messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump())); } // Final safety check: Ensure no message has null content (Jinja templates require strings) @@ -1988,7 +1991,7 @@ class BackendServiceImpl final : public backend::Backend::Service { if (!body_json.contains("chat_template_kwargs")) { body_json["chat_template_kwargs"] = json::object(); } - for (auto& el : ctk.items()) { + for (auto el : ctk.items()) { body_json["chat_template_kwargs"][el.key()] = el.value(); } } @@ -2074,30 +2077,27 @@ class BackendServiceImpl final : public backend::Backend::Service { // If not using chat templates, extract files from image_data/audio_data fields // (If using chat templates, files were already extracted by oaicompat_chat_params_parse) if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) { - const auto &images_data = data.find("image_data"); - if (images_data != data.end() && images_data->is_array()) + if (data.contains("image_data") && data.at("image_data").is_array()) { - for (const auto &img : *images_data) + for (const auto &img : data.at("image_data")) { auto decoded_data = base64_decode(img["data"].get()); files.push_back(decoded_data); } } - const auto &audio_data = data.find("audio_data"); - if (audio_data != data.end() && audio_data->is_array()) + if (data.contains("audio_data") && data.at("audio_data").is_array()) { - for (const auto &audio : *audio_data) + for (const auto &audio : data.at("audio_data")) { auto decoded_data = base64_decode(audio["data"].get()); files.push_back(decoded_data); } } - const auto &video_data = data.find("video_data"); - if (video_data != data.end() && video_data->is_array()) + if (data.contains("video_data") && data.at("video_data").is_array()) { - for (const auto &video : *video_data) + for (const auto &video : data.at("video_data")) { auto decoded_data = base64_decode(video["data"].get()); files.push_back(decoded_data); @@ -2370,7 +2370,7 @@ class BackendServiceImpl final : public backend::Backend::Service { for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j)); for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j)); } - messages_json.push_back(llama_grpc::build_reconstructed_message(rin)); + messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump())); } // Final safety check: Ensure no message has null content (Jinja templates require strings) @@ -2563,7 +2563,7 @@ class BackendServiceImpl final : public backend::Backend::Service { if (!body_json.contains("chat_template_kwargs")) { body_json["chat_template_kwargs"] = json::object(); } - for (auto& el : ctk.items()) { + for (auto el : ctk.items()) { body_json["chat_template_kwargs"][el.key()] = el.value(); } } @@ -2649,11 +2649,10 @@ class BackendServiceImpl final : public backend::Backend::Service { // If not using chat templates, extract files from image_data/audio_data fields // (If using chat templates, files were already extracted by oaicompat_chat_params_parse) if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) { - const auto &images_data = data.find("image_data"); - if (images_data != data.end() && images_data->is_array()) + if (data.contains("image_data") && data.at("image_data").is_array()) { - std::cout << "[PREDICT] Processing " << images_data->size() << " images" << std::endl; - for (const auto &img : *images_data) + std::cout << "[PREDICT] Processing " << data.at("image_data").size() << " images" << std::endl; + for (const auto &img : data.at("image_data")) { std::cout << "[PREDICT] Processing image" << std::endl; auto decoded_data = base64_decode(img["data"].get()); @@ -2661,20 +2660,18 @@ class BackendServiceImpl final : public backend::Backend::Service { } } - const auto &audio_data = data.find("audio_data"); - if (audio_data != data.end() && audio_data->is_array()) + if (data.contains("audio_data") && data.at("audio_data").is_array()) { - for (const auto &audio : *audio_data) + for (const auto &audio : data.at("audio_data")) { auto decoded_data = base64_decode(audio["data"].get()); files.push_back(decoded_data); } } - const auto &video_data = data.find("video_data"); - if (video_data != data.end() && video_data->is_array()) + if (data.contains("video_data") && data.at("video_data").is_array()) { - for (const auto &video : *video_data) + for (const auto &video : data.at("video_data")) { auto decoded_data = base64_decode(video["data"].get()); files.push_back(decoded_data); @@ -3005,7 +3002,7 @@ class BackendServiceImpl final : public backend::Backend::Service { } // Collect responses - json responses = json::array(); + std::vector responses; for (auto & res : all_results.results) { GGML_ASSERT(dynamic_cast(res.get()) != nullptr); responses.push_back(res->to_json()); @@ -3018,7 +3015,7 @@ class BackendServiceImpl final : public backend::Backend::Service { // Crop results by request.top_n if specified int top_n = request->top_n(); if (top_n > 0 && top_n < static_cast(responses.size())) { - responses = json(responses.begin(), responses.begin() + top_n); + responses.resize(top_n); } // Set usage information backend::Usage* usage = rerankResult->mutable_usage(); diff --git a/backend/cpp/llama-cpp/message_content.h b/backend/cpp/llama-cpp/message_content.h index 4c7317ecdfd4..0b70c8b569ba 100644 --- a/backend/cpp/llama-cpp/message_content.h +++ b/backend/cpp/llama-cpp/message_content.h @@ -52,14 +52,15 @@ inline nlohmann::ordered_json normalize_message_content(const std::string& role, // (#7528). A multimodal user message legitimately carries a typed-part array // ({type:text}, {type:image_url}, ...), which must be left intact. Shared by the // streaming and non-streaming paths so this invariant cannot drift between them. -inline void normalize_template_message(nlohmann::ordered_json& msg) { +template +inline void normalize_template_message(Json& msg) { if (!msg.contains("content")) { msg["content"] = ""; // templates expect the field to exist return; } - nlohmann::ordered_json& content = msg["content"]; + auto& content = msg["content"]; const std::string role = (msg.contains("role") && msg["role"].is_string()) - ? msg["role"].get() + ? msg["role"].template get() : std::string(); if (content.is_null()) { content = ""; // #7324: null would crash content[:N] slicing diff --git a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch index f056d47cec81..253e8da5fe17 100644 --- a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch +++ b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch @@ -6,10 +6,9 @@ Subject: [PATCH 1/2] score-patch --- common/common.cpp | 6 +- common/common.h | 3 + - tools/CMakeLists.txt | 1 + tools/server/server-context.cpp | 358 +++++++++++++++++++++++++++++++- tools/server/server-task.h | 47 +++++ - 5 files changed, 406 insertions(+), 9 deletions(-) + 4 files changed, 405 insertions(+), 9 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 2e3f14c..0cec0dc 100644 @@ -42,15 +41,6 @@ index 878534d..4001df2 100644 int32_t n_sequences = 1; // number of sequences to decode int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch) int32_t n_outputs_max_per_seq = 1; // max outputs per sequence -diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt -index 780df32..1d2fe8f 100644 ---- a/tools/CMakeLists.txt -+++ b/tools/CMakeLists.txt -@@ -41,3 +41,4 @@ else() - add_subdirectory(fit-params) - add_subdirectory(results) - endif() -+add_subdirectory(grpc-server) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 3b5f6a1..d0e18e6 100644 --- a/tools/server/server-context.cpp diff --git a/backend/cpp/turboquant/patch-grpc-server.sh b/backend/cpp/turboquant/patch-grpc-server.sh index fa11897dd43f..8d1e355782fd 100755 --- a/backend/cpp/turboquant/patch-grpc-server.sh +++ b/backend/cpp/turboquant/patch-grpc-server.sh @@ -8,6 +8,8 @@ # so the grpc-server option parser skips the two references to # common_params::checkpoint_min_step (the default and the option handler). # That field does not exist in the fork yet; drop this once it does. +# 3. Use nlohmann's parse_error type in JSON catch clauses because the fork +# predates upstream's common_json_error wrapper. # # The fork used to lag upstream on the whole common_params_speculative refactor # (ggml-org/llama.cpp#22397/#22838/#22964), the model_tgt rename (#22838) and @@ -100,4 +102,16 @@ else echo "==> LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP define OK" fi +# 3. The shared source follows current upstream and catches common_json_error. +# TurboQuant still exposes nlohmann::json directly, so its equivalent parse +# failures use json::parse_error instead. +if grep -q 'common_json_error' "$SRC"; then + echo "==> patching $SRC to use the TurboQuant JSON exception type" + awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp" + mv "$SRC.tmp" "$SRC" + echo "==> TurboQuant JSON exception patch OK" +else + echo "==> $SRC already uses a TurboQuant-compatible JSON exception type, skipping" +fi + echo "==> all patches applied" diff --git a/scripts/build/bonsai-json-compat_test.sh b/scripts/build/bonsai-json-compat_test.sh new file mode 100644 index 000000000000..c0cfc4f885c4 --- /dev/null +++ b/scripts/build/bonsai-json-compat_test.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -euo pipefail + +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +PATCHER="$ROOT/backend/cpp/bonsai/patch-grpc-server.sh" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +cat > "$WORK/grpc-server.cpp" <<'EOF' +try { + json::parse("{"); +} catch (const common_json_error& e) { +} +EOF + +bash "$PATCHER" "$WORK/grpc-server.cpp" +grep -q 'catch (const json::parse_error& e)' "$WORK/grpc-server.cpp" +! grep -q 'common_json_error' "$WORK/grpc-server.cpp" + +# A repeated preparation pass must not change the generated source. +cp "$WORK/grpc-server.cpp" "$WORK/once.cpp" +bash "$PATCHER" "$WORK/grpc-server.cpp" +cmp "$WORK/once.cpp" "$WORK/grpc-server.cpp" + +echo "PASS: Bonsai uses its fork-compatible JSON exception"