[pull] master from mudler:master

backend/cpp/audio-cpp/Makefile

@@ -9,7 +9,7 @@
 # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean
 # rebuild and so the bump bot can see the pin.
 
-AUDIO_CPP_VERSION?=288a2712316470847a730e55db9ac9e5062a2b03
+AUDIO_CPP_VERSION?=d25ffac094a9d5a240940b4955ea79ad9b7b4c78
 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))

backend/cpp/bonsai/Makefile

@@ -41,6 +41,7 @@ define bonsai-build
 	# and are applied by apply-patches.sh below.
 	rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/patches
 	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build purge
+	bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp
 	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp
 	bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp
 	$(info $(GREEN)I bonsai build info:$(1)$(RESET))
@@ -79,6 +80,7 @@ bonsai-cpu-all:
 	# and are applied by apply-patches.sh below.
 	rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/patches
 	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build purge
+	bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp
 	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp
 	bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp
 	$(info $(GREEN)I bonsai build info:cpu-all-variants$(RESET))

backend/cpp/bonsai/patch-grpc-server.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Adapt the shared llama.cpp gRPC source to the older JSON API in Bonsai.
+
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+    echo "usage: $0 <grpc-server.cpp>" >&2
+    exit 2
+fi
+
+SRC=$1
+if [[ ! -f "$SRC" ]]; then
+    echo "grpc-server.cpp not found at $SRC" >&2
+    exit 2
+fi
+
+if grep -q 'common_json_error' "$SRC"; then
+    echo "==> patching $SRC to use the Bonsai JSON exception type"
+    awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp"
+    mv "$SRC.tmp" "$SRC"
+    echo "==> Bonsai JSON exception patch OK"
+else
+    echo "==> $SRC already uses a Bonsai-compatible JSON exception type, skipping"
+fi

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=8337e4cd3861406fc04e0854b1409cd1b027fbc9
+IK_LLAMA_VERSION?=0ed847d3140baead542abe3e5e6fe841013e7340
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=d59d455fd8ea09e5a2e87ce2a9d668267ffb5ccd
+LLAMA_VERSION?=f280b26983ad0fdb705a0d9ebf0503e76f2899b0
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/grpc-server.cpp

@@ -294,7 +294,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
             } else {
                 SRV_WRN("[TOOLS DEBUG] parse_options: Parsed tools JSON is not an array: %s\n", tools_json.dump().c_str());
             }
-        } catch (const json::parse_error& e) {
+        } catch (const common_json_error& e) {
             SRV_WRN("Failed to parse tools JSON from proto: %s\n", e.what());
             SRV_WRN("[TOOLS DEBUG] parse_options: Tools string that failed to parse: %s\n", predict->tools().c_str());
         }
@@ -324,7 +324,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
                 SRV_DBG("[TOOLS DEBUG] Received tool_choice object from Go layer: %s\n", tool_choice_json.dump().c_str());
             }
             SRV_INF("Extracted tool_choice from proto: %s\n", predict->toolchoice().c_str());
-        } catch (const json::parse_error& e) {
+        } catch (const common_json_error& e) {
             // If parsing fails, treat as string
             data["tool_choice"] = predict->toolchoice();
             SRV_INF("Extracted tool_choice as string: %s\n", predict->toolchoice().c_str());
@@ -353,7 +353,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
             // Add to data - llama.cpp server expects it as an object (map)
             data["logit_bias"] = logit_bias_json;
             SRV_INF("Using logit_bias: %s\n", predict->logitbias().c_str());
-        } catch (const json::parse_error& e) {
+        } catch (const common_json_error& e) {
             SRV_ERR("Failed to parse logit_bias JSON from proto: %s\n", e.what());
         }
     }
@@ -398,7 +398,10 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
             });
     }
 
-    data["stop"] = predict->stopprompts();
+    data["stop"] = json::array();
+    for (const auto & stop : predict->stopprompts()) {
+        data["stop"].push_back(stop);
+    }
     // data["n_probs"] = predict->nprobs();
     //TODO: images,
 
@@ -1795,7 +1798,7 @@ class BackendServiceImpl final : public backend::Backend::Service {
                         for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j));
                         for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j));
                     }
-                    messages_json.push_back(llama_grpc::build_reconstructed_message(rin));
+                    messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump()));
                 }
 
                 // Final safety check: Ensure no message has null content (Jinja templates require strings)
@@ -1988,7 +1991,7 @@ class BackendServiceImpl final : public backend::Backend::Service {
                             if (!body_json.contains("chat_template_kwargs")) {
                                 body_json["chat_template_kwargs"] = json::object();
                             }
-                            for (auto& el : ctk.items()) {
+                            for (auto el : ctk.items()) {
                                 body_json["chat_template_kwargs"][el.key()] = el.value();
                             }
                         }
@@ -2074,30 +2077,27 @@ class BackendServiceImpl final : public backend::Backend::Service {
             // If not using chat templates, extract files from image_data/audio_data fields
             // (If using chat templates, files were already extracted by oaicompat_chat_params_parse)
             if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) {
-                const auto &images_data = data.find("image_data");
-                if (images_data != data.end() && images_data->is_array())
+                if (data.contains("image_data") && data.at("image_data").is_array())
                 {
-                    for (const auto &img : *images_data)
+                    for (const auto &img : data.at("image_data"))
                     {
                         auto decoded_data = base64_decode(img["data"].get<std::string>());
                         files.push_back(decoded_data);
                     }
                 }
 
-                const auto &audio_data = data.find("audio_data");
-                if (audio_data != data.end() && audio_data->is_array())
+                if (data.contains("audio_data") && data.at("audio_data").is_array())
                 {
-                    for (const auto &audio : *audio_data)
+                    for (const auto &audio : data.at("audio_data"))
                     {
                         auto decoded_data = base64_decode(audio["data"].get<std::string>());
                         files.push_back(decoded_data);
                     }
                 }
 
-                const auto &video_data = data.find("video_data");
-                if (video_data != data.end() && video_data->is_array())
+                if (data.contains("video_data") && data.at("video_data").is_array())
                 {
-                    for (const auto &video : *video_data)
+                    for (const auto &video : data.at("video_data"))
                     {
                         auto decoded_data = base64_decode(video["data"].get<std::string>());
                         files.push_back(decoded_data);
@@ -2370,7 +2370,7 @@ class BackendServiceImpl final : public backend::Backend::Service {
                         for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j));
                         for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j));
                     }
-                    messages_json.push_back(llama_grpc::build_reconstructed_message(rin));
+                    messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump()));
                 }
 
                 // Final safety check: Ensure no message has null content (Jinja templates require strings)
@@ -2563,7 +2563,7 @@ class BackendServiceImpl final : public backend::Backend::Service {
                             if (!body_json.contains("chat_template_kwargs")) {
                                 body_json["chat_template_kwargs"] = json::object();
                             }
-                            for (auto& el : ctk.items()) {
+                            for (auto el : ctk.items()) {
                                 body_json["chat_template_kwargs"][el.key()] = el.value();
                             }
                         }
@@ -2649,32 +2649,29 @@ class BackendServiceImpl final : public backend::Backend::Service {
             // If not using chat templates, extract files from image_data/audio_data fields
             // (If using chat templates, files were already extracted by oaicompat_chat_params_parse)
             if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) {
-                const auto &images_data = data.find("image_data");
-                if (images_data != data.end() && images_data->is_array())
+                if (data.contains("image_data") && data.at("image_data").is_array())
                 {
-                    std::cout << "[PREDICT] Processing " << images_data->size() << " images" << std::endl;
-                    for (const auto &img : *images_data)
+                    std::cout << "[PREDICT] Processing " << data.at("image_data").size() << " images" << std::endl;
+                    for (const auto &img : data.at("image_data"))
                     {
                         std::cout << "[PREDICT] Processing image" << std::endl;
                         auto decoded_data = base64_decode(img["data"].get<std::string>());
                         files.push_back(decoded_data);
                     }
                 }
 
-                const auto &audio_data = data.find("audio_data");
-                if (audio_data != data.end() && audio_data->is_array())
+                if (data.contains("audio_data") && data.at("audio_data").is_array())
                 {
-                    for (const auto &audio : *audio_data)
+                    for (const auto &audio : data.at("audio_data"))
                     {
                         auto decoded_data = base64_decode(audio["data"].get<std::string>());
                         files.push_back(decoded_data);
                     }
                 }
 
-                const auto &video_data = data.find("video_data");
-                if (video_data != data.end() && video_data->is_array())
+                if (data.contains("video_data") && data.at("video_data").is_array())
                 {
-                    for (const auto &video : *video_data)
+                    for (const auto &video : data.at("video_data"))
                     {
                         auto decoded_data = base64_decode(video["data"].get<std::string>());
                         files.push_back(decoded_data);
@@ -3005,7 +3002,7 @@ class BackendServiceImpl final : public backend::Backend::Service {
         }
 
         // Collect responses
-        json responses = json::array();
+        std::vector<json> responses;
         for (auto & res : all_results.results) {
             GGML_ASSERT(dynamic_cast<server_task_result_rerank*>(res.get()) != nullptr);
             responses.push_back(res->to_json());
@@ -3018,7 +3015,7 @@ class BackendServiceImpl final : public backend::Backend::Service {
         // Crop results by request.top_n if specified
         int top_n = request->top_n();
         if (top_n > 0 && top_n < static_cast<int>(responses.size())) {
-            responses = json(responses.begin(), responses.begin() + top_n);
+            responses.resize(top_n);
         }
         // Set usage information
         backend::Usage* usage = rerankResult->mutable_usage();

backend/cpp/llama-cpp/message_content.h

@@ -52,14 +52,15 @@ inline nlohmann::ordered_json normalize_message_content(const std::string& role,
 // (#7528). A multimodal user message legitimately carries a typed-part array
 // ({type:text}, {type:image_url}, ...), which must be left intact. Shared by the
 // streaming and non-streaming paths so this invariant cannot drift between them.
-inline void normalize_template_message(nlohmann::ordered_json& msg) {
+template <typename Json>
+inline void normalize_template_message(Json& msg) {
     if (!msg.contains("content")) {
         msg["content"] = ""; // templates expect the field to exist
         return;
     }
-    nlohmann::ordered_json& content = msg["content"];
+    auto& content = msg["content"];
     const std::string role = (msg.contains("role") && msg["role"].is_string())
-                                 ? msg["role"].get<std::string>()
+                                 ? msg["role"].template get<std::string>()
                                  : std::string();
     if (content.is_null()) {
         content = ""; // #7324: null would crash content[:N] slicing

backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch

@@ -6,10 +6,9 @@ Subject: [PATCH 1/2] score-patch
 ---
  common/common.cpp               |   6 +-
  common/common.h                 |   3 +
- tools/CMakeLists.txt            |   1 +
  tools/server/server-context.cpp | 358 +++++++++++++++++++++++++++++++-
  tools/server/server-task.h      |  47 +++++
- 5 files changed, 406 insertions(+), 9 deletions(-)
+ 4 files changed, 405 insertions(+), 9 deletions(-)
 
 diff --git a/common/common.cpp b/common/common.cpp
 index 2e3f14c..0cec0dc 100644
@@ -42,15 +41,6 @@ index 878534d..4001df2 100644
      int32_t n_sequences           =     1; // number of sequences to decode
      int32_t n_outputs_max         =     0; // max outputs in a batch (0 = n_batch)
      int32_t n_outputs_max_per_seq =     1; // max outputs per sequence
-diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
-index 780df32..1d2fe8f 100644
---- a/tools/CMakeLists.txt
-+++ b/tools/CMakeLists.txt
-@@ -41,3 +41,4 @@ else()
-     add_subdirectory(fit-params)
-     add_subdirectory(results)
- endif()
-+add_subdirectory(grpc-server)
 diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
 index 3b5f6a1..d0e18e6 100644
 --- a/tools/server/server-context.cpp

backend/cpp/turboquant/patch-grpc-server.sh

@@ -8,6 +8,8 @@
 #      so the grpc-server option parser skips the two references to
 #      common_params::checkpoint_min_step (the default and the option handler).
 #      That field does not exist in the fork yet; drop this once it does.
+#   3. Use nlohmann's parse_error type in JSON catch clauses because the fork
+#      predates upstream's common_json_error wrapper.
 #
 # The fork used to lag upstream on the whole common_params_speculative refactor
 # (ggml-org/llama.cpp#22397/#22838/#22964), the model_tgt rename (#22838) and
@@ -100,4 +102,16 @@ else
     echo "==> LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP define OK"
 fi
 
+# 3. The shared source follows current upstream and catches common_json_error.
+#    TurboQuant still exposes nlohmann::json directly, so its equivalent parse
+#    failures use json::parse_error instead.
+if grep -q 'common_json_error' "$SRC"; then
+    echo "==> patching $SRC to use the TurboQuant JSON exception type"
+    awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp"
+    mv "$SRC.tmp" "$SRC"
+    echo "==> TurboQuant JSON exception patch OK"
+else
+    echo "==> $SRC already uses a TurboQuant-compatible JSON exception type, skipping"
+fi
+
 echo "==> all patches applied"

backend/python/sglang/backend.py

@@ -363,8 +363,9 @@ def _build_prompt(self, request) -> str:
                 template_kwargs["tools"] = json.loads(request.Tools)
             except json.JSONDecodeError:
                 pass
-        if request.Metadata.get("enable_thinking", "").lower() == "true":
-            template_kwargs["enable_thinking"] = True
+        _thinking = request.Metadata.get("enable_thinking", "").lower()
+        if _thinking in ("true", "false"):
+            template_kwargs["enable_thinking"] = (_thinking == "true")
 
         try:
             return self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs)

backend/python/sglang/test.py

@@ -96,6 +96,38 @@ def test_apply_engine_args_non_object_raises(self):
             servicer._apply_engine_args({}, "[1,2,3]")
         self.assertIn("must be a JSON object", str(ctx.exception))
 
+    def test_build_prompt_forwards_enable_thinking(self):
+        from types import SimpleNamespace
+
+        class Tok:
+            def __init__(self):
+                self.kwargs = None
+
+            def apply_chat_template(self, messages, **kwargs):
+                self.kwargs = kwargs
+                return "PROMPT"
+
+        def kwargs_for(metadata):
+            servicer = self._servicer()
+            tok = Tok()
+            servicer.tokenizer = tok
+            msg = SimpleNamespace(
+                role="user", content="hi", name="",
+                tool_call_id="", reasoning_content="", tool_calls="",
+            )
+            req = SimpleNamespace(
+                Prompt="", UseTokenizerTemplate=True,
+                Messages=[msg], Tools="", Metadata=metadata,
+            )
+            self.assertEqual(servicer._build_prompt(req), "PROMPT")
+            return tok.kwargs
+
+        self.assertIs(kwargs_for({"enable_thinking": "true"})["enable_thinking"], True)
+        # "false" used to be dropped, so Qwen3 kept thinking on
+        self.assertIs(kwargs_for({"enable_thinking": "false"})["enable_thinking"], False)
+        self.assertNotIn("enable_thinking", kwargs_for({}))
+        self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False)
+
 
 if __name__ == "__main__":
     unittest.main()

backend/python/vllm/backend.py

@@ -587,9 +587,9 @@ async def _predict(self, request, context, streaming=False):
                 except json.JSONDecodeError:
                     pass
 
-            # Enable thinking mode if requested
-            if request.Metadata.get("enable_thinking", "").lower() == "true":
-                template_kwargs["enable_thinking"] = True
+            _thinking = request.Metadata.get("enable_thinking", "").lower()
+            if _thinking in ("true", "false"):
+                template_kwargs["enable_thinking"] = (_thinking == "true")
 
             try:
                 prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs)