From 1db8db762d31cb6e8cd365c3f52715242c58d6cf Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:20:09 +0200 Subject: [PATCH 1/5] chore: :arrow_up: Update mudler/vllm.cpp to `150b37852c123f7855fb219b37347572ca9427e7` (#11745) :arrow_up: Update mudler/vllm.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/vllm-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index d403319f55f2..1ab67324a4e3 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=6738e0b4639199f3ff0998815e4d32bfa7fe5be2 +VLLM_CPP_VERSION?=150b37852c123f7855fb219b37347572ca9427e7 # MLX GEMM provider (darwin/metal only; see the metal branch below for why). # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun From 893a45141c5a3d94b9c86d2b79cbe7c4fc3dc835 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sat, 29 Aug 2026 21:28:37 +0200 Subject: [PATCH 2/5] fix(realtime): accept GA WebRTC signaling (#11778) OpenAI GA clients send multipart or raw SDP requests. They expect a bare SDP answer. LocalAI only accepted its legacy JSON envelope, so signaling failed before media setup. Keep the JSON contract for existing clients. Accept both GA request shapes and choose the matching response format. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- core/http/endpoints/openai/realtime_webrtc.go | 67 ++++++++++++-- .../openai/realtime_webrtc_request_test.go | 91 +++++++++++++++++++ docs/content/features/openai-realtime.md | 16 +++- 3 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 core/http/endpoints/openai/realtime_webrtc_request_test.go diff --git a/core/http/endpoints/openai/realtime_webrtc.go b/core/http/endpoints/openai/realtime_webrtc.go index 4f13862c80ac..eca3aa2c6b14 100644 --- a/core/http/endpoints/openai/realtime_webrtc.go +++ b/core/http/endpoints/openai/realtime_webrtc.go @@ -1,6 +1,9 @@ package openai import ( + "encoding/json" + "io" + "mime" "net/http" "time" @@ -27,6 +30,61 @@ type RealtimeCallResponse struct { SessionID string `json:"session_id"` } +func decodeRealtimeCallRequest(c echo.Context) (RealtimeCallRequest, bool, error) { + var req RealtimeCallRequest + mediaType := "" + contentType := c.Request().Header.Get(echo.HeaderContentType) + if contentType != "" { + var err error + mediaType, _, err = mime.ParseMediaType(contentType) + if err != nil { + return req, false, err + } + } + + switch mediaType { + case echo.MIMEMultipartForm: + if err := c.Request().ParseMultipartForm(32 << 20); err != nil { + return req, true, err + } + req.SDP = c.FormValue("sdp") + var session struct { + Model string `json:"model"` + LocalAIAssistant bool `json:"localai_assistant,omitempty"` + } + if err := json.Unmarshal([]byte(c.FormValue("session")), &session); err != nil { + return req, true, err + } + req.Model = session.Model + req.LocalAIAssistant = session.LocalAIAssistant + return req, true, nil + case "application/sdp": + sdp, err := readRealtimeSDP(c.Request().Body) + req.SDP = sdp + req.Model = c.QueryParam("model") + return req, true, err + default: + err := c.Bind(&req) + return req, false, err + } +} + +func readRealtimeSDP(body io.Reader) (string, error) { + data, err := io.ReadAll(body) + return string(data), err +} + +func writeRealtimeCallResponse(c echo.Context, plainSDPResponse bool, sdp, sessionID string) error { + if plainSDPResponse { + return c.Blob(http.StatusCreated, "application/sdp", []byte(sdp)) + } + + return c.JSON(http.StatusCreated, RealtimeCallResponse{ + SDP: sdp, + SessionID: sessionID, + }) +} + // RealtimeCalls handles POST /v1/realtime/calls for WebRTC signaling. func RealtimeCalls(application *application.Application) echo.HandlerFunc { se, settingEngineErr := webRTCSettingEngine(application.ApplicationConfig()) @@ -38,8 +96,8 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc { if settingEngineErr != nil { return c.JSON(http.StatusInternalServerError, map[string]string{"error": settingEngineErr.Error()}) } - var req RealtimeCallRequest - if err := c.Bind(&req); err != nil { + req, plainSDPResponse, err := decodeRealtimeCallRequest(c) + if err != nil { return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid request body"}) } if req.SDP == "" { @@ -189,10 +247,7 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc { runRealtimeSession(application, transport, req.Model, evaluator, opts) }() - return c.JSON(http.StatusCreated, RealtimeCallResponse{ - SDP: localDesc.SDP, - SessionID: sessionID, - }) + return writeRealtimeCallResponse(c, plainSDPResponse, localDesc.SDP, sessionID) } } diff --git a/core/http/endpoints/openai/realtime_webrtc_request_test.go b/core/http/endpoints/openai/realtime_webrtc_request_test.go new file mode 100644 index 000000000000..1875553296a6 --- /dev/null +++ b/core/http/endpoints/openai/realtime_webrtc_request_test.go @@ -0,0 +1,91 @@ +package openai + +import ( + "bytes" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/textproto" + + "github.com/labstack/echo/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("decodeRealtimeCallRequest", func() { + It("decodes the legacy JSON request", func() { + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", bytes.NewBufferString(`{"sdp":"offer","model":"voice","localai_assistant":true}`)) + request.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true})) + Expect(plainSDPResponse).To(BeFalse()) + }) + + It("decodes the OpenAI multipart request", func() { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + sdpHeader := make(textproto.MIMEHeader) + sdpHeader.Set("Content-Disposition", `form-data; name="sdp"`) + sdpHeader.Set("Content-Type", "application/sdp") + sdpPart, err := writer.CreatePart(sdpHeader) + Expect(err).NotTo(HaveOccurred()) + _, err = sdpPart.Write([]byte("offer")) + Expect(err).NotTo(HaveOccurred()) + sessionHeader := make(textproto.MIMEHeader) + sessionHeader.Set("Content-Disposition", `form-data; name="session"`) + sessionHeader.Set("Content-Type", echo.MIMEApplicationJSON) + sessionPart, err := writer.CreatePart(sessionHeader) + Expect(err).NotTo(HaveOccurred()) + _, err = sessionPart.Write([]byte(`{"type":"realtime","model":"voice","localai_assistant":true}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) + + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", &body) + request.Header.Set(echo.HeaderContentType, writer.FormDataContentType()) + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true})) + Expect(plainSDPResponse).To(BeTrue()) + }) + + It("decodes a raw SDP request with the model query parameter", func() { + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls?model=voice", bytes.NewBufferString("offer")) + request.Header.Set(echo.HeaderContentType, "application/sdp") + + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice"})) + Expect(plainSDPResponse).To(BeTrue()) + }) +}) + +var _ = Describe("writeRealtimeCallResponse", func() { + It("writes the bare SDP answer for OpenAI request formats", func() { + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil) + context := echo.New().NewContext(request, response) + + Expect(writeRealtimeCallResponse(context, true, "answer", "session-id")).To(Succeed()) + + Expect(response.Code).To(Equal(http.StatusCreated)) + Expect(response.Header().Get(echo.HeaderContentType)).To(Equal("application/sdp")) + Expect(response.Body.String()).To(Equal("answer")) + }) + + It("preserves the JSON response for legacy requests", func() { + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil) + context := echo.New().NewContext(request, response) + + Expect(writeRealtimeCallResponse(context, false, "answer", "session-id")).To(Succeed()) + + Expect(response.Code).To(Equal(http.StatusCreated)) + Expect(response.Header().Get(echo.HeaderContentType)).To(Equal(echo.MIMEApplicationJSON)) + Expect(response.Body.String()).To(MatchJSON(`{"sdp":"answer","session_id":"session-id"}`)) + }) +}) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index 417fab33104a..54bec58abe43 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -266,16 +266,26 @@ Audio is sent and received as raw PCM in the WebSocket messages, following the O ### WebRTC -The WebRTC transport enables browser-based voice conversations with lower latency. Connect by POSTing an SDP offer to the REST endpoint: +The WebRTC transport enables browser-based voice conversations with lower latency. OpenAI-compatible clients can send a raw SDP offer and select the model with the query parameter: ``` -POST http://localhost:8080/v1/realtime?model=gpt-realtime +POST http://localhost:8080/v1/realtime/calls?model=gpt-realtime Content-Type: application/sdp ``` -The response contains the SDP answer to complete the WebRTC handshake. +The response has the `application/sdp` content type and contains the bare SDP answer. + +The unified OpenAI interface is also supported. Send `multipart/form-data` with an `sdp` field that contains the offer and a JSON `session` field. LocalAI reads the model from the session object: + +```bash +curl http://localhost:8080/v1/realtime/calls \ + -F "sdp= Date: Sat, 29 Aug 2026 21:29:00 +0200 Subject: [PATCH 3/5] feat(gallery): add WeMM embedding variants (#11775) Tencent released three WeMM sizes with direct Sentence Transformers support. Add each safetensor repository so users can select the quality and resource tradeoff. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- gallery/index.yaml | 80 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index a7f62bd8d790..d4163bd310d8 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -124,6 +124,86 @@ - filename: llama-cpp/mmproj/qwen3.8-flash-next/mmproj-BF16.gguf uri: huggingface://unsloth/Qwen3.8-Flash-Next-GGUF/mmproj-BF16.gguf sha256: 2e788f8c511d8093c7b43cb87b2fd7e14228340318057f8fb20c86df2efe2355 +- &wemm-embedding-2b + name: "wemm-embedding-2b" + variants: + - model: wemm-embedding-4b + - model: wemm-embedding-9b + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/tencent/WeMM-Embedding-2B + description: | + WeMM-Embedding-2B is Tencent's Apache-2.0 multilingual embedding model + built on Qwen3.5. This entry serves the original bfloat16 safetensors with + LocalAI's Transformers backend and produces 2,048-dimensional normalized + embeddings for text retrieval, semantic search, and RAG. + + The upstream model can also embed images and videos. LocalAI currently + exposes text input through its embeddings API for this backend. + license: "apache-2.0" + tags: + - embeddings + - safetensors + - transformers + - multilingual + - retrieval + - rag + - gpu + - cpu + last_checked: "2026-08-29" + overrides: + backend: transformers + embeddings: true + trust_remote_code: true + type: SentenceTransformer + known_usecases: + - embeddings + parameters: + model: tencent/WeMM-Embedding-2B +- !!merge <<: *wemm-embedding-2b + name: "wemm-embedding-4b" + variants: [] + urls: + - https://huggingface.co/tencent/WeMM-Embedding-4B + description: | + WeMM-Embedding-4B is Tencent's mid-sized Apache-2.0 multilingual embedding + model built on Qwen3.5. This entry serves the original bfloat16 safetensors + with LocalAI's Transformers backend and produces 2,560-dimensional + normalized embeddings for text retrieval, semantic search, and RAG. + + The upstream model can also embed images and videos. LocalAI currently + exposes text input through its embeddings API for this backend. + overrides: + backend: transformers + embeddings: true + trust_remote_code: true + type: SentenceTransformer + known_usecases: + - embeddings + parameters: + model: tencent/WeMM-Embedding-4B +- !!merge <<: *wemm-embedding-2b + name: "wemm-embedding-9b" + variants: [] + urls: + - https://huggingface.co/tencent/WeMM-Embedding-9B + description: | + WeMM-Embedding-9B is Tencent's largest Apache-2.0 multilingual embedding + model built on Qwen3.5. This entry serves the original bfloat16 safetensors + with LocalAI's Transformers backend and produces 4,096-dimensional + normalized embeddings for text retrieval, semantic search, and RAG. + + The upstream model can also embed images and videos. LocalAI currently + exposes text input through its embeddings API for this backend. + overrides: + backend: transformers + embeddings: true + trust_remote_code: true + type: SentenceTransformer + known_usecases: + - embeddings + parameters: + model: tencent/WeMM-Embedding-9B - &granite-4-2-3b name: "granite-4.2-3b-q4" variants: From 572a12768211becf81cdfdd378fe7632b0bd2e24 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:29:16 +0200 Subject: [PATCH 4/5] feat(stablediffusion-ggml): build a ROCm variant (#11774) The Makefile already had a hipblas branch, but no CI row built it and the gallery's `amd:` mapping stayed commented out. On an AMD host the capability lookup found no `amd` key and fell back to `default`, so these users silently ran the CPU build. Add the hipblas row to the backend matrix and the two gallery entries it publishes, then point `amd:` at them. Drop `-DGGML_HIPBLAS=ON` while here. `SD_HIPBLAS` sets `GGML_HIP` itself, and `GGML_HIPBLAS` is the name ggml used before the rename, so the flag only produced an unused-variable warning. Add gfx1151 to the local target list to match the value the workflows pass in. Assisted-by: Claude Code:claude-opus-5[1m] Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- .github/backend-matrix.yml | 13 +++++++++++++ backend/go/stablediffusion-ggml/Makefile | 7 +++++-- backend/index.yaml | 14 ++++++++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index e7bada4b4e91..3d3b2b804cd3 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -3754,6 +3754,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'hipblas' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-rocm-hipblas-stablediffusion-ggml' + runs-on: 'ubuntu-latest' + base-image: "rocm/dev-ubuntu-24.04:7.2.1" + skip-drivers: 'false' + backend: "stablediffusion-ggml" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'sycl_f16' cuda-major-version: "" cuda-minor-version: "" diff --git a/backend/go/stablediffusion-ggml/Makefile b/backend/go/stablediffusion-ggml/Makefile index 89e533059ccf..fd82adbb0965 100644 --- a/backend/go/stablediffusion-ggml/Makefile +++ b/backend/go/stablediffusion-ggml/Makefile @@ -38,8 +38,11 @@ else ifeq ($(BUILD_TYPE),hipblas) ROCM_PATH ?= /opt/rocm export CXX=$(ROCM_HOME)/llvm/bin/clang++ export CC=$(ROCM_HOME)/llvm/bin/clang - AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201 - CMAKE_ARGS+=-DSD_HIPBLAS=ON -DGGML_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS) + AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201 + # SD_HIPBLAS turns on ggml's HIP backend itself; GGML_HIPBLAS is the name ggml + # used before it was renamed to GGML_HIP, so passing it here only produced an + # unused-variable warning. + CMAKE_ARGS+=-DSD_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS) else ifeq ($(BUILD_TYPE),vulkan) CMAKE_ARGS+=-DSD_VULKAN=ON -DGGML_VULKAN=ON else ifeq ($(BUILD_TYPE),metal) diff --git a/backend/index.yaml b/backend/index.yaml index 3420d85496b7..6e3519f66f3d 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -510,7 +510,7 @@ default: "cpu-stablediffusion-ggml" nvidia: "cuda12-stablediffusion-ggml" intel: "intel-sycl-f16-stablediffusion-ggml" - # amd: "rocm-stablediffusion-ggml" + amd: "rocm-stablediffusion-ggml" vulkan: "vulkan-stablediffusion-ggml" nvidia-l4t: "nvidia-l4t-arm64-stablediffusion-ggml" metal: "metal-stablediffusion-ggml" @@ -2109,7 +2109,7 @@ default: "cpu-stablediffusion-ggml-development" nvidia: "cuda12-stablediffusion-ggml-development" intel: "intel-sycl-f16-stablediffusion-ggml-development" - # amd: "rocm-stablediffusion-ggml-development" + amd: "rocm-stablediffusion-ggml-development" vulkan: "vulkan-stablediffusion-ggml-development" nvidia-l4t: "nvidia-l4t-arm64-stablediffusion-ggml-development" metal: "metal-stablediffusion-ggml-development" @@ -3904,6 +3904,11 @@ uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-stablediffusion-ggml" mirrors: - localai/localai-backends:latest-gpu-nvidia-cuda-12-stablediffusion-ggml +- !!merge <<: *stablediffusionggml + name: "rocm-stablediffusion-ggml" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-stablediffusion-ggml" + mirrors: + - localai/localai-backends:latest-gpu-rocm-hipblas-stablediffusion-ggml - !!merge <<: *stablediffusionggml name: "intel-sycl-f32-stablediffusion-ggml" uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-stablediffusion-ggml" @@ -3917,6 +3922,11 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-stablediffusion-ggml" mirrors: - localai/localai-backends:master-gpu-nvidia-cuda-12-stablediffusion-ggml +- !!merge <<: *stablediffusionggml + name: "rocm-stablediffusion-ggml-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-stablediffusion-ggml" + mirrors: + - localai/localai-backends:master-gpu-rocm-hipblas-stablediffusion-ggml - !!merge <<: *stablediffusionggml name: "intel-sycl-f32-stablediffusion-ggml-development" uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f32-stablediffusion-ggml" From a7cc5873ef5b7c909fc9ff7d349d51738ba9bb05 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:29:32 +0200 Subject: [PATCH 5/5] chore(model gallery): :robot: add 1 new models via gallery agent (#11777) chore(model gallery): :robot: add new models via gallery agent Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- gallery/index.yaml | 105 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/gallery/index.yaml b/gallery/index.yaml index d4163bd310d8..cab4117e092b 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -1,4 +1,109 @@ --- +- name: "glm-5.3" + url: "github:mudler/LocalAI/gallery/virtual.yaml@master" + urls: + - https://huggingface.co/unsloth/GLM-5.3-GGUF + description: | + # GLM-5.3 + + GLM-5.3 uses the same base model as GLM-5.2 — every gain comes from post-training. Compared with GLM-5.2, it is much better at complex coding and long-horizon tasks: + + + Stronger Coding: GLM-5.3 is the most capable open-weights model for coding, with a 50% improvement over GLM-5.2 on our in-house Z.ai Code Bench. It also achieve open-source SOTA on public benchmarks including Terminal Bench 3.0 and Agents' Last Exam. + + Emergent Cyber Capability: As we scaled post-training, cyber capability developed faster than we expected. GLM-5.3 is state of the art on CyberGym for vulnerability discovery, and its gains are largest further up the exploitation chain, where it more than doubles GLM-5.2 on exploitation benchmarks. + + ## Benchmark + + ### Serve GLM-5.3 Locally + + GLM-5.3 supports deployment with the following frameworks. Feel free to try them out: + + - SGLang — see cookbook + - vLLM — see recipes + - TokenSpeed — see here + - Transformers — see transformers docs + - KTransformers — see tutorial + - Unsloth — see guide + - For deployment on the `Ascend NPU` platform, inference frameworks such as vLLM-Ascend, xLLM and SGLang are supported — see here. + + ### Note + + ... + license: "other" + tags: + - llm + - gguf + icon: https://raw.githubusercontent.com/zai-org/GLM-5/refs/heads/main/resources/bench_53_2.png + overrides: + backend: llama-cpp + function: + automatic_tool_parsing_fallback: true + grammar: + disable: true + known_usecases: + - chat + options: + - use_jinja:true + - spec_type:draft-mtp + - spec_n_max:6 + - spec_p_min:0.75 + parameters: + min_p: 0.01 + model: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00001-of-00016.gguf + repeat_penalty: 1 + temperature: 1 + top_k: -1 + top_p: 0.95 + template: + use_tokenizer_template: true + files: + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00001-of-00016.gguf + sha256: d7f982efbb767fe9ce8d9e61e908d16c6c5f9eb470ee354273010b3aae734fea + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00001-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00002-of-00016.gguf + sha256: 710a95e0b38573ba205873caa3b7511b8f0b2a7b4c121698e4f251584087e62e + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00002-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00003-of-00016.gguf + sha256: bc5ba47c593594bcc0d8f3dc504c70661b6659bade502d4fc78ee780ec7ec672 + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00003-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00004-of-00016.gguf + sha256: e766dfbb18e0dc27bd45196fad1bd70634664ef6bb61f0d0625c8df76b38eb36 + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00004-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00005-of-00016.gguf + sha256: a74c1184952552a66234af849aee8e8720ec8c4beb4605a0482d090fa5366d22 + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00005-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00006-of-00016.gguf + sha256: e28dcb4b00c4d3d2b850cba8252ae030017a33d5f283bc87a891d3d49f66e0ed + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00006-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00007-of-00016.gguf + sha256: cd60331983b19c5456fba9da439b4ea1dcdabe84e10239c404a43b25fbb76206 + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00007-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00008-of-00016.gguf + sha256: a5042fdb3764e52ab717d6a8ebe5a8e06abff8f4cebe3c9c0dd51cba32722133 + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00008-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00009-of-00016.gguf + sha256: 5bc47574214ef4dfe89ac1c5623dcc29e81056aed9d5fd3cbe16d5e8d93d42db + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00009-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00010-of-00016.gguf + sha256: 294b62b307ef8c8e39e233a977ed1c34c5f7191c1d69c098c0997b4bfa24bfcb + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00010-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00011-of-00016.gguf + sha256: 40b23e5b14423cfc20bd47e1ce77a434bd51beb428c123f421b4bf7d377e052f + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00011-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00012-of-00016.gguf + sha256: 3833e99de0ffd1673daf306c2a27a67ed0d3b252aec9f0ca8a7f18e7122501f0 + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00012-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00013-of-00016.gguf + sha256: 87f0b9630eabf16cf64fe3003b177bb620a917ff1dff5d1b7d63041ea626b5ff + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00013-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00014-of-00016.gguf + sha256: e598ca6d9a61adae5a339bcaf0ecda8443665cc2f3972daa0c909a9ecf45352a + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00014-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00015-of-00016.gguf + sha256: e3430564ded510ab658d44bd6b412220ab91eadeb0f7e295a3f3360bfa1f4249 + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00015-of-00016.gguf + - filename: llama-cpp/models/GLM-5.3-UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00016-of-00016.gguf + sha256: 79af2211278ac07dfe4c789751d7de7d7caf0a4516486f8a9b6eedb1f3fb6e9b + uri: https://huggingface.co/unsloth/GLM-5.3-GGUF/resolve/main/UD-Q6_K_XL/GLM-5.3-UD-Q6_K_XL-00016-of-00016.gguf - &qwen3-8-flash-next name: "qwen3.8-flash-next-q4" variants: