[pull] master from mudler:master

.github/backend-matrix.yml

@@ -3754,6 +3754,19 @@ include:
     dockerfile: "./backend/Dockerfile.golang"
     context: "./"
     ubuntu-version: '2404'
+  - build-type: 'hipblas'
+    cuda-major-version: ""
+    cuda-minor-version: ""
+    platforms: 'linux/amd64'
+    tag-latest: 'auto'
+    tag-suffix: '-gpu-rocm-hipblas-stablediffusion-ggml'
+    runs-on: 'ubuntu-latest'
+    base-image: "rocm/dev-ubuntu-24.04:7.2.1"
+    skip-drivers: 'false'
+    backend: "stablediffusion-ggml"
+    dockerfile: "./backend/Dockerfile.golang"
+    context: "./"
+    ubuntu-version: '2404'
   - build-type: 'sycl_f16'
     cuda-major-version: ""
     cuda-minor-version: ""

backend/go/stablediffusion-ggml/Makefile

@@ -38,8 +38,11 @@ else ifeq ($(BUILD_TYPE),hipblas)
 	ROCM_PATH ?= /opt/rocm
 	export CXX=$(ROCM_HOME)/llvm/bin/clang++
 	export CC=$(ROCM_HOME)/llvm/bin/clang
-	AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
-	CMAKE_ARGS+=-DSD_HIPBLAS=ON -DGGML_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
+	AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201
+	# SD_HIPBLAS turns on ggml's HIP backend itself; GGML_HIPBLAS is the name ggml
+	# used before it was renamed to GGML_HIP, so passing it here only produced an
+	# unused-variable warning.
+	CMAKE_ARGS+=-DSD_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
 else ifeq ($(BUILD_TYPE),vulkan)
 	CMAKE_ARGS+=-DSD_VULKAN=ON -DGGML_VULKAN=ON
 else ifeq ($(BUILD_TYPE),metal)

backend/go/vllm-cpp/Makefile

@@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e
 
 # vllm.cpp version
 VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
-VLLM_CPP_VERSION?=6738e0b4639199f3ff0998815e4d32bfa7fe5be2
+VLLM_CPP_VERSION?=150b37852c123f7855fb219b37347572ca9427e7
 
 # MLX GEMM provider (darwin/metal only; see the metal branch below for why).
 # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun

backend/index.yaml

@@ -510,7 +510,7 @@
     default: "cpu-stablediffusion-ggml"
     nvidia: "cuda12-stablediffusion-ggml"
     intel: "intel-sycl-f16-stablediffusion-ggml"
-    # amd: "rocm-stablediffusion-ggml"
+    amd: "rocm-stablediffusion-ggml"
     vulkan: "vulkan-stablediffusion-ggml"
     nvidia-l4t: "nvidia-l4t-arm64-stablediffusion-ggml"
     metal: "metal-stablediffusion-ggml"
@@ -2109,7 +2109,7 @@
     default: "cpu-stablediffusion-ggml-development"
     nvidia: "cuda12-stablediffusion-ggml-development"
     intel: "intel-sycl-f16-stablediffusion-ggml-development"
-    # amd: "rocm-stablediffusion-ggml-development"
+    amd: "rocm-stablediffusion-ggml-development"
     vulkan: "vulkan-stablediffusion-ggml-development"
     nvidia-l4t: "nvidia-l4t-arm64-stablediffusion-ggml-development"
     metal: "metal-stablediffusion-ggml-development"
@@ -3904,6 +3904,11 @@
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-stablediffusion-ggml"
   mirrors:
     - localai/localai-backends:latest-gpu-nvidia-cuda-12-stablediffusion-ggml
+- !!merge <<: *stablediffusionggml
+  name: "rocm-stablediffusion-ggml"
+  uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-stablediffusion-ggml"
+  mirrors:
+    - localai/localai-backends:latest-gpu-rocm-hipblas-stablediffusion-ggml
 - !!merge <<: *stablediffusionggml
   name: "intel-sycl-f32-stablediffusion-ggml"
   uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-stablediffusion-ggml"
@@ -3917,6 +3922,11 @@
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-stablediffusion-ggml"
   mirrors:
     - localai/localai-backends:master-gpu-nvidia-cuda-12-stablediffusion-ggml
+- !!merge <<: *stablediffusionggml
+  name: "rocm-stablediffusion-ggml-development"
+  uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-stablediffusion-ggml"
+  mirrors:
+    - localai/localai-backends:master-gpu-rocm-hipblas-stablediffusion-ggml
 - !!merge <<: *stablediffusionggml
   name: "intel-sycl-f32-stablediffusion-ggml-development"
   uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f32-stablediffusion-ggml"

core/http/endpoints/openai/realtime_webrtc.go

@@ -1,6 +1,9 @@
 package openai
 
 import (
+	"encoding/json"
+	"io"
+	"mime"
 	"net/http"
 	"time"
 
@@ -27,6 +30,61 @@ type RealtimeCallResponse struct {
 	SessionID string `json:"session_id"`
 }
 
+func decodeRealtimeCallRequest(c echo.Context) (RealtimeCallRequest, bool, error) {
+	var req RealtimeCallRequest
+	mediaType := ""
+	contentType := c.Request().Header.Get(echo.HeaderContentType)
+	if contentType != "" {
+		var err error
+		mediaType, _, err = mime.ParseMediaType(contentType)
+		if err != nil {
+			return req, false, err
+		}
+	}
+
+	switch mediaType {
+	case echo.MIMEMultipartForm:
+		if err := c.Request().ParseMultipartForm(32 << 20); err != nil {
+			return req, true, err
+		}
+		req.SDP = c.FormValue("sdp")
+		var session struct {
+			Model            string `json:"model"`
+			LocalAIAssistant bool   `json:"localai_assistant,omitempty"`
+		}
+		if err := json.Unmarshal([]byte(c.FormValue("session")), &session); err != nil {
+			return req, true, err
+		}
+		req.Model = session.Model
+		req.LocalAIAssistant = session.LocalAIAssistant
+		return req, true, nil
+	case "application/sdp":
+		sdp, err := readRealtimeSDP(c.Request().Body)
+		req.SDP = sdp
+		req.Model = c.QueryParam("model")
+		return req, true, err
+	default:
+		err := c.Bind(&req)
+		return req, false, err
+	}
+}
+
+func readRealtimeSDP(body io.Reader) (string, error) {
+	data, err := io.ReadAll(body)
+	return string(data), err
+}
+
+func writeRealtimeCallResponse(c echo.Context, plainSDPResponse bool, sdp, sessionID string) error {
+	if plainSDPResponse {
+		return c.Blob(http.StatusCreated, "application/sdp", []byte(sdp))
+	}
+
+	return c.JSON(http.StatusCreated, RealtimeCallResponse{
+		SDP:       sdp,
+		SessionID: sessionID,
+	})
+}
+
 // RealtimeCalls handles POST /v1/realtime/calls for WebRTC signaling.
 func RealtimeCalls(application *application.Application) echo.HandlerFunc {
 	se, settingEngineErr := webRTCSettingEngine(application.ApplicationConfig())
@@ -38,8 +96,8 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc {
 		if settingEngineErr != nil {
 			return c.JSON(http.StatusInternalServerError, map[string]string{"error": settingEngineErr.Error()})
 		}
-		var req RealtimeCallRequest
-		if err := c.Bind(&req); err != nil {
+		req, plainSDPResponse, err := decodeRealtimeCallRequest(c)
+		if err != nil {
 			return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid request body"})
 		}
 		if req.SDP == "" {
@@ -189,10 +247,7 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc {
 			runRealtimeSession(application, transport, req.Model, evaluator, opts)
 		}()
 
-		return c.JSON(http.StatusCreated, RealtimeCallResponse{
-			SDP:       localDesc.SDP,
-			SessionID: sessionID,
-		})
+		return writeRealtimeCallResponse(c, plainSDPResponse, localDesc.SDP, sessionID)
 	}
 }
 

core/http/endpoints/openai/realtime_webrtc_request_test.go

@@ -0,0 +1,91 @@
+package openai
+
+import (
+	"bytes"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"net/textproto"
+
+	"github.com/labstack/echo/v4"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("decodeRealtimeCallRequest", func() {
+	It("decodes the legacy JSON request", func() {
+		request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", bytes.NewBufferString(`{"sdp":"offer","model":"voice","localai_assistant":true}`))
+		request.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
+
+		req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder()))
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true}))
+		Expect(plainSDPResponse).To(BeFalse())
+	})
+
+	It("decodes the OpenAI multipart request", func() {
+		var body bytes.Buffer
+		writer := multipart.NewWriter(&body)
+		sdpHeader := make(textproto.MIMEHeader)
+		sdpHeader.Set("Content-Disposition", `form-data; name="sdp"`)
+		sdpHeader.Set("Content-Type", "application/sdp")
+		sdpPart, err := writer.CreatePart(sdpHeader)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = sdpPart.Write([]byte("offer"))
+		Expect(err).NotTo(HaveOccurred())
+		sessionHeader := make(textproto.MIMEHeader)
+		sessionHeader.Set("Content-Disposition", `form-data; name="session"`)
+		sessionHeader.Set("Content-Type", echo.MIMEApplicationJSON)
+		sessionPart, err := writer.CreatePart(sessionHeader)
+		Expect(err).NotTo(HaveOccurred())
+		_, err = sessionPart.Write([]byte(`{"type":"realtime","model":"voice","localai_assistant":true}`))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(writer.Close()).To(Succeed())
+
+		request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", &body)
+		request.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
+		req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder()))
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true}))
+		Expect(plainSDPResponse).To(BeTrue())
+	})
+
+	It("decodes a raw SDP request with the model query parameter", func() {
+		request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls?model=voice", bytes.NewBufferString("offer"))
+		request.Header.Set(echo.HeaderContentType, "application/sdp")
+
+		req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder()))
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice"}))
+		Expect(plainSDPResponse).To(BeTrue())
+	})
+})
+
+var _ = Describe("writeRealtimeCallResponse", func() {
+	It("writes the bare SDP answer for OpenAI request formats", func() {
+		response := httptest.NewRecorder()
+		request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil)
+		context := echo.New().NewContext(request, response)
+
+		Expect(writeRealtimeCallResponse(context, true, "answer", "session-id")).To(Succeed())
+
+		Expect(response.Code).To(Equal(http.StatusCreated))
+		Expect(response.Header().Get(echo.HeaderContentType)).To(Equal("application/sdp"))
+		Expect(response.Body.String()).To(Equal("answer"))
+	})
+
+	It("preserves the JSON response for legacy requests", func() {
+		response := httptest.NewRecorder()
+		request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil)
+		context := echo.New().NewContext(request, response)
+
+		Expect(writeRealtimeCallResponse(context, false, "answer", "session-id")).To(Succeed())
+
+		Expect(response.Code).To(Equal(http.StatusCreated))
+		Expect(response.Header().Get(echo.HeaderContentType)).To(Equal(echo.MIMEApplicationJSON))
+		Expect(response.Body.String()).To(MatchJSON(`{"sdp":"answer","session_id":"session-id"}`))
+	})
+})

docs/content/features/openai-realtime.md

@@ -266,16 +266,26 @@ Audio is sent and received as raw PCM in the WebSocket messages, following the O
 
 ### WebRTC
 
-The WebRTC transport enables browser-based voice conversations with lower latency. Connect by POSTing an SDP offer to the REST endpoint:
+The WebRTC transport enables browser-based voice conversations with lower latency. OpenAI-compatible clients can send a raw SDP offer and select the model with the query parameter:
 
 ```
-POST http://localhost:8080/v1/realtime?model=gpt-realtime
+POST http://localhost:8080/v1/realtime/calls?model=gpt-realtime
 Content-Type: application/sdp
 
 <SDP offer body>
 ```
 
-The response contains the SDP answer to complete the WebRTC handshake.
+The response has the `application/sdp` content type and contains the bare SDP answer.
+
+The unified OpenAI interface is also supported. Send `multipart/form-data` with an `sdp` field that contains the offer and a JSON `session` field. LocalAI reads the model from the session object:
+
+```bash
+curl http://localhost:8080/v1/realtime/calls \
+  -F "sdp=<offer.sdp;type=application/sdp" \
+  -F 'session={"type":"realtime","model":"gpt-realtime"};type=application/json'
+```
+
+LocalAI also accepts its original JSON request format for compatibility. A JSON request contains top-level `sdp` and `model` fields and receives a JSON response with `sdp` and `session_id` fields.
 
 #### Opus backend requirement