diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index e7bada4b4e91..3d3b2b804cd3 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -3754,6 +3754,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'hipblas' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-rocm-hipblas-stablediffusion-ggml' + runs-on: 'ubuntu-latest' + base-image: "rocm/dev-ubuntu-24.04:7.2.1" + skip-drivers: 'false' + backend: "stablediffusion-ggml" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'sycl_f16' cuda-major-version: "" cuda-minor-version: "" diff --git a/backend/go/stablediffusion-ggml/Makefile b/backend/go/stablediffusion-ggml/Makefile index 89e533059ccf..fd82adbb0965 100644 --- a/backend/go/stablediffusion-ggml/Makefile +++ b/backend/go/stablediffusion-ggml/Makefile @@ -38,8 +38,11 @@ else ifeq ($(BUILD_TYPE),hipblas) ROCM_PATH ?= /opt/rocm export CXX=$(ROCM_HOME)/llvm/bin/clang++ export CC=$(ROCM_HOME)/llvm/bin/clang - AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201 - CMAKE_ARGS+=-DSD_HIPBLAS=ON -DGGML_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS) + AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201 + # SD_HIPBLAS turns on ggml's HIP backend itself; GGML_HIPBLAS is the name ggml + # used before it was renamed to GGML_HIP, so passing it here only produced an + # unused-variable warning. + CMAKE_ARGS+=-DSD_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS) else ifeq ($(BUILD_TYPE),vulkan) CMAKE_ARGS+=-DSD_VULKAN=ON -DGGML_VULKAN=ON else ifeq ($(BUILD_TYPE),metal) diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index d403319f55f2..1ab67324a4e3 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=6738e0b4639199f3ff0998815e4d32bfa7fe5be2 +VLLM_CPP_VERSION?=150b37852c123f7855fb219b37347572ca9427e7 # MLX GEMM provider (darwin/metal only; see the metal branch below for why). # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun diff --git a/backend/index.yaml b/backend/index.yaml index 3420d85496b7..6e3519f66f3d 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -510,7 +510,7 @@ default: "cpu-stablediffusion-ggml" nvidia: "cuda12-stablediffusion-ggml" intel: "intel-sycl-f16-stablediffusion-ggml" - # amd: "rocm-stablediffusion-ggml" + amd: "rocm-stablediffusion-ggml" vulkan: "vulkan-stablediffusion-ggml" nvidia-l4t: "nvidia-l4t-arm64-stablediffusion-ggml" metal: "metal-stablediffusion-ggml" @@ -2109,7 +2109,7 @@ default: "cpu-stablediffusion-ggml-development" nvidia: "cuda12-stablediffusion-ggml-development" intel: "intel-sycl-f16-stablediffusion-ggml-development" - # amd: "rocm-stablediffusion-ggml-development" + amd: "rocm-stablediffusion-ggml-development" vulkan: "vulkan-stablediffusion-ggml-development" nvidia-l4t: "nvidia-l4t-arm64-stablediffusion-ggml-development" metal: "metal-stablediffusion-ggml-development" @@ -3904,6 +3904,11 @@ uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-stablediffusion-ggml" mirrors: - localai/localai-backends:latest-gpu-nvidia-cuda-12-stablediffusion-ggml +- !!merge <<: *stablediffusionggml + name: "rocm-stablediffusion-ggml" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-stablediffusion-ggml" + mirrors: + - localai/localai-backends:latest-gpu-rocm-hipblas-stablediffusion-ggml - !!merge <<: *stablediffusionggml name: "intel-sycl-f32-stablediffusion-ggml" uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-stablediffusion-ggml" @@ -3917,6 +3922,11 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-stablediffusion-ggml" mirrors: - localai/localai-backends:master-gpu-nvidia-cuda-12-stablediffusion-ggml +- !!merge <<: *stablediffusionggml + name: "rocm-stablediffusion-ggml-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-stablediffusion-ggml" + mirrors: + - localai/localai-backends:master-gpu-rocm-hipblas-stablediffusion-ggml - !!merge <<: *stablediffusionggml name: "intel-sycl-f32-stablediffusion-ggml-development" uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f32-stablediffusion-ggml" diff --git a/core/http/endpoints/openai/realtime_webrtc.go b/core/http/endpoints/openai/realtime_webrtc.go index 4f13862c80ac..eca3aa2c6b14 100644 --- a/core/http/endpoints/openai/realtime_webrtc.go +++ b/core/http/endpoints/openai/realtime_webrtc.go @@ -1,6 +1,9 @@ package openai import ( + "encoding/json" + "io" + "mime" "net/http" "time" @@ -27,6 +30,61 @@ type RealtimeCallResponse struct { SessionID string `json:"session_id"` } +func decodeRealtimeCallRequest(c echo.Context) (RealtimeCallRequest, bool, error) { + var req RealtimeCallRequest + mediaType := "" + contentType := c.Request().Header.Get(echo.HeaderContentType) + if contentType != "" { + var err error + mediaType, _, err = mime.ParseMediaType(contentType) + if err != nil { + return req, false, err + } + } + + switch mediaType { + case echo.MIMEMultipartForm: + if err := c.Request().ParseMultipartForm(32 << 20); err != nil { + return req, true, err + } + req.SDP = c.FormValue("sdp") + var session struct { + Model string `json:"model"` + LocalAIAssistant bool `json:"localai_assistant,omitempty"` + } + if err := json.Unmarshal([]byte(c.FormValue("session")), &session); err != nil { + return req, true, err + } + req.Model = session.Model + req.LocalAIAssistant = session.LocalAIAssistant + return req, true, nil + case "application/sdp": + sdp, err := readRealtimeSDP(c.Request().Body) + req.SDP = sdp + req.Model = c.QueryParam("model") + return req, true, err + default: + err := c.Bind(&req) + return req, false, err + } +} + +func readRealtimeSDP(body io.Reader) (string, error) { + data, err := io.ReadAll(body) + return string(data), err +} + +func writeRealtimeCallResponse(c echo.Context, plainSDPResponse bool, sdp, sessionID string) error { + if plainSDPResponse { + return c.Blob(http.StatusCreated, "application/sdp", []byte(sdp)) + } + + return c.JSON(http.StatusCreated, RealtimeCallResponse{ + SDP: sdp, + SessionID: sessionID, + }) +} + // RealtimeCalls handles POST /v1/realtime/calls for WebRTC signaling. func RealtimeCalls(application *application.Application) echo.HandlerFunc { se, settingEngineErr := webRTCSettingEngine(application.ApplicationConfig()) @@ -38,8 +96,8 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc { if settingEngineErr != nil { return c.JSON(http.StatusInternalServerError, map[string]string{"error": settingEngineErr.Error()}) } - var req RealtimeCallRequest - if err := c.Bind(&req); err != nil { + req, plainSDPResponse, err := decodeRealtimeCallRequest(c) + if err != nil { return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid request body"}) } if req.SDP == "" { @@ -189,10 +247,7 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc { runRealtimeSession(application, transport, req.Model, evaluator, opts) }() - return c.JSON(http.StatusCreated, RealtimeCallResponse{ - SDP: localDesc.SDP, - SessionID: sessionID, - }) + return writeRealtimeCallResponse(c, plainSDPResponse, localDesc.SDP, sessionID) } } diff --git a/core/http/endpoints/openai/realtime_webrtc_request_test.go b/core/http/endpoints/openai/realtime_webrtc_request_test.go new file mode 100644 index 000000000000..1875553296a6 --- /dev/null +++ b/core/http/endpoints/openai/realtime_webrtc_request_test.go @@ -0,0 +1,91 @@ +package openai + +import ( + "bytes" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/textproto" + + "github.com/labstack/echo/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("decodeRealtimeCallRequest", func() { + It("decodes the legacy JSON request", func() { + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", bytes.NewBufferString(`{"sdp":"offer","model":"voice","localai_assistant":true}`)) + request.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true})) + Expect(plainSDPResponse).To(BeFalse()) + }) + + It("decodes the OpenAI multipart request", func() { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + sdpHeader := make(textproto.MIMEHeader) + sdpHeader.Set("Content-Disposition", `form-data; name="sdp"`) + sdpHeader.Set("Content-Type", "application/sdp") + sdpPart, err := writer.CreatePart(sdpHeader) + Expect(err).NotTo(HaveOccurred()) + _, err = sdpPart.Write([]byte("offer")) + Expect(err).NotTo(HaveOccurred()) + sessionHeader := make(textproto.MIMEHeader) + sessionHeader.Set("Content-Disposition", `form-data; name="session"`) + sessionHeader.Set("Content-Type", echo.MIMEApplicationJSON) + sessionPart, err := writer.CreatePart(sessionHeader) + Expect(err).NotTo(HaveOccurred()) + _, err = sessionPart.Write([]byte(`{"type":"realtime","model":"voice","localai_assistant":true}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) + + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", &body) + request.Header.Set(echo.HeaderContentType, writer.FormDataContentType()) + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true})) + Expect(plainSDPResponse).To(BeTrue()) + }) + + It("decodes a raw SDP request with the model query parameter", func() { + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls?model=voice", bytes.NewBufferString("offer")) + request.Header.Set(echo.HeaderContentType, "application/sdp") + + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice"})) + Expect(plainSDPResponse).To(BeTrue()) + }) +}) + +var _ = Describe("writeRealtimeCallResponse", func() { + It("writes the bare SDP answer for OpenAI request formats", func() { + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil) + context := echo.New().NewContext(request, response) + + Expect(writeRealtimeCallResponse(context, true, "answer", "session-id")).To(Succeed()) + + Expect(response.Code).To(Equal(http.StatusCreated)) + Expect(response.Header().Get(echo.HeaderContentType)).To(Equal("application/sdp")) + Expect(response.Body.String()).To(Equal("answer")) + }) + + It("preserves the JSON response for legacy requests", func() { + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil) + context := echo.New().NewContext(request, response) + + Expect(writeRealtimeCallResponse(context, false, "answer", "session-id")).To(Succeed()) + + Expect(response.Code).To(Equal(http.StatusCreated)) + Expect(response.Header().Get(echo.HeaderContentType)).To(Equal(echo.MIMEApplicationJSON)) + Expect(response.Body.String()).To(MatchJSON(`{"sdp":"answer","session_id":"session-id"}`)) + }) +}) diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md index 417fab33104a..54bec58abe43 100644 --- a/docs/content/features/openai-realtime.md +++ b/docs/content/features/openai-realtime.md @@ -266,16 +266,26 @@ Audio is sent and received as raw PCM in the WebSocket messages, following the O ### WebRTC -The WebRTC transport enables browser-based voice conversations with lower latency. Connect by POSTing an SDP offer to the REST endpoint: +The WebRTC transport enables browser-based voice conversations with lower latency. OpenAI-compatible clients can send a raw SDP offer and select the model with the query parameter: ``` -POST http://localhost:8080/v1/realtime?model=gpt-realtime +POST http://localhost:8080/v1/realtime/calls?model=gpt-realtime Content-Type: application/sdp ``` -The response contains the SDP answer to complete the WebRTC handshake. +The response has the `application/sdp` content type and contains the bare SDP answer. + +The unified OpenAI interface is also supported. Send `multipart/form-data` with an `sdp` field that contains the offer and a JSON `session` field. LocalAI reads the model from the session object: + +```bash +curl http://localhost:8080/v1/realtime/calls \ + -F "sdp=