gemma4: add image and audio support

cmake/local.cmake

@@ -546,7 +546,8 @@ function(ollama_add_mlx_build name)
         SOURCE_DIR ${CMAKE_SOURCE_DIR}/cmake/mlx
         BINARY_DIR ${_build_dir}
         CONFIGURE_COMMAND ${_configure_command}
-        BUILD_COMMAND ${OLLAMA_NATIVE_BUILD_TOOL_COMMAND}
+        BUILD_COMMAND ${_configure_command}
+            COMMAND ${OLLAMA_NATIVE_BUILD_TOOL_COMMAND}
             ${OLLAMA_NATIVE_CONFIG_ARG}
             ${OLLAMA_NATIVE_BUILD_TARGET_ARG} mlx
             ${OLLAMA_NATIVE_BUILD_TARGET_ARG} mlxc

docs/third-party/mlx-vlm.md

@@ -0,0 +1,44 @@
+# MLX-VLM
+
+Portions of `x/models/gemma4/vision.go` are adapted from the Gemma 4
+implementation in MLX-VLM:
+
+- Repository: https://github.com/Blaizzy/mlx-vlm
+- Revision: `61990c9054f2bc7bb8f32541e3238b4a58fe64e5`
+- Source paths: `mlx_vlm/models/gemma4/gemma4.py`,
+  `mlx_vlm/models/gemma4/vision.py`, and
+  `mlx_vlm/models/gemma4_unified/gemma4_unified.py`
+
+The native Gemma 4 audio encoder in `x/models/gemma4/audio.go` is adapted from
+the same project's Gemma 4 Conformer implementation at revision
+`84f43753380355c0455a2bafb291d4b7cbcf81d1` (MLX-VLM v0.6.5), source path
+`mlx_vlm/models/gemma4/audio.py`. Its architecture and numerical behavior were
+also cross-checked against Hugging Face Transformers revision
+`dff4572dfa4bfa9f00cc8414e4b84877552fefe9`. No MLX-VLM or Transformers runtime
+dependency is included.
+
+The adapted implementation is distributed under the following license.
+
+```text
+MIT License
+
+Copyright © 2025 Prince Canuma
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+```

integration/audio_test.go

@@ -6,18 +6,28 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"io"
 	"mime/multipart"
 	"net/http"
+	"regexp"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/ollama/ollama/api"
 )
 
+var defaultAudioModels = []string{
+	"nemotron3:33b",
+	"gemma4:e2b",
+	"gemma4:e4b",
+}
+
+var catResponsePattern = regexp.MustCompile(`\bcats?\b`)
+
 // decodeTestAudio returns the test audio clip ("Why is the sky blue?", 16kHz mono WAV).
 func decodeTestAudio(t *testing.T) api.ImageData {
 	t.Helper()
@@ -28,6 +38,58 @@ func decodeTestAudio(t *testing.T) api.ImageData {
 	return data
 }
 
+func silentTestAudio(t *testing.T) api.ImageData {
+	t.Helper()
+	const (
+		sampleRate = 16_000
+		samples    = sampleRate / 2
+	)
+	dataSize := samples * 2
+	var out bytes.Buffer
+	for _, value := range []any{
+		[]byte("RIFF"), uint32(36 + dataSize), []byte("WAVE"),
+		[]byte("fmt "), uint32(16), uint16(1), uint16(1),
+		uint32(sampleRate), uint32(sampleRate * 2), uint16(2), uint16(16),
+		[]byte("data"), uint32(dataSize), make([]byte, dataSize),
+	} {
+		if err := binary.Write(&out, binary.LittleEndian, value); err != nil {
+			t.Fatalf("encode silent WAV: %v", err)
+		}
+	}
+	return out.Bytes()
+}
+
+func requireResponseContains(t *testing.T, response string, words ...string) {
+	t.Helper()
+	lower := strings.ToLower(response)
+	for _, word := range words {
+		if word == "cat" {
+			if catResponsePattern.MatchString(lower) {
+				return
+			}
+			continue
+		}
+		if strings.Contains(lower, word) {
+			return
+		}
+	}
+	t.Fatalf("none of %v found in %q", words, response)
+}
+
+func requireLabeledImageOrder(t *testing.T, response string, firstWords, secondWords []string) {
+	t.Helper()
+	lower := strings.ToLower(response)
+	firstAt := strings.Index(lower, "first:")
+	secondAt := strings.Index(lower, "second:")
+	if firstAt < 0 || secondAt <= firstAt {
+		t.Fatalf("response does not contain ordered FIRST:/SECOND: labels: %q", response)
+	}
+	first := response[firstAt+len("first:") : secondAt]
+	second := response[secondAt+len("second:"):]
+	requireResponseContains(t, first, firstWords...)
+	requireResponseContains(t, second, secondWords...)
+}
+
 // setupAudioModel pulls the model, preloads it, and skips if it doesn't support audio.
 func setupAudioModel(ctx context.Context, t *testing.T, client *api.Client, model string) {
 	t.Helper()
@@ -250,3 +312,180 @@ func runOpenAIChatWithAudio(t *testing.T, models []string) {
 		})
 	}
 }
+
+// TestGemma4MultipleMedia exercises ordered multi-audio, multi-image, mixed
+// image/audio, OpenAI interleaving, and retained-history media through MLX.
+func TestGemma4MultipleMedia(t *testing.T) {
+	models := testModels([]string{"gemma4:e2b"})
+	for _, model := range models {
+		t.Run(model, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+			defer cancel()
+			client, endpoint, cleanup := InitServerConnection(ctx, t)
+			defer cleanup()
+
+			setupAudioModel(ctx, t, client, model)
+			requireCapability(ctx, t, client, model, "vision")
+			speech := decodeTestAudio(t)
+			silence := silentTestAudio(t)
+			abbeyRoad, docs, _ := decodeTestImages(t)
+			noThink := &api.ThinkValue{Value: false}
+
+			for _, tc := range []struct {
+				name  string
+				media []api.ImageData
+			}{
+				{name: "speech_then_silence", media: []api.ImageData{speech, silence}},
+				{name: "silence_then_speech", media: []api.ImageData{silence, speech}},
+			} {
+				t.Run(tc.name, func(t *testing.T) {
+					req := api.ChatRequest{
+						Model: model,
+						Think: noThink,
+						Messages: []api.Message{{
+							Role:    "user",
+							Content: "Two audio clips are attached. Transcribe only the clip containing speech.",
+							Images:  tc.media,
+						}},
+						Options: map[string]any{"temperature": 0, "seed": 123, "num_predict": 80},
+					}
+					response := DoChat(ctx, t, client, req, []string{"sky", "blue"}, 90*time.Second, 20*time.Second)
+					requireResponseContains(t, response.Content, "sky", "blue")
+				})
+			}
+
+			for _, tc := range []struct {
+				name        string
+				media       []api.ImageData
+				firstWords  []string
+				secondWords []string
+			}{
+				{
+					name: "abbey_then_docs", media: []api.ImageData{abbeyRoad, docs},
+					firstWords:  []string{"road", "street", "cross", "walk", "beatles", "stripe", "ollamas"},
+					secondWords: []string{"laptop", "book", "read", "sleep", "documentation", "document", "desk", "work", "study", "activity", "office"},
+				},
+				{
+					name: "docs_then_abbey", media: []api.ImageData{docs, abbeyRoad},
+					firstWords:  []string{"laptop", "book", "read", "sleep", "documentation", "desk"},
+					secondWords: []string{"road", "street", "cross", "walk", "beatles", "stripe"},
+				},
+			} {
+				t.Run(tc.name, func(t *testing.T) {
+					req := api.ChatRequest{
+						Model: model,
+						Think: noThink,
+						Messages: []api.Message{{
+							Role: "user",
+							Content: "Describe both pictures in order. Reply with exactly two labeled lines: " +
+								"FIRST: the first picture. SECOND: the second picture.",
+							Images: tc.media,
+						}},
+						Options: map[string]any{"temperature": 0, "seed": 123, "num_predict": 120},
+					}
+					response := DoChat(ctx, t, client, req, append(tc.firstWords, tc.secondWords...), 120*time.Second, 20*time.Second)
+					requireLabeledImageOrder(t, response.Content, tc.firstWords, tc.secondWords)
+				})
+			}
+
+			t.Run("mixed_same_message", func(t *testing.T) {
+				req := api.ChatRequest{
+					Model: model,
+					Think: noThink,
+					Messages: []api.Message{{
+						Role:    "user",
+						Content: "First [img] is a picture and second [img] is audio. Identify the picture subject and transcribe the spoken question.",
+						Images:  []api.ImageData{docs, speech},
+					}},
+					Options: map[string]any{"temperature": 0, "seed": 123, "num_predict": 120},
+				}
+				response := DoChat(ctx, t, client, req, []string{"llama", "alpaca", "sky", "blue"}, 120*time.Second, 20*time.Second)
+				requireResponseContains(t, response.Content, "llama", "alpaca", "animal", "cartoon", "bear", "character", "cat")
+				requireResponseContains(t, response.Content, "sky", "blue")
+			})
+
+			t.Run("openai_mixed_same_message", func(t *testing.T) {
+				body, err := json.Marshal(map[string]any{
+					"model": model,
+					"messages": []any{map[string]any{
+						"role": "user",
+						"content": []any{
+							map[string]any{"type": "text", "text": "First "},
+							map[string]any{"type": "image_url", "image_url": map[string]any{
+								"url": "data:image/png;base64," + base64.StdEncoding.EncodeToString(docs),
+							}},
+							map[string]any{"type": "text", "text": " is a picture. Second "},
+							map[string]any{"type": "input_audio", "input_audio": map[string]any{
+								"data": base64.StdEncoding.EncodeToString(speech), "format": "wav",
+							}},
+							map[string]any{"type": "text", "text": " is audio. Identify the picture subject and transcribe the spoken question."},
+						},
+					}},
+					"temperature":      0,
+					"seed":             123,
+					"max_tokens":       200,
+					"reasoning_effort": "none",
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
+				req, err := http.NewRequestWithContext(ctx, http.MethodPost,
+					fmt.Sprintf("http://%s/v1/chat/completions", endpoint), bytes.NewReader(body))
+				if err != nil {
+					t.Fatal(err)
+				}
+				req.Header.Set("Content-Type", "application/json")
+				resp, err := http.DefaultClient.Do(req)
+				if err != nil {
+					t.Fatal(err)
+				}
+				defer resp.Body.Close()
+				responseBody, err := io.ReadAll(resp.Body)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if resp.StatusCode != http.StatusOK {
+					t.Fatalf("OpenAI mixed-media request returned %s: %s", resp.Status, responseBody)
+				}
+				var result struct {
+					Choices []struct {
+						Message struct {
+							Content   string `json:"content"`
+							Reasoning string `json:"reasoning"`
+						} `json:"message"`
+					} `json:"choices"`
+				}
+				if err := json.Unmarshal(responseBody, &result); err != nil {
+					t.Fatal(err)
+				}
+				if len(result.Choices) != 1 {
+					t.Fatalf("OpenAI mixed-media choices = %d, want 1", len(result.Choices))
+				}
+				text := result.Choices[0].Message.Content + " " + result.Choices[0].Message.Reasoning
+				requireResponseContains(t, text, "llama", "alpaca", "animal", "cartoon", "bear", "character", "cat")
+				requireResponseContains(t, text, "sky", "blue")
+			})
+
+			t.Run("mixed_across_history", func(t *testing.T) {
+				req := api.ChatRequest{
+					Model: model,
+					Think: noThink,
+					Messages: []api.Message{
+						{Role: "user", Content: "Remember this picture.", Images: []api.ImageData{docs}},
+						{Role: "assistant", Content: "I will retain the picture for the next instruction."},
+						{
+							Role: "user",
+							Content: "Use both media inputs. Reply with exactly two labeled lines: " +
+								"AUDIO: the exact spoken question. IMAGE: the picture subject.",
+							Images: []api.ImageData{speech},
+						},
+					},
+					Options: map[string]any{"temperature": 0, "seed": 123, "num_predict": 120},
+				}
+				response := DoChat(ctx, t, client, req, []string{"llama", "alpaca", "sky", "blue"}, 120*time.Second, 20*time.Second)
+				requireResponseContains(t, response.Content, "llama", "alpaca", "animal", "cartoon", "bear", "character", "cat")
+				requireResponseContains(t, response.Content, "sky", "blue")
+			})
+		})
+	}
+}

middleware/openai.go

@@ -1018,7 +1018,7 @@ func (w *WebSearchResponsesWriter) writeWebSearchResponse(final api.ChatResponse
 	response := openai.ToResponse(w.inner.model, w.inner.responseID, w.inner.itemID, final, w.req)
 	completedAt := time.Now().Unix()
 	response.CompletedAt = &completedAt
-	response.Output = buildResponsesWebSearchOutput(response.Output, w.preSearchThinking, w.preSearchContent, calls, w.otherToolCalls)
+	response.Output = buildResponsesWebSearchOutput(w.req, response.Output, w.preSearchThinking, w.preSearchContent, calls, w.otherToolCalls)
 	if response.Usage != nil {
 		response.Usage.InputTokens = usage.PromptEvalCount
 		response.Usage.OutputTokens = usage.EvalCount
@@ -1032,7 +1032,7 @@ func (w *WebSearchResponsesWriter) writeWebSearchResponse(final api.ChatResponse
 // buildResponsesWebSearchOutput assembles the final non-streaming output in
 // model-leg order: pre-search reasoning/text, server and mixed tool calls, then
 // the final model output.
-func buildResponsesWebSearchOutput(output []openai.ResponsesOutputItem, preSearchThinking, preSearchContent string, searchCalls []openai.ResponsesWebSearchCall, otherToolCalls []api.ToolCall) []openai.ResponsesOutputItem {
+func buildResponsesWebSearchOutput(request openai.ResponsesRequest, output []openai.ResponsesOutputItem, preSearchThinking, preSearchContent string, searchCalls []openai.ResponsesWebSearchCall, otherToolCalls []api.ToolCall) []openai.ResponsesOutputItem {
 	items := make([]openai.ResponsesOutputItem, 0, len(output)+len(searchCalls)+len(otherToolCalls)+2)
 	if preSearchThinking != "" {
 		items = append(items, openai.ResponsesOutputItem{
@@ -1061,17 +1061,7 @@ func buildResponsesWebSearchOutput(output []openai.ResponsesOutputItem, preSearc
 		items = append(items, openai.WebSearchCallOutputItem(call))
 	}
 	// function_call items from mixed responses
-	convertedCalls := openai.ToToolCalls(otherToolCalls)
-	for i, tc := range convertedCalls {
-		items = append(items, openai.ResponsesOutputItem{
-			ID:        fmt.Sprintf("fc_mixed_%d", i),
-			Type:      "function_call",
-			Status:    "completed",
-			CallID:    tc.ID,
-			Name:      tc.Function.Name,
-			Arguments: tc.Function.Arguments,
-		})
-	}
+	items = append(items, openai.ResponsesFunctionCallOutputItems(request, "fc_mixed_", otherToolCalls)...)
 	// remaining items (final reasoning, message, or function calls)
 	items = append(items, output...)
 	return items