media: harden ordered image, audio, MP3, and WebP handling

cmd/interactive.go

@@ -18,6 +18,7 @@ import (
 	"github.com/ollama/ollama/api"
 	"github.com/ollama/ollama/envconfig"
 	"github.com/ollama/ollama/internal/modelref"
+	"github.com/ollama/ollama/llm"
 	"github.com/ollama/ollama/readline"
 	"github.com/ollama/ollama/types/errtypes"
 	"github.com/ollama/ollama/types/model"
@@ -47,7 +48,7 @@ func generateInteractive(cmd *cobra.Command, opts runOptions) error {
 		fmt.Fprintln(os.Stderr, "Use \"\"\" to begin a multi-line message.")
 
 		if opts.MultiModal {
-			fmt.Fprintf(os.Stderr, "Use %s to include .jpg, .png, .webp images, or .wav audio files.\n", filepath.FromSlash("/path/to/file"))
+			fmt.Fprintf(os.Stderr, "Use %s to include .jpg, .jpeg, .png, .webp images, or .wav and .mp3 audio files.\n", filepath.FromSlash("/path/to/file"))
 		}
 
 		fmt.Fprintln(os.Stderr, "")
@@ -610,7 +611,7 @@ func extractFileNames(input string) []string {
 	// Regex to match file paths starting with optional drive letter, / ./ \ or .\ and include escaped or unescaped spaces (\ or %20)
 	// and followed by more characters and a file extension
 	// This will capture non filename strings, but we'll check for file existence to remove mismatches
-	regexPattern := `(?:[a-zA-Z]:)?(?:\./|/|\\)[\S\\ ]+?\.(?i:jpg|jpeg|png|webp|wav)\b`
+	regexPattern := `(?:[a-zA-Z]:)?(?:\./|/|\\)[\S\\ ]+?\.(?i:jpg|jpeg|png|webp|wav|mp3)\b`
 	re := regexp.MustCompile(regexPattern)
 
 	return re.FindAllString(input, -1)
@@ -631,7 +632,7 @@ func extractFileData(input string) (string, []api.ImageData, error) {
 		}
 		ext := strings.ToLower(filepath.Ext(nfp))
 		switch ext {
-		case ".wav":
+		case ".wav", ".mp3":
 			fmt.Fprintf(os.Stderr, "Added audio '%s'\n", nfp)
 		default:
 			fmt.Fprintf(os.Stderr, "Added image '%s'\n", nfp)
@@ -711,8 +712,9 @@ func getImageData(filePath string) ([]byte, error) {
 	}
 
 	contentType := http.DetectContentType(buf)
-	allowedTypes := []string{"image/jpeg", "image/jpg", "image/png", "image/webp", "audio/wave"}
-	if !slices.Contains(allowedTypes, contentType) {
+	allowedTypes := []string{"image/jpeg", "image/jpg", "image/png", "image/webp", "audio/wave", "audio/mpeg", "audio/mp3"}
+	_, audioOK := llm.AudioFormat(buf)
+	if !audioOK && !slices.Contains(allowedTypes, contentType) {
 		return nil, fmt.Errorf("invalid file type: %s", contentType)
 	}
 

cmd/interactive_test.go

@@ -147,3 +147,19 @@ func TestExtractFileDataWAV(t *testing.T) {
 	assert.Len(t, imgs, 1)
 	assert.Equal(t, "before  after", cleaned)
 }
+
+func TestExtractFileDataMP3(t *testing.T) {
+	dir := t.TempDir()
+	for _, name := range []string{"sample.mp3", "upper.MP3"} {
+		fp := filepath.Join(dir, name)
+		data := make([]byte, 600)
+		copy(data, []byte{0xff, 0xfb, 0xd4, 0xc4})
+		if err := os.WriteFile(fp, data, 0o600); err != nil {
+			t.Fatal(err)
+		}
+		cleaned, media, err := extractFileData("before " + fp + " after")
+		assert.NoError(t, err)
+		assert.Len(t, media, 1)
+		assert.Equal(t, "before  after", cleaned)
+	}
+}

docs/capabilities/vision.mdx

@@ -2,7 +2,7 @@
 title: Vision
 ---
 
-Vision models accept images alongside text so the model can describe, classify, and answer questions about what it sees.
+Vision models accept JPEG, PNG, and WebP images alongside text so the model can describe, classify, and answer questions about what it sees.
 
 ## Quick start
 

docs/cli.mdx

@@ -67,6 +67,14 @@ I'm a basic program that prints the famous "Hello, world!" message to the consol
 ollama run gemma4 "What's in this image? /Users/jmorgan/Desktop/smile.png"
 ```
 
+Media-capable models accept JPEG, PNG, and WebP images and WAV and MP3 audio.
+Multiple file paths can be included in one prompt in the order they should be
+presented to the model.
+
+```
+ollama run gemma4 "Summarize this recording: ./meeting.mp3"
+```
+
 ### Generate embeddings
 
 ```

docs/third-party/miniaudio.md

@@ -0,0 +1,34 @@
+# miniaudio
+
+Ollama vendors miniaudio v0.11.25 for in-process MP3 decoding in native MLX
+media runners. The source is `x/mlxrunner/model/audio/miniaudio.h`, copied from the
+same miniaudio revision present in llama.cpp b9888, pinned by
+`LLAMA_CPP_VERSION` when this copy was introduced.
+
+- Upstream: <https://github.com/mackron/miniaudio>
+- Version: 0.11.25 (2026-03-04)
+- SHA-256: `ac7af4de748b7e26b777f37e01cee313a308a7296a3eb080e2906b320cc55c89`
+- License selection: MIT No Attribution (MIT-0)
+
+The vendored header contains the complete upstream license statements. Ollama
+uses the MIT-0 option reproduced below:
+
+> Copyright 2026 David Reid
+>
+> Permission is hereby granted, free of charge, to any person obtaining a copy
+> of this software and associated documentation files (the "Software"), to deal
+> in the Software without restriction, including without limitation the rights
+> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+> copies of the Software, and to permit persons to whom the Software is
+> furnished to do so.
+>
+> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+> SOFTWARE.
+
+The shared audio package uses miniaudio only for MP3. Its existing Go WAV
+decoder remains independent so its pinned processor outputs do not change.

integration/audio_test.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -28,6 +29,50 @@ func decodeTestAudio(t *testing.T) api.ImageData {
 	return data
 }
 
+func silentTestAudio(t *testing.T) api.ImageData {
+	t.Helper()
+	const (
+		sampleRate = 16_000
+		samples    = sampleRate / 2
+	)
+	dataSize := samples * 2
+	var out bytes.Buffer
+	for _, value := range []any{
+		[]byte("RIFF"), uint32(36 + dataSize), []byte("WAVE"),
+		[]byte("fmt "), uint32(16), uint16(1), uint16(1),
+		uint32(sampleRate), uint32(sampleRate * 2), uint16(2), uint16(16),
+		[]byte("data"), uint32(dataSize), make([]byte, dataSize),
+	} {
+		if err := binary.Write(&out, binary.LittleEndian, value); err != nil {
+			t.Fatalf("encode silent WAV: %v", err)
+		}
+	}
+	return out.Bytes()
+}
+
+func requireResponseContains(t *testing.T, response string, words ...string) {
+	t.Helper()
+	lower := strings.ToLower(response)
+	for _, word := range words {
+		if strings.Contains(lower, word) {
+			return
+		}
+	}
+	t.Fatalf("none of %v found in %q", words, response)
+}
+
+func requireOrderedImageResponse(t *testing.T, response string, firstWords, secondWords []string) {
+	t.Helper()
+	lower := strings.ToLower(response)
+	first := strings.Index(lower, "first:")
+	second := strings.Index(lower, "second:")
+	if first < 0 || second < 0 || first >= second {
+		t.Fatalf("response does not contain ordered FIRST:/SECOND: sections: %q", response)
+	}
+	requireResponseContains(t, lower[first:second], firstWords...)
+	requireResponseContains(t, lower[second:], secondWords...)
+}
+
 // setupAudioModel pulls the model, preloads it, and skips if it doesn't support audio.
 func setupAudioModel(ctx context.Context, t *testing.T, client *api.Client, model string) {
 	t.Helper()
@@ -250,3 +295,180 @@ func runOpenAIChatWithAudio(t *testing.T, models []string) {
 		})
 	}
 }
+
+// TestGemma4MultipleMedia exercises ordered multi-audio, multi-image, mixed
+// image/audio, OpenAI interleaving, and retained-history media through MLX.
+func TestGemma4MultipleMedia(t *testing.T) {
+	models := testModels([]string{"gemma4:e2b"})
+	for _, model := range models {
+		t.Run(model, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+			defer cancel()
+			client, endpoint, cleanup := InitServerConnection(ctx, t)
+			defer cleanup()
+
+			setupAudioModel(ctx, t, client, model)
+			requireCapability(ctx, t, client, model, "vision")
+			speech := decodeTestAudio(t)
+			silence := silentTestAudio(t)
+			abbeyRoad, docs, _ := decodeTestImages(t)
+			noThink := &api.ThinkValue{Value: false}
+
+			for _, tc := range []struct {
+				name  string
+				media []api.ImageData
+			}{
+				{name: "speech_then_silence", media: []api.ImageData{speech, silence}},
+				{name: "silence_then_speech", media: []api.ImageData{silence, speech}},
+			} {
+				t.Run(tc.name, func(t *testing.T) {
+					req := api.ChatRequest{
+						Model: model,
+						Think: noThink,
+						Messages: []api.Message{{
+							Role:    "user",
+							Content: "Two audio clips are attached. Transcribe only the clip containing speech.",
+							Images:  tc.media,
+						}},
+						Options: map[string]any{"temperature": 0, "seed": 123, "num_predict": 80},
+					}
+					response := DoChat(ctx, t, client, req, []string{"sky", "blue"}, 90*time.Second, 20*time.Second)
+					requireResponseContains(t, response.Content, "sky", "blue")
+				})
+			}
+
+			for _, tc := range []struct {
+				name        string
+				media       []api.ImageData
+				firstWords  []string
+				secondWords []string
+			}{
+				{
+					name: "abbey_then_docs", media: []api.ImageData{abbeyRoad, docs},
+					firstWords:  []string{"road", "street", "cross", "walk", "beatles"},
+					secondWords: []string{"laptop", "book", "read", "sleep", "documentation", "desk"},
+				},
+				{
+					name: "docs_then_abbey", media: []api.ImageData{docs, abbeyRoad},
+					firstWords:  []string{"laptop", "book", "read", "sleep", "documentation", "desk"},
+					secondWords: []string{"road", "street", "cross", "walk", "beatles"},
+				},
+			} {
+				t.Run(tc.name, func(t *testing.T) {
+					req := api.ChatRequest{
+						Model: model,
+						Think: noThink,
+						Messages: []api.Message{{
+							Role: "user",
+							Content: "Describe both pictures in order. Reply with exactly two labeled lines: " +
+								"FIRST: the first picture. SECOND: the second picture.",
+							Images: tc.media,
+						}},
+						Options: map[string]any{"temperature": 0, "seed": 123, "num_predict": 120},
+					}
+					response := DoChat(ctx, t, client, req, append(tc.firstWords, tc.secondWords...), 120*time.Second, 20*time.Second)
+					requireOrderedImageResponse(t, response.Content, tc.firstWords, tc.secondWords)
+				})
+			}
+
+			t.Run("mixed_same_message", func(t *testing.T) {
+				req := api.ChatRequest{
+					Model: model,
+					Think: noThink,
+					Messages: []api.Message{{
+						Role:    "user",
+						Content: "First [img] is a picture and second [img] is audio. Identify the picture subject and transcribe the spoken question.",
+						Images:  []api.ImageData{docs, speech},
+					}},
+					Options: map[string]any{"temperature": 0, "seed": 123, "num_predict": 120},
+				}
+				response := DoChat(ctx, t, client, req, []string{"llama", "alpaca", "sky", "blue"}, 120*time.Second, 20*time.Second)
+				requireResponseContains(t, response.Content, "llama", "alpaca", "animal", "cartoon", "bear", "character")
+				requireResponseContains(t, response.Content, "sky", "blue")
+			})
+
+			t.Run("openai_mixed_same_message", func(t *testing.T) {
+				body, err := json.Marshal(map[string]any{
+					"model": model,
+					"messages": []any{map[string]any{
+						"role": "user",
+						"content": []any{
+							map[string]any{"type": "text", "text": "First "},
+							map[string]any{"type": "image_url", "image_url": map[string]any{
+								"url": "data:image/png;base64," + base64.StdEncoding.EncodeToString(docs),
+							}},
+							map[string]any{"type": "text", "text": " is a picture. Second "},
+							map[string]any{"type": "input_audio", "input_audio": map[string]any{
+								"data": base64.StdEncoding.EncodeToString(speech), "format": "wav",
+							}},
+							map[string]any{"type": "text", "text": " is audio. Identify the picture subject and transcribe the spoken question."},
+						},
+					}},
+					"temperature":      0,
+					"seed":             123,
+					"max_tokens":       200,
+					"reasoning_effort": "none",
+				})
+				if err != nil {
+					t.Fatal(err)
+				}
+				req, err := http.NewRequestWithContext(ctx, http.MethodPost,
+					fmt.Sprintf("http://%s/v1/chat/completions", endpoint), bytes.NewReader(body))
+				if err != nil {
+					t.Fatal(err)
+				}
+				req.Header.Set("Content-Type", "application/json")
+				resp, err := http.DefaultClient.Do(req)
+				if err != nil {
+					t.Fatal(err)
+				}
+				defer resp.Body.Close()
+				responseBody, err := io.ReadAll(resp.Body)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if resp.StatusCode != http.StatusOK {
+					t.Fatalf("OpenAI mixed-media request returned %s: %s", resp.Status, responseBody)
+				}
+				var result struct {
+					Choices []struct {
+						Message struct {
+							Content   string `json:"content"`
+							Reasoning string `json:"reasoning"`
+						} `json:"message"`
+					} `json:"choices"`
+				}
+				if err := json.Unmarshal(responseBody, &result); err != nil {
+					t.Fatal(err)
+				}
+				if len(result.Choices) != 1 {
+					t.Fatalf("OpenAI mixed-media choices = %d, want 1", len(result.Choices))
+				}
+				text := result.Choices[0].Message.Content + " " + result.Choices[0].Message.Reasoning
+				requireResponseContains(t, text, "llama", "alpaca", "animal", "cartoon", "bear", "character")
+				requireResponseContains(t, text, "sky", "blue")
+			})
+
+			t.Run("mixed_across_history", func(t *testing.T) {
+				req := api.ChatRequest{
+					Model: model,
+					Think: noThink,
+					Messages: []api.Message{
+						{Role: "user", Content: "Remember this picture.", Images: []api.ImageData{docs}},
+						{Role: "assistant", Content: "I will retain the picture for the next instruction."},
+						{
+							Role: "user",
+							Content: "Use both media inputs. Reply with exactly two labeled lines: " +
+								"AUDIO: the exact spoken question. IMAGE: the picture subject.",
+							Images: []api.ImageData{speech},
+						},
+					},
+					Options: map[string]any{"temperature": 0, "seed": 123, "num_predict": 120},
+				}
+				response := DoChat(ctx, t, client, req, []string{"llama", "alpaca", "sky", "blue"}, 120*time.Second, 20*time.Second)
+				requireResponseContains(t, response.Content, "llama", "alpaca", "animal", "cartoon", "bear", "character")
+				requireResponseContains(t, response.Content, "sky", "blue")
+			})
+		})
+	}
+}