From 97f6fc0fbde40d24c0e825934cb39a60b4d0801a Mon Sep 17 00:00:00 2001 From: octo-patch <266937838+octo-patch@users.noreply.github.com> Date: Wed, 29 Jul 2026 07:59:06 +0000 Subject: [PATCH] feat(client): add video_url content part for multimodal video input --- internal/client/types.go | 12 +++++ internal/client/types_test.go | 89 +++++++++++++++++++++++++++++++++++ internal/orchestrator/base.go | 22 +++++++-- internal/tui/update.go | 15 ++++-- 4 files changed, 129 insertions(+), 9 deletions(-) create mode 100644 internal/client/types_test.go diff --git a/internal/client/types.go b/internal/client/types.go index 2514b0f..ba1f051 100644 --- a/internal/client/types.go +++ b/internal/client/types.go @@ -95,20 +95,32 @@ type ContentPartType string const ( ContentPartText ContentPartType = "text" ContentPartImageURL ContentPartType = "image_url" + ContentPartVideoURL ContentPartType = "video_url" ) type ContentPart struct { Type ContentPartType `json:"type"` Text string `json:"text,omitempty"` ImageURL *ImageURL `json:"image_url,omitempty"` + VideoURL *VideoURL `json:"video_url,omitempty"` IsAttachment bool `json:"-"` // Purely for UI filtering } +// ImageURL mirrors the OpenAI-compatible "image_url" content part payload. type ImageURL struct { URL string `json:"url"` Detail string `json:"detail,omitempty"` } +// VideoURL mirrors the OpenAI-compatible "video_url" content part payload, +// enabling video input for multimodal models (e.g. MiniMax-M3) alongside the +// existing image_url handling. The schema matches image_url: a "url" (remote +// URL, data URL, or provider file reference) and an optional "detail" level. +type VideoURL struct { + URL string `json:"url"` + Detail string `json:"detail,omitempty"` +} + type ToolCall struct { Index int `json:"index"` ID string `json:"id"` diff --git a/internal/client/types_test.go b/internal/client/types_test.go new file mode 100644 index 0000000..821d2bb --- /dev/null +++ b/internal/client/types_test.go @@ -0,0 +1,89 @@ +package client + +import ( + "encoding/json" + "testing" +) + +// TestContentPartVideoURLSerialization verifies that a video_url content part +// serializes to the OpenAI-compatible "video_url" schema so multimodal models +// (e.g. MiniMax-M3) can receive video input alongside existing text/image parts. +func TestContentPartVideoURLSerialization(t *testing.T) { + part := ContentPart{ + Type: ContentPartVideoURL, + VideoURL: &VideoURL{ + URL: "https://example.com/clip.mp4", + Detail: "high", + }, + } + raw, err := json.Marshal(part) + if err != nil { + t.Fatalf("marshal video part: %v", err) + } + + var got map[string]json.RawMessage + if err := json.Unmarshal(raw, &got); err != nil { + t.Fatalf("unmarshal video part: %v", err) + } + + if typ, _ := json.Marshal(got["type"]); string(typ) != `"video_url"` { + t.Errorf("expected type \"video_url\", got %s", typ) + } + + var vu VideoURL + if err := json.Unmarshal(got["video_url"], &vu); err != nil { + t.Fatalf("unmarshal video_url field: %v", err) + } + if vu.URL != "https://example.com/clip.mp4" { + t.Errorf("expected video url, got %q", vu.URL) + } + if vu.Detail != "high" { + t.Errorf("expected detail high, got %q", vu.Detail) + } + + // image_url field must not leak into a video part. + if _, ok := got["image_url"]; ok { + t.Errorf("video part must not serialize an image_url field") + } +} + +// TestMessageContentVideoRoundTrip ensures a multimodal message containing text, +// image, and video parts marshals and unmarshals without losing the video part. +func TestMessageContentVideoRoundTrip(t *testing.T) { + content := MessageContent{Parts: []ContentPart{ + {Type: ContentPartText, Text: "describe this clip"}, + {Type: ContentPartImageURL, ImageURL: &ImageURL{URL: "data:image/png;base64,AAA"}}, + {Type: ContentPartVideoURL, VideoURL: &VideoURL{URL: "data:video/mp4;base64,BBB"}}, + }} + + raw, err := json.Marshal(content) + if err != nil { + t.Fatalf("marshal content: %v", err) + } + + var back MessageContent + if err := json.Unmarshal(raw, &back); err != nil { + t.Fatalf("unmarshal content: %v", err) + } + if len(back.Parts) != 3 { + t.Fatalf("expected 3 parts, got %d", len(back.Parts)) + } + if back.Parts[2].Type != ContentPartVideoURL { + t.Errorf("expected third part video_url, got %q", back.Parts[2].Type) + } + if back.Parts[2].VideoURL == nil || back.Parts[2].VideoURL.URL != "data:video/mp4;base64,BBB" { + t.Errorf("video url not preserved on round-trip") + } +} + +// TestTextContentStillString verifies that pure-text content still serializes as +// a plain JSON string, preserving the existing text-only message handling. +func TestTextContentStillString(t *testing.T) { + raw, err := json.Marshal(TextContent("hello")) + if err != nil { + t.Fatalf("marshal text: %v", err) + } + if string(raw) != `"hello"` { + t.Errorf("expected plain string, got %s", raw) + } +} diff --git a/internal/orchestrator/base.go b/internal/orchestrator/base.go index b0062e4..ecde931 100644 --- a/internal/orchestrator/base.go +++ b/internal/orchestrator/base.go @@ -124,8 +124,9 @@ func (o *BaseOrchestrator) Submit(text string, images []string) error { } mimeType := http.DetectContentType(data) - // Only attach to LLM content if it's an image AND the model supports vision - if strings.HasPrefix(mimeType, "image/") && supportsVision { + switch { + // Attach images as image_url content parts when the model supports vision. + case strings.HasPrefix(mimeType, "image/") && supportsVision: encoded := base64.StdEncoding.EncodeToString(data) parts = append(parts, client.ContentPart{ Type: client.ContentPartImageURL, @@ -133,13 +134,26 @@ func (o *BaseOrchestrator) Submit(text string, images []string) error { URL: fmt.Sprintf("data:%s;base64,%s", mimeType, encoded), }, }) - } else { + // Attach videos as video_url content parts, mirroring the image path, + // so multimodal models that accept video input (e.g. MiniMax-M3) can + // receive them through the same OpenAI-compatible message schema. + case strings.HasPrefix(mimeType, "video/") && supportsVision: + encoded := base64.StdEncoding.EncodeToString(data) + parts = append(parts, client.ContentPart{ + Type: client.ContentPartVideoURL, + VideoURL: &client.VideoURL{ + URL: fmt.Sprintf("data:%s;base64,%s", mimeType, encoded), + }, + }) + default: // Treat as text if it looks like text or has a common extension. - // If it's an image but the model doesn't support vision, just include a note. + // If it's an image/video but the model doesn't support vision, just include a note. content := "" isAttachment := true if strings.HasPrefix(mimeType, "image/") { content = fmt.Sprintf("\nAttached Image: %s (Vision not supported by current model)\n", filepath.Base(imgPath)) + } else if strings.HasPrefix(mimeType, "video/") { + content = fmt.Sprintf("\nAttached Video: %s (Vision not supported by current model)\n", filepath.Base(imgPath)) } else { content = fmt.Sprintf("\nFilename: %s\nContent: ```\n%s\n```\n", filepath.Base(imgPath), string(data)) } diff --git a/internal/tui/update.go b/internal/tui/update.go index 5a03c51..6789237 100644 --- a/internal/tui/update.go +++ b/internal/tui/update.go @@ -335,16 +335,21 @@ func (m Model) updateInternal(msg tea.Msg) (Model, tea.Cmd) { return m, fpCmd } - // Content-based validation for image support + // Content-based validation for image/video support data, err := os.ReadFile(file) if err != nil { m.Err = fmt.Errorf("failed to read file: %w", err) } else { mimeType := http.DetectContentType(data) isImage := strings.HasPrefix(mimeType, "image/") - if isImage && !m.Focused.SupportsVision() { + isVideo := strings.HasPrefix(mimeType, "video/") + if (isImage || isVideo) && !m.Focused.SupportsVision() { focusedState := m.GetAgentState(m.Focused.ID()) - focusedState.StatusText = "Images not supported by current model" + statusText := "Images not supported by current model" + if isVideo { + statusText = "Video not supported by current model" + } + focusedState.StatusText = statusText } else { m.AttachedFiles = append(m.AttachedFiles, file) m.ShowFilePicker = false @@ -732,13 +737,13 @@ func (m Model) updateChat(msg tea.Msg) (Model, tea.Cmd) { continue } mimeType := http.DetectContentType(data) - if !strings.HasPrefix(mimeType, "image/") { + if !strings.HasPrefix(mimeType, "image/") && !strings.HasPrefix(mimeType, "video/") { filtered = append(filtered, f) } } if len(filtered) != len(m.AttachedFiles) { m.AttachedFiles = filtered - focusedState.StatusText = "Images dropped: model no longer supports vision" + focusedState.StatusText = "Media dropped: model no longer supports vision" return m, nil } }