Changeset 0.12.0 (#63)

server/providers/anthropic.go

@@ -263,6 +263,40 @@ func (p *AnthropicProvider) doRequest(ctx context.Context, cfg ProviderConfig, r
 					Content:   m.Content,
 				},
 			}
+		} else if len(m.ContentParts) > 0 && m.Role == "user" {
+			// Multimodal user message: convert ContentParts to Anthropic blocks
+			for _, p := range m.ContentParts {
+				switch p.Type {
+				case "image_url":
+					if p.ImageURL != nil {
+						// Parse data URI: "data:image/jpeg;base64,{data}"
+						mediaType, b64Data := parseDataURI(p.ImageURL.URL)
+						if b64Data != "" {
+							antMsg.Content = append(antMsg.Content, anthropicContentBlock{
+								Type: "image",
+								Source: &anthropicImageSource{
+									Type:      "base64",
+									MediaType: mediaType,
+									Data:      b64Data,
+								},
+							})
+						}
+					}
+				default: // "text", "document"
+					if p.Text != "" {
+						antMsg.Content = append(antMsg.Content, anthropicContentBlock{
+							Type: "text",
+							Text: p.Text,
+						})
+					}
+				}
+			}
+			if len(antMsg.Content) == 0 {
+				// Fallback: shouldn't happen, but safety net
+				antMsg.Content = []anthropicContentBlock{
+					{Type: "text", Text: m.Content},
+				}
+			}
 		} else {
 			// Regular text message
 			antMsg.Content = []anthropicContentBlock{
@@ -354,6 +388,8 @@ type anthropicContentBlock struct {
 	Type string `json:"type"`
 	// text
 	Text string `json:"text,omitempty"`
+	// image (type="image")
+	Source *anthropicImageSource `json:"source,omitempty"`
 	// tool_use
 	ID    string          `json:"id,omitempty"`
 	Name  string          `json:"name,omitempty"`
@@ -363,6 +399,15 @@ type anthropicContentBlock struct {
 	Content   string `json:"content,omitempty"`
 }
 
+// anthropicImageSource is the Anthropic-native image format.
+// Anthropic uses {"type":"base64","media_type":"image/jpeg","data":"..."}
+// rather than OpenAI's data URI approach.
+type anthropicImageSource struct {
+	Type      string `json:"type"`       // "base64"
+	MediaType string `json:"media_type"` // "image/jpeg", "image/png", etc.
+	Data      string `json:"data"`       // raw base64 (no data: prefix)
+}
+
 type anthropicMessage struct {
 	Role    string                  `json:"role"`
 	Content []anthropicContentBlock `json:"content"`
@@ -425,3 +470,33 @@ type anthropicStreamEvent struct {
 		Name string `json:"name"`
 	} `json:"content_block,omitempty"`
 }
+
+// parseDataURI splits a data URI into media type and base64 data.
+// Input:  "data:image/jpeg;base64,/9j/4AAQ..."
+// Output: "image/jpeg", "/9j/4AAQ..."
+// Returns empty strings if the URI is malformed.
+func parseDataURI(uri string) (mediaType, data string) {
+	// Strip "data:" prefix
+	if !strings.HasPrefix(uri, "data:") {
+		return "", ""
+	}
+	rest := uri[5:]
+
+	// Split on comma (separates header from data)
+	commaIdx := strings.Index(rest, ",")
+	if commaIdx < 0 {
+		return "", ""
+	}
+
+	header := rest[:commaIdx]
+	data = rest[commaIdx+1:]
+
+	// Extract media type from header (before ";base64")
+	if semiIdx := strings.Index(header, ";"); semiIdx >= 0 {
+		mediaType = header[:semiIdx]
+	} else {
+		mediaType = header
+	}
+
+	return mediaType, data
+}

server/providers/multimodal_test.go

@@ -0,0 +1,202 @@
+package providers
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+// ── ContentPart Tests ──────────────────────
+
+func TestContentPart_TextOnly(t *testing.T) {
+	msg := Message{
+		Role:    "user",
+		Content: "hello",
+		ContentParts: []ContentPart{
+			{Type: "text", Text: "hello"},
+		},
+	}
+	if len(msg.ContentParts) != 1 {
+		t.Fatalf("expected 1 part, got %d", len(msg.ContentParts))
+	}
+	if msg.ContentParts[0].Type != "text" {
+		t.Errorf("type = %q, want text", msg.ContentParts[0].Type)
+	}
+}
+
+func TestContentPart_WithImage(t *testing.T) {
+	msg := Message{
+		Role:    "user",
+		Content: "describe this",
+		ContentParts: []ContentPart{
+			{Type: "text", Text: "describe this"},
+			{Type: "image_url", ImageURL: &ImageURL{URL: "data:image/png;base64,abc123", Detail: "auto"}},
+		},
+	}
+	if len(msg.ContentParts) != 2 {
+		t.Fatalf("expected 2 parts, got %d", len(msg.ContentParts))
+	}
+	if msg.ContentParts[1].ImageURL == nil {
+		t.Fatal("expected non-nil ImageURL")
+	}
+	if msg.ContentParts[1].ImageURL.URL != "data:image/png;base64,abc123" {
+		t.Errorf("URL = %q", msg.ContentParts[1].ImageURL.URL)
+	}
+}
+
+// ── OpenAI Multimodal Serialization ────────
+
+func TestOpenAI_TextOnlyContent_String(t *testing.T) {
+	// When Content is a string, it should serialize as a JSON string
+	msg := openaiMessage{
+		Role:    "user",
+		Content: "hello world",
+	}
+	data, err := json.Marshal(msg)
+	if err != nil {
+		t.Fatalf("Marshal: %v", err)
+	}
+
+	var raw map[string]interface{}
+	json.Unmarshal(data, &raw)
+
+	content := raw["content"]
+	if s, ok := content.(string); !ok || s != "hello world" {
+		t.Errorf("content = %v (%T), want string 'hello world'", content, content)
+	}
+}
+
+func TestOpenAI_MultimodalContent_Array(t *testing.T) {
+	// When Content is an array, it should serialize as a JSON array
+	msg := openaiMessage{
+		Role: "user",
+		Content: []openaiContentPart{
+			{Type: "text", Text: "describe this"},
+			{Type: "image_url", ImageURL: &openaiImageURL{URL: "data:image/png;base64,abc", Detail: "auto"}},
+		},
+	}
+	data, err := json.Marshal(msg)
+	if err != nil {
+		t.Fatalf("Marshal: %v", err)
+	}
+
+	var raw map[string]interface{}
+	json.Unmarshal(data, &raw)
+
+	content := raw["content"]
+	arr, ok := content.([]interface{})
+	if !ok {
+		t.Fatalf("content is %T, want array", content)
+	}
+	if len(arr) != 2 {
+		t.Fatalf("content has %d elements, want 2", len(arr))
+	}
+
+	// Verify first part is text
+	part0 := arr[0].(map[string]interface{})
+	if part0["type"] != "text" {
+		t.Errorf("part[0].type = %v, want text", part0["type"])
+	}
+
+	// Verify second part is image_url
+	part1 := arr[1].(map[string]interface{})
+	if part1["type"] != "image_url" {
+		t.Errorf("part[1].type = %v, want image_url", part1["type"])
+	}
+}
+
+// ── Anthropic Image Source ─────────────────
+
+func TestAnthropic_ImageSourceBlock(t *testing.T) {
+	block := anthropicContentBlock{
+		Type: "image",
+		Source: &anthropicImageSource{
+			Type:      "base64",
+			MediaType: "image/jpeg",
+			Data:      "abc123",
+		},
+	}
+	data, err := json.Marshal(block)
+	if err != nil {
+		t.Fatalf("Marshal: %v", err)
+	}
+
+	var raw map[string]interface{}
+	json.Unmarshal(data, &raw)
+
+	if raw["type"] != "image" {
+		t.Errorf("type = %v", raw["type"])
+	}
+	source := raw["source"].(map[string]interface{})
+	if source["type"] != "base64" {
+		t.Errorf("source.type = %v", source["type"])
+	}
+	if source["media_type"] != "image/jpeg" {
+		t.Errorf("source.media_type = %v", source["media_type"])
+	}
+	if source["data"] != "abc123" {
+		t.Errorf("source.data = %v", source["data"])
+	}
+}
+
+func TestAnthropic_TextBlockUnchanged(t *testing.T) {
+	block := anthropicContentBlock{
+		Type: "text",
+		Text: "hello",
+	}
+	data, err := json.Marshal(block)
+	if err != nil {
+		t.Fatalf("Marshal: %v", err)
+	}
+
+	var raw map[string]interface{}
+	json.Unmarshal(data, &raw)
+
+	if raw["type"] != "text" {
+		t.Errorf("type = %v", raw["type"])
+	}
+	if raw["text"] != "hello" {
+		t.Errorf("text = %v", raw["text"])
+	}
+	// Should NOT have source field
+	if _, ok := raw["source"]; ok {
+		t.Error("text block should not have source field")
+	}
+}
+
+// ── parseDataURI ───────────────────────────
+
+func TestParseDataURI_Valid(t *testing.T) {
+	tests := []struct {
+		uri       string
+		wantType  string
+		wantData  string
+	}{
+		{"data:image/jpeg;base64,/9j/4AAQ", "image/jpeg", "/9j/4AAQ"},
+		{"data:image/png;base64,abc123", "image/png", "abc123"},
+		{"data:image/webp;base64,RIFF", "image/webp", "RIFF"},
+	}
+	for _, tc := range tests {
+		mediaType, data := parseDataURI(tc.uri)
+		if mediaType != tc.wantType {
+			t.Errorf("parseDataURI(%q) mediaType = %q, want %q", tc.uri, mediaType, tc.wantType)
+		}
+		if data != tc.wantData {
+			t.Errorf("parseDataURI(%q) data = %q, want %q", tc.uri, data, tc.wantData)
+		}
+	}
+}
+
+func TestParseDataURI_Invalid(t *testing.T) {
+	tests := []string{
+		"",
+		"not-a-data-uri",
+		"data:nocomma",
+		"https://example.com/image.png",
+	}
+	for _, uri := range tests {
+		mediaType, data := parseDataURI(uri)
+		if mediaType != "" || data != "" {
+			t.Errorf("parseDataURI(%q) = (%q, %q), want empty", uri, mediaType, data)
+		}
+	}
+}

server/providers/openai.go

@@ -37,8 +37,14 @@ func (p *OpenAIProvider) ChatCompletion(ctx context.Context, cfg ProviderConfig,
 		return nil, fmt.Errorf("no choices in response")
 	}
 
+	// Extract content string from response (responses are always string, not array)
+	var contentStr string
+	if s, ok := resp.Choices[0].Message.Content.(string); ok {
+		contentStr = s
+	}
+
 	result := &CompletionResponse{
-		Content:      resp.Choices[0].Message.Content,
+		Content:      contentStr,
 		Model:        resp.Model,
 		FinishReason: resp.Choices[0].FinishReason,
 		InputTokens:  resp.Usage.PromptTokens,
@@ -360,11 +366,42 @@ func (p *OpenAIProvider) doRequest(ctx context.Context, cfg ProviderConfig, req
 		})
 	}
 
-	// Convert messages — handle all roles including tool
+	// Convert messages — handle all roles including tool and multimodal
 	for _, m := range req.Messages {
 		oaiMsg := openaiMessage{
-			Role:    m.Role,
-			Content: m.Content,
+			Role: m.Role,
+		}
+
+		// Build content: multimodal parts or plain string
+		if len(m.ContentParts) > 0 && m.Role == "user" {
+			// Multimodal: convert to OpenAI content array format
+			parts := make([]openaiContentPart, 0, len(m.ContentParts))
+			for _, p := range m.ContentParts {
+				switch p.Type {
+				case "image_url":
+					if p.ImageURL != nil {
+						detail := p.ImageURL.Detail
+						if detail == "" {
+							detail = "auto"
+						}
+						parts = append(parts, openaiContentPart{
+							Type: "image_url",
+							ImageURL: &openaiImageURL{
+								URL:    p.ImageURL.URL,
+								Detail: detail,
+							},
+						})
+					}
+				default: // "text", "document" → text
+					parts = append(parts, openaiContentPart{
+						Type: "text",
+						Text: p.Text,
+					})
+				}
+			}
+			oaiMsg.Content = parts
+		} else {
+			oaiMsg.Content = m.Content
 		}
 
 		// Assistant messages with tool calls
@@ -425,11 +462,24 @@ func (p *OpenAIProvider) doRequest(ctx context.Context, cfg ProviderConfig, req
 // ── OpenAI Wire Types ───────────────────────
 
 type openaiMessage struct {
-	Role       string              `json:"role"`
-	Content    string              `json:"content,omitempty"`
-	ToolCalls  []openaiToolCall    `json:"tool_calls,omitempty"`   // assistant requesting tools
-	ToolCallID string              `json:"tool_call_id,omitempty"` // role=tool result
-	Name       string              `json:"name,omitempty"`         // tool name for role=tool
+	Role         string              `json:"role"`
+	Content      interface{}         `json:"content,omitempty"`      // string or []openaiContentPart
+	ToolCalls    []openaiToolCall    `json:"tool_calls,omitempty"`   // assistant requesting tools
+	ToolCallID   string              `json:"tool_call_id,omitempty"` // role=tool result
+	Name         string              `json:"name,omitempty"`         // tool name for role=tool
+}
+
+// openaiContentPart represents a single element in a multimodal content array.
+// OpenAI format: [{"type":"text","text":"..."}, {"type":"image_url","image_url":{"url":"..."}}]
+type openaiContentPart struct {
+	Type     string          `json:"type"`
+	Text     string          `json:"text,omitempty"`
+	ImageURL *openaiImageURL `json:"image_url,omitempty"`
+}
+
+type openaiImageURL struct {
+	URL    string `json:"url"`
+	Detail string `json:"detail,omitempty"`
 }
 
 type openaiToolCallFunction struct {

server/providers/provider.go

@@ -52,12 +52,36 @@ type Message struct {
 	Role    string `json:"role"`    // user, assistant, system, tool
 	Content string `json:"content"`
 
+	// Multimodal content parts (v0.12.0+). When set, providers use these
+	// instead of Content for the user message. Content is still set as
+	// a fallback and for message persistence (extracted text summary).
+	ContentParts []ContentPart `json:"content_parts,omitempty"`
+
 	// Tool calling fields (assistant → tool_calls, tool → tool_call_id)
 	ToolCalls  []ToolCall `json:"tool_calls,omitempty"`   // set when assistant requests tools
 	ToolCallID string     `json:"tool_call_id,omitempty"` // set for role="tool" result messages
 	Name       string     `json:"name,omitempty"`         // tool name for role="tool"
 }
 
+// ContentPart is a single element in a multimodal message.
+// Providers convert these to their native wire format.
+type ContentPart struct {
+	Type string `json:"type"` // "text", "image_url", "document"
+
+	// Text content (type="text")
+	Text string `json:"text,omitempty"`
+
+	// Image content (type="image_url") — base64 data URI
+	ImageURL *ImageURL `json:"image_url,omitempty"`
+}
+
+// ImageURL holds image data for multimodal requests.
+// URL is a data URI: "data:image/jpeg;base64,..."
+type ImageURL struct {
+	URL    string `json:"url"`
+	Detail string `json:"detail,omitempty"` // "auto", "low", "high"
+}
+
 // ToolCall represents a function call requested by the LLM.
 type ToolCall struct {
 	ID       string       `json:"id"`