Changeset 0.12.0 (#63)
This commit is contained in:
@@ -3,49 +3,54 @@ package handlers
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
capspkg "git.gobha.me/xcaliber/chat-switchboard/capabilities"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/crypto"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/database"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/events"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/providers"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/crypto"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/storage"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
capspkg "git.gobha.me/xcaliber/chat-switchboard/capabilities"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/tools"
|
||||
)
|
||||
|
||||
// ── Request Types ───────────────────────────
|
||||
|
||||
type completionRequest struct {
|
||||
ChannelID string `json:"channel_id"` // preferred; validated manually below
|
||||
ChatID string `json:"chat_id"` // deprecated alias — maps to channel_id
|
||||
Content string `json:"content" binding:"required"`
|
||||
Model string `json:"model,omitempty"`
|
||||
PresetID string `json:"preset_id,omitempty"` // if set, unwraps preset → base model + config
|
||||
APIConfigID string `json:"provider_config_id,omitempty"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
Stream *bool `json:"stream,omitempty"`
|
||||
ChannelID string `json:"channel_id"` // preferred; validated manually below
|
||||
ChatID string `json:"chat_id"` // deprecated alias — maps to channel_id
|
||||
Content string `json:"content" binding:"required"`
|
||||
Model string `json:"model,omitempty"`
|
||||
PresetID string `json:"preset_id,omitempty"` // if set, unwraps preset → base model + config
|
||||
APIConfigID string `json:"provider_config_id,omitempty"`
|
||||
MaxTokens int `json:"max_tokens,omitempty"`
|
||||
Temperature *float64 `json:"temperature,omitempty"`
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
Stream *bool `json:"stream,omitempty"`
|
||||
AttachmentIDs []string `json:"attachment_ids,omitempty"` // staged attachment UUIDs to include in request
|
||||
}
|
||||
|
||||
// CompletionHandler proxies LLM requests through the backend.
|
||||
type CompletionHandler struct {
|
||||
vault *crypto.KeyResolver
|
||||
stores store.Stores
|
||||
hub *events.Hub // WebSocket hub for browser tool bridge
|
||||
vault *crypto.KeyResolver
|
||||
stores store.Stores
|
||||
hub *events.Hub // WebSocket hub for browser tool bridge
|
||||
objStore storage.ObjectStore // file storage for attachment content (nil = disabled)
|
||||
}
|
||||
|
||||
// NewCompletionHandler creates a new handler.
|
||||
func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub) *CompletionHandler {
|
||||
return &CompletionHandler{vault: vault, stores: stores, hub: hub}
|
||||
func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub, objStore storage.ObjectStore) *CompletionHandler {
|
||||
return &CompletionHandler{vault: vault, stores: stores, hub: hub, objStore: objStore}
|
||||
}
|
||||
|
||||
// ── Chat Completion ─────────────────────────
|
||||
@@ -138,26 +143,53 @@ func (h *CompletionHandler) Complete(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// Add the new user message
|
||||
messages = append(messages, providers.Message{
|
||||
// Resolve capabilities early — needed for vision gating below
|
||||
caps := h.getModelCapabilities(c, model, configID)
|
||||
|
||||
// Build user message — multimodal if attachments are present
|
||||
userMsg := providers.Message{
|
||||
Role: "user",
|
||||
Content: req.Content,
|
||||
})
|
||||
}
|
||||
|
||||
// Persist user message
|
||||
if _, err := h.persistMessage(channelID, userID, "user", req.Content, "", 0, 0, nil, nil); err != nil {
|
||||
if len(req.AttachmentIDs) > 0 && h.objStore != nil {
|
||||
parts, augContent, _, err := h.buildMultimodalParts(c, channelID, req.Content, req.AttachmentIDs, caps)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
if len(parts) > 0 {
|
||||
// Multimodal (images present): use ContentParts
|
||||
userMsg.ContentParts = parts
|
||||
} else if augContent != req.Content {
|
||||
// Doc-only: use augmented text content
|
||||
userMsg.Content = augContent
|
||||
}
|
||||
}
|
||||
|
||||
messages = append(messages, userMsg)
|
||||
|
||||
// Persist user message (text-only for storage — multimodal parts are ephemeral)
|
||||
msgID, err := h.persistMessage(channelID, userID, "user", req.Content, "", 0, 0, nil, nil)
|
||||
if err != nil {
|
||||
log.Printf("Failed to persist user message: %v", err)
|
||||
}
|
||||
|
||||
// Link attachments to the persisted message
|
||||
if msgID != "" && len(req.AttachmentIDs) > 0 {
|
||||
for _, attID := range req.AttachmentIDs {
|
||||
if err := h.stores.Attachments.SetMessageID(c.Request.Context(), attID, msgID); err != nil {
|
||||
log.Printf("Failed to link attachment %s to message %s: %v", attID, msgID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build provider request
|
||||
provReq := providers.CompletionRequest{
|
||||
Model: model,
|
||||
Messages: messages,
|
||||
}
|
||||
|
||||
// Resolve capabilities for this model — auto-set defaults
|
||||
caps := h.getModelCapabilities(c, model, configID)
|
||||
|
||||
if req.MaxTokens > 0 {
|
||||
provReq.MaxTokens = req.MaxTokens
|
||||
} else {
|
||||
@@ -405,6 +437,128 @@ func (h *CompletionHandler) getModelCapabilities(c *gin.Context, model, apiConfi
|
||||
return ResolveModelCaps(c, model, apiConfigID)
|
||||
}
|
||||
|
||||
// ── Multimodal Assembly ─────────────────────
|
||||
// Builds content parts from attachments for the user message.
|
||||
//
|
||||
// Returns:
|
||||
// - parts: ContentParts array (non-nil only when images are present)
|
||||
// - augContent: enriched text content with document context (for doc-only case)
|
||||
// - validIDs: attachment IDs that were successfully processed
|
||||
// - error: if any attachment is invalid or vision is needed but missing
|
||||
//
|
||||
// Rules:
|
||||
// - Images → base64 data URI (requires vision capability)
|
||||
// - Documents with extracted_text → text injection
|
||||
// - Documents without extraction → filename placeholder
|
||||
// - Text is always the first part
|
||||
|
||||
func (h *CompletionHandler) buildMultimodalParts(
|
||||
c *gin.Context,
|
||||
channelID, textContent string,
|
||||
attachmentIDs []string,
|
||||
caps models.ModelCapabilities,
|
||||
) ([]providers.ContentPart, string, []string, error) {
|
||||
|
||||
parts := []providers.ContentPart{
|
||||
{Type: "text", Text: textContent},
|
||||
}
|
||||
var docTexts []string
|
||||
var validIDs []string
|
||||
hasImage := false
|
||||
|
||||
for _, attID := range attachmentIDs {
|
||||
att, err := h.stores.Attachments.GetByID(c.Request.Context(), attID)
|
||||
if err != nil {
|
||||
return nil, "", nil, fmt.Errorf("attachment %s not found", attID)
|
||||
}
|
||||
|
||||
// Security: verify attachment belongs to this channel
|
||||
if att.ChannelID != channelID {
|
||||
return nil, "", nil, fmt.Errorf("attachment %s does not belong to this channel", attID)
|
||||
}
|
||||
|
||||
if isImageContentType(att.ContentType) {
|
||||
// Vision gating: reject images if model lacks vision
|
||||
if !caps.Vision {
|
||||
return nil, "", nil, fmt.Errorf("model does not support image input; remove image attachments or choose a vision-capable model")
|
||||
}
|
||||
|
||||
// Read image from storage, base64 encode, build data URI
|
||||
reader, _, _, err := h.objStore.Get(c.Request.Context(), att.StorageKey)
|
||||
if err != nil {
|
||||
log.Printf("Failed to read attachment %s from storage: %v", attID, err)
|
||||
parts = append(parts, providers.ContentPart{
|
||||
Type: "text",
|
||||
Text: fmt.Sprintf("[Image: %s — failed to read from storage]", att.Filename),
|
||||
})
|
||||
validIDs = append(validIDs, attID)
|
||||
continue
|
||||
}
|
||||
data, err := io.ReadAll(reader)
|
||||
reader.Close()
|
||||
if err != nil {
|
||||
log.Printf("Failed to read attachment %s bytes: %v", attID, err)
|
||||
validIDs = append(validIDs, attID)
|
||||
continue
|
||||
}
|
||||
|
||||
b64 := base64.StdEncoding.EncodeToString(data)
|
||||
dataURI := fmt.Sprintf("data:%s;base64,%s", att.ContentType, b64)
|
||||
|
||||
parts = append(parts, providers.ContentPart{
|
||||
Type: "image_url",
|
||||
ImageURL: &providers.ImageURL{
|
||||
URL: dataURI,
|
||||
Detail: "auto",
|
||||
},
|
||||
})
|
||||
hasImage = true
|
||||
|
||||
} else if att.ExtractedText != nil && *att.ExtractedText != "" {
|
||||
// Document with extracted text → inject as context
|
||||
docText := fmt.Sprintf("[Document: %s]\n%s", att.Filename, *att.ExtractedText)
|
||||
parts = append(parts, providers.ContentPart{
|
||||
Type: "text",
|
||||
Text: docText,
|
||||
})
|
||||
docTexts = append(docTexts, docText)
|
||||
|
||||
} else {
|
||||
// Document without extraction (pending, failed, or not extractable)
|
||||
status := "pending"
|
||||
if s, ok := att.Metadata["extraction_status"].(string); ok {
|
||||
status = s
|
||||
}
|
||||
placeholder := fmt.Sprintf("[Attached file: %s (extraction %s)]", att.Filename, status)
|
||||
parts = append(parts, providers.ContentPart{
|
||||
Type: "text",
|
||||
Text: placeholder,
|
||||
})
|
||||
docTexts = append(docTexts, placeholder)
|
||||
}
|
||||
|
||||
validIDs = append(validIDs, attID)
|
||||
}
|
||||
|
||||
// If images present → use ContentParts (multimodal array)
|
||||
if hasImage {
|
||||
return parts, textContent, validIDs, nil
|
||||
}
|
||||
|
||||
// Doc-only: merge document context into a single text string (more efficient)
|
||||
augmented := textContent
|
||||
for _, dt := range docTexts {
|
||||
augmented += "\n\n" + dt
|
||||
}
|
||||
return nil, augmented, validIDs, nil
|
||||
}
|
||||
|
||||
// isImageContentType returns true for MIME types that should be sent as
|
||||
// base64 image content parts rather than text extraction.
|
||||
func isImageContentType(ct string) bool {
|
||||
return strings.HasPrefix(ct, "image/")
|
||||
}
|
||||
|
||||
// ── Config Resolution ───────────────────────
|
||||
// Priority: request.provider_config_id → chat.provider_config_id → user's first active config
|
||||
|
||||
|
||||
Reference in New Issue
Block a user