249 lines
9.7 KiB
Go
249 lines
9.7 KiB
Go
package providers
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"log"
|
|
"net/http"
|
|
"net/url"
|
|
"sync"
|
|
"time"
|
|
|
|
"chat-switchboard/models"
|
|
)
|
|
|
|
// ── Errors ─────────────────────────────────
|
|
|
|
var ErrNotSupported = errors.New("operation not supported by this provider")
|
|
|
|
// ── Provider Interface ──────────────────────
|
|
|
|
// Provider is the contract for LLM API adapters.
|
|
type Provider interface {
|
|
// ID returns the provider identifier (e.g. "openai", "anthropic").
|
|
ID() string
|
|
|
|
// ChatCompletion sends a non-streaming request and returns the full response.
|
|
ChatCompletion(ctx context.Context, cfg ProviderConfig, req CompletionRequest) (*CompletionResponse, error)
|
|
|
|
// StreamCompletion sends a streaming request and returns a channel of events.
|
|
// The channel is closed when the stream ends. Errors are sent as StreamEvents.
|
|
StreamCompletion(ctx context.Context, cfg ProviderConfig, req CompletionRequest) (<-chan StreamEvent, error)
|
|
|
|
// ListModels returns available models from this provider.
|
|
ListModels(ctx context.Context, cfg ProviderConfig) ([]Model, error)
|
|
|
|
// Embed generates embeddings for the given input texts.
|
|
// Returns ErrNotSupported if the provider has no embedding endpoint.
|
|
Embed(ctx context.Context, cfg ProviderConfig, req EmbeddingRequest) (*EmbeddingResponse, error)
|
|
}
|
|
|
|
// ── Configuration ───────────────────────────
|
|
|
|
// ProviderConfig holds credentials and endpoint for a configured provider.
|
|
// Populated from the provider_configs table at call time.
|
|
type ProviderConfig struct {
|
|
Endpoint string
|
|
APIKey string
|
|
CustomHeaders map[string]string // Extra HTTP headers (e.g. OpenRouter HTTP-Referer)
|
|
Settings map[string]interface{} // Provider-specific settings from config JSONB
|
|
ProxyMode string // "system" (default), "direct", "custom"
|
|
ProxyURL string // Only used when ProxyMode == "custom"
|
|
}
|
|
|
|
// ── HTTP Client Pool ───────────────────────
|
|
//
|
|
// Provider HTTP clients are pooled by (ProxyMode, ProxyURL) tuple.
|
|
// Previously Client() created a new http.Transport per call, leaking
|
|
// idle connections and preventing TCP reuse across requests to the same
|
|
// provider. With pooling, a Venice model sync followed by a completion
|
|
// reuses the same TCP connection to api.venice.ai.
|
|
|
|
// clientPool holds shared *http.Client instances keyed by proxy config.
|
|
var clientPool sync.Map // map[string]*http.Client
|
|
|
|
// Client returns an *http.Client configured for this provider's proxy settings.
|
|
// Clients are cached and reused across calls with the same proxy configuration.
|
|
// system = http.ProxyFromEnvironment (Go default), direct = no proxy,
|
|
// custom = explicit proxy URL (http/https/socks5).
|
|
func (c ProviderConfig) Client() *http.Client {
|
|
key := c.ProxyMode + "|" + c.ProxyURL
|
|
if v, ok := clientPool.Load(key); ok {
|
|
return v.(*http.Client)
|
|
}
|
|
|
|
transport := &http.Transport{
|
|
MaxIdleConns: 100,
|
|
MaxIdleConnsPerHost: 10,
|
|
IdleConnTimeout: 90 * time.Second,
|
|
TLSHandshakeTimeout: 10 * time.Second,
|
|
}
|
|
switch c.ProxyMode {
|
|
case "direct":
|
|
transport.Proxy = nil
|
|
case "custom":
|
|
if c.ProxyURL != "" {
|
|
u, err := url.Parse(c.ProxyURL)
|
|
if err != nil {
|
|
// Fall back to system rather than silently going direct —
|
|
// a bad custom URL in a mandatory-proxy environment is a
|
|
// security/audit hole if it silently bypasses.
|
|
log.Printf("[proxy] invalid custom proxy URL %q: %v — falling back to system proxy", c.ProxyURL, err)
|
|
transport.Proxy = http.ProxyFromEnvironment
|
|
} else {
|
|
transport.Proxy = http.ProxyURL(u)
|
|
}
|
|
} else {
|
|
transport.Proxy = http.ProxyFromEnvironment
|
|
}
|
|
default: // "system" or empty
|
|
transport.Proxy = http.ProxyFromEnvironment
|
|
}
|
|
|
|
client := &http.Client{Transport: transport}
|
|
actual, _ := clientPool.LoadOrStore(key, client)
|
|
return actual.(*http.Client)
|
|
}
|
|
|
|
// ── Request / Response Types ────────────────
|
|
|
|
// Message represents a chat message in the conversation.
|
|
type Message struct {
|
|
Role string `json:"role"` // user, assistant, system, tool
|
|
Content string `json:"content"`
|
|
|
|
// Multimodal content parts (v0.12.0+). When set, providers use these
|
|
// instead of Content for the user message. Content is still set as
|
|
// a fallback and for message persistence (extracted text summary).
|
|
ContentParts []ContentPart `json:"content_parts,omitempty"`
|
|
|
|
// Tool calling fields (assistant → tool_calls, tool → tool_call_id)
|
|
ToolCalls []ToolCall `json:"tool_calls,omitempty"` // set when assistant requests tools
|
|
ToolCallID string `json:"tool_call_id,omitempty"` // set for role="tool" result messages
|
|
Name string `json:"name,omitempty"` // speaker name (tool result, or persona attribution v0.23.0)
|
|
}
|
|
|
|
// ContentPart is a single element in a multimodal message.
|
|
// Providers convert these to their native wire format.
|
|
type ContentPart struct {
|
|
Type string `json:"type"` // "text", "image_url", "document"
|
|
|
|
// Text content (type="text")
|
|
Text string `json:"text,omitempty"`
|
|
|
|
// Image content (type="image_url") — base64 data URI
|
|
ImageURL *ImageURL `json:"image_url,omitempty"`
|
|
}
|
|
|
|
// ImageURL holds image data for multimodal requests.
|
|
// URL is a data URI: "data:image/jpeg;base64,..."
|
|
type ImageURL struct {
|
|
URL string `json:"url"`
|
|
Detail string `json:"detail,omitempty"` // "auto", "low", "high"
|
|
}
|
|
|
|
// ToolCall represents a function call requested by the LLM.
|
|
type ToolCall struct {
|
|
ID string `json:"id"`
|
|
Type string `json:"type"` // "function"
|
|
Function FunctionCall `json:"function"`
|
|
}
|
|
|
|
// FunctionCall is the function name and JSON arguments.
|
|
type FunctionCall struct {
|
|
Name string `json:"name"`
|
|
Arguments string `json:"arguments"`
|
|
}
|
|
|
|
// ToolDef describes a tool available to the LLM.
|
|
type ToolDef struct {
|
|
Type string `json:"type"` // "function"
|
|
Function FunctionDef `json:"function"`
|
|
}
|
|
|
|
// FunctionDef is the schema for a tool function.
|
|
type FunctionDef struct {
|
|
Name string `json:"name"`
|
|
Description string `json:"description"`
|
|
Parameters json.RawMessage `json:"parameters"`
|
|
}
|
|
|
|
// CompletionRequest is the normalized request sent to any provider.
|
|
type CompletionRequest struct {
|
|
Model string `json:"model"`
|
|
Messages []Message `json:"messages"`
|
|
MaxTokens int `json:"max_tokens,omitempty"`
|
|
Temperature *float64 `json:"temperature,omitempty"`
|
|
TopP *float64 `json:"top_p,omitempty"`
|
|
Stream bool `json:"stream,omitempty"`
|
|
Tools []ToolDef `json:"tools,omitempty"` // available tools for function calling
|
|
ExtraBody map[string]interface{} `json:"-"` // provider-specific wire fields (v0.22.1)
|
|
}
|
|
|
|
// CompletionResponse is the normalized non-streaming response.
|
|
type CompletionResponse struct {
|
|
Content string `json:"content"`
|
|
Model string `json:"model"`
|
|
FinishReason string `json:"finish_reason"`
|
|
InputTokens int `json:"input_tokens"`
|
|
OutputTokens int `json:"output_tokens"`
|
|
CacheCreationTokens int `json:"cache_creation_tokens,omitempty"`
|
|
CacheReadTokens int `json:"cache_read_tokens,omitempty"`
|
|
ToolCalls []ToolCall `json:"tool_calls,omitempty"` // set when finish_reason is "tool_calls"
|
|
}
|
|
|
|
// StreamEvent is a single chunk from a streaming response.
|
|
type StreamEvent struct {
|
|
Delta string `json:"delta,omitempty"`
|
|
Reasoning string `json:"reasoning,omitempty"`
|
|
Done bool `json:"done,omitempty"`
|
|
FinishReason string `json:"finish_reason,omitempty"`
|
|
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
|
|
Model string `json:"model,omitempty"`
|
|
Error error `json:"-"`
|
|
InputTokens int `json:"input_tokens,omitempty"`
|
|
OutputTokens int `json:"output_tokens,omitempty"`
|
|
CacheCreationTokens int `json:"cache_creation_tokens,omitempty"`
|
|
CacheReadTokens int `json:"cache_read_tokens,omitempty"`
|
|
}
|
|
|
|
// ── Embedding Types ────────────────────────
|
|
|
|
// EmbeddingRequest is the normalized embedding request.
|
|
type EmbeddingRequest struct {
|
|
Model string `json:"model"`
|
|
Input []string `json:"input"` // batch of texts to embed
|
|
}
|
|
|
|
// EmbeddingResponse is the normalized embedding response.
|
|
type EmbeddingResponse struct {
|
|
Embeddings [][]float64 `json:"embeddings"`
|
|
Model string `json:"model"`
|
|
InputTokens int `json:"input_tokens"`
|
|
}
|
|
|
|
// Model represents an available model from a provider, with capabilities.
|
|
type Model struct {
|
|
ID string `json:"id"`
|
|
Name string `json:"name,omitempty"`
|
|
Type string `json:"type,omitempty"` // "chat", "embedding", "image" — from provider API
|
|
OwnedBy string `json:"owned_by,omitempty"`
|
|
Capabilities models.ModelCapabilities `json:"capabilities"`
|
|
Pricing *models.ModelPricing `json:"pricing,omitempty"`
|
|
}
|
|
|
|
// mergeExtraBody merges arbitrary key-value pairs into a marshalled JSON object.
|
|
// Used by provider doRequest methods to inject hook-supplied fields (e.g.
|
|
// venice_parameters, thinking config) into the wire-format request body.
|
|
func mergeExtraBody(body []byte, extra map[string]interface{}) ([]byte, error) {
|
|
var base map[string]interface{}
|
|
if err := json.Unmarshal(body, &base); err != nil {
|
|
return nil, err
|
|
}
|
|
for k, v := range extra {
|
|
base[k] = v
|
|
}
|
|
return json.Marshal(base)
|
|
}
|