From ad13c08f920fd9da7306a98e4eb95e4d21e5e2e4 Mon Sep 17 00:00:00 2001 From: xcaliber Date: Tue, 3 Feb 2026 22:40:16 +0000 Subject: [PATCH] Add backend features and extensions architecture doc --- docs/BACKEND_FEATURES.md | 698 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 698 insertions(+) create mode 100644 docs/BACKEND_FEATURES.md diff --git a/docs/BACKEND_FEATURES.md b/docs/BACKEND_FEATURES.md new file mode 100644 index 0000000..d075251 --- /dev/null +++ b/docs/BACKEND_FEATURES.md @@ -0,0 +1,698 @@ +# ⚙️ Backend Features & Extensions Architecture + +## Overview +This document defines the **server-side architecture** for Chat Switchboard, focusing on persistent storage, multi-model routing, function/tool calling, and backend extension capabilities. + +--- + +## 🏗️ Core Backend Features + +### 1. **Multi-Model Router** +Route requests to different LLM providers based on model, capabilities, cost, or custom logic. + +#### Architecture +``` +┌─────────────────┐ +│ Frontend │ +└────────┬────────┘ + │ OpenAI-compatible API + │ +┌────────▼────────────────────────────────────────┐ +│ Chat Switchboard Server │ +│ ┌──────────────────────────────────────────┐ │ +│ │ Model Router │ │ +│ │ - Route selection logic │ │ +│ │ - Fallback handling │ │ +│ │ - Load balancing │ │ +│ └──────────┬───────────────────────────────┘ │ +│ │ │ +│ ┌──────────▼───────────────────────────────┐ │ +│ │ Provider Adapters │ │ +│ ├──────────────────────────────────────────┤ │ +│ │ OpenAI │ Anthropic │ Local │ Custom │ │ +│ └──────┬─────┴─────┬─────┴───┬───┴────┬────┘ │ +└─────────┼───────────┼─────────┼────────┼───────┘ + │ │ │ │ + OpenAI API Claude API Ollama Custom +``` + +#### Implementation (`server/router/model_router.go`) +```go +type ModelRouter struct { + providers map[string]Provider + config *RouterConfig +} + +type Provider interface { + Name() string + Models() []string + ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error) + StreamCompletion(ctx context.Context, req *ChatRequest) (<-chan *ChatChunk, error) + SupportsTools() bool +} + +type RouterConfig struct { + DefaultProvider string + FallbackChain []string + ModelMapping map[string]string // model -> provider + LoadBalancing bool + CostOptimization bool +} + +func (r *ModelRouter) Route(model string, req *ChatRequest) (Provider, error) { + // 1. Check explicit mapping + if provider, ok := r.config.ModelMapping[model]; ok { + return r.providers[provider], nil + } + + // 2. Query each provider for model support + for _, provider := range r.providers { + if provider.SupportsModel(model) { + return provider, nil + } + } + + // 3. Fallback chain + if len(r.config.FallbackChain) > 0 { + return r.providers[r.config.FallbackChain[0]], nil + } + + return nil, ErrModelNotFound +} +``` + +--- + +### 2. **Tool/Function Calling System** +Enable LLMs to call external functions and tools. + +#### Tool Registry (`server/tools/registry.go`) +```go +type Tool struct { + Name string `json:"name"` + Description string `json:"description"` + Parameters map[string]interface{} `json:"parameters"` + Handler ToolHandler +} + +type ToolHandler func(ctx context.Context, args map[string]interface{}) (interface{}, error) + +type ToolRegistry struct { + tools map[string]*Tool + mu sync.RWMutex +} + +func (r *ToolRegistry) Register(tool *Tool) { + r.mu.Lock() + defer r.mu.Unlock() + r.tools[tool.Name] = tool +} + +func (r *ToolRegistry) Execute(name string, args map[string]interface{}) (interface{}, error) { + r.mu.RLock() + tool, ok := r.tools[name] + r.mu.RUnlock() + + if !ok { + return nil, ErrToolNotFound + } + + return tool.Handler(context.Background(), args) +} + +func (r *ToolRegistry) GetOpenAIFunctions() []map[string]interface{} { + // Convert tools to OpenAI function calling format + functions := []map[string]interface{}{} + for _, tool := range r.tools { + functions = append(functions, map[string]interface{}{ + "name": tool.Name, + "description": tool.Description, + "parameters": tool.Parameters, + }) + } + return functions +} +``` + +#### Built-in Tools +```go +// Web Search Tool +func init() { + toolRegistry.Register(&Tool{ + Name: "web_search", + Description: "Search the web for current information", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "query": map[string]interface{}{ + "type": "string", + "description": "Search query", + }, + }, + "required": []string{"query"}, + }, + Handler: func(ctx context.Context, args map[string]interface{}) (interface{}, error) { + query := args["query"].(string) + // Implement web search (DuckDuckGo, Google, etc.) + results := performWebSearch(query) + return results, nil + }, + }) +} + +// Code Execution Tool +func init() { + toolRegistry.Register(&Tool{ + Name: "execute_code", + Description: "Execute code in a sandboxed environment", + Parameters: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "language": map[string]string{ + "type": "string", + "enum": []string{"python", "javascript", "bash"}, + }, + "code": map[string]string{ + "type": "string", + }, + }, + "required": []string{"language", "code"}, + }, + Handler: func(ctx context.Context, args map[string]interface{}) (interface{}, error) { + lang := args["language"].(string) + code := args["code"].(string) + + // Execute in Docker container or isolated runtime + result := executeSandboxed(lang, code) + return result, nil + }, + }) +} +``` + +#### Function Calling Flow +```go +func (h *ChatHandler) HandleFunctionCalling(req *ChatRequest) (*ChatResponse, error) { + // 1. Send initial request with tools + req.Tools = h.toolRegistry.GetOpenAIFunctions() + resp, err := h.provider.ChatCompletion(req) + + // 2. Check if model wants to call a function + if resp.FunctionCall != nil { + // 3. Execute the function + result, err := h.toolRegistry.Execute( + resp.FunctionCall.Name, + resp.FunctionCall.Arguments, + ) + + // 4. Send function result back to model + req.Messages = append(req.Messages, Message{ + Role: "function", + Name: resp.FunctionCall.Name, + Content: jsonStringify(result), + }) + + // 5. Get final response + return h.provider.ChatCompletion(req) + } + + return resp, nil +} +``` + +--- + +### 3. **Backend Extension System** +Plugin architecture for server-side extensions. + +#### Extension Interface (`server/extensions/extension.go`) +```go +type Extension interface { + ID() string + Name() string + Version() string + + // Lifecycle + Initialize(ctx context.Context, config map[string]interface{}) error + Shutdown() error + + // Hooks + OnChatRequest(ctx context.Context, req *ChatRequest) error + OnChatResponse(ctx context.Context, resp *ChatResponse) error + OnToolCall(ctx context.Context, tool string, args map[string]interface{}) error + + // Optional: Provide custom tools + RegisterTools() []*Tool + + // Optional: Provide custom routes + RegisterRoutes(router *gin.Engine) +} + +type BaseExtension struct { + id string + name string + version string +} + +func (e *BaseExtension) ID() string { return e.id } +func (e *BaseExtension) Name() string { return e.name } +func (e *BaseExtension) Version() string { return e.version } + +// Default no-op implementations +func (e *BaseExtension) Initialize(ctx context.Context, config map[string]interface{}) error { + return nil +} +func (e *BaseExtension) Shutdown() error { return nil } +func (e *BaseExtension) OnChatRequest(ctx context.Context, req *ChatRequest) error { + return nil +} +func (e *BaseExtension) OnChatResponse(ctx context.Context, resp *ChatResponse) error { + return nil +} +func (e *BaseExtension) OnToolCall(ctx context.Context, tool string, args map[string]interface{}) error { + return nil +} +func (e *BaseExtension) RegisterTools() []*Tool { return nil } +func (e *BaseExtension) RegisterRoutes(router *gin.Engine) {} +``` + +#### Extension Manager +```go +type ExtensionManager struct { + extensions map[string]Extension + mu sync.RWMutex +} + +func (m *ExtensionManager) Load(ext Extension, config map[string]interface{}) error { + if err := ext.Initialize(context.Background(), config); err != nil { + return err + } + + m.mu.Lock() + m.extensions[ext.ID()] = ext + m.mu.Unlock() + + // Register any tools provided by extension + for _, tool := range ext.RegisterTools() { + toolRegistry.Register(tool) + } + + return nil +} + +func (m *ExtensionManager) ExecuteHook(hookName string, args ...interface{}) error { + m.mu.RLock() + defer m.mu.RUnlock() + + for _, ext := range m.extensions { + switch hookName { + case "OnChatRequest": + if err := ext.OnChatRequest(args[0].(context.Context), args[1].(*ChatRequest)); err != nil { + return err + } + case "OnChatResponse": + if err := ext.OnChatResponse(args[0].(context.Context), args[1].(*ChatResponse)); err != nil { + return err + } + } + } + return nil +} +``` + +--- + +### 4. **Example Backend Extensions** + +#### A. Logging Extension +```go +type LoggingExtension struct { + BaseExtension + db *sql.DB +} + +func NewLoggingExtension() *LoggingExtension { + return &LoggingExtension{ + BaseExtension: BaseExtension{ + id: "logging", + name: "Request Logging", + version: "1.0.0", + }, + } +} + +func (e *LoggingExtension) OnChatRequest(ctx context.Context, req *ChatRequest) error { + log.Printf("[CHAT] Model: %s, Messages: %d", req.Model, len(req.Messages)) + + // Store in database + _, err := e.db.Exec(` + INSERT INTO request_logs (user_id, model, message_count, timestamp) + VALUES ($1, $2, $3, NOW()) + `, getUserID(ctx), req.Model, len(req.Messages)) + + return err +} +``` + +#### B. Rate Limiting Extension +```go +type RateLimitExtension struct { + BaseExtension + limiter *rate.Limiter +} + +func (e *RateLimitExtension) OnChatRequest(ctx context.Context, req *ChatRequest) error { + if !e.limiter.Allow() { + return errors.New("rate limit exceeded") + } + return nil +} +``` + +#### C. Cost Tracking Extension +```go +type CostTrackingExtension struct { + BaseExtension + pricing map[string]float64 // model -> price per 1K tokens +} + +func (e *CostTrackingExtension) OnChatResponse(ctx context.Context, resp *ChatResponse) error { + model := resp.Model + tokens := resp.Usage.TotalTokens + + cost := (float64(tokens) / 1000.0) * e.pricing[model] + + log.Printf("[COST] Model: %s, Tokens: %d, Cost: $%.4f", model, tokens, cost) + + // Store in user account + return e.updateUserBalance(getUserID(ctx), cost) +} +``` + +#### D. Content Moderation Extension +```go +type ModerationExtension struct { + BaseExtension + moderationAPI string +} + +func (e *ModerationExtension) OnChatRequest(ctx context.Context, req *ChatRequest) error { + lastMsg := req.Messages[len(req.Messages)-1] + + flagged, err := e.checkModeration(lastMsg.Content) + if err != nil { + return err + } + + if flagged { + return errors.New("content violates policy") + } + + return nil +} +``` + +--- + +### 5. **Database Schema** + +#### Core Tables (`migrations/001_initial.sql`) +```sql +-- Users +CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + email VARCHAR(255) UNIQUE NOT NULL, + password_hash VARCHAR(255) NOT NULL, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); + +-- API Configurations (multiple API keys per user) +CREATE TABLE api_configs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID REFERENCES users(id) ON DELETE CASCADE, + name VARCHAR(100) NOT NULL, + endpoint VARCHAR(255) NOT NULL, + api_key_encrypted TEXT NOT NULL, + provider VARCHAR(50) NOT NULL, -- 'openai', 'anthropic', 'local', etc. + is_default BOOLEAN DEFAULT false, + created_at TIMESTAMP DEFAULT NOW() +); + +-- Chats +CREATE TABLE chats ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID REFERENCES users(id) ON DELETE CASCADE, + title VARCHAR(500), + model VARCHAR(100), + api_config_id UUID REFERENCES api_configs(id), + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() +); + +-- Messages +CREATE TABLE messages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id UUID REFERENCES chats(id) ON DELETE CASCADE, + role VARCHAR(20) NOT NULL, -- 'system', 'user', 'assistant', 'function' + content TEXT NOT NULL, + name VARCHAR(100), -- for function messages + function_call JSONB, -- { "name": "...", "arguments": "..." } + metadata JSONB, -- extensible metadata + created_at TIMESTAMP DEFAULT NOW() +); + +-- User Settings +CREATE TABLE user_settings ( + user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + settings JSONB NOT NULL DEFAULT '{}', + updated_at TIMESTAMP DEFAULT NOW() +); + +-- Extension Data (for extensions to store persistent data) +CREATE TABLE extension_data ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + extension_id VARCHAR(100) NOT NULL, + user_id UUID REFERENCES users(id) ON DELETE CASCADE, + key VARCHAR(255) NOT NULL, + value JSONB NOT NULL, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW(), + UNIQUE(extension_id, user_id, key) +); + +-- Request Logs (for analytics) +CREATE TABLE request_logs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID REFERENCES users(id) ON DELETE CASCADE, + model VARCHAR(100), + message_count INT, + tokens_used INT, + cost DECIMAL(10, 6), + duration_ms INT, + timestamp TIMESTAMP DEFAULT NOW() +); + +-- Indexes +CREATE INDEX idx_chats_user_id ON chats(user_id); +CREATE INDEX idx_messages_chat_id ON messages(chat_id); +CREATE INDEX idx_request_logs_user_id ON request_logs(user_id); +CREATE INDEX idx_request_logs_timestamp ON request_logs(timestamp); +CREATE INDEX idx_extension_data_lookup ON extension_data(extension_id, user_id, key); +``` + +--- + +### 6. **Provider Adapters** + +#### OpenAI Provider (`server/providers/openai.go`) +```go +type OpenAIProvider struct { + apiKey string + endpoint string + client *http.Client +} + +func (p *OpenAIProvider) ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error) { + // Standard OpenAI API implementation +} + +func (p *OpenAIProvider) StreamCompletion(ctx context.Context, req *ChatRequest) (<-chan *ChatChunk, error) { + // SSE streaming implementation +} + +func (p *OpenAIProvider) SupportsTools() bool { + return true +} +``` + +#### Anthropic Provider (`server/providers/anthropic.go`) +```go +type AnthropicProvider struct { + apiKey string + client *http.Client +} + +func (p *AnthropicProvider) ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error) { + // Convert to Anthropic message format + anthropicReq := p.convertRequest(req) + + // Call Anthropic API + resp := p.callAPI(anthropicReq) + + // Convert back to standard format + return p.convertResponse(resp), nil +} +``` + +#### Local/Ollama Provider (`server/providers/ollama.go`) +```go +type OllamaProvider struct { + endpoint string // http://localhost:11434 +} + +func (p *OllamaProvider) ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error) { + // Call local Ollama instance +} +``` + +--- + +### 7. **API Endpoints** + +#### Unified Chat Endpoint +```go +// POST /api/v1/chat/completions +// OpenAI-compatible endpoint +func (h *ChatHandler) CreateCompletion(c *gin.Context) { + var req ChatRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(400, gin.H{"error": err.Error()}) + return + } + + // Execute extension hooks + if err := extensionManager.ExecuteHook("OnChatRequest", c.Request.Context(), &req); err != nil { + c.JSON(400, gin.H{"error": err.Error()}) + return + } + + // Route to appropriate provider + provider, err := modelRouter.Route(req.Model, &req) + if err != nil { + c.JSON(404, gin.H{"error": "model not found"}) + return + } + + // Handle streaming vs non-streaming + if req.Stream { + h.streamResponse(c, provider, &req) + } else { + resp, err := provider.ChatCompletion(c.Request.Context(), &req) + if err != nil { + c.JSON(500, gin.H{"error": err.Error()}) + return + } + + // Execute response hooks + extensionManager.ExecuteHook("OnChatResponse", c.Request.Context(), resp) + + c.JSON(200, resp) + } +} +``` + +#### Tool Execution Endpoint +```go +// POST /api/v1/tools/execute +func (h *ToolHandler) Execute(c *gin.Context) { + var req struct { + Tool string `json:"tool"` + Args map[string]interface{} `json:"args"` + } + + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(400, gin.H{"error": err.Error()}) + return + } + + result, err := toolRegistry.Execute(req.Tool, req.Args) + if err != nil { + c.JSON(500, gin.H{"error": err.Error()}) + return + } + + c.JSON(200, gin.H{"result": result}) +} +``` + +--- + +### 8. **Configuration** + +#### Server Config (`server/config.yaml`) +```yaml +server: + port: 8080 + cors: + allowed_origins: ["*"] + +database: + host: localhost + port: 5432 + name: chat_switchboard + user: postgres + password: ${DB_PASSWORD} + +providers: + openai: + endpoint: https://api.openai.com/v1 + api_key: ${OPENAI_API_KEY} + anthropic: + endpoint: https://api.anthropic.com/v1 + api_key: ${ANTHROPIC_API_KEY} + ollama: + endpoint: http://localhost:11434 + +router: + default_provider: openai + fallback_chain: + - openai + - anthropic + - ollama + model_mapping: + gpt-4: openai + claude-3-opus: anthropic + llama2: ollama + +extensions: + enabled: + - logging + - cost_tracking + - rate_limiting + logging: + level: info + cost_tracking: + pricing: + gpt-4: 0.03 + gpt-3.5-turbo: 0.002 + claude-3-opus: 0.015 + rate_limiting: + requests_per_minute: 60 +``` + +--- + +## 🚀 Next Steps + +1. **Implement provider adapters** (`server/providers/`) +2. **Build tool registry** (`server/tools/`) +3. **Create extension manager** (`server/extensions/`) +4. **Implement database layer** (`server/models/`) +5. **Add authentication/JWT** (`server/middleware/`) +6. **Build admin panel** for extension management +7. **Create Docker setup** for easy deployment + +--- + +**This backend architecture gives you a production-ready, infinitely extensible multi-model LLM router with function calling.** \ No newline at end of file