20 KiB
20 KiB
⚙️ Backend Features & Extensions Architecture
Overview
This document defines the server-side architecture for Chat Switchboard, focusing on persistent storage, multi-model routing, function/tool calling, and backend extension capabilities.
🏗️ Core Backend Features
1. Multi-Model Router
Route requests to different LLM providers based on model, capabilities, cost, or custom logic.
Architecture
┌─────────────────┐
│ Frontend │
└────────┬────────┘
│ OpenAI-compatible API
│
┌────────▼────────────────────────────────────────┐
│ Chat Switchboard Server │
│ ┌──────────────────────────────────────────┐ │
│ │ Model Router │ │
│ │ - Route selection logic │ │
│ │ - Fallback handling │ │
│ │ - Load balancing │ │
│ └──────────┬───────────────────────────────┘ │
│ │ │
│ ┌──────────▼───────────────────────────────┐ │
│ │ Provider Adapters │ │
│ ├──────────────────────────────────────────┤ │
│ │ OpenAI │ Anthropic │ Local │ Custom │ │
│ └──────┬─────┴─────┬─────┴───┬───┴────┬────┘ │
└─────────┼───────────┼─────────┼────────┼───────┘
│ │ │ │
OpenAI API Claude API Ollama Custom
Implementation (server/router/model_router.go)
type ModelRouter struct {
providers map[string]Provider
config *RouterConfig
}
type Provider interface {
Name() string
Models() []string
ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
StreamCompletion(ctx context.Context, req *ChatRequest) (<-chan *ChatChunk, error)
SupportsTools() bool
}
type RouterConfig struct {
DefaultProvider string
FallbackChain []string
ModelMapping map[string]string // model -> provider
LoadBalancing bool
CostOptimization bool
}
func (r *ModelRouter) Route(model string, req *ChatRequest) (Provider, error) {
// 1. Check explicit mapping
if provider, ok := r.config.ModelMapping[model]; ok {
return r.providers[provider], nil
}
// 2. Query each provider for model support
for _, provider := range r.providers {
if provider.SupportsModel(model) {
return provider, nil
}
}
// 3. Fallback chain
if len(r.config.FallbackChain) > 0 {
return r.providers[r.config.FallbackChain[0]], nil
}
return nil, ErrModelNotFound
}
2. Tool/Function Calling System
Enable LLMs to call external functions and tools.
Tool Registry (server/tools/registry.go)
type Tool struct {
Name string `json:"name"`
Description string `json:"description"`
Parameters map[string]interface{} `json:"parameters"`
Handler ToolHandler
}
type ToolHandler func(ctx context.Context, args map[string]interface{}) (interface{}, error)
type ToolRegistry struct {
tools map[string]*Tool
mu sync.RWMutex
}
func (r *ToolRegistry) Register(tool *Tool) {
r.mu.Lock()
defer r.mu.Unlock()
r.tools[tool.Name] = tool
}
func (r *ToolRegistry) Execute(name string, args map[string]interface{}) (interface{}, error) {
r.mu.RLock()
tool, ok := r.tools[name]
r.mu.RUnlock()
if !ok {
return nil, ErrToolNotFound
}
return tool.Handler(context.Background(), args)
}
func (r *ToolRegistry) GetOpenAIFunctions() []map[string]interface{} {
// Convert tools to OpenAI function calling format
functions := []map[string]interface{}{}
for _, tool := range r.tools {
functions = append(functions, map[string]interface{}{
"name": tool.Name,
"description": tool.Description,
"parameters": tool.Parameters,
})
}
return functions
}
Built-in Tools
// Web Search Tool
func init() {
toolRegistry.Register(&Tool{
Name: "web_search",
Description: "Search the web for current information",
Parameters: map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"query": map[string]interface{}{
"type": "string",
"description": "Search query",
},
},
"required": []string{"query"},
},
Handler: func(ctx context.Context, args map[string]interface{}) (interface{}, error) {
query := args["query"].(string)
// Implement web search (DuckDuckGo, Google, etc.)
results := performWebSearch(query)
return results, nil
},
})
}
// Code Execution Tool
func init() {
toolRegistry.Register(&Tool{
Name: "execute_code",
Description: "Execute code in a sandboxed environment",
Parameters: map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
"language": map[string]string{
"type": "string",
"enum": []string{"python", "javascript", "bash"},
},
"code": map[string]string{
"type": "string",
},
},
"required": []string{"language", "code"},
},
Handler: func(ctx context.Context, args map[string]interface{}) (interface{}, error) {
lang := args["language"].(string)
code := args["code"].(string)
// Execute in Docker container or isolated runtime
result := executeSandboxed(lang, code)
return result, nil
},
})
}
Function Calling Flow
func (h *ChatHandler) HandleFunctionCalling(req *ChatRequest) (*ChatResponse, error) {
// 1. Send initial request with tools
req.Tools = h.toolRegistry.GetOpenAIFunctions()
resp, err := h.provider.ChatCompletion(req)
// 2. Check if model wants to call a function
if resp.FunctionCall != nil {
// 3. Execute the function
result, err := h.toolRegistry.Execute(
resp.FunctionCall.Name,
resp.FunctionCall.Arguments,
)
// 4. Send function result back to model
req.Messages = append(req.Messages, Message{
Role: "function",
Name: resp.FunctionCall.Name,
Content: jsonStringify(result),
})
// 5. Get final response
return h.provider.ChatCompletion(req)
}
return resp, nil
}
3. Backend Extension System
Plugin architecture for server-side extensions.
Extension Interface (server/extensions/extension.go)
type Extension interface {
ID() string
Name() string
Version() string
// Lifecycle
Initialize(ctx context.Context, config map[string]interface{}) error
Shutdown() error
// Hooks
OnChatRequest(ctx context.Context, req *ChatRequest) error
OnChatResponse(ctx context.Context, resp *ChatResponse) error
OnToolCall(ctx context.Context, tool string, args map[string]interface{}) error
// Optional: Provide custom tools
RegisterTools() []*Tool
// Optional: Provide custom routes
RegisterRoutes(router *gin.Engine)
}
type BaseExtension struct {
id string
name string
version string
}
func (e *BaseExtension) ID() string { return e.id }
func (e *BaseExtension) Name() string { return e.name }
func (e *BaseExtension) Version() string { return e.version }
// Default no-op implementations
func (e *BaseExtension) Initialize(ctx context.Context, config map[string]interface{}) error {
return nil
}
func (e *BaseExtension) Shutdown() error { return nil }
func (e *BaseExtension) OnChatRequest(ctx context.Context, req *ChatRequest) error {
return nil
}
func (e *BaseExtension) OnChatResponse(ctx context.Context, resp *ChatResponse) error {
return nil
}
func (e *BaseExtension) OnToolCall(ctx context.Context, tool string, args map[string]interface{}) error {
return nil
}
func (e *BaseExtension) RegisterTools() []*Tool { return nil }
func (e *BaseExtension) RegisterRoutes(router *gin.Engine) {}
Extension Manager
type ExtensionManager struct {
extensions map[string]Extension
mu sync.RWMutex
}
func (m *ExtensionManager) Load(ext Extension, config map[string]interface{}) error {
if err := ext.Initialize(context.Background(), config); err != nil {
return err
}
m.mu.Lock()
m.extensions[ext.ID()] = ext
m.mu.Unlock()
// Register any tools provided by extension
for _, tool := range ext.RegisterTools() {
toolRegistry.Register(tool)
}
return nil
}
func (m *ExtensionManager) ExecuteHook(hookName string, args ...interface{}) error {
m.mu.RLock()
defer m.mu.RUnlock()
for _, ext := range m.extensions {
switch hookName {
case "OnChatRequest":
if err := ext.OnChatRequest(args[0].(context.Context), args[1].(*ChatRequest)); err != nil {
return err
}
case "OnChatResponse":
if err := ext.OnChatResponse(args[0].(context.Context), args[1].(*ChatResponse)); err != nil {
return err
}
}
}
return nil
}
4. Example Backend Extensions
A. Logging Extension
type LoggingExtension struct {
BaseExtension
db *sql.DB
}
func NewLoggingExtension() *LoggingExtension {
return &LoggingExtension{
BaseExtension: BaseExtension{
id: "logging",
name: "Request Logging",
version: "1.0.0",
},
}
}
func (e *LoggingExtension) OnChatRequest(ctx context.Context, req *ChatRequest) error {
log.Printf("[CHAT] Model: %s, Messages: %d", req.Model, len(req.Messages))
// Store in database
_, err := e.db.Exec(`
INSERT INTO request_logs (user_id, model, message_count, timestamp)
VALUES ($1, $2, $3, NOW())
`, getUserID(ctx), req.Model, len(req.Messages))
return err
}
B. Rate Limiting Extension
type RateLimitExtension struct {
BaseExtension
limiter *rate.Limiter
}
func (e *RateLimitExtension) OnChatRequest(ctx context.Context, req *ChatRequest) error {
if !e.limiter.Allow() {
return errors.New("rate limit exceeded")
}
return nil
}
C. Cost Tracking Extension
type CostTrackingExtension struct {
BaseExtension
pricing map[string]float64 // model -> price per 1K tokens
}
func (e *CostTrackingExtension) OnChatResponse(ctx context.Context, resp *ChatResponse) error {
model := resp.Model
tokens := resp.Usage.TotalTokens
cost := (float64(tokens) / 1000.0) * e.pricing[model]
log.Printf("[COST] Model: %s, Tokens: %d, Cost: $%.4f", model, tokens, cost)
// Store in user account
return e.updateUserBalance(getUserID(ctx), cost)
}
D. Content Moderation Extension
type ModerationExtension struct {
BaseExtension
moderationAPI string
}
func (e *ModerationExtension) OnChatRequest(ctx context.Context, req *ChatRequest) error {
lastMsg := req.Messages[len(req.Messages)-1]
flagged, err := e.checkModeration(lastMsg.Content)
if err != nil {
return err
}
if flagged {
return errors.New("content violates policy")
}
return nil
}
5. Database Schema
Core Tables (migrations/001_initial.sql)
-- Users
CREATE TABLE users (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
email VARCHAR(255) UNIQUE NOT NULL,
password_hash VARCHAR(255) NOT NULL,
created_at TIMESTAMP DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW()
);
-- API Configurations (multiple API keys per user)
CREATE TABLE api_configs (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id UUID REFERENCES users(id) ON DELETE CASCADE,
name VARCHAR(100) NOT NULL,
endpoint VARCHAR(255) NOT NULL,
api_key_encrypted TEXT NOT NULL,
provider VARCHAR(50) NOT NULL, -- 'openai', 'anthropic', 'local', etc.
is_default BOOLEAN DEFAULT false,
created_at TIMESTAMP DEFAULT NOW()
);
-- Chats
CREATE TABLE chats (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id UUID REFERENCES users(id) ON DELETE CASCADE,
title VARCHAR(500),
model VARCHAR(100),
api_config_id UUID REFERENCES api_configs(id),
created_at TIMESTAMP DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW()
);
-- Messages
CREATE TABLE messages (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
chat_id UUID REFERENCES chats(id) ON DELETE CASCADE,
role VARCHAR(20) NOT NULL, -- 'system', 'user', 'assistant', 'function'
content TEXT NOT NULL,
name VARCHAR(100), -- for function messages
function_call JSONB, -- { "name": "...", "arguments": "..." }
metadata JSONB, -- extensible metadata
created_at TIMESTAMP DEFAULT NOW()
);
-- User Settings
CREATE TABLE user_settings (
user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
settings JSONB NOT NULL DEFAULT '{}',
updated_at TIMESTAMP DEFAULT NOW()
);
-- Extension Data (for extensions to store persistent data)
CREATE TABLE extension_data (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
extension_id VARCHAR(100) NOT NULL,
user_id UUID REFERENCES users(id) ON DELETE CASCADE,
key VARCHAR(255) NOT NULL,
value JSONB NOT NULL,
created_at TIMESTAMP DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW(),
UNIQUE(extension_id, user_id, key)
);
-- Request Logs (for analytics)
CREATE TABLE request_logs (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id UUID REFERENCES users(id) ON DELETE CASCADE,
model VARCHAR(100),
message_count INT,
tokens_used INT,
cost DECIMAL(10, 6),
duration_ms INT,
timestamp TIMESTAMP DEFAULT NOW()
);
-- Indexes
CREATE INDEX idx_chats_user_id ON chats(user_id);
CREATE INDEX idx_messages_chat_id ON messages(chat_id);
CREATE INDEX idx_request_logs_user_id ON request_logs(user_id);
CREATE INDEX idx_request_logs_timestamp ON request_logs(timestamp);
CREATE INDEX idx_extension_data_lookup ON extension_data(extension_id, user_id, key);
6. Provider Adapters
OpenAI Provider (server/providers/openai.go)
type OpenAIProvider struct {
apiKey string
endpoint string
client *http.Client
}
func (p *OpenAIProvider) ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error) {
// Standard OpenAI API implementation
}
func (p *OpenAIProvider) StreamCompletion(ctx context.Context, req *ChatRequest) (<-chan *ChatChunk, error) {
// SSE streaming implementation
}
func (p *OpenAIProvider) SupportsTools() bool {
return true
}
Anthropic Provider (server/providers/anthropic.go)
type AnthropicProvider struct {
apiKey string
client *http.Client
}
func (p *AnthropicProvider) ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error) {
// Convert to Anthropic message format
anthropicReq := p.convertRequest(req)
// Call Anthropic API
resp := p.callAPI(anthropicReq)
// Convert back to standard format
return p.convertResponse(resp), nil
}
Local/Ollama Provider (server/providers/ollama.go)
type OllamaProvider struct {
endpoint string // http://localhost:11434
}
func (p *OllamaProvider) ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error) {
// Call local Ollama instance
}
7. API Endpoints
Unified Chat Endpoint
// POST /api/v1/chat/completions
// OpenAI-compatible endpoint
func (h *ChatHandler) CreateCompletion(c *gin.Context) {
var req ChatRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(400, gin.H{"error": err.Error()})
return
}
// Execute extension hooks
if err := extensionManager.ExecuteHook("OnChatRequest", c.Request.Context(), &req); err != nil {
c.JSON(400, gin.H{"error": err.Error()})
return
}
// Route to appropriate provider
provider, err := modelRouter.Route(req.Model, &req)
if err != nil {
c.JSON(404, gin.H{"error": "model not found"})
return
}
// Handle streaming vs non-streaming
if req.Stream {
h.streamResponse(c, provider, &req)
} else {
resp, err := provider.ChatCompletion(c.Request.Context(), &req)
if err != nil {
c.JSON(500, gin.H{"error": err.Error()})
return
}
// Execute response hooks
extensionManager.ExecuteHook("OnChatResponse", c.Request.Context(), resp)
c.JSON(200, resp)
}
}
Tool Execution Endpoint
// POST /api/v1/tools/execute
func (h *ToolHandler) Execute(c *gin.Context) {
var req struct {
Tool string `json:"tool"`
Args map[string]interface{} `json:"args"`
}
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(400, gin.H{"error": err.Error()})
return
}
result, err := toolRegistry.Execute(req.Tool, req.Args)
if err != nil {
c.JSON(500, gin.H{"error": err.Error()})
return
}
c.JSON(200, gin.H{"result": result})
}
8. Configuration
Server Config (server/config.yaml)
server:
port: 8080
cors:
allowed_origins: ["*"]
database:
host: localhost
port: 5432
name: chat_switchboard
user: postgres
password: ${DB_PASSWORD}
providers:
openai:
endpoint: https://api.openai.com/v1
api_key: ${OPENAI_API_KEY}
anthropic:
endpoint: https://api.anthropic.com/v1
api_key: ${ANTHROPIC_API_KEY}
ollama:
endpoint: http://localhost:11434
router:
default_provider: openai
fallback_chain:
- openai
- anthropic
- ollama
model_mapping:
gpt-4: openai
claude-3-opus: anthropic
llama2: ollama
extensions:
enabled:
- logging
- cost_tracking
- rate_limiting
logging:
level: info
cost_tracking:
pricing:
gpt-4: 0.03
gpt-3.5-turbo: 0.002
claude-3-opus: 0.015
rate_limiting:
requests_per_minute: 60
🚀 Next Steps
- Implement provider adapters (
server/providers/) - Build tool registry (
server/tools/) - Create extension manager (
server/extensions/) - Implement database layer (
server/models/) - Add authentication/JWT (
server/middleware/) - Build admin panel for extension management
- Create Docker setup for easy deployment
This backend architecture gives you a production-ready, infinitely extensible multi-model LLM router with function calling.