Add backend features and extensions architecture doc
This commit is contained in:
698
docs/BACKEND_FEATURES.md
Normal file
698
docs/BACKEND_FEATURES.md
Normal file
@@ -0,0 +1,698 @@
|
||||
# ⚙️ Backend Features & Extensions Architecture
|
||||
|
||||
## Overview
|
||||
This document defines the **server-side architecture** for Chat Switchboard, focusing on persistent storage, multi-model routing, function/tool calling, and backend extension capabilities.
|
||||
|
||||
---
|
||||
|
||||
## 🏗️ Core Backend Features
|
||||
|
||||
### 1. **Multi-Model Router**
|
||||
Route requests to different LLM providers based on model, capabilities, cost, or custom logic.
|
||||
|
||||
#### Architecture
|
||||
```
|
||||
┌─────────────────┐
|
||||
│ Frontend │
|
||||
└────────┬────────┘
|
||||
│ OpenAI-compatible API
|
||||
│
|
||||
┌────────▼────────────────────────────────────────┐
|
||||
│ Chat Switchboard Server │
|
||||
│ ┌──────────────────────────────────────────┐ │
|
||||
│ │ Model Router │ │
|
||||
│ │ - Route selection logic │ │
|
||||
│ │ - Fallback handling │ │
|
||||
│ │ - Load balancing │ │
|
||||
│ └──────────┬───────────────────────────────┘ │
|
||||
│ │ │
|
||||
│ ┌──────────▼───────────────────────────────┐ │
|
||||
│ │ Provider Adapters │ │
|
||||
│ ├──────────────────────────────────────────┤ │
|
||||
│ │ OpenAI │ Anthropic │ Local │ Custom │ │
|
||||
│ └──────┬─────┴─────┬─────┴───┬───┴────┬────┘ │
|
||||
└─────────┼───────────┼─────────┼────────┼───────┘
|
||||
│ │ │ │
|
||||
OpenAI API Claude API Ollama Custom
|
||||
```
|
||||
|
||||
#### Implementation (`server/router/model_router.go`)
|
||||
```go
|
||||
type ModelRouter struct {
|
||||
providers map[string]Provider
|
||||
config *RouterConfig
|
||||
}
|
||||
|
||||
type Provider interface {
|
||||
Name() string
|
||||
Models() []string
|
||||
ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
|
||||
StreamCompletion(ctx context.Context, req *ChatRequest) (<-chan *ChatChunk, error)
|
||||
SupportsTools() bool
|
||||
}
|
||||
|
||||
type RouterConfig struct {
|
||||
DefaultProvider string
|
||||
FallbackChain []string
|
||||
ModelMapping map[string]string // model -> provider
|
||||
LoadBalancing bool
|
||||
CostOptimization bool
|
||||
}
|
||||
|
||||
func (r *ModelRouter) Route(model string, req *ChatRequest) (Provider, error) {
|
||||
// 1. Check explicit mapping
|
||||
if provider, ok := r.config.ModelMapping[model]; ok {
|
||||
return r.providers[provider], nil
|
||||
}
|
||||
|
||||
// 2. Query each provider for model support
|
||||
for _, provider := range r.providers {
|
||||
if provider.SupportsModel(model) {
|
||||
return provider, nil
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Fallback chain
|
||||
if len(r.config.FallbackChain) > 0 {
|
||||
return r.providers[r.config.FallbackChain[0]], nil
|
||||
}
|
||||
|
||||
return nil, ErrModelNotFound
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 2. **Tool/Function Calling System**
|
||||
Enable LLMs to call external functions and tools.
|
||||
|
||||
#### Tool Registry (`server/tools/registry.go`)
|
||||
```go
|
||||
type Tool struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Parameters map[string]interface{} `json:"parameters"`
|
||||
Handler ToolHandler
|
||||
}
|
||||
|
||||
type ToolHandler func(ctx context.Context, args map[string]interface{}) (interface{}, error)
|
||||
|
||||
type ToolRegistry struct {
|
||||
tools map[string]*Tool
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
func (r *ToolRegistry) Register(tool *Tool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.tools[tool.Name] = tool
|
||||
}
|
||||
|
||||
func (r *ToolRegistry) Execute(name string, args map[string]interface{}) (interface{}, error) {
|
||||
r.mu.RLock()
|
||||
tool, ok := r.tools[name]
|
||||
r.mu.RUnlock()
|
||||
|
||||
if !ok {
|
||||
return nil, ErrToolNotFound
|
||||
}
|
||||
|
||||
return tool.Handler(context.Background(), args)
|
||||
}
|
||||
|
||||
func (r *ToolRegistry) GetOpenAIFunctions() []map[string]interface{} {
|
||||
// Convert tools to OpenAI function calling format
|
||||
functions := []map[string]interface{}{}
|
||||
for _, tool := range r.tools {
|
||||
functions = append(functions, map[string]interface{}{
|
||||
"name": tool.Name,
|
||||
"description": tool.Description,
|
||||
"parameters": tool.Parameters,
|
||||
})
|
||||
}
|
||||
return functions
|
||||
}
|
||||
```
|
||||
|
||||
#### Built-in Tools
|
||||
```go
|
||||
// Web Search Tool
|
||||
func init() {
|
||||
toolRegistry.Register(&Tool{
|
||||
Name: "web_search",
|
||||
Description: "Search the web for current information",
|
||||
Parameters: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"query": map[string]interface{}{
|
||||
"type": "string",
|
||||
"description": "Search query",
|
||||
},
|
||||
},
|
||||
"required": []string{"query"},
|
||||
},
|
||||
Handler: func(ctx context.Context, args map[string]interface{}) (interface{}, error) {
|
||||
query := args["query"].(string)
|
||||
// Implement web search (DuckDuckGo, Google, etc.)
|
||||
results := performWebSearch(query)
|
||||
return results, nil
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Code Execution Tool
|
||||
func init() {
|
||||
toolRegistry.Register(&Tool{
|
||||
Name: "execute_code",
|
||||
Description: "Execute code in a sandboxed environment",
|
||||
Parameters: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"language": map[string]string{
|
||||
"type": "string",
|
||||
"enum": []string{"python", "javascript", "bash"},
|
||||
},
|
||||
"code": map[string]string{
|
||||
"type": "string",
|
||||
},
|
||||
},
|
||||
"required": []string{"language", "code"},
|
||||
},
|
||||
Handler: func(ctx context.Context, args map[string]interface{}) (interface{}, error) {
|
||||
lang := args["language"].(string)
|
||||
code := args["code"].(string)
|
||||
|
||||
// Execute in Docker container or isolated runtime
|
||||
result := executeSandboxed(lang, code)
|
||||
return result, nil
|
||||
},
|
||||
})
|
||||
}
|
||||
```
|
||||
|
||||
#### Function Calling Flow
|
||||
```go
|
||||
func (h *ChatHandler) HandleFunctionCalling(req *ChatRequest) (*ChatResponse, error) {
|
||||
// 1. Send initial request with tools
|
||||
req.Tools = h.toolRegistry.GetOpenAIFunctions()
|
||||
resp, err := h.provider.ChatCompletion(req)
|
||||
|
||||
// 2. Check if model wants to call a function
|
||||
if resp.FunctionCall != nil {
|
||||
// 3. Execute the function
|
||||
result, err := h.toolRegistry.Execute(
|
||||
resp.FunctionCall.Name,
|
||||
resp.FunctionCall.Arguments,
|
||||
)
|
||||
|
||||
// 4. Send function result back to model
|
||||
req.Messages = append(req.Messages, Message{
|
||||
Role: "function",
|
||||
Name: resp.FunctionCall.Name,
|
||||
Content: jsonStringify(result),
|
||||
})
|
||||
|
||||
// 5. Get final response
|
||||
return h.provider.ChatCompletion(req)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 3. **Backend Extension System**
|
||||
Plugin architecture for server-side extensions.
|
||||
|
||||
#### Extension Interface (`server/extensions/extension.go`)
|
||||
```go
|
||||
type Extension interface {
|
||||
ID() string
|
||||
Name() string
|
||||
Version() string
|
||||
|
||||
// Lifecycle
|
||||
Initialize(ctx context.Context, config map[string]interface{}) error
|
||||
Shutdown() error
|
||||
|
||||
// Hooks
|
||||
OnChatRequest(ctx context.Context, req *ChatRequest) error
|
||||
OnChatResponse(ctx context.Context, resp *ChatResponse) error
|
||||
OnToolCall(ctx context.Context, tool string, args map[string]interface{}) error
|
||||
|
||||
// Optional: Provide custom tools
|
||||
RegisterTools() []*Tool
|
||||
|
||||
// Optional: Provide custom routes
|
||||
RegisterRoutes(router *gin.Engine)
|
||||
}
|
||||
|
||||
type BaseExtension struct {
|
||||
id string
|
||||
name string
|
||||
version string
|
||||
}
|
||||
|
||||
func (e *BaseExtension) ID() string { return e.id }
|
||||
func (e *BaseExtension) Name() string { return e.name }
|
||||
func (e *BaseExtension) Version() string { return e.version }
|
||||
|
||||
// Default no-op implementations
|
||||
func (e *BaseExtension) Initialize(ctx context.Context, config map[string]interface{}) error {
|
||||
return nil
|
||||
}
|
||||
func (e *BaseExtension) Shutdown() error { return nil }
|
||||
func (e *BaseExtension) OnChatRequest(ctx context.Context, req *ChatRequest) error {
|
||||
return nil
|
||||
}
|
||||
func (e *BaseExtension) OnChatResponse(ctx context.Context, resp *ChatResponse) error {
|
||||
return nil
|
||||
}
|
||||
func (e *BaseExtension) OnToolCall(ctx context.Context, tool string, args map[string]interface{}) error {
|
||||
return nil
|
||||
}
|
||||
func (e *BaseExtension) RegisterTools() []*Tool { return nil }
|
||||
func (e *BaseExtension) RegisterRoutes(router *gin.Engine) {}
|
||||
```
|
||||
|
||||
#### Extension Manager
|
||||
```go
|
||||
type ExtensionManager struct {
|
||||
extensions map[string]Extension
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
func (m *ExtensionManager) Load(ext Extension, config map[string]interface{}) error {
|
||||
if err := ext.Initialize(context.Background(), config); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
m.extensions[ext.ID()] = ext
|
||||
m.mu.Unlock()
|
||||
|
||||
// Register any tools provided by extension
|
||||
for _, tool := range ext.RegisterTools() {
|
||||
toolRegistry.Register(tool)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ExtensionManager) ExecuteHook(hookName string, args ...interface{}) error {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
for _, ext := range m.extensions {
|
||||
switch hookName {
|
||||
case "OnChatRequest":
|
||||
if err := ext.OnChatRequest(args[0].(context.Context), args[1].(*ChatRequest)); err != nil {
|
||||
return err
|
||||
}
|
||||
case "OnChatResponse":
|
||||
if err := ext.OnChatResponse(args[0].(context.Context), args[1].(*ChatResponse)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 4. **Example Backend Extensions**
|
||||
|
||||
#### A. Logging Extension
|
||||
```go
|
||||
type LoggingExtension struct {
|
||||
BaseExtension
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
func NewLoggingExtension() *LoggingExtension {
|
||||
return &LoggingExtension{
|
||||
BaseExtension: BaseExtension{
|
||||
id: "logging",
|
||||
name: "Request Logging",
|
||||
version: "1.0.0",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (e *LoggingExtension) OnChatRequest(ctx context.Context, req *ChatRequest) error {
|
||||
log.Printf("[CHAT] Model: %s, Messages: %d", req.Model, len(req.Messages))
|
||||
|
||||
// Store in database
|
||||
_, err := e.db.Exec(`
|
||||
INSERT INTO request_logs (user_id, model, message_count, timestamp)
|
||||
VALUES ($1, $2, $3, NOW())
|
||||
`, getUserID(ctx), req.Model, len(req.Messages))
|
||||
|
||||
return err
|
||||
}
|
||||
```
|
||||
|
||||
#### B. Rate Limiting Extension
|
||||
```go
|
||||
type RateLimitExtension struct {
|
||||
BaseExtension
|
||||
limiter *rate.Limiter
|
||||
}
|
||||
|
||||
func (e *RateLimitExtension) OnChatRequest(ctx context.Context, req *ChatRequest) error {
|
||||
if !e.limiter.Allow() {
|
||||
return errors.New("rate limit exceeded")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
#### C. Cost Tracking Extension
|
||||
```go
|
||||
type CostTrackingExtension struct {
|
||||
BaseExtension
|
||||
pricing map[string]float64 // model -> price per 1K tokens
|
||||
}
|
||||
|
||||
func (e *CostTrackingExtension) OnChatResponse(ctx context.Context, resp *ChatResponse) error {
|
||||
model := resp.Model
|
||||
tokens := resp.Usage.TotalTokens
|
||||
|
||||
cost := (float64(tokens) / 1000.0) * e.pricing[model]
|
||||
|
||||
log.Printf("[COST] Model: %s, Tokens: %d, Cost: $%.4f", model, tokens, cost)
|
||||
|
||||
// Store in user account
|
||||
return e.updateUserBalance(getUserID(ctx), cost)
|
||||
}
|
||||
```
|
||||
|
||||
#### D. Content Moderation Extension
|
||||
```go
|
||||
type ModerationExtension struct {
|
||||
BaseExtension
|
||||
moderationAPI string
|
||||
}
|
||||
|
||||
func (e *ModerationExtension) OnChatRequest(ctx context.Context, req *ChatRequest) error {
|
||||
lastMsg := req.Messages[len(req.Messages)-1]
|
||||
|
||||
flagged, err := e.checkModeration(lastMsg.Content)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if flagged {
|
||||
return errors.New("content violates policy")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 5. **Database Schema**
|
||||
|
||||
#### Core Tables (`migrations/001_initial.sql`)
|
||||
```sql
|
||||
-- Users
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
email VARCHAR(255) UNIQUE NOT NULL,
|
||||
password_hash VARCHAR(255) NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- API Configurations (multiple API keys per user)
|
||||
CREATE TABLE api_configs (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id UUID REFERENCES users(id) ON DELETE CASCADE,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
endpoint VARCHAR(255) NOT NULL,
|
||||
api_key_encrypted TEXT NOT NULL,
|
||||
provider VARCHAR(50) NOT NULL, -- 'openai', 'anthropic', 'local', etc.
|
||||
is_default BOOLEAN DEFAULT false,
|
||||
created_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Chats
|
||||
CREATE TABLE chats (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id UUID REFERENCES users(id) ON DELETE CASCADE,
|
||||
title VARCHAR(500),
|
||||
model VARCHAR(100),
|
||||
api_config_id UUID REFERENCES api_configs(id),
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Messages
|
||||
CREATE TABLE messages (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
chat_id UUID REFERENCES chats(id) ON DELETE CASCADE,
|
||||
role VARCHAR(20) NOT NULL, -- 'system', 'user', 'assistant', 'function'
|
||||
content TEXT NOT NULL,
|
||||
name VARCHAR(100), -- for function messages
|
||||
function_call JSONB, -- { "name": "...", "arguments": "..." }
|
||||
metadata JSONB, -- extensible metadata
|
||||
created_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- User Settings
|
||||
CREATE TABLE user_settings (
|
||||
user_id UUID PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
|
||||
settings JSONB NOT NULL DEFAULT '{}',
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Extension Data (for extensions to store persistent data)
|
||||
CREATE TABLE extension_data (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
extension_id VARCHAR(100) NOT NULL,
|
||||
user_id UUID REFERENCES users(id) ON DELETE CASCADE,
|
||||
key VARCHAR(255) NOT NULL,
|
||||
value JSONB NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW(),
|
||||
UNIQUE(extension_id, user_id, key)
|
||||
);
|
||||
|
||||
-- Request Logs (for analytics)
|
||||
CREATE TABLE request_logs (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
user_id UUID REFERENCES users(id) ON DELETE CASCADE,
|
||||
model VARCHAR(100),
|
||||
message_count INT,
|
||||
tokens_used INT,
|
||||
cost DECIMAL(10, 6),
|
||||
duration_ms INT,
|
||||
timestamp TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Indexes
|
||||
CREATE INDEX idx_chats_user_id ON chats(user_id);
|
||||
CREATE INDEX idx_messages_chat_id ON messages(chat_id);
|
||||
CREATE INDEX idx_request_logs_user_id ON request_logs(user_id);
|
||||
CREATE INDEX idx_request_logs_timestamp ON request_logs(timestamp);
|
||||
CREATE INDEX idx_extension_data_lookup ON extension_data(extension_id, user_id, key);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 6. **Provider Adapters**
|
||||
|
||||
#### OpenAI Provider (`server/providers/openai.go`)
|
||||
```go
|
||||
type OpenAIProvider struct {
|
||||
apiKey string
|
||||
endpoint string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (p *OpenAIProvider) ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error) {
|
||||
// Standard OpenAI API implementation
|
||||
}
|
||||
|
||||
func (p *OpenAIProvider) StreamCompletion(ctx context.Context, req *ChatRequest) (<-chan *ChatChunk, error) {
|
||||
// SSE streaming implementation
|
||||
}
|
||||
|
||||
func (p *OpenAIProvider) SupportsTools() bool {
|
||||
return true
|
||||
}
|
||||
```
|
||||
|
||||
#### Anthropic Provider (`server/providers/anthropic.go`)
|
||||
```go
|
||||
type AnthropicProvider struct {
|
||||
apiKey string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func (p *AnthropicProvider) ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error) {
|
||||
// Convert to Anthropic message format
|
||||
anthropicReq := p.convertRequest(req)
|
||||
|
||||
// Call Anthropic API
|
||||
resp := p.callAPI(anthropicReq)
|
||||
|
||||
// Convert back to standard format
|
||||
return p.convertResponse(resp), nil
|
||||
}
|
||||
```
|
||||
|
||||
#### Local/Ollama Provider (`server/providers/ollama.go`)
|
||||
```go
|
||||
type OllamaProvider struct {
|
||||
endpoint string // http://localhost:11434
|
||||
}
|
||||
|
||||
func (p *OllamaProvider) ChatCompletion(ctx context.Context, req *ChatRequest) (*ChatResponse, error) {
|
||||
// Call local Ollama instance
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 7. **API Endpoints**
|
||||
|
||||
#### Unified Chat Endpoint
|
||||
```go
|
||||
// POST /api/v1/chat/completions
|
||||
// OpenAI-compatible endpoint
|
||||
func (h *ChatHandler) CreateCompletion(c *gin.Context) {
|
||||
var req ChatRequest
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// Execute extension hooks
|
||||
if err := extensionManager.ExecuteHook("OnChatRequest", c.Request.Context(), &req); err != nil {
|
||||
c.JSON(400, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// Route to appropriate provider
|
||||
provider, err := modelRouter.Route(req.Model, &req)
|
||||
if err != nil {
|
||||
c.JSON(404, gin.H{"error": "model not found"})
|
||||
return
|
||||
}
|
||||
|
||||
// Handle streaming vs non-streaming
|
||||
if req.Stream {
|
||||
h.streamResponse(c, provider, &req)
|
||||
} else {
|
||||
resp, err := provider.ChatCompletion(c.Request.Context(), &req)
|
||||
if err != nil {
|
||||
c.JSON(500, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// Execute response hooks
|
||||
extensionManager.ExecuteHook("OnChatResponse", c.Request.Context(), resp)
|
||||
|
||||
c.JSON(200, resp)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Tool Execution Endpoint
|
||||
```go
|
||||
// POST /api/v1/tools/execute
|
||||
func (h *ToolHandler) Execute(c *gin.Context) {
|
||||
var req struct {
|
||||
Tool string `json:"tool"`
|
||||
Args map[string]interface{} `json:"args"`
|
||||
}
|
||||
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
result, err := toolRegistry.Execute(req.Tool, req.Args)
|
||||
if err != nil {
|
||||
c.JSON(500, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(200, gin.H{"result": result})
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 8. **Configuration**
|
||||
|
||||
#### Server Config (`server/config.yaml`)
|
||||
```yaml
|
||||
server:
|
||||
port: 8080
|
||||
cors:
|
||||
allowed_origins: ["*"]
|
||||
|
||||
database:
|
||||
host: localhost
|
||||
port: 5432
|
||||
name: chat_switchboard
|
||||
user: postgres
|
||||
password: ${DB_PASSWORD}
|
||||
|
||||
providers:
|
||||
openai:
|
||||
endpoint: https://api.openai.com/v1
|
||||
api_key: ${OPENAI_API_KEY}
|
||||
anthropic:
|
||||
endpoint: https://api.anthropic.com/v1
|
||||
api_key: ${ANTHROPIC_API_KEY}
|
||||
ollama:
|
||||
endpoint: http://localhost:11434
|
||||
|
||||
router:
|
||||
default_provider: openai
|
||||
fallback_chain:
|
||||
- openai
|
||||
- anthropic
|
||||
- ollama
|
||||
model_mapping:
|
||||
gpt-4: openai
|
||||
claude-3-opus: anthropic
|
||||
llama2: ollama
|
||||
|
||||
extensions:
|
||||
enabled:
|
||||
- logging
|
||||
- cost_tracking
|
||||
- rate_limiting
|
||||
logging:
|
||||
level: info
|
||||
cost_tracking:
|
||||
pricing:
|
||||
gpt-4: 0.03
|
||||
gpt-3.5-turbo: 0.002
|
||||
claude-3-opus: 0.015
|
||||
rate_limiting:
|
||||
requests_per_minute: 60
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Next Steps
|
||||
|
||||
1. **Implement provider adapters** (`server/providers/`)
|
||||
2. **Build tool registry** (`server/tools/`)
|
||||
3. **Create extension manager** (`server/extensions/`)
|
||||
4. **Implement database layer** (`server/models/`)
|
||||
5. **Add authentication/JWT** (`server/middleware/`)
|
||||
6. **Build admin panel** for extension management
|
||||
7. **Create Docker setup** for easy deployment
|
||||
|
||||
---
|
||||
|
||||
**This backend architecture gives you a production-ready, infinitely extensible multi-model LLM router with function calling.**
|
||||
Reference in New Issue
Block a user