Changeset 0.22.2 (#96)
This commit is contained in:
40
server/routing/convert.go
Normal file
40
server/routing/convert.go
Normal file
@@ -0,0 +1,40 @@
|
||||
package routing
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
// FromModel converts a DB model to a routing Policy for evaluation.
|
||||
func FromModel(m *models.RoutingPolicy) Policy {
|
||||
p := Policy{
|
||||
ID: m.ID,
|
||||
Name: m.Name,
|
||||
Scope: m.Scope,
|
||||
TeamID: m.TeamID,
|
||||
Priority: m.Priority,
|
||||
Type: PolicyType(m.Type),
|
||||
IsActive: m.IsActive,
|
||||
CreatedAt: m.CreatedAt,
|
||||
UpdatedAt: m.UpdatedAt,
|
||||
}
|
||||
|
||||
// Convert JSONMap → PolicyConfig via JSON round-trip.
|
||||
// This is safe because PolicyConfig fields match the JSONB keys.
|
||||
if m.Config != nil {
|
||||
raw, _ := json.Marshal(m.Config)
|
||||
json.Unmarshal(raw, &p.Config)
|
||||
}
|
||||
|
||||
return p
|
||||
}
|
||||
|
||||
// FromModels converts a slice of DB models to routing Policies.
|
||||
func FromModels(ms []models.RoutingPolicy) []Policy {
|
||||
out := make([]Policy, len(ms))
|
||||
for i := range ms {
|
||||
out[i] = FromModel(&ms[i])
|
||||
}
|
||||
return out
|
||||
}
|
||||
280
server/routing/evaluator.go
Normal file
280
server/routing/evaluator.go
Normal file
@@ -0,0 +1,280 @@
|
||||
package routing
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
// Evaluator processes routing policies to produce a ranked candidate list.
|
||||
type Evaluator struct{}
|
||||
|
||||
// NewEvaluator creates a new routing evaluator.
|
||||
func NewEvaluator() *Evaluator {
|
||||
return &Evaluator{}
|
||||
}
|
||||
|
||||
// Evaluate applies active policies to the available candidates and returns
|
||||
// a ranked list. Candidates are filtered and reordered based on policy rules
|
||||
// and health status. The first candidate in the result is the preferred choice.
|
||||
//
|
||||
// Policies are evaluated in priority order (lower priority number = first).
|
||||
// The first matching policy wins — subsequent policies of the same type are skipped.
|
||||
func (e *Evaluator) Evaluate(ctx *Context, policies []Policy, candidates []Candidate) ([]Candidate, *Decision) {
|
||||
if len(candidates) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Start with all candidates
|
||||
result := make([]Candidate, len(candidates))
|
||||
copy(result, candidates)
|
||||
|
||||
var decision Decision
|
||||
decision.Candidates = len(candidates)
|
||||
|
||||
// Sort policies by priority
|
||||
sorted := make([]Policy, len(policies))
|
||||
copy(sorted, policies)
|
||||
sort.Slice(sorted, func(i, j int) bool {
|
||||
return sorted[i].Priority < sorted[j].Priority
|
||||
})
|
||||
|
||||
// Evaluate policies in priority order
|
||||
appliedTypes := map[PolicyType]bool{}
|
||||
for _, p := range sorted {
|
||||
if !p.IsActive {
|
||||
continue
|
||||
}
|
||||
// Skip if a policy of this type already matched
|
||||
if appliedTypes[p.Type] {
|
||||
continue
|
||||
}
|
||||
// Scope check: global applies to all; team only to matching teams
|
||||
if p.Scope == "team" && p.TeamID != nil {
|
||||
if !contains(ctx.TeamIDs, *p.TeamID) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
matched := false
|
||||
switch p.Type {
|
||||
case PolicyProviderPrefer:
|
||||
result, matched = applyProviderPrefer(result, p.Config)
|
||||
case PolicyTeamRoute:
|
||||
result, matched = applyTeamRoute(result, p.Config)
|
||||
case PolicyCostLimit:
|
||||
result, matched = applyCostLimit(result, p.Config, ctx)
|
||||
case PolicyModelAlias:
|
||||
result, matched = applyModelAlias(result, p.Config, ctx)
|
||||
}
|
||||
|
||||
if matched {
|
||||
appliedTypes[p.Type] = true
|
||||
decision.PolicyID = p.ID
|
||||
decision.PolicyName = p.Name
|
||||
decision.PolicyType = string(p.Type)
|
||||
}
|
||||
}
|
||||
|
||||
// Global health-aware sorting: healthy > degraded > unknown > down.
|
||||
// Also skip "down" providers if any policy requested it, or by default
|
||||
// when there are healthy alternatives.
|
||||
skipDown := anyPolicySkipsDown(sorted)
|
||||
result = sortByHealth(result, ctx.HealthStatus, skipDown)
|
||||
|
||||
if len(result) > 0 {
|
||||
decision.Selected = result[0].ConfigID
|
||||
}
|
||||
|
||||
return result, &decision
|
||||
}
|
||||
|
||||
// ── Policy Implementations ──────────────────
|
||||
|
||||
// applyProviderPrefer reorders candidates to match the preferred provider list.
|
||||
// Providers not in the preferred list are appended after, in original order.
|
||||
func applyProviderPrefer(candidates []Candidate, cfg PolicyConfig) ([]Candidate, bool) {
|
||||
if len(cfg.PreferredProviders) == 0 {
|
||||
return candidates, false
|
||||
}
|
||||
|
||||
// Build index: configID → position in preferred list
|
||||
rank := make(map[string]int, len(cfg.PreferredProviders))
|
||||
for i, id := range cfg.PreferredProviders {
|
||||
rank[id] = i
|
||||
}
|
||||
|
||||
preferred := make([]Candidate, 0, len(candidates))
|
||||
rest := make([]Candidate, 0)
|
||||
|
||||
for _, c := range candidates {
|
||||
if _, ok := rank[c.ConfigID]; ok {
|
||||
c.Reason = "preferred provider"
|
||||
preferred = append(preferred, c)
|
||||
} else {
|
||||
c.Reason = "fallback"
|
||||
rest = append(rest, c)
|
||||
}
|
||||
}
|
||||
|
||||
// Sort preferred by their order in the preference list
|
||||
sort.Slice(preferred, func(i, j int) bool {
|
||||
return rank[preferred[i].ConfigID] < rank[preferred[j].ConfigID]
|
||||
})
|
||||
|
||||
return append(preferred, rest...), len(preferred) > 0
|
||||
}
|
||||
|
||||
// applyTeamRoute filters candidates to only allowed providers for this team.
|
||||
func applyTeamRoute(candidates []Candidate, cfg PolicyConfig) ([]Candidate, bool) {
|
||||
if len(cfg.AllowedProviders) == 0 {
|
||||
return candidates, false
|
||||
}
|
||||
|
||||
allowed := make(map[string]bool, len(cfg.AllowedProviders))
|
||||
for _, id := range cfg.AllowedProviders {
|
||||
allowed[id] = true
|
||||
}
|
||||
|
||||
filtered := make([]Candidate, 0, len(candidates))
|
||||
for _, c := range candidates {
|
||||
if allowed[c.ConfigID] {
|
||||
c.Reason = "team-allowed provider"
|
||||
filtered = append(filtered, c)
|
||||
}
|
||||
}
|
||||
|
||||
// Only apply if at least one allowed provider is available
|
||||
if len(filtered) > 0 {
|
||||
return filtered, true
|
||||
}
|
||||
return candidates, false
|
||||
}
|
||||
|
||||
// applyCostLimit removes candidates whose estimated cost exceeds the limit.
|
||||
func applyCostLimit(candidates []Candidate, cfg PolicyConfig, ctx *Context) ([]Candidate, bool) {
|
||||
if cfg.MaxCostUSD == nil || *cfg.MaxCostUSD <= 0 {
|
||||
return candidates, false
|
||||
}
|
||||
|
||||
// For cost estimation we'd need request token count, which isn't
|
||||
// available at routing time. Instead, filter out providers with
|
||||
// pricing above the threshold per-million-tokens as a heuristic.
|
||||
// A 4K-token request at $10/M = $0.04, so if max is $0.10 we'd
|
||||
// filter out providers above ~$25/M. For now, just ensure pricing
|
||||
// exists and tag the decision.
|
||||
maxPerM := *cfg.MaxCostUSD * 1_000_000 / 4096 // rough heuristic: 4K tokens avg
|
||||
|
||||
filtered := make([]Candidate, 0, len(candidates))
|
||||
for _, c := range candidates {
|
||||
key := c.ConfigID + ":" + c.Model
|
||||
if pricing, ok := ctx.Pricing[key]; ok && pricing != nil {
|
||||
if pricing.InputPerM > maxPerM {
|
||||
continue // too expensive
|
||||
}
|
||||
}
|
||||
filtered = append(filtered, c)
|
||||
}
|
||||
|
||||
if len(filtered) > 0 && len(filtered) < len(candidates) {
|
||||
return filtered, true
|
||||
}
|
||||
return candidates, false
|
||||
}
|
||||
|
||||
// applyModelAlias replaces the model in matching candidates.
|
||||
func applyModelAlias(candidates []Candidate, cfg PolicyConfig, ctx *Context) ([]Candidate, bool) {
|
||||
if cfg.AliasFrom == "" || cfg.AliasTo == "" {
|
||||
return candidates, false
|
||||
}
|
||||
|
||||
// Only apply if the requested model matches the alias source
|
||||
if !strings.EqualFold(ctx.Model, cfg.AliasFrom) {
|
||||
return candidates, false
|
||||
}
|
||||
|
||||
// If a specific provider is targeted, reorder to prefer it
|
||||
if cfg.AliasProvider != "" {
|
||||
for i, c := range candidates {
|
||||
if c.ConfigID == cfg.AliasProvider {
|
||||
candidates[i].Model = cfg.AliasTo
|
||||
candidates[i].Reason = "model alias: " + cfg.AliasFrom + " → " + cfg.AliasTo
|
||||
// Move to front
|
||||
reordered := make([]Candidate, 0, len(candidates))
|
||||
reordered = append(reordered, candidates[i])
|
||||
reordered = append(reordered, candidates[:i]...)
|
||||
reordered = append(reordered, candidates[i+1:]...)
|
||||
return reordered, true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No specific provider — just rewrite the model on all candidates
|
||||
for i := range candidates {
|
||||
candidates[i].Model = cfg.AliasTo
|
||||
candidates[i].Reason = "model alias: " + cfg.AliasFrom + " → " + cfg.AliasTo
|
||||
}
|
||||
return candidates, true
|
||||
}
|
||||
|
||||
// ── Health Sorting ──────────────────────────
|
||||
|
||||
var statusOrder = map[models.ProviderStatus]int{
|
||||
models.StatusHealthy: 0,
|
||||
models.StatusDegraded: 1,
|
||||
models.StatusUnknown: 2,
|
||||
models.StatusDown: 3,
|
||||
}
|
||||
|
||||
func sortByHealth(candidates []Candidate, health map[string]models.ProviderStatus, skipDown bool) []Candidate {
|
||||
// Annotate status
|
||||
for i, c := range candidates {
|
||||
if s, ok := health[c.ConfigID]; ok {
|
||||
candidates[i].Status = s
|
||||
} else {
|
||||
candidates[i].Status = models.StatusUnknown
|
||||
}
|
||||
}
|
||||
|
||||
// Filter down if requested and alternatives exist
|
||||
if skipDown {
|
||||
healthy := make([]Candidate, 0, len(candidates))
|
||||
for _, c := range candidates {
|
||||
if c.Status != models.StatusDown {
|
||||
healthy = append(healthy, c)
|
||||
}
|
||||
}
|
||||
if len(healthy) > 0 {
|
||||
candidates = healthy
|
||||
}
|
||||
// If all are down, keep them — better to try than fail immediately.
|
||||
}
|
||||
|
||||
// Stable sort by health status
|
||||
sort.SliceStable(candidates, func(i, j int) bool {
|
||||
return statusOrder[candidates[i].Status] < statusOrder[candidates[j].Status]
|
||||
})
|
||||
|
||||
return candidates
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────
|
||||
|
||||
func contains(slice []string, item string) bool {
|
||||
for _, s := range slice {
|
||||
if s == item {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func anyPolicySkipsDown(policies []Policy) bool {
|
||||
for _, p := range policies {
|
||||
if p.IsActive && p.Config.SkipDown {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
294
server/routing/evaluator_test.go
Normal file
294
server/routing/evaluator_test.go
Normal file
@@ -0,0 +1,294 @@
|
||||
package routing
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
func strPtr(s string) *string { return &s }
|
||||
func f64Ptr(f float64) *float64 { return &f }
|
||||
|
||||
func baseCandidates() []Candidate {
|
||||
return []Candidate{
|
||||
{ConfigID: "cfg-openai", ProviderID: "openai", Model: "gpt-4o"},
|
||||
{ConfigID: "cfg-anthropic", ProviderID: "anthropic", Model: "claude-sonnet-4-20250514"},
|
||||
{ConfigID: "cfg-venice", ProviderID: "venice", Model: "llama-3.1-405b"},
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_NoPolices(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{UserID: "u1", Model: "gpt-4o", HealthStatus: map[string]models.ProviderStatus{}}
|
||||
result, decision := e.Evaluate(ctx, nil, baseCandidates())
|
||||
|
||||
if len(result) != 3 {
|
||||
t.Fatalf("expected 3 candidates, got %d", len(result))
|
||||
}
|
||||
if decision.Selected != "cfg-openai" {
|
||||
t.Errorf("expected first candidate selected, got %s", decision.Selected)
|
||||
}
|
||||
if decision.FallbackDepth != 0 {
|
||||
t.Errorf("expected fallback depth 0, got %d", decision.FallbackDepth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_ProviderPrefer(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{UserID: "u1", Model: "gpt-4o", HealthStatus: map[string]models.ProviderStatus{}}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "Prefer Anthropic", Priority: 1,
|
||||
Type: PolicyProviderPrefer, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{
|
||||
PreferredProviders: []string{"cfg-anthropic", "cfg-venice"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, decision := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
if result[0].ConfigID != "cfg-anthropic" {
|
||||
t.Errorf("expected anthropic first, got %s", result[0].ConfigID)
|
||||
}
|
||||
if result[1].ConfigID != "cfg-venice" {
|
||||
t.Errorf("expected venice second, got %s", result[1].ConfigID)
|
||||
}
|
||||
// openai should still be present as fallback
|
||||
if result[2].ConfigID != "cfg-openai" {
|
||||
t.Errorf("expected openai last, got %s", result[2].ConfigID)
|
||||
}
|
||||
if decision.PolicyID != "p1" {
|
||||
t.Errorf("expected policy p1, got %s", decision.PolicyID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_TeamRoute(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
teamID := "team-1"
|
||||
ctx := &Context{
|
||||
UserID: "u1",
|
||||
TeamIDs: []string{"team-1"},
|
||||
Model: "gpt-4o",
|
||||
HealthStatus: map[string]models.ProviderStatus{},
|
||||
}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "Team uses Anthropic only", Priority: 1,
|
||||
Type: PolicyTeamRoute, IsActive: true, Scope: "team", TeamID: &teamID,
|
||||
Config: PolicyConfig{
|
||||
AllowedProviders: []string{"cfg-anthropic"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
if len(result) != 1 {
|
||||
t.Fatalf("expected 1 candidate after team filter, got %d", len(result))
|
||||
}
|
||||
if result[0].ConfigID != "cfg-anthropic" {
|
||||
t.Errorf("expected anthropic only, got %s", result[0].ConfigID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_TeamRoute_WrongTeam(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
teamID := "team-2"
|
||||
ctx := &Context{
|
||||
UserID: "u1",
|
||||
TeamIDs: []string{"team-1"}, // user is NOT in team-2
|
||||
Model: "gpt-4o",
|
||||
HealthStatus: map[string]models.ProviderStatus{},
|
||||
}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "Team 2 only", Priority: 1,
|
||||
Type: PolicyTeamRoute, IsActive: true, Scope: "team", TeamID: &teamID,
|
||||
Config: PolicyConfig{AllowedProviders: []string{"cfg-anthropic"}},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// Policy should not apply — all 3 candidates remain
|
||||
if len(result) != 3 {
|
||||
t.Fatalf("expected 3 candidates (policy doesn't apply), got %d", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_ModelAlias(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{
|
||||
UserID: "u1", Model: "fast",
|
||||
HealthStatus: map[string]models.ProviderStatus{},
|
||||
}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "fast = haiku", Priority: 1,
|
||||
Type: PolicyModelAlias, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{
|
||||
AliasFrom: "fast",
|
||||
AliasTo: "claude-haiku-4-20250414",
|
||||
AliasProvider: "cfg-anthropic",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
if result[0].ConfigID != "cfg-anthropic" {
|
||||
t.Errorf("expected anthropic first for alias, got %s", result[0].ConfigID)
|
||||
}
|
||||
if result[0].Model != "claude-haiku-4-20250414" {
|
||||
t.Errorf("expected model rewritten to haiku, got %s", result[0].Model)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_HealthSkipDown(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{
|
||||
UserID: "u1", Model: "gpt-4o",
|
||||
HealthStatus: map[string]models.ProviderStatus{
|
||||
"cfg-openai": models.StatusDown,
|
||||
"cfg-anthropic": models.StatusHealthy,
|
||||
"cfg-venice": models.StatusDegraded,
|
||||
},
|
||||
}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "Skip down", Priority: 1,
|
||||
Type: PolicyProviderPrefer, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{
|
||||
PreferredProviders: []string{"cfg-openai", "cfg-anthropic"},
|
||||
SkipDown: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// openai is down and should be skipped
|
||||
if result[0].ConfigID != "cfg-anthropic" {
|
||||
t.Errorf("expected anthropic first (openai is down), got %s", result[0].ConfigID)
|
||||
}
|
||||
// openai should not be in results at all
|
||||
for _, c := range result {
|
||||
if c.ConfigID == "cfg-openai" {
|
||||
t.Error("down provider should be skipped")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_HealthSorting(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{
|
||||
UserID: "u1", Model: "gpt-4o",
|
||||
HealthStatus: map[string]models.ProviderStatus{
|
||||
"cfg-openai": models.StatusDegraded,
|
||||
"cfg-anthropic": models.StatusHealthy,
|
||||
"cfg-venice": models.StatusUnknown,
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, nil, baseCandidates())
|
||||
|
||||
// Should be sorted: healthy (anthropic) > degraded (openai) > unknown (venice)
|
||||
if result[0].ConfigID != "cfg-anthropic" {
|
||||
t.Errorf("expected healthy provider first, got %s", result[0].ConfigID)
|
||||
}
|
||||
if result[1].ConfigID != "cfg-openai" {
|
||||
t.Errorf("expected degraded provider second, got %s", result[1].ConfigID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_AllDown_KeepsCandidates(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{
|
||||
UserID: "u1", Model: "gpt-4o",
|
||||
HealthStatus: map[string]models.ProviderStatus{
|
||||
"cfg-openai": models.StatusDown,
|
||||
"cfg-anthropic": models.StatusDown,
|
||||
"cfg-venice": models.StatusDown,
|
||||
},
|
||||
}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Priority: 1, Type: PolicyProviderPrefer, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{SkipDown: true},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// When ALL are down, should keep them rather than returning empty
|
||||
if len(result) == 0 {
|
||||
t.Error("should keep all candidates when all are down")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_PriorityOrder(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{
|
||||
UserID: "u1",
|
||||
TeamIDs: []string{"team-1"},
|
||||
Model: "gpt-4o",
|
||||
HealthStatus: map[string]models.ProviderStatus{},
|
||||
}
|
||||
teamID := "team-1"
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p-low", Name: "Low priority prefer", Priority: 10,
|
||||
Type: PolicyProviderPrefer, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{PreferredProviders: []string{"cfg-venice"}},
|
||||
},
|
||||
{
|
||||
ID: "p-high", Name: "High priority team route", Priority: 1,
|
||||
Type: PolicyTeamRoute, IsActive: true, Scope: "team", TeamID: &teamID,
|
||||
Config: PolicyConfig{AllowedProviders: []string{"cfg-anthropic", "cfg-openai"}},
|
||||
},
|
||||
}
|
||||
|
||||
result, decision := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// Team route (priority 1) should filter first, then prefer (priority 10) reorders
|
||||
if decision.PolicyType != string(PolicyTeamRoute) {
|
||||
t.Errorf("expected team_route to be recorded (first match), got %s", decision.PolicyType)
|
||||
}
|
||||
// Venice should be filtered out by team route
|
||||
for _, c := range result {
|
||||
if c.ConfigID == "cfg-venice" {
|
||||
t.Error("venice should be filtered out by team route")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_EmptyCandidates(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{UserID: "u1"}
|
||||
result, decision := e.Evaluate(ctx, nil, nil)
|
||||
if result != nil {
|
||||
t.Error("expected nil result for empty candidates")
|
||||
}
|
||||
if decision != nil {
|
||||
t.Error("expected nil decision for empty candidates")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_InactivePolicy(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{UserID: "u1", Model: "gpt-4o", HealthStatus: map[string]models.ProviderStatus{}}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Priority: 1, Type: PolicyTeamRoute, IsActive: false, Scope: "global",
|
||||
Config: PolicyConfig{AllowedProviders: []string{"cfg-anthropic"}},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// Inactive policy should not filter
|
||||
if len(result) != 3 {
|
||||
t.Fatalf("inactive policy should not filter, got %d candidates", len(result))
|
||||
}
|
||||
}
|
||||
59
server/routing/fallback.go
Normal file
59
server/routing/fallback.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package routing
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
)
|
||||
|
||||
// ── Fallback Runner ─────────────────────────
|
||||
|
||||
// DispatchFunc is called by the fallback runner to attempt a request
|
||||
// against a specific candidate. Returns nil on success, error on failure.
|
||||
// The caller is responsible for setting up SSE headers, streaming, etc.
|
||||
// This is NOT the actual dispatch — it's the "try this candidate" callback.
|
||||
type DispatchFunc func(candidate Candidate) error
|
||||
|
||||
// RunWithFallback tries candidates in order until one succeeds.
|
||||
// Returns the candidate that succeeded, the decision (updated with fallback
|
||||
// depth), and any error if all candidates fail.
|
||||
//
|
||||
// maxRetries controls how many fallback attempts are made beyond the first.
|
||||
// maxRetries=0 means only try the first candidate (no fallback).
|
||||
// maxRetries=-1 means try all candidates.
|
||||
func RunWithFallback(candidates []Candidate, decision *Decision, maxRetries int, dispatch DispatchFunc) (Candidate, *Decision, error) {
|
||||
if len(candidates) == 0 {
|
||||
return Candidate{}, decision, fmt.Errorf("no routing candidates available")
|
||||
}
|
||||
|
||||
// Determine how many candidates to try
|
||||
limit := 1 // just the first
|
||||
if maxRetries < 0 {
|
||||
limit = len(candidates) // try all
|
||||
} else if maxRetries > 0 {
|
||||
limit = 1 + maxRetries
|
||||
if limit > len(candidates) {
|
||||
limit = len(candidates)
|
||||
}
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for i := 0; i < limit; i++ {
|
||||
c := candidates[i]
|
||||
err := dispatch(c)
|
||||
if err == nil {
|
||||
// Success
|
||||
if decision != nil {
|
||||
decision.Selected = c.ConfigID
|
||||
decision.FallbackDepth = i
|
||||
}
|
||||
return c, decision, nil
|
||||
}
|
||||
|
||||
lastErr = err
|
||||
if i < limit-1 {
|
||||
log.Printf("[routing] candidate %s (%s) failed: %v — trying next", c.ConfigID, c.ProviderID, err)
|
||||
}
|
||||
}
|
||||
|
||||
return Candidate{}, decision, fmt.Errorf("all %d routing candidates failed; last error: %w", limit, lastErr)
|
||||
}
|
||||
107
server/routing/types.go
Normal file
107
server/routing/types.go
Normal file
@@ -0,0 +1,107 @@
|
||||
// Package routing implements rules-based request routing for LLM providers.
|
||||
// Policies are evaluated between "what model was requested" and "which
|
||||
// provider config serves it," producing a ranked list of candidates.
|
||||
// The fallback runner dispatches to candidates in order, retrying on failure.
|
||||
package routing
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
// ── Policy ─────────────────────────────────
|
||||
|
||||
// PolicyType identifies the routing strategy.
|
||||
type PolicyType string
|
||||
|
||||
const (
|
||||
// PolicyProviderPrefer routes to a preferred provider, falling back to others.
|
||||
PolicyProviderPrefer PolicyType = "provider_prefer"
|
||||
// PolicyTeamRoute forces a team to use specific providers.
|
||||
PolicyTeamRoute PolicyType = "team_route"
|
||||
// PolicyCostLimit caps per-request cost.
|
||||
PolicyCostLimit PolicyType = "cost_limit"
|
||||
// PolicyModelAlias maps a model alias to a specific provider+model pair.
|
||||
PolicyModelAlias PolicyType = "model_alias"
|
||||
)
|
||||
|
||||
// Policy is one routing rule stored in the routing_policies table.
|
||||
type Policy struct {
|
||||
ID string `json:"id" db:"id"`
|
||||
Name string `json:"name" db:"name"`
|
||||
Scope string `json:"scope" db:"scope"` // "global" or "team"
|
||||
TeamID *string `json:"team_id,omitempty" db:"team_id"`
|
||||
Priority int `json:"priority" db:"priority"` // lower = evaluated first
|
||||
Type PolicyType `json:"policy_type" db:"policy_type"`
|
||||
Config PolicyConfig `json:"config" db:"config"`
|
||||
IsActive bool `json:"is_active" db:"is_active"`
|
||||
CreatedAt time.Time `json:"created_at" db:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
|
||||
}
|
||||
|
||||
// PolicyConfig holds the type-specific parameters for a routing policy.
|
||||
// Stored as JSONB. Each PolicyType uses a subset of these fields.
|
||||
type PolicyConfig struct {
|
||||
// provider_prefer: ordered list of provider config IDs
|
||||
PreferredProviders []string `json:"preferred_providers,omitempty"`
|
||||
|
||||
// team_route: required provider config IDs for this team
|
||||
AllowedProviders []string `json:"allowed_providers,omitempty"`
|
||||
|
||||
// cost_limit: max cost per request in USD
|
||||
MaxCostUSD *float64 `json:"max_cost_usd,omitempty"`
|
||||
|
||||
// model_alias: maps alias → provider+model
|
||||
AliasFrom string `json:"alias_from,omitempty"` // e.g. "default", "fast", "smart"
|
||||
AliasTo string `json:"alias_to,omitempty"` // model ID
|
||||
AliasProvider string `json:"alias_provider,omitempty"` // config ID
|
||||
|
||||
// Shared: health requirements
|
||||
SkipDown bool `json:"skip_down,omitempty"` // skip providers with status "down"
|
||||
PreferHealthy bool `json:"prefer_healthy,omitempty"` // rank healthy > degraded
|
||||
MaxRetries int `json:"max_retries,omitempty"` // 0 = no retry, default = 1
|
||||
}
|
||||
|
||||
// ── Evaluation Context ─────────────────────
|
||||
|
||||
// Context carries the information needed to evaluate routing policies.
|
||||
type Context struct {
|
||||
UserID string
|
||||
TeamIDs []string // teams the user belongs to
|
||||
Model string // requested model ID
|
||||
ConfigID string // explicitly requested provider config (may be empty)
|
||||
|
||||
// Health status for all active providers (keyed by config ID).
|
||||
// Populated from the health accumulator at evaluation time.
|
||||
HealthStatus map[string]models.ProviderStatus
|
||||
|
||||
// Pricing for candidate models (keyed by "configID:modelID").
|
||||
// Populated lazily from the pricing store.
|
||||
Pricing map[string]*models.ModelPricing
|
||||
}
|
||||
|
||||
// ── Candidates ─────────────────────────────
|
||||
|
||||
// Candidate is one possible (provider, model) pair the request could be routed to.
|
||||
type Candidate struct {
|
||||
ConfigID string `json:"config_id"`
|
||||
ProviderID string `json:"provider_id"` // "openai", "anthropic", etc.
|
||||
Model string `json:"model"`
|
||||
Endpoint string `json:"-"` // internal, not exposed in API
|
||||
Status models.ProviderStatus `json:"status"`
|
||||
Reason string `json:"reason"` // why this candidate was selected
|
||||
}
|
||||
|
||||
// ── Routing Decision ───────────────────────
|
||||
|
||||
// Decision records which candidate was selected and why.
|
||||
// Stored in the usage_log for observability.
|
||||
type Decision struct {
|
||||
PolicyID string `json:"policy_id,omitempty"` // which policy matched
|
||||
PolicyName string `json:"policy_name,omitempty"` // human-readable
|
||||
PolicyType string `json:"policy_type,omitempty"`
|
||||
Selected string `json:"selected"` // config ID that served the request
|
||||
FallbackDepth int `json:"fallback_depth"` // 0 = first choice, 1+ = fallback
|
||||
Candidates int `json:"candidates"` // total candidates evaluated
|
||||
}
|
||||
Reference in New Issue
Block a user