Jeffrey Smith
11fd8c1e57
- go.mod module name
- All 714 import references across 289 Go files
- VERSION: 0.1.0
- CI DB names: switchboard_core_{ci,dev,test}
- Docker image: gobha/switchboard-core
- Test fixtures: JWT issuer, repo names
- .env.example, docker-compose container name
- Compiles clean (go build exit 0)
499 lines
14 KiB
Go
499 lines
14 KiB
Go
package routing
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"switchboard-core/models"
|
|
)
|
|
|
|
func strPtr(s string) *string { return &s }
|
|
func f64Ptr(f float64) *float64 { return &f }
|
|
|
|
func baseCandidates() []Candidate {
|
|
return []Candidate{
|
|
{ConfigID: "cfg-openai", ProviderID: "openai", Model: "gpt-4o"},
|
|
{ConfigID: "cfg-anthropic", ProviderID: "anthropic", Model: "claude-sonnet-4-20250514"},
|
|
{ConfigID: "cfg-venice", ProviderID: "venice", Model: "llama-3.1-405b"},
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_NoPolices(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := &Context{UserID: "u1", Model: "gpt-4o", HealthStatus: map[string]models.ProviderStatus{}}
|
|
result, decision := e.Evaluate(ctx, nil, baseCandidates())
|
|
|
|
if len(result) != 3 {
|
|
t.Fatalf("expected 3 candidates, got %d", len(result))
|
|
}
|
|
if decision.Selected != "cfg-openai" {
|
|
t.Errorf("expected first candidate selected, got %s", decision.Selected)
|
|
}
|
|
if decision.FallbackDepth != 0 {
|
|
t.Errorf("expected fallback depth 0, got %d", decision.FallbackDepth)
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_ProviderPrefer(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := &Context{UserID: "u1", Model: "gpt-4o", HealthStatus: map[string]models.ProviderStatus{}}
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Name: "Prefer Anthropic", Priority: 1,
|
|
Type: PolicyProviderPrefer, IsActive: true, Scope: "global",
|
|
Config: PolicyConfig{
|
|
PreferredProviders: []string{"cfg-anthropic", "cfg-venice"},
|
|
},
|
|
},
|
|
}
|
|
|
|
result, decision := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
if result[0].ConfigID != "cfg-anthropic" {
|
|
t.Errorf("expected anthropic first, got %s", result[0].ConfigID)
|
|
}
|
|
if result[1].ConfigID != "cfg-venice" {
|
|
t.Errorf("expected venice second, got %s", result[1].ConfigID)
|
|
}
|
|
// openai should still be present as fallback
|
|
if result[2].ConfigID != "cfg-openai" {
|
|
t.Errorf("expected openai last, got %s", result[2].ConfigID)
|
|
}
|
|
if decision.PolicyID != "p1" {
|
|
t.Errorf("expected policy p1, got %s", decision.PolicyID)
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_TeamRoute(t *testing.T) {
|
|
e := NewEvaluator()
|
|
teamID := "team-1"
|
|
ctx := &Context{
|
|
UserID: "u1",
|
|
TeamIDs: []string{"team-1"},
|
|
Model: "gpt-4o",
|
|
HealthStatus: map[string]models.ProviderStatus{},
|
|
}
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Name: "Team uses Anthropic only", Priority: 1,
|
|
Type: PolicyTeamRoute, IsActive: true, Scope: "team", TeamID: &teamID,
|
|
Config: PolicyConfig{
|
|
AllowedProviders: []string{"cfg-anthropic"},
|
|
},
|
|
},
|
|
}
|
|
|
|
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
if len(result) != 1 {
|
|
t.Fatalf("expected 1 candidate after team filter, got %d", len(result))
|
|
}
|
|
if result[0].ConfigID != "cfg-anthropic" {
|
|
t.Errorf("expected anthropic only, got %s", result[0].ConfigID)
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_TeamRoute_WrongTeam(t *testing.T) {
|
|
e := NewEvaluator()
|
|
teamID := "team-2"
|
|
ctx := &Context{
|
|
UserID: "u1",
|
|
TeamIDs: []string{"team-1"}, // user is NOT in team-2
|
|
Model: "gpt-4o",
|
|
HealthStatus: map[string]models.ProviderStatus{},
|
|
}
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Name: "Team 2 only", Priority: 1,
|
|
Type: PolicyTeamRoute, IsActive: true, Scope: "team", TeamID: &teamID,
|
|
Config: PolicyConfig{AllowedProviders: []string{"cfg-anthropic"}},
|
|
},
|
|
}
|
|
|
|
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
// Policy should not apply — all 3 candidates remain
|
|
if len(result) != 3 {
|
|
t.Fatalf("expected 3 candidates (policy doesn't apply), got %d", len(result))
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_ModelAlias(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := &Context{
|
|
UserID: "u1", Model: "fast",
|
|
HealthStatus: map[string]models.ProviderStatus{},
|
|
}
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Name: "fast = haiku", Priority: 1,
|
|
Type: PolicyModelAlias, IsActive: true, Scope: "global",
|
|
Config: PolicyConfig{
|
|
AliasFrom: "fast",
|
|
AliasTo: "claude-haiku-4-20250414",
|
|
AliasProvider: "cfg-anthropic",
|
|
},
|
|
},
|
|
}
|
|
|
|
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
if result[0].ConfigID != "cfg-anthropic" {
|
|
t.Errorf("expected anthropic first for alias, got %s", result[0].ConfigID)
|
|
}
|
|
if result[0].Model != "claude-haiku-4-20250414" {
|
|
t.Errorf("expected model rewritten to haiku, got %s", result[0].Model)
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_HealthSkipDown(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := &Context{
|
|
UserID: "u1", Model: "gpt-4o",
|
|
HealthStatus: map[string]models.ProviderStatus{
|
|
"cfg-openai": models.StatusDown,
|
|
"cfg-anthropic": models.StatusHealthy,
|
|
"cfg-venice": models.StatusDegraded,
|
|
},
|
|
}
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Name: "Skip down", Priority: 1,
|
|
Type: PolicyProviderPrefer, IsActive: true, Scope: "global",
|
|
Config: PolicyConfig{
|
|
PreferredProviders: []string{"cfg-openai", "cfg-anthropic"},
|
|
SkipDown: true,
|
|
},
|
|
},
|
|
}
|
|
|
|
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
// openai is down and should be skipped
|
|
if result[0].ConfigID != "cfg-anthropic" {
|
|
t.Errorf("expected anthropic first (openai is down), got %s", result[0].ConfigID)
|
|
}
|
|
// openai should not be in results at all
|
|
for _, c := range result {
|
|
if c.ConfigID == "cfg-openai" {
|
|
t.Error("down provider should be skipped")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_HealthSorting(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := &Context{
|
|
UserID: "u1", Model: "gpt-4o",
|
|
HealthStatus: map[string]models.ProviderStatus{
|
|
"cfg-openai": models.StatusDegraded,
|
|
"cfg-anthropic": models.StatusHealthy,
|
|
"cfg-venice": models.StatusUnknown,
|
|
},
|
|
}
|
|
|
|
result, _ := e.Evaluate(ctx, nil, baseCandidates())
|
|
|
|
// Should be sorted: healthy (anthropic) > degraded (openai) > unknown (venice)
|
|
if result[0].ConfigID != "cfg-anthropic" {
|
|
t.Errorf("expected healthy provider first, got %s", result[0].ConfigID)
|
|
}
|
|
if result[1].ConfigID != "cfg-openai" {
|
|
t.Errorf("expected degraded provider second, got %s", result[1].ConfigID)
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_AllDown_KeepsCandidates(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := &Context{
|
|
UserID: "u1", Model: "gpt-4o",
|
|
HealthStatus: map[string]models.ProviderStatus{
|
|
"cfg-openai": models.StatusDown,
|
|
"cfg-anthropic": models.StatusDown,
|
|
"cfg-venice": models.StatusDown,
|
|
},
|
|
}
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Priority: 1, Type: PolicyProviderPrefer, IsActive: true, Scope: "global",
|
|
Config: PolicyConfig{SkipDown: true},
|
|
},
|
|
}
|
|
|
|
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
// When ALL are down, should keep them rather than returning empty
|
|
if len(result) == 0 {
|
|
t.Error("should keep all candidates when all are down")
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_PriorityOrder(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := &Context{
|
|
UserID: "u1",
|
|
TeamIDs: []string{"team-1"},
|
|
Model: "gpt-4o",
|
|
HealthStatus: map[string]models.ProviderStatus{},
|
|
}
|
|
teamID := "team-1"
|
|
policies := []Policy{
|
|
{
|
|
ID: "p-low", Name: "Low priority prefer", Priority: 10,
|
|
Type: PolicyProviderPrefer, IsActive: true, Scope: "global",
|
|
Config: PolicyConfig{PreferredProviders: []string{"cfg-venice"}},
|
|
},
|
|
{
|
|
ID: "p-high", Name: "High priority team route", Priority: 1,
|
|
Type: PolicyTeamRoute, IsActive: true, Scope: "team", TeamID: &teamID,
|
|
Config: PolicyConfig{AllowedProviders: []string{"cfg-anthropic", "cfg-openai"}},
|
|
},
|
|
}
|
|
|
|
result, decision := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
// Team route (priority 1) should filter first, then prefer (priority 10) reorders
|
|
if decision.PolicyType != string(PolicyTeamRoute) {
|
|
t.Errorf("expected team_route to be recorded (first match), got %s", decision.PolicyType)
|
|
}
|
|
// Venice should be filtered out by team route
|
|
for _, c := range result {
|
|
if c.ConfigID == "cfg-venice" {
|
|
t.Error("venice should be filtered out by team route")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_EmptyCandidates(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := &Context{UserID: "u1"}
|
|
result, decision := e.Evaluate(ctx, nil, nil)
|
|
if result != nil {
|
|
t.Error("expected nil result for empty candidates")
|
|
}
|
|
if decision != nil {
|
|
t.Error("expected nil decision for empty candidates")
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_InactivePolicy(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := &Context{UserID: "u1", Model: "gpt-4o", HealthStatus: map[string]models.ProviderStatus{}}
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Priority: 1, Type: PolicyTeamRoute, IsActive: false, Scope: "global",
|
|
Config: PolicyConfig{AllowedProviders: []string{"cfg-anthropic"}},
|
|
},
|
|
}
|
|
|
|
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
// Inactive policy should not filter
|
|
if len(result) != 3 {
|
|
t.Fatalf("inactive policy should not filter, got %d candidates", len(result))
|
|
}
|
|
}
|
|
|
|
// ── capability_match tests ──────────────────
|
|
|
|
func capContext() *Context {
|
|
return &Context{
|
|
UserID: "u1", Model: "any",
|
|
HealthStatus: map[string]models.ProviderStatus{},
|
|
Capabilities: map[string]*models.ModelCapabilities{
|
|
"cfg-openai:gpt-4o": {ToolCalling: true, Vision: true, Streaming: true},
|
|
"cfg-anthropic:claude-sonnet-4-20250514": {ToolCalling: true, Vision: true, Thinking: true, Streaming: true},
|
|
"cfg-venice:llama-3.1-405b": {ToolCalling: false, Vision: false, Streaming: true},
|
|
},
|
|
Pricing: map[string]*models.ModelPricing{
|
|
"cfg-openai:gpt-4o": {OutputPerM: 15.0},
|
|
"cfg-anthropic:claude-sonnet-4-20250514": {OutputPerM: 3.0},
|
|
"cfg-venice:llama-3.1-405b": {OutputPerM: 0.5},
|
|
},
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_CapabilityMatch_ToolCalling(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := capContext()
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Name: "Needs tool calling", Priority: 1,
|
|
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
|
Config: PolicyConfig{RequiredCaps: []string{"tool_calling"}},
|
|
},
|
|
}
|
|
|
|
result, decision := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
// Venice (llama) has no tool_calling — should be filtered out
|
|
if len(result) != 2 {
|
|
t.Fatalf("expected 2 candidates with tool_calling, got %d", len(result))
|
|
}
|
|
for _, c := range result {
|
|
if c.ConfigID == "cfg-venice" {
|
|
t.Error("venice should be filtered (no tool_calling)")
|
|
}
|
|
}
|
|
if decision.PolicyID != "p1" {
|
|
t.Errorf("expected policy p1, got %s", decision.PolicyID)
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_CapabilityMatch_MultipleCaps(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := capContext()
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Name: "Needs thinking + tool_calling", Priority: 1,
|
|
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
|
Config: PolicyConfig{RequiredCaps: []string{"thinking", "tool_calling"}},
|
|
},
|
|
}
|
|
|
|
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
// Only anthropic has both thinking AND tool_calling
|
|
if len(result) != 1 {
|
|
t.Fatalf("expected 1 candidate with thinking+tool_calling, got %d", len(result))
|
|
}
|
|
if result[0].ConfigID != "cfg-anthropic" {
|
|
t.Errorf("expected anthropic, got %s", result[0].ConfigID)
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_CapabilityMatch_PreferCheapest(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := capContext()
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Name: "Tool calling, cheapest first", Priority: 1,
|
|
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
|
Config: PolicyConfig{
|
|
RequiredCaps: []string{"tool_calling"},
|
|
PreferCheapest: true,
|
|
},
|
|
},
|
|
}
|
|
|
|
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
if len(result) != 2 {
|
|
t.Fatalf("expected 2 candidates, got %d", len(result))
|
|
}
|
|
// Anthropic ($3/M) should be before OpenAI ($15/M)
|
|
if result[0].ConfigID != "cfg-anthropic" {
|
|
t.Errorf("expected cheapest (anthropic) first, got %s", result[0].ConfigID)
|
|
}
|
|
if result[1].ConfigID != "cfg-openai" {
|
|
t.Errorf("expected openai second, got %s", result[1].ConfigID)
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_CapabilityMatch_NoneMatch_FallsBack(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := capContext()
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Name: "Needs web_search", Priority: 1,
|
|
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
|
Config: PolicyConfig{RequiredCaps: []string{"web_search"}},
|
|
},
|
|
}
|
|
|
|
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
// None of the candidates have web_search — should fall back to all 3
|
|
if len(result) != 3 {
|
|
t.Fatalf("expected 3 candidates (fallback), got %d", len(result))
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_CapabilityMatch_StreamingOnly(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := capContext()
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Name: "Streaming only", Priority: 1,
|
|
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
|
Config: PolicyConfig{RequiredCaps: []string{"streaming"}},
|
|
},
|
|
}
|
|
|
|
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
// All three have streaming — all should pass
|
|
if len(result) != 3 {
|
|
t.Fatalf("expected 3 candidates (all have streaming), got %d", len(result))
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_CapabilityMatch_NoCapsData(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := &Context{
|
|
UserID: "u1", Model: "any",
|
|
HealthStatus: map[string]models.ProviderStatus{},
|
|
Capabilities: nil, // no capability data
|
|
}
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Priority: 1,
|
|
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
|
Config: PolicyConfig{RequiredCaps: []string{"tool_calling"}},
|
|
},
|
|
}
|
|
|
|
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
// No caps data — policy should not match, all candidates kept
|
|
if len(result) != 3 {
|
|
t.Fatalf("expected 3 candidates (no caps data), got %d", len(result))
|
|
}
|
|
}
|
|
|
|
func TestEvaluate_CapabilityMatch_EmptyRequiredCaps(t *testing.T) {
|
|
e := NewEvaluator()
|
|
ctx := capContext()
|
|
policies := []Policy{
|
|
{
|
|
ID: "p1", Priority: 1,
|
|
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
|
Config: PolicyConfig{RequiredCaps: []string{}},
|
|
},
|
|
}
|
|
|
|
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
|
|
|
// Empty required caps — no filtering
|
|
if len(result) != 3 {
|
|
t.Fatalf("expected 3 candidates (empty caps), got %d", len(result))
|
|
}
|
|
}
|
|
|
|
func TestHasAllCaps(t *testing.T) {
|
|
caps := &models.ModelCapabilities{
|
|
ToolCalling: true, Vision: true, Streaming: true,
|
|
}
|
|
|
|
if !hasAllCaps(caps, []string{"tool_calling"}) {
|
|
t.Error("should match tool_calling")
|
|
}
|
|
if !hasAllCaps(caps, []string{"tool_calling", "vision"}) {
|
|
t.Error("should match tool_calling+vision")
|
|
}
|
|
if hasAllCaps(caps, []string{"thinking"}) {
|
|
t.Error("should not match thinking")
|
|
}
|
|
if !hasAllCaps(caps, []string{"unknown_future_cap"}) {
|
|
t.Error("unknown caps should be ignored (forward compat)")
|
|
}
|
|
if hasAllCaps(nil, []string{"tool_calling"}) {
|
|
t.Error("nil caps should return false")
|
|
}
|
|
if !hasAllCaps(caps, nil) {
|
|
t.Error("nil required should return true")
|
|
}
|
|
if !hasAllCaps(caps, []string{}) {
|
|
t.Error("empty required should return true")
|
|
}
|
|
}
|