Changeset 0.22.2 (#96)
This commit is contained in:
294
server/routing/evaluator_test.go
Normal file
294
server/routing/evaluator_test.go
Normal file
@@ -0,0 +1,294 @@
|
||||
package routing
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
func strPtr(s string) *string { return &s }
|
||||
func f64Ptr(f float64) *float64 { return &f }
|
||||
|
||||
func baseCandidates() []Candidate {
|
||||
return []Candidate{
|
||||
{ConfigID: "cfg-openai", ProviderID: "openai", Model: "gpt-4o"},
|
||||
{ConfigID: "cfg-anthropic", ProviderID: "anthropic", Model: "claude-sonnet-4-20250514"},
|
||||
{ConfigID: "cfg-venice", ProviderID: "venice", Model: "llama-3.1-405b"},
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_NoPolices(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{UserID: "u1", Model: "gpt-4o", HealthStatus: map[string]models.ProviderStatus{}}
|
||||
result, decision := e.Evaluate(ctx, nil, baseCandidates())
|
||||
|
||||
if len(result) != 3 {
|
||||
t.Fatalf("expected 3 candidates, got %d", len(result))
|
||||
}
|
||||
if decision.Selected != "cfg-openai" {
|
||||
t.Errorf("expected first candidate selected, got %s", decision.Selected)
|
||||
}
|
||||
if decision.FallbackDepth != 0 {
|
||||
t.Errorf("expected fallback depth 0, got %d", decision.FallbackDepth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_ProviderPrefer(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{UserID: "u1", Model: "gpt-4o", HealthStatus: map[string]models.ProviderStatus{}}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "Prefer Anthropic", Priority: 1,
|
||||
Type: PolicyProviderPrefer, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{
|
||||
PreferredProviders: []string{"cfg-anthropic", "cfg-venice"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, decision := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
if result[0].ConfigID != "cfg-anthropic" {
|
||||
t.Errorf("expected anthropic first, got %s", result[0].ConfigID)
|
||||
}
|
||||
if result[1].ConfigID != "cfg-venice" {
|
||||
t.Errorf("expected venice second, got %s", result[1].ConfigID)
|
||||
}
|
||||
// openai should still be present as fallback
|
||||
if result[2].ConfigID != "cfg-openai" {
|
||||
t.Errorf("expected openai last, got %s", result[2].ConfigID)
|
||||
}
|
||||
if decision.PolicyID != "p1" {
|
||||
t.Errorf("expected policy p1, got %s", decision.PolicyID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_TeamRoute(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
teamID := "team-1"
|
||||
ctx := &Context{
|
||||
UserID: "u1",
|
||||
TeamIDs: []string{"team-1"},
|
||||
Model: "gpt-4o",
|
||||
HealthStatus: map[string]models.ProviderStatus{},
|
||||
}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "Team uses Anthropic only", Priority: 1,
|
||||
Type: PolicyTeamRoute, IsActive: true, Scope: "team", TeamID: &teamID,
|
||||
Config: PolicyConfig{
|
||||
AllowedProviders: []string{"cfg-anthropic"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
if len(result) != 1 {
|
||||
t.Fatalf("expected 1 candidate after team filter, got %d", len(result))
|
||||
}
|
||||
if result[0].ConfigID != "cfg-anthropic" {
|
||||
t.Errorf("expected anthropic only, got %s", result[0].ConfigID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_TeamRoute_WrongTeam(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
teamID := "team-2"
|
||||
ctx := &Context{
|
||||
UserID: "u1",
|
||||
TeamIDs: []string{"team-1"}, // user is NOT in team-2
|
||||
Model: "gpt-4o",
|
||||
HealthStatus: map[string]models.ProviderStatus{},
|
||||
}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "Team 2 only", Priority: 1,
|
||||
Type: PolicyTeamRoute, IsActive: true, Scope: "team", TeamID: &teamID,
|
||||
Config: PolicyConfig{AllowedProviders: []string{"cfg-anthropic"}},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// Policy should not apply — all 3 candidates remain
|
||||
if len(result) != 3 {
|
||||
t.Fatalf("expected 3 candidates (policy doesn't apply), got %d", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_ModelAlias(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{
|
||||
UserID: "u1", Model: "fast",
|
||||
HealthStatus: map[string]models.ProviderStatus{},
|
||||
}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "fast = haiku", Priority: 1,
|
||||
Type: PolicyModelAlias, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{
|
||||
AliasFrom: "fast",
|
||||
AliasTo: "claude-haiku-4-20250414",
|
||||
AliasProvider: "cfg-anthropic",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
if result[0].ConfigID != "cfg-anthropic" {
|
||||
t.Errorf("expected anthropic first for alias, got %s", result[0].ConfigID)
|
||||
}
|
||||
if result[0].Model != "claude-haiku-4-20250414" {
|
||||
t.Errorf("expected model rewritten to haiku, got %s", result[0].Model)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_HealthSkipDown(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{
|
||||
UserID: "u1", Model: "gpt-4o",
|
||||
HealthStatus: map[string]models.ProviderStatus{
|
||||
"cfg-openai": models.StatusDown,
|
||||
"cfg-anthropic": models.StatusHealthy,
|
||||
"cfg-venice": models.StatusDegraded,
|
||||
},
|
||||
}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "Skip down", Priority: 1,
|
||||
Type: PolicyProviderPrefer, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{
|
||||
PreferredProviders: []string{"cfg-openai", "cfg-anthropic"},
|
||||
SkipDown: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// openai is down and should be skipped
|
||||
if result[0].ConfigID != "cfg-anthropic" {
|
||||
t.Errorf("expected anthropic first (openai is down), got %s", result[0].ConfigID)
|
||||
}
|
||||
// openai should not be in results at all
|
||||
for _, c := range result {
|
||||
if c.ConfigID == "cfg-openai" {
|
||||
t.Error("down provider should be skipped")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_HealthSorting(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{
|
||||
UserID: "u1", Model: "gpt-4o",
|
||||
HealthStatus: map[string]models.ProviderStatus{
|
||||
"cfg-openai": models.StatusDegraded,
|
||||
"cfg-anthropic": models.StatusHealthy,
|
||||
"cfg-venice": models.StatusUnknown,
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, nil, baseCandidates())
|
||||
|
||||
// Should be sorted: healthy (anthropic) > degraded (openai) > unknown (venice)
|
||||
if result[0].ConfigID != "cfg-anthropic" {
|
||||
t.Errorf("expected healthy provider first, got %s", result[0].ConfigID)
|
||||
}
|
||||
if result[1].ConfigID != "cfg-openai" {
|
||||
t.Errorf("expected degraded provider second, got %s", result[1].ConfigID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_AllDown_KeepsCandidates(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{
|
||||
UserID: "u1", Model: "gpt-4o",
|
||||
HealthStatus: map[string]models.ProviderStatus{
|
||||
"cfg-openai": models.StatusDown,
|
||||
"cfg-anthropic": models.StatusDown,
|
||||
"cfg-venice": models.StatusDown,
|
||||
},
|
||||
}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Priority: 1, Type: PolicyProviderPrefer, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{SkipDown: true},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// When ALL are down, should keep them rather than returning empty
|
||||
if len(result) == 0 {
|
||||
t.Error("should keep all candidates when all are down")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_PriorityOrder(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{
|
||||
UserID: "u1",
|
||||
TeamIDs: []string{"team-1"},
|
||||
Model: "gpt-4o",
|
||||
HealthStatus: map[string]models.ProviderStatus{},
|
||||
}
|
||||
teamID := "team-1"
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p-low", Name: "Low priority prefer", Priority: 10,
|
||||
Type: PolicyProviderPrefer, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{PreferredProviders: []string{"cfg-venice"}},
|
||||
},
|
||||
{
|
||||
ID: "p-high", Name: "High priority team route", Priority: 1,
|
||||
Type: PolicyTeamRoute, IsActive: true, Scope: "team", TeamID: &teamID,
|
||||
Config: PolicyConfig{AllowedProviders: []string{"cfg-anthropic", "cfg-openai"}},
|
||||
},
|
||||
}
|
||||
|
||||
result, decision := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// Team route (priority 1) should filter first, then prefer (priority 10) reorders
|
||||
if decision.PolicyType != string(PolicyTeamRoute) {
|
||||
t.Errorf("expected team_route to be recorded (first match), got %s", decision.PolicyType)
|
||||
}
|
||||
// Venice should be filtered out by team route
|
||||
for _, c := range result {
|
||||
if c.ConfigID == "cfg-venice" {
|
||||
t.Error("venice should be filtered out by team route")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_EmptyCandidates(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{UserID: "u1"}
|
||||
result, decision := e.Evaluate(ctx, nil, nil)
|
||||
if result != nil {
|
||||
t.Error("expected nil result for empty candidates")
|
||||
}
|
||||
if decision != nil {
|
||||
t.Error("expected nil decision for empty candidates")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_InactivePolicy(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{UserID: "u1", Model: "gpt-4o", HealthStatus: map[string]models.ProviderStatus{}}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Priority: 1, Type: PolicyTeamRoute, IsActive: false, Scope: "global",
|
||||
Config: PolicyConfig{AllowedProviders: []string{"cfg-anthropic"}},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// Inactive policy should not filter
|
||||
if len(result) != 3 {
|
||||
t.Fatalf("inactive policy should not filter, got %d candidates", len(result))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user