Changeset 0.29.1 (#196)
This commit is contained in:
@@ -292,3 +292,207 @@ func TestEvaluate_InactivePolicy(t *testing.T) {
|
||||
t.Fatalf("inactive policy should not filter, got %d candidates", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
// ── capability_match tests ──────────────────
|
||||
|
||||
func capContext() *Context {
|
||||
return &Context{
|
||||
UserID: "u1", Model: "any",
|
||||
HealthStatus: map[string]models.ProviderStatus{},
|
||||
Capabilities: map[string]*models.ModelCapabilities{
|
||||
"cfg-openai:gpt-4o": {ToolCalling: true, Vision: true, Streaming: true},
|
||||
"cfg-anthropic:claude-sonnet-4-20250514": {ToolCalling: true, Vision: true, Thinking: true, Streaming: true},
|
||||
"cfg-venice:llama-3.1-405b": {ToolCalling: false, Vision: false, Streaming: true},
|
||||
},
|
||||
Pricing: map[string]*models.ModelPricing{
|
||||
"cfg-openai:gpt-4o": {OutputPerM: 15.0},
|
||||
"cfg-anthropic:claude-sonnet-4-20250514": {OutputPerM: 3.0},
|
||||
"cfg-venice:llama-3.1-405b": {OutputPerM: 0.5},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_CapabilityMatch_ToolCalling(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := capContext()
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "Needs tool calling", Priority: 1,
|
||||
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{RequiredCaps: []string{"tool_calling"}},
|
||||
},
|
||||
}
|
||||
|
||||
result, decision := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// Venice (llama) has no tool_calling — should be filtered out
|
||||
if len(result) != 2 {
|
||||
t.Fatalf("expected 2 candidates with tool_calling, got %d", len(result))
|
||||
}
|
||||
for _, c := range result {
|
||||
if c.ConfigID == "cfg-venice" {
|
||||
t.Error("venice should be filtered (no tool_calling)")
|
||||
}
|
||||
}
|
||||
if decision.PolicyID != "p1" {
|
||||
t.Errorf("expected policy p1, got %s", decision.PolicyID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_CapabilityMatch_MultipleCaps(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := capContext()
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "Needs thinking + tool_calling", Priority: 1,
|
||||
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{RequiredCaps: []string{"thinking", "tool_calling"}},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// Only anthropic has both thinking AND tool_calling
|
||||
if len(result) != 1 {
|
||||
t.Fatalf("expected 1 candidate with thinking+tool_calling, got %d", len(result))
|
||||
}
|
||||
if result[0].ConfigID != "cfg-anthropic" {
|
||||
t.Errorf("expected anthropic, got %s", result[0].ConfigID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_CapabilityMatch_PreferCheapest(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := capContext()
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "Tool calling, cheapest first", Priority: 1,
|
||||
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{
|
||||
RequiredCaps: []string{"tool_calling"},
|
||||
PreferCheapest: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
if len(result) != 2 {
|
||||
t.Fatalf("expected 2 candidates, got %d", len(result))
|
||||
}
|
||||
// Anthropic ($3/M) should be before OpenAI ($15/M)
|
||||
if result[0].ConfigID != "cfg-anthropic" {
|
||||
t.Errorf("expected cheapest (anthropic) first, got %s", result[0].ConfigID)
|
||||
}
|
||||
if result[1].ConfigID != "cfg-openai" {
|
||||
t.Errorf("expected openai second, got %s", result[1].ConfigID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_CapabilityMatch_NoneMatch_FallsBack(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := capContext()
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "Needs web_search", Priority: 1,
|
||||
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{RequiredCaps: []string{"web_search"}},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// None of the candidates have web_search — should fall back to all 3
|
||||
if len(result) != 3 {
|
||||
t.Fatalf("expected 3 candidates (fallback), got %d", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_CapabilityMatch_StreamingOnly(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := capContext()
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Name: "Streaming only", Priority: 1,
|
||||
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{RequiredCaps: []string{"streaming"}},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// All three have streaming — all should pass
|
||||
if len(result) != 3 {
|
||||
t.Fatalf("expected 3 candidates (all have streaming), got %d", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_CapabilityMatch_NoCapsData(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := &Context{
|
||||
UserID: "u1", Model: "any",
|
||||
HealthStatus: map[string]models.ProviderStatus{},
|
||||
Capabilities: nil, // no capability data
|
||||
}
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Priority: 1,
|
||||
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{RequiredCaps: []string{"tool_calling"}},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// No caps data — policy should not match, all candidates kept
|
||||
if len(result) != 3 {
|
||||
t.Fatalf("expected 3 candidates (no caps data), got %d", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluate_CapabilityMatch_EmptyRequiredCaps(t *testing.T) {
|
||||
e := NewEvaluator()
|
||||
ctx := capContext()
|
||||
policies := []Policy{
|
||||
{
|
||||
ID: "p1", Priority: 1,
|
||||
Type: PolicyCapabilityMatch, IsActive: true, Scope: "global",
|
||||
Config: PolicyConfig{RequiredCaps: []string{}},
|
||||
},
|
||||
}
|
||||
|
||||
result, _ := e.Evaluate(ctx, policies, baseCandidates())
|
||||
|
||||
// Empty required caps — no filtering
|
||||
if len(result) != 3 {
|
||||
t.Fatalf("expected 3 candidates (empty caps), got %d", len(result))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasAllCaps(t *testing.T) {
|
||||
caps := &models.ModelCapabilities{
|
||||
ToolCalling: true, Vision: true, Streaming: true,
|
||||
}
|
||||
|
||||
if !hasAllCaps(caps, []string{"tool_calling"}) {
|
||||
t.Error("should match tool_calling")
|
||||
}
|
||||
if !hasAllCaps(caps, []string{"tool_calling", "vision"}) {
|
||||
t.Error("should match tool_calling+vision")
|
||||
}
|
||||
if hasAllCaps(caps, []string{"thinking"}) {
|
||||
t.Error("should not match thinking")
|
||||
}
|
||||
if !hasAllCaps(caps, []string{"unknown_future_cap"}) {
|
||||
t.Error("unknown caps should be ignored (forward compat)")
|
||||
}
|
||||
if hasAllCaps(nil, []string{"tool_calling"}) {
|
||||
t.Error("nil caps should return false")
|
||||
}
|
||||
if !hasAllCaps(caps, nil) {
|
||||
t.Error("nil required should return true")
|
||||
}
|
||||
if !hasAllCaps(caps, []string{}) {
|
||||
t.Error("empty required should return true")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user