package capabilities import ( "testing" "git.gobha.me/xcaliber/chat-switchboard/models" ) func TestLookupKnownModel_AlwaysFalse(t *testing.T) { // Known table removed in 0.9.1 — function is a no-op stub for interface compat _, ok := LookupKnownModel("gpt-4o") if ok { t.Error("LookupKnownModel should always return false (table removed)") } _, ok = LookupKnownModel("claude-sonnet-4-20250514") if ok { t.Error("LookupKnownModel should always return false (table removed)") } } func TestInferCapabilities_ToolCalling(t *testing.T) { tests := []struct { model string want bool }{ {"llama-3.1-70b-instruct", true}, {"mistral-7b", true}, {"qwen-2.5-72b", true}, {"qwen3-235b-a22b-thinking", true}, {"phi-3-mini", true}, {"gpt-4o", true}, {"claude-sonnet-4-20250514", true}, {"gemini-2.5-pro", true}, {"grok-41-fast", true}, {"kimi-k2-thinking", true}, {"minimax-m2.5", true}, {"random-smallmodel", false}, } for _, tt := range tests { caps := InferCapabilities(tt.model) if caps.ToolCalling != tt.want { t.Errorf("InferCapabilities(%q).ToolCalling = %v, want %v", tt.model, caps.ToolCalling, tt.want) } } } func TestInferCapabilities_Vision(t *testing.T) { tests := []struct { model string want bool }{ {"llava-v1.6-34b", true}, {"gemini-2.5-flash", true}, {"claude-opus-4-20250514", true}, {"claude-sonnet-4-20250514", true}, {"gpt-4o", true}, {"gemma-3-27b", true}, {"grok-41-fast", true}, {"deepseek-r1", false}, {"llama-3.1-70b", false}, } for _, tt := range tests { caps := InferCapabilities(tt.model) if caps.Vision != tt.want { t.Errorf("InferCapabilities(%q).Vision = %v, want %v", tt.model, caps.Vision, tt.want) } } } func TestInferCapabilities_Reasoning(t *testing.T) { tests := []struct { model string want bool }{ {"deepseek-r1-distill-llama-70b", true}, {"qwq-32b", true}, {"o3-mini", true}, {"qwen3-235b-a22b-thinking-2507", true}, {"grok-41-fast", true}, {"glm-5-32b", true}, {"gpt-4o", false}, {"llama-3.1-70b", false}, } for _, tt := range tests { caps := InferCapabilities(tt.model) if caps.Reasoning != tt.want { t.Errorf("InferCapabilities(%q).Reasoning = %v, want %v", tt.model, caps.Reasoning, tt.want) } } } func TestInferCapabilities_CodeOptimized(t *testing.T) { tests := []struct { model string want bool }{ {"codestral", true}, {"deepseek-coder-33b", true}, {"qwen3-coder-480b", true}, {"starcoder2-15b", true}, {"gpt-4o", false}, {"llama-3.1-70b", false}, } for _, tt := range tests { caps := InferCapabilities(tt.model) if caps.CodeOptimized != tt.want { t.Errorf("InferCapabilities(%q).CodeOptimized = %v, want %v", tt.model, caps.CodeOptimized, tt.want) } } } func TestResolveMaxOutput_FromCaps(t *testing.T) { caps := models.ModelCapabilities{MaxOutputTokens: 32000} got := ResolveMaxOutput("whatever-model", caps) if got != 32000 { t.Errorf("got %d, want 32000", got) } } func TestResolveMaxOutput_FromContext(t *testing.T) { caps := models.ModelCapabilities{MaxContext: 32768} got := ResolveMaxOutput("unknown-model-abc", caps) if got != 4096 { t.Errorf("got %d, want 4096 (derived from context/8)", got) } } func TestResolveMaxOutput_ContextClamp(t *testing.T) { caps := models.ModelCapabilities{MaxContext: 1048576} got := ResolveMaxOutput("unknown-large-model", caps) if got != 16384 { t.Errorf("got %d, want 16384 (clamped)", got) } caps = models.ModelCapabilities{MaxContext: 4096} got = ResolveMaxOutput("unknown-tiny-model", caps) if got != 2048 { t.Errorf("got %d, want 2048 (clamped)", got) } } func TestResolveMaxOutput_LastResort(t *testing.T) { caps := models.ModelCapabilities{} got := ResolveMaxOutput("mystery-model", caps) if got != 4096 { t.Errorf("got %d, want 4096 (last resort)", got) } } func TestResolveIntrinsic_CatalogWins(t *testing.T) { // Provider reports tool_calling and max_output — authoritative. // Heuristic should fill vision for claude (regex match). providerCaps := models.ModelCapabilities{ ToolCalling: true, MaxOutputTokens: 16384, } merged := ResolveIntrinsic("claude-sonnet-4-20250514", &providerCaps, nil) if !merged.ToolCalling { t.Error("tool_calling should be preserved from provider") } if merged.MaxOutputTokens != 16384 { t.Errorf("max_output should be 16384 from provider, got %d", merged.MaxOutputTokens) } if !merged.Vision { t.Error("vision should be filled from heuristic (claude-sonnet matches)") } } func TestResolveIntrinsic_ProviderDataPreserved(t *testing.T) { // Provider says no vision — heuristic shouldn't override. // mergeGaps only fills false→true, never overrides true→false. // But: provider set vision=false explicitly. Since mergeGaps uses // "fill zero", and false IS zero, heuristic CAN fill it. // This is the expected behavior: heuristic is additive best-effort. providerCaps := models.ModelCapabilities{ ToolCalling: true, Vision: false, MaxOutputTokens: 8192, MaxContext: 65536, } merged := ResolveIntrinsic("some-unknown-model", &providerCaps, nil) if !merged.ToolCalling { t.Error("tool_calling should be preserved from provider") } if merged.MaxContext != 65536 { t.Errorf("max_context should be preserved from provider, got %d", merged.MaxContext) } } func TestResolveIntrinsic_NilProvider(t *testing.T) { // Nil provider caps — falls through entirely to heuristic merged := ResolveIntrinsic("gpt-4o", nil, nil) if !merged.ToolCalling { t.Error("should get tool_calling from heuristic (gpt-4 pattern)") } if !merged.Vision { t.Error("should get vision from heuristic (gpt-4o pattern)") } } func TestResolveIntrinsic_HeuristicOnly(t *testing.T) { // No catalog, no known table — pure heuristic merged := ResolveIntrinsic("deepseek-r1-distill-llama-70b", nil, nil) if !merged.Reasoning { t.Error("should infer reasoning from deepseek-r1 pattern") } if !merged.Streaming { t.Error("should infer streaming (everything streams)") } } func TestHasProviderData(t *testing.T) { empty := models.ModelCapabilities{} if empty.HasProviderData() { t.Error("empty caps should not have provider data") } withTool := models.ModelCapabilities{ToolCalling: true} if !withTool.HasProviderData() { t.Error("caps with tool_calling should have provider data") } withContext := models.ModelCapabilities{MaxContext: 128000} if !withContext.HasProviderData() { t.Error("caps with max_context should have provider data") } } func TestResolveIntrinsic_AdminOverrides(t *testing.T) { // Catalog says no vision, heuristic says no vision, // but admin override forces vision=true. catalogCaps := &models.ModelCapabilities{ ToolCalling: true, Vision: false, } overrides := []models.CapabilityOverride{ {Field: "vision", Value: "true"}, } merged := ResolveIntrinsic("some-custom-model", catalogCaps, overrides) if !merged.Vision { t.Error("admin override should force vision=true") } if !merged.ToolCalling { t.Error("catalog tool_calling should be preserved") } } func TestResolveIntrinsic_AdminOverrides_DisableCapability(t *testing.T) { // Heuristic infers tool_calling for gpt-4o, but admin says no. overrides := []models.CapabilityOverride{ {Field: "tool_calling", Value: "false"}, } merged := ResolveIntrinsic("gpt-4o", nil, overrides) if merged.ToolCalling { t.Error("admin override should disable tool_calling") } } func TestResolveIntrinsic_AdminOverrides_IntValues(t *testing.T) { overrides := []models.CapabilityOverride{ {Field: "max_context", Value: "200000"}, {Field: "max_output_tokens", Value: "16384"}, } merged := ResolveIntrinsic("some-model", nil, overrides) if merged.MaxContext != 200000 { t.Errorf("expected max_context=200000, got %d", merged.MaxContext) } if merged.MaxOutputTokens != 16384 { t.Errorf("expected max_output_tokens=16384, got %d", merged.MaxOutputTokens) } } func TestResolveIntrinsic_OverridePriority(t *testing.T) { // Catalog says max_context=128000, admin overrides to 256000. // Admin should win. catalogCaps := &models.ModelCapabilities{ MaxContext: 128000, } overrides := []models.CapabilityOverride{ {Field: "max_context", Value: "256000"}, } merged := ResolveIntrinsic("some-model", catalogCaps, overrides) if merged.MaxContext != 256000 { t.Errorf("admin override should win over catalog: expected 256000, got %d", merged.MaxContext) } }