package capabilities import ( "testing" "git.gobha.me/xcaliber/chat-switchboard/models" ) func TestLookupKnownModel_AlwaysFalse(t *testing.T) { // Known table removed in 0.9.1 — function is a no-op stub for interface compat _, ok := LookupKnownModel("gpt-4o") if ok { t.Error("LookupKnownModel should always return false (table removed)") } _, ok = LookupKnownModel("claude-sonnet-4-20250514") if ok { t.Error("LookupKnownModel should always return false (table removed)") } } func TestInferCapabilities_ToolCalling(t *testing.T) { tests := []struct { model string want bool }{ {"llama-3.1-70b-instruct", true}, {"mistral-7b", true}, {"qwen-2.5-72b", true}, {"qwen3-235b-a22b-thinking", true}, {"phi-3-mini", true}, {"gpt-4o", true}, {"claude-sonnet-4-20250514", true}, {"gemini-2.5-pro", true}, {"grok-41-fast", true}, {"kimi-k2-thinking", true}, {"minimax-m2.5", true}, {"random-smallmodel", false}, } for _, tt := range tests { caps := InferCapabilities(tt.model) if caps.ToolCalling != tt.want { t.Errorf("InferCapabilities(%q).ToolCalling = %v, want %v", tt.model, caps.ToolCalling, tt.want) } } } func TestInferCapabilities_Vision(t *testing.T) { tests := []struct { model string want bool }{ {"llava-v1.6-34b", true}, {"gemini-2.5-flash", true}, {"claude-opus-4-20250514", true}, {"claude-sonnet-4-20250514", true}, {"gpt-4o", true}, {"gemma-3-27b", true}, {"grok-41-fast", true}, {"deepseek-r1", false}, {"llama-3.1-70b", false}, } for _, tt := range tests { caps := InferCapabilities(tt.model) if caps.Vision != tt.want { t.Errorf("InferCapabilities(%q).Vision = %v, want %v", tt.model, caps.Vision, tt.want) } } } func TestInferCapabilities_Reasoning(t *testing.T) { tests := []struct { model string want bool }{ {"deepseek-r1-distill-llama-70b", true}, {"qwq-32b", true}, {"o3-mini", true}, {"qwen3-235b-a22b-thinking-2507", true}, {"grok-41-fast", true}, {"glm-5-32b", true}, {"gpt-4o", false}, {"llama-3.1-70b", false}, } for _, tt := range tests { caps := InferCapabilities(tt.model) if caps.Reasoning != tt.want { t.Errorf("InferCapabilities(%q).Reasoning = %v, want %v", tt.model, caps.Reasoning, tt.want) } } } func TestInferCapabilities_CodeOptimized(t *testing.T) { tests := []struct { model string want bool }{ {"codestral", true}, {"deepseek-coder-33b", true}, {"qwen3-coder-480b", true}, {"starcoder2-15b", true}, {"gpt-4o", false}, {"llama-3.1-70b", false}, } for _, tt := range tests { caps := InferCapabilities(tt.model) if caps.CodeOptimized != tt.want { t.Errorf("InferCapabilities(%q).CodeOptimized = %v, want %v", tt.model, caps.CodeOptimized, tt.want) } } } func TestResolveMaxOutput_FromCaps(t *testing.T) { caps := models.ModelCapabilities{MaxOutputTokens: 32000} got := ResolveMaxOutput("whatever-model", caps) if got != 32000 { t.Errorf("got %d, want 32000", got) } } func TestResolveMaxOutput_FromContext(t *testing.T) { caps := models.ModelCapabilities{MaxContext: 32768} got := ResolveMaxOutput("unknown-model-abc", caps) if got != 4096 { t.Errorf("got %d, want 4096 (derived from context/8)", got) } } func TestResolveMaxOutput_ContextClamp(t *testing.T) { caps := models.ModelCapabilities{MaxContext: 1048576} got := ResolveMaxOutput("unknown-large-model", caps) if got != 16384 { t.Errorf("got %d, want 16384 (clamped)", got) } caps = models.ModelCapabilities{MaxContext: 4096} got = ResolveMaxOutput("unknown-tiny-model", caps) if got != 2048 { t.Errorf("got %d, want 2048 (clamped)", got) } } func TestResolveMaxOutput_LastResort(t *testing.T) { caps := models.ModelCapabilities{} got := ResolveMaxOutput("mystery-model", caps) if got != 4096 { t.Errorf("got %d, want 4096 (last resort)", got) } } func TestResolveIntrinsic_CatalogWins(t *testing.T) { // Provider reports tool_calling and max_output — authoritative. // Heuristic should fill vision for claude (regex match). providerCaps := models.ModelCapabilities{ ToolCalling: true, MaxOutputTokens: 16384, } merged := ResolveIntrinsic("claude-sonnet-4-20250514", &providerCaps) if !merged.ToolCalling { t.Error("tool_calling should be preserved from provider") } if merged.MaxOutputTokens != 16384 { t.Errorf("max_output should be 16384 from provider, got %d", merged.MaxOutputTokens) } if !merged.Vision { t.Error("vision should be filled from heuristic (claude-sonnet matches)") } } func TestResolveIntrinsic_ProviderDataPreserved(t *testing.T) { // Provider says no vision — heuristic shouldn't override. // mergeGaps only fills false→true, never overrides true→false. // But: provider set vision=false explicitly. Since mergeGaps uses // "fill zero", and false IS zero, heuristic CAN fill it. // This is the expected behavior: heuristic is additive best-effort. providerCaps := models.ModelCapabilities{ ToolCalling: true, Vision: false, MaxOutputTokens: 8192, MaxContext: 65536, } merged := ResolveIntrinsic("some-unknown-model", &providerCaps) if !merged.ToolCalling { t.Error("tool_calling should be preserved from provider") } if merged.MaxContext != 65536 { t.Errorf("max_context should be preserved from provider, got %d", merged.MaxContext) } } func TestResolveIntrinsic_NilProvider(t *testing.T) { // Nil provider caps — falls through entirely to heuristic merged := ResolveIntrinsic("gpt-4o", nil) if !merged.ToolCalling { t.Error("should get tool_calling from heuristic (gpt-4 pattern)") } if !merged.Vision { t.Error("should get vision from heuristic (gpt-4o pattern)") } } func TestResolveIntrinsic_HeuristicOnly(t *testing.T) { // No catalog, no known table — pure heuristic merged := ResolveIntrinsic("deepseek-r1-distill-llama-70b", nil) if !merged.Reasoning { t.Error("should infer reasoning from deepseek-r1 pattern") } if !merged.Streaming { t.Error("should infer streaming (everything streams)") } } func TestHasProviderData(t *testing.T) { empty := models.ModelCapabilities{} if empty.HasProviderData() { t.Error("empty caps should not have provider data") } withTool := models.ModelCapabilities{ToolCalling: true} if !withTool.HasProviderData() { t.Error("caps with tool_calling should have provider data") } withContext := models.ModelCapabilities{MaxContext: 128000} if !withContext.HasProviderData() { t.Error("caps with max_context should have provider data") } }