300 lines
8.3 KiB
Go
300 lines
8.3 KiB
Go
package capabilities
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"git.gobha.me/xcaliber/chat-switchboard/models"
|
|
)
|
|
|
|
func TestLookupKnownModel_AlwaysFalse(t *testing.T) {
|
|
// Known table removed in 0.9.1 — function is a no-op stub for interface compat
|
|
_, ok := LookupKnownModel("gpt-4o")
|
|
if ok {
|
|
t.Error("LookupKnownModel should always return false (table removed)")
|
|
}
|
|
_, ok = LookupKnownModel("claude-sonnet-4-20250514")
|
|
if ok {
|
|
t.Error("LookupKnownModel should always return false (table removed)")
|
|
}
|
|
}
|
|
|
|
func TestInferCapabilities_ToolCalling(t *testing.T) {
|
|
tests := []struct {
|
|
model string
|
|
want bool
|
|
}{
|
|
{"llama-3.1-70b-instruct", true},
|
|
{"mistral-7b", true},
|
|
{"qwen-2.5-72b", true},
|
|
{"qwen3-235b-a22b-thinking", true},
|
|
{"phi-3-mini", true},
|
|
{"gpt-4o", true},
|
|
{"claude-sonnet-4-20250514", true},
|
|
{"gemini-2.5-pro", true},
|
|
{"grok-41-fast", true},
|
|
{"kimi-k2-thinking", true},
|
|
{"minimax-m2.5", true},
|
|
{"random-smallmodel", false},
|
|
}
|
|
for _, tt := range tests {
|
|
caps := InferCapabilities(tt.model)
|
|
if caps.ToolCalling != tt.want {
|
|
t.Errorf("InferCapabilities(%q).ToolCalling = %v, want %v", tt.model, caps.ToolCalling, tt.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestInferCapabilities_Vision(t *testing.T) {
|
|
tests := []struct {
|
|
model string
|
|
want bool
|
|
}{
|
|
{"llava-v1.6-34b", true},
|
|
{"gemini-2.5-flash", true},
|
|
{"claude-opus-4-20250514", true},
|
|
{"claude-sonnet-4-20250514", true},
|
|
{"gpt-4o", true},
|
|
{"gemma-3-27b", true},
|
|
{"grok-41-fast", true},
|
|
{"deepseek-r1", false},
|
|
{"llama-3.1-70b", false},
|
|
}
|
|
for _, tt := range tests {
|
|
caps := InferCapabilities(tt.model)
|
|
if caps.Vision != tt.want {
|
|
t.Errorf("InferCapabilities(%q).Vision = %v, want %v", tt.model, caps.Vision, tt.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestInferCapabilities_Reasoning(t *testing.T) {
|
|
tests := []struct {
|
|
model string
|
|
want bool
|
|
}{
|
|
{"deepseek-r1-distill-llama-70b", true},
|
|
{"qwq-32b", true},
|
|
{"o3-mini", true},
|
|
{"qwen3-235b-a22b-thinking-2507", true},
|
|
{"grok-41-fast", true},
|
|
{"glm-5-32b", true},
|
|
{"gpt-4o", false},
|
|
{"llama-3.1-70b", false},
|
|
}
|
|
for _, tt := range tests {
|
|
caps := InferCapabilities(tt.model)
|
|
if caps.Reasoning != tt.want {
|
|
t.Errorf("InferCapabilities(%q).Reasoning = %v, want %v", tt.model, caps.Reasoning, tt.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestInferCapabilities_CodeOptimized(t *testing.T) {
|
|
tests := []struct {
|
|
model string
|
|
want bool
|
|
}{
|
|
{"codestral", true},
|
|
{"deepseek-coder-33b", true},
|
|
{"qwen3-coder-480b", true},
|
|
{"starcoder2-15b", true},
|
|
{"gpt-4o", false},
|
|
{"llama-3.1-70b", false},
|
|
}
|
|
for _, tt := range tests {
|
|
caps := InferCapabilities(tt.model)
|
|
if caps.CodeOptimized != tt.want {
|
|
t.Errorf("InferCapabilities(%q).CodeOptimized = %v, want %v", tt.model, caps.CodeOptimized, tt.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestResolveMaxOutput_FromCaps(t *testing.T) {
|
|
caps := models.ModelCapabilities{MaxOutputTokens: 32000}
|
|
got := ResolveMaxOutput("whatever-model", caps)
|
|
if got != 32000 {
|
|
t.Errorf("got %d, want 32000", got)
|
|
}
|
|
}
|
|
|
|
func TestResolveMaxOutput_FromContext(t *testing.T) {
|
|
caps := models.ModelCapabilities{MaxContext: 32768}
|
|
got := ResolveMaxOutput("unknown-model-abc", caps)
|
|
if got != 4096 {
|
|
t.Errorf("got %d, want 4096 (derived from context/8)", got)
|
|
}
|
|
}
|
|
|
|
func TestResolveMaxOutput_ContextClamp(t *testing.T) {
|
|
caps := models.ModelCapabilities{MaxContext: 1048576}
|
|
got := ResolveMaxOutput("unknown-large-model", caps)
|
|
if got != 16384 {
|
|
t.Errorf("got %d, want 16384 (clamped)", got)
|
|
}
|
|
|
|
caps = models.ModelCapabilities{MaxContext: 4096}
|
|
got = ResolveMaxOutput("unknown-tiny-model", caps)
|
|
if got != 2048 {
|
|
t.Errorf("got %d, want 2048 (clamped)", got)
|
|
}
|
|
}
|
|
|
|
func TestResolveMaxOutput_LastResort(t *testing.T) {
|
|
caps := models.ModelCapabilities{}
|
|
got := ResolveMaxOutput("mystery-model", caps)
|
|
if got != 4096 {
|
|
t.Errorf("got %d, want 4096 (last resort)", got)
|
|
}
|
|
}
|
|
|
|
func TestResolveIntrinsic_CatalogWins(t *testing.T) {
|
|
// Provider reports tool_calling and max_output — authoritative.
|
|
// Heuristic should fill vision for claude (regex match).
|
|
providerCaps := models.ModelCapabilities{
|
|
ToolCalling: true,
|
|
MaxOutputTokens: 16384,
|
|
}
|
|
merged := ResolveIntrinsic("claude-sonnet-4-20250514", &providerCaps, nil)
|
|
|
|
if !merged.ToolCalling {
|
|
t.Error("tool_calling should be preserved from provider")
|
|
}
|
|
if merged.MaxOutputTokens != 16384 {
|
|
t.Errorf("max_output should be 16384 from provider, got %d", merged.MaxOutputTokens)
|
|
}
|
|
if !merged.Vision {
|
|
t.Error("vision should be filled from heuristic (claude-sonnet matches)")
|
|
}
|
|
}
|
|
|
|
func TestResolveIntrinsic_ProviderDataPreserved(t *testing.T) {
|
|
// Provider says no vision — heuristic shouldn't override.
|
|
// mergeGaps only fills false→true, never overrides true→false.
|
|
// But: provider set vision=false explicitly. Since mergeGaps uses
|
|
// "fill zero", and false IS zero, heuristic CAN fill it.
|
|
// This is the expected behavior: heuristic is additive best-effort.
|
|
providerCaps := models.ModelCapabilities{
|
|
ToolCalling: true,
|
|
Vision: false,
|
|
MaxOutputTokens: 8192,
|
|
MaxContext: 65536,
|
|
}
|
|
merged := ResolveIntrinsic("some-unknown-model", &providerCaps, nil)
|
|
|
|
if !merged.ToolCalling {
|
|
t.Error("tool_calling should be preserved from provider")
|
|
}
|
|
if merged.MaxContext != 65536 {
|
|
t.Errorf("max_context should be preserved from provider, got %d", merged.MaxContext)
|
|
}
|
|
}
|
|
|
|
func TestResolveIntrinsic_NilProvider(t *testing.T) {
|
|
// Nil provider caps — falls through entirely to heuristic
|
|
merged := ResolveIntrinsic("gpt-4o", nil, nil)
|
|
|
|
if !merged.ToolCalling {
|
|
t.Error("should get tool_calling from heuristic (gpt-4 pattern)")
|
|
}
|
|
if !merged.Vision {
|
|
t.Error("should get vision from heuristic (gpt-4o pattern)")
|
|
}
|
|
}
|
|
|
|
func TestResolveIntrinsic_HeuristicOnly(t *testing.T) {
|
|
// No catalog, no known table — pure heuristic
|
|
merged := ResolveIntrinsic("deepseek-r1-distill-llama-70b", nil, nil)
|
|
|
|
if !merged.Reasoning {
|
|
t.Error("should infer reasoning from deepseek-r1 pattern")
|
|
}
|
|
if !merged.Streaming {
|
|
t.Error("should infer streaming (everything streams)")
|
|
}
|
|
}
|
|
|
|
func TestHasProviderData(t *testing.T) {
|
|
empty := models.ModelCapabilities{}
|
|
if empty.HasProviderData() {
|
|
t.Error("empty caps should not have provider data")
|
|
}
|
|
|
|
withTool := models.ModelCapabilities{ToolCalling: true}
|
|
if !withTool.HasProviderData() {
|
|
t.Error("caps with tool_calling should have provider data")
|
|
}
|
|
|
|
withContext := models.ModelCapabilities{MaxContext: 128000}
|
|
if !withContext.HasProviderData() {
|
|
t.Error("caps with max_context should have provider data")
|
|
}
|
|
}
|
|
|
|
func TestResolveIntrinsic_AdminOverrides(t *testing.T) {
|
|
// Catalog says no vision, heuristic says no vision,
|
|
// but admin override forces vision=true.
|
|
catalogCaps := &models.ModelCapabilities{
|
|
ToolCalling: true,
|
|
Vision: false,
|
|
}
|
|
overrides := []models.CapabilityOverride{
|
|
{Field: "vision", Value: "true"},
|
|
}
|
|
|
|
merged := ResolveIntrinsic("some-custom-model", catalogCaps, overrides)
|
|
|
|
if !merged.Vision {
|
|
t.Error("admin override should force vision=true")
|
|
}
|
|
if !merged.ToolCalling {
|
|
t.Error("catalog tool_calling should be preserved")
|
|
}
|
|
}
|
|
|
|
func TestResolveIntrinsic_AdminOverrides_DisableCapability(t *testing.T) {
|
|
// Heuristic infers tool_calling for gpt-4o, but admin says no.
|
|
overrides := []models.CapabilityOverride{
|
|
{Field: "tool_calling", Value: "false"},
|
|
}
|
|
|
|
merged := ResolveIntrinsic("gpt-4o", nil, overrides)
|
|
|
|
if merged.ToolCalling {
|
|
t.Error("admin override should disable tool_calling")
|
|
}
|
|
}
|
|
|
|
func TestResolveIntrinsic_AdminOverrides_IntValues(t *testing.T) {
|
|
overrides := []models.CapabilityOverride{
|
|
{Field: "max_context", Value: "200000"},
|
|
{Field: "max_output_tokens", Value: "16384"},
|
|
}
|
|
|
|
merged := ResolveIntrinsic("some-model", nil, overrides)
|
|
|
|
if merged.MaxContext != 200000 {
|
|
t.Errorf("expected max_context=200000, got %d", merged.MaxContext)
|
|
}
|
|
if merged.MaxOutputTokens != 16384 {
|
|
t.Errorf("expected max_output_tokens=16384, got %d", merged.MaxOutputTokens)
|
|
}
|
|
}
|
|
|
|
func TestResolveIntrinsic_OverridePriority(t *testing.T) {
|
|
// Catalog says max_context=128000, admin overrides to 256000.
|
|
// Admin should win.
|
|
catalogCaps := &models.ModelCapabilities{
|
|
MaxContext: 128000,
|
|
}
|
|
overrides := []models.CapabilityOverride{
|
|
{Field: "max_context", Value: "256000"},
|
|
}
|
|
|
|
merged := ResolveIntrinsic("some-model", catalogCaps, overrides)
|
|
|
|
if merged.MaxContext != 256000 {
|
|
t.Errorf("admin override should win over catalog: expected 256000, got %d", merged.MaxContext)
|
|
}
|
|
}
|
|
|