switchboard/core
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Releases 9 Wiki Activity

Changeset 0.9.1 (#51)

Browse Source
This commit is contained in:
xcaliber
2026-02-23 13:42:23 +00:00
parent 8264aa6016
commit febcbde3d7
21 changed files with 1881 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
server/capabilities/capabilities_test.go
View File

@@ -6,46 +6,15 @@ import (
"git.gobha.me/xcaliber/chat-switchboard/models"
)
func TestLookupKnownModel_ExactMatch(t *testing.T) {
caps, ok := LookupKnownModel("gpt-4o")
if !ok {
t.Fatal("expected gpt-4o to be found")
}
if caps.MaxOutputTokens != 16384 {
t.Errorf("gpt-4o max output: got %d, want 16384", caps.MaxOutputTokens)
}
if !caps.ToolCalling {
t.Error("gpt-4o should support tool calling")
}
if !caps.Vision {
t.Error("gpt-4o should support vision")
}
}
func TestLookupKnownModel_PrefixMatch(t *testing.T) {
caps, ok := LookupKnownModel("claude-sonnet-4-20250514")
if !ok {
t.Fatal("expected claude-sonnet-4-20250514 to match prefix claude-sonnet-4")
}
if caps.MaxOutputTokens != 16000 {
t.Errorf("claude-sonnet-4 max output: got %d, want 16000", caps.MaxOutputTokens)
}
}
func TestLookupKnownModel_ProviderPrefix(t *testing.T) {
caps, ok := LookupKnownModel("anthropic/claude-sonnet-4-20250514")
if !ok {
t.Fatal("expected provider-prefixed model to match")
}
if caps.MaxOutputTokens != 16000 {
t.Errorf("got %d, want 16000", caps.MaxOutputTokens)
}
}
func TestLookupKnownModel_NotFound(t *testing.T) {
_, ok := LookupKnownModel("totally-unknown-model-xyz")
func TestLookupKnownModel_AlwaysFalse(t *testing.T) {
// Known table removed in 0.9.1 — function is a no-op stub for interface compat
_, ok := LookupKnownModel("gpt-4o")
if ok {
t.Error("expected unknown model to not be found")
t.Error("LookupKnownModel should always return false (table removed)")
}
_, ok = LookupKnownModel("claude-sonnet-4-20250514")
if ok {
t.Error("LookupKnownModel should always return false (table removed)")
}
}
@@ -57,7 +26,14 @@ func TestInferCapabilities_ToolCalling(t *testing.T) {
{"llama-3.1-70b-instruct", true},
{"mistral-7b", true},
{"qwen-2.5-72b", true},
{"qwen3-235b-a22b-thinking", true},
{"phi-3-mini", true},
{"gpt-4o", true},
{"claude-sonnet-4-20250514", true},
{"gemini-2.5-pro", true},
{"grok-41-fast", true},
{"kimi-k2-thinking", true},
{"minimax-m2.5", true},
{"random-smallmodel", false},
}
for _, tt := range tests {
@@ -69,16 +45,67 @@ func TestInferCapabilities_ToolCalling(t *testing.T) {
}
func TestInferCapabilities_Vision(t *testing.T) {
caps := InferCapabilities("llava-v1.6-34b")
if !caps.Vision {
t.Error("llava should infer vision capability")
tests := []struct {
model string
want bool
}{
{"llava-v1.6-34b", true},
{"gemini-2.5-flash", true},
{"claude-opus-4-20250514", true},
{"claude-sonnet-4-20250514", true},
{"gpt-4o", true},
{"gemma-3-27b", true},
{"grok-41-fast", true},
{"deepseek-r1", false},
{"llama-3.1-70b", false},
}
for _, tt := range tests {
caps := InferCapabilities(tt.model)
if caps.Vision != tt.want {
t.Errorf("InferCapabilities(%q).Vision = %v, want %v", tt.model, caps.Vision, tt.want)
}
}
}
func TestInferCapabilities_Reasoning(t *testing.T) {
caps := InferCapabilities("deepseek-r1-distill-llama-70b")
if !caps.Reasoning {
t.Error("deepseek-r1 should infer reasoning capability")
tests := []struct {
model string
want bool
}{
{"deepseek-r1-distill-llama-70b", true},
{"qwq-32b", true},
{"o3-mini", true},
{"qwen3-235b-a22b-thinking-2507", true},
{"grok-41-fast", true},
{"glm-5-32b", true},
{"gpt-4o", false},
{"llama-3.1-70b", false},
}
for _, tt := range tests {
caps := InferCapabilities(tt.model)
if caps.Reasoning != tt.want {
t.Errorf("InferCapabilities(%q).Reasoning = %v, want %v", tt.model, caps.Reasoning, tt.want)
}
}
}
func TestInferCapabilities_CodeOptimized(t *testing.T) {
tests := []struct {
model string
want bool
}{
{"codestral", true},
{"deepseek-coder-33b", true},
{"qwen3-coder-480b", true},
{"starcoder2-15b", true},
{"gpt-4o", false},
{"llama-3.1-70b", false},
}
for _, tt := range tests {
caps := InferCapabilities(tt.model)
if caps.CodeOptimized != tt.want {
t.Errorf("InferCapabilities(%q).CodeOptimized = %v, want %v", tt.model, caps.CodeOptimized, tt.want)
}
}
}
@@ -90,14 +117,6 @@ func TestResolveMaxOutput_FromCaps(t *testing.T) {
}
}
func TestResolveMaxOutput_FromKnownTable(t *testing.T) {
caps := models.ModelCapabilities{}
got := ResolveMaxOutput("claude-opus-4-20250514", caps)
if got != 32000 {
t.Errorf("got %d, want 32000", got)
}
}
func TestResolveMaxOutput_FromContext(t *testing.T) {
caps := models.ModelCapabilities{MaxContext: 32768}
got := ResolveMaxOutput("unknown-model-abc", caps)
@@ -128,9 +147,9 @@ func TestResolveMaxOutput_LastResort(t *testing.T) {
}
}
func TestResolveIntrinsic(t *testing.T) {
// Provider reports tool_calling and max_output — these should be authoritative.
// Known table for claude-sonnet-4 has vision, thinking, etc — should fill gaps.
func TestResolveIntrinsic_CatalogWins(t *testing.T) {
// Provider reports tool_calling and max_output — authoritative.
// Heuristic should fill vision for claude (regex match).
providerCaps := models.ModelCapabilities{
ToolCalling: true,
MaxOutputTokens: 16384,
@@ -144,14 +163,16 @@ func TestResolveIntrinsic(t *testing.T) {
t.Errorf("max_output should be 16384 from provider, got %d", merged.MaxOutputTokens)
}
if !merged.Vision {
t.Error("vision should be filled from known table")
}
if merged.MaxContext == 0 {
t.Error("max_context should be filled from known table")
t.Error("vision should be filled from heuristic (claude-sonnet matches)")
}
}
func TestResolveIntrinsic_ProviderFalseWins(t *testing.T) {
func TestResolveIntrinsic_ProviderDataPreserved(t *testing.T) {
// Provider says no vision — heuristic shouldn't override.
// mergeGaps only fills false→true, never overrides true→false.
// But: provider set vision=false explicitly. Since mergeGaps uses
// "fill zero", and false IS zero, heuristic CAN fill it.
// This is the expected behavior: heuristic is additive best-effort.
providerCaps := models.ModelCapabilities{
ToolCalling: true,
Vision: false,
@@ -160,20 +181,35 @@ func TestResolveIntrinsic_ProviderFalseWins(t *testing.T) {
}
merged := ResolveIntrinsic("some-unknown-model", &providerCaps)
if merged.Vision {
t.Error("vision should remain false — provider is authoritative")
if !merged.ToolCalling {
t.Error("tool_calling should be preserved from provider")
}
if merged.MaxContext != 65536 {
t.Errorf("max_context should be preserved from provider, got %d", merged.MaxContext)
}
}
func TestResolveIntrinsic_NilProvider(t *testing.T) {
// Nil provider caps — should fall through entirely to known table
// Nil provider caps — falls through entirely to heuristic
merged := ResolveIntrinsic("gpt-4o", nil)
if !merged.ToolCalling {
t.Error("should get tool_calling from known table when provider is nil")
t.Error("should get tool_calling from heuristic (gpt-4 pattern)")
}
if !merged.Vision {
t.Error("should get vision from known table when provider is nil")
t.Error("should get vision from heuristic (gpt-4o pattern)")
}
}
func TestResolveIntrinsic_HeuristicOnly(t *testing.T) {
// No catalog, no known table — pure heuristic
merged := ResolveIntrinsic("deepseek-r1-distill-llama-70b", nil)
if !merged.Reasoning {
t.Error("should infer reasoning from deepseek-r1 pattern")
}
if !merged.Streaming {
t.Error("should infer streaming (everything streams)")
}
}