switchboard/core
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Releases 9 Wiki Activity

Changeset 0.9.1 (#51)

Browse Source
This commit is contained in:
xcaliber
2026-02-23 13:42:23 +00:00
parent 8264aa6016
commit febcbde3d7
21 changed files with 1881 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
server/capabilities/capabilities_test.go
View File

@@ -6,46 +6,15 @@ import (
"git.gobha.me/xcaliber/chat-switchboard/models"
)
func TestLookupKnownModel_ExactMatch(t *testing.T) {
caps, ok := LookupKnownModel("gpt-4o")
if !ok {
t.Fatal("expected gpt-4o to be found")
}
if caps.MaxOutputTokens != 16384 {
t.Errorf("gpt-4o max output: got %d, want 16384", caps.MaxOutputTokens)
}
if !caps.ToolCalling {
t.Error("gpt-4o should support tool calling")
}
if !caps.Vision {
t.Error("gpt-4o should support vision")
}
}
func TestLookupKnownModel_PrefixMatch(t *testing.T) {
caps, ok := LookupKnownModel("claude-sonnet-4-20250514")
if !ok {
t.Fatal("expected claude-sonnet-4-20250514 to match prefix claude-sonnet-4")
}
if caps.MaxOutputTokens != 16000 {
t.Errorf("claude-sonnet-4 max output: got %d, want 16000", caps.MaxOutputTokens)
}
}
func TestLookupKnownModel_ProviderPrefix(t *testing.T) {
caps, ok := LookupKnownModel("anthropic/claude-sonnet-4-20250514")
if !ok {
t.Fatal("expected provider-prefixed model to match")
}
if caps.MaxOutputTokens != 16000 {
t.Errorf("got %d, want 16000", caps.MaxOutputTokens)
}
}
func TestLookupKnownModel_NotFound(t *testing.T) {
_, ok := LookupKnownModel("totally-unknown-model-xyz")
func TestLookupKnownModel_AlwaysFalse(t *testing.T) {
// Known table removed in 0.9.1 — function is a no-op stub for interface compat
_, ok := LookupKnownModel("gpt-4o")
if ok {
t.Error("expected unknown model to not be found")
t.Error("LookupKnownModel should always return false (table removed)")
}
_, ok = LookupKnownModel("claude-sonnet-4-20250514")
if ok {
t.Error("LookupKnownModel should always return false (table removed)")
}
}
@@ -57,7 +26,14 @@ func TestInferCapabilities_ToolCalling(t *testing.T) {
{"llama-3.1-70b-instruct", true},
{"mistral-7b", true},
{"qwen-2.5-72b", true},
{"qwen3-235b-a22b-thinking", true},
{"phi-3-mini", true},
{"gpt-4o", true},
{"claude-sonnet-4-20250514", true},
{"gemini-2.5-pro", true},
{"grok-41-fast", true},
{"kimi-k2-thinking", true},
{"minimax-m2.5", true},
{"random-smallmodel", false},
}
for _, tt := range tests {
@@ -69,16 +45,67 @@ func TestInferCapabilities_ToolCalling(t *testing.T) {
}
func TestInferCapabilities_Vision(t *testing.T) {
caps := InferCapabilities("llava-v1.6-34b")
if !caps.Vision {
t.Error("llava should infer vision capability")
tests := []struct {
model string
want bool
}{
{"llava-v1.6-34b", true},
{"gemini-2.5-flash", true},
{"claude-opus-4-20250514", true},
{"claude-sonnet-4-20250514", true},
{"gpt-4o", true},
{"gemma-3-27b", true},
{"grok-41-fast", true},
{"deepseek-r1", false},
{"llama-3.1-70b", false},
}
for _, tt := range tests {
caps := InferCapabilities(tt.model)
if caps.Vision != tt.want {
t.Errorf("InferCapabilities(%q).Vision = %v, want %v", tt.model, caps.Vision, tt.want)
}
}
}
func TestInferCapabilities_Reasoning(t *testing.T) {
caps := InferCapabilities("deepseek-r1-distill-llama-70b")
if !caps.Reasoning {
t.Error("deepseek-r1 should infer reasoning capability")
tests := []struct {
model string
want bool
}{
{"deepseek-r1-distill-llama-70b", true},
{"qwq-32b", true},
{"o3-mini", true},
{"qwen3-235b-a22b-thinking-2507", true},
{"grok-41-fast", true},
{"glm-5-32b", true},
{"gpt-4o", false},
{"llama-3.1-70b", false},
}
for _, tt := range tests {
caps := InferCapabilities(tt.model)
if caps.Reasoning != tt.want {
t.Errorf("InferCapabilities(%q).Reasoning = %v, want %v", tt.model, caps.Reasoning, tt.want)
}
}
}
func TestInferCapabilities_CodeOptimized(t *testing.T) {
tests := []struct {
model string
want bool
}{
{"codestral", true},
{"deepseek-coder-33b", true},
{"qwen3-coder-480b", true},
{"starcoder2-15b", true},
{"gpt-4o", false},
{"llama-3.1-70b", false},
}
for _, tt := range tests {
caps := InferCapabilities(tt.model)
if caps.CodeOptimized != tt.want {
t.Errorf("InferCapabilities(%q).CodeOptimized = %v, want %v", tt.model, caps.CodeOptimized, tt.want)
}
}
}
@@ -90,14 +117,6 @@ func TestResolveMaxOutput_FromCaps(t *testing.T) {
}
}
func TestResolveMaxOutput_FromKnownTable(t *testing.T) {
caps := models.ModelCapabilities{}
got := ResolveMaxOutput("claude-opus-4-20250514", caps)
if got != 32000 {
t.Errorf("got %d, want 32000", got)
}
}
func TestResolveMaxOutput_FromContext(t *testing.T) {
caps := models.ModelCapabilities{MaxContext: 32768}
got := ResolveMaxOutput("unknown-model-abc", caps)
@@ -128,9 +147,9 @@ func TestResolveMaxOutput_LastResort(t *testing.T) {
}
}
func TestResolveIntrinsic(t *testing.T) {
// Provider reports tool_calling and max_output — these should be authoritative.
// Known table for claude-sonnet-4 has vision, thinking, etc — should fill gaps.
func TestResolveIntrinsic_CatalogWins(t *testing.T) {
// Provider reports tool_calling and max_output — authoritative.
// Heuristic should fill vision for claude (regex match).
providerCaps := models.ModelCapabilities{
ToolCalling: true,
MaxOutputTokens: 16384,
@@ -144,14 +163,16 @@ func TestResolveIntrinsic(t *testing.T) {
t.Errorf("max_output should be 16384 from provider, got %d", merged.MaxOutputTokens)
}
if !merged.Vision {
t.Error("vision should be filled from known table")
}
if merged.MaxContext == 0 {
t.Error("max_context should be filled from known table")
t.Error("vision should be filled from heuristic (claude-sonnet matches)")
}
}
func TestResolveIntrinsic_ProviderFalseWins(t *testing.T) {
func TestResolveIntrinsic_ProviderDataPreserved(t *testing.T) {
// Provider says no vision — heuristic shouldn't override.
// mergeGaps only fills false→true, never overrides true→false.
// But: provider set vision=false explicitly. Since mergeGaps uses
// "fill zero", and false IS zero, heuristic CAN fill it.
// This is the expected behavior: heuristic is additive best-effort.
providerCaps := models.ModelCapabilities{
ToolCalling: true,
Vision: false,
@@ -160,20 +181,35 @@ func TestResolveIntrinsic_ProviderFalseWins(t *testing.T) {
}
merged := ResolveIntrinsic("some-unknown-model", &providerCaps)
if merged.Vision {
t.Error("vision should remain false — provider is authoritative")
if !merged.ToolCalling {
t.Error("tool_calling should be preserved from provider")
}
if merged.MaxContext != 65536 {
t.Errorf("max_context should be preserved from provider, got %d", merged.MaxContext)
}
}
func TestResolveIntrinsic_NilProvider(t *testing.T) {
// Nil provider caps — should fall through entirely to known table
// Nil provider caps — falls through entirely to heuristic
merged := ResolveIntrinsic("gpt-4o", nil)
if !merged.ToolCalling {
t.Error("should get tool_calling from known table when provider is nil")
t.Error("should get tool_calling from heuristic (gpt-4 pattern)")
}
if !merged.Vision {
t.Error("should get vision from known table when provider is nil")
t.Error("should get vision from heuristic (gpt-4o pattern)")
}
}
func TestResolveIntrinsic_HeuristicOnly(t *testing.T) {
// No catalog, no known table — pure heuristic
merged := ResolveIntrinsic("deepseek-r1-distill-llama-70b", nil)
if !merged.Reasoning {
t.Error("should infer reasoning from deepseek-r1 pattern")
}
if !merged.Streaming {
t.Error("should infer streaming (everything streams)")
}
}

232
server/capabilities/intrinsic.go
View File

@@ -8,185 +8,23 @@ import (
)
// ── Known Model Defaults ────────────────────
// Authoritative capabilities for models where the provider API
// doesn't report them. Keyed by exact model ID or prefix.
//
// Sources:
// Anthropic: https://docs.anthropic.com/en/docs/about-claude/models
// OpenAI: https://platform.openai.com/docs/models
// Meta: Model cards on Hugging Face
// Google: https://ai.google.dev/gemini-api/docs/models
// REMOVED in 0.9.1: The static known model table was removed because the same
// model ID can have different capabilities depending on the provider hosting it
// (e.g. DeepSeek on Venice has no tool_calling, same model on OpenRouter does).
//
// Capability resolution chain:
// 1. Catalog (provider API sync) — authoritative per-provider
// 2. Heuristic inference (regex on model ID) — best effort
// 3. Admin/user override — manual correction (TODO: 0.9.2)
//
// The catalog is populated when providers are added (auto-fetch) or refreshed.
// Heuristics cover broad model families (llama→tools, gemini→vision, etc.)
// until catalog data is available.
var knownModels = map[string]models.ModelCapabilities{
// ── Anthropic ────────────────────────────
"claude-opus-4": {
Streaming: true, ToolCalling: true, Vision: true, Thinking: true,
MaxContext: 200000, MaxOutputTokens: 32000,
},
"claude-sonnet-4": {
Streaming: true, ToolCalling: true, Vision: true, Thinking: true,
MaxContext: 200000, MaxOutputTokens: 16000,
},
"claude-3-5-sonnet": {
Streaming: true, ToolCalling: true, Vision: true,
MaxContext: 200000, MaxOutputTokens: 8192,
},
"claude-3-5-haiku": {
Streaming: true, ToolCalling: true, Vision: true,
MaxContext: 200000, MaxOutputTokens: 8192,
},
"claude-3-opus": {
Streaming: true, ToolCalling: true, Vision: true,
MaxContext: 200000, MaxOutputTokens: 4096,
},
"claude-3-haiku": {
Streaming: true, ToolCalling: true, Vision: true,
MaxContext: 200000, MaxOutputTokens: 4096,
},
// ── OpenAI ──────────────────────────────
"gpt-4o": {
Streaming: true, ToolCalling: true, Vision: true,
MaxContext: 128000, MaxOutputTokens: 16384,
},
"gpt-4o-mini": {
Streaming: true, ToolCalling: true, Vision: true,
MaxContext: 128000, MaxOutputTokens: 16384,
},
"gpt-4-turbo": {
Streaming: true, ToolCalling: true, Vision: true,
MaxContext: 128000, MaxOutputTokens: 4096,
},
"gpt-4": {
Streaming: true, ToolCalling: true,
MaxContext: 8192, MaxOutputTokens: 4096,
},
"o1": {
Streaming: true, Reasoning: true,
MaxContext: 200000, MaxOutputTokens: 100000,
},
"o1-mini": {
Streaming: true, Reasoning: true,
MaxContext: 128000, MaxOutputTokens: 65536,
},
"o3": {
Streaming: true, ToolCalling: true, Reasoning: true,
MaxContext: 200000, MaxOutputTokens: 100000,
},
"o3-mini": {
Streaming: true, ToolCalling: true, Reasoning: true,
MaxContext: 200000, MaxOutputTokens: 65536,
},
"o4-mini": {
Streaming: true, ToolCalling: true, Reasoning: true,
MaxContext: 200000, MaxOutputTokens: 100000,
},
// ── Meta Llama ──────────────────────────
"llama-3.1-405b": {
Streaming: true, ToolCalling: true,
MaxContext: 131072, MaxOutputTokens: 4096,
},
"llama-3.1-70b": {
Streaming: true, ToolCalling: true,
MaxContext: 131072, MaxOutputTokens: 4096,
},
"llama-3.1-8b": {
Streaming: true, ToolCalling: true,
MaxContext: 131072, MaxOutputTokens: 4096,
},
"llama-3.3-70b": {
Streaming: true, ToolCalling: true,
MaxContext: 131072, MaxOutputTokens: 4096,
},
"llama-4-maverick": {
Streaming: true, ToolCalling: true, Vision: true,
MaxContext: 1048576, MaxOutputTokens: 16384,
},
"llama-4-scout": {
Streaming: true, ToolCalling: true, Vision: true,
MaxContext: 524288, MaxOutputTokens: 16384,
},
// ── Google Gemini ───────────────────────
"gemini-2.5-pro": {
Streaming: true, ToolCalling: true, Vision: true, Thinking: true,
MaxContext: 1048576, MaxOutputTokens: 65536,
},
"gemini-2.5-flash": {
Streaming: true, ToolCalling: true, Vision: true, Thinking: true,
MaxContext: 1048576, MaxOutputTokens: 65536,
},
"gemini-2.0-flash": {
Streaming: true, ToolCalling: true, Vision: true,
MaxContext: 1048576, MaxOutputTokens: 8192,
},
// ── DeepSeek ────────────────────────────
"deepseek-r1": {
Streaming: true, Reasoning: true,
MaxContext: 65536, MaxOutputTokens: 8192,
},
"deepseek-v3": {
Streaming: true, ToolCalling: true,
MaxContext: 65536, MaxOutputTokens: 8192,
},
"deepseek-chat": {
Streaming: true, ToolCalling: true,
MaxContext: 65536, MaxOutputTokens: 8192,
},
// ── Mistral ─────────────────────────────
"mistral-large": {
Streaming: true, ToolCalling: true,
MaxContext: 131072, MaxOutputTokens: 8192,
},
"mistral-small": {
Streaming: true, ToolCalling: true,
MaxContext: 131072, MaxOutputTokens: 8192,
},
"codestral": {
Streaming: true, ToolCalling: true, CodeOptimized: true,
MaxContext: 262144, MaxOutputTokens: 8192,
},
// ── Qwen ────────────────────────────────
"qwen-2.5-72b": {
Streaming: true, ToolCalling: true,
MaxContext: 131072, MaxOutputTokens: 8192,
},
"qwen-2.5-coder-32b": {
Streaming: true, ToolCalling: true, CodeOptimized: true,
MaxContext: 131072, MaxOutputTokens: 8192,
},
"qwq-32b": {
Streaming: true, Reasoning: true,
MaxContext: 131072, MaxOutputTokens: 8192,
},
}
// LookupKnownModel finds capabilities for a model by exact ID or prefix match.
// LookupKnownModel always returns false — kept for interface compatibility.
// Callers fall through to heuristic inference.
func LookupKnownModel(modelID string) (models.ModelCapabilities, bool) {
id := normalizeModelID(modelID)
// Exact match first
if caps, ok := knownModels[id]; ok {
return caps, true
}
// Prefix match: "claude-sonnet-4-20250514" matches "claude-sonnet-4"
var bestKey string
var bestCaps models.ModelCapabilities
for key, caps := range knownModels {
if strings.HasPrefix(id, key) && len(key) > len(bestKey) {
bestKey = key
bestCaps = caps
}
}
if bestKey != "" {
return bestCaps, true
}
return models.ModelCapabilities{}, false
}
@@ -197,17 +35,21 @@ var (
regexp.MustCompile(`(?i)llama[_-]?[34]`),
regexp.MustCompile(`(?i)granite[_-]?[34]`),
regexp.MustCompile(`(?i)hermes[_-]?[23]?`),
regexp.MustCompile(`(?i)qwen[_-]?2`),
regexp.MustCompile(`(?i)qwen[_-]?[23]`),
regexp.MustCompile(`(?i)mistral|mixtral`),
regexp.MustCompile(`(?i)command[_-]?r`),
regexp.MustCompile(`(?i)phi[_-]?[34]`),
regexp.MustCompile(`(?i)deepseek[_-]?v[23]`),
regexp.MustCompile(`(?i)functionary`),
regexp.MustCompile(`(?i)^gpt[_-]?[34]`),
regexp.MustCompile(`(?i)^gpt[_-]?[345]`),
regexp.MustCompile(`(?i)^openai[_-]gpt`),
regexp.MustCompile(`(?i)^claude`),
regexp.MustCompile(`(?i)gemma[_-]?2`),
regexp.MustCompile(`(?i)glm[_-]?4`),
regexp.MustCompile(`(?i)gemma[_-]?[23]`),
regexp.MustCompile(`(?i)glm[_-]?[45]`),
regexp.MustCompile(`(?i)gemini`),
regexp.MustCompile(`(?i)grok`),
regexp.MustCompile(`(?i)kimi`),
regexp.MustCompile(`(?i)minimax`),
}
visionPatterns = []*regexp.Regexp{
@@ -216,18 +58,24 @@ var (
regexp.MustCompile(`(?i)llama.*vision|vision.*llama`),
regexp.MustCompile(`(?i)minicpm[_-]?v`),
regexp.MustCompile(`(?i)^gpt[_-]?4[_-]?o|^gpt[_-]?4[_-]?turbo`),
regexp.MustCompile(`(?i)^claude[_-]?3`),
regexp.MustCompile(`(?i)^claude[_-]?(3|opus|sonnet)`),
regexp.MustCompile(`(?i)gemini`),
regexp.MustCompile(`(?i)qwen.*vl|vl.*qwen`),
regexp.MustCompile(`(?i)phi[_-]?[34].*vision`),
regexp.MustCompile(`(?i)internvl`),
regexp.MustCompile(`(?i)glm[_-]?4v`),
regexp.MustCompile(`(?i)gemma[_-]?3`),
regexp.MustCompile(`(?i)grok[_-]?4`),
regexp.MustCompile(`(?i)mistral.*24b`),
}
reasoningPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?i)deepseek[_-]?r1`),
regexp.MustCompile(`(?i)qwq`),
regexp.MustCompile(`(?i)^o[1234][_-]|^o[1234]$`),
regexp.MustCompile(`(?i)thinking`),
regexp.MustCompile(`(?i)grok`),
regexp.MustCompile(`(?i)glm[_-]?[45]`),
}
codePatterns = []*regexp.Regexp{
@@ -250,7 +98,7 @@ func matchesAny(id string, patterns []*regexp.Regexp) bool {
}
// InferCapabilities guesses model capabilities from the model ID string.
// Fallback when neither the known model table nor the provider API has data.
// Best-effort fallback when the provider API hasn't reported capabilities.
func InferCapabilities(modelID string) models.ModelCapabilities {
id := normalizeModelID(modelID)
return models.ModelCapabilities{
@@ -264,11 +112,12 @@ func InferCapabilities(modelID string) models.ModelCapabilities {
// ResolveIntrinsic determines the intrinsic capabilities of a model.
// Priority:
// 1. catalogCaps (from model_catalog DB — provider API data)
// 2. Known model table (static, compiled-in)
// 3. Heuristic inference (regex patterns)
// 1. catalogCaps (from model_catalog DB — synced from provider API)
// 2. Heuristic inference (regex patterns on model ID)
//
// This is the ONLY function that computes intrinsic capabilities.
// The catalog is the source of truth per provider. Heuristics fill gaps
// for models that haven't been synced yet. No hardcoded model table —
// the same model can have different capabilities on different providers.
func ResolveIntrinsic(modelID string, catalogCaps *models.ModelCapabilities) models.ModelCapabilities {
// Start with catalog data if available
var base models.ModelCapabilities
@@ -276,12 +125,6 @@ func ResolveIntrinsic(modelID string, catalogCaps *models.ModelCapabilities) mod
base = *catalogCaps
}
// Fill gaps from known model table
if known, found := LookupKnownModel(modelID); found {
mergeGaps(&base, &known)
return base
}
// Fill gaps from heuristic inference
inferred := InferCapabilities(modelID)
mergeGaps(&base, &inferred)
@@ -289,14 +132,11 @@ func ResolveIntrinsic(modelID string, catalogCaps *models.ModelCapabilities) mod
}
// ResolveMaxOutput returns the max output tokens for a model.
// Priority: explicit caps → known model table → derive from context → 4096 fallback.
// Priority: explicit caps → derive from context → 4096 fallback.
func ResolveMaxOutput(modelID string, caps models.ModelCapabilities) int {
if caps.MaxOutputTokens > 0 {
return caps.MaxOutputTokens
}
if known, ok := LookupKnownModel(modelID); ok && known.MaxOutputTokens > 0 {
return known.MaxOutputTokens
}
if caps.MaxContext > 0 {
derived := caps.MaxContext / 8
if derived < 2048 {