This repository has been archived on 2026-04-03. You can view files and clone it. You cannot open issues or pull requests or push a commit.
Files
core/server/handlers/live_provider_test.go
2026-02-24 14:50:53 +00:00

654 lines
23 KiB
Go

package handlers
import (
"context"
"fmt"
"net/http"
"os"
"testing"
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/providers"
)
// ═══════════════════════════════════════════
// Live Provider Integration Tests
// ═══════════════════════════════════════════
// These tests require:
// - TEST_DATABASE_URL or PGHOST+PGUSER
// - VENICE_API_KEY secret
//
// They exercise the full flow: create provider →
// fetch models → enable model → resolve → complete.
//
// Model: qwen3-4b (Venice Small) — cheapest at $0.05/$0.15 per 1M tokens
// ═══════════════════════════════════════════
const veniceTestModel = "qwen3-4b"
func requireVeniceKey(t *testing.T) string {
t.Helper()
key := os.Getenv("VENICE_API_KEY")
if key == "" {
t.Skip("VENICE_API_KEY not set — skipping live provider test")
}
return key
}
// setupVeniceWithModel creates a Venice provider, fetches models, and enables
// the specified model. Returns (configID, catalogEntryID).
func setupVeniceWithModel(t *testing.T, h *testHarness, adminToken, apiKey, modelID string) (string, string) {
t.Helper()
// Create provider
w := h.request("POST", "/api/v1/admin/configs", adminToken, map[string]interface{}{
"name": "Venice Test", "provider": "venice",
"endpoint": "https://api.venice.ai/api/v1", "api_key": apiKey,
})
if w.Code != http.StatusCreated {
t.Fatalf("create venice config: want 201, got %d: %s", w.Code, w.Body.String())
}
var cfg map[string]interface{}
decode(w, &cfg)
configID := cfg["id"].(string)
// Fetch models
w = h.request("POST", "/api/v1/admin/models/fetch", adminToken,
map[string]interface{}{"provider_config_id": configID})
if w.Code != http.StatusOK {
t.Fatalf("fetch models: %d: %s", w.Code, w.Body.String())
}
// Find and enable target model
w = h.request("GET", "/api/v1/admin/models", adminToken, nil)
var modelsResp map[string]interface{}
decode(w, &modelsResp)
var catalogID string
for _, raw := range modelsResp["models"].([]interface{}) {
m := raw.(map[string]interface{})
if m["model_id"].(string) == modelID {
catalogID = m["id"].(string)
break
}
}
if catalogID == "" {
t.Fatalf("model %s not found in Venice catalog", modelID)
}
w = h.request("PUT", "/api/v1/admin/models/"+catalogID, adminToken,
map[string]interface{}{"visibility": "enabled"})
if w.Code != http.StatusOK {
t.Fatalf("enable model: %d: %s", w.Code, w.Body.String())
}
t.Logf(" Venice provider %s ready, model %s enabled", configID, modelID)
return configID, catalogID
}
// TestLive_VeniceProviderFullFlow exercises the complete admin workflow:
// create provider → fetch models → enable a model → user sees it → chat completion
func TestLive_VeniceProviderFullFlow(t *testing.T) {
h := setupHarness(t)
veniceKey := requireVeniceKey(t)
_, adminToken := h.createAdminUser("admin", "admin@test.com")
// ── 1. Create Venice provider config ────
t.Log("Step 1: Creating Venice provider config")
w := h.request("POST", "/api/v1/admin/configs", adminToken, map[string]interface{}{
"name": "Venice Live Test",
"provider": "venice",
"endpoint": "https://api.venice.ai/api/v1",
"api_key": veniceKey,
})
if w.Code != http.StatusCreated {
t.Fatalf("create venice config: want 201, got %d: %s", w.Code, w.Body.String())
}
var configResp map[string]interface{}
decode(w, &configResp)
configID := configResp["id"].(string)
t.Logf(" Created config: %s", configID)
// ── 2. Fetch models from Venice ─────────
t.Log("Step 2: Fetching models from Venice API")
w = h.request("POST", "/api/v1/admin/models/fetch", adminToken, map[string]interface{}{
"provider_config_id": configID,
})
if w.Code != http.StatusOK {
t.Fatalf("fetch models: want 200, got %d: %s", w.Code, w.Body.String())
}
var fetchResp map[string]interface{}
decode(w, &fetchResp)
totalFetched := fetchResp["total"].(float64)
if totalFetched < 1 {
t.Fatalf("Venice should return at least 1 model, got %.0f", totalFetched)
}
t.Logf(" Fetched %.0f models from Venice", totalFetched)
// ── 3. List catalog models (all disabled by default) ──
t.Log("Step 3: Listing catalog models")
w = h.request("GET", "/api/v1/admin/models", adminToken, nil)
if w.Code != http.StatusOK {
t.Fatalf("list models: want 200, got %d: %s", w.Code, w.Body.String())
}
var modelsResp map[string]interface{}
decode(w, &modelsResp)
catalogModels := modelsResp["models"].([]interface{})
if len(catalogModels) < 1 {
t.Fatal("catalog should have models after fetch")
}
// Find a text model to enable (prefer a small/fast one)
var enableID string
var enableModelID string
for _, raw := range catalogModels {
m := raw.(map[string]interface{})
modelID := m["model_id"].(string)
vis := m["visibility"].(string)
if vis == "disabled" {
enableID = m["id"].(string)
enableModelID = modelID
break
}
}
if enableID == "" {
t.Fatal("no disabled model found to enable")
}
t.Logf(" Will enable: %s (catalog ID: %s)", enableModelID, enableID)
// ── 4. Enable the model ─────────────────
t.Log("Step 4: Enabling model")
w = h.request("PUT", "/api/v1/admin/models/"+enableID, adminToken,
map[string]interface{}{"visibility": "enabled"})
if w.Code != http.StatusOK {
t.Fatalf("enable model: want 200, got %d: %s", w.Code, w.Body.String())
}
// ── 5. Verify models/enabled returns it (admin) ──
t.Log("Step 5: Verifying models/enabled (admin)")
w = h.request("GET", "/api/v1/models/enabled", adminToken, nil)
if w.Code != http.StatusOK {
t.Fatalf("models/enabled: want 200, got %d: %s", w.Code, w.Body.String())
}
var enabledResp map[string]interface{}
decode(w, &enabledResp)
enabledModels := enabledResp["models"].([]interface{})
if len(enabledModels) < 1 {
t.Fatal("models/enabled should return at least 1 model after enabling")
}
// Verify our model is in the list
found := false
for _, raw := range enabledModels {
m := raw.(map[string]interface{})
if m["model_id"] == enableModelID {
found = true
t.Logf(" ✓ Found %s in enabled models", enableModelID)
// Verify it has the required fields for the frontend
if m["config_id"] == nil || m["config_id"] == "" {
t.Error("enabled model must have config_id for composite ID")
}
if m["provider_name"] == nil || m["provider_name"] == "" {
t.Error("enabled model must have provider_name for display")
}
break
}
}
if !found {
t.Errorf("enabled model %s not found in models/enabled response", enableModelID)
}
// ── 6. Verify a REGULAR USER also sees the model ──
t.Log("Step 6: Verifying models/enabled (regular user)")
userID := database.SeedTestUser(t, "liveuser", "liveuser@test.com")
database.TestDB.Exec("UPDATE users SET is_active = true WHERE id = $1", userID)
userToken := makeToken(userID, "liveuser@test.com", "user")
w = h.request("GET", "/api/v1/models/enabled", userToken, nil)
if w.Code != http.StatusOK {
t.Fatalf("user models/enabled: want 200, got %d: %s", w.Code, w.Body.String())
}
var userResp map[string]interface{}
decode(w, &userResp)
userModels := userResp["models"].([]interface{})
if len(userModels) < 1 {
t.Fatalf("regular user should see at least 1 enabled model, got %d — "+
"admin can see models but regular user cannot; check ListVisible query",
len(userModels))
}
userFound := false
for _, raw := range userModels {
m := raw.(map[string]interface{})
if m["model_id"] == enableModelID {
userFound = true
t.Logf(" ✓ Regular user can see %s", enableModelID)
// Verify same fields available for regular user
if m["config_id"] == nil || m["config_id"] == "" {
t.Error("user: enabled model must have config_id")
}
if m["provider_name"] == nil || m["provider_name"] == "" {
t.Error("user: enabled model must have provider_name")
}
break
}
}
if !userFound {
t.Errorf("regular user cannot see %s — admin→user visibility broken", enableModelID)
}
}
// TestLive_VeniceFetchModelsCapabilities verifies that Venice model
// capabilities are correctly parsed into the catalog.
func TestLive_VeniceFetchModelsCapabilities(t *testing.T) {
h := setupHarness(t)
veniceKey := requireVeniceKey(t)
_, adminToken := h.createAdminUser("admin", "admin@test.com")
// Create provider + fetch
w := h.request("POST", "/api/v1/admin/configs", adminToken, map[string]interface{}{
"name": "Venice Caps Test", "provider": "venice",
"endpoint": "https://api.venice.ai/api/v1", "api_key": veniceKey,
})
var cfg map[string]interface{}
decode(w, &cfg)
configID := cfg["id"].(string)
h.request("POST", "/api/v1/admin/models/fetch", adminToken,
map[string]interface{}{"provider_config_id": configID})
// Read catalog and check capabilities
w = h.request("GET", "/api/v1/admin/models", adminToken, nil)
var resp map[string]interface{}
decode(w, &resp)
for _, raw := range resp["models"].([]interface{}) {
m := raw.(map[string]interface{})
caps, ok := m["capabilities"].(map[string]interface{})
if !ok {
t.Errorf("model %s: capabilities must be an object", m["model_id"])
continue
}
// streaming should always be true for Venice
if caps["streaming"] != true {
t.Errorf("model %s: streaming should be true", m["model_id"])
}
// Verify capabilities are actual booleans (not strings)
for _, key := range []string{"streaming", "vision", "tool_calling", "reasoning"} {
if v, exists := caps[key]; exists {
if _, ok := v.(bool); !ok {
t.Errorf("model %s: capability %s should be bool, got %T", m["model_id"], key, v)
}
}
}
}
}
// TestLive_VeniceChatCompletion sends an actual non-streaming chat completion
// using the cheapest model (qwen3-4b = $0.05/$0.15 per 1M tokens).
func TestLive_VeniceChatCompletion(t *testing.T) {
h := setupHarness(t)
veniceKey := requireVeniceKey(t)
_, adminToken := h.createAdminUser("admin", "admin@test.com")
configID, _ := setupVeniceWithModel(t, h, adminToken, veniceKey, veniceTestModel)
// Create channel
w := h.request("POST", "/api/v1/channels", adminToken, map[string]interface{}{
"title": "Chat Test", "type": "direct",
})
if w.Code != http.StatusCreated {
t.Skipf("channel creation failed: %d %s", w.Code, w.Body.String())
}
var ch map[string]interface{}
decode(w, &ch)
channelID := ch["id"].(string)
// Non-streaming completion with correct field names
stream := false
w = h.request("POST", "/api/v1/chat/completions", adminToken, map[string]interface{}{
"channel_id": channelID,
"content": "Say ok",
"model": veniceTestModel,
"provider_config_id": configID,
"stream": &stream,
"max_tokens": 10,
})
if w.Code != http.StatusOK {
t.Fatalf("completion: want 200, got %d: %s", w.Code, w.Body.String())
}
t.Logf(" ✓ Completion succeeded: %s", w.Body.String()[:min(200, w.Body.Len())])
}
// TestLive_VeniceUsageLogging verifies that a non-streaming completion
// creates a usage_log row with token counts from the provider.
func TestLive_VeniceUsageLogging(t *testing.T) {
h := setupHarness(t)
veniceKey := requireVeniceKey(t)
_, adminToken := h.createAdminUser("admin", "admin@test.com")
configID, _ := setupVeniceWithModel(t, h, adminToken, veniceKey, veniceTestModel)
// Create channel
w := h.request("POST", "/api/v1/channels", adminToken, map[string]interface{}{
"title": "Usage Test", "type": "direct",
})
if w.Code != http.StatusCreated {
t.Skipf("channel creation failed: %d %s", w.Code, w.Body.String())
}
var ch map[string]interface{}
decode(w, &ch)
channelID := ch["id"].(string)
// Non-streaming — providers reliably return usage in non-streaming mode
stream := false
w = h.request("POST", "/api/v1/chat/completions", adminToken, map[string]interface{}{
"channel_id": channelID,
"content": "Say ok",
"model": veniceTestModel,
"provider_config_id": configID,
"stream": &stream,
"max_tokens": 10,
})
if w.Code != http.StatusOK {
t.Fatalf("completion: %d: %s", w.Code, w.Body.String())
}
// Verify usage_log row exists
var rowCount int
var promptTokens, completionTokens int
err := database.TestDB.QueryRow(`
SELECT COUNT(*), COALESCE(SUM(prompt_tokens), 0), COALESCE(SUM(completion_tokens), 0)
FROM usage_log WHERE provider_config_id = $1
`, configID).Scan(&rowCount, &promptTokens, &completionTokens)
if err != nil {
t.Fatalf("query usage_log: %v", err)
}
if rowCount == 0 {
t.Fatal("usage_log should have a row after non-streaming completion")
}
t.Logf(" ✓ Usage logged: %d row(s), prompt=%d completion=%d", rowCount, promptTokens, completionTokens)
if promptTokens == 0 {
t.Fatal("non-streaming completion should report prompt tokens — check provider response parsing")
}
t.Logf(" ✓ Token counts: prompt=%d completion=%d", promptTokens, completionTokens)
}
// TestLive_VeniceStreamingUsageLogging verifies that streaming completions
// create a usage_log row with actual token counts. Venice supports
// stream_options.include_usage — the parser must capture the usage chunk
// that arrives after finish_reason but before [DONE].
func TestLive_VeniceStreamingUsageLogging(t *testing.T) {
h := setupHarness(t)
veniceKey := requireVeniceKey(t)
_, adminToken := h.createAdminUser("admin", "admin@test.com")
configID, _ := setupVeniceWithModel(t, h, adminToken, veniceKey, veniceTestModel)
// Create channel
w := h.request("POST", "/api/v1/channels", adminToken, map[string]interface{}{
"title": "Stream Usage Test", "type": "direct",
})
if w.Code != http.StatusCreated {
t.Skipf("channel creation failed: %d %s", w.Code, w.Body.String())
}
var ch map[string]interface{}
decode(w, &ch)
channelID := ch["id"].(string)
// Streaming completion
stream := true
w = h.request("POST", "/api/v1/chat/completions", adminToken, map[string]interface{}{
"channel_id": channelID,
"content": "Say ok",
"model": veniceTestModel,
"provider_config_id": configID,
"stream": &stream,
"max_tokens": 10,
})
// Streaming returns 200 with SSE — the recorder captures the full body
if w.Code != http.StatusOK {
t.Fatalf("streaming completion: %d: %s", w.Code, w.Body.String())
}
// Verify usage_log row exists (even if tokens are 0)
var rowCount int
var promptTokens, completionTokens int
err := database.TestDB.QueryRow(`
SELECT COUNT(*), COALESCE(SUM(prompt_tokens), 0), COALESCE(SUM(completion_tokens), 0)
FROM usage_log WHERE provider_config_id = $1
`, configID).Scan(&rowCount, &promptTokens, &completionTokens)
if err != nil {
t.Fatalf("query usage_log: %v", err)
}
if rowCount == 0 {
t.Fatal("usage_log should have a row after streaming completion")
}
if promptTokens == 0 {
t.Fatal("streaming completion should report prompt tokens — check pendingFinish logic in openai.go parser")
}
t.Logf(" ✓ Streaming usage logged: %d row(s), prompt=%d completion=%d", rowCount, promptTokens, completionTokens)
}
// TestLive_VenicePricingFromCatalog verifies that model sync populates
// the model_pricing table from Venice's pricing data.
func TestLive_VenicePricingFromCatalog(t *testing.T) {
h := setupHarness(t)
veniceKey := requireVeniceKey(t)
_, adminToken := h.createAdminUser("admin", "admin@test.com")
configID, _ := setupVeniceWithModel(t, h, adminToken, veniceKey, veniceTestModel)
// Check if pricing was populated during fetch
var pricingCount int
database.TestDB.QueryRow(`
SELECT COUNT(*) FROM model_pricing WHERE provider_config_id = $1
`, configID).Scan(&pricingCount)
if pricingCount == 0 {
t.Skip("Venice model sync did not populate pricing — may need provider pricing support")
}
t.Logf(" ✓ Catalog sync populated %d pricing entries", pricingCount)
// Verify our test model has pricing
var inputPerM, outputPerM float64
err := database.TestDB.QueryRow(`
SELECT COALESCE(input_per_m, 0), COALESCE(output_per_m, 0)
FROM model_pricing
WHERE provider_config_id = $1 AND model_id = $2
`, configID, veniceTestModel).Scan(&inputPerM, &outputPerM)
if err != nil {
t.Logf(" ⚠ No pricing for %s specifically (may use different model ID)", veniceTestModel)
} else {
t.Logf(" ✓ %s pricing: $%.4f input, $%.4f output (per 1M)", veniceTestModel, inputPerM, outputPerM)
}
// Admin pricing list should show these (scope=global, not personal)
w := h.request("GET", "/api/v1/admin/pricing", adminToken, nil)
if w.Code != http.StatusOK {
t.Fatalf("admin pricing: %d: %s", w.Code, w.Body.String())
}
var entries []interface{}
decode(w, &entries)
if len(entries) == 0 {
t.Error("admin pricing API should return catalog-synced entries")
} else {
t.Logf(" ✓ Admin pricing API returns %d entries", len(entries))
}
}
// TestLive_VeniceEmbeddings tests the Venice embeddings endpoint directly
// using the BGE-M3 model ($0.15 per 1M tokens).
func TestLive_VeniceEmbeddings(t *testing.T) {
veniceKey := requireVeniceKey(t)
provider := &providers.VeniceProvider{}
cfg := providers.ProviderConfig{
Endpoint: "https://api.venice.ai/api/v1",
APIKey: veniceKey,
}
resp, err := provider.Embed(
context.Background(), cfg,
providers.EmbeddingRequest{
Model: "text-embedding-bge-m3",
Input: []string{"test embedding"},
},
)
if err != nil {
t.Fatalf("Venice Embed: %v", err)
}
if len(resp.Embeddings) == 0 {
t.Fatal("expected at least 1 embedding vector")
}
if len(resp.Embeddings[0]) == 0 {
t.Fatal("embedding vector should not be empty")
}
t.Logf(" ✓ Embedding returned %d dimensions, input_tokens=%d",
len(resp.Embeddings[0]), resp.InputTokens)
}
// TestLive_VeniceModelDeletion tests cleanup: delete provider removes catalog entries
func TestLive_VeniceModelDeletion(t *testing.T) {
h := setupHarness(t)
veniceKey := requireVeniceKey(t)
_, adminToken := h.createAdminUser("admin", "admin@test.com")
// Create + fetch
w := h.request("POST", "/api/v1/admin/configs", adminToken, map[string]interface{}{
"name": "Venice Delete Test", "provider": "venice",
"endpoint": "https://api.venice.ai/api/v1", "api_key": veniceKey,
})
var cfg map[string]interface{}
decode(w, &cfg)
configID := cfg["id"].(string)
h.request("POST", "/api/v1/admin/models/fetch", adminToken,
map[string]interface{}{"provider_config_id": configID})
// Verify models exist
var count int
database.TestDB.QueryRow("SELECT COUNT(*) FROM model_catalog WHERE provider_config_id = $1", configID).Scan(&count)
if count == 0 {
t.Fatal("catalog should have models after fetch")
}
t.Logf(" %d models in catalog before delete", count)
// Delete the provider
w = h.request("DELETE", "/api/v1/admin/configs/"+configID, adminToken, nil)
if w.Code != http.StatusOK {
t.Fatalf("delete config: want 200, got %d: %s", w.Code, w.Body.String())
}
// Verify cascade: catalog entries should be gone
database.TestDB.QueryRow("SELECT COUNT(*) FROM model_catalog WHERE provider_config_id = $1", configID).Scan(&count)
if count != 0 {
t.Errorf("catalog should be empty after provider delete, got %d entries", count)
}
t.Log(" ✓ Cascade delete cleaned up catalog entries")
}
// TestLive_VeniceBYOK_AutoFetch exercises the ACTUAL user experience:
// user creates a BYOK provider → auto-fetch triggers → models appear in /models/enabled
//
// This is the definitive test. No simulated data. Real Venice API.
func TestLive_VeniceBYOK_AutoFetch(t *testing.T) {
h := setupHarness(t)
veniceKey := requireVeniceKey(t)
_, adminToken := h.createAdminUser("admin", "admin@test.com")
// Enable BYOK policy
h.request("PUT", "/api/v1/admin/settings/allow_user_byok", adminToken,
map[string]interface{}{"value": "true"})
// Create regular user
userID := database.SeedTestUser(t, "byokuser", "byokuser@test.com")
database.TestDB.Exec("UPDATE users SET is_active = true WHERE id = $1", userID)
userToken := makeToken(userID, "byokuser@test.com", "user")
// ── Step 1: User creates BYOK provider (the ONLY user action) ──
t.Log("Step 1: User creates BYOK Venice provider")
w := h.request("POST", "/api/v1/api-configs", userToken, map[string]interface{}{
"name": "My Venice",
"provider": "venice",
"endpoint": "https://api.venice.ai/api/v1",
"api_key": veniceKey,
})
if w.Code != http.StatusCreated {
t.Fatalf("create BYOK provider: want 201, got %d: %s", w.Code, w.Body.String())
}
var created map[string]interface{}
decode(w, &created)
cfgID := created["id"].(string)
t.Logf(" Created provider: %s", cfgID)
// ── Step 2: Verify auto-fetch happened ──
if created["warning"] != nil {
t.Fatalf("auto-fetch should succeed with real Venice key, got warning: %v", created["warning"])
}
modelsFetched := created["models_fetched"]
if modelsFetched == nil || modelsFetched.(float64) < 1 {
t.Fatalf("auto-fetch should return models_fetched > 0, got: %v", modelsFetched)
}
t.Logf(" Auto-fetched %.0f models", modelsFetched.(float64))
// ── Step 3: User's models/enabled shows personal models ──
t.Log("Step 3: Verify user sees BYOK models in /models/enabled")
w = h.request("GET", "/api/v1/models/enabled", userToken, nil)
if w.Code != http.StatusOK {
t.Fatalf("models/enabled: %d: %s", w.Code, w.Body.String())
}
var resp map[string]interface{}
decode(w, &resp)
userModels := resp["models"].([]interface{})
personalCount := 0
for _, raw := range userModels {
m := raw.(map[string]interface{})
if m["scope"] == "personal" {
personalCount++
}
}
if personalCount < 1 {
t.Fatalf("user should see personal BYOK models, got %d personal out of %d total\n"+
" model IDs: %v",
personalCount, len(userModels), func() []string {
ids := make([]string, 0)
for _, raw := range userModels {
m := raw.(map[string]interface{})
ids = append(ids, fmt.Sprintf("%s(scope=%s)", m["model_id"], m["scope"]))
}
return ids
}())
}
t.Logf(" ✓ User sees %d personal BYOK models (out of %d total)", personalCount, len(userModels))
// ── Step 4: Verify model fields for frontend ──
t.Log("Step 4: Verify frontend-required fields on BYOK models")
for _, raw := range userModels {
m := raw.(map[string]interface{})
if m["scope"] != "personal" {
continue
}
if m["config_id"] == nil || m["config_id"] == "" {
t.Errorf("personal model %s missing config_id", m["model_id"])
}
if m["provider_name"] == nil || m["provider_name"] == "" {
t.Errorf("personal model %s missing provider_name", m["model_id"])
}
if m["model_id"] == nil || m["model_id"] == "" {
t.Errorf("personal model missing model_id")
}
break // check first personal model only
}
// ── Cleanup ──
h.request("DELETE", fmt.Sprintf("/api/v1/api-configs/%s", cfgID), userToken, nil)
}