package handlers import ( "context" "fmt" "net/http" "os" "testing" "git.gobha.me/xcaliber/chat-switchboard/database" "git.gobha.me/xcaliber/chat-switchboard/providers" ) // ═══════════════════════════════════════════ // Live Provider Integration Tests // ═══════════════════════════════════════════ // These tests require: // - TEST_DATABASE_URL or PGHOST+PGUSER // - VENICE_API_KEY secret // // They exercise the full flow: create provider → // fetch models → enable model → resolve → complete. // // Model: qwen3-4b (Venice Small) — cheapest at $0.05/$0.15 per 1M tokens // ═══════════════════════════════════════════ const veniceTestModel = "qwen3-4b" func requireVeniceKey(t *testing.T) string { t.Helper() key := os.Getenv("VENICE_API_KEY") if key == "" { t.Skip("VENICE_API_KEY not set — skipping live provider test") } return key } // setupVeniceWithModel creates a Venice provider, fetches models, and enables // the specified model. Returns (configID, catalogEntryID). func setupVeniceWithModel(t *testing.T, h *testHarness, adminToken, apiKey, modelID string) (string, string) { t.Helper() // Create provider w := h.request("POST", "/api/v1/admin/configs", adminToken, map[string]interface{}{ "name": "Venice Test", "provider": "venice", "endpoint": "https://api.venice.ai/api/v1", "api_key": apiKey, }) if w.Code != http.StatusCreated { t.Fatalf("create venice config: want 201, got %d: %s", w.Code, w.Body.String()) } var cfg map[string]interface{} decode(w, &cfg) configID := cfg["id"].(string) // Fetch models w = h.request("POST", "/api/v1/admin/models/fetch", adminToken, map[string]interface{}{"provider_config_id": configID}) if w.Code != http.StatusOK { t.Fatalf("fetch models: %d: %s", w.Code, w.Body.String()) } // Find and enable target model w = h.request("GET", "/api/v1/admin/models", adminToken, nil) var modelsResp map[string]interface{} decode(w, &modelsResp) var catalogID string for _, raw := range modelsResp["models"].([]interface{}) { m := raw.(map[string]interface{}) if m["model_id"].(string) == modelID { catalogID = m["id"].(string) break } } if catalogID == "" { t.Fatalf("model %s not found in Venice catalog", modelID) } w = h.request("PUT", "/api/v1/admin/models/"+catalogID, adminToken, map[string]interface{}{"visibility": "enabled"}) if w.Code != http.StatusOK { t.Fatalf("enable model: %d: %s", w.Code, w.Body.String()) } t.Logf(" Venice provider %s ready, model %s enabled", configID, modelID) return configID, catalogID } // TestLive_VeniceProviderFullFlow exercises the complete admin workflow: // create provider → fetch models → enable a model → user sees it → chat completion func TestLive_VeniceProviderFullFlow(t *testing.T) { h := setupHarness(t) veniceKey := requireVeniceKey(t) _, adminToken := h.createAdminUser("admin", "admin@test.com") // ── 1. Create Venice provider config ──── t.Log("Step 1: Creating Venice provider config") w := h.request("POST", "/api/v1/admin/configs", adminToken, map[string]interface{}{ "name": "Venice Live Test", "provider": "venice", "endpoint": "https://api.venice.ai/api/v1", "api_key": veniceKey, }) if w.Code != http.StatusCreated { t.Fatalf("create venice config: want 201, got %d: %s", w.Code, w.Body.String()) } var configResp map[string]interface{} decode(w, &configResp) configID := configResp["id"].(string) t.Logf(" Created config: %s", configID) // ── 2. Fetch models from Venice ───────── t.Log("Step 2: Fetching models from Venice API") w = h.request("POST", "/api/v1/admin/models/fetch", adminToken, map[string]interface{}{ "provider_config_id": configID, }) if w.Code != http.StatusOK { t.Fatalf("fetch models: want 200, got %d: %s", w.Code, w.Body.String()) } var fetchResp map[string]interface{} decode(w, &fetchResp) totalFetched := fetchResp["total"].(float64) if totalFetched < 1 { t.Fatalf("Venice should return at least 1 model, got %.0f", totalFetched) } t.Logf(" Fetched %.0f models from Venice", totalFetched) // ── 3. List catalog models (all disabled by default) ── t.Log("Step 3: Listing catalog models") w = h.request("GET", "/api/v1/admin/models", adminToken, nil) if w.Code != http.StatusOK { t.Fatalf("list models: want 200, got %d: %s", w.Code, w.Body.String()) } var modelsResp map[string]interface{} decode(w, &modelsResp) catalogModels := modelsResp["models"].([]interface{}) if len(catalogModels) < 1 { t.Fatal("catalog should have models after fetch") } // Find a text model to enable (prefer a small/fast one) var enableID string var enableModelID string for _, raw := range catalogModels { m := raw.(map[string]interface{}) modelID := m["model_id"].(string) vis := m["visibility"].(string) if vis == "disabled" { enableID = m["id"].(string) enableModelID = modelID break } } if enableID == "" { t.Fatal("no disabled model found to enable") } t.Logf(" Will enable: %s (catalog ID: %s)", enableModelID, enableID) // ── 4. Enable the model ───────────────── t.Log("Step 4: Enabling model") w = h.request("PUT", "/api/v1/admin/models/"+enableID, adminToken, map[string]interface{}{"visibility": "enabled"}) if w.Code != http.StatusOK { t.Fatalf("enable model: want 200, got %d: %s", w.Code, w.Body.String()) } // ── 5. Verify models/enabled returns it (admin) ── t.Log("Step 5: Verifying models/enabled (admin)") w = h.request("GET", "/api/v1/models/enabled", adminToken, nil) if w.Code != http.StatusOK { t.Fatalf("models/enabled: want 200, got %d: %s", w.Code, w.Body.String()) } var enabledResp map[string]interface{} decode(w, &enabledResp) enabledModels := enabledResp["models"].([]interface{}) if len(enabledModels) < 1 { t.Fatal("models/enabled should return at least 1 model after enabling") } // Verify our model is in the list found := false for _, raw := range enabledModels { m := raw.(map[string]interface{}) if m["model_id"] == enableModelID { found = true t.Logf(" ✓ Found %s in enabled models", enableModelID) // Verify it has the required fields for the frontend if m["config_id"] == nil || m["config_id"] == "" { t.Error("enabled model must have config_id for composite ID") } if m["provider_name"] == nil || m["provider_name"] == "" { t.Error("enabled model must have provider_name for display") } break } } if !found { t.Errorf("enabled model %s not found in models/enabled response", enableModelID) } // ── 6. Verify a REGULAR USER also sees the model ── t.Log("Step 6: Verifying models/enabled (regular user)") userID := database.SeedTestUser(t, "liveuser", "liveuser@test.com") database.TestDB.Exec("UPDATE users SET is_active = true WHERE id = $1", userID) userToken := makeToken(userID, "liveuser@test.com", "user") w = h.request("GET", "/api/v1/models/enabled", userToken, nil) if w.Code != http.StatusOK { t.Fatalf("user models/enabled: want 200, got %d: %s", w.Code, w.Body.String()) } var userResp map[string]interface{} decode(w, &userResp) userModels := userResp["models"].([]interface{}) if len(userModels) < 1 { t.Fatalf("regular user should see at least 1 enabled model, got %d — "+ "admin can see models but regular user cannot; check ListVisible query", len(userModels)) } userFound := false for _, raw := range userModels { m := raw.(map[string]interface{}) if m["model_id"] == enableModelID { userFound = true t.Logf(" ✓ Regular user can see %s", enableModelID) // Verify same fields available for regular user if m["config_id"] == nil || m["config_id"] == "" { t.Error("user: enabled model must have config_id") } if m["provider_name"] == nil || m["provider_name"] == "" { t.Error("user: enabled model must have provider_name") } break } } if !userFound { t.Errorf("regular user cannot see %s — admin→user visibility broken", enableModelID) } } // TestLive_VeniceFetchModelsCapabilities verifies that Venice model // capabilities are correctly parsed into the catalog. func TestLive_VeniceFetchModelsCapabilities(t *testing.T) { h := setupHarness(t) veniceKey := requireVeniceKey(t) _, adminToken := h.createAdminUser("admin", "admin@test.com") // Create provider + fetch w := h.request("POST", "/api/v1/admin/configs", adminToken, map[string]interface{}{ "name": "Venice Caps Test", "provider": "venice", "endpoint": "https://api.venice.ai/api/v1", "api_key": veniceKey, }) var cfg map[string]interface{} decode(w, &cfg) configID := cfg["id"].(string) h.request("POST", "/api/v1/admin/models/fetch", adminToken, map[string]interface{}{"provider_config_id": configID}) // Read catalog and check capabilities w = h.request("GET", "/api/v1/admin/models", adminToken, nil) var resp map[string]interface{} decode(w, &resp) for _, raw := range resp["models"].([]interface{}) { m := raw.(map[string]interface{}) caps, ok := m["capabilities"].(map[string]interface{}) if !ok { t.Errorf("model %s: capabilities must be an object", m["model_id"]) continue } // streaming should always be true for Venice if caps["streaming"] != true { t.Errorf("model %s: streaming should be true", m["model_id"]) } // Verify capabilities are actual booleans (not strings) for _, key := range []string{"streaming", "vision", "tool_calling", "reasoning"} { if v, exists := caps[key]; exists { if _, ok := v.(bool); !ok { t.Errorf("model %s: capability %s should be bool, got %T", m["model_id"], key, v) } } } } } // TestLive_VeniceChatCompletion sends an actual non-streaming chat completion // using the cheapest model (qwen3-4b = $0.05/$0.15 per 1M tokens). func TestLive_VeniceChatCompletion(t *testing.T) { h := setupHarness(t) veniceKey := requireVeniceKey(t) _, adminToken := h.createAdminUser("admin", "admin@test.com") configID, _ := setupVeniceWithModel(t, h, adminToken, veniceKey, veniceTestModel) // Create channel w := h.request("POST", "/api/v1/channels", adminToken, map[string]interface{}{ "title": "Chat Test", "type": "direct", }) if w.Code != http.StatusCreated { t.Skipf("channel creation failed: %d %s", w.Code, w.Body.String()) } var ch map[string]interface{} decode(w, &ch) channelID := ch["id"].(string) // Non-streaming completion with correct field names stream := false w = h.request("POST", "/api/v1/chat/completions", adminToken, map[string]interface{}{ "channel_id": channelID, "content": "Say ok", "model": veniceTestModel, "provider_config_id": configID, "stream": &stream, "max_tokens": 10, }) if w.Code != http.StatusOK { t.Fatalf("completion: want 200, got %d: %s", w.Code, w.Body.String()) } t.Logf(" ✓ Completion succeeded: %s", w.Body.String()[:min(200, w.Body.Len())]) } // TestLive_VeniceUsageLogging verifies that a non-streaming completion // creates a usage_log row with token counts from the provider. func TestLive_VeniceUsageLogging(t *testing.T) { h := setupHarness(t) veniceKey := requireVeniceKey(t) _, adminToken := h.createAdminUser("admin", "admin@test.com") configID, _ := setupVeniceWithModel(t, h, adminToken, veniceKey, veniceTestModel) // Create channel w := h.request("POST", "/api/v1/channels", adminToken, map[string]interface{}{ "title": "Usage Test", "type": "direct", }) if w.Code != http.StatusCreated { t.Skipf("channel creation failed: %d %s", w.Code, w.Body.String()) } var ch map[string]interface{} decode(w, &ch) channelID := ch["id"].(string) // Non-streaming — providers reliably return usage in non-streaming mode stream := false w = h.request("POST", "/api/v1/chat/completions", adminToken, map[string]interface{}{ "channel_id": channelID, "content": "Say ok", "model": veniceTestModel, "provider_config_id": configID, "stream": &stream, "max_tokens": 10, }) if w.Code != http.StatusOK { t.Fatalf("completion: %d: %s", w.Code, w.Body.String()) } // Verify usage_log row exists var rowCount int var promptTokens, completionTokens int err := database.TestDB.QueryRow(` SELECT COUNT(*), COALESCE(SUM(prompt_tokens), 0), COALESCE(SUM(completion_tokens), 0) FROM usage_log WHERE provider_config_id = $1 `, configID).Scan(&rowCount, &promptTokens, &completionTokens) if err != nil { t.Fatalf("query usage_log: %v", err) } if rowCount == 0 { t.Fatal("usage_log should have a row after non-streaming completion") } t.Logf(" ✓ Usage logged: %d row(s), prompt=%d completion=%d", rowCount, promptTokens, completionTokens) if promptTokens == 0 { t.Fatal("non-streaming completion should report prompt tokens — check provider response parsing") } t.Logf(" ✓ Token counts: prompt=%d completion=%d", promptTokens, completionTokens) } // TestLive_VeniceStreamingUsageLogging verifies that streaming completions // create a usage_log row with actual token counts. Venice supports // stream_options.include_usage — the parser must capture the usage chunk // that arrives after finish_reason but before [DONE]. func TestLive_VeniceStreamingUsageLogging(t *testing.T) { h := setupHarness(t) veniceKey := requireVeniceKey(t) _, adminToken := h.createAdminUser("admin", "admin@test.com") configID, _ := setupVeniceWithModel(t, h, adminToken, veniceKey, veniceTestModel) // Create channel w := h.request("POST", "/api/v1/channels", adminToken, map[string]interface{}{ "title": "Stream Usage Test", "type": "direct", }) if w.Code != http.StatusCreated { t.Skipf("channel creation failed: %d %s", w.Code, w.Body.String()) } var ch map[string]interface{} decode(w, &ch) channelID := ch["id"].(string) // Streaming completion stream := true w = h.request("POST", "/api/v1/chat/completions", adminToken, map[string]interface{}{ "channel_id": channelID, "content": "Say ok", "model": veniceTestModel, "provider_config_id": configID, "stream": &stream, "max_tokens": 10, }) // Streaming returns 200 with SSE — the recorder captures the full body if w.Code != http.StatusOK { t.Fatalf("streaming completion: %d: %s", w.Code, w.Body.String()) } // Verify usage_log row exists (even if tokens are 0) var rowCount int var promptTokens, completionTokens int err := database.TestDB.QueryRow(` SELECT COUNT(*), COALESCE(SUM(prompt_tokens), 0), COALESCE(SUM(completion_tokens), 0) FROM usage_log WHERE provider_config_id = $1 `, configID).Scan(&rowCount, &promptTokens, &completionTokens) if err != nil { t.Fatalf("query usage_log: %v", err) } if rowCount == 0 { t.Fatal("usage_log should have a row after streaming completion") } if promptTokens == 0 { t.Fatal("streaming completion should report prompt tokens — check pendingFinish logic in openai.go parser") } t.Logf(" ✓ Streaming usage logged: %d row(s), prompt=%d completion=%d", rowCount, promptTokens, completionTokens) } // TestLive_VenicePricingFromCatalog verifies that model sync populates // the model_pricing table from Venice's pricing data. func TestLive_VenicePricingFromCatalog(t *testing.T) { h := setupHarness(t) veniceKey := requireVeniceKey(t) _, adminToken := h.createAdminUser("admin", "admin@test.com") configID, _ := setupVeniceWithModel(t, h, adminToken, veniceKey, veniceTestModel) // Check if pricing was populated during fetch var pricingCount int database.TestDB.QueryRow(` SELECT COUNT(*) FROM model_pricing WHERE provider_config_id = $1 `, configID).Scan(&pricingCount) if pricingCount == 0 { t.Skip("Venice model sync did not populate pricing — may need provider pricing support") } t.Logf(" ✓ Catalog sync populated %d pricing entries", pricingCount) // Verify our test model has pricing var inputPerM, outputPerM float64 err := database.TestDB.QueryRow(` SELECT COALESCE(input_per_m, 0), COALESCE(output_per_m, 0) FROM model_pricing WHERE provider_config_id = $1 AND model_id = $2 `, configID, veniceTestModel).Scan(&inputPerM, &outputPerM) if err != nil { t.Logf(" ⚠ No pricing for %s specifically (may use different model ID)", veniceTestModel) } else { t.Logf(" ✓ %s pricing: $%.4f input, $%.4f output (per 1M)", veniceTestModel, inputPerM, outputPerM) } // Admin pricing list should show these (scope=global, not personal) w := h.request("GET", "/api/v1/admin/pricing", adminToken, nil) if w.Code != http.StatusOK { t.Fatalf("admin pricing: %d: %s", w.Code, w.Body.String()) } var entries []interface{} decode(w, &entries) if len(entries) == 0 { t.Error("admin pricing API should return catalog-synced entries") } else { t.Logf(" ✓ Admin pricing API returns %d entries", len(entries)) } } // TestLive_VeniceEmbeddings tests the Venice embeddings endpoint directly // using the BGE-M3 model ($0.15 per 1M tokens). func TestLive_VeniceEmbeddings(t *testing.T) { veniceKey := requireVeniceKey(t) provider := &providers.VeniceProvider{} cfg := providers.ProviderConfig{ Endpoint: "https://api.venice.ai/api/v1", APIKey: veniceKey, } resp, err := provider.Embed( context.Background(), cfg, providers.EmbeddingRequest{ Model: "text-embedding-bge-m3", Input: []string{"test embedding"}, }, ) if err != nil { t.Fatalf("Venice Embed: %v", err) } if len(resp.Embeddings) == 0 { t.Fatal("expected at least 1 embedding vector") } if len(resp.Embeddings[0]) == 0 { t.Fatal("embedding vector should not be empty") } t.Logf(" ✓ Embedding returned %d dimensions, input_tokens=%d", len(resp.Embeddings[0]), resp.InputTokens) } // TestLive_VeniceModelDeletion tests cleanup: delete provider removes catalog entries func TestLive_VeniceModelDeletion(t *testing.T) { h := setupHarness(t) veniceKey := requireVeniceKey(t) _, adminToken := h.createAdminUser("admin", "admin@test.com") // Create + fetch w := h.request("POST", "/api/v1/admin/configs", adminToken, map[string]interface{}{ "name": "Venice Delete Test", "provider": "venice", "endpoint": "https://api.venice.ai/api/v1", "api_key": veniceKey, }) var cfg map[string]interface{} decode(w, &cfg) configID := cfg["id"].(string) h.request("POST", "/api/v1/admin/models/fetch", adminToken, map[string]interface{}{"provider_config_id": configID}) // Verify models exist var count int database.TestDB.QueryRow("SELECT COUNT(*) FROM model_catalog WHERE provider_config_id = $1", configID).Scan(&count) if count == 0 { t.Fatal("catalog should have models after fetch") } t.Logf(" %d models in catalog before delete", count) // Delete the provider w = h.request("DELETE", "/api/v1/admin/configs/"+configID, adminToken, nil) if w.Code != http.StatusOK { t.Fatalf("delete config: want 200, got %d: %s", w.Code, w.Body.String()) } // Verify cascade: catalog entries should be gone database.TestDB.QueryRow("SELECT COUNT(*) FROM model_catalog WHERE provider_config_id = $1", configID).Scan(&count) if count != 0 { t.Errorf("catalog should be empty after provider delete, got %d entries", count) } t.Log(" ✓ Cascade delete cleaned up catalog entries") } // TestLive_VeniceBYOK_AutoFetch exercises the ACTUAL user experience: // user creates a BYOK provider → auto-fetch triggers → models appear in /models/enabled // // This is the definitive test. No simulated data. Real Venice API. func TestLive_VeniceBYOK_AutoFetch(t *testing.T) { h := setupHarness(t) veniceKey := requireVeniceKey(t) _, adminToken := h.createAdminUser("admin", "admin@test.com") // Enable BYOK policy h.request("PUT", "/api/v1/admin/settings/allow_user_byok", adminToken, map[string]interface{}{"value": "true"}) // Create regular user userID := database.SeedTestUser(t, "byokuser", "byokuser@test.com") database.TestDB.Exec("UPDATE users SET is_active = true WHERE id = $1", userID) userToken := makeToken(userID, "byokuser@test.com", "user") // ── Step 1: User creates BYOK provider (the ONLY user action) ── t.Log("Step 1: User creates BYOK Venice provider") w := h.request("POST", "/api/v1/api-configs", userToken, map[string]interface{}{ "name": "My Venice", "provider": "venice", "endpoint": "https://api.venice.ai/api/v1", "api_key": veniceKey, }) if w.Code != http.StatusCreated { t.Fatalf("create BYOK provider: want 201, got %d: %s", w.Code, w.Body.String()) } var created map[string]interface{} decode(w, &created) cfgID := created["id"].(string) t.Logf(" Created provider: %s", cfgID) // ── Step 2: Verify auto-fetch happened ── if created["warning"] != nil { t.Fatalf("auto-fetch should succeed with real Venice key, got warning: %v", created["warning"]) } modelsFetched := created["models_fetched"] if modelsFetched == nil || modelsFetched.(float64) < 1 { t.Fatalf("auto-fetch should return models_fetched > 0, got: %v", modelsFetched) } t.Logf(" Auto-fetched %.0f models", modelsFetched.(float64)) // ── Step 3: User's models/enabled shows personal models ── t.Log("Step 3: Verify user sees BYOK models in /models/enabled") w = h.request("GET", "/api/v1/models/enabled", userToken, nil) if w.Code != http.StatusOK { t.Fatalf("models/enabled: %d: %s", w.Code, w.Body.String()) } var resp map[string]interface{} decode(w, &resp) userModels := resp["models"].([]interface{}) personalCount := 0 for _, raw := range userModels { m := raw.(map[string]interface{}) if m["scope"] == "personal" { personalCount++ } } if personalCount < 1 { t.Fatalf("user should see personal BYOK models, got %d personal out of %d total\n"+ " model IDs: %v", personalCount, len(userModels), func() []string { ids := make([]string, 0) for _, raw := range userModels { m := raw.(map[string]interface{}) ids = append(ids, fmt.Sprintf("%s(scope=%s)", m["model_id"], m["scope"])) } return ids }()) } t.Logf(" ✓ User sees %d personal BYOK models (out of %d total)", personalCount, len(userModels)) // ── Step 4: Verify model fields for frontend ── t.Log("Step 4: Verify frontend-required fields on BYOK models") for _, raw := range userModels { m := raw.(map[string]interface{}) if m["scope"] != "personal" { continue } if m["config_id"] == nil || m["config_id"] == "" { t.Errorf("personal model %s missing config_id", m["model_id"]) } if m["provider_name"] == nil || m["provider_name"] == "" { t.Errorf("personal model %s missing provider_name", m["model_id"]) } if m["model_id"] == nil || m["model_id"] == "" { t.Errorf("personal model missing model_id") } break // check first personal model only } // ── Cleanup ── h.request("DELETE", fmt.Sprintf("/api/v1/api-configs/%s", cfgID), userToken, nil) }