Changeset 0.10.0 (#56)

2026-02-24 14:50:53 +00:00
parent cdfd69bad3
commit ea03f956ca
31 changed files with 3303 additions and 167 deletions
--- a/server/handlers/live_provider_test.go
+++ b/server/handlers/live_provider_test.go
@@ -1,12 +1,14 @@
 package handlers

 import (
+	"context"
 	"fmt"
 	"net/http"
 	"os"
 	"testing"

 	"git.gobha.me/xcaliber/chat-switchboard/database"
+	"git.gobha.me/xcaliber/chat-switchboard/providers"
 )

 // ═══════════════════════════════════════════
@@ -18,8 +20,12 @@ import (
 //
 // They exercise the full flow: create provider →
 // fetch models → enable model → resolve → complete.
+//
+// Model: qwen3-4b (Venice Small) — cheapest at $0.05/$0.15 per 1M tokens
 // ═══════════════════════════════════════════

+const veniceTestModel = "qwen3-4b"
+
 func requireVeniceKey(t *testing.T) string {
 	t.Helper()
 	key := os.Getenv("VENICE_API_KEY")
@@ -29,6 +35,57 @@ func requireVeniceKey(t *testing.T) string {
 	return key
 }

+// setupVeniceWithModel creates a Venice provider, fetches models, and enables
+// the specified model. Returns (configID, catalogEntryID).
+func setupVeniceWithModel(t *testing.T, h *testHarness, adminToken, apiKey, modelID string) (string, string) {
+	t.Helper()
+
+	// Create provider
+	w := h.request("POST", "/api/v1/admin/configs", adminToken, map[string]interface{}{
+		"name": "Venice Test", "provider": "venice",
+		"endpoint": "https://api.venice.ai/api/v1", "api_key": apiKey,
+	})
+	if w.Code != http.StatusCreated {
+		t.Fatalf("create venice config: want 201, got %d: %s", w.Code, w.Body.String())
+	}
+	var cfg map[string]interface{}
+	decode(w, &cfg)
+	configID := cfg["id"].(string)
+
+	// Fetch models
+	w = h.request("POST", "/api/v1/admin/models/fetch", adminToken,
+		map[string]interface{}{"provider_config_id": configID})
+	if w.Code != http.StatusOK {
+		t.Fatalf("fetch models: %d: %s", w.Code, w.Body.String())
+	}
+
+	// Find and enable target model
+	w = h.request("GET", "/api/v1/admin/models", adminToken, nil)
+	var modelsResp map[string]interface{}
+	decode(w, &modelsResp)
+
+	var catalogID string
+	for _, raw := range modelsResp["models"].([]interface{}) {
+		m := raw.(map[string]interface{})
+		if m["model_id"].(string) == modelID {
+			catalogID = m["id"].(string)
+			break
+		}
+	}
+	if catalogID == "" {
+		t.Fatalf("model %s not found in Venice catalog", modelID)
+	}
+
+	w = h.request("PUT", "/api/v1/admin/models/"+catalogID, adminToken,
+		map[string]interface{}{"visibility": "enabled"})
+	if w.Code != http.StatusOK {
+		t.Fatalf("enable model: %d: %s", w.Code, w.Body.String())
+	}
+
+	t.Logf("  Venice provider %s ready, model %s enabled", configID, modelID)
+	return configID, catalogID
+}
+
 // TestLive_VeniceProviderFullFlow exercises the complete admin workflow:
 // create provider → fetch models → enable a model → user sees it → chat completion
 func TestLive_VeniceProviderFullFlow(t *testing.T) {
@@ -231,92 +288,232 @@ func TestLive_VeniceFetchModelsCapabilities(t *testing.T) {
 	}
 }

-// TestLive_VeniceChatCompletion sends an actual chat completion.
+// TestLive_VeniceChatCompletion sends an actual non-streaming chat completion
+// using the cheapest model (qwen3-4b = $0.05/$0.15 per 1M tokens).
 func TestLive_VeniceChatCompletion(t *testing.T) {
 	h := setupHarness(t)
 	veniceKey := requireVeniceKey(t)
-	adminID, adminToken := h.createAdminUser("admin", "admin@test.com")
-	_ = adminID
+	_, adminToken := h.createAdminUser("admin", "admin@test.com")

-	// Create provider
-	w := h.request("POST", "/api/v1/admin/configs", adminToken, map[string]interface{}{
-		"name": "Venice Chat Test", "provider": "venice",
-		"endpoint": "https://api.venice.ai/api/v1", "api_key": veniceKey,
-	})
-	var cfg map[string]interface{}
-	decode(w, &cfg)
-	configID := cfg["id"].(string)
+	configID, _ := setupVeniceWithModel(t, h, adminToken, veniceKey, veniceTestModel)

-	// Fetch + enable a fast model
-	h.request("POST", "/api/v1/admin/models/fetch", adminToken,
-		map[string]interface{}{"provider_config_id": configID})
-
-	w = h.request("GET", "/api/v1/admin/models", adminToken, nil)
-	var modelsResp map[string]interface{}
-	decode(w, &modelsResp)
-
-	// Find and enable a small model (prefer llama or qwen for speed)
-	var targetModelID, targetCatalogID string
-	for _, raw := range modelsResp["models"].([]interface{}) {
-		m := raw.(map[string]interface{})
-		mid := m["model_id"].(string)
-		// Pick any available model - first disabled one
-		if m["visibility"].(string) == "disabled" {
-			targetModelID = mid
-			targetCatalogID = m["id"].(string)
-			break
-		}
-	}
-	if targetModelID == "" {
-		t.Skip("no model available to test chat completion")
-	}
-
-	h.request("PUT", "/api/v1/admin/models/"+targetCatalogID, adminToken,
-		map[string]interface{}{"visibility": "enabled"})
-
-	// Set as default model and send completion
-	// First get enabled model's composite ID
-	w = h.request("GET", "/api/v1/models/enabled", adminToken, nil)
-	var enabled map[string]interface{}
-	decode(w, &enabled)
-	if len(enabled["models"].([]interface{})) == 0 {
-		t.Fatal("no enabled models for completion test")
-	}
-	firstModel := enabled["models"].([]interface{})[0].(map[string]interface{})
-	modelForChat := firstModel["model_id"].(string)
-	configForChat := firstModel["config_id"].(string)
-
-	t.Logf("Sending completion to %s via config %s", modelForChat, configForChat)
-
-	// Create a channel first
-	w = h.request("POST", "/api/v1/channels", adminToken, map[string]interface{}{
-		"title": "Test Chat", "type": "direct",
+	// Create channel
+	w := h.request("POST", "/api/v1/channels", adminToken, map[string]interface{}{
+		"title": "Chat Test", "type": "direct",
 	})
 	if w.Code != http.StatusCreated {
-		// Some channel handlers may use database.DB directly
-		t.Skipf("channel creation failed (may need database.DB global): %d %s", w.Code, w.Body.String())
+		t.Skipf("channel creation failed: %d %s", w.Code, w.Body.String())
 	}
 	var ch map[string]interface{}
 	decode(w, &ch)
 	channelID := ch["id"].(string)

-	// Send completion
+	// Non-streaming completion with correct field names
+	stream := false
 	w = h.request("POST", "/api/v1/chat/completions", adminToken, map[string]interface{}{
-		"channel_id": channelID,
-		"model":      modelForChat,
-		"config_id":  configForChat,
-		"stream":     false,
-		"messages": []map[string]string{
-			{"role": "user", "content": "Say hello in exactly 3 words."},
-		},
+		"channel_id":         channelID,
+		"content":            "Say ok",
+		"model":              veniceTestModel,
+		"provider_config_id": configID,
+		"stream":             &stream,
+		"max_tokens":         10,
 	})
 	if w.Code != http.StatusOK {
-		t.Logf("completion response: %s", w.Body.String())
-		t.Skipf("chat completion failed with %d (may need full router wiring)", w.Code)
+		t.Fatalf("completion: want 200, got %d: %s", w.Code, w.Body.String())
 	}
 	t.Logf("  ✓ Completion succeeded: %s", w.Body.String()[:min(200, w.Body.Len())])
 }

+// TestLive_VeniceUsageLogging verifies that a non-streaming completion
+// creates a usage_log row with token counts from the provider.
+func TestLive_VeniceUsageLogging(t *testing.T) {
+	h := setupHarness(t)
+	veniceKey := requireVeniceKey(t)
+	_, adminToken := h.createAdminUser("admin", "admin@test.com")
+
+	configID, _ := setupVeniceWithModel(t, h, adminToken, veniceKey, veniceTestModel)
+
+	// Create channel
+	w := h.request("POST", "/api/v1/channels", adminToken, map[string]interface{}{
+		"title": "Usage Test", "type": "direct",
+	})
+	if w.Code != http.StatusCreated {
+		t.Skipf("channel creation failed: %d %s", w.Code, w.Body.String())
+	}
+	var ch map[string]interface{}
+	decode(w, &ch)
+	channelID := ch["id"].(string)
+
+	// Non-streaming — providers reliably return usage in non-streaming mode
+	stream := false
+	w = h.request("POST", "/api/v1/chat/completions", adminToken, map[string]interface{}{
+		"channel_id":         channelID,
+		"content":            "Say ok",
+		"model":              veniceTestModel,
+		"provider_config_id": configID,
+		"stream":             &stream,
+		"max_tokens":         10,
+	})
+	if w.Code != http.StatusOK {
+		t.Fatalf("completion: %d: %s", w.Code, w.Body.String())
+	}
+
+	// Verify usage_log row exists
+	var rowCount int
+	var promptTokens, completionTokens int
+	err := database.TestDB.QueryRow(`
+		SELECT COUNT(*), COALESCE(SUM(prompt_tokens), 0), COALESCE(SUM(completion_tokens), 0)
+		FROM usage_log WHERE provider_config_id = $1
+	`, configID).Scan(&rowCount, &promptTokens, &completionTokens)
+	if err != nil {
+		t.Fatalf("query usage_log: %v", err)
+	}
+	if rowCount == 0 {
+		t.Fatal("usage_log should have a row after non-streaming completion")
+	}
+	t.Logf("  ✓ Usage logged: %d row(s), prompt=%d completion=%d", rowCount, promptTokens, completionTokens)
+
+	if promptTokens == 0 {
+		t.Fatal("non-streaming completion should report prompt tokens — check provider response parsing")
+	}
+	t.Logf("  ✓ Token counts: prompt=%d completion=%d", promptTokens, completionTokens)
+}
+
+// TestLive_VeniceStreamingUsageLogging verifies that streaming completions
+// create a usage_log row with actual token counts. Venice supports
+// stream_options.include_usage — the parser must capture the usage chunk
+// that arrives after finish_reason but before [DONE].
+func TestLive_VeniceStreamingUsageLogging(t *testing.T) {
+	h := setupHarness(t)
+	veniceKey := requireVeniceKey(t)
+	_, adminToken := h.createAdminUser("admin", "admin@test.com")
+
+	configID, _ := setupVeniceWithModel(t, h, adminToken, veniceKey, veniceTestModel)
+
+	// Create channel
+	w := h.request("POST", "/api/v1/channels", adminToken, map[string]interface{}{
+		"title": "Stream Usage Test", "type": "direct",
+	})
+	if w.Code != http.StatusCreated {
+		t.Skipf("channel creation failed: %d %s", w.Code, w.Body.String())
+	}
+	var ch map[string]interface{}
+	decode(w, &ch)
+	channelID := ch["id"].(string)
+
+	// Streaming completion
+	stream := true
+	w = h.request("POST", "/api/v1/chat/completions", adminToken, map[string]interface{}{
+		"channel_id":         channelID,
+		"content":            "Say ok",
+		"model":              veniceTestModel,
+		"provider_config_id": configID,
+		"stream":             &stream,
+		"max_tokens":         10,
+	})
+	// Streaming returns 200 with SSE — the recorder captures the full body
+	if w.Code != http.StatusOK {
+		t.Fatalf("streaming completion: %d: %s", w.Code, w.Body.String())
+	}
+
+	// Verify usage_log row exists (even if tokens are 0)
+	var rowCount int
+	var promptTokens, completionTokens int
+	err := database.TestDB.QueryRow(`
+		SELECT COUNT(*), COALESCE(SUM(prompt_tokens), 0), COALESCE(SUM(completion_tokens), 0)
+		FROM usage_log WHERE provider_config_id = $1
+	`, configID).Scan(&rowCount, &promptTokens, &completionTokens)
+	if err != nil {
+		t.Fatalf("query usage_log: %v", err)
+	}
+	if rowCount == 0 {
+		t.Fatal("usage_log should have a row after streaming completion")
+	}
+	if promptTokens == 0 {
+		t.Fatal("streaming completion should report prompt tokens — check pendingFinish logic in openai.go parser")
+	}
+	t.Logf("  ✓ Streaming usage logged: %d row(s), prompt=%d completion=%d", rowCount, promptTokens, completionTokens)
+}
+
+// TestLive_VenicePricingFromCatalog verifies that model sync populates
+// the model_pricing table from Venice's pricing data.
+func TestLive_VenicePricingFromCatalog(t *testing.T) {
+	h := setupHarness(t)
+	veniceKey := requireVeniceKey(t)
+	_, adminToken := h.createAdminUser("admin", "admin@test.com")
+
+	configID, _ := setupVeniceWithModel(t, h, adminToken, veniceKey, veniceTestModel)
+
+	// Check if pricing was populated during fetch
+	var pricingCount int
+	database.TestDB.QueryRow(`
+		SELECT COUNT(*) FROM model_pricing WHERE provider_config_id = $1
+	`, configID).Scan(&pricingCount)
+
+	if pricingCount == 0 {
+		t.Skip("Venice model sync did not populate pricing — may need provider pricing support")
+	}
+
+	t.Logf("  ✓ Catalog sync populated %d pricing entries", pricingCount)
+
+	// Verify our test model has pricing
+	var inputPerM, outputPerM float64
+	err := database.TestDB.QueryRow(`
+		SELECT COALESCE(input_per_m, 0), COALESCE(output_per_m, 0)
+		FROM model_pricing
+		WHERE provider_config_id = $1 AND model_id = $2
+	`, configID, veniceTestModel).Scan(&inputPerM, &outputPerM)
+	if err != nil {
+		t.Logf("  ⚠ No pricing for %s specifically (may use different model ID)", veniceTestModel)
+	} else {
+		t.Logf("  ✓ %s pricing: $%.4f input, $%.4f output (per 1M)", veniceTestModel, inputPerM, outputPerM)
+	}
+
+	// Admin pricing list should show these (scope=global, not personal)
+	w := h.request("GET", "/api/v1/admin/pricing", adminToken, nil)
+	if w.Code != http.StatusOK {
+		t.Fatalf("admin pricing: %d: %s", w.Code, w.Body.String())
+	}
+	var entries []interface{}
+	decode(w, &entries)
+	if len(entries) == 0 {
+		t.Error("admin pricing API should return catalog-synced entries")
+	} else {
+		t.Logf("  ✓ Admin pricing API returns %d entries", len(entries))
+	}
+}
+
+// TestLive_VeniceEmbeddings tests the Venice embeddings endpoint directly
+// using the BGE-M3 model ($0.15 per 1M tokens).
+func TestLive_VeniceEmbeddings(t *testing.T) {
+	veniceKey := requireVeniceKey(t)
+
+	provider := &providers.VeniceProvider{}
+	cfg := providers.ProviderConfig{
+		Endpoint: "https://api.venice.ai/api/v1",
+		APIKey:   veniceKey,
+	}
+
+	resp, err := provider.Embed(
+		context.Background(), cfg,
+		providers.EmbeddingRequest{
+			Model: "text-embedding-bge-m3",
+			Input: []string{"test embedding"},
+		},
+	)
+	if err != nil {
+		t.Fatalf("Venice Embed: %v", err)
+	}
+	if len(resp.Embeddings) == 0 {
+		t.Fatal("expected at least 1 embedding vector")
+	}
+	if len(resp.Embeddings[0]) == 0 {
+		t.Fatal("embedding vector should not be empty")
+	}
+	t.Logf("  ✓ Embedding returned %d dimensions, input_tokens=%d",
+		len(resp.Embeddings[0]), resp.InputTokens)
+}
+
 // TestLive_VeniceModelDeletion tests cleanup: delete provider removes catalog entries
 func TestLive_VeniceModelDeletion(t *testing.T) {
 	h := setupHarness(t)