Changeset 0.22.0.1 (#94)

2026-03-02 01:55:19 +00:00
parent 12e03cec8b
commit 06c4e2a5a1
33 changed files with 1929 additions and 418 deletions
--- a/server/handlers/capabilities.go
+++ b/server/handlers/capabilities.go
@@ -94,7 +94,7 @@ func capsFromCatalog(modelID, configID string) (models.ModelCapabilities, bool)
 	}

 	// Merge with known data to fill gaps
-	resolved := capspkg.ResolveIntrinsic(modelID, &caps)
+	resolved := capspkg.ResolveIntrinsic(modelID, &caps, nil)
 	return resolved, true
 }

@@ -140,7 +140,7 @@ func liveQueryModelCaps(c *gin.Context, configID, modelID string) (models.ModelC

 	for _, m := range modelList {
 		if m.ID == modelID {
-			resolved := capspkg.ResolveIntrinsic(modelID, &m.Capabilities)
+			resolved := capspkg.ResolveIntrinsic(modelID, &m.Capabilities, nil)
 			return resolved, true
 		}
 	}
--- a/server/handlers/completion.go
+++ b/server/handlers/completion.go
@@ -10,6 +10,7 @@ import (
 	"log"
 	"net/http"
 	"strings"
+	"time"

 	"github.com/gin-gonic/gin"

@@ -50,6 +51,15 @@ type CompletionHandler struct {
 	hub      *events.Hub          // WebSocket hub for browser tool bridge
 	objStore storage.ObjectStore  // file storage for attachment content (nil = disabled)
 	embedder *knowledge.Embedder  // for memory semantic recall (v0.18.0)
+	health   HealthRecorder       // provider health tracking (v0.22.0, nil = disabled)
+}
+
+// HealthRecorder is the interface for recording provider call outcomes.
+// Matches health.Accumulator methods without importing the package.
+type HealthRecorder interface {
+	RecordSuccess(providerConfigID string, latencyMs int)
+	RecordError(providerConfigID string, latencyMs int, errMsg string)
+	RecordTimeout(providerConfigID string, latencyMs int, errMsg string)
 }

 // NewCompletionHandler creates a new handler.
@@ -57,6 +67,11 @@ func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *e
 	return &CompletionHandler{vault: vault, stores: stores, hub: hub, objStore: objStore, embedder: embedder}
 }

+// SetHealthRecorder attaches the provider health accumulator.
+func (h *CompletionHandler) SetHealthRecorder(hr HealthRecorder) {
+	h.health = hr
+}
+
 // ── Chat Completion ─────────────────────────
 // POST /api/v1/chat/completions
 //
@@ -371,7 +386,7 @@ func (h *CompletionHandler) multiModelStream(
 		}

 		// Stream this model's response using the shared streaming core
-		result := streamModelResponse(c, provider, providerCfg, &provReq, model, target.DisplayName, userID, channelID, personaID, workspaceID, h.hub)
+		result := streamModelResponse(c, provider, providerCfg, &provReq, model, target.DisplayName, userID, channelID, personaID, workspaceID, configID, h.hub, h.health)

 		// Persist assistant message with model attribution
 		if result.Content != "" {
@@ -536,7 +551,7 @@ func (h *CompletionHandler) streamCompletion(
 	req providers.CompletionRequest,
 	channelID, userID, model, configID, providerScope, personaID, workspaceID string,
 ) {
-	result := streamWithToolLoop(c, provider, cfg, &req, model, userID, channelID, personaID, workspaceID, h.hub)
+	result := streamWithToolLoop(c, provider, cfg, &req, model, userID, channelID, personaID, workspaceID, configID, h.hub, h.health)

 	// Persist assistant response
 	if result.Content != "" {
@@ -566,7 +581,9 @@ func (h *CompletionHandler) syncCompletion(
 	var allToolActivity []map[string]interface{}

 	for iteration := 0; iteration < maxToolIterations; iteration++ {
+		callStart := time.Now()
 		resp, err := provider.ChatCompletion(c.Request.Context(), cfg, req)
+		h.recordHealth(configID, callStart, err)
 		if err != nil {
 			c.JSON(http.StatusBadGateway, gin.H{"error": "provider error: " + err.Error()})
 			return
@@ -1175,3 +1192,21 @@ func calcCost(tokens int, pricePerM *float64) *float64 {
 	cost := float64(tokens) * *pricePerM / 1_000_000.0
 	return &cost
 }
+
+// recordHealth records provider call outcome for health tracking (v0.22.0).
+func (h *CompletionHandler) recordHealth(configID string, start time.Time, err error) {
+	if h.health == nil || configID == "" {
+		return
+	}
+	latencyMs := int(time.Since(start).Milliseconds())
+	if err != nil {
+		errMsg := err.Error()
+		if strings.Contains(errMsg, "context deadline exceeded") || strings.Contains(errMsg, "timeout") {
+			h.health.RecordTimeout(configID, latencyMs, errMsg)
+		} else {
+			h.health.RecordError(configID, latencyMs, errMsg)
+		}
+	} else {
+		h.health.RecordSuccess(configID, latencyMs)
+	}
+}
--- a/server/handlers/health_admin.go
+++ b/server/handlers/health_admin.go
@@ -0,0 +1,280 @@
+package handlers
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+
+	capspkg "git.gobha.me/xcaliber/chat-switchboard/capabilities"
+	"git.gobha.me/xcaliber/chat-switchboard/health"
+	"git.gobha.me/xcaliber/chat-switchboard/models"
+	"git.gobha.me/xcaliber/chat-switchboard/store"
+)
+
+// ── Health Admin Handler ────────────────────
+
+type HealthAdminHandler struct {
+	healthStore health.Store
+	stores      store.Stores
+}
+
+func NewHealthAdminHandler(hs health.Store, stores store.Stores) *HealthAdminHandler {
+	return &HealthAdminHandler{healthStore: hs, stores: stores}
+}
+
+// GetProviderHealth returns health metrics for a single provider.
+// GET /api/v1/admin/providers/:id/health?hours=24
+func (h *HealthAdminHandler) GetProviderHealth(c *gin.Context) {
+	providerID := c.Param("id")
+	hours, _ := strconv.Atoi(c.DefaultQuery("hours", "24"))
+	if hours < 1 || hours > 168 {
+		hours = 24
+	}
+
+	windows, err := h.healthStore.ListWindows(c.Request.Context(), providerID, hours)
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to fetch health data"})
+		return
+	}
+
+	// Derive current status from most recent window
+	var summary models.ProviderHealthSummary
+	summary.ProviderConfigID = providerID
+
+	if len(windows) > 0 {
+		latest := windows[0]
+		summary.RequestCount = latest.RequestCount
+		summary.ErrorRate = latest.ErrorRate()
+		summary.AvgLatencyMs = latest.AvgLatencyMs()
+		summary.MaxLatencyMs = latest.MaxLatencyMs
+		summary.LastError = latest.LastError
+		summary.LastErrorAt = latest.LastErrorAt
+		summary.Status = health.DeriveStatus(summary.ErrorRate, summary.RequestCount)
+	} else {
+		summary.Status = models.StatusUnknown
+	}
+
+	SafeJSON(c, http.StatusOK, gin.H{
+		"summary": summary,
+		"windows": windows,
+	})
+}
+
+// GetAllProviderHealth returns current health for all providers.
+// GET /api/v1/admin/providers/health
+func (h *HealthAdminHandler) GetAllProviderHealth(c *gin.Context) {
+	windows, err := h.healthStore.ListAllCurrent(c.Request.Context())
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to fetch health data"})
+		return
+	}
+
+	// Build summaries
+	summaries := make([]models.ProviderHealthSummary, 0, len(windows))
+	for _, w := range windows {
+		s := models.ProviderHealthSummary{
+			ProviderConfigID: w.ProviderConfigID,
+			Status:           health.DeriveStatus(w.ErrorRate(), w.RequestCount),
+			RequestCount:     w.RequestCount,
+			ErrorRate:        w.ErrorRate(),
+			AvgLatencyMs:     w.AvgLatencyMs(),
+			MaxLatencyMs:     w.MaxLatencyMs,
+			LastError:        w.LastError,
+			LastErrorAt:      w.LastErrorAt,
+		}
+		summaries = append(summaries, s)
+	}
+
+	SafeJSON(c, http.StatusOK, gin.H{
+		"data": summaries,
+	})
+}
+
+// ── Capability Override Admin Handler ────────
+
+type CapOverrideAdminHandler struct {
+	stores store.Stores
+}
+
+func NewCapOverrideAdminHandler(stores store.Stores) *CapOverrideAdminHandler {
+	return &CapOverrideAdminHandler{stores: stores}
+}
+
+// GetModelCapabilities returns resolved capabilities with source annotation.
+// GET /api/v1/admin/models/:id/capabilities?provider_config_id=...
+func (h *CapOverrideAdminHandler) GetModelCapabilities(c *gin.Context) {
+	modelID := c.Param("id")
+	providerConfigID := c.Query("provider_config_id")
+
+	// Get catalog capabilities
+	var catalogCaps *models.ModelCapabilities
+	if providerConfigID != "" {
+		if entry, err := h.stores.Catalog.GetByModelID(c.Request.Context(), providerConfigID, modelID); err == nil {
+			catalogCaps = &entry.Capabilities
+		}
+	} else {
+		if entry, err := h.stores.Catalog.GetByModelIDAny(c.Request.Context(), modelID); err == nil {
+			catalogCaps = &entry.Capabilities
+		}
+	}
+
+	// Get admin overrides
+	var overrides []models.CapabilityOverride
+	var err error
+	if providerConfigID != "" {
+		overrides, err = h.stores.CapOverrides.ListForProviderModel(c.Request.Context(), providerConfigID, modelID)
+	} else {
+		overrides, err = h.stores.CapOverrides.ListForModel(c.Request.Context(), modelID)
+	}
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to fetch overrides"})
+		return
+	}
+
+	// Resolve through three tiers
+	heuristic := capspkg.InferCapabilities(modelID)
+	resolved := capspkg.ResolveIntrinsic(modelID, catalogCaps, overrides)
+
+	// Build source annotations
+	sources := buildSourceAnnotations(catalogCaps, &heuristic, overrides)
+
+	SafeJSON(c, http.StatusOK, gin.H{
+		"model_id":    modelID,
+		"resolved":    resolved,
+		"catalog":     catalogCaps,
+		"heuristic":   heuristic,
+		"overrides":   overrides,
+		"sources":     sources,
+	})
+}
+
+// SetModelCapability sets an admin override for a specific capability.
+// PUT /api/v1/admin/models/:id/capabilities
+func (h *CapOverrideAdminHandler) SetModelCapability(c *gin.Context) {
+	modelID := c.Param("id")
+	userID := c.GetString("user_id")
+
+	var req struct {
+		ProviderConfigID *string `json:"provider_config_id"`
+		Field            string  `json:"field" binding:"required"`
+		Value            string  `json:"value" binding:"required"`
+	}
+	if err := c.ShouldBindJSON(&req); err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+		return
+	}
+
+	// Validate field name
+	validFields := map[string]bool{
+		"streaming": true, "tool_calling": true, "vision": true,
+		"thinking": true, "reasoning": true, "code_optimized": true,
+		"web_search": true, "max_context": true, "max_output_tokens": true,
+	}
+	if !validFields[req.Field] {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "invalid capability field: " + req.Field})
+		return
+	}
+
+	// Validate value
+	switch req.Field {
+	case "max_context", "max_output_tokens":
+		if _, err := strconv.Atoi(req.Value); err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "value must be a number for " + req.Field})
+			return
+		}
+	default:
+		if req.Value != "true" && req.Value != "false" {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "value must be 'true' or 'false' for " + req.Field})
+			return
+		}
+	}
+
+	override := &models.CapabilityOverride{
+		ProviderConfigID: req.ProviderConfigID,
+		ModelID:          modelID,
+		Field:            req.Field,
+		Value:            req.Value,
+		SetBy:            &userID,
+	}
+
+	if err := h.stores.CapOverrides.Set(c.Request.Context(), override); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to set override"})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"status": "ok"})
+}
+
+// DeleteModelCapability removes a specific override.
+// DELETE /api/v1/admin/models/:id/capabilities/:overrideId
+func (h *CapOverrideAdminHandler) DeleteModelCapability(c *gin.Context) {
+	overrideID := c.Param("overrideId")
+
+	if err := h.stores.CapOverrides.Delete(c.Request.Context(), overrideID); err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete override"})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"status": "ok"})
+}
+
+// ListAllOverrides returns all capability overrides (admin view).
+// GET /api/v1/admin/capability-overrides
+func (h *CapOverrideAdminHandler) ListAllOverrides(c *gin.Context) {
+	overrides, err := h.stores.CapOverrides.ListAll(c.Request.Context())
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to fetch overrides"})
+		return
+	}
+
+	SafeJSON(c, http.StatusOK, gin.H{
+		"data":  overrides,
+		"total": len(overrides),
+	})
+}
+
+// ── Source Annotation Helpers ────────────────
+
+type capSource struct {
+	Field  string `json:"field"`
+	Source string `json:"source"` // "catalog", "heuristic", "override"
+}
+
+func buildSourceAnnotations(catalog *models.ModelCapabilities, heuristic *models.ModelCapabilities, overrides []models.CapabilityOverride) []capSource {
+	// Build override set for quick lookup
+	overrideSet := make(map[string]bool, len(overrides))
+	for _, o := range overrides {
+		overrideSet[o.Field] = true
+	}
+
+	fields := []struct {
+		name       string
+		hasCatalog bool
+		hasHeur    bool
+	}{
+		{"streaming", catalog != nil && catalog.Streaming, heuristic.Streaming},
+		{"tool_calling", catalog != nil && catalog.ToolCalling, heuristic.ToolCalling},
+		{"vision", catalog != nil && catalog.Vision, heuristic.Vision},
+		{"thinking", catalog != nil && catalog.Thinking, heuristic.Thinking},
+		{"reasoning", catalog != nil && catalog.Reasoning, heuristic.Reasoning},
+		{"code_optimized", catalog != nil && catalog.CodeOptimized, heuristic.CodeOptimized},
+		{"web_search", catalog != nil && catalog.WebSearch, heuristic.WebSearch},
+		{"max_context", catalog != nil && catalog.MaxContext > 0, heuristic.MaxContext > 0},
+		{"max_output_tokens", catalog != nil && catalog.MaxOutputTokens > 0, heuristic.MaxOutputTokens > 0},
+	}
+
+	var sources []capSource
+	for _, f := range fields {
+		src := "default"
+		if overrideSet[f.name] {
+			src = "override"
+		} else if f.hasCatalog {
+			src = "catalog"
+		} else if f.hasHeur {
+			src = "heuristic"
+		}
+		sources = append(sources, capSource{Field: f.name, Source: src})
+	}
+	return sources
+}
--- a/server/handlers/messages.go
+++ b/server/handlers/messages.go
@@ -516,7 +516,7 @@ func (h *MessageHandler) Regenerate(c *gin.Context) {

 	// ── Stream the response (shared loop handles tools, reasoning, SSE) ──

-	result := streamWithToolLoop(c, provider, providerCfg, &provReq, model, userID, channelID, personaID, workspaceID, h.hub)
+	result := streamWithToolLoop(c, provider, providerCfg, &provReq, model, userID, channelID, personaID, workspaceID, configID, h.hub, comp.health)

 	// Persist as sibling (regen) with tool activity
 	if result.Content != "" {
--- a/server/handlers/stream_loop.go
+++ b/server/handlers/stream_loop.go
@@ -27,6 +27,20 @@ type streamResult struct {
 	CacheReadTokens     int
 }

+// recordHealthFn records provider health from standalone streaming functions.
+func recordHealthFn(hr HealthRecorder, configID string, start time.Time, err error) {
+	if hr == nil || configID == "" {
+		return
+	}
+	latencyMs := int(time.Since(start).Milliseconds())
+	if err != nil {
+		errMsg := err.Error()
+		hr.RecordError(configID, latencyMs, errMsg)
+	} else {
+		hr.RecordSuccess(configID, latencyMs)
+	}
+}
+
 // streamWithToolLoop is the single, canonical streaming implementation.
 // It handles SSE setup, multi-round tool execution, reasoning_content
 // forwarding, and client notifications. Returns the accumulated content
@@ -43,8 +57,9 @@ func streamWithToolLoop(
 	provider providers.Provider,
 	cfg providers.ProviderConfig,
 	req *providers.CompletionRequest,
-	model, userID, channelID, personaID, workspaceID string,
+	model, userID, channelID, personaID, workspaceID, configID string,
 	hub *events.Hub,
+	health HealthRecorder,
 ) streamResult {
 	// Set SSE headers
 	c.Header("Content-Type", "text/event-stream")
@@ -74,8 +89,10 @@ func streamWithToolLoop(
 	var result streamResult

 	for iteration := 0; iteration < maxToolIterations; iteration++ {
+		callStart := time.Now()
 		ch, err := provider.StreamCompletion(c.Request.Context(), cfg, *req)
 		if err != nil {
+			recordHealthFn(health, configID, callStart, err)
 			sendSSE(fmt.Sprintf(`{"error":"%s"}`, escapeJSON(err.Error())))
 			return result
 		}
@@ -86,6 +103,7 @@ func streamWithToolLoop(

 		for event := range ch {
 			if event.Error != nil {
+				recordHealthFn(health, configID, callStart, event.Error)
 				sendSSE(fmt.Sprintf(`{"error":"%s"}`, escapeJSON(event.Error.Error())))
 				return result
 			}
@@ -107,8 +125,9 @@ func streamWithToolLoop(
 			}

 			if event.Done {
+				recordHealthFn(health, configID, callStart, nil)
 				if event.FinishReason == "tool_calls" && len(event.ToolCalls) > 0 {
-					toolCalls = event.ToolCalls
+						toolCalls = event.ToolCalls
 					// Accumulate tokens from this tool-call iteration
 					result.InputTokens += event.InputTokens
 					result.OutputTokens += event.OutputTokens
@@ -268,8 +287,9 @@ func streamModelResponse(
 	provider providers.Provider,
 	cfg providers.ProviderConfig,
 	req *providers.CompletionRequest,
-	model, displayName, userID, channelID, personaID, workspaceID string,
+	model, displayName, userID, channelID, personaID, workspaceID, configID string,
 	hub *events.Hub,
+	health HealthRecorder,
 ) streamResult {
 	flusher, _ := c.Writer.(http.Flusher)
 	flush := func() {
@@ -289,8 +309,10 @@ func streamModelResponse(
 	var result streamResult

 	for iteration := 0; iteration < maxToolIterations; iteration++ {
+		callStart := time.Now()
 		ch, err := provider.StreamCompletion(c.Request.Context(), cfg, *req)
 		if err != nil {
+			recordHealthFn(health, configID, callStart, err)
 			sendSSE(fmt.Sprintf(`{"error":"%s"}`, escapeJSON(err.Error())))
 			return result
 		}
@@ -301,6 +323,7 @@ func streamModelResponse(

 		for event := range ch {
 			if event.Error != nil {
+				recordHealthFn(health, configID, callStart, event.Error)
 				sendSSE(fmt.Sprintf(`{"error":"%s"}`, escapeJSON(event.Error.Error())))
 				return result
 			}
@@ -320,6 +343,7 @@ func streamModelResponse(
 			}

 			if event.Done {
+				recordHealthFn(health, configID, callStart, nil)
 				if event.FinishReason == "tool_calls" && len(event.ToolCalls) > 0 {
 					toolCalls = event.ToolCalls
 					result.InputTokens += event.InputTokens
--- a/server/handlers/team_providers.go
+++ b/server/handlers/team_providers.go
@@ -310,7 +310,7 @@ func (h *TeamHandler) ListTeamProviderModels(c *gin.Context) {

 	out := make([]modelInfo, 0, len(modelList))
 	for _, m := range modelList {
-		caps := capspkg.ResolveIntrinsic(m.ID, &m.Capabilities)
+		caps := capspkg.ResolveIntrinsic(m.ID, &m.Capabilities, nil)
 		caps.MaxOutputTokens = capspkg.ResolveMaxOutput(m.ID, caps)
 		out = append(out, modelInfo{ID: m.ID, Capabilities: caps})
 	}