Changeset 0.22.0.1 (#94)

This commit is contained in:
2026-03-02 01:55:19 +00:00
parent 12e03cec8b
commit 06c4e2a5a1
33 changed files with 1929 additions and 418 deletions

View File

@@ -154,7 +154,7 @@ func TestResolveIntrinsic_CatalogWins(t *testing.T) {
ToolCalling: true,
MaxOutputTokens: 16384,
}
merged := ResolveIntrinsic("claude-sonnet-4-20250514", &providerCaps)
merged := ResolveIntrinsic("claude-sonnet-4-20250514", &providerCaps, nil)
if !merged.ToolCalling {
t.Error("tool_calling should be preserved from provider")
@@ -179,7 +179,7 @@ func TestResolveIntrinsic_ProviderDataPreserved(t *testing.T) {
MaxOutputTokens: 8192,
MaxContext: 65536,
}
merged := ResolveIntrinsic("some-unknown-model", &providerCaps)
merged := ResolveIntrinsic("some-unknown-model", &providerCaps, nil)
if !merged.ToolCalling {
t.Error("tool_calling should be preserved from provider")
@@ -191,7 +191,7 @@ func TestResolveIntrinsic_ProviderDataPreserved(t *testing.T) {
func TestResolveIntrinsic_NilProvider(t *testing.T) {
// Nil provider caps — falls through entirely to heuristic
merged := ResolveIntrinsic("gpt-4o", nil)
merged := ResolveIntrinsic("gpt-4o", nil, nil)
if !merged.ToolCalling {
t.Error("should get tool_calling from heuristic (gpt-4 pattern)")
@@ -203,7 +203,7 @@ func TestResolveIntrinsic_NilProvider(t *testing.T) {
func TestResolveIntrinsic_HeuristicOnly(t *testing.T) {
// No catalog, no known table — pure heuristic
merged := ResolveIntrinsic("deepseek-r1-distill-llama-70b", nil)
merged := ResolveIntrinsic("deepseek-r1-distill-llama-70b", nil, nil)
if !merged.Reasoning {
t.Error("should infer reasoning from deepseek-r1 pattern")
@@ -229,3 +229,71 @@ func TestHasProviderData(t *testing.T) {
t.Error("caps with max_context should have provider data")
}
}
func TestResolveIntrinsic_AdminOverrides(t *testing.T) {
// Catalog says no vision, heuristic says no vision,
// but admin override forces vision=true.
catalogCaps := &models.ModelCapabilities{
ToolCalling: true,
Vision: false,
}
overrides := []models.CapabilityOverride{
{Field: "vision", Value: "true"},
}
merged := ResolveIntrinsic("some-custom-model", catalogCaps, overrides)
if !merged.Vision {
t.Error("admin override should force vision=true")
}
if !merged.ToolCalling {
t.Error("catalog tool_calling should be preserved")
}
}
func TestResolveIntrinsic_AdminOverrides_DisableCapability(t *testing.T) {
// Heuristic infers tool_calling for gpt-4o, but admin says no.
overrides := []models.CapabilityOverride{
{Field: "tool_calling", Value: "false"},
}
merged := ResolveIntrinsic("gpt-4o", nil, overrides)
if merged.ToolCalling {
t.Error("admin override should disable tool_calling")
}
}
func TestResolveIntrinsic_AdminOverrides_IntValues(t *testing.T) {
overrides := []models.CapabilityOverride{
{Field: "max_context", Value: "200000"},
{Field: "max_output_tokens", Value: "16384"},
}
merged := ResolveIntrinsic("some-model", nil, overrides)
if merged.MaxContext != 200000 {
t.Errorf("expected max_context=200000, got %d", merged.MaxContext)
}
if merged.MaxOutputTokens != 16384 {
t.Errorf("expected max_output_tokens=16384, got %d", merged.MaxOutputTokens)
}
}
func TestResolveIntrinsic_OverridePriority(t *testing.T) {
// Catalog says max_context=128000, admin overrides to 256000.
// Admin should win.
catalogCaps := &models.ModelCapabilities{
MaxContext: 128000,
}
overrides := []models.CapabilityOverride{
{Field: "max_context", Value: "256000"},
}
merged := ResolveIntrinsic("some-model", catalogCaps, overrides)
if merged.MaxContext != 256000 {
t.Errorf("admin override should win over catalog: expected 256000, got %d", merged.MaxContext)
}
}

View File

@@ -111,14 +111,16 @@ func InferCapabilities(modelID string) models.ModelCapabilities {
}
// ResolveIntrinsic determines the intrinsic capabilities of a model.
// Priority:
// 1. catalogCaps (from model_catalog DB — synced from provider API)
// 2. Heuristic inference (regex patterns on model ID)
// Priority (highest wins):
// 1. Admin overrides (manual corrections from capability_overrides table)
// 2. Catalog (from model_catalog DB — synced from provider API)
// 3. Heuristic inference (regex patterns on model ID)
//
// The catalog is the source of truth per provider. Heuristics fill gaps
// for models that haven't been synced yet. No hardcoded model table —
// for models that haven't been synced yet. Admin overrides correct
// provider-reported or heuristic errors. No hardcoded model table —
// the same model can have different capabilities on different providers.
func ResolveIntrinsic(modelID string, catalogCaps *models.ModelCapabilities) models.ModelCapabilities {
func ResolveIntrinsic(modelID string, catalogCaps *models.ModelCapabilities, overrides []models.CapabilityOverride) models.ModelCapabilities {
// Start with catalog data if available
var base models.ModelCapabilities
if catalogCaps != nil && catalogCaps.HasProviderData() {
@@ -128,9 +130,56 @@ func ResolveIntrinsic(modelID string, catalogCaps *models.ModelCapabilities) mod
// Fill gaps from heuristic inference
inferred := InferCapabilities(modelID)
mergeGaps(&base, &inferred)
// Apply admin overrides (highest priority — can flip any field)
applyOverrides(&base, overrides)
return base
}
// applyOverrides applies admin capability corrections to the resolved capabilities.
func applyOverrides(caps *models.ModelCapabilities, overrides []models.CapabilityOverride) {
for _, o := range overrides {
boolVal := o.Value == "true" || o.Value == "1"
switch o.Field {
case "streaming":
caps.Streaming = boolVal
case "tool_calling":
caps.ToolCalling = boolVal
case "vision":
caps.Vision = boolVal
case "thinking":
caps.Thinking = boolVal
case "reasoning":
caps.Reasoning = boolVal
case "code_optimized":
caps.CodeOptimized = boolVal
case "web_search":
caps.WebSearch = boolVal
case "max_context":
if n := parseInt(o.Value); n > 0 {
caps.MaxContext = n
}
case "max_output_tokens":
if n := parseInt(o.Value); n > 0 {
caps.MaxOutputTokens = n
}
}
}
}
// parseInt parses a string to int, returning 0 on failure.
func parseInt(s string) int {
n := 0
for _, c := range s {
if c < '0' || c > '9' {
return 0
}
n = n*10 + int(c-'0')
}
return n
}
// ResolveMaxOutput returns the max output tokens for a model.
// Priority: explicit caps → derive from context → 4096 fallback.
func ResolveMaxOutput(modelID string, caps models.ModelCapabilities) int {

View File

@@ -63,7 +63,7 @@ func ModelsForUser(ctx context.Context, stores store.Stores, userID string) ([]m
countGlobal := len(globalModels)
for _, entry := range globalModels {
caps := ResolveIntrinsic(entry.ModelID, &entry.Capabilities)
caps := ResolveIntrinsic(entry.ModelID, &entry.Capabilities, nil)
prov := providerMap[entry.ProviderConfigID]
result = append(result, models.UserModel{
@@ -100,7 +100,7 @@ func ModelsForUser(ctx context.Context, stores store.Stores, userID string) ([]m
continue
}
for _, entry := range entries {
caps := ResolveIntrinsic(entry.ModelID, &entry.Capabilities)
caps := ResolveIntrinsic(entry.ModelID, &entry.Capabilities, nil)
result = append(result, models.UserModel{
ID: entry.ModelID,
DisplayName: displayName(entry.DisplayName, entry.ModelID),
@@ -138,7 +138,7 @@ func ModelsForUser(ctx context.Context, stores store.Stores, userID string) ([]m
continue
}
for _, entry := range entries {
caps := ResolveIntrinsic(entry.ModelID, &entry.Capabilities)
caps := ResolveIntrinsic(entry.ModelID, &entry.Capabilities, nil)
result = append(result, models.UserModel{
ID: entry.ModelID,
DisplayName: displayName(entry.DisplayName, entry.ModelID),
@@ -181,7 +181,7 @@ func ModelsForUser(ctx context.Context, stores store.Stores, userID string) ([]m
catalogCaps = &entry.Capabilities
}
}
caps := ResolveIntrinsic(p.BaseModelID, catalogCaps)
caps := ResolveIntrinsic(p.BaseModelID, catalogCaps, nil)
// Load tool grants
toolGrants, _ := stores.Personas.GetToolGrants(ctx, p.ID)
@@ -251,7 +251,7 @@ func ResolveForPersona(ctx context.Context, stores store.Stores, persona *models
catalogCaps = &entry.Capabilities
}
}
caps := ResolveIntrinsic(persona.BaseModelID, catalogCaps)
caps := ResolveIntrinsic(persona.BaseModelID, catalogCaps, nil)
toolGrants, err := stores.Personas.GetToolGrants(ctx, persona.ID)
if err != nil {

View File

@@ -0,0 +1,43 @@
-- v0.22.0: Provider health tracking + capability admin overrides.
-- ── Provider Health ─────────────────────────
-- Hourly bucketed health metrics per provider config.
-- In-memory accumulator flushes to this table every 60s.
-- Background job prunes rows older than 7 days.
CREATE TABLE IF NOT EXISTS provider_health (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
provider_config_id UUID NOT NULL REFERENCES provider_configs(id) ON DELETE CASCADE,
window_start TIMESTAMPTZ NOT NULL, -- hourly bucket floor
request_count INT NOT NULL DEFAULT 0,
error_count INT NOT NULL DEFAULT 0,
timeout_count INT NOT NULL DEFAULT 0,
total_latency_ms BIGINT NOT NULL DEFAULT 0, -- sum for avg calculation
max_latency_ms INT NOT NULL DEFAULT 0,
last_error TEXT,
last_error_at TIMESTAMPTZ,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
UNIQUE (provider_config_id, window_start)
);
CREATE INDEX IF NOT EXISTS idx_provider_health_window
ON provider_health (provider_config_id, window_start DESC);
-- ── Capability Overrides ────────────────────
-- Admin corrections for model capabilities. Highest priority in the
-- three-tier resolution chain: catalog → heuristic → admin override.
CREATE TABLE IF NOT EXISTS capability_overrides (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
provider_config_id UUID REFERENCES provider_configs(id) ON DELETE CASCADE,
model_id TEXT NOT NULL,
field TEXT NOT NULL, -- tool_calling, vision, thinking, reasoning, etc.
value TEXT NOT NULL, -- "true"/"false" for bools, numeric string for ints
set_by UUID REFERENCES users(id) ON DELETE SET NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
-- Unique per (provider, model, field). NULL provider = applies to model globally.
UNIQUE (provider_config_id, model_id, field)
);
CREATE INDEX IF NOT EXISTS idx_capability_overrides_model
ON capability_overrides (model_id);

View File

@@ -0,0 +1,35 @@
-- v0.22.0: Provider health tracking + capability admin overrides.
CREATE TABLE IF NOT EXISTS provider_health (
id TEXT PRIMARY KEY,
provider_config_id TEXT NOT NULL,
window_start TEXT NOT NULL,
request_count INTEGER NOT NULL DEFAULT 0,
error_count INTEGER NOT NULL DEFAULT 0,
timeout_count INTEGER NOT NULL DEFAULT 0,
total_latency_ms INTEGER NOT NULL DEFAULT 0,
max_latency_ms INTEGER NOT NULL DEFAULT 0,
last_error TEXT,
last_error_at TEXT,
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
UNIQUE (provider_config_id, window_start)
);
CREATE INDEX IF NOT EXISTS idx_provider_health_window
ON provider_health (provider_config_id, window_start);
CREATE TABLE IF NOT EXISTS capability_overrides (
id TEXT PRIMARY KEY,
provider_config_id TEXT,
model_id TEXT NOT NULL,
field TEXT NOT NULL,
value TEXT NOT NULL,
set_by TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
UNIQUE (provider_config_id, model_id, field)
);
CREATE INDEX IF NOT EXISTS idx_capability_overrides_model
ON capability_overrides (model_id);

View File

@@ -94,7 +94,7 @@ func capsFromCatalog(modelID, configID string) (models.ModelCapabilities, bool)
}
// Merge with known data to fill gaps
resolved := capspkg.ResolveIntrinsic(modelID, &caps)
resolved := capspkg.ResolveIntrinsic(modelID, &caps, nil)
return resolved, true
}
@@ -140,7 +140,7 @@ func liveQueryModelCaps(c *gin.Context, configID, modelID string) (models.ModelC
for _, m := range modelList {
if m.ID == modelID {
resolved := capspkg.ResolveIntrinsic(modelID, &m.Capabilities)
resolved := capspkg.ResolveIntrinsic(modelID, &m.Capabilities, nil)
return resolved, true
}
}

View File

@@ -10,6 +10,7 @@ import (
"log"
"net/http"
"strings"
"time"
"github.com/gin-gonic/gin"
@@ -50,6 +51,15 @@ type CompletionHandler struct {
hub *events.Hub // WebSocket hub for browser tool bridge
objStore storage.ObjectStore // file storage for attachment content (nil = disabled)
embedder *knowledge.Embedder // for memory semantic recall (v0.18.0)
health HealthRecorder // provider health tracking (v0.22.0, nil = disabled)
}
// HealthRecorder is the interface for recording provider call outcomes.
// Matches health.Accumulator methods without importing the package.
type HealthRecorder interface {
RecordSuccess(providerConfigID string, latencyMs int)
RecordError(providerConfigID string, latencyMs int, errMsg string)
RecordTimeout(providerConfigID string, latencyMs int, errMsg string)
}
// NewCompletionHandler creates a new handler.
@@ -57,6 +67,11 @@ func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *e
return &CompletionHandler{vault: vault, stores: stores, hub: hub, objStore: objStore, embedder: embedder}
}
// SetHealthRecorder attaches the provider health accumulator.
func (h *CompletionHandler) SetHealthRecorder(hr HealthRecorder) {
h.health = hr
}
// ── Chat Completion ─────────────────────────
// POST /api/v1/chat/completions
//
@@ -371,7 +386,7 @@ func (h *CompletionHandler) multiModelStream(
}
// Stream this model's response using the shared streaming core
result := streamModelResponse(c, provider, providerCfg, &provReq, model, target.DisplayName, userID, channelID, personaID, workspaceID, h.hub)
result := streamModelResponse(c, provider, providerCfg, &provReq, model, target.DisplayName, userID, channelID, personaID, workspaceID, configID, h.hub, h.health)
// Persist assistant message with model attribution
if result.Content != "" {
@@ -536,7 +551,7 @@ func (h *CompletionHandler) streamCompletion(
req providers.CompletionRequest,
channelID, userID, model, configID, providerScope, personaID, workspaceID string,
) {
result := streamWithToolLoop(c, provider, cfg, &req, model, userID, channelID, personaID, workspaceID, h.hub)
result := streamWithToolLoop(c, provider, cfg, &req, model, userID, channelID, personaID, workspaceID, configID, h.hub, h.health)
// Persist assistant response
if result.Content != "" {
@@ -566,7 +581,9 @@ func (h *CompletionHandler) syncCompletion(
var allToolActivity []map[string]interface{}
for iteration := 0; iteration < maxToolIterations; iteration++ {
callStart := time.Now()
resp, err := provider.ChatCompletion(c.Request.Context(), cfg, req)
h.recordHealth(configID, callStart, err)
if err != nil {
c.JSON(http.StatusBadGateway, gin.H{"error": "provider error: " + err.Error()})
return
@@ -1175,3 +1192,21 @@ func calcCost(tokens int, pricePerM *float64) *float64 {
cost := float64(tokens) * *pricePerM / 1_000_000.0
return &cost
}
// recordHealth records provider call outcome for health tracking (v0.22.0).
func (h *CompletionHandler) recordHealth(configID string, start time.Time, err error) {
if h.health == nil || configID == "" {
return
}
latencyMs := int(time.Since(start).Milliseconds())
if err != nil {
errMsg := err.Error()
if strings.Contains(errMsg, "context deadline exceeded") || strings.Contains(errMsg, "timeout") {
h.health.RecordTimeout(configID, latencyMs, errMsg)
} else {
h.health.RecordError(configID, latencyMs, errMsg)
}
} else {
h.health.RecordSuccess(configID, latencyMs)
}
}

View File

@@ -0,0 +1,280 @@
package handlers
import (
"net/http"
"strconv"
"github.com/gin-gonic/gin"
capspkg "git.gobha.me/xcaliber/chat-switchboard/capabilities"
"git.gobha.me/xcaliber/chat-switchboard/health"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// ── Health Admin Handler ────────────────────
type HealthAdminHandler struct {
healthStore health.Store
stores store.Stores
}
func NewHealthAdminHandler(hs health.Store, stores store.Stores) *HealthAdminHandler {
return &HealthAdminHandler{healthStore: hs, stores: stores}
}
// GetProviderHealth returns health metrics for a single provider.
// GET /api/v1/admin/providers/:id/health?hours=24
func (h *HealthAdminHandler) GetProviderHealth(c *gin.Context) {
providerID := c.Param("id")
hours, _ := strconv.Atoi(c.DefaultQuery("hours", "24"))
if hours < 1 || hours > 168 {
hours = 24
}
windows, err := h.healthStore.ListWindows(c.Request.Context(), providerID, hours)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to fetch health data"})
return
}
// Derive current status from most recent window
var summary models.ProviderHealthSummary
summary.ProviderConfigID = providerID
if len(windows) > 0 {
latest := windows[0]
summary.RequestCount = latest.RequestCount
summary.ErrorRate = latest.ErrorRate()
summary.AvgLatencyMs = latest.AvgLatencyMs()
summary.MaxLatencyMs = latest.MaxLatencyMs
summary.LastError = latest.LastError
summary.LastErrorAt = latest.LastErrorAt
summary.Status = health.DeriveStatus(summary.ErrorRate, summary.RequestCount)
} else {
summary.Status = models.StatusUnknown
}
SafeJSON(c, http.StatusOK, gin.H{
"summary": summary,
"windows": windows,
})
}
// GetAllProviderHealth returns current health for all providers.
// GET /api/v1/admin/providers/health
func (h *HealthAdminHandler) GetAllProviderHealth(c *gin.Context) {
windows, err := h.healthStore.ListAllCurrent(c.Request.Context())
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to fetch health data"})
return
}
// Build summaries
summaries := make([]models.ProviderHealthSummary, 0, len(windows))
for _, w := range windows {
s := models.ProviderHealthSummary{
ProviderConfigID: w.ProviderConfigID,
Status: health.DeriveStatus(w.ErrorRate(), w.RequestCount),
RequestCount: w.RequestCount,
ErrorRate: w.ErrorRate(),
AvgLatencyMs: w.AvgLatencyMs(),
MaxLatencyMs: w.MaxLatencyMs,
LastError: w.LastError,
LastErrorAt: w.LastErrorAt,
}
summaries = append(summaries, s)
}
SafeJSON(c, http.StatusOK, gin.H{
"data": summaries,
})
}
// ── Capability Override Admin Handler ────────
type CapOverrideAdminHandler struct {
stores store.Stores
}
func NewCapOverrideAdminHandler(stores store.Stores) *CapOverrideAdminHandler {
return &CapOverrideAdminHandler{stores: stores}
}
// GetModelCapabilities returns resolved capabilities with source annotation.
// GET /api/v1/admin/models/:id/capabilities?provider_config_id=...
func (h *CapOverrideAdminHandler) GetModelCapabilities(c *gin.Context) {
modelID := c.Param("id")
providerConfigID := c.Query("provider_config_id")
// Get catalog capabilities
var catalogCaps *models.ModelCapabilities
if providerConfigID != "" {
if entry, err := h.stores.Catalog.GetByModelID(c.Request.Context(), providerConfigID, modelID); err == nil {
catalogCaps = &entry.Capabilities
}
} else {
if entry, err := h.stores.Catalog.GetByModelIDAny(c.Request.Context(), modelID); err == nil {
catalogCaps = &entry.Capabilities
}
}
// Get admin overrides
var overrides []models.CapabilityOverride
var err error
if providerConfigID != "" {
overrides, err = h.stores.CapOverrides.ListForProviderModel(c.Request.Context(), providerConfigID, modelID)
} else {
overrides, err = h.stores.CapOverrides.ListForModel(c.Request.Context(), modelID)
}
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to fetch overrides"})
return
}
// Resolve through three tiers
heuristic := capspkg.InferCapabilities(modelID)
resolved := capspkg.ResolveIntrinsic(modelID, catalogCaps, overrides)
// Build source annotations
sources := buildSourceAnnotations(catalogCaps, &heuristic, overrides)
SafeJSON(c, http.StatusOK, gin.H{
"model_id": modelID,
"resolved": resolved,
"catalog": catalogCaps,
"heuristic": heuristic,
"overrides": overrides,
"sources": sources,
})
}
// SetModelCapability sets an admin override for a specific capability.
// PUT /api/v1/admin/models/:id/capabilities
func (h *CapOverrideAdminHandler) SetModelCapability(c *gin.Context) {
modelID := c.Param("id")
userID := c.GetString("user_id")
var req struct {
ProviderConfigID *string `json:"provider_config_id"`
Field string `json:"field" binding:"required"`
Value string `json:"value" binding:"required"`
}
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
// Validate field name
validFields := map[string]bool{
"streaming": true, "tool_calling": true, "vision": true,
"thinking": true, "reasoning": true, "code_optimized": true,
"web_search": true, "max_context": true, "max_output_tokens": true,
}
if !validFields[req.Field] {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid capability field: " + req.Field})
return
}
// Validate value
switch req.Field {
case "max_context", "max_output_tokens":
if _, err := strconv.Atoi(req.Value); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "value must be a number for " + req.Field})
return
}
default:
if req.Value != "true" && req.Value != "false" {
c.JSON(http.StatusBadRequest, gin.H{"error": "value must be 'true' or 'false' for " + req.Field})
return
}
}
override := &models.CapabilityOverride{
ProviderConfigID: req.ProviderConfigID,
ModelID: modelID,
Field: req.Field,
Value: req.Value,
SetBy: &userID,
}
if err := h.stores.CapOverrides.Set(c.Request.Context(), override); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to set override"})
return
}
c.JSON(http.StatusOK, gin.H{"status": "ok"})
}
// DeleteModelCapability removes a specific override.
// DELETE /api/v1/admin/models/:id/capabilities/:overrideId
func (h *CapOverrideAdminHandler) DeleteModelCapability(c *gin.Context) {
overrideID := c.Param("overrideId")
if err := h.stores.CapOverrides.Delete(c.Request.Context(), overrideID); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete override"})
return
}
c.JSON(http.StatusOK, gin.H{"status": "ok"})
}
// ListAllOverrides returns all capability overrides (admin view).
// GET /api/v1/admin/capability-overrides
func (h *CapOverrideAdminHandler) ListAllOverrides(c *gin.Context) {
overrides, err := h.stores.CapOverrides.ListAll(c.Request.Context())
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to fetch overrides"})
return
}
SafeJSON(c, http.StatusOK, gin.H{
"data": overrides,
"total": len(overrides),
})
}
// ── Source Annotation Helpers ────────────────
type capSource struct {
Field string `json:"field"`
Source string `json:"source"` // "catalog", "heuristic", "override"
}
func buildSourceAnnotations(catalog *models.ModelCapabilities, heuristic *models.ModelCapabilities, overrides []models.CapabilityOverride) []capSource {
// Build override set for quick lookup
overrideSet := make(map[string]bool, len(overrides))
for _, o := range overrides {
overrideSet[o.Field] = true
}
fields := []struct {
name string
hasCatalog bool
hasHeur bool
}{
{"streaming", catalog != nil && catalog.Streaming, heuristic.Streaming},
{"tool_calling", catalog != nil && catalog.ToolCalling, heuristic.ToolCalling},
{"vision", catalog != nil && catalog.Vision, heuristic.Vision},
{"thinking", catalog != nil && catalog.Thinking, heuristic.Thinking},
{"reasoning", catalog != nil && catalog.Reasoning, heuristic.Reasoning},
{"code_optimized", catalog != nil && catalog.CodeOptimized, heuristic.CodeOptimized},
{"web_search", catalog != nil && catalog.WebSearch, heuristic.WebSearch},
{"max_context", catalog != nil && catalog.MaxContext > 0, heuristic.MaxContext > 0},
{"max_output_tokens", catalog != nil && catalog.MaxOutputTokens > 0, heuristic.MaxOutputTokens > 0},
}
var sources []capSource
for _, f := range fields {
src := "default"
if overrideSet[f.name] {
src = "override"
} else if f.hasCatalog {
src = "catalog"
} else if f.hasHeur {
src = "heuristic"
}
sources = append(sources, capSource{Field: f.name, Source: src})
}
return sources
}

View File

@@ -516,7 +516,7 @@ func (h *MessageHandler) Regenerate(c *gin.Context) {
// ── Stream the response (shared loop handles tools, reasoning, SSE) ──
result := streamWithToolLoop(c, provider, providerCfg, &provReq, model, userID, channelID, personaID, workspaceID, h.hub)
result := streamWithToolLoop(c, provider, providerCfg, &provReq, model, userID, channelID, personaID, workspaceID, configID, h.hub, comp.health)
// Persist as sibling (regen) with tool activity
if result.Content != "" {

View File

@@ -27,6 +27,20 @@ type streamResult struct {
CacheReadTokens int
}
// recordHealthFn records provider health from standalone streaming functions.
func recordHealthFn(hr HealthRecorder, configID string, start time.Time, err error) {
if hr == nil || configID == "" {
return
}
latencyMs := int(time.Since(start).Milliseconds())
if err != nil {
errMsg := err.Error()
hr.RecordError(configID, latencyMs, errMsg)
} else {
hr.RecordSuccess(configID, latencyMs)
}
}
// streamWithToolLoop is the single, canonical streaming implementation.
// It handles SSE setup, multi-round tool execution, reasoning_content
// forwarding, and client notifications. Returns the accumulated content
@@ -43,8 +57,9 @@ func streamWithToolLoop(
provider providers.Provider,
cfg providers.ProviderConfig,
req *providers.CompletionRequest,
model, userID, channelID, personaID, workspaceID string,
model, userID, channelID, personaID, workspaceID, configID string,
hub *events.Hub,
health HealthRecorder,
) streamResult {
// Set SSE headers
c.Header("Content-Type", "text/event-stream")
@@ -74,8 +89,10 @@ func streamWithToolLoop(
var result streamResult
for iteration := 0; iteration < maxToolIterations; iteration++ {
callStart := time.Now()
ch, err := provider.StreamCompletion(c.Request.Context(), cfg, *req)
if err != nil {
recordHealthFn(health, configID, callStart, err)
sendSSE(fmt.Sprintf(`{"error":"%s"}`, escapeJSON(err.Error())))
return result
}
@@ -86,6 +103,7 @@ func streamWithToolLoop(
for event := range ch {
if event.Error != nil {
recordHealthFn(health, configID, callStart, event.Error)
sendSSE(fmt.Sprintf(`{"error":"%s"}`, escapeJSON(event.Error.Error())))
return result
}
@@ -107,8 +125,9 @@ func streamWithToolLoop(
}
if event.Done {
recordHealthFn(health, configID, callStart, nil)
if event.FinishReason == "tool_calls" && len(event.ToolCalls) > 0 {
toolCalls = event.ToolCalls
toolCalls = event.ToolCalls
// Accumulate tokens from this tool-call iteration
result.InputTokens += event.InputTokens
result.OutputTokens += event.OutputTokens
@@ -268,8 +287,9 @@ func streamModelResponse(
provider providers.Provider,
cfg providers.ProviderConfig,
req *providers.CompletionRequest,
model, displayName, userID, channelID, personaID, workspaceID string,
model, displayName, userID, channelID, personaID, workspaceID, configID string,
hub *events.Hub,
health HealthRecorder,
) streamResult {
flusher, _ := c.Writer.(http.Flusher)
flush := func() {
@@ -289,8 +309,10 @@ func streamModelResponse(
var result streamResult
for iteration := 0; iteration < maxToolIterations; iteration++ {
callStart := time.Now()
ch, err := provider.StreamCompletion(c.Request.Context(), cfg, *req)
if err != nil {
recordHealthFn(health, configID, callStart, err)
sendSSE(fmt.Sprintf(`{"error":"%s"}`, escapeJSON(err.Error())))
return result
}
@@ -301,6 +323,7 @@ func streamModelResponse(
for event := range ch {
if event.Error != nil {
recordHealthFn(health, configID, callStart, event.Error)
sendSSE(fmt.Sprintf(`{"error":"%s"}`, escapeJSON(event.Error.Error())))
return result
}
@@ -320,6 +343,7 @@ func streamModelResponse(
}
if event.Done {
recordHealthFn(health, configID, callStart, nil)
if event.FinishReason == "tool_calls" && len(event.ToolCalls) > 0 {
toolCalls = event.ToolCalls
result.InputTokens += event.InputTokens

View File

@@ -310,7 +310,7 @@ func (h *TeamHandler) ListTeamProviderModels(c *gin.Context) {
out := make([]modelInfo, 0, len(modelList))
for _, m := range modelList {
caps := capspkg.ResolveIntrinsic(m.ID, &m.Capabilities)
caps := capspkg.ResolveIntrinsic(m.ID, &m.Capabilities, nil)
caps.MaxOutputTokens = capspkg.ResolveMaxOutput(m.ID, caps)
out = append(out, modelInfo{ID: m.ID, Capabilities: caps})
}

View File

@@ -0,0 +1,219 @@
// Package health provides in-memory accumulation of provider health
// metrics with periodic flush to the database. The accumulator is
// goroutine-safe and designed for high-throughput recording from
// concurrent completion handlers.
package health
import (
"context"
"log"
"sync"
"time"
"git.gobha.me/xcaliber/chat-switchboard/models"
)
// ── Thresholds ──────────────────────────────
const (
// DegradedThreshold — error rate above this = degraded.
DegradedThreshold = 0.05 // 5%
// DownThreshold — error rate above this = down.
DownThreshold = 0.25 // 25%
// FlushInterval — how often in-memory counters are flushed to DB.
FlushInterval = 60 * time.Second
// PruneAge — health windows older than this are deleted.
PruneAge = 7 * 24 * time.Hour
)
// ── Store Interface ─────────────────────────
// Store is the persistence layer for health data.
type Store interface {
// UpsertWindow merges in-memory counters into the hourly bucket row.
UpsertWindow(ctx context.Context, w *models.ProviderHealthWindow) error
// GetCurrentWindow returns the current hourly bucket for a provider.
GetCurrentWindow(ctx context.Context, providerConfigID string) (*models.ProviderHealthWindow, error)
// ListWindows returns the last N hourly buckets for a provider, newest first.
ListWindows(ctx context.Context, providerConfigID string, hours int) ([]models.ProviderHealthWindow, error)
// ListAllCurrent returns the current-hour bucket for every provider.
ListAllCurrent(ctx context.Context) ([]models.ProviderHealthWindow, error)
// Prune deletes rows older than the given time.
Prune(ctx context.Context, before time.Time) (int64, error)
}
// ── In-Memory Bucket ────────────────────────
// bucket holds counters for one provider within one flush interval.
type bucket struct {
providerConfigID string
requestCount int
errorCount int
timeoutCount int
totalLatencyMs int64
maxLatencyMs int
lastError string
lastErrorAt time.Time
}
// ── Accumulator ─────────────────────────────
// Accumulator collects health metrics in memory and periodically
// flushes them to the database. One instance per process.
type Accumulator struct {
mu sync.Mutex
buckets map[string]*bucket // keyed by provider_config_id
store Store
stopCh chan struct{}
wg sync.WaitGroup
}
// NewAccumulator creates and starts the background flush loop.
func NewAccumulator(store Store) *Accumulator {
a := &Accumulator{
buckets: make(map[string]*bucket),
store: store,
stopCh: make(chan struct{}),
}
a.wg.Add(1)
go a.flushLoop()
return a
}
// RecordSuccess records a successful provider call.
func (a *Accumulator) RecordSuccess(providerConfigID string, latencyMs int) {
a.mu.Lock()
defer a.mu.Unlock()
b := a.getBucket(providerConfigID)
b.requestCount++
b.totalLatencyMs += int64(latencyMs)
if latencyMs > b.maxLatencyMs {
b.maxLatencyMs = latencyMs
}
}
// RecordError records a failed provider call.
func (a *Accumulator) RecordError(providerConfigID string, latencyMs int, errMsg string) {
a.mu.Lock()
defer a.mu.Unlock()
b := a.getBucket(providerConfigID)
b.requestCount++
b.errorCount++
b.totalLatencyMs += int64(latencyMs)
if latencyMs > b.maxLatencyMs {
b.maxLatencyMs = latencyMs
}
b.lastError = errMsg
b.lastErrorAt = time.Now().UTC()
}
// RecordTimeout records a timed-out provider call (counted as both error and timeout).
func (a *Accumulator) RecordTimeout(providerConfigID string, latencyMs int, errMsg string) {
a.mu.Lock()
defer a.mu.Unlock()
b := a.getBucket(providerConfigID)
b.requestCount++
b.errorCount++
b.timeoutCount++
b.totalLatencyMs += int64(latencyMs)
if latencyMs > b.maxLatencyMs {
b.maxLatencyMs = latencyMs
}
b.lastError = errMsg
b.lastErrorAt = time.Now().UTC()
}
// Stop halts the flush loop and performs a final flush.
func (a *Accumulator) Stop() {
close(a.stopCh)
a.wg.Wait()
}
// DeriveStatus computes the provider status from an error rate.
func DeriveStatus(errorRate float64, requestCount int) models.ProviderStatus {
if requestCount == 0 {
return models.StatusUnknown
}
if errorRate >= DownThreshold {
return models.StatusDown
}
if errorRate >= DegradedThreshold {
return models.StatusDegraded
}
return models.StatusHealthy
}
// ── Internal ────────────────────────────────
func (a *Accumulator) getBucket(providerConfigID string) *bucket {
b, ok := a.buckets[providerConfigID]
if !ok {
b = &bucket{providerConfigID: providerConfigID}
a.buckets[providerConfigID] = b
}
return b
}
func (a *Accumulator) flushLoop() {
defer a.wg.Done()
ticker := time.NewTicker(FlushInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
a.flush()
case <-a.stopCh:
a.flush() // final flush
return
}
}
}
func (a *Accumulator) flush() {
a.mu.Lock()
if len(a.buckets) == 0 {
a.mu.Unlock()
return
}
// Swap out current buckets so we don't hold the lock during DB writes.
snap := a.buckets
a.buckets = make(map[string]*bucket, len(snap))
a.mu.Unlock()
windowStart := hourFloor(time.Now().UTC())
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
for _, b := range snap {
if b.requestCount == 0 {
continue
}
w := &models.ProviderHealthWindow{
ProviderConfigID: b.providerConfigID,
WindowStart: windowStart,
RequestCount: b.requestCount,
ErrorCount: b.errorCount,
TimeoutCount: b.timeoutCount,
TotalLatencyMs: b.totalLatencyMs,
MaxLatencyMs: b.maxLatencyMs,
}
if b.lastError != "" {
w.LastError = &b.lastError
ts := b.lastErrorAt.Format(time.RFC3339)
w.LastErrorAt = &ts
}
if err := a.store.UpsertWindow(ctx, w); err != nil {
log.Printf("⚠ health: flush failed for provider %s: %v", b.providerConfigID, err)
}
}
}
// hourFloor truncates a time to the start of its hour.
func hourFloor(t time.Time) time.Time {
return t.Truncate(time.Hour)
}

View File

@@ -0,0 +1,239 @@
package health
import (
"context"
"sync"
"testing"
"time"
"git.gobha.me/xcaliber/chat-switchboard/models"
)
// ── Mock Store ──────────────────────────────
type mockStore struct {
mu sync.Mutex
windows map[string]*models.ProviderHealthWindow // keyed by provider_config_id
}
func newMockStore() *mockStore {
return &mockStore{windows: make(map[string]*models.ProviderHealthWindow)}
}
func (m *mockStore) UpsertWindow(_ context.Context, w *models.ProviderHealthWindow) error {
m.mu.Lock()
defer m.mu.Unlock()
existing, ok := m.windows[w.ProviderConfigID]
if ok {
existing.RequestCount += w.RequestCount
existing.ErrorCount += w.ErrorCount
existing.TimeoutCount += w.TimeoutCount
existing.TotalLatencyMs += w.TotalLatencyMs
if w.MaxLatencyMs > existing.MaxLatencyMs {
existing.MaxLatencyMs = w.MaxLatencyMs
}
if w.LastError != nil {
existing.LastError = w.LastError
existing.LastErrorAt = w.LastErrorAt
}
} else {
cp := *w
m.windows[w.ProviderConfigID] = &cp
}
return nil
}
func (m *mockStore) GetCurrentWindow(_ context.Context, id string) (*models.ProviderHealthWindow, error) {
m.mu.Lock()
defer m.mu.Unlock()
w, ok := m.windows[id]
if !ok {
return nil, nil
}
cp := *w
return &cp, nil
}
func (m *mockStore) ListWindows(_ context.Context, _ string, _ int) ([]models.ProviderHealthWindow, error) {
return nil, nil
}
func (m *mockStore) ListAllCurrent(_ context.Context) ([]models.ProviderHealthWindow, error) {
m.mu.Lock()
defer m.mu.Unlock()
var result []models.ProviderHealthWindow
for _, w := range m.windows {
result = append(result, *w)
}
return result, nil
}
func (m *mockStore) Prune(_ context.Context, _ time.Time) (int64, error) { return 0, nil }
// ── Tests ───────────────────────────────────
func TestRecordSuccess(t *testing.T) {
ms := newMockStore()
a := &Accumulator{
buckets: make(map[string]*bucket),
store: ms,
stopCh: make(chan struct{}),
}
a.RecordSuccess("prov-1", 100)
a.RecordSuccess("prov-1", 200)
a.RecordSuccess("prov-2", 50)
a.flush()
w1, _ := ms.GetCurrentWindow(context.Background(), "prov-1")
if w1 == nil {
t.Fatal("expected prov-1 window")
}
if w1.RequestCount != 2 {
t.Fatalf("expected 2 requests, got %d", w1.RequestCount)
}
if w1.ErrorCount != 0 {
t.Fatalf("expected 0 errors, got %d", w1.ErrorCount)
}
if w1.TotalLatencyMs != 300 {
t.Fatalf("expected 300ms total latency, got %d", w1.TotalLatencyMs)
}
if w1.MaxLatencyMs != 200 {
t.Fatalf("expected 200ms max latency, got %d", w1.MaxLatencyMs)
}
w2, _ := ms.GetCurrentWindow(context.Background(), "prov-2")
if w2 == nil {
t.Fatal("expected prov-2 window")
}
if w2.RequestCount != 1 {
t.Fatalf("expected 1 request, got %d", w2.RequestCount)
}
}
func TestRecordError(t *testing.T) {
ms := newMockStore()
a := &Accumulator{
buckets: make(map[string]*bucket),
store: ms,
stopCh: make(chan struct{}),
}
a.RecordSuccess("prov-1", 100)
a.RecordError("prov-1", 500, "connection refused")
a.RecordSuccess("prov-1", 150)
a.flush()
w, _ := ms.GetCurrentWindow(context.Background(), "prov-1")
if w.RequestCount != 3 {
t.Fatalf("expected 3 requests, got %d", w.RequestCount)
}
if w.ErrorCount != 1 {
t.Fatalf("expected 1 error, got %d", w.ErrorCount)
}
if w.LastError == nil || *w.LastError != "connection refused" {
t.Fatalf("expected last_error='connection refused', got %v", w.LastError)
}
}
func TestRecordTimeout(t *testing.T) {
ms := newMockStore()
a := &Accumulator{
buckets: make(map[string]*bucket),
store: ms,
stopCh: make(chan struct{}),
}
a.RecordTimeout("prov-1", 30000, "context deadline exceeded")
a.flush()
w, _ := ms.GetCurrentWindow(context.Background(), "prov-1")
if w.RequestCount != 1 {
t.Fatalf("expected 1 request, got %d", w.RequestCount)
}
if w.ErrorCount != 1 {
t.Fatalf("expected 1 error, got %d", w.ErrorCount)
}
if w.TimeoutCount != 1 {
t.Fatalf("expected 1 timeout, got %d", w.TimeoutCount)
}
}
func TestDeriveStatus(t *testing.T) {
tests := []struct {
rate float64
count int
expected models.ProviderStatus
}{
{0, 0, models.StatusUnknown},
{0, 100, models.StatusHealthy},
{0.03, 100, models.StatusHealthy},
{0.05, 100, models.StatusDegraded},
{0.15, 100, models.StatusDegraded},
{0.25, 100, models.StatusDown},
{0.50, 100, models.StatusDown},
{1.0, 1, models.StatusDown},
}
for _, tt := range tests {
got := DeriveStatus(tt.rate, tt.count)
if got != tt.expected {
t.Errorf("DeriveStatus(%v, %d) = %s, want %s", tt.rate, tt.count, got, tt.expected)
}
}
}
func TestFlushClearsBuckets(t *testing.T) {
ms := newMockStore()
a := &Accumulator{
buckets: make(map[string]*bucket),
store: ms,
stopCh: make(chan struct{}),
}
a.RecordSuccess("prov-1", 100)
a.flush()
// After flush, buckets should be empty
a.mu.Lock()
count := len(a.buckets)
a.mu.Unlock()
if count != 0 {
t.Fatalf("expected 0 buckets after flush, got %d", count)
}
}
func TestConcurrentRecording(t *testing.T) {
ms := newMockStore()
a := &Accumulator{
buckets: make(map[string]*bucket),
store: ms,
stopCh: make(chan struct{}),
}
var wg sync.WaitGroup
for i := 0; i < 100; i++ {
wg.Add(1)
go func() {
defer wg.Done()
a.RecordSuccess("prov-1", 50)
}()
}
wg.Wait()
a.flush()
w, _ := ms.GetCurrentWindow(context.Background(), "prov-1")
if w.RequestCount != 100 {
t.Fatalf("expected 100 concurrent requests, got %d", w.RequestCount)
}
}
func TestHourFloor(t *testing.T) {
ts := time.Date(2026, 3, 1, 14, 37, 42, 0, time.UTC)
floor := hourFloor(ts)
expected := time.Date(2026, 3, 1, 14, 0, 0, 0, time.UTC)
if !floor.Equal(expected) {
t.Fatalf("expected %v, got %v", expected, floor)
}
}

View File

@@ -18,6 +18,7 @@ import (
"git.gobha.me/xcaliber/chat-switchboard/events"
"git.gobha.me/xcaliber/chat-switchboard/extraction"
"git.gobha.me/xcaliber/chat-switchboard/handlers"
"git.gobha.me/xcaliber/chat-switchboard/health"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/memory"
"git.gobha.me/xcaliber/chat-switchboard/middleware"
@@ -54,6 +55,8 @@ func main() {
uekCache := crypto.NewUEKCache()
var keyResolver *crypto.KeyResolver
var objStore storage.ObjectStore
var healthAccum *health.Accumulator
var healthStore *postgres.HealthStore
if err := database.Connect(cfg); err != nil {
log.Printf("⚠ Database unavailable: %v", err)
@@ -89,6 +92,26 @@ func main() {
// Initialize store layer
stores = postgres.NewStores(database.DB)
// Provider health accumulator (v0.22.0)
healthStore = postgres.NewHealthStore(database.DB)
healthAccum = health.NewAccumulator(healthStore)
defer healthAccum.Stop()
// Background health cleanup: prune windows older than 7 days
go func() {
ticker := time.NewTicker(6 * time.Hour)
defer ticker.Stop()
for range ticker.C {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
if n, err := healthStore.Prune(ctx, time.Now().UTC().Add(-health.PruneAge)); err != nil {
log.Printf("⚠ health prune failed: %v", err)
} else if n > 0 {
log.Printf("🧹 health: pruned %d old windows", n)
}
cancel()
}
}()
// Bootstrap admin from env (K8s secret) — upserts on every restart
handlers.BootstrapAdmin(cfg, stores)
@@ -350,6 +373,9 @@ func main() {
// Chat Completions
comp := handlers.NewCompletionHandler(keyResolver, stores, hub, objStore, kbEmbedder)
if healthAccum != nil {
comp.SetHealthRecorder(healthAccum)
}
protected.POST("/chat/completions", comp.Complete)
protected.GET("/tools", comp.ListTools)
@@ -722,6 +748,18 @@ func main() {
admin.POST("/extensions", extAdm.AdminInstallExtension)
admin.PUT("/extensions/:id", extAdm.AdminUpdateExtension)
admin.DELETE("/extensions/:id", extAdm.AdminUninstallExtension)
// Provider Health (admin — v0.22.0)
healthAdm := handlers.NewHealthAdminHandler(healthStore, stores)
admin.GET("/providers/health", healthAdm.GetAllProviderHealth)
admin.GET("/providers/:id/health", healthAdm.GetProviderHealth)
// Capability Overrides (admin — v0.22.0)
capAdm := handlers.NewCapOverrideAdminHandler(stores)
admin.GET("/models/:id/capabilities", capAdm.GetModelCapabilities)
admin.PUT("/models/:id/capabilities", capAdm.SetModelCapability)
admin.DELETE("/models/:id/capabilities/:overrideId", capAdm.DeleteModelCapability)
admin.GET("/capability-overrides", capAdm.ListAllOverrides)
}
}
@@ -744,6 +782,7 @@ func main() {
log.Printf(" Storage: disabled")
}
log.Printf(" EventBus: ready, WebSocket on %s/ws", cfg.BasePath)
log.Printf(" Health: provider tracking active (flush=%s, prune=%s)", health.FlushInterval, health.PruneAge)
if err := r.Run(":" + cfg.Port); err != nil {
log.Fatalf("Failed to start server: %v", err)
}

View File

@@ -864,3 +864,76 @@ type ChannelKB struct {
Enabled bool `json:"enabled" db:"enabled"`
DocumentCount int `json:"document_count" db:"document_count"`
}
// =========================================
// PROVIDER HEALTH (v0.22.0)
// =========================================
// ProviderStatus represents the derived health state.
type ProviderStatus string
const (
StatusHealthy ProviderStatus = "healthy"
StatusDegraded ProviderStatus = "degraded"
StatusDown ProviderStatus = "down"
StatusUnknown ProviderStatus = "unknown"
)
// ProviderHealthWindow is one hourly bucket of health metrics.
type ProviderHealthWindow struct {
ID string `json:"id" db:"id"`
ProviderConfigID string `json:"provider_config_id" db:"provider_config_id"`
WindowStart time.Time `json:"window_start" db:"window_start"`
RequestCount int `json:"request_count" db:"request_count"`
ErrorCount int `json:"error_count" db:"error_count"`
TimeoutCount int `json:"timeout_count" db:"timeout_count"`
TotalLatencyMs int64 `json:"total_latency_ms" db:"total_latency_ms"`
MaxLatencyMs int `json:"max_latency_ms" db:"max_latency_ms"`
LastError *string `json:"last_error,omitempty" db:"last_error"`
LastErrorAt *string `json:"last_error_at,omitempty" db:"last_error_at"`
}
// AvgLatencyMs returns the average latency, or 0 if no requests.
func (w *ProviderHealthWindow) AvgLatencyMs() int {
if w.RequestCount == 0 {
return 0
}
return int(w.TotalLatencyMs / int64(w.RequestCount))
}
// ErrorRate returns the error fraction, or 0 if no requests.
func (w *ProviderHealthWindow) ErrorRate() float64 {
if w.RequestCount == 0 {
return 0
}
return float64(w.ErrorCount) / float64(w.RequestCount)
}
// ProviderHealthSummary is the API response for a provider's current health.
type ProviderHealthSummary struct {
ProviderConfigID string `json:"provider_config_id"`
ProviderName string `json:"provider_name,omitempty"`
Status ProviderStatus `json:"status"`
RequestCount int `json:"request_count"` // last hour
ErrorRate float64 `json:"error_rate"` // last hour
AvgLatencyMs int `json:"avg_latency_ms"` // last hour
MaxLatencyMs int `json:"max_latency_ms"` // last hour
LastError *string `json:"last_error,omitempty"`
LastErrorAt *string `json:"last_error_at,omitempty"`
}
// =========================================
// CAPABILITY OVERRIDES (v0.22.0)
// =========================================
// CapabilityOverride is an admin correction for a model's capabilities.
type CapabilityOverride struct {
ID string `json:"id" db:"id"`
ProviderConfigID *string `json:"provider_config_id,omitempty" db:"provider_config_id"`
ModelID string `json:"model_id" db:"model_id"`
Field string `json:"field" db:"field"`
Value string `json:"value" db:"value"`
SetBy *string `json:"set_by,omitempty" db:"set_by"`
CreatedAt string `json:"created_at" db:"created_at"`
}

View File

@@ -44,6 +44,7 @@ type Stores struct {
NotifPrefs NotificationPreferenceStore
Workspaces WorkspaceStore
GitCredentials GitCredentialStore
CapOverrides CapabilityOverrideStore
}
// =========================================
@@ -562,6 +563,30 @@ type GitCredentialStore interface {
Delete(ctx context.Context, id, userID string) error
}
// =========================================
// CAPABILITY OVERRIDES (v0.22.0)
// =========================================
type CapabilityOverrideStore interface {
// Set creates or updates an override for (provider, model, field).
Set(ctx context.Context, o *models.CapabilityOverride) error
// Delete removes a specific override.
Delete(ctx context.Context, id string) error
// ListForModel returns all overrides for a model ID (across all providers + global).
ListForModel(ctx context.Context, modelID string) ([]models.CapabilityOverride, error)
// ListForProviderModel returns overrides for a specific provider+model combination.
ListForProviderModel(ctx context.Context, providerConfigID, modelID string) ([]models.CapabilityOverride, error)
// ListAll returns every override (admin view).
ListAll(ctx context.Context) ([]models.CapabilityOverride, error)
// DeleteForProvider removes all overrides for a provider (cascade cleanup).
DeleteForProvider(ctx context.Context, providerConfigID string) error
}
// =========================================
// SHARED TYPES
// =========================================

View File

@@ -0,0 +1,85 @@
package postgres
import (
"context"
"database/sql"
"git.gobha.me/xcaliber/chat-switchboard/models"
)
// CapOverrideStore implements store.CapabilityOverrideStore for Postgres.
type CapOverrideStore struct {
db *sql.DB
}
func NewCapOverrideStore(db *sql.DB) *CapOverrideStore {
return &CapOverrideStore{db: db}
}
func (s *CapOverrideStore) Set(ctx context.Context, o *models.CapabilityOverride) error {
_, err := s.db.ExecContext(ctx, `
INSERT INTO capability_overrides (provider_config_id, model_id, field, value, set_by)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (provider_config_id, model_id, field) DO UPDATE SET
value = EXCLUDED.value,
set_by = EXCLUDED.set_by,
created_at = now()
`, o.ProviderConfigID, o.ModelID, o.Field, o.Value, o.SetBy)
return err
}
func (s *CapOverrideStore) Delete(ctx context.Context, id string) error {
_, err := s.db.ExecContext(ctx, `DELETE FROM capability_overrides WHERE id = $1`, id)
return err
}
func (s *CapOverrideStore) ListForModel(ctx context.Context, modelID string) ([]models.CapabilityOverride, error) {
return s.query(ctx, `
SELECT id, provider_config_id, model_id, field, value, set_by, created_at
FROM capability_overrides
WHERE model_id = $1
ORDER BY provider_config_id NULLS LAST, field
`, modelID)
}
func (s *CapOverrideStore) ListForProviderModel(ctx context.Context, providerConfigID, modelID string) ([]models.CapabilityOverride, error) {
return s.query(ctx, `
SELECT id, provider_config_id, model_id, field, value, set_by, created_at
FROM capability_overrides
WHERE model_id = $1 AND (provider_config_id = $2 OR provider_config_id IS NULL)
ORDER BY provider_config_id NULLS LAST, field
`, modelID, providerConfigID)
}
func (s *CapOverrideStore) ListAll(ctx context.Context) ([]models.CapabilityOverride, error) {
return s.query(ctx, `
SELECT id, provider_config_id, model_id, field, value, set_by, created_at
FROM capability_overrides
ORDER BY model_id, provider_config_id NULLS LAST, field
`)
}
func (s *CapOverrideStore) DeleteForProvider(ctx context.Context, providerConfigID string) error {
_, err := s.db.ExecContext(ctx, `
DELETE FROM capability_overrides WHERE provider_config_id = $1
`, providerConfigID)
return err
}
func (s *CapOverrideStore) query(ctx context.Context, q string, args ...interface{}) ([]models.CapabilityOverride, error) {
rows, err := s.db.QueryContext(ctx, q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var result []models.CapabilityOverride
for rows.Next() {
var o models.CapabilityOverride
if err := rows.Scan(&o.ID, &o.ProviderConfigID, &o.ModelID, &o.Field, &o.Value, &o.SetBy, &o.CreatedAt); err != nil {
return nil, err
}
result = append(result, o)
}
return result, rows.Err()
}

View File

@@ -0,0 +1,130 @@
package postgres
import (
"context"
"database/sql"
"time"
"git.gobha.me/xcaliber/chat-switchboard/models"
)
// HealthStore implements health.Store for Postgres.
type HealthStore struct {
db *sql.DB
}
func NewHealthStore(db *sql.DB) *HealthStore {
return &HealthStore{db: db}
}
func (s *HealthStore) UpsertWindow(ctx context.Context, w *models.ProviderHealthWindow) error {
_, err := s.db.ExecContext(ctx, `
INSERT INTO provider_health (provider_config_id, window_start,
request_count, error_count, timeout_count,
total_latency_ms, max_latency_ms, last_error, last_error_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, now())
ON CONFLICT (provider_config_id, window_start) DO UPDATE SET
request_count = provider_health.request_count + EXCLUDED.request_count,
error_count = provider_health.error_count + EXCLUDED.error_count,
timeout_count = provider_health.timeout_count + EXCLUDED.timeout_count,
total_latency_ms = provider_health.total_latency_ms + EXCLUDED.total_latency_ms,
max_latency_ms = GREATEST(provider_health.max_latency_ms, EXCLUDED.max_latency_ms),
last_error = COALESCE(EXCLUDED.last_error, provider_health.last_error),
last_error_at = COALESCE(EXCLUDED.last_error_at, provider_health.last_error_at),
updated_at = now()
`, w.ProviderConfigID, w.WindowStart,
w.RequestCount, w.ErrorCount, w.TimeoutCount,
w.TotalLatencyMs, w.MaxLatencyMs, w.LastError, w.LastErrorAt,
)
return err
}
func (s *HealthStore) GetCurrentWindow(ctx context.Context, providerConfigID string) (*models.ProviderHealthWindow, error) {
windowStart := time.Now().UTC().Truncate(time.Hour)
var w models.ProviderHealthWindow
err := s.db.QueryRowContext(ctx, `
SELECT id, provider_config_id, window_start,
request_count, error_count, timeout_count,
total_latency_ms, max_latency_ms, last_error, last_error_at
FROM provider_health
WHERE provider_config_id = $1 AND window_start = $2
`, providerConfigID, windowStart).Scan(
&w.ID, &w.ProviderConfigID, &w.WindowStart,
&w.RequestCount, &w.ErrorCount, &w.TimeoutCount,
&w.TotalLatencyMs, &w.MaxLatencyMs, &w.LastError, &w.LastErrorAt,
)
if err == sql.ErrNoRows {
return nil, nil
}
return &w, err
}
func (s *HealthStore) ListWindows(ctx context.Context, providerConfigID string, hours int) ([]models.ProviderHealthWindow, error) {
rows, err := s.db.QueryContext(ctx, `
SELECT id, provider_config_id, window_start,
request_count, error_count, timeout_count,
total_latency_ms, max_latency_ms, last_error, last_error_at
FROM provider_health
WHERE provider_config_id = $1
ORDER BY window_start DESC
LIMIT $2
`, providerConfigID, hours)
if err != nil {
return nil, err
}
defer rows.Close()
var result []models.ProviderHealthWindow
for rows.Next() {
var w models.ProviderHealthWindow
if err := rows.Scan(
&w.ID, &w.ProviderConfigID, &w.WindowStart,
&w.RequestCount, &w.ErrorCount, &w.TimeoutCount,
&w.TotalLatencyMs, &w.MaxLatencyMs, &w.LastError, &w.LastErrorAt,
); err != nil {
return nil, err
}
result = append(result, w)
}
return result, rows.Err()
}
func (s *HealthStore) ListAllCurrent(ctx context.Context) ([]models.ProviderHealthWindow, error) {
windowStart := time.Now().UTC().Truncate(time.Hour)
rows, err := s.db.QueryContext(ctx, `
SELECT id, provider_config_id, window_start,
request_count, error_count, timeout_count,
total_latency_ms, max_latency_ms, last_error, last_error_at
FROM provider_health
WHERE window_start = $1
ORDER BY provider_config_id
`, windowStart)
if err != nil {
return nil, err
}
defer rows.Close()
var result []models.ProviderHealthWindow
for rows.Next() {
var w models.ProviderHealthWindow
if err := rows.Scan(
&w.ID, &w.ProviderConfigID, &w.WindowStart,
&w.RequestCount, &w.ErrorCount, &w.TimeoutCount,
&w.TotalLatencyMs, &w.MaxLatencyMs, &w.LastError, &w.LastErrorAt,
); err != nil {
return nil, err
}
result = append(result, w)
}
return result, rows.Err()
}
func (s *HealthStore) Prune(ctx context.Context, before time.Time) (int64, error) {
result, err := s.db.ExecContext(ctx, `
DELETE FROM provider_health WHERE window_start < $1
`, before)
if err != nil {
return 0, err
}
return result.RowsAffected()
}

View File

@@ -37,5 +37,6 @@ func NewStores(db *sql.DB) store.Stores {
NotifPrefs: NewNotificationPreferenceStore(),
Workspaces: NewWorkspaceStore(),
GitCredentials: &GitCredentialStore{},
CapOverrides: NewCapOverrideStore(db),
}
}

View File

@@ -0,0 +1,90 @@
package sqlite
import (
"context"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
type CapOverrideStore struct{}
func NewCapOverrideStore() *CapOverrideStore { return &CapOverrideStore{} }
func (s *CapOverrideStore) Set(ctx context.Context, o *models.CapabilityOverride) error {
id := store.NewID()
_, err := DB.ExecContext(ctx, `
INSERT INTO capability_overrides (id, provider_config_id, model_id, field, value, set_by)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (provider_config_id, model_id, field) DO UPDATE SET
value = excluded.value,
set_by = excluded.set_by,
created_at = datetime('now')
`, id, o.ProviderConfigID, o.ModelID, o.Field, o.Value, o.SetBy)
return err
}
func (s *CapOverrideStore) Delete(ctx context.Context, id string) error {
_, err := DB.ExecContext(ctx, `DELETE FROM capability_overrides WHERE id = ?`, id)
return err
}
func (s *CapOverrideStore) ListForModel(ctx context.Context, modelID string) ([]models.CapabilityOverride, error) {
return s.query(ctx, `
SELECT id, provider_config_id, model_id, field, value, set_by, created_at
FROM capability_overrides
WHERE model_id = ?
ORDER BY provider_config_id, field
`, modelID)
}
func (s *CapOverrideStore) ListForProviderModel(ctx context.Context, providerConfigID, modelID string) ([]models.CapabilityOverride, error) {
return s.query(ctx, `
SELECT id, provider_config_id, model_id, field, value, set_by, created_at
FROM capability_overrides
WHERE model_id = ? AND (provider_config_id = ? OR provider_config_id IS NULL)
ORDER BY provider_config_id, field
`, modelID, providerConfigID)
}
func (s *CapOverrideStore) ListAll(ctx context.Context) ([]models.CapabilityOverride, error) {
return s.query(ctx, `
SELECT id, provider_config_id, model_id, field, value, set_by, created_at
FROM capability_overrides
ORDER BY model_id, provider_config_id, field
`)
}
func (s *CapOverrideStore) DeleteForProvider(ctx context.Context, providerConfigID string) error {
_, err := DB.ExecContext(ctx, `
DELETE FROM capability_overrides WHERE provider_config_id = ?
`, providerConfigID)
return err
}
func (s *CapOverrideStore) query(ctx context.Context, q string, args ...interface{}) ([]models.CapabilityOverride, error) {
rows, err := DB.QueryContext(ctx, q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var result []models.CapabilityOverride
for rows.Next() {
var o models.CapabilityOverride
if err := rows.Scan(&o.ID, &o.ProviderConfigID, &o.ModelID, &o.Field, &o.Value, &o.SetBy, &o.CreatedAt); err != nil {
return nil, err
}
result = append(result, o)
}
return result, rows.Err()
}
// ensure interface compliance (compile-time check)
var _ store.CapabilityOverrideStore = (*CapOverrideStore)(nil)
// also check the health store interface at this package level
// (health.Store is in the health package, but we verify via the sql.Rows pattern)
func init() {
// compile-time interface checks happen via the var _ lines above
}

View File

@@ -0,0 +1,125 @@
package sqlite
import (
"context"
"database/sql"
"time"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
type HealthStore struct{}
func NewHealthStore() *HealthStore { return &HealthStore{} }
func (s *HealthStore) UpsertWindow(ctx context.Context, w *models.ProviderHealthWindow) error {
id := store.NewID()
windowStr := w.WindowStart.Format(timeFmt)
_, err := DB.ExecContext(ctx, `
INSERT INTO provider_health (id, provider_config_id, window_start,
request_count, error_count, timeout_count,
total_latency_ms, max_latency_ms, last_error, last_error_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))
ON CONFLICT (provider_config_id, window_start) DO UPDATE SET
request_count = provider_health.request_count + excluded.request_count,
error_count = provider_health.error_count + excluded.error_count,
timeout_count = provider_health.timeout_count + excluded.timeout_count,
total_latency_ms = provider_health.total_latency_ms + excluded.total_latency_ms,
max_latency_ms = MAX(provider_health.max_latency_ms, excluded.max_latency_ms),
last_error = COALESCE(excluded.last_error, provider_health.last_error),
last_error_at = COALESCE(excluded.last_error_at, provider_health.last_error_at),
updated_at = datetime('now')
`, id, w.ProviderConfigID, windowStr,
w.RequestCount, w.ErrorCount, w.TimeoutCount,
w.TotalLatencyMs, w.MaxLatencyMs, w.LastError, w.LastErrorAt,
)
return err
}
func (s *HealthStore) GetCurrentWindow(ctx context.Context, providerConfigID string) (*models.ProviderHealthWindow, error) {
windowStr := time.Now().UTC().Truncate(time.Hour).Format(timeFmt)
var w models.ProviderHealthWindow
var windowStartStr string
err := DB.QueryRowContext(ctx, `
SELECT id, provider_config_id, window_start,
request_count, error_count, timeout_count,
total_latency_ms, max_latency_ms, last_error, last_error_at
FROM provider_health
WHERE provider_config_id = ? AND window_start = ?
`, providerConfigID, windowStr).Scan(
&w.ID, &w.ProviderConfigID, &windowStartStr,
&w.RequestCount, &w.ErrorCount, &w.TimeoutCount,
&w.TotalLatencyMs, &w.MaxLatencyMs, &w.LastError, &w.LastErrorAt,
)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, err
}
w.WindowStart, _ = time.Parse(timeFmt, windowStartStr)
return &w, nil
}
func (s *HealthStore) ListWindows(ctx context.Context, providerConfigID string, hours int) ([]models.ProviderHealthWindow, error) {
rows, err := DB.QueryContext(ctx, `
SELECT id, provider_config_id, window_start,
request_count, error_count, timeout_count,
total_latency_ms, max_latency_ms, last_error, last_error_at
FROM provider_health
WHERE provider_config_id = ?
ORDER BY window_start DESC
LIMIT ?
`, providerConfigID, hours)
if err != nil {
return nil, err
}
defer rows.Close()
return scanHealthRows(rows)
}
func (s *HealthStore) ListAllCurrent(ctx context.Context) ([]models.ProviderHealthWindow, error) {
windowStr := time.Now().UTC().Truncate(time.Hour).Format(timeFmt)
rows, err := DB.QueryContext(ctx, `
SELECT id, provider_config_id, window_start,
request_count, error_count, timeout_count,
total_latency_ms, max_latency_ms, last_error, last_error_at
FROM provider_health
WHERE window_start = ?
ORDER BY provider_config_id
`, windowStr)
if err != nil {
return nil, err
}
defer rows.Close()
return scanHealthRows(rows)
}
func (s *HealthStore) Prune(ctx context.Context, before time.Time) (int64, error) {
result, err := DB.ExecContext(ctx, `
DELETE FROM provider_health WHERE window_start < ?
`, before.Format(timeFmt))
if err != nil {
return 0, err
}
return result.RowsAffected()
}
func scanHealthRows(rows *sql.Rows) ([]models.ProviderHealthWindow, error) {
var result []models.ProviderHealthWindow
for rows.Next() {
var w models.ProviderHealthWindow
var windowStartStr string
if err := rows.Scan(
&w.ID, &w.ProviderConfigID, &windowStartStr,
&w.RequestCount, &w.ErrorCount, &w.TimeoutCount,
&w.TotalLatencyMs, &w.MaxLatencyMs, &w.LastError, &w.LastErrorAt,
); err != nil {
return nil, err
}
w.WindowStart, _ = time.Parse(timeFmt, windowStartStr)
result = append(result, w)
}
return result, rows.Err()
}

View File

@@ -37,5 +37,6 @@ func NewStores(db *sql.DB) store.Stores {
NotifPrefs: NewNotificationPreferenceStore(),
Workspaces: NewWorkspaceStore(),
GitCredentials: &GitCredentialStore{},
CapOverrides: NewCapOverrideStore(),
}
}