Changeset 0.22.0.1 (#94)
This commit is contained in:
@@ -154,7 +154,7 @@ func TestResolveIntrinsic_CatalogWins(t *testing.T) {
|
||||
ToolCalling: true,
|
||||
MaxOutputTokens: 16384,
|
||||
}
|
||||
merged := ResolveIntrinsic("claude-sonnet-4-20250514", &providerCaps)
|
||||
merged := ResolveIntrinsic("claude-sonnet-4-20250514", &providerCaps, nil)
|
||||
|
||||
if !merged.ToolCalling {
|
||||
t.Error("tool_calling should be preserved from provider")
|
||||
@@ -179,7 +179,7 @@ func TestResolveIntrinsic_ProviderDataPreserved(t *testing.T) {
|
||||
MaxOutputTokens: 8192,
|
||||
MaxContext: 65536,
|
||||
}
|
||||
merged := ResolveIntrinsic("some-unknown-model", &providerCaps)
|
||||
merged := ResolveIntrinsic("some-unknown-model", &providerCaps, nil)
|
||||
|
||||
if !merged.ToolCalling {
|
||||
t.Error("tool_calling should be preserved from provider")
|
||||
@@ -191,7 +191,7 @@ func TestResolveIntrinsic_ProviderDataPreserved(t *testing.T) {
|
||||
|
||||
func TestResolveIntrinsic_NilProvider(t *testing.T) {
|
||||
// Nil provider caps — falls through entirely to heuristic
|
||||
merged := ResolveIntrinsic("gpt-4o", nil)
|
||||
merged := ResolveIntrinsic("gpt-4o", nil, nil)
|
||||
|
||||
if !merged.ToolCalling {
|
||||
t.Error("should get tool_calling from heuristic (gpt-4 pattern)")
|
||||
@@ -203,7 +203,7 @@ func TestResolveIntrinsic_NilProvider(t *testing.T) {
|
||||
|
||||
func TestResolveIntrinsic_HeuristicOnly(t *testing.T) {
|
||||
// No catalog, no known table — pure heuristic
|
||||
merged := ResolveIntrinsic("deepseek-r1-distill-llama-70b", nil)
|
||||
merged := ResolveIntrinsic("deepseek-r1-distill-llama-70b", nil, nil)
|
||||
|
||||
if !merged.Reasoning {
|
||||
t.Error("should infer reasoning from deepseek-r1 pattern")
|
||||
@@ -229,3 +229,71 @@ func TestHasProviderData(t *testing.T) {
|
||||
t.Error("caps with max_context should have provider data")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveIntrinsic_AdminOverrides(t *testing.T) {
|
||||
// Catalog says no vision, heuristic says no vision,
|
||||
// but admin override forces vision=true.
|
||||
catalogCaps := &models.ModelCapabilities{
|
||||
ToolCalling: true,
|
||||
Vision: false,
|
||||
}
|
||||
overrides := []models.CapabilityOverride{
|
||||
{Field: "vision", Value: "true"},
|
||||
}
|
||||
|
||||
merged := ResolveIntrinsic("some-custom-model", catalogCaps, overrides)
|
||||
|
||||
if !merged.Vision {
|
||||
t.Error("admin override should force vision=true")
|
||||
}
|
||||
if !merged.ToolCalling {
|
||||
t.Error("catalog tool_calling should be preserved")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveIntrinsic_AdminOverrides_DisableCapability(t *testing.T) {
|
||||
// Heuristic infers tool_calling for gpt-4o, but admin says no.
|
||||
overrides := []models.CapabilityOverride{
|
||||
{Field: "tool_calling", Value: "false"},
|
||||
}
|
||||
|
||||
merged := ResolveIntrinsic("gpt-4o", nil, overrides)
|
||||
|
||||
if merged.ToolCalling {
|
||||
t.Error("admin override should disable tool_calling")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveIntrinsic_AdminOverrides_IntValues(t *testing.T) {
|
||||
overrides := []models.CapabilityOverride{
|
||||
{Field: "max_context", Value: "200000"},
|
||||
{Field: "max_output_tokens", Value: "16384"},
|
||||
}
|
||||
|
||||
merged := ResolveIntrinsic("some-model", nil, overrides)
|
||||
|
||||
if merged.MaxContext != 200000 {
|
||||
t.Errorf("expected max_context=200000, got %d", merged.MaxContext)
|
||||
}
|
||||
if merged.MaxOutputTokens != 16384 {
|
||||
t.Errorf("expected max_output_tokens=16384, got %d", merged.MaxOutputTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveIntrinsic_OverridePriority(t *testing.T) {
|
||||
// Catalog says max_context=128000, admin overrides to 256000.
|
||||
// Admin should win.
|
||||
catalogCaps := &models.ModelCapabilities{
|
||||
MaxContext: 128000,
|
||||
}
|
||||
overrides := []models.CapabilityOverride{
|
||||
{Field: "max_context", Value: "256000"},
|
||||
}
|
||||
|
||||
merged := ResolveIntrinsic("some-model", catalogCaps, overrides)
|
||||
|
||||
if merged.MaxContext != 256000 {
|
||||
t.Errorf("admin override should win over catalog: expected 256000, got %d", merged.MaxContext)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -111,14 +111,16 @@ func InferCapabilities(modelID string) models.ModelCapabilities {
|
||||
}
|
||||
|
||||
// ResolveIntrinsic determines the intrinsic capabilities of a model.
|
||||
// Priority:
|
||||
// 1. catalogCaps (from model_catalog DB — synced from provider API)
|
||||
// 2. Heuristic inference (regex patterns on model ID)
|
||||
// Priority (highest wins):
|
||||
// 1. Admin overrides (manual corrections from capability_overrides table)
|
||||
// 2. Catalog (from model_catalog DB — synced from provider API)
|
||||
// 3. Heuristic inference (regex patterns on model ID)
|
||||
//
|
||||
// The catalog is the source of truth per provider. Heuristics fill gaps
|
||||
// for models that haven't been synced yet. No hardcoded model table —
|
||||
// for models that haven't been synced yet. Admin overrides correct
|
||||
// provider-reported or heuristic errors. No hardcoded model table —
|
||||
// the same model can have different capabilities on different providers.
|
||||
func ResolveIntrinsic(modelID string, catalogCaps *models.ModelCapabilities) models.ModelCapabilities {
|
||||
func ResolveIntrinsic(modelID string, catalogCaps *models.ModelCapabilities, overrides []models.CapabilityOverride) models.ModelCapabilities {
|
||||
// Start with catalog data if available
|
||||
var base models.ModelCapabilities
|
||||
if catalogCaps != nil && catalogCaps.HasProviderData() {
|
||||
@@ -128,9 +130,56 @@ func ResolveIntrinsic(modelID string, catalogCaps *models.ModelCapabilities) mod
|
||||
// Fill gaps from heuristic inference
|
||||
inferred := InferCapabilities(modelID)
|
||||
mergeGaps(&base, &inferred)
|
||||
|
||||
// Apply admin overrides (highest priority — can flip any field)
|
||||
applyOverrides(&base, overrides)
|
||||
|
||||
return base
|
||||
}
|
||||
|
||||
// applyOverrides applies admin capability corrections to the resolved capabilities.
|
||||
func applyOverrides(caps *models.ModelCapabilities, overrides []models.CapabilityOverride) {
|
||||
for _, o := range overrides {
|
||||
boolVal := o.Value == "true" || o.Value == "1"
|
||||
switch o.Field {
|
||||
case "streaming":
|
||||
caps.Streaming = boolVal
|
||||
case "tool_calling":
|
||||
caps.ToolCalling = boolVal
|
||||
case "vision":
|
||||
caps.Vision = boolVal
|
||||
case "thinking":
|
||||
caps.Thinking = boolVal
|
||||
case "reasoning":
|
||||
caps.Reasoning = boolVal
|
||||
case "code_optimized":
|
||||
caps.CodeOptimized = boolVal
|
||||
case "web_search":
|
||||
caps.WebSearch = boolVal
|
||||
case "max_context":
|
||||
if n := parseInt(o.Value); n > 0 {
|
||||
caps.MaxContext = n
|
||||
}
|
||||
case "max_output_tokens":
|
||||
if n := parseInt(o.Value); n > 0 {
|
||||
caps.MaxOutputTokens = n
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parseInt parses a string to int, returning 0 on failure.
|
||||
func parseInt(s string) int {
|
||||
n := 0
|
||||
for _, c := range s {
|
||||
if c < '0' || c > '9' {
|
||||
return 0
|
||||
}
|
||||
n = n*10 + int(c-'0')
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// ResolveMaxOutput returns the max output tokens for a model.
|
||||
// Priority: explicit caps → derive from context → 4096 fallback.
|
||||
func ResolveMaxOutput(modelID string, caps models.ModelCapabilities) int {
|
||||
|
||||
@@ -63,7 +63,7 @@ func ModelsForUser(ctx context.Context, stores store.Stores, userID string) ([]m
|
||||
|
||||
countGlobal := len(globalModels)
|
||||
for _, entry := range globalModels {
|
||||
caps := ResolveIntrinsic(entry.ModelID, &entry.Capabilities)
|
||||
caps := ResolveIntrinsic(entry.ModelID, &entry.Capabilities, nil)
|
||||
prov := providerMap[entry.ProviderConfigID]
|
||||
|
||||
result = append(result, models.UserModel{
|
||||
@@ -100,7 +100,7 @@ func ModelsForUser(ctx context.Context, stores store.Stores, userID string) ([]m
|
||||
continue
|
||||
}
|
||||
for _, entry := range entries {
|
||||
caps := ResolveIntrinsic(entry.ModelID, &entry.Capabilities)
|
||||
caps := ResolveIntrinsic(entry.ModelID, &entry.Capabilities, nil)
|
||||
result = append(result, models.UserModel{
|
||||
ID: entry.ModelID,
|
||||
DisplayName: displayName(entry.DisplayName, entry.ModelID),
|
||||
@@ -138,7 +138,7 @@ func ModelsForUser(ctx context.Context, stores store.Stores, userID string) ([]m
|
||||
continue
|
||||
}
|
||||
for _, entry := range entries {
|
||||
caps := ResolveIntrinsic(entry.ModelID, &entry.Capabilities)
|
||||
caps := ResolveIntrinsic(entry.ModelID, &entry.Capabilities, nil)
|
||||
result = append(result, models.UserModel{
|
||||
ID: entry.ModelID,
|
||||
DisplayName: displayName(entry.DisplayName, entry.ModelID),
|
||||
@@ -181,7 +181,7 @@ func ModelsForUser(ctx context.Context, stores store.Stores, userID string) ([]m
|
||||
catalogCaps = &entry.Capabilities
|
||||
}
|
||||
}
|
||||
caps := ResolveIntrinsic(p.BaseModelID, catalogCaps)
|
||||
caps := ResolveIntrinsic(p.BaseModelID, catalogCaps, nil)
|
||||
|
||||
// Load tool grants
|
||||
toolGrants, _ := stores.Personas.GetToolGrants(ctx, p.ID)
|
||||
@@ -251,7 +251,7 @@ func ResolveForPersona(ctx context.Context, stores store.Stores, persona *models
|
||||
catalogCaps = &entry.Capabilities
|
||||
}
|
||||
}
|
||||
caps := ResolveIntrinsic(persona.BaseModelID, catalogCaps)
|
||||
caps := ResolveIntrinsic(persona.BaseModelID, catalogCaps, nil)
|
||||
|
||||
toolGrants, err := stores.Personas.GetToolGrants(ctx, persona.ID)
|
||||
if err != nil {
|
||||
|
||||
43
server/database/migrations/013_v0220_health.sql
Normal file
43
server/database/migrations/013_v0220_health.sql
Normal file
@@ -0,0 +1,43 @@
|
||||
-- v0.22.0: Provider health tracking + capability admin overrides.
|
||||
|
||||
-- ── Provider Health ─────────────────────────
|
||||
-- Hourly bucketed health metrics per provider config.
|
||||
-- In-memory accumulator flushes to this table every 60s.
|
||||
-- Background job prunes rows older than 7 days.
|
||||
CREATE TABLE IF NOT EXISTS provider_health (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
provider_config_id UUID NOT NULL REFERENCES provider_configs(id) ON DELETE CASCADE,
|
||||
window_start TIMESTAMPTZ NOT NULL, -- hourly bucket floor
|
||||
request_count INT NOT NULL DEFAULT 0,
|
||||
error_count INT NOT NULL DEFAULT 0,
|
||||
timeout_count INT NOT NULL DEFAULT 0,
|
||||
total_latency_ms BIGINT NOT NULL DEFAULT 0, -- sum for avg calculation
|
||||
max_latency_ms INT NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
last_error_at TIMESTAMPTZ,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
UNIQUE (provider_config_id, window_start)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_provider_health_window
|
||||
ON provider_health (provider_config_id, window_start DESC);
|
||||
|
||||
-- ── Capability Overrides ────────────────────
|
||||
-- Admin corrections for model capabilities. Highest priority in the
|
||||
-- three-tier resolution chain: catalog → heuristic → admin override.
|
||||
CREATE TABLE IF NOT EXISTS capability_overrides (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
provider_config_id UUID REFERENCES provider_configs(id) ON DELETE CASCADE,
|
||||
model_id TEXT NOT NULL,
|
||||
field TEXT NOT NULL, -- tool_calling, vision, thinking, reasoning, etc.
|
||||
value TEXT NOT NULL, -- "true"/"false" for bools, numeric string for ints
|
||||
set_by UUID REFERENCES users(id) ON DELETE SET NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
-- Unique per (provider, model, field). NULL provider = applies to model globally.
|
||||
UNIQUE (provider_config_id, model_id, field)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_capability_overrides_model
|
||||
ON capability_overrides (model_id);
|
||||
35
server/database/migrations/sqlite/012_v0220_health.sql
Normal file
35
server/database/migrations/sqlite/012_v0220_health.sql
Normal file
@@ -0,0 +1,35 @@
|
||||
-- v0.22.0: Provider health tracking + capability admin overrides.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS provider_health (
|
||||
id TEXT PRIMARY KEY,
|
||||
provider_config_id TEXT NOT NULL,
|
||||
window_start TEXT NOT NULL,
|
||||
request_count INTEGER NOT NULL DEFAULT 0,
|
||||
error_count INTEGER NOT NULL DEFAULT 0,
|
||||
timeout_count INTEGER NOT NULL DEFAULT 0,
|
||||
total_latency_ms INTEGER NOT NULL DEFAULT 0,
|
||||
max_latency_ms INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
last_error_at TEXT,
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
|
||||
UNIQUE (provider_config_id, window_start)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_provider_health_window
|
||||
ON provider_health (provider_config_id, window_start);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS capability_overrides (
|
||||
id TEXT PRIMARY KEY,
|
||||
provider_config_id TEXT,
|
||||
model_id TEXT NOT NULL,
|
||||
field TEXT NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
set_by TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
|
||||
UNIQUE (provider_config_id, model_id, field)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_capability_overrides_model
|
||||
ON capability_overrides (model_id);
|
||||
@@ -94,7 +94,7 @@ func capsFromCatalog(modelID, configID string) (models.ModelCapabilities, bool)
|
||||
}
|
||||
|
||||
// Merge with known data to fill gaps
|
||||
resolved := capspkg.ResolveIntrinsic(modelID, &caps)
|
||||
resolved := capspkg.ResolveIntrinsic(modelID, &caps, nil)
|
||||
return resolved, true
|
||||
}
|
||||
|
||||
@@ -140,7 +140,7 @@ func liveQueryModelCaps(c *gin.Context, configID, modelID string) (models.ModelC
|
||||
|
||||
for _, m := range modelList {
|
||||
if m.ID == modelID {
|
||||
resolved := capspkg.ResolveIntrinsic(modelID, &m.Capabilities)
|
||||
resolved := capspkg.ResolveIntrinsic(modelID, &m.Capabilities, nil)
|
||||
return resolved, true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
@@ -50,6 +51,15 @@ type CompletionHandler struct {
|
||||
hub *events.Hub // WebSocket hub for browser tool bridge
|
||||
objStore storage.ObjectStore // file storage for attachment content (nil = disabled)
|
||||
embedder *knowledge.Embedder // for memory semantic recall (v0.18.0)
|
||||
health HealthRecorder // provider health tracking (v0.22.0, nil = disabled)
|
||||
}
|
||||
|
||||
// HealthRecorder is the interface for recording provider call outcomes.
|
||||
// Matches health.Accumulator methods without importing the package.
|
||||
type HealthRecorder interface {
|
||||
RecordSuccess(providerConfigID string, latencyMs int)
|
||||
RecordError(providerConfigID string, latencyMs int, errMsg string)
|
||||
RecordTimeout(providerConfigID string, latencyMs int, errMsg string)
|
||||
}
|
||||
|
||||
// NewCompletionHandler creates a new handler.
|
||||
@@ -57,6 +67,11 @@ func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *e
|
||||
return &CompletionHandler{vault: vault, stores: stores, hub: hub, objStore: objStore, embedder: embedder}
|
||||
}
|
||||
|
||||
// SetHealthRecorder attaches the provider health accumulator.
|
||||
func (h *CompletionHandler) SetHealthRecorder(hr HealthRecorder) {
|
||||
h.health = hr
|
||||
}
|
||||
|
||||
// ── Chat Completion ─────────────────────────
|
||||
// POST /api/v1/chat/completions
|
||||
//
|
||||
@@ -371,7 +386,7 @@ func (h *CompletionHandler) multiModelStream(
|
||||
}
|
||||
|
||||
// Stream this model's response using the shared streaming core
|
||||
result := streamModelResponse(c, provider, providerCfg, &provReq, model, target.DisplayName, userID, channelID, personaID, workspaceID, h.hub)
|
||||
result := streamModelResponse(c, provider, providerCfg, &provReq, model, target.DisplayName, userID, channelID, personaID, workspaceID, configID, h.hub, h.health)
|
||||
|
||||
// Persist assistant message with model attribution
|
||||
if result.Content != "" {
|
||||
@@ -536,7 +551,7 @@ func (h *CompletionHandler) streamCompletion(
|
||||
req providers.CompletionRequest,
|
||||
channelID, userID, model, configID, providerScope, personaID, workspaceID string,
|
||||
) {
|
||||
result := streamWithToolLoop(c, provider, cfg, &req, model, userID, channelID, personaID, workspaceID, h.hub)
|
||||
result := streamWithToolLoop(c, provider, cfg, &req, model, userID, channelID, personaID, workspaceID, configID, h.hub, h.health)
|
||||
|
||||
// Persist assistant response
|
||||
if result.Content != "" {
|
||||
@@ -566,7 +581,9 @@ func (h *CompletionHandler) syncCompletion(
|
||||
var allToolActivity []map[string]interface{}
|
||||
|
||||
for iteration := 0; iteration < maxToolIterations; iteration++ {
|
||||
callStart := time.Now()
|
||||
resp, err := provider.ChatCompletion(c.Request.Context(), cfg, req)
|
||||
h.recordHealth(configID, callStart, err)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadGateway, gin.H{"error": "provider error: " + err.Error()})
|
||||
return
|
||||
@@ -1175,3 +1192,21 @@ func calcCost(tokens int, pricePerM *float64) *float64 {
|
||||
cost := float64(tokens) * *pricePerM / 1_000_000.0
|
||||
return &cost
|
||||
}
|
||||
|
||||
// recordHealth records provider call outcome for health tracking (v0.22.0).
|
||||
func (h *CompletionHandler) recordHealth(configID string, start time.Time, err error) {
|
||||
if h.health == nil || configID == "" {
|
||||
return
|
||||
}
|
||||
latencyMs := int(time.Since(start).Milliseconds())
|
||||
if err != nil {
|
||||
errMsg := err.Error()
|
||||
if strings.Contains(errMsg, "context deadline exceeded") || strings.Contains(errMsg, "timeout") {
|
||||
h.health.RecordTimeout(configID, latencyMs, errMsg)
|
||||
} else {
|
||||
h.health.RecordError(configID, latencyMs, errMsg)
|
||||
}
|
||||
} else {
|
||||
h.health.RecordSuccess(configID, latencyMs)
|
||||
}
|
||||
}
|
||||
|
||||
280
server/handlers/health_admin.go
Normal file
280
server/handlers/health_admin.go
Normal file
@@ -0,0 +1,280 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
capspkg "git.gobha.me/xcaliber/chat-switchboard/capabilities"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/health"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
// ── Health Admin Handler ────────────────────
|
||||
|
||||
type HealthAdminHandler struct {
|
||||
healthStore health.Store
|
||||
stores store.Stores
|
||||
}
|
||||
|
||||
func NewHealthAdminHandler(hs health.Store, stores store.Stores) *HealthAdminHandler {
|
||||
return &HealthAdminHandler{healthStore: hs, stores: stores}
|
||||
}
|
||||
|
||||
// GetProviderHealth returns health metrics for a single provider.
|
||||
// GET /api/v1/admin/providers/:id/health?hours=24
|
||||
func (h *HealthAdminHandler) GetProviderHealth(c *gin.Context) {
|
||||
providerID := c.Param("id")
|
||||
hours, _ := strconv.Atoi(c.DefaultQuery("hours", "24"))
|
||||
if hours < 1 || hours > 168 {
|
||||
hours = 24
|
||||
}
|
||||
|
||||
windows, err := h.healthStore.ListWindows(c.Request.Context(), providerID, hours)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to fetch health data"})
|
||||
return
|
||||
}
|
||||
|
||||
// Derive current status from most recent window
|
||||
var summary models.ProviderHealthSummary
|
||||
summary.ProviderConfigID = providerID
|
||||
|
||||
if len(windows) > 0 {
|
||||
latest := windows[0]
|
||||
summary.RequestCount = latest.RequestCount
|
||||
summary.ErrorRate = latest.ErrorRate()
|
||||
summary.AvgLatencyMs = latest.AvgLatencyMs()
|
||||
summary.MaxLatencyMs = latest.MaxLatencyMs
|
||||
summary.LastError = latest.LastError
|
||||
summary.LastErrorAt = latest.LastErrorAt
|
||||
summary.Status = health.DeriveStatus(summary.ErrorRate, summary.RequestCount)
|
||||
} else {
|
||||
summary.Status = models.StatusUnknown
|
||||
}
|
||||
|
||||
SafeJSON(c, http.StatusOK, gin.H{
|
||||
"summary": summary,
|
||||
"windows": windows,
|
||||
})
|
||||
}
|
||||
|
||||
// GetAllProviderHealth returns current health for all providers.
|
||||
// GET /api/v1/admin/providers/health
|
||||
func (h *HealthAdminHandler) GetAllProviderHealth(c *gin.Context) {
|
||||
windows, err := h.healthStore.ListAllCurrent(c.Request.Context())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to fetch health data"})
|
||||
return
|
||||
}
|
||||
|
||||
// Build summaries
|
||||
summaries := make([]models.ProviderHealthSummary, 0, len(windows))
|
||||
for _, w := range windows {
|
||||
s := models.ProviderHealthSummary{
|
||||
ProviderConfigID: w.ProviderConfigID,
|
||||
Status: health.DeriveStatus(w.ErrorRate(), w.RequestCount),
|
||||
RequestCount: w.RequestCount,
|
||||
ErrorRate: w.ErrorRate(),
|
||||
AvgLatencyMs: w.AvgLatencyMs(),
|
||||
MaxLatencyMs: w.MaxLatencyMs,
|
||||
LastError: w.LastError,
|
||||
LastErrorAt: w.LastErrorAt,
|
||||
}
|
||||
summaries = append(summaries, s)
|
||||
}
|
||||
|
||||
SafeJSON(c, http.StatusOK, gin.H{
|
||||
"data": summaries,
|
||||
})
|
||||
}
|
||||
|
||||
// ── Capability Override Admin Handler ────────
|
||||
|
||||
type CapOverrideAdminHandler struct {
|
||||
stores store.Stores
|
||||
}
|
||||
|
||||
func NewCapOverrideAdminHandler(stores store.Stores) *CapOverrideAdminHandler {
|
||||
return &CapOverrideAdminHandler{stores: stores}
|
||||
}
|
||||
|
||||
// GetModelCapabilities returns resolved capabilities with source annotation.
|
||||
// GET /api/v1/admin/models/:id/capabilities?provider_config_id=...
|
||||
func (h *CapOverrideAdminHandler) GetModelCapabilities(c *gin.Context) {
|
||||
modelID := c.Param("id")
|
||||
providerConfigID := c.Query("provider_config_id")
|
||||
|
||||
// Get catalog capabilities
|
||||
var catalogCaps *models.ModelCapabilities
|
||||
if providerConfigID != "" {
|
||||
if entry, err := h.stores.Catalog.GetByModelID(c.Request.Context(), providerConfigID, modelID); err == nil {
|
||||
catalogCaps = &entry.Capabilities
|
||||
}
|
||||
} else {
|
||||
if entry, err := h.stores.Catalog.GetByModelIDAny(c.Request.Context(), modelID); err == nil {
|
||||
catalogCaps = &entry.Capabilities
|
||||
}
|
||||
}
|
||||
|
||||
// Get admin overrides
|
||||
var overrides []models.CapabilityOverride
|
||||
var err error
|
||||
if providerConfigID != "" {
|
||||
overrides, err = h.stores.CapOverrides.ListForProviderModel(c.Request.Context(), providerConfigID, modelID)
|
||||
} else {
|
||||
overrides, err = h.stores.CapOverrides.ListForModel(c.Request.Context(), modelID)
|
||||
}
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to fetch overrides"})
|
||||
return
|
||||
}
|
||||
|
||||
// Resolve through three tiers
|
||||
heuristic := capspkg.InferCapabilities(modelID)
|
||||
resolved := capspkg.ResolveIntrinsic(modelID, catalogCaps, overrides)
|
||||
|
||||
// Build source annotations
|
||||
sources := buildSourceAnnotations(catalogCaps, &heuristic, overrides)
|
||||
|
||||
SafeJSON(c, http.StatusOK, gin.H{
|
||||
"model_id": modelID,
|
||||
"resolved": resolved,
|
||||
"catalog": catalogCaps,
|
||||
"heuristic": heuristic,
|
||||
"overrides": overrides,
|
||||
"sources": sources,
|
||||
})
|
||||
}
|
||||
|
||||
// SetModelCapability sets an admin override for a specific capability.
|
||||
// PUT /api/v1/admin/models/:id/capabilities
|
||||
func (h *CapOverrideAdminHandler) SetModelCapability(c *gin.Context) {
|
||||
modelID := c.Param("id")
|
||||
userID := c.GetString("user_id")
|
||||
|
||||
var req struct {
|
||||
ProviderConfigID *string `json:"provider_config_id"`
|
||||
Field string `json:"field" binding:"required"`
|
||||
Value string `json:"value" binding:"required"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
// Validate field name
|
||||
validFields := map[string]bool{
|
||||
"streaming": true, "tool_calling": true, "vision": true,
|
||||
"thinking": true, "reasoning": true, "code_optimized": true,
|
||||
"web_search": true, "max_context": true, "max_output_tokens": true,
|
||||
}
|
||||
if !validFields[req.Field] {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid capability field: " + req.Field})
|
||||
return
|
||||
}
|
||||
|
||||
// Validate value
|
||||
switch req.Field {
|
||||
case "max_context", "max_output_tokens":
|
||||
if _, err := strconv.Atoi(req.Value); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "value must be a number for " + req.Field})
|
||||
return
|
||||
}
|
||||
default:
|
||||
if req.Value != "true" && req.Value != "false" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "value must be 'true' or 'false' for " + req.Field})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
override := &models.CapabilityOverride{
|
||||
ProviderConfigID: req.ProviderConfigID,
|
||||
ModelID: modelID,
|
||||
Field: req.Field,
|
||||
Value: req.Value,
|
||||
SetBy: &userID,
|
||||
}
|
||||
|
||||
if err := h.stores.CapOverrides.Set(c.Request.Context(), override); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to set override"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"status": "ok"})
|
||||
}
|
||||
|
||||
// DeleteModelCapability removes a specific override.
|
||||
// DELETE /api/v1/admin/models/:id/capabilities/:overrideId
|
||||
func (h *CapOverrideAdminHandler) DeleteModelCapability(c *gin.Context) {
|
||||
overrideID := c.Param("overrideId")
|
||||
|
||||
if err := h.stores.CapOverrides.Delete(c.Request.Context(), overrideID); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete override"})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"status": "ok"})
|
||||
}
|
||||
|
||||
// ListAllOverrides returns all capability overrides (admin view).
|
||||
// GET /api/v1/admin/capability-overrides
|
||||
func (h *CapOverrideAdminHandler) ListAllOverrides(c *gin.Context) {
|
||||
overrides, err := h.stores.CapOverrides.ListAll(c.Request.Context())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to fetch overrides"})
|
||||
return
|
||||
}
|
||||
|
||||
SafeJSON(c, http.StatusOK, gin.H{
|
||||
"data": overrides,
|
||||
"total": len(overrides),
|
||||
})
|
||||
}
|
||||
|
||||
// ── Source Annotation Helpers ────────────────
|
||||
|
||||
type capSource struct {
|
||||
Field string `json:"field"`
|
||||
Source string `json:"source"` // "catalog", "heuristic", "override"
|
||||
}
|
||||
|
||||
func buildSourceAnnotations(catalog *models.ModelCapabilities, heuristic *models.ModelCapabilities, overrides []models.CapabilityOverride) []capSource {
|
||||
// Build override set for quick lookup
|
||||
overrideSet := make(map[string]bool, len(overrides))
|
||||
for _, o := range overrides {
|
||||
overrideSet[o.Field] = true
|
||||
}
|
||||
|
||||
fields := []struct {
|
||||
name string
|
||||
hasCatalog bool
|
||||
hasHeur bool
|
||||
}{
|
||||
{"streaming", catalog != nil && catalog.Streaming, heuristic.Streaming},
|
||||
{"tool_calling", catalog != nil && catalog.ToolCalling, heuristic.ToolCalling},
|
||||
{"vision", catalog != nil && catalog.Vision, heuristic.Vision},
|
||||
{"thinking", catalog != nil && catalog.Thinking, heuristic.Thinking},
|
||||
{"reasoning", catalog != nil && catalog.Reasoning, heuristic.Reasoning},
|
||||
{"code_optimized", catalog != nil && catalog.CodeOptimized, heuristic.CodeOptimized},
|
||||
{"web_search", catalog != nil && catalog.WebSearch, heuristic.WebSearch},
|
||||
{"max_context", catalog != nil && catalog.MaxContext > 0, heuristic.MaxContext > 0},
|
||||
{"max_output_tokens", catalog != nil && catalog.MaxOutputTokens > 0, heuristic.MaxOutputTokens > 0},
|
||||
}
|
||||
|
||||
var sources []capSource
|
||||
for _, f := range fields {
|
||||
src := "default"
|
||||
if overrideSet[f.name] {
|
||||
src = "override"
|
||||
} else if f.hasCatalog {
|
||||
src = "catalog"
|
||||
} else if f.hasHeur {
|
||||
src = "heuristic"
|
||||
}
|
||||
sources = append(sources, capSource{Field: f.name, Source: src})
|
||||
}
|
||||
return sources
|
||||
}
|
||||
@@ -516,7 +516,7 @@ func (h *MessageHandler) Regenerate(c *gin.Context) {
|
||||
|
||||
// ── Stream the response (shared loop handles tools, reasoning, SSE) ──
|
||||
|
||||
result := streamWithToolLoop(c, provider, providerCfg, &provReq, model, userID, channelID, personaID, workspaceID, h.hub)
|
||||
result := streamWithToolLoop(c, provider, providerCfg, &provReq, model, userID, channelID, personaID, workspaceID, configID, h.hub, comp.health)
|
||||
|
||||
// Persist as sibling (regen) with tool activity
|
||||
if result.Content != "" {
|
||||
|
||||
@@ -27,6 +27,20 @@ type streamResult struct {
|
||||
CacheReadTokens int
|
||||
}
|
||||
|
||||
// recordHealthFn records provider health from standalone streaming functions.
|
||||
func recordHealthFn(hr HealthRecorder, configID string, start time.Time, err error) {
|
||||
if hr == nil || configID == "" {
|
||||
return
|
||||
}
|
||||
latencyMs := int(time.Since(start).Milliseconds())
|
||||
if err != nil {
|
||||
errMsg := err.Error()
|
||||
hr.RecordError(configID, latencyMs, errMsg)
|
||||
} else {
|
||||
hr.RecordSuccess(configID, latencyMs)
|
||||
}
|
||||
}
|
||||
|
||||
// streamWithToolLoop is the single, canonical streaming implementation.
|
||||
// It handles SSE setup, multi-round tool execution, reasoning_content
|
||||
// forwarding, and client notifications. Returns the accumulated content
|
||||
@@ -43,8 +57,9 @@ func streamWithToolLoop(
|
||||
provider providers.Provider,
|
||||
cfg providers.ProviderConfig,
|
||||
req *providers.CompletionRequest,
|
||||
model, userID, channelID, personaID, workspaceID string,
|
||||
model, userID, channelID, personaID, workspaceID, configID string,
|
||||
hub *events.Hub,
|
||||
health HealthRecorder,
|
||||
) streamResult {
|
||||
// Set SSE headers
|
||||
c.Header("Content-Type", "text/event-stream")
|
||||
@@ -74,8 +89,10 @@ func streamWithToolLoop(
|
||||
var result streamResult
|
||||
|
||||
for iteration := 0; iteration < maxToolIterations; iteration++ {
|
||||
callStart := time.Now()
|
||||
ch, err := provider.StreamCompletion(c.Request.Context(), cfg, *req)
|
||||
if err != nil {
|
||||
recordHealthFn(health, configID, callStart, err)
|
||||
sendSSE(fmt.Sprintf(`{"error":"%s"}`, escapeJSON(err.Error())))
|
||||
return result
|
||||
}
|
||||
@@ -86,6 +103,7 @@ func streamWithToolLoop(
|
||||
|
||||
for event := range ch {
|
||||
if event.Error != nil {
|
||||
recordHealthFn(health, configID, callStart, event.Error)
|
||||
sendSSE(fmt.Sprintf(`{"error":"%s"}`, escapeJSON(event.Error.Error())))
|
||||
return result
|
||||
}
|
||||
@@ -107,8 +125,9 @@ func streamWithToolLoop(
|
||||
}
|
||||
|
||||
if event.Done {
|
||||
recordHealthFn(health, configID, callStart, nil)
|
||||
if event.FinishReason == "tool_calls" && len(event.ToolCalls) > 0 {
|
||||
toolCalls = event.ToolCalls
|
||||
toolCalls = event.ToolCalls
|
||||
// Accumulate tokens from this tool-call iteration
|
||||
result.InputTokens += event.InputTokens
|
||||
result.OutputTokens += event.OutputTokens
|
||||
@@ -268,8 +287,9 @@ func streamModelResponse(
|
||||
provider providers.Provider,
|
||||
cfg providers.ProviderConfig,
|
||||
req *providers.CompletionRequest,
|
||||
model, displayName, userID, channelID, personaID, workspaceID string,
|
||||
model, displayName, userID, channelID, personaID, workspaceID, configID string,
|
||||
hub *events.Hub,
|
||||
health HealthRecorder,
|
||||
) streamResult {
|
||||
flusher, _ := c.Writer.(http.Flusher)
|
||||
flush := func() {
|
||||
@@ -289,8 +309,10 @@ func streamModelResponse(
|
||||
var result streamResult
|
||||
|
||||
for iteration := 0; iteration < maxToolIterations; iteration++ {
|
||||
callStart := time.Now()
|
||||
ch, err := provider.StreamCompletion(c.Request.Context(), cfg, *req)
|
||||
if err != nil {
|
||||
recordHealthFn(health, configID, callStart, err)
|
||||
sendSSE(fmt.Sprintf(`{"error":"%s"}`, escapeJSON(err.Error())))
|
||||
return result
|
||||
}
|
||||
@@ -301,6 +323,7 @@ func streamModelResponse(
|
||||
|
||||
for event := range ch {
|
||||
if event.Error != nil {
|
||||
recordHealthFn(health, configID, callStart, event.Error)
|
||||
sendSSE(fmt.Sprintf(`{"error":"%s"}`, escapeJSON(event.Error.Error())))
|
||||
return result
|
||||
}
|
||||
@@ -320,6 +343,7 @@ func streamModelResponse(
|
||||
}
|
||||
|
||||
if event.Done {
|
||||
recordHealthFn(health, configID, callStart, nil)
|
||||
if event.FinishReason == "tool_calls" && len(event.ToolCalls) > 0 {
|
||||
toolCalls = event.ToolCalls
|
||||
result.InputTokens += event.InputTokens
|
||||
|
||||
@@ -310,7 +310,7 @@ func (h *TeamHandler) ListTeamProviderModels(c *gin.Context) {
|
||||
|
||||
out := make([]modelInfo, 0, len(modelList))
|
||||
for _, m := range modelList {
|
||||
caps := capspkg.ResolveIntrinsic(m.ID, &m.Capabilities)
|
||||
caps := capspkg.ResolveIntrinsic(m.ID, &m.Capabilities, nil)
|
||||
caps.MaxOutputTokens = capspkg.ResolveMaxOutput(m.ID, caps)
|
||||
out = append(out, modelInfo{ID: m.ID, Capabilities: caps})
|
||||
}
|
||||
|
||||
219
server/health/accumulator.go
Normal file
219
server/health/accumulator.go
Normal file
@@ -0,0 +1,219 @@
|
||||
// Package health provides in-memory accumulation of provider health
|
||||
// metrics with periodic flush to the database. The accumulator is
|
||||
// goroutine-safe and designed for high-throughput recording from
|
||||
// concurrent completion handlers.
|
||||
package health
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
// ── Thresholds ──────────────────────────────
|
||||
|
||||
const (
|
||||
// DegradedThreshold — error rate above this = degraded.
|
||||
DegradedThreshold = 0.05 // 5%
|
||||
// DownThreshold — error rate above this = down.
|
||||
DownThreshold = 0.25 // 25%
|
||||
// FlushInterval — how often in-memory counters are flushed to DB.
|
||||
FlushInterval = 60 * time.Second
|
||||
// PruneAge — health windows older than this are deleted.
|
||||
PruneAge = 7 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// ── Store Interface ─────────────────────────
|
||||
|
||||
// Store is the persistence layer for health data.
|
||||
type Store interface {
|
||||
// UpsertWindow merges in-memory counters into the hourly bucket row.
|
||||
UpsertWindow(ctx context.Context, w *models.ProviderHealthWindow) error
|
||||
|
||||
// GetCurrentWindow returns the current hourly bucket for a provider.
|
||||
GetCurrentWindow(ctx context.Context, providerConfigID string) (*models.ProviderHealthWindow, error)
|
||||
|
||||
// ListWindows returns the last N hourly buckets for a provider, newest first.
|
||||
ListWindows(ctx context.Context, providerConfigID string, hours int) ([]models.ProviderHealthWindow, error)
|
||||
|
||||
// ListAllCurrent returns the current-hour bucket for every provider.
|
||||
ListAllCurrent(ctx context.Context) ([]models.ProviderHealthWindow, error)
|
||||
|
||||
// Prune deletes rows older than the given time.
|
||||
Prune(ctx context.Context, before time.Time) (int64, error)
|
||||
}
|
||||
|
||||
// ── In-Memory Bucket ────────────────────────
|
||||
|
||||
// bucket holds counters for one provider within one flush interval.
|
||||
type bucket struct {
|
||||
providerConfigID string
|
||||
requestCount int
|
||||
errorCount int
|
||||
timeoutCount int
|
||||
totalLatencyMs int64
|
||||
maxLatencyMs int
|
||||
lastError string
|
||||
lastErrorAt time.Time
|
||||
}
|
||||
|
||||
// ── Accumulator ─────────────────────────────
|
||||
|
||||
// Accumulator collects health metrics in memory and periodically
|
||||
// flushes them to the database. One instance per process.
|
||||
type Accumulator struct {
|
||||
mu sync.Mutex
|
||||
buckets map[string]*bucket // keyed by provider_config_id
|
||||
store Store
|
||||
stopCh chan struct{}
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewAccumulator creates and starts the background flush loop.
|
||||
func NewAccumulator(store Store) *Accumulator {
|
||||
a := &Accumulator{
|
||||
buckets: make(map[string]*bucket),
|
||||
store: store,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
a.wg.Add(1)
|
||||
go a.flushLoop()
|
||||
return a
|
||||
}
|
||||
|
||||
// RecordSuccess records a successful provider call.
|
||||
func (a *Accumulator) RecordSuccess(providerConfigID string, latencyMs int) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
b := a.getBucket(providerConfigID)
|
||||
b.requestCount++
|
||||
b.totalLatencyMs += int64(latencyMs)
|
||||
if latencyMs > b.maxLatencyMs {
|
||||
b.maxLatencyMs = latencyMs
|
||||
}
|
||||
}
|
||||
|
||||
// RecordError records a failed provider call.
|
||||
func (a *Accumulator) RecordError(providerConfigID string, latencyMs int, errMsg string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
b := a.getBucket(providerConfigID)
|
||||
b.requestCount++
|
||||
b.errorCount++
|
||||
b.totalLatencyMs += int64(latencyMs)
|
||||
if latencyMs > b.maxLatencyMs {
|
||||
b.maxLatencyMs = latencyMs
|
||||
}
|
||||
b.lastError = errMsg
|
||||
b.lastErrorAt = time.Now().UTC()
|
||||
}
|
||||
|
||||
// RecordTimeout records a timed-out provider call (counted as both error and timeout).
|
||||
func (a *Accumulator) RecordTimeout(providerConfigID string, latencyMs int, errMsg string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
b := a.getBucket(providerConfigID)
|
||||
b.requestCount++
|
||||
b.errorCount++
|
||||
b.timeoutCount++
|
||||
b.totalLatencyMs += int64(latencyMs)
|
||||
if latencyMs > b.maxLatencyMs {
|
||||
b.maxLatencyMs = latencyMs
|
||||
}
|
||||
b.lastError = errMsg
|
||||
b.lastErrorAt = time.Now().UTC()
|
||||
}
|
||||
|
||||
// Stop halts the flush loop and performs a final flush.
|
||||
func (a *Accumulator) Stop() {
|
||||
close(a.stopCh)
|
||||
a.wg.Wait()
|
||||
}
|
||||
|
||||
// DeriveStatus computes the provider status from an error rate.
|
||||
func DeriveStatus(errorRate float64, requestCount int) models.ProviderStatus {
|
||||
if requestCount == 0 {
|
||||
return models.StatusUnknown
|
||||
}
|
||||
if errorRate >= DownThreshold {
|
||||
return models.StatusDown
|
||||
}
|
||||
if errorRate >= DegradedThreshold {
|
||||
return models.StatusDegraded
|
||||
}
|
||||
return models.StatusHealthy
|
||||
}
|
||||
|
||||
// ── Internal ────────────────────────────────
|
||||
|
||||
func (a *Accumulator) getBucket(providerConfigID string) *bucket {
|
||||
b, ok := a.buckets[providerConfigID]
|
||||
if !ok {
|
||||
b = &bucket{providerConfigID: providerConfigID}
|
||||
a.buckets[providerConfigID] = b
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (a *Accumulator) flushLoop() {
|
||||
defer a.wg.Done()
|
||||
ticker := time.NewTicker(FlushInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
a.flush()
|
||||
case <-a.stopCh:
|
||||
a.flush() // final flush
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Accumulator) flush() {
|
||||
a.mu.Lock()
|
||||
if len(a.buckets) == 0 {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
// Swap out current buckets so we don't hold the lock during DB writes.
|
||||
snap := a.buckets
|
||||
a.buckets = make(map[string]*bucket, len(snap))
|
||||
a.mu.Unlock()
|
||||
|
||||
windowStart := hourFloor(time.Now().UTC())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
for _, b := range snap {
|
||||
if b.requestCount == 0 {
|
||||
continue
|
||||
}
|
||||
w := &models.ProviderHealthWindow{
|
||||
ProviderConfigID: b.providerConfigID,
|
||||
WindowStart: windowStart,
|
||||
RequestCount: b.requestCount,
|
||||
ErrorCount: b.errorCount,
|
||||
TimeoutCount: b.timeoutCount,
|
||||
TotalLatencyMs: b.totalLatencyMs,
|
||||
MaxLatencyMs: b.maxLatencyMs,
|
||||
}
|
||||
if b.lastError != "" {
|
||||
w.LastError = &b.lastError
|
||||
ts := b.lastErrorAt.Format(time.RFC3339)
|
||||
w.LastErrorAt = &ts
|
||||
}
|
||||
if err := a.store.UpsertWindow(ctx, w); err != nil {
|
||||
log.Printf("⚠ health: flush failed for provider %s: %v", b.providerConfigID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hourFloor truncates a time to the start of its hour.
|
||||
func hourFloor(t time.Time) time.Time {
|
||||
return t.Truncate(time.Hour)
|
||||
}
|
||||
239
server/health/accumulator_test.go
Normal file
239
server/health/accumulator_test.go
Normal file
@@ -0,0 +1,239 @@
|
||||
package health
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
// ── Mock Store ──────────────────────────────
|
||||
|
||||
type mockStore struct {
|
||||
mu sync.Mutex
|
||||
windows map[string]*models.ProviderHealthWindow // keyed by provider_config_id
|
||||
}
|
||||
|
||||
func newMockStore() *mockStore {
|
||||
return &mockStore{windows: make(map[string]*models.ProviderHealthWindow)}
|
||||
}
|
||||
|
||||
func (m *mockStore) UpsertWindow(_ context.Context, w *models.ProviderHealthWindow) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
existing, ok := m.windows[w.ProviderConfigID]
|
||||
if ok {
|
||||
existing.RequestCount += w.RequestCount
|
||||
existing.ErrorCount += w.ErrorCount
|
||||
existing.TimeoutCount += w.TimeoutCount
|
||||
existing.TotalLatencyMs += w.TotalLatencyMs
|
||||
if w.MaxLatencyMs > existing.MaxLatencyMs {
|
||||
existing.MaxLatencyMs = w.MaxLatencyMs
|
||||
}
|
||||
if w.LastError != nil {
|
||||
existing.LastError = w.LastError
|
||||
existing.LastErrorAt = w.LastErrorAt
|
||||
}
|
||||
} else {
|
||||
cp := *w
|
||||
m.windows[w.ProviderConfigID] = &cp
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockStore) GetCurrentWindow(_ context.Context, id string) (*models.ProviderHealthWindow, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
w, ok := m.windows[id]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
cp := *w
|
||||
return &cp, nil
|
||||
}
|
||||
|
||||
func (m *mockStore) ListWindows(_ context.Context, _ string, _ int) ([]models.ProviderHealthWindow, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockStore) ListAllCurrent(_ context.Context) ([]models.ProviderHealthWindow, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var result []models.ProviderHealthWindow
|
||||
for _, w := range m.windows {
|
||||
result = append(result, *w)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (m *mockStore) Prune(_ context.Context, _ time.Time) (int64, error) { return 0, nil }
|
||||
|
||||
// ── Tests ───────────────────────────────────
|
||||
|
||||
func TestRecordSuccess(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
a := &Accumulator{
|
||||
buckets: make(map[string]*bucket),
|
||||
store: ms,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
a.RecordSuccess("prov-1", 100)
|
||||
a.RecordSuccess("prov-1", 200)
|
||||
a.RecordSuccess("prov-2", 50)
|
||||
|
||||
a.flush()
|
||||
|
||||
w1, _ := ms.GetCurrentWindow(context.Background(), "prov-1")
|
||||
if w1 == nil {
|
||||
t.Fatal("expected prov-1 window")
|
||||
}
|
||||
if w1.RequestCount != 2 {
|
||||
t.Fatalf("expected 2 requests, got %d", w1.RequestCount)
|
||||
}
|
||||
if w1.ErrorCount != 0 {
|
||||
t.Fatalf("expected 0 errors, got %d", w1.ErrorCount)
|
||||
}
|
||||
if w1.TotalLatencyMs != 300 {
|
||||
t.Fatalf("expected 300ms total latency, got %d", w1.TotalLatencyMs)
|
||||
}
|
||||
if w1.MaxLatencyMs != 200 {
|
||||
t.Fatalf("expected 200ms max latency, got %d", w1.MaxLatencyMs)
|
||||
}
|
||||
|
||||
w2, _ := ms.GetCurrentWindow(context.Background(), "prov-2")
|
||||
if w2 == nil {
|
||||
t.Fatal("expected prov-2 window")
|
||||
}
|
||||
if w2.RequestCount != 1 {
|
||||
t.Fatalf("expected 1 request, got %d", w2.RequestCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordError(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
a := &Accumulator{
|
||||
buckets: make(map[string]*bucket),
|
||||
store: ms,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
a.RecordSuccess("prov-1", 100)
|
||||
a.RecordError("prov-1", 500, "connection refused")
|
||||
a.RecordSuccess("prov-1", 150)
|
||||
|
||||
a.flush()
|
||||
|
||||
w, _ := ms.GetCurrentWindow(context.Background(), "prov-1")
|
||||
if w.RequestCount != 3 {
|
||||
t.Fatalf("expected 3 requests, got %d", w.RequestCount)
|
||||
}
|
||||
if w.ErrorCount != 1 {
|
||||
t.Fatalf("expected 1 error, got %d", w.ErrorCount)
|
||||
}
|
||||
if w.LastError == nil || *w.LastError != "connection refused" {
|
||||
t.Fatalf("expected last_error='connection refused', got %v", w.LastError)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordTimeout(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
a := &Accumulator{
|
||||
buckets: make(map[string]*bucket),
|
||||
store: ms,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
a.RecordTimeout("prov-1", 30000, "context deadline exceeded")
|
||||
a.flush()
|
||||
|
||||
w, _ := ms.GetCurrentWindow(context.Background(), "prov-1")
|
||||
if w.RequestCount != 1 {
|
||||
t.Fatalf("expected 1 request, got %d", w.RequestCount)
|
||||
}
|
||||
if w.ErrorCount != 1 {
|
||||
t.Fatalf("expected 1 error, got %d", w.ErrorCount)
|
||||
}
|
||||
if w.TimeoutCount != 1 {
|
||||
t.Fatalf("expected 1 timeout, got %d", w.TimeoutCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveStatus(t *testing.T) {
|
||||
tests := []struct {
|
||||
rate float64
|
||||
count int
|
||||
expected models.ProviderStatus
|
||||
}{
|
||||
{0, 0, models.StatusUnknown},
|
||||
{0, 100, models.StatusHealthy},
|
||||
{0.03, 100, models.StatusHealthy},
|
||||
{0.05, 100, models.StatusDegraded},
|
||||
{0.15, 100, models.StatusDegraded},
|
||||
{0.25, 100, models.StatusDown},
|
||||
{0.50, 100, models.StatusDown},
|
||||
{1.0, 1, models.StatusDown},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := DeriveStatus(tt.rate, tt.count)
|
||||
if got != tt.expected {
|
||||
t.Errorf("DeriveStatus(%v, %d) = %s, want %s", tt.rate, tt.count, got, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushClearsBuckets(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
a := &Accumulator{
|
||||
buckets: make(map[string]*bucket),
|
||||
store: ms,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
a.RecordSuccess("prov-1", 100)
|
||||
a.flush()
|
||||
|
||||
// After flush, buckets should be empty
|
||||
a.mu.Lock()
|
||||
count := len(a.buckets)
|
||||
a.mu.Unlock()
|
||||
if count != 0 {
|
||||
t.Fatalf("expected 0 buckets after flush, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentRecording(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
a := &Accumulator{
|
||||
buckets: make(map[string]*bucket),
|
||||
store: ms,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 100; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
a.RecordSuccess("prov-1", 50)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
a.flush()
|
||||
|
||||
w, _ := ms.GetCurrentWindow(context.Background(), "prov-1")
|
||||
if w.RequestCount != 100 {
|
||||
t.Fatalf("expected 100 concurrent requests, got %d", w.RequestCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHourFloor(t *testing.T) {
|
||||
ts := time.Date(2026, 3, 1, 14, 37, 42, 0, time.UTC)
|
||||
floor := hourFloor(ts)
|
||||
expected := time.Date(2026, 3, 1, 14, 0, 0, 0, time.UTC)
|
||||
if !floor.Equal(expected) {
|
||||
t.Fatalf("expected %v, got %v", expected, floor)
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"git.gobha.me/xcaliber/chat-switchboard/events"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/extraction"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/handlers"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/health"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/memory"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/middleware"
|
||||
@@ -54,6 +55,8 @@ func main() {
|
||||
uekCache := crypto.NewUEKCache()
|
||||
var keyResolver *crypto.KeyResolver
|
||||
var objStore storage.ObjectStore
|
||||
var healthAccum *health.Accumulator
|
||||
var healthStore *postgres.HealthStore
|
||||
|
||||
if err := database.Connect(cfg); err != nil {
|
||||
log.Printf("⚠ Database unavailable: %v", err)
|
||||
@@ -89,6 +92,26 @@ func main() {
|
||||
// Initialize store layer
|
||||
stores = postgres.NewStores(database.DB)
|
||||
|
||||
// Provider health accumulator (v0.22.0)
|
||||
healthStore = postgres.NewHealthStore(database.DB)
|
||||
healthAccum = health.NewAccumulator(healthStore)
|
||||
defer healthAccum.Stop()
|
||||
|
||||
// Background health cleanup: prune windows older than 7 days
|
||||
go func() {
|
||||
ticker := time.NewTicker(6 * time.Hour)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
if n, err := healthStore.Prune(ctx, time.Now().UTC().Add(-health.PruneAge)); err != nil {
|
||||
log.Printf("⚠ health prune failed: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("🧹 health: pruned %d old windows", n)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}()
|
||||
|
||||
// Bootstrap admin from env (K8s secret) — upserts on every restart
|
||||
handlers.BootstrapAdmin(cfg, stores)
|
||||
|
||||
@@ -350,6 +373,9 @@ func main() {
|
||||
|
||||
// Chat Completions
|
||||
comp := handlers.NewCompletionHandler(keyResolver, stores, hub, objStore, kbEmbedder)
|
||||
if healthAccum != nil {
|
||||
comp.SetHealthRecorder(healthAccum)
|
||||
}
|
||||
protected.POST("/chat/completions", comp.Complete)
|
||||
protected.GET("/tools", comp.ListTools)
|
||||
|
||||
@@ -722,6 +748,18 @@ func main() {
|
||||
admin.POST("/extensions", extAdm.AdminInstallExtension)
|
||||
admin.PUT("/extensions/:id", extAdm.AdminUpdateExtension)
|
||||
admin.DELETE("/extensions/:id", extAdm.AdminUninstallExtension)
|
||||
|
||||
// Provider Health (admin — v0.22.0)
|
||||
healthAdm := handlers.NewHealthAdminHandler(healthStore, stores)
|
||||
admin.GET("/providers/health", healthAdm.GetAllProviderHealth)
|
||||
admin.GET("/providers/:id/health", healthAdm.GetProviderHealth)
|
||||
|
||||
// Capability Overrides (admin — v0.22.0)
|
||||
capAdm := handlers.NewCapOverrideAdminHandler(stores)
|
||||
admin.GET("/models/:id/capabilities", capAdm.GetModelCapabilities)
|
||||
admin.PUT("/models/:id/capabilities", capAdm.SetModelCapability)
|
||||
admin.DELETE("/models/:id/capabilities/:overrideId", capAdm.DeleteModelCapability)
|
||||
admin.GET("/capability-overrides", capAdm.ListAllOverrides)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -744,6 +782,7 @@ func main() {
|
||||
log.Printf(" Storage: disabled")
|
||||
}
|
||||
log.Printf(" EventBus: ready, WebSocket on %s/ws", cfg.BasePath)
|
||||
log.Printf(" Health: provider tracking active (flush=%s, prune=%s)", health.FlushInterval, health.PruneAge)
|
||||
if err := r.Run(":" + cfg.Port); err != nil {
|
||||
log.Fatalf("Failed to start server: %v", err)
|
||||
}
|
||||
|
||||
@@ -864,3 +864,76 @@ type ChannelKB struct {
|
||||
Enabled bool `json:"enabled" db:"enabled"`
|
||||
DocumentCount int `json:"document_count" db:"document_count"`
|
||||
}
|
||||
|
||||
// =========================================
|
||||
// PROVIDER HEALTH (v0.22.0)
|
||||
// =========================================
|
||||
|
||||
// ProviderStatus represents the derived health state.
|
||||
type ProviderStatus string
|
||||
|
||||
const (
|
||||
StatusHealthy ProviderStatus = "healthy"
|
||||
StatusDegraded ProviderStatus = "degraded"
|
||||
StatusDown ProviderStatus = "down"
|
||||
StatusUnknown ProviderStatus = "unknown"
|
||||
)
|
||||
|
||||
// ProviderHealthWindow is one hourly bucket of health metrics.
|
||||
type ProviderHealthWindow struct {
|
||||
ID string `json:"id" db:"id"`
|
||||
ProviderConfigID string `json:"provider_config_id" db:"provider_config_id"`
|
||||
WindowStart time.Time `json:"window_start" db:"window_start"`
|
||||
RequestCount int `json:"request_count" db:"request_count"`
|
||||
ErrorCount int `json:"error_count" db:"error_count"`
|
||||
TimeoutCount int `json:"timeout_count" db:"timeout_count"`
|
||||
TotalLatencyMs int64 `json:"total_latency_ms" db:"total_latency_ms"`
|
||||
MaxLatencyMs int `json:"max_latency_ms" db:"max_latency_ms"`
|
||||
LastError *string `json:"last_error,omitempty" db:"last_error"`
|
||||
LastErrorAt *string `json:"last_error_at,omitempty" db:"last_error_at"`
|
||||
}
|
||||
|
||||
// AvgLatencyMs returns the average latency, or 0 if no requests.
|
||||
func (w *ProviderHealthWindow) AvgLatencyMs() int {
|
||||
if w.RequestCount == 0 {
|
||||
return 0
|
||||
}
|
||||
return int(w.TotalLatencyMs / int64(w.RequestCount))
|
||||
}
|
||||
|
||||
// ErrorRate returns the error fraction, or 0 if no requests.
|
||||
func (w *ProviderHealthWindow) ErrorRate() float64 {
|
||||
if w.RequestCount == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(w.ErrorCount) / float64(w.RequestCount)
|
||||
}
|
||||
|
||||
// ProviderHealthSummary is the API response for a provider's current health.
|
||||
type ProviderHealthSummary struct {
|
||||
ProviderConfigID string `json:"provider_config_id"`
|
||||
ProviderName string `json:"provider_name,omitempty"`
|
||||
Status ProviderStatus `json:"status"`
|
||||
RequestCount int `json:"request_count"` // last hour
|
||||
ErrorRate float64 `json:"error_rate"` // last hour
|
||||
AvgLatencyMs int `json:"avg_latency_ms"` // last hour
|
||||
MaxLatencyMs int `json:"max_latency_ms"` // last hour
|
||||
LastError *string `json:"last_error,omitempty"`
|
||||
LastErrorAt *string `json:"last_error_at,omitempty"`
|
||||
}
|
||||
|
||||
// =========================================
|
||||
// CAPABILITY OVERRIDES (v0.22.0)
|
||||
// =========================================
|
||||
|
||||
// CapabilityOverride is an admin correction for a model's capabilities.
|
||||
type CapabilityOverride struct {
|
||||
ID string `json:"id" db:"id"`
|
||||
ProviderConfigID *string `json:"provider_config_id,omitempty" db:"provider_config_id"`
|
||||
ModelID string `json:"model_id" db:"model_id"`
|
||||
Field string `json:"field" db:"field"`
|
||||
Value string `json:"value" db:"value"`
|
||||
SetBy *string `json:"set_by,omitempty" db:"set_by"`
|
||||
CreatedAt string `json:"created_at" db:"created_at"`
|
||||
}
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@ type Stores struct {
|
||||
NotifPrefs NotificationPreferenceStore
|
||||
Workspaces WorkspaceStore
|
||||
GitCredentials GitCredentialStore
|
||||
CapOverrides CapabilityOverrideStore
|
||||
}
|
||||
|
||||
// =========================================
|
||||
@@ -562,6 +563,30 @@ type GitCredentialStore interface {
|
||||
Delete(ctx context.Context, id, userID string) error
|
||||
}
|
||||
|
||||
// =========================================
|
||||
// CAPABILITY OVERRIDES (v0.22.0)
|
||||
// =========================================
|
||||
|
||||
type CapabilityOverrideStore interface {
|
||||
// Set creates or updates an override for (provider, model, field).
|
||||
Set(ctx context.Context, o *models.CapabilityOverride) error
|
||||
|
||||
// Delete removes a specific override.
|
||||
Delete(ctx context.Context, id string) error
|
||||
|
||||
// ListForModel returns all overrides for a model ID (across all providers + global).
|
||||
ListForModel(ctx context.Context, modelID string) ([]models.CapabilityOverride, error)
|
||||
|
||||
// ListForProviderModel returns overrides for a specific provider+model combination.
|
||||
ListForProviderModel(ctx context.Context, providerConfigID, modelID string) ([]models.CapabilityOverride, error)
|
||||
|
||||
// ListAll returns every override (admin view).
|
||||
ListAll(ctx context.Context) ([]models.CapabilityOverride, error)
|
||||
|
||||
// DeleteForProvider removes all overrides for a provider (cascade cleanup).
|
||||
DeleteForProvider(ctx context.Context, providerConfigID string) error
|
||||
}
|
||||
|
||||
// =========================================
|
||||
// SHARED TYPES
|
||||
// =========================================
|
||||
|
||||
85
server/store/postgres/cap_override.go
Normal file
85
server/store/postgres/cap_override.go
Normal file
@@ -0,0 +1,85 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
// CapOverrideStore implements store.CapabilityOverrideStore for Postgres.
|
||||
type CapOverrideStore struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
func NewCapOverrideStore(db *sql.DB) *CapOverrideStore {
|
||||
return &CapOverrideStore{db: db}
|
||||
}
|
||||
|
||||
func (s *CapOverrideStore) Set(ctx context.Context, o *models.CapabilityOverride) error {
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO capability_overrides (provider_config_id, model_id, field, value, set_by)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (provider_config_id, model_id, field) DO UPDATE SET
|
||||
value = EXCLUDED.value,
|
||||
set_by = EXCLUDED.set_by,
|
||||
created_at = now()
|
||||
`, o.ProviderConfigID, o.ModelID, o.Field, o.Value, o.SetBy)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *CapOverrideStore) Delete(ctx context.Context, id string) error {
|
||||
_, err := s.db.ExecContext(ctx, `DELETE FROM capability_overrides WHERE id = $1`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *CapOverrideStore) ListForModel(ctx context.Context, modelID string) ([]models.CapabilityOverride, error) {
|
||||
return s.query(ctx, `
|
||||
SELECT id, provider_config_id, model_id, field, value, set_by, created_at
|
||||
FROM capability_overrides
|
||||
WHERE model_id = $1
|
||||
ORDER BY provider_config_id NULLS LAST, field
|
||||
`, modelID)
|
||||
}
|
||||
|
||||
func (s *CapOverrideStore) ListForProviderModel(ctx context.Context, providerConfigID, modelID string) ([]models.CapabilityOverride, error) {
|
||||
return s.query(ctx, `
|
||||
SELECT id, provider_config_id, model_id, field, value, set_by, created_at
|
||||
FROM capability_overrides
|
||||
WHERE model_id = $1 AND (provider_config_id = $2 OR provider_config_id IS NULL)
|
||||
ORDER BY provider_config_id NULLS LAST, field
|
||||
`, modelID, providerConfigID)
|
||||
}
|
||||
|
||||
func (s *CapOverrideStore) ListAll(ctx context.Context) ([]models.CapabilityOverride, error) {
|
||||
return s.query(ctx, `
|
||||
SELECT id, provider_config_id, model_id, field, value, set_by, created_at
|
||||
FROM capability_overrides
|
||||
ORDER BY model_id, provider_config_id NULLS LAST, field
|
||||
`)
|
||||
}
|
||||
|
||||
func (s *CapOverrideStore) DeleteForProvider(ctx context.Context, providerConfigID string) error {
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
DELETE FROM capability_overrides WHERE provider_config_id = $1
|
||||
`, providerConfigID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *CapOverrideStore) query(ctx context.Context, q string, args ...interface{}) ([]models.CapabilityOverride, error) {
|
||||
rows, err := s.db.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []models.CapabilityOverride
|
||||
for rows.Next() {
|
||||
var o models.CapabilityOverride
|
||||
if err := rows.Scan(&o.ID, &o.ProviderConfigID, &o.ModelID, &o.Field, &o.Value, &o.SetBy, &o.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, o)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
130
server/store/postgres/health.go
Normal file
130
server/store/postgres/health.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
// HealthStore implements health.Store for Postgres.
|
||||
type HealthStore struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
func NewHealthStore(db *sql.DB) *HealthStore {
|
||||
return &HealthStore{db: db}
|
||||
}
|
||||
|
||||
func (s *HealthStore) UpsertWindow(ctx context.Context, w *models.ProviderHealthWindow) error {
|
||||
_, err := s.db.ExecContext(ctx, `
|
||||
INSERT INTO provider_health (provider_config_id, window_start,
|
||||
request_count, error_count, timeout_count,
|
||||
total_latency_ms, max_latency_ms, last_error, last_error_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, now())
|
||||
ON CONFLICT (provider_config_id, window_start) DO UPDATE SET
|
||||
request_count = provider_health.request_count + EXCLUDED.request_count,
|
||||
error_count = provider_health.error_count + EXCLUDED.error_count,
|
||||
timeout_count = provider_health.timeout_count + EXCLUDED.timeout_count,
|
||||
total_latency_ms = provider_health.total_latency_ms + EXCLUDED.total_latency_ms,
|
||||
max_latency_ms = GREATEST(provider_health.max_latency_ms, EXCLUDED.max_latency_ms),
|
||||
last_error = COALESCE(EXCLUDED.last_error, provider_health.last_error),
|
||||
last_error_at = COALESCE(EXCLUDED.last_error_at, provider_health.last_error_at),
|
||||
updated_at = now()
|
||||
`, w.ProviderConfigID, w.WindowStart,
|
||||
w.RequestCount, w.ErrorCount, w.TimeoutCount,
|
||||
w.TotalLatencyMs, w.MaxLatencyMs, w.LastError, w.LastErrorAt,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *HealthStore) GetCurrentWindow(ctx context.Context, providerConfigID string) (*models.ProviderHealthWindow, error) {
|
||||
windowStart := time.Now().UTC().Truncate(time.Hour)
|
||||
var w models.ProviderHealthWindow
|
||||
err := s.db.QueryRowContext(ctx, `
|
||||
SELECT id, provider_config_id, window_start,
|
||||
request_count, error_count, timeout_count,
|
||||
total_latency_ms, max_latency_ms, last_error, last_error_at
|
||||
FROM provider_health
|
||||
WHERE provider_config_id = $1 AND window_start = $2
|
||||
`, providerConfigID, windowStart).Scan(
|
||||
&w.ID, &w.ProviderConfigID, &w.WindowStart,
|
||||
&w.RequestCount, &w.ErrorCount, &w.TimeoutCount,
|
||||
&w.TotalLatencyMs, &w.MaxLatencyMs, &w.LastError, &w.LastErrorAt,
|
||||
)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return &w, err
|
||||
}
|
||||
|
||||
func (s *HealthStore) ListWindows(ctx context.Context, providerConfigID string, hours int) ([]models.ProviderHealthWindow, error) {
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, provider_config_id, window_start,
|
||||
request_count, error_count, timeout_count,
|
||||
total_latency_ms, max_latency_ms, last_error, last_error_at
|
||||
FROM provider_health
|
||||
WHERE provider_config_id = $1
|
||||
ORDER BY window_start DESC
|
||||
LIMIT $2
|
||||
`, providerConfigID, hours)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []models.ProviderHealthWindow
|
||||
for rows.Next() {
|
||||
var w models.ProviderHealthWindow
|
||||
if err := rows.Scan(
|
||||
&w.ID, &w.ProviderConfigID, &w.WindowStart,
|
||||
&w.RequestCount, &w.ErrorCount, &w.TimeoutCount,
|
||||
&w.TotalLatencyMs, &w.MaxLatencyMs, &w.LastError, &w.LastErrorAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, w)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func (s *HealthStore) ListAllCurrent(ctx context.Context) ([]models.ProviderHealthWindow, error) {
|
||||
windowStart := time.Now().UTC().Truncate(time.Hour)
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
SELECT id, provider_config_id, window_start,
|
||||
request_count, error_count, timeout_count,
|
||||
total_latency_ms, max_latency_ms, last_error, last_error_at
|
||||
FROM provider_health
|
||||
WHERE window_start = $1
|
||||
ORDER BY provider_config_id
|
||||
`, windowStart)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []models.ProviderHealthWindow
|
||||
for rows.Next() {
|
||||
var w models.ProviderHealthWindow
|
||||
if err := rows.Scan(
|
||||
&w.ID, &w.ProviderConfigID, &w.WindowStart,
|
||||
&w.RequestCount, &w.ErrorCount, &w.TimeoutCount,
|
||||
&w.TotalLatencyMs, &w.MaxLatencyMs, &w.LastError, &w.LastErrorAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, w)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func (s *HealthStore) Prune(ctx context.Context, before time.Time) (int64, error) {
|
||||
result, err := s.db.ExecContext(ctx, `
|
||||
DELETE FROM provider_health WHERE window_start < $1
|
||||
`, before)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
@@ -37,5 +37,6 @@ func NewStores(db *sql.DB) store.Stores {
|
||||
NotifPrefs: NewNotificationPreferenceStore(),
|
||||
Workspaces: NewWorkspaceStore(),
|
||||
GitCredentials: &GitCredentialStore{},
|
||||
CapOverrides: NewCapOverrideStore(db),
|
||||
}
|
||||
}
|
||||
|
||||
90
server/store/sqlite/cap_override.go
Normal file
90
server/store/sqlite/cap_override.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
type CapOverrideStore struct{}
|
||||
|
||||
func NewCapOverrideStore() *CapOverrideStore { return &CapOverrideStore{} }
|
||||
|
||||
func (s *CapOverrideStore) Set(ctx context.Context, o *models.CapabilityOverride) error {
|
||||
id := store.NewID()
|
||||
_, err := DB.ExecContext(ctx, `
|
||||
INSERT INTO capability_overrides (id, provider_config_id, model_id, field, value, set_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (provider_config_id, model_id, field) DO UPDATE SET
|
||||
value = excluded.value,
|
||||
set_by = excluded.set_by,
|
||||
created_at = datetime('now')
|
||||
`, id, o.ProviderConfigID, o.ModelID, o.Field, o.Value, o.SetBy)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *CapOverrideStore) Delete(ctx context.Context, id string) error {
|
||||
_, err := DB.ExecContext(ctx, `DELETE FROM capability_overrides WHERE id = ?`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *CapOverrideStore) ListForModel(ctx context.Context, modelID string) ([]models.CapabilityOverride, error) {
|
||||
return s.query(ctx, `
|
||||
SELECT id, provider_config_id, model_id, field, value, set_by, created_at
|
||||
FROM capability_overrides
|
||||
WHERE model_id = ?
|
||||
ORDER BY provider_config_id, field
|
||||
`, modelID)
|
||||
}
|
||||
|
||||
func (s *CapOverrideStore) ListForProviderModel(ctx context.Context, providerConfigID, modelID string) ([]models.CapabilityOverride, error) {
|
||||
return s.query(ctx, `
|
||||
SELECT id, provider_config_id, model_id, field, value, set_by, created_at
|
||||
FROM capability_overrides
|
||||
WHERE model_id = ? AND (provider_config_id = ? OR provider_config_id IS NULL)
|
||||
ORDER BY provider_config_id, field
|
||||
`, modelID, providerConfigID)
|
||||
}
|
||||
|
||||
func (s *CapOverrideStore) ListAll(ctx context.Context) ([]models.CapabilityOverride, error) {
|
||||
return s.query(ctx, `
|
||||
SELECT id, provider_config_id, model_id, field, value, set_by, created_at
|
||||
FROM capability_overrides
|
||||
ORDER BY model_id, provider_config_id, field
|
||||
`)
|
||||
}
|
||||
|
||||
func (s *CapOverrideStore) DeleteForProvider(ctx context.Context, providerConfigID string) error {
|
||||
_, err := DB.ExecContext(ctx, `
|
||||
DELETE FROM capability_overrides WHERE provider_config_id = ?
|
||||
`, providerConfigID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *CapOverrideStore) query(ctx context.Context, q string, args ...interface{}) ([]models.CapabilityOverride, error) {
|
||||
rows, err := DB.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []models.CapabilityOverride
|
||||
for rows.Next() {
|
||||
var o models.CapabilityOverride
|
||||
if err := rows.Scan(&o.ID, &o.ProviderConfigID, &o.ModelID, &o.Field, &o.Value, &o.SetBy, &o.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, o)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
// ensure interface compliance (compile-time check)
|
||||
var _ store.CapabilityOverrideStore = (*CapOverrideStore)(nil)
|
||||
|
||||
// also check the health store interface at this package level
|
||||
// (health.Store is in the health package, but we verify via the sql.Rows pattern)
|
||||
func init() {
|
||||
// compile-time interface checks happen via the var _ lines above
|
||||
}
|
||||
125
server/store/sqlite/health.go
Normal file
125
server/store/sqlite/health.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
type HealthStore struct{}
|
||||
|
||||
func NewHealthStore() *HealthStore { return &HealthStore{} }
|
||||
|
||||
func (s *HealthStore) UpsertWindow(ctx context.Context, w *models.ProviderHealthWindow) error {
|
||||
id := store.NewID()
|
||||
windowStr := w.WindowStart.Format(timeFmt)
|
||||
_, err := DB.ExecContext(ctx, `
|
||||
INSERT INTO provider_health (id, provider_config_id, window_start,
|
||||
request_count, error_count, timeout_count,
|
||||
total_latency_ms, max_latency_ms, last_error, last_error_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))
|
||||
ON CONFLICT (provider_config_id, window_start) DO UPDATE SET
|
||||
request_count = provider_health.request_count + excluded.request_count,
|
||||
error_count = provider_health.error_count + excluded.error_count,
|
||||
timeout_count = provider_health.timeout_count + excluded.timeout_count,
|
||||
total_latency_ms = provider_health.total_latency_ms + excluded.total_latency_ms,
|
||||
max_latency_ms = MAX(provider_health.max_latency_ms, excluded.max_latency_ms),
|
||||
last_error = COALESCE(excluded.last_error, provider_health.last_error),
|
||||
last_error_at = COALESCE(excluded.last_error_at, provider_health.last_error_at),
|
||||
updated_at = datetime('now')
|
||||
`, id, w.ProviderConfigID, windowStr,
|
||||
w.RequestCount, w.ErrorCount, w.TimeoutCount,
|
||||
w.TotalLatencyMs, w.MaxLatencyMs, w.LastError, w.LastErrorAt,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *HealthStore) GetCurrentWindow(ctx context.Context, providerConfigID string) (*models.ProviderHealthWindow, error) {
|
||||
windowStr := time.Now().UTC().Truncate(time.Hour).Format(timeFmt)
|
||||
var w models.ProviderHealthWindow
|
||||
var windowStartStr string
|
||||
err := DB.QueryRowContext(ctx, `
|
||||
SELECT id, provider_config_id, window_start,
|
||||
request_count, error_count, timeout_count,
|
||||
total_latency_ms, max_latency_ms, last_error, last_error_at
|
||||
FROM provider_health
|
||||
WHERE provider_config_id = ? AND window_start = ?
|
||||
`, providerConfigID, windowStr).Scan(
|
||||
&w.ID, &w.ProviderConfigID, &windowStartStr,
|
||||
&w.RequestCount, &w.ErrorCount, &w.TimeoutCount,
|
||||
&w.TotalLatencyMs, &w.MaxLatencyMs, &w.LastError, &w.LastErrorAt,
|
||||
)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w.WindowStart, _ = time.Parse(timeFmt, windowStartStr)
|
||||
return &w, nil
|
||||
}
|
||||
|
||||
func (s *HealthStore) ListWindows(ctx context.Context, providerConfigID string, hours int) ([]models.ProviderHealthWindow, error) {
|
||||
rows, err := DB.QueryContext(ctx, `
|
||||
SELECT id, provider_config_id, window_start,
|
||||
request_count, error_count, timeout_count,
|
||||
total_latency_ms, max_latency_ms, last_error, last_error_at
|
||||
FROM provider_health
|
||||
WHERE provider_config_id = ?
|
||||
ORDER BY window_start DESC
|
||||
LIMIT ?
|
||||
`, providerConfigID, hours)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanHealthRows(rows)
|
||||
}
|
||||
|
||||
func (s *HealthStore) ListAllCurrent(ctx context.Context) ([]models.ProviderHealthWindow, error) {
|
||||
windowStr := time.Now().UTC().Truncate(time.Hour).Format(timeFmt)
|
||||
rows, err := DB.QueryContext(ctx, `
|
||||
SELECT id, provider_config_id, window_start,
|
||||
request_count, error_count, timeout_count,
|
||||
total_latency_ms, max_latency_ms, last_error, last_error_at
|
||||
FROM provider_health
|
||||
WHERE window_start = ?
|
||||
ORDER BY provider_config_id
|
||||
`, windowStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanHealthRows(rows)
|
||||
}
|
||||
|
||||
func (s *HealthStore) Prune(ctx context.Context, before time.Time) (int64, error) {
|
||||
result, err := DB.ExecContext(ctx, `
|
||||
DELETE FROM provider_health WHERE window_start < ?
|
||||
`, before.Format(timeFmt))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
func scanHealthRows(rows *sql.Rows) ([]models.ProviderHealthWindow, error) {
|
||||
var result []models.ProviderHealthWindow
|
||||
for rows.Next() {
|
||||
var w models.ProviderHealthWindow
|
||||
var windowStartStr string
|
||||
if err := rows.Scan(
|
||||
&w.ID, &w.ProviderConfigID, &windowStartStr,
|
||||
&w.RequestCount, &w.ErrorCount, &w.TimeoutCount,
|
||||
&w.TotalLatencyMs, &w.MaxLatencyMs, &w.LastError, &w.LastErrorAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
w.WindowStart, _ = time.Parse(timeFmt, windowStartStr)
|
||||
result = append(result, w)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -37,5 +37,6 @@ func NewStores(db *sql.DB) store.Stores {
|
||||
NotifPrefs: NewNotificationPreferenceStore(),
|
||||
Workspaces: NewWorkspaceStore(),
|
||||
GitCredentials: &GitCredentialStore{},
|
||||
CapOverrides: NewCapOverrideStore(),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user