Changeset 0.22.0.1 (#94)
This commit is contained in:
219
server/health/accumulator.go
Normal file
219
server/health/accumulator.go
Normal file
@@ -0,0 +1,219 @@
|
||||
// Package health provides in-memory accumulation of provider health
|
||||
// metrics with periodic flush to the database. The accumulator is
|
||||
// goroutine-safe and designed for high-throughput recording from
|
||||
// concurrent completion handlers.
|
||||
package health
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
// ── Thresholds ──────────────────────────────
|
||||
|
||||
const (
|
||||
// DegradedThreshold — error rate above this = degraded.
|
||||
DegradedThreshold = 0.05 // 5%
|
||||
// DownThreshold — error rate above this = down.
|
||||
DownThreshold = 0.25 // 25%
|
||||
// FlushInterval — how often in-memory counters are flushed to DB.
|
||||
FlushInterval = 60 * time.Second
|
||||
// PruneAge — health windows older than this are deleted.
|
||||
PruneAge = 7 * 24 * time.Hour
|
||||
)
|
||||
|
||||
// ── Store Interface ─────────────────────────
|
||||
|
||||
// Store is the persistence layer for health data.
|
||||
type Store interface {
|
||||
// UpsertWindow merges in-memory counters into the hourly bucket row.
|
||||
UpsertWindow(ctx context.Context, w *models.ProviderHealthWindow) error
|
||||
|
||||
// GetCurrentWindow returns the current hourly bucket for a provider.
|
||||
GetCurrentWindow(ctx context.Context, providerConfigID string) (*models.ProviderHealthWindow, error)
|
||||
|
||||
// ListWindows returns the last N hourly buckets for a provider, newest first.
|
||||
ListWindows(ctx context.Context, providerConfigID string, hours int) ([]models.ProviderHealthWindow, error)
|
||||
|
||||
// ListAllCurrent returns the current-hour bucket for every provider.
|
||||
ListAllCurrent(ctx context.Context) ([]models.ProviderHealthWindow, error)
|
||||
|
||||
// Prune deletes rows older than the given time.
|
||||
Prune(ctx context.Context, before time.Time) (int64, error)
|
||||
}
|
||||
|
||||
// ── In-Memory Bucket ────────────────────────
|
||||
|
||||
// bucket holds counters for one provider within one flush interval.
|
||||
type bucket struct {
|
||||
providerConfigID string
|
||||
requestCount int
|
||||
errorCount int
|
||||
timeoutCount int
|
||||
totalLatencyMs int64
|
||||
maxLatencyMs int
|
||||
lastError string
|
||||
lastErrorAt time.Time
|
||||
}
|
||||
|
||||
// ── Accumulator ─────────────────────────────
|
||||
|
||||
// Accumulator collects health metrics in memory and periodically
|
||||
// flushes them to the database. One instance per process.
|
||||
type Accumulator struct {
|
||||
mu sync.Mutex
|
||||
buckets map[string]*bucket // keyed by provider_config_id
|
||||
store Store
|
||||
stopCh chan struct{}
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewAccumulator creates and starts the background flush loop.
|
||||
func NewAccumulator(store Store) *Accumulator {
|
||||
a := &Accumulator{
|
||||
buckets: make(map[string]*bucket),
|
||||
store: store,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
a.wg.Add(1)
|
||||
go a.flushLoop()
|
||||
return a
|
||||
}
|
||||
|
||||
// RecordSuccess records a successful provider call.
|
||||
func (a *Accumulator) RecordSuccess(providerConfigID string, latencyMs int) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
b := a.getBucket(providerConfigID)
|
||||
b.requestCount++
|
||||
b.totalLatencyMs += int64(latencyMs)
|
||||
if latencyMs > b.maxLatencyMs {
|
||||
b.maxLatencyMs = latencyMs
|
||||
}
|
||||
}
|
||||
|
||||
// RecordError records a failed provider call.
|
||||
func (a *Accumulator) RecordError(providerConfigID string, latencyMs int, errMsg string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
b := a.getBucket(providerConfigID)
|
||||
b.requestCount++
|
||||
b.errorCount++
|
||||
b.totalLatencyMs += int64(latencyMs)
|
||||
if latencyMs > b.maxLatencyMs {
|
||||
b.maxLatencyMs = latencyMs
|
||||
}
|
||||
b.lastError = errMsg
|
||||
b.lastErrorAt = time.Now().UTC()
|
||||
}
|
||||
|
||||
// RecordTimeout records a timed-out provider call (counted as both error and timeout).
|
||||
func (a *Accumulator) RecordTimeout(providerConfigID string, latencyMs int, errMsg string) {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
b := a.getBucket(providerConfigID)
|
||||
b.requestCount++
|
||||
b.errorCount++
|
||||
b.timeoutCount++
|
||||
b.totalLatencyMs += int64(latencyMs)
|
||||
if latencyMs > b.maxLatencyMs {
|
||||
b.maxLatencyMs = latencyMs
|
||||
}
|
||||
b.lastError = errMsg
|
||||
b.lastErrorAt = time.Now().UTC()
|
||||
}
|
||||
|
||||
// Stop halts the flush loop and performs a final flush.
|
||||
func (a *Accumulator) Stop() {
|
||||
close(a.stopCh)
|
||||
a.wg.Wait()
|
||||
}
|
||||
|
||||
// DeriveStatus computes the provider status from an error rate.
|
||||
func DeriveStatus(errorRate float64, requestCount int) models.ProviderStatus {
|
||||
if requestCount == 0 {
|
||||
return models.StatusUnknown
|
||||
}
|
||||
if errorRate >= DownThreshold {
|
||||
return models.StatusDown
|
||||
}
|
||||
if errorRate >= DegradedThreshold {
|
||||
return models.StatusDegraded
|
||||
}
|
||||
return models.StatusHealthy
|
||||
}
|
||||
|
||||
// ── Internal ────────────────────────────────
|
||||
|
||||
func (a *Accumulator) getBucket(providerConfigID string) *bucket {
|
||||
b, ok := a.buckets[providerConfigID]
|
||||
if !ok {
|
||||
b = &bucket{providerConfigID: providerConfigID}
|
||||
a.buckets[providerConfigID] = b
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (a *Accumulator) flushLoop() {
|
||||
defer a.wg.Done()
|
||||
ticker := time.NewTicker(FlushInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
a.flush()
|
||||
case <-a.stopCh:
|
||||
a.flush() // final flush
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *Accumulator) flush() {
|
||||
a.mu.Lock()
|
||||
if len(a.buckets) == 0 {
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
// Swap out current buckets so we don't hold the lock during DB writes.
|
||||
snap := a.buckets
|
||||
a.buckets = make(map[string]*bucket, len(snap))
|
||||
a.mu.Unlock()
|
||||
|
||||
windowStart := hourFloor(time.Now().UTC())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
for _, b := range snap {
|
||||
if b.requestCount == 0 {
|
||||
continue
|
||||
}
|
||||
w := &models.ProviderHealthWindow{
|
||||
ProviderConfigID: b.providerConfigID,
|
||||
WindowStart: windowStart,
|
||||
RequestCount: b.requestCount,
|
||||
ErrorCount: b.errorCount,
|
||||
TimeoutCount: b.timeoutCount,
|
||||
TotalLatencyMs: b.totalLatencyMs,
|
||||
MaxLatencyMs: b.maxLatencyMs,
|
||||
}
|
||||
if b.lastError != "" {
|
||||
w.LastError = &b.lastError
|
||||
ts := b.lastErrorAt.Format(time.RFC3339)
|
||||
w.LastErrorAt = &ts
|
||||
}
|
||||
if err := a.store.UpsertWindow(ctx, w); err != nil {
|
||||
log.Printf("⚠ health: flush failed for provider %s: %v", b.providerConfigID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hourFloor truncates a time to the start of its hour.
|
||||
func hourFloor(t time.Time) time.Time {
|
||||
return t.Truncate(time.Hour)
|
||||
}
|
||||
239
server/health/accumulator_test.go
Normal file
239
server/health/accumulator_test.go
Normal file
@@ -0,0 +1,239 @@
|
||||
package health
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
// ── Mock Store ──────────────────────────────
|
||||
|
||||
type mockStore struct {
|
||||
mu sync.Mutex
|
||||
windows map[string]*models.ProviderHealthWindow // keyed by provider_config_id
|
||||
}
|
||||
|
||||
func newMockStore() *mockStore {
|
||||
return &mockStore{windows: make(map[string]*models.ProviderHealthWindow)}
|
||||
}
|
||||
|
||||
func (m *mockStore) UpsertWindow(_ context.Context, w *models.ProviderHealthWindow) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
existing, ok := m.windows[w.ProviderConfigID]
|
||||
if ok {
|
||||
existing.RequestCount += w.RequestCount
|
||||
existing.ErrorCount += w.ErrorCount
|
||||
existing.TimeoutCount += w.TimeoutCount
|
||||
existing.TotalLatencyMs += w.TotalLatencyMs
|
||||
if w.MaxLatencyMs > existing.MaxLatencyMs {
|
||||
existing.MaxLatencyMs = w.MaxLatencyMs
|
||||
}
|
||||
if w.LastError != nil {
|
||||
existing.LastError = w.LastError
|
||||
existing.LastErrorAt = w.LastErrorAt
|
||||
}
|
||||
} else {
|
||||
cp := *w
|
||||
m.windows[w.ProviderConfigID] = &cp
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockStore) GetCurrentWindow(_ context.Context, id string) (*models.ProviderHealthWindow, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
w, ok := m.windows[id]
|
||||
if !ok {
|
||||
return nil, nil
|
||||
}
|
||||
cp := *w
|
||||
return &cp, nil
|
||||
}
|
||||
|
||||
func (m *mockStore) ListWindows(_ context.Context, _ string, _ int) ([]models.ProviderHealthWindow, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockStore) ListAllCurrent(_ context.Context) ([]models.ProviderHealthWindow, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
var result []models.ProviderHealthWindow
|
||||
for _, w := range m.windows {
|
||||
result = append(result, *w)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (m *mockStore) Prune(_ context.Context, _ time.Time) (int64, error) { return 0, nil }
|
||||
|
||||
// ── Tests ───────────────────────────────────
|
||||
|
||||
func TestRecordSuccess(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
a := &Accumulator{
|
||||
buckets: make(map[string]*bucket),
|
||||
store: ms,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
a.RecordSuccess("prov-1", 100)
|
||||
a.RecordSuccess("prov-1", 200)
|
||||
a.RecordSuccess("prov-2", 50)
|
||||
|
||||
a.flush()
|
||||
|
||||
w1, _ := ms.GetCurrentWindow(context.Background(), "prov-1")
|
||||
if w1 == nil {
|
||||
t.Fatal("expected prov-1 window")
|
||||
}
|
||||
if w1.RequestCount != 2 {
|
||||
t.Fatalf("expected 2 requests, got %d", w1.RequestCount)
|
||||
}
|
||||
if w1.ErrorCount != 0 {
|
||||
t.Fatalf("expected 0 errors, got %d", w1.ErrorCount)
|
||||
}
|
||||
if w1.TotalLatencyMs != 300 {
|
||||
t.Fatalf("expected 300ms total latency, got %d", w1.TotalLatencyMs)
|
||||
}
|
||||
if w1.MaxLatencyMs != 200 {
|
||||
t.Fatalf("expected 200ms max latency, got %d", w1.MaxLatencyMs)
|
||||
}
|
||||
|
||||
w2, _ := ms.GetCurrentWindow(context.Background(), "prov-2")
|
||||
if w2 == nil {
|
||||
t.Fatal("expected prov-2 window")
|
||||
}
|
||||
if w2.RequestCount != 1 {
|
||||
t.Fatalf("expected 1 request, got %d", w2.RequestCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordError(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
a := &Accumulator{
|
||||
buckets: make(map[string]*bucket),
|
||||
store: ms,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
a.RecordSuccess("prov-1", 100)
|
||||
a.RecordError("prov-1", 500, "connection refused")
|
||||
a.RecordSuccess("prov-1", 150)
|
||||
|
||||
a.flush()
|
||||
|
||||
w, _ := ms.GetCurrentWindow(context.Background(), "prov-1")
|
||||
if w.RequestCount != 3 {
|
||||
t.Fatalf("expected 3 requests, got %d", w.RequestCount)
|
||||
}
|
||||
if w.ErrorCount != 1 {
|
||||
t.Fatalf("expected 1 error, got %d", w.ErrorCount)
|
||||
}
|
||||
if w.LastError == nil || *w.LastError != "connection refused" {
|
||||
t.Fatalf("expected last_error='connection refused', got %v", w.LastError)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordTimeout(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
a := &Accumulator{
|
||||
buckets: make(map[string]*bucket),
|
||||
store: ms,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
a.RecordTimeout("prov-1", 30000, "context deadline exceeded")
|
||||
a.flush()
|
||||
|
||||
w, _ := ms.GetCurrentWindow(context.Background(), "prov-1")
|
||||
if w.RequestCount != 1 {
|
||||
t.Fatalf("expected 1 request, got %d", w.RequestCount)
|
||||
}
|
||||
if w.ErrorCount != 1 {
|
||||
t.Fatalf("expected 1 error, got %d", w.ErrorCount)
|
||||
}
|
||||
if w.TimeoutCount != 1 {
|
||||
t.Fatalf("expected 1 timeout, got %d", w.TimeoutCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveStatus(t *testing.T) {
|
||||
tests := []struct {
|
||||
rate float64
|
||||
count int
|
||||
expected models.ProviderStatus
|
||||
}{
|
||||
{0, 0, models.StatusUnknown},
|
||||
{0, 100, models.StatusHealthy},
|
||||
{0.03, 100, models.StatusHealthy},
|
||||
{0.05, 100, models.StatusDegraded},
|
||||
{0.15, 100, models.StatusDegraded},
|
||||
{0.25, 100, models.StatusDown},
|
||||
{0.50, 100, models.StatusDown},
|
||||
{1.0, 1, models.StatusDown},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := DeriveStatus(tt.rate, tt.count)
|
||||
if got != tt.expected {
|
||||
t.Errorf("DeriveStatus(%v, %d) = %s, want %s", tt.rate, tt.count, got, tt.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushClearsBuckets(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
a := &Accumulator{
|
||||
buckets: make(map[string]*bucket),
|
||||
store: ms,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
a.RecordSuccess("prov-1", 100)
|
||||
a.flush()
|
||||
|
||||
// After flush, buckets should be empty
|
||||
a.mu.Lock()
|
||||
count := len(a.buckets)
|
||||
a.mu.Unlock()
|
||||
if count != 0 {
|
||||
t.Fatalf("expected 0 buckets after flush, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentRecording(t *testing.T) {
|
||||
ms := newMockStore()
|
||||
a := &Accumulator{
|
||||
buckets: make(map[string]*bucket),
|
||||
store: ms,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 100; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
a.RecordSuccess("prov-1", 50)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
a.flush()
|
||||
|
||||
w, _ := ms.GetCurrentWindow(context.Background(), "prov-1")
|
||||
if w.RequestCount != 100 {
|
||||
t.Fatalf("expected 100 concurrent requests, got %d", w.RequestCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHourFloor(t *testing.T) {
|
||||
ts := time.Date(2026, 3, 1, 14, 37, 42, 0, time.UTC)
|
||||
floor := hourFloor(ts)
|
||||
expected := time.Date(2026, 3, 1, 14, 0, 0, 0, time.UTC)
|
||||
if !floor.Equal(expected) {
|
||||
t.Fatalf("expected %v, got %v", expected, floor)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user