Changeset 0.17.0 (#75)

server/roles/roles.go

@@ -5,8 +5,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"sync"
+	"time"
 
 	"git.gobha.me/xcaliber/chat-switchboard/crypto"
+	"git.gobha.me/xcaliber/chat-switchboard/events"
+	"git.gobha.me/xcaliber/chat-switchboard/models"
 	"git.gobha.me/xcaliber/chat-switchboard/providers"
 	"git.gobha.me/xcaliber/chat-switchboard/store"
 )
@@ -89,11 +93,24 @@ var (
 type Resolver struct {
 	stores store.Stores
 	vault  *crypto.KeyResolver
+	bus    *events.Bus // optional — nil-safe
+
+	// Fallback cooldown: suppress duplicate alerts per role
+	coolMu   sync.Mutex
+	cooldown map[string]time.Time // role → last alert timestamp
 }
 
+const fallbackCooldown = 5 * time.Minute
+
 // NewResolver creates a role resolver with access to stores and vault.
 func NewResolver(s store.Stores, vault *crypto.KeyResolver) *Resolver {
-	return &Resolver{stores: s, vault: vault}
+	return &Resolver{stores: s, vault: vault, cooldown: make(map[string]time.Time)}
+}
+
+// WithBus attaches an event bus for fallback alert publishing.
+func (r *Resolver) WithBus(bus *events.Bus) *Resolver {
+	r.bus = bus
+	return r
 }
 
 // Complete sends a chat completion using the named role.
@@ -118,6 +135,7 @@ func (r *Resolver) Complete(ctx context.Context, role string, userID string, tea
 		result, err := r.doComplete(ctx, role, cfg.Fallback, messages)
 		if err == nil {
 			result.UsedFallback = true
+			r.onFallback(ctx, role, "completion", cfg.Primary, cfg.Fallback)
 			return result, nil
 		}
 		return nil, fmt.Errorf("role %q: both primary and fallback failed: %w", role, err)
@@ -150,6 +168,7 @@ func (r *Resolver) Embed(ctx context.Context, role string, userID string, teamID
 		result, err := r.doEmbed(ctx, role, cfg.Fallback, input)
 		if err == nil {
 			result.UsedFallback = true
+			r.onFallback(ctx, role, "embedding", cfg.Primary, cfg.Fallback)
 			return result, nil
 		}
 		return nil, fmt.Errorf("role %q: both primary and fallback embed failed: %w", role, err)
@@ -387,3 +406,66 @@ func parseRoleConfig(data interface{}) (*RoleConfig, error) {
 	}
 	return &cfg, nil
 }
+
+// ── Fallback Alerting ──────────────────────
+
+// onFallback fires on successful fallback activation. It emits:
+//   - log line (always)
+//   - audit_log entry (always)
+//   - event bus "role.fallback" (once per cooldown window)
+//
+// The cooldown prevents flooding the admin UI with identical alerts
+// when a primary is down and every request triggers the fallback.
+func (r *Resolver) onFallback(ctx context.Context, role, opType string, primary, fallback *RoleBinding) {
+	primaryModel := ""
+	fallbackModel := ""
+	if primary != nil {
+		primaryModel = primary.ModelID
+	}
+	if fallback != nil {
+		fallbackModel = fallback.ModelID
+	}
+
+	log.Printf("⚠ Role %q fallback activated: %s → %s (%s)", role, primaryModel, fallbackModel, opType)
+
+	// Audit log — always written (one row per fallback fire)
+	_ = r.stores.Audit.Log(ctx, &models.AuditEntry{
+		Action:       "role.fallback",
+		ResourceType: "role",
+		ResourceID:   role,
+		Metadata: models.JSONMap{
+			"operation":      opType,
+			"primary_model":  primaryModel,
+			"fallback_model": fallbackModel,
+		},
+	})
+
+	// Bus event — cooldown-gated
+	if r.bus == nil {
+		return
+	}
+
+	r.coolMu.Lock()
+	last, exists := r.cooldown[role]
+	now := time.Now()
+	if exists && now.Sub(last) < fallbackCooldown {
+		r.coolMu.Unlock()
+		return
+	}
+	r.cooldown[role] = now
+	r.coolMu.Unlock()
+
+	payload, _ := json.Marshal(map[string]string{
+		"role":           role,
+		"operation":      opType,
+		"primary_model":  primaryModel,
+		"fallback_model": fallbackModel,
+		"message":        fmt.Sprintf("Role %q primary (%s) failed — using fallback (%s)", role, primaryModel, fallbackModel),
+	})
+	r.bus.PublishAsync(events.Event{
+		Label:   "role.fallback",
+		Room:    "admin", // admin-targeted
+		Payload: payload,
+		Ts:      now.UnixMilli(),
+	})
+}