Feat v0.6.0 cluster registry (#36)
All checks were successful
CI/CD / detect-changes (push) Successful in 4s
CI/CD / test-frontend (push) Successful in 6s
CI/CD / test-go-pg (push) Successful in 2m40s
CI/CD / test-sqlite (push) Successful in 3m1s
CI/CD / build-and-deploy (push) Successful in 1m22s

Co-authored-by: Jeffrey Smith <jasafpro@gmail.com>
Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>
This commit was merged in pull request #36.
This commit is contained in:
2026-03-30 22:57:11 +00:00
committed by xcaliber
parent 768f15b3cd
commit 1a7f41493d
21 changed files with 1101 additions and 19 deletions

143
server/cluster/registry.go Normal file
View File

@@ -0,0 +1,143 @@
package cluster
import (
"context"
"encoding/json"
"log"
"os"
"runtime"
"sync"
"time"
"switchboard-core/store"
)
// ConnCounter provides WebSocket connection count (satisfied by events.Hub).
type ConnCounter interface {
ConnCount() int
}
// RegistryConfig holds tunable cluster registry parameters.
type RegistryConfig struct {
HeartbeatInterval time.Duration
StaleThreshold time.Duration
}
// Registry manages node self-registration, heartbeat, and stale sweep.
// Follows the workflow.Scanner pattern: stopCh + wg + ticker.
type Registry struct {
nodeID string
endpoint string
store store.ClusterStore
hub ConnCounter
cfg RegistryConfig
startTime time.Time
stopCh chan struct{}
wg sync.WaitGroup
}
// NewRegistry creates a cluster registry instance.
func NewRegistry(nodeID, endpoint string, s store.ClusterStore, hub ConnCounter, cfg RegistryConfig) *Registry {
return &Registry{
nodeID: nodeID,
endpoint: endpoint,
store: s,
hub: hub,
cfg: cfg,
startTime: time.Now(),
stopCh: make(chan struct{}),
}
}
// NodeID returns this node's identifier.
func (r *Registry) NodeID() string { return r.nodeID }
// Start registers the node and launches the heartbeat goroutine.
func (r *Registry) Start() error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := r.store.Register(ctx, r.nodeID, r.endpoint); err != nil {
return err
}
r.wg.Add(1)
go func() {
defer r.wg.Done()
ticker := time.NewTicker(r.cfg.HeartbeatInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
r.tick()
case <-r.stopCh:
return
}
}
}()
log.Printf("[cluster] started (node=%s, heartbeat=%s, stale=%s)",
r.nodeID, r.cfg.HeartbeatInterval, r.cfg.StaleThreshold)
return nil
}
// Stop deregisters the node and waits for the heartbeat goroutine to exit.
func (r *Registry) Stop() {
close(r.stopCh)
r.wg.Wait()
// Best-effort deregister so peers don't have to wait for stale sweep.
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
if err := r.store.Deregister(ctx, r.nodeID); err != nil {
log.Printf("[cluster] deregister failed: %v", err)
}
log.Printf("[cluster] stopped (node=%s)", r.nodeID)
}
// tick runs one heartbeat + sweep cycle.
func (r *Registry) tick() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
stats := r.collectStats()
rows, err := r.store.Heartbeat(ctx, r.nodeID, stats)
if err != nil {
log.Printf("[cluster] heartbeat error: %v", err)
return
}
if rows == 0 {
// Self-eviction: this node was swept by a peer.
// Kubernetes will restart the process → re-register.
log.Printf("[cluster] FATAL: node %s evicted from registry — shutting down", r.nodeID)
os.Exit(1)
}
swept, err := r.store.SweepStale(ctx, r.cfg.StaleThreshold)
if err != nil {
log.Printf("[cluster] sweep error: %v", err)
return
}
if swept > 0 {
log.Printf("[cluster] swept %d stale node(s)", swept)
}
}
// collectStats gathers runtime metrics for the stats JSONB column.
func (r *Registry) collectStats() json.RawMessage {
var m runtime.MemStats
runtime.ReadMemStats(&m)
stats := map[string]any{
"goroutines": runtime.NumGoroutine(),
"heap_alloc": m.HeapAlloc,
"heap_sys": m.HeapSys,
"gc_cycles": m.NumGC,
"gc_pause_ns": m.PauseNs[(m.NumGC+255)%256],
"uptime_sec": time.Since(r.startTime).Seconds(),
"ws_clients": r.hub.ConnCount(),
}
data, _ := json.Marshal(stats)
return data
}

View File

@@ -0,0 +1,112 @@
package cluster
import (
"context"
"encoding/json"
"testing"
"time"
"switchboard-core/store"
)
// mockClusterStore implements store.ClusterStore for unit tests.
type mockClusterStore struct {
nodes []store.ClusterNode
heartbeatRows int64
heartbeatErr error
sweepCalled bool
registerCalled bool
lastStats json.RawMessage
}
func (m *mockClusterStore) Register(_ context.Context, nodeID, endpoint string) error {
m.registerCalled = true
return nil
}
func (m *mockClusterStore) Heartbeat(_ context.Context, nodeID string, stats json.RawMessage) (int64, error) {
m.lastStats = stats
return m.heartbeatRows, m.heartbeatErr
}
func (m *mockClusterStore) SweepStale(_ context.Context, _ time.Duration) (int64, error) {
m.sweepCalled = true
return 0, nil
}
func (m *mockClusterStore) ListNodes(_ context.Context) ([]store.ClusterNode, error) {
return m.nodes, nil
}
func (m *mockClusterStore) Deregister(_ context.Context, _ string) error {
return nil
}
// mockHub implements ConnCounter.
type mockHub struct{ count int }
func (h *mockHub) ConnCount() int { return h.count }
func TestCollectStats(t *testing.T) {
hub := &mockHub{count: 5}
reg := NewRegistry("test-node", "http://localhost:8080", nil, hub, RegistryConfig{
HeartbeatInterval: 10 * time.Second,
StaleThreshold: 30 * time.Second,
})
data := reg.collectStats()
var stats map[string]any
if err := json.Unmarshal(data, &stats); err != nil {
t.Fatalf("collectStats returned invalid JSON: %v", err)
}
// Verify expected keys
expectedKeys := []string{"goroutines", "heap_alloc", "heap_sys", "gc_cycles", "gc_pause_ns", "uptime_sec", "ws_clients"}
for _, key := range expectedKeys {
if _, ok := stats[key]; !ok {
t.Errorf("missing expected stats key: %s", key)
}
}
// ws_clients should reflect hub count
if ws, ok := stats["ws_clients"].(float64); !ok || int(ws) != 5 {
t.Errorf("ws_clients = %v, want 5", stats["ws_clients"])
}
}
func TestRegistryStartStop(t *testing.T) {
ms := &mockClusterStore{heartbeatRows: 1}
hub := &mockHub{count: 0}
reg := NewRegistry("test-node", "", ms, hub, RegistryConfig{
HeartbeatInterval: 50 * time.Millisecond,
StaleThreshold: 150 * time.Millisecond,
})
if err := reg.Start(); err != nil {
t.Fatalf("Start() error: %v", err)
}
if !ms.registerCalled {
t.Error("Register was not called on Start")
}
// Let at least one tick fire
time.Sleep(100 * time.Millisecond)
reg.Stop()
if !ms.sweepCalled {
t.Error("SweepStale was never called during tick")
}
if ms.lastStats == nil {
t.Error("Heartbeat was never called with stats")
}
}
func TestNodeID(t *testing.T) {
reg := NewRegistry("my-node-123", "", nil, &mockHub{}, RegistryConfig{})
if got := reg.NodeID(); got != "my-node-123" {
t.Errorf("NodeID() = %q, want %q", got, "my-node-123")
}
}

View File

@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"strconv"
"time"
"github.com/joho/godotenv"
)
@@ -85,6 +86,17 @@ type Config struct {
OIDCRolesClaim string // JWT claim for role mapping (default "realm_access.roles")
OIDCGroupsClaim string // JWT claim for group sync (default "groups")
OIDCAdminRole string // IdP role value that maps to admin (default "admin")
// Cluster registry (v0.6.0)
// PG-backed node registry for horizontal scaling. No-op on SQLite.
// CLUSTER_NODE_ID: override for deterministic identity (default: hostname-PID).
// CLUSTER_HEARTBEAT_INTERVAL: tick frequency (default "10s").
// CLUSTER_STALE_THRESHOLD: 3x heartbeat = stale (default "30s").
// CLUSTER_ENDPOINT: advertised address for peer mesh (Phase 2, auto-detect).
ClusterNodeID string
ClusterHeartbeatInterval time.Duration
ClusterStaleThreshold time.Duration
ClusterEndpoint string
}
// Load reads configuration from environment variables.
@@ -143,6 +155,12 @@ func Load() *Config {
OIDCRolesClaim: getEnv("OIDC_ROLES_CLAIM", "realm_access.roles"),
OIDCGroupsClaim: getEnv("OIDC_GROUPS_CLAIM", "groups"),
OIDCAdminRole: getEnv("OIDC_ADMIN_ROLE", "admin"),
// Cluster
ClusterNodeID: getEnv("CLUSTER_NODE_ID", ""),
ClusterHeartbeatInterval: getEnvDuration("CLUSTER_HEARTBEAT_INTERVAL", 10*time.Second),
ClusterStaleThreshold: getEnvDuration("CLUSTER_STALE_THRESHOLD", 30*time.Second),
ClusterEndpoint: getEnv("CLUSTER_ENDPOINT", ""),
}
}
@@ -204,6 +222,15 @@ func getEnvInt(key string, fallback int) int {
return fallback
}
func getEnvDuration(key string, fallback time.Duration) time.Duration {
if v := os.Getenv(key); v != "" {
if d, err := time.ParseDuration(v); err == nil {
return d
}
}
return fallback
}
func getEnvBool(key string, fallback bool) bool {
if v := os.Getenv(key); v != "" {
b, err := strconv.ParseBool(v)

View File

@@ -0,0 +1,13 @@
-- 013_cluster_registry.sql — v0.6.0
-- PG-backed cluster registry for node self-assembly.
-- UNLOGGED: no WAL overhead, contents lost on PG crash (correct behavior —
-- all nodes are dead anyway and re-register on restart).
CREATE UNLOGGED TABLE IF NOT EXISTS node_registry (
node_id TEXT PRIMARY KEY,
endpoint TEXT NOT NULL DEFAULT '',
seq SERIAL,
registered_at TIMESTAMPTZ DEFAULT now(),
heartbeat TIMESTAMPTZ DEFAULT now(),
stats JSONB DEFAULT '{}'
);

View File

@@ -0,0 +1,32 @@
package handlers
import (
"net/http"
"github.com/gin-gonic/gin"
"switchboard-core/store"
)
// ClusterHandler serves the admin cluster API.
type ClusterHandler struct {
stores store.Stores
}
func NewClusterHandler(stores store.Stores) *ClusterHandler {
return &ClusterHandler{stores: stores}
}
// ListNodes returns all registered cluster nodes.
// GET /api/v1/admin/cluster
func (h *ClusterHandler) ListNodes(c *gin.Context) {
nodes, err := h.stores.Cluster.ListNodes(c.Request.Context())
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list cluster nodes"})
return
}
if nodes == nil {
nodes = []store.ClusterNode{}
}
c.JSON(http.StatusOK, gin.H{"data": nodes})
}

View File

@@ -0,0 +1,114 @@
package handlers
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/gin-gonic/gin"
"switchboard-core/store"
)
// mockClusterStore implements store.ClusterStore for handler tests.
type mockClusterStore struct {
nodes []store.ClusterNode
}
func (m *mockClusterStore) Register(_ context.Context, _, _ string) error { return nil }
func (m *mockClusterStore) Heartbeat(_ context.Context, _ string, _ json.RawMessage) (int64, error) {
return 1, nil
}
func (m *mockClusterStore) SweepStale(_ context.Context, _ time.Duration) (int64, error) { return 0, nil }
func (m *mockClusterStore) ListNodes(_ context.Context) ([]store.ClusterNode, error) {
return m.nodes, nil
}
func (m *mockClusterStore) Deregister(_ context.Context, _ string) error { return nil }
func TestClusterListNodes(t *testing.T) {
gin.SetMode(gin.TestMode)
mock := &mockClusterStore{
nodes: []store.ClusterNode{
{
NodeID: "node-1",
Endpoint: "http://node-1:8080",
Seq: 1,
RegisteredAt: time.Now(),
Heartbeat: time.Now(),
Stats: json.RawMessage(`{"ws_clients":3}`),
},
{
NodeID: "node-2",
Endpoint: "http://node-2:8080",
Seq: 2,
RegisteredAt: time.Now(),
Heartbeat: time.Now(),
Stats: json.RawMessage(`{"ws_clients":7}`),
},
},
}
stores := store.Stores{Cluster: mock}
h := NewClusterHandler(stores)
r := gin.New()
r.GET("/api/v1/admin/cluster", h.ListNodes)
req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/cluster", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want %d", w.Code, http.StatusOK)
}
var resp struct {
Data []store.ClusterNode `json:"data"`
}
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("unmarshal response: %v", err)
}
if len(resp.Data) != 2 {
t.Fatalf("data length = %d, want 2", len(resp.Data))
}
if resp.Data[0].NodeID != "node-1" {
t.Errorf("data[0].node_id = %q, want %q", resp.Data[0].NodeID, "node-1")
}
if resp.Data[1].NodeID != "node-2" {
t.Errorf("data[1].node_id = %q, want %q", resp.Data[1].NodeID, "node-2")
}
}
func TestClusterListNodesEmpty(t *testing.T) {
gin.SetMode(gin.TestMode)
mock := &mockClusterStore{nodes: nil}
stores := store.Stores{Cluster: mock}
h := NewClusterHandler(stores)
r := gin.New()
r.GET("/api/v1/admin/cluster", h.ListNodes)
req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/cluster", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want %d", w.Code, http.StatusOK)
}
var resp struct {
Data []json.RawMessage `json:"data"`
}
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatalf("unmarshal: %v", err)
}
// Should be empty array, not null
if resp.Data == nil {
t.Error("data should be [], not null")
}
}

View File

@@ -23,13 +23,19 @@ import (
)
// defaultBundledPackages is the curated set of packages installed by default.
// This is the dev/test bundle — includes demos and workflow examples.
// Other packages still ship in the Docker image but require BUNDLED_PACKAGES
// to be set explicitly (or "*" for all).
//
// Recommended production override (lean):
// BUNDLED_PACKAGES=notes,chat,chat-core,cluster-dashboard
var defaultBundledPackages = map[string]bool{
"notes": true,
"chat": true,
"chat-core": true,
"workflow-chat": true,
"dashboard": true,
"cluster-dashboard": true,
"workflow-demo": true,
"bug-report-triage": true,
"content-approval": true,

View File

@@ -16,6 +16,7 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"switchboard-core/auth"
"switchboard-core/cluster"
"switchboard-core/config"
"switchboard-core/crypto"
"switchboard-core/database"
@@ -209,6 +210,27 @@ func main() {
// ── WebSocket Hub ─────────────────────────
hub := events.NewHub(bus, middleware.GetAllowedOrigins(cfg))
// ── Cluster Registry (v0.6.0) ────────────
// PG-backed node self-registration + heartbeat. No-op on SQLite.
var clusterReg *cluster.Registry
if database.IsPostgres() && stores.Cluster != nil {
nodeID := cfg.ClusterNodeID
if nodeID == "" {
hostname, _ := os.Hostname()
nodeID = fmt.Sprintf("%s-%d", hostname, os.Getpid())
}
clusterReg = cluster.NewRegistry(nodeID, cfg.ClusterEndpoint, stores.Cluster, hub, cluster.RegistryConfig{
HeartbeatInterval: cfg.ClusterHeartbeatInterval,
StaleThreshold: cfg.ClusterStaleThreshold,
})
if err := clusterReg.Start(); err != nil {
log.Printf("⚠ Cluster registry failed to start: %v", err)
clusterReg = nil
} else {
defer clusterReg.Stop()
}
}
// ── WebSocket Ticket Store (v0.32.0: PG-backed for cross-pod) ─────
ticketAdapter := &events.TicketValidatorAdapter{Store: stores.Tickets}
@@ -244,13 +266,15 @@ func main() {
// Health check (k8s probes hit this directly)
base.GET("/health", func(c *gin.Context) {
c.JSON(200, gin.H{
info := gin.H{
"status": "ok",
"version": Version,
"database": database.IsConnected(),
"database_name": database.Name(),
"schema_version": database.SchemaVersion(),
})
}
appendClusterHealth(info, clusterReg, stores)
c.JSON(200, info)
})
// v0.32.0: Kubernetes probe endpoints
@@ -345,6 +369,7 @@ func main() {
if database.IsConnected() {
info["registration_enabled"] = handlers.IsRegistrationEnabled(stores)
}
appendClusterHealth(info, clusterReg, stores)
c.JSON(200, info)
})
@@ -762,6 +787,12 @@ func main() {
admin.PUT("/schedules/:id/disable", schedH.AdminDisableSchedule)
admin.DELETE("/schedules/:id", schedH.AdminDeleteSchedule)
// ── Cluster (v0.6.0) ─────────────────
if stores.Cluster != nil {
clusterH := handlers.NewClusterHandler(stores)
admin.GET("/cluster", clusterH.ListNodes)
}
// Surface aliases (backward compat — same handlers)
admin.GET("/surfaces", pkgAdm.ListPackages)
admin.GET("/surfaces/:id", pkgAdm.GetPackage)
@@ -841,6 +872,32 @@ func main() {
}
}
// appendClusterHealth adds cluster info to a health response if the registry is active.
func appendClusterHealth(info gin.H, reg *cluster.Registry, stores store.Stores) {
if reg == nil || stores.Cluster == nil {
return
}
nodes, err := stores.Cluster.ListNodes(context.Background())
if err != nil {
return
}
var peers []string
var heartbeatAge time.Duration
for _, n := range nodes {
if n.NodeID != reg.NodeID() {
peers = append(peers, n.NodeID)
} else {
heartbeatAge = time.Since(n.Heartbeat)
}
}
info["node_id"] = reg.NodeID()
info["cluster"] = gin.H{
"size": len(nodes),
"peers": peers,
"heartbeat_age_ms": heartbeatAge.Milliseconds(),
}
}
// ── Vault CLI Commands ──────────────────────
func runVaultCommand(subcmd string) {

View File

@@ -0,0 +1,37 @@
package store
import (
"context"
"encoding/json"
"time"
)
// ClusterNode represents a registered node in the cluster registry.
type ClusterNode struct {
NodeID string `json:"node_id"`
Endpoint string `json:"endpoint"`
Seq int `json:"seq"`
RegisteredAt time.Time `json:"registered_at"`
Heartbeat time.Time `json:"heartbeat"`
Stats json.RawMessage `json:"stats"`
}
// ClusterStore manages the node_registry table for cluster self-assembly.
// Postgres-only — SQLite deployments set this to nil.
type ClusterStore interface {
// Register inserts or re-registers a node (idempotent).
Register(ctx context.Context, nodeID, endpoint string) error
// Heartbeat updates the node's heartbeat timestamp and stats.
// Returns rows affected — 0 means the node was swept (self-eviction).
Heartbeat(ctx context.Context, nodeID string, stats json.RawMessage) (int64, error)
// SweepStale deletes nodes whose heartbeat is older than threshold.
SweepStale(ctx context.Context, threshold time.Duration) (int64, error)
// ListNodes returns all registered nodes ordered by sequence.
ListNodes(ctx context.Context) ([]ClusterNode, error)
// Deregister removes a node (best-effort cleanup on shutdown).
Deregister(ctx context.Context, nodeID string) error
}

View File

@@ -54,6 +54,7 @@ type Stores struct {
RateLimits RateLimitStore // v0.32.0: Distributed rate limiting
Triggers TriggerStore // v0.2.2: Extension event/webhook triggers
ScheduledTasks ScheduledTaskStore // v0.2.2: User-created cron tasks
Cluster ClusterStore // v0.6.0: PG-backed cluster node registry (nil on SQLite)
}
// TeamAvailableModel is returned by CatalogStore.ListTeamAvailable.

View File

@@ -0,0 +1,78 @@
package postgres
import (
"context"
"encoding/json"
"fmt"
"time"
"switchboard-core/store"
)
// ClusterStore manages node_registry — Postgres-only UNLOGGED table.
type ClusterStore struct{}
func NewClusterStore() *ClusterStore { return &ClusterStore{} }
func (s *ClusterStore) Register(ctx context.Context, nodeID, endpoint string) error {
_, err := DB.ExecContext(ctx, `
INSERT INTO node_registry (node_id, endpoint)
VALUES ($1, $2)
ON CONFLICT (node_id) DO UPDATE
SET endpoint = EXCLUDED.endpoint,
registered_at = now(),
heartbeat = now(),
stats = '{}'
`, nodeID, endpoint)
return err
}
func (s *ClusterStore) Heartbeat(ctx context.Context, nodeID string, stats json.RawMessage) (int64, error) {
result, err := DB.ExecContext(ctx, `
UPDATE node_registry
SET heartbeat = now(), stats = $2
WHERE node_id = $1
`, nodeID, stats)
if err != nil {
return 0, err
}
return result.RowsAffected()
}
func (s *ClusterStore) SweepStale(ctx context.Context, threshold time.Duration) (int64, error) {
result, err := DB.ExecContext(ctx, `
DELETE FROM node_registry
WHERE heartbeat < now() - $1 * interval '1 second'
`, threshold.Seconds())
if err != nil {
return 0, err
}
return result.RowsAffected()
}
func (s *ClusterStore) ListNodes(ctx context.Context) ([]store.ClusterNode, error) {
rows, err := DB.QueryContext(ctx, `
SELECT node_id, endpoint, seq, registered_at, heartbeat, stats
FROM node_registry
ORDER BY seq
`)
if err != nil {
return nil, err
}
defer rows.Close()
var nodes []store.ClusterNode
for rows.Next() {
var n store.ClusterNode
if err := rows.Scan(&n.NodeID, &n.Endpoint, &n.Seq, &n.RegisteredAt, &n.Heartbeat, &n.Stats); err != nil {
return nil, fmt.Errorf("scan cluster node: %w", err)
}
nodes = append(nodes, n)
}
return nodes, rows.Err()
}
func (s *ClusterStore) Deregister(ctx context.Context, nodeID string) error {
_, err := DB.ExecContext(ctx, `DELETE FROM node_registry WHERE node_id = $1`, nodeID)
return err
}

View File

@@ -32,5 +32,6 @@ func NewStores(db *sql.DB) store.Stores {
RateLimits: NewRateLimitStore(),
Triggers: NewTriggerStore(),
ScheduledTasks: NewScheduledTaskStore(),
Cluster: NewClusterStore(),
}
}