Feat v0.6.0 cluster registry (#36)
All checks were successful
All checks were successful
Co-authored-by: Jeffrey Smith <jasafpro@gmail.com> Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>
This commit was merged in pull request #36.
This commit is contained in:
143
server/cluster/registry.go
Normal file
143
server/cluster/registry.go
Normal file
@@ -0,0 +1,143 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"switchboard-core/store"
|
||||
)
|
||||
|
||||
// ConnCounter provides WebSocket connection count (satisfied by events.Hub).
|
||||
type ConnCounter interface {
|
||||
ConnCount() int
|
||||
}
|
||||
|
||||
// RegistryConfig holds tunable cluster registry parameters.
|
||||
type RegistryConfig struct {
|
||||
HeartbeatInterval time.Duration
|
||||
StaleThreshold time.Duration
|
||||
}
|
||||
|
||||
// Registry manages node self-registration, heartbeat, and stale sweep.
|
||||
// Follows the workflow.Scanner pattern: stopCh + wg + ticker.
|
||||
type Registry struct {
|
||||
nodeID string
|
||||
endpoint string
|
||||
store store.ClusterStore
|
||||
hub ConnCounter
|
||||
cfg RegistryConfig
|
||||
startTime time.Time
|
||||
stopCh chan struct{}
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewRegistry creates a cluster registry instance.
|
||||
func NewRegistry(nodeID, endpoint string, s store.ClusterStore, hub ConnCounter, cfg RegistryConfig) *Registry {
|
||||
return &Registry{
|
||||
nodeID: nodeID,
|
||||
endpoint: endpoint,
|
||||
store: s,
|
||||
hub: hub,
|
||||
cfg: cfg,
|
||||
startTime: time.Now(),
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// NodeID returns this node's identifier.
|
||||
func (r *Registry) NodeID() string { return r.nodeID }
|
||||
|
||||
// Start registers the node and launches the heartbeat goroutine.
|
||||
func (r *Registry) Start() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := r.store.Register(ctx, r.nodeID, r.endpoint); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
r.wg.Add(1)
|
||||
go func() {
|
||||
defer r.wg.Done()
|
||||
ticker := time.NewTicker(r.cfg.HeartbeatInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
r.tick()
|
||||
case <-r.stopCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("[cluster] started (node=%s, heartbeat=%s, stale=%s)",
|
||||
r.nodeID, r.cfg.HeartbeatInterval, r.cfg.StaleThreshold)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop deregisters the node and waits for the heartbeat goroutine to exit.
|
||||
func (r *Registry) Stop() {
|
||||
close(r.stopCh)
|
||||
r.wg.Wait()
|
||||
|
||||
// Best-effort deregister so peers don't have to wait for stale sweep.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
if err := r.store.Deregister(ctx, r.nodeID); err != nil {
|
||||
log.Printf("[cluster] deregister failed: %v", err)
|
||||
}
|
||||
log.Printf("[cluster] stopped (node=%s)", r.nodeID)
|
||||
}
|
||||
|
||||
// tick runs one heartbeat + sweep cycle.
|
||||
func (r *Registry) tick() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
stats := r.collectStats()
|
||||
|
||||
rows, err := r.store.Heartbeat(ctx, r.nodeID, stats)
|
||||
if err != nil {
|
||||
log.Printf("[cluster] heartbeat error: %v", err)
|
||||
return
|
||||
}
|
||||
if rows == 0 {
|
||||
// Self-eviction: this node was swept by a peer.
|
||||
// Kubernetes will restart the process → re-register.
|
||||
log.Printf("[cluster] FATAL: node %s evicted from registry — shutting down", r.nodeID)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
swept, err := r.store.SweepStale(ctx, r.cfg.StaleThreshold)
|
||||
if err != nil {
|
||||
log.Printf("[cluster] sweep error: %v", err)
|
||||
return
|
||||
}
|
||||
if swept > 0 {
|
||||
log.Printf("[cluster] swept %d stale node(s)", swept)
|
||||
}
|
||||
}
|
||||
|
||||
// collectStats gathers runtime metrics for the stats JSONB column.
|
||||
func (r *Registry) collectStats() json.RawMessage {
|
||||
var m runtime.MemStats
|
||||
runtime.ReadMemStats(&m)
|
||||
|
||||
stats := map[string]any{
|
||||
"goroutines": runtime.NumGoroutine(),
|
||||
"heap_alloc": m.HeapAlloc,
|
||||
"heap_sys": m.HeapSys,
|
||||
"gc_cycles": m.NumGC,
|
||||
"gc_pause_ns": m.PauseNs[(m.NumGC+255)%256],
|
||||
"uptime_sec": time.Since(r.startTime).Seconds(),
|
||||
"ws_clients": r.hub.ConnCount(),
|
||||
}
|
||||
|
||||
data, _ := json.Marshal(stats)
|
||||
return data
|
||||
}
|
||||
112
server/cluster/registry_test.go
Normal file
112
server/cluster/registry_test.go
Normal file
@@ -0,0 +1,112 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"switchboard-core/store"
|
||||
)
|
||||
|
||||
// mockClusterStore implements store.ClusterStore for unit tests.
|
||||
type mockClusterStore struct {
|
||||
nodes []store.ClusterNode
|
||||
heartbeatRows int64
|
||||
heartbeatErr error
|
||||
sweepCalled bool
|
||||
registerCalled bool
|
||||
lastStats json.RawMessage
|
||||
}
|
||||
|
||||
func (m *mockClusterStore) Register(_ context.Context, nodeID, endpoint string) error {
|
||||
m.registerCalled = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockClusterStore) Heartbeat(_ context.Context, nodeID string, stats json.RawMessage) (int64, error) {
|
||||
m.lastStats = stats
|
||||
return m.heartbeatRows, m.heartbeatErr
|
||||
}
|
||||
|
||||
func (m *mockClusterStore) SweepStale(_ context.Context, _ time.Duration) (int64, error) {
|
||||
m.sweepCalled = true
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (m *mockClusterStore) ListNodes(_ context.Context) ([]store.ClusterNode, error) {
|
||||
return m.nodes, nil
|
||||
}
|
||||
|
||||
func (m *mockClusterStore) Deregister(_ context.Context, _ string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// mockHub implements ConnCounter.
|
||||
type mockHub struct{ count int }
|
||||
|
||||
func (h *mockHub) ConnCount() int { return h.count }
|
||||
|
||||
func TestCollectStats(t *testing.T) {
|
||||
hub := &mockHub{count: 5}
|
||||
reg := NewRegistry("test-node", "http://localhost:8080", nil, hub, RegistryConfig{
|
||||
HeartbeatInterval: 10 * time.Second,
|
||||
StaleThreshold: 30 * time.Second,
|
||||
})
|
||||
|
||||
data := reg.collectStats()
|
||||
|
||||
var stats map[string]any
|
||||
if err := json.Unmarshal(data, &stats); err != nil {
|
||||
t.Fatalf("collectStats returned invalid JSON: %v", err)
|
||||
}
|
||||
|
||||
// Verify expected keys
|
||||
expectedKeys := []string{"goroutines", "heap_alloc", "heap_sys", "gc_cycles", "gc_pause_ns", "uptime_sec", "ws_clients"}
|
||||
for _, key := range expectedKeys {
|
||||
if _, ok := stats[key]; !ok {
|
||||
t.Errorf("missing expected stats key: %s", key)
|
||||
}
|
||||
}
|
||||
|
||||
// ws_clients should reflect hub count
|
||||
if ws, ok := stats["ws_clients"].(float64); !ok || int(ws) != 5 {
|
||||
t.Errorf("ws_clients = %v, want 5", stats["ws_clients"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryStartStop(t *testing.T) {
|
||||
ms := &mockClusterStore{heartbeatRows: 1}
|
||||
hub := &mockHub{count: 0}
|
||||
|
||||
reg := NewRegistry("test-node", "", ms, hub, RegistryConfig{
|
||||
HeartbeatInterval: 50 * time.Millisecond,
|
||||
StaleThreshold: 150 * time.Millisecond,
|
||||
})
|
||||
|
||||
if err := reg.Start(); err != nil {
|
||||
t.Fatalf("Start() error: %v", err)
|
||||
}
|
||||
if !ms.registerCalled {
|
||||
t.Error("Register was not called on Start")
|
||||
}
|
||||
|
||||
// Let at least one tick fire
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
reg.Stop()
|
||||
|
||||
if !ms.sweepCalled {
|
||||
t.Error("SweepStale was never called during tick")
|
||||
}
|
||||
if ms.lastStats == nil {
|
||||
t.Error("Heartbeat was never called with stats")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeID(t *testing.T) {
|
||||
reg := NewRegistry("my-node-123", "", nil, &mockHub{}, RegistryConfig{})
|
||||
if got := reg.NodeID(); got != "my-node-123" {
|
||||
t.Errorf("NodeID() = %q, want %q", got, "my-node-123")
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
@@ -85,6 +86,17 @@ type Config struct {
|
||||
OIDCRolesClaim string // JWT claim for role mapping (default "realm_access.roles")
|
||||
OIDCGroupsClaim string // JWT claim for group sync (default "groups")
|
||||
OIDCAdminRole string // IdP role value that maps to admin (default "admin")
|
||||
|
||||
// Cluster registry (v0.6.0)
|
||||
// PG-backed node registry for horizontal scaling. No-op on SQLite.
|
||||
// CLUSTER_NODE_ID: override for deterministic identity (default: hostname-PID).
|
||||
// CLUSTER_HEARTBEAT_INTERVAL: tick frequency (default "10s").
|
||||
// CLUSTER_STALE_THRESHOLD: 3x heartbeat = stale (default "30s").
|
||||
// CLUSTER_ENDPOINT: advertised address for peer mesh (Phase 2, auto-detect).
|
||||
ClusterNodeID string
|
||||
ClusterHeartbeatInterval time.Duration
|
||||
ClusterStaleThreshold time.Duration
|
||||
ClusterEndpoint string
|
||||
}
|
||||
|
||||
// Load reads configuration from environment variables.
|
||||
@@ -143,6 +155,12 @@ func Load() *Config {
|
||||
OIDCRolesClaim: getEnv("OIDC_ROLES_CLAIM", "realm_access.roles"),
|
||||
OIDCGroupsClaim: getEnv("OIDC_GROUPS_CLAIM", "groups"),
|
||||
OIDCAdminRole: getEnv("OIDC_ADMIN_ROLE", "admin"),
|
||||
|
||||
// Cluster
|
||||
ClusterNodeID: getEnv("CLUSTER_NODE_ID", ""),
|
||||
ClusterHeartbeatInterval: getEnvDuration("CLUSTER_HEARTBEAT_INTERVAL", 10*time.Second),
|
||||
ClusterStaleThreshold: getEnvDuration("CLUSTER_STALE_THRESHOLD", 30*time.Second),
|
||||
ClusterEndpoint: getEnv("CLUSTER_ENDPOINT", ""),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -204,6 +222,15 @@ func getEnvInt(key string, fallback int) int {
|
||||
return fallback
|
||||
}
|
||||
|
||||
func getEnvDuration(key string, fallback time.Duration) time.Duration {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func getEnvBool(key string, fallback bool) bool {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
b, err := strconv.ParseBool(v)
|
||||
|
||||
13
server/database/migrations/postgres/013_cluster_registry.sql
Normal file
13
server/database/migrations/postgres/013_cluster_registry.sql
Normal file
@@ -0,0 +1,13 @@
|
||||
-- 013_cluster_registry.sql — v0.6.0
|
||||
-- PG-backed cluster registry for node self-assembly.
|
||||
-- UNLOGGED: no WAL overhead, contents lost on PG crash (correct behavior —
|
||||
-- all nodes are dead anyway and re-register on restart).
|
||||
|
||||
CREATE UNLOGGED TABLE IF NOT EXISTS node_registry (
|
||||
node_id TEXT PRIMARY KEY,
|
||||
endpoint TEXT NOT NULL DEFAULT '',
|
||||
seq SERIAL,
|
||||
registered_at TIMESTAMPTZ DEFAULT now(),
|
||||
heartbeat TIMESTAMPTZ DEFAULT now(),
|
||||
stats JSONB DEFAULT '{}'
|
||||
);
|
||||
32
server/handlers/cluster.go
Normal file
32
server/handlers/cluster.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"switchboard-core/store"
|
||||
)
|
||||
|
||||
// ClusterHandler serves the admin cluster API.
|
||||
type ClusterHandler struct {
|
||||
stores store.Stores
|
||||
}
|
||||
|
||||
func NewClusterHandler(stores store.Stores) *ClusterHandler {
|
||||
return &ClusterHandler{stores: stores}
|
||||
}
|
||||
|
||||
// ListNodes returns all registered cluster nodes.
|
||||
// GET /api/v1/admin/cluster
|
||||
func (h *ClusterHandler) ListNodes(c *gin.Context) {
|
||||
nodes, err := h.stores.Cluster.ListNodes(c.Request.Context())
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list cluster nodes"})
|
||||
return
|
||||
}
|
||||
if nodes == nil {
|
||||
nodes = []store.ClusterNode{}
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"data": nodes})
|
||||
}
|
||||
114
server/handlers/cluster_test.go
Normal file
114
server/handlers/cluster_test.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"switchboard-core/store"
|
||||
)
|
||||
|
||||
// mockClusterStore implements store.ClusterStore for handler tests.
|
||||
type mockClusterStore struct {
|
||||
nodes []store.ClusterNode
|
||||
}
|
||||
|
||||
func (m *mockClusterStore) Register(_ context.Context, _, _ string) error { return nil }
|
||||
func (m *mockClusterStore) Heartbeat(_ context.Context, _ string, _ json.RawMessage) (int64, error) {
|
||||
return 1, nil
|
||||
}
|
||||
func (m *mockClusterStore) SweepStale(_ context.Context, _ time.Duration) (int64, error) { return 0, nil }
|
||||
func (m *mockClusterStore) ListNodes(_ context.Context) ([]store.ClusterNode, error) {
|
||||
return m.nodes, nil
|
||||
}
|
||||
func (m *mockClusterStore) Deregister(_ context.Context, _ string) error { return nil }
|
||||
|
||||
func TestClusterListNodes(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
mock := &mockClusterStore{
|
||||
nodes: []store.ClusterNode{
|
||||
{
|
||||
NodeID: "node-1",
|
||||
Endpoint: "http://node-1:8080",
|
||||
Seq: 1,
|
||||
RegisteredAt: time.Now(),
|
||||
Heartbeat: time.Now(),
|
||||
Stats: json.RawMessage(`{"ws_clients":3}`),
|
||||
},
|
||||
{
|
||||
NodeID: "node-2",
|
||||
Endpoint: "http://node-2:8080",
|
||||
Seq: 2,
|
||||
RegisteredAt: time.Now(),
|
||||
Heartbeat: time.Now(),
|
||||
Stats: json.RawMessage(`{"ws_clients":7}`),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
stores := store.Stores{Cluster: mock}
|
||||
h := NewClusterHandler(stores)
|
||||
|
||||
r := gin.New()
|
||||
r.GET("/api/v1/admin/cluster", h.ListNodes)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/cluster", nil)
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", w.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
var resp struct {
|
||||
Data []store.ClusterNode `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("unmarshal response: %v", err)
|
||||
}
|
||||
if len(resp.Data) != 2 {
|
||||
t.Fatalf("data length = %d, want 2", len(resp.Data))
|
||||
}
|
||||
if resp.Data[0].NodeID != "node-1" {
|
||||
t.Errorf("data[0].node_id = %q, want %q", resp.Data[0].NodeID, "node-1")
|
||||
}
|
||||
if resp.Data[1].NodeID != "node-2" {
|
||||
t.Errorf("data[1].node_id = %q, want %q", resp.Data[1].NodeID, "node-2")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClusterListNodesEmpty(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
mock := &mockClusterStore{nodes: nil}
|
||||
stores := store.Stores{Cluster: mock}
|
||||
h := NewClusterHandler(stores)
|
||||
|
||||
r := gin.New()
|
||||
r.GET("/api/v1/admin/cluster", h.ListNodes)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/cluster", nil)
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, req)
|
||||
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want %d", w.Code, http.StatusOK)
|
||||
}
|
||||
|
||||
var resp struct {
|
||||
Data []json.RawMessage `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
// Should be empty array, not null
|
||||
if resp.Data == nil {
|
||||
t.Error("data should be [], not null")
|
||||
}
|
||||
}
|
||||
@@ -23,13 +23,19 @@ import (
|
||||
)
|
||||
|
||||
// defaultBundledPackages is the curated set of packages installed by default.
|
||||
// This is the dev/test bundle — includes demos and workflow examples.
|
||||
// Other packages still ship in the Docker image but require BUNDLED_PACKAGES
|
||||
// to be set explicitly (or "*" for all).
|
||||
//
|
||||
// Recommended production override (lean):
|
||||
// BUNDLED_PACKAGES=notes,chat,chat-core,cluster-dashboard
|
||||
var defaultBundledPackages = map[string]bool{
|
||||
"notes": true,
|
||||
"chat": true,
|
||||
"chat-core": true,
|
||||
"workflow-chat": true,
|
||||
"dashboard": true,
|
||||
"cluster-dashboard": true,
|
||||
"workflow-demo": true,
|
||||
"bug-report-triage": true,
|
||||
"content-approval": true,
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
"switchboard-core/auth"
|
||||
"switchboard-core/cluster"
|
||||
"switchboard-core/config"
|
||||
"switchboard-core/crypto"
|
||||
"switchboard-core/database"
|
||||
@@ -209,6 +210,27 @@ func main() {
|
||||
// ── WebSocket Hub ─────────────────────────
|
||||
hub := events.NewHub(bus, middleware.GetAllowedOrigins(cfg))
|
||||
|
||||
// ── Cluster Registry (v0.6.0) ────────────
|
||||
// PG-backed node self-registration + heartbeat. No-op on SQLite.
|
||||
var clusterReg *cluster.Registry
|
||||
if database.IsPostgres() && stores.Cluster != nil {
|
||||
nodeID := cfg.ClusterNodeID
|
||||
if nodeID == "" {
|
||||
hostname, _ := os.Hostname()
|
||||
nodeID = fmt.Sprintf("%s-%d", hostname, os.Getpid())
|
||||
}
|
||||
clusterReg = cluster.NewRegistry(nodeID, cfg.ClusterEndpoint, stores.Cluster, hub, cluster.RegistryConfig{
|
||||
HeartbeatInterval: cfg.ClusterHeartbeatInterval,
|
||||
StaleThreshold: cfg.ClusterStaleThreshold,
|
||||
})
|
||||
if err := clusterReg.Start(); err != nil {
|
||||
log.Printf("⚠ Cluster registry failed to start: %v", err)
|
||||
clusterReg = nil
|
||||
} else {
|
||||
defer clusterReg.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
// ── WebSocket Ticket Store (v0.32.0: PG-backed for cross-pod) ─────
|
||||
ticketAdapter := &events.TicketValidatorAdapter{Store: stores.Tickets}
|
||||
|
||||
@@ -244,13 +266,15 @@ func main() {
|
||||
|
||||
// Health check (k8s probes hit this directly)
|
||||
base.GET("/health", func(c *gin.Context) {
|
||||
c.JSON(200, gin.H{
|
||||
info := gin.H{
|
||||
"status": "ok",
|
||||
"version": Version,
|
||||
"database": database.IsConnected(),
|
||||
"database_name": database.Name(),
|
||||
"schema_version": database.SchemaVersion(),
|
||||
})
|
||||
}
|
||||
appendClusterHealth(info, clusterReg, stores)
|
||||
c.JSON(200, info)
|
||||
})
|
||||
|
||||
// v0.32.0: Kubernetes probe endpoints
|
||||
@@ -345,6 +369,7 @@ func main() {
|
||||
if database.IsConnected() {
|
||||
info["registration_enabled"] = handlers.IsRegistrationEnabled(stores)
|
||||
}
|
||||
appendClusterHealth(info, clusterReg, stores)
|
||||
c.JSON(200, info)
|
||||
})
|
||||
|
||||
@@ -762,6 +787,12 @@ func main() {
|
||||
admin.PUT("/schedules/:id/disable", schedH.AdminDisableSchedule)
|
||||
admin.DELETE("/schedules/:id", schedH.AdminDeleteSchedule)
|
||||
|
||||
// ── Cluster (v0.6.0) ─────────────────
|
||||
if stores.Cluster != nil {
|
||||
clusterH := handlers.NewClusterHandler(stores)
|
||||
admin.GET("/cluster", clusterH.ListNodes)
|
||||
}
|
||||
|
||||
// Surface aliases (backward compat — same handlers)
|
||||
admin.GET("/surfaces", pkgAdm.ListPackages)
|
||||
admin.GET("/surfaces/:id", pkgAdm.GetPackage)
|
||||
@@ -841,6 +872,32 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// appendClusterHealth adds cluster info to a health response if the registry is active.
|
||||
func appendClusterHealth(info gin.H, reg *cluster.Registry, stores store.Stores) {
|
||||
if reg == nil || stores.Cluster == nil {
|
||||
return
|
||||
}
|
||||
nodes, err := stores.Cluster.ListNodes(context.Background())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var peers []string
|
||||
var heartbeatAge time.Duration
|
||||
for _, n := range nodes {
|
||||
if n.NodeID != reg.NodeID() {
|
||||
peers = append(peers, n.NodeID)
|
||||
} else {
|
||||
heartbeatAge = time.Since(n.Heartbeat)
|
||||
}
|
||||
}
|
||||
info["node_id"] = reg.NodeID()
|
||||
info["cluster"] = gin.H{
|
||||
"size": len(nodes),
|
||||
"peers": peers,
|
||||
"heartbeat_age_ms": heartbeatAge.Milliseconds(),
|
||||
}
|
||||
}
|
||||
|
||||
// ── Vault CLI Commands ──────────────────────
|
||||
|
||||
func runVaultCommand(subcmd string) {
|
||||
|
||||
37
server/store/cluster_iface.go
Normal file
37
server/store/cluster_iface.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ClusterNode represents a registered node in the cluster registry.
|
||||
type ClusterNode struct {
|
||||
NodeID string `json:"node_id"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Seq int `json:"seq"`
|
||||
RegisteredAt time.Time `json:"registered_at"`
|
||||
Heartbeat time.Time `json:"heartbeat"`
|
||||
Stats json.RawMessage `json:"stats"`
|
||||
}
|
||||
|
||||
// ClusterStore manages the node_registry table for cluster self-assembly.
|
||||
// Postgres-only — SQLite deployments set this to nil.
|
||||
type ClusterStore interface {
|
||||
// Register inserts or re-registers a node (idempotent).
|
||||
Register(ctx context.Context, nodeID, endpoint string) error
|
||||
|
||||
// Heartbeat updates the node's heartbeat timestamp and stats.
|
||||
// Returns rows affected — 0 means the node was swept (self-eviction).
|
||||
Heartbeat(ctx context.Context, nodeID string, stats json.RawMessage) (int64, error)
|
||||
|
||||
// SweepStale deletes nodes whose heartbeat is older than threshold.
|
||||
SweepStale(ctx context.Context, threshold time.Duration) (int64, error)
|
||||
|
||||
// ListNodes returns all registered nodes ordered by sequence.
|
||||
ListNodes(ctx context.Context) ([]ClusterNode, error)
|
||||
|
||||
// Deregister removes a node (best-effort cleanup on shutdown).
|
||||
Deregister(ctx context.Context, nodeID string) error
|
||||
}
|
||||
@@ -54,6 +54,7 @@ type Stores struct {
|
||||
RateLimits RateLimitStore // v0.32.0: Distributed rate limiting
|
||||
Triggers TriggerStore // v0.2.2: Extension event/webhook triggers
|
||||
ScheduledTasks ScheduledTaskStore // v0.2.2: User-created cron tasks
|
||||
Cluster ClusterStore // v0.6.0: PG-backed cluster node registry (nil on SQLite)
|
||||
}
|
||||
|
||||
// TeamAvailableModel is returned by CatalogStore.ListTeamAvailable.
|
||||
|
||||
78
server/store/postgres/cluster.go
Normal file
78
server/store/postgres/cluster.go
Normal file
@@ -0,0 +1,78 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"switchboard-core/store"
|
||||
)
|
||||
|
||||
// ClusterStore manages node_registry — Postgres-only UNLOGGED table.
|
||||
type ClusterStore struct{}
|
||||
|
||||
func NewClusterStore() *ClusterStore { return &ClusterStore{} }
|
||||
|
||||
func (s *ClusterStore) Register(ctx context.Context, nodeID, endpoint string) error {
|
||||
_, err := DB.ExecContext(ctx, `
|
||||
INSERT INTO node_registry (node_id, endpoint)
|
||||
VALUES ($1, $2)
|
||||
ON CONFLICT (node_id) DO UPDATE
|
||||
SET endpoint = EXCLUDED.endpoint,
|
||||
registered_at = now(),
|
||||
heartbeat = now(),
|
||||
stats = '{}'
|
||||
`, nodeID, endpoint)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *ClusterStore) Heartbeat(ctx context.Context, nodeID string, stats json.RawMessage) (int64, error) {
|
||||
result, err := DB.ExecContext(ctx, `
|
||||
UPDATE node_registry
|
||||
SET heartbeat = now(), stats = $2
|
||||
WHERE node_id = $1
|
||||
`, nodeID, stats)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
func (s *ClusterStore) SweepStale(ctx context.Context, threshold time.Duration) (int64, error) {
|
||||
result, err := DB.ExecContext(ctx, `
|
||||
DELETE FROM node_registry
|
||||
WHERE heartbeat < now() - $1 * interval '1 second'
|
||||
`, threshold.Seconds())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
func (s *ClusterStore) ListNodes(ctx context.Context) ([]store.ClusterNode, error) {
|
||||
rows, err := DB.QueryContext(ctx, `
|
||||
SELECT node_id, endpoint, seq, registered_at, heartbeat, stats
|
||||
FROM node_registry
|
||||
ORDER BY seq
|
||||
`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var nodes []store.ClusterNode
|
||||
for rows.Next() {
|
||||
var n store.ClusterNode
|
||||
if err := rows.Scan(&n.NodeID, &n.Endpoint, &n.Seq, &n.RegisteredAt, &n.Heartbeat, &n.Stats); err != nil {
|
||||
return nil, fmt.Errorf("scan cluster node: %w", err)
|
||||
}
|
||||
nodes = append(nodes, n)
|
||||
}
|
||||
return nodes, rows.Err()
|
||||
}
|
||||
|
||||
func (s *ClusterStore) Deregister(ctx context.Context, nodeID string) error {
|
||||
_, err := DB.ExecContext(ctx, `DELETE FROM node_registry WHERE node_id = $1`, nodeID)
|
||||
return err
|
||||
}
|
||||
@@ -32,5 +32,6 @@ func NewStores(db *sql.DB) store.Stores {
|
||||
RateLimits: NewRateLimitStore(),
|
||||
Triggers: NewTriggerStore(),
|
||||
ScheduledTasks: NewScheduledTaskStore(),
|
||||
Cluster: NewClusterStore(),
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user