From 4bc11c2f4e0a60c995df83de2f7273321daab583 Mon Sep 17 00:00:00 2001 From: Jeffrey Smith Date: Tue, 31 Mar 2026 13:06:09 +0000 Subject: [PATCH 1/2] Feat v0.6.4 admin health/metrics tab + cluster merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Admin Health tab under Monitoring with auto-refreshing runtime, DB, cluster, and extension metrics panels. New GET /api/v1/admin/metrics endpoint. Fattened heartbeat JSONB with stack, GC CPU%, extension count, sandbox stats, trigger fires. Sandbox runner and trigger engine now track cumulative execution counters. Event bus tracks publish/deliver counts. Health endpoints consolidated via shared builder. Block renderers (mermaid, katex, csv-table, diff-viewer) no longer require chat. cluster-dashboard package retired — merged into admin Health tab. Co-Authored-By: Claude Opus 4.6 (1M context) --- CHANGELOG.md | 44 ++++ ROADMAP.md | 20 +- VERSION | 2 +- packages/cluster-dashboard/css/main.css | 100 --------- packages/cluster-dashboard/js/main.js | 124 ----------- packages/cluster-dashboard/manifest.json | 13 -- packages/csv-table/manifest.json | 2 +- packages/diff-viewer/manifest.json | 2 +- packages/katex-renderer/manifest.json | 2 +- packages/mermaid-renderer/manifest.json | 2 +- server/cluster/registry.go | 49 ++++- server/cluster/registry_test.go | 40 ++++ server/events/bus.go | 12 ++ server/handlers/admin_metrics.go | 24 +++ server/handlers/admin_metrics_test.go | 179 ++++++++++++++++ server/main.go | 55 +++-- server/metrics/collector.go | 253 +++++++++++++++++++++++ server/sandbox/runner.go | 52 ++++- server/triggers/engine.go | 9 +- src/js/sw/sdk/api-domains.js | 1 + src/js/sw/surfaces/admin/health.js | 206 ++++++++++++++++++ src/js/sw/surfaces/admin/index.js | 7 +- 22 files changed, 917 insertions(+), 281 deletions(-) delete mode 100644 packages/cluster-dashboard/css/main.css delete mode 100644 packages/cluster-dashboard/js/main.js delete mode 100644 packages/cluster-dashboard/manifest.json create mode 100644 server/handlers/admin_metrics.go create mode 100644 server/handlers/admin_metrics_test.go create mode 100644 server/metrics/collector.go create mode 100644 src/js/sw/surfaces/admin/health.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 9047700..8402376 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,50 @@ All notable changes to Switchboard Core are documented here. +## v0.6.4 — Admin Health/Metrics Tab + Cluster Merge + +Structural consolidation: cluster dashboard merges into Admin as a Health tab. +Comprehensive metrics endpoint for runtime, DB, cluster, and extension stats. + +### Added + +- **Admin Health tab**: New "Health" section under Monitoring in the Admin + surface. Auto-refreshing panels for runtime, database, cluster, and extension + metrics with configurable poll interval (5/10/30/60s). +- **`GET /api/v1/admin/metrics`**: Single JSON endpoint returning all platform + metrics — runtime (goroutines, heap, GC, uptime, FDs), DB pool (latency, + active/idle/max, PG-only dead tuples + active backends), cluster nodes + (when PG multi-node), and extension stats (Starlark exec/errors/duration, + trigger fires, event bus publish/deliver counts). +- **Fattened heartbeat payload**: Cluster heartbeat JSONB now carries stack + usage, GC CPU%, extensions loaded, Starlark execution counters, and trigger + fire count — visible in per-node cluster cards. +- **Sandbox execution tracking**: `ExecPackage` and `CallEntryPoint` now + increment atomic counters (exec, errors, duration) and the previously + declared but unused `SandboxExecutionsTotal` Prometheus counter. +- **Event bus counters**: `Bus.PublishCount()` and `Bus.DeliverCount()` track + cumulative publish/deliver operations. +- **Trigger fire counter**: `Engine.FireCount()` tracks cumulative trigger + fires across webhook, event, and scheduled triggers. +- 4 new handler tests (metrics SQLite shape, cluster shape, extension counters). +- 1 new cluster test (fattened heartbeat payload). + +### Changed + +- **Health endpoint consolidation**: `/health` and `/api/v1/health` now use a + shared `buildHealthResponse()` function returning identical JSON. Both now + include `registration_enabled` when the database is connected. +- **Block renderer `requires` removed**: `mermaid-renderer`, `katex-renderer`, + `csv-table`, and `diff-viewer` no longer require `["chat"]` — they activate + on any surface (notes, docs, etc.). + +### Removed + +- **`cluster-dashboard` package**: Standalone surface package retired. Cluster + node visibility is now in Admin > Monitoring > Health. + +--- + ## v0.6.3 — Dead Code Sweep + Registry Fix Pre-fork hardening: fix broken registry install, add registry settings UI, diff --git a/ROADMAP.md b/ROADMAP.md index 7255721..48ee961 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,6 +1,6 @@ # Switchboard Core — Roadmap -## Current: v0.6.3 — Dead Code Sweep + Registry Fix +## Current: v0.6.4 — Admin Health/Metrics Tab + Cluster Merge Self-hosted extensible platform. Auth, identity, packages, Starlark sandbox, storage, realtime, and ops are kernel primitives. Everything else is an extension. @@ -98,15 +98,15 @@ Structural move: cluster dashboard becomes an Admin tab. Better home for health/ | Step | Status | Description | |------|--------|-------------| -| "Health / Metrics" admin tab | ☐ | New tab in Admin surface. DB-agnostic metrics for all deployments. Cluster cards conditional on PG + multi-node detection. | -| Runtime metrics | ☐ | Per-node: goroutines, heap alloc/sys, stack in use, GC cycles, last GC pause, GC CPU %, uptime, WebSocket clients, extensions loaded, open FDs. | -| DB pool metrics | ☐ | All deployments: DB latency (`SELECT 1` round-trip), pool active/idle/max, wait count, wait duration. PG-only: table bloat (`n_dead_tup`), active backends (`pg_stat_activity`). | -| Cluster metrics | ☐ | PG multi-node only: cluster size, peer list with endpoint + uptime, heartbeat age per node, event bus publish/deliver rates. | -| Extension runtime metrics | ☐ | Starlark exec/min, errors/min, avg duration, HTTP outbound requests/min, trigger fires/min, schedule overruns. | -| Fatten heartbeat payload | ☐ | Heartbeat JSONB carries full metric set. `GET /api/v1/admin/metrics` for single-node SQLite fallback (same shape). | -| Retire `cluster-dashboard` | ☐ | Remove package once Admin Health tab ships. Update `defaultBundledPackages`. | -| Fix block renderer `requires` | ☐ | `mermaid-renderer`, `katex-renderer`, `csv-table`, `diff-viewer` all have `"requires": ["chat"]`. These are content renderers, not chat features. Remove constraint — they should activate without chat. | -| Health endpoint consolidation | ☐ | `/health` and `/api/v1/health` return near-identical JSON. Merge or clearly differentiate with docs. | +| "Health / Metrics" admin tab | ✅ | New tab in Admin surface. DB-agnostic metrics for all deployments. Cluster cards conditional on PG + multi-node detection. | +| Runtime metrics | ✅ | Per-node: goroutines, heap alloc/sys, stack in use, GC cycles, last GC pause, GC CPU %, uptime, WebSocket clients, extensions loaded, open FDs. | +| DB pool metrics | ✅ | All deployments: DB latency (`SELECT 1` round-trip), pool active/idle/max, wait count, wait duration. PG-only: table bloat (`n_dead_tup`), active backends (`pg_stat_activity`). | +| Cluster metrics | ✅ | PG multi-node only: cluster size, peer list with endpoint + uptime, heartbeat age per node, event bus publish/deliver rates. | +| Extension runtime metrics | ✅ | Starlark exec/min, errors/min, avg duration, HTTP outbound requests/min, trigger fires/min, schedule overruns. | +| Fatten heartbeat payload | ✅ | Heartbeat JSONB carries full metric set. `GET /api/v1/admin/metrics` for single-node SQLite fallback (same shape). | +| Retire `cluster-dashboard` | ✅ | Remove package once Admin Health tab ships. Update `defaultBundledPackages`. | +| Fix block renderer `requires` | ✅ | `mermaid-renderer`, `katex-renderer`, `csv-table`, `diff-viewer` all have `"requires": ["chat"]`. These are content renderers, not chat features. Remove constraint — they should activate without chat. | +| Health endpoint consolidation | ✅ | `/health` and `/api/v1/health` return near-identical JSON. Merge or clearly differentiate with docs. | ### v0.6.5 — Renderer Pipeline + Docs Rewrite diff --git a/VERSION b/VERSION index 844f6a9..d2b13eb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.3 +0.6.4 diff --git a/packages/cluster-dashboard/css/main.css b/packages/cluster-dashboard/css/main.css deleted file mode 100644 index f4cda2b..0000000 --- a/packages/cluster-dashboard/css/main.css +++ /dev/null @@ -1,100 +0,0 @@ -.cluster-dashboard { - max-width: 1200px; - margin: 0 auto; - padding: 2rem 1.5rem; -} - -.cluster-header { - display: flex; - align-items: center; - gap: 1rem; - margin-bottom: 1.5rem; -} - -.cluster-header h1 { - margin: 0; - font-size: 1.5rem; - color: var(--text-primary, #111); -} - -.cluster-status { - font-size: 0.85rem; - color: var(--text-secondary, #666); - background: var(--bg-secondary, #f3f4f6); - padding: 0.25rem 0.75rem; - border-radius: 999px; -} - -.cluster-grid { - display: grid; - grid-template-columns: repeat(auto-fill, minmax(320px, 1fr)); - gap: 1rem; -} - -.cluster-card { - background: var(--bg-primary, #fff); - border: 1px solid var(--border, #e5e7eb); - border-radius: 8px; - padding: 1.25rem; -} - -.card-header { - display: flex; - justify-content: space-between; - align-items: center; - margin-bottom: 0.5rem; -} - -.node-id { - font-weight: 600; - font-size: 0.9rem; - font-family: var(--font-mono, monospace); - color: var(--text-primary, #111); -} - -.heartbeat-badge { - font-size: 0.75rem; - color: var(--text-secondary, #666); - background: var(--bg-secondary, #f3f4f6); - padding: 0.15rem 0.5rem; - border-radius: 4px; -} - -.node-endpoint { - font-size: 0.8rem; - color: var(--text-secondary, #666); - margin-bottom: 0.75rem; - font-family: var(--font-mono, monospace); -} - -.card-stats { - display: grid; - grid-template-columns: 1fr 1fr 1fr; - gap: 0.5rem; -} - -.stat { - display: flex; - flex-direction: column; - gap: 0.15rem; -} - -.stat-label { - font-size: 0.7rem; - text-transform: uppercase; - letter-spacing: 0.03em; - color: var(--text-secondary, #999); -} - -.stat-value { - font-size: 0.9rem; - font-weight: 500; - font-family: var(--font-mono, monospace); - color: var(--text-primary, #111); -} - -.cluster-empty { - color: var(--text-secondary, #666); - text-align: center; - padding: 3rem 1rem; -} diff --git a/packages/cluster-dashboard/js/main.js b/packages/cluster-dashboard/js/main.js deleted file mode 100644 index efde770..0000000 --- a/packages/cluster-dashboard/js/main.js +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Cluster Dashboard — admin surface showing registered cluster nodes. - * - * Polls GET /api/v1/admin/cluster every 10s. Renders one card per node - * with runtime stats from the JSONB stats column. New keys added to - * stats are rendered automatically — no code change required. - */ -(function () { - 'use strict'; - - var mount = document.getElementById('extension-mount'); - if (!mount) return; - - var REFRESH_MS = 10000; - var timer = null; - - mount.innerHTML = - '
' + - '
' + - '

Cluster

' + - 'loading...' + - '
' + - '
' + - '
'; - - load(); - timer = setInterval(load, REFRESH_MS); - - // Cleanup on navigation (SPA) - window.addEventListener('beforeunload', function () { - if (timer) clearInterval(timer); - }); - - function load() { - var api = (window.sw && window.sw.api) || (typeof API !== 'undefined' && API); - if (!api) return; - var get = api._get || api.get; - if (!get) return; - get.call(api, '/api/v1/admin/cluster').then(render).catch(function () { - // 404 = SQLite (no cluster route registered), or other fetch error - render({ data: [] }); - }); - } - - function render(resp) { - var nodes = (resp && resp.data) || []; - var status = document.getElementById('clusterStatus'); - var grid = document.getElementById('clusterGrid'); - - status.textContent = nodes.length + ' node' + (nodes.length !== 1 ? 's' : '') + ' registered'; - - if (nodes.length === 0) { - grid.innerHTML = '

No nodes registered. The cluster registry requires PostgreSQL — SQLite runs single-node only.

'; - return; - } - - grid.innerHTML = nodes.map(nodeCard).join(''); - } - - function nodeCard(node) { - var stats = {}; - try { stats = typeof node.stats === 'string' ? JSON.parse(node.stats) : (node.stats || {}); } catch (e) { /* ignore */ } - - var uptimeSec = stats.uptime_sec || 0; - var uptime = formatDuration(uptimeSec); - var heap = formatBytes(stats.heap_alloc || 0); - var gcPause = ((stats.gc_pause_ns || 0) / 1e6).toFixed(1) + ' ms'; - var wsClients = stats.ws_clients != null ? stats.ws_clients : '-'; - var goroutines = stats.goroutines != null ? stats.goroutines : '-'; - var gcCycles = stats.gc_cycles != null ? stats.gc_cycles : '-'; - var heartbeatAge = timeSince(node.heartbeat); - - return ( - '
' + - '
' + - '' + esc(node.node_id) + '' + - '' + heartbeatAge + '' + - '
' + - (node.endpoint ? '
' + esc(node.endpoint) + '
' : '') + - '
' + - stat('Uptime', uptime) + - stat('WS Clients', wsClients) + - stat('Goroutines', goroutines) + - stat('Heap', heap) + - stat('GC Pause', gcPause) + - stat('GC Cycles', gcCycles) + - '
' + - '
' - ); - } - - function stat(label, value) { - return '
' + label + '' + value + '
'; - } - - function formatBytes(b) { - if (b < 1024) return b + ' B'; - if (b < 1024 * 1024) return (b / 1024).toFixed(1) + ' KB'; - return (b / (1024 * 1024)).toFixed(1) + ' MB'; - } - - function formatDuration(sec) { - if (sec < 60) return Math.floor(sec) + 's'; - if (sec < 3600) return Math.floor(sec / 60) + 'm ' + Math.floor(sec % 60) + 's'; - var h = Math.floor(sec / 3600); - var m = Math.floor((sec % 3600) / 60); - return h + 'h ' + m + 'm'; - } - - function timeSince(iso) { - if (!iso) return '-'; - var ms = Date.now() - new Date(iso).getTime(); - if (ms < 0) ms = 0; - var sec = Math.floor(ms / 1000); - if (sec < 60) return sec + 's ago'; - return Math.floor(sec / 60) + 'm ago'; - } - - function esc(s) { - var d = document.createElement('div'); - d.appendChild(document.createTextNode(s || '')); - return d.innerHTML; - } -})(); diff --git a/packages/cluster-dashboard/manifest.json b/packages/cluster-dashboard/manifest.json deleted file mode 100644 index 003541b..0000000 --- a/packages/cluster-dashboard/manifest.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "id": "cluster-dashboard", - "icon": "🔗", - "type": "surface", - "title": "Cluster", - "route": "/s/cluster-dashboard", - "auth": "admin", - "layout": "single", - "components": [], - "hooks": ["surface"], - "version": "0.6.0", - "description": "Cluster node registry — health, runtime stats, and peer visibility." -} diff --git a/packages/csv-table/manifest.json b/packages/csv-table/manifest.json index aa520fd..2597cef 100644 --- a/packages/csv-table/manifest.json +++ b/packages/csv-table/manifest.json @@ -6,7 +6,7 @@ "tier": "browser", "author": "switchboard", "description": "Renders ```csv code blocks as sortable, interactive HTML tables", - "requires": ["chat"], + "requires": [], "permissions": [], "tools": [], "surfaces": [], diff --git a/packages/diff-viewer/manifest.json b/packages/diff-viewer/manifest.json index e8fea94..e3b8f84 100644 --- a/packages/diff-viewer/manifest.json +++ b/packages/diff-viewer/manifest.json @@ -6,7 +6,7 @@ "tier": "browser", "author": "switchboard", "description": "Renders ```diff code blocks with syntax-highlighted additions, deletions, and context lines", - "requires": ["chat"], + "requires": [], "permissions": [], "tools": [], "surfaces": [], diff --git a/packages/katex-renderer/manifest.json b/packages/katex-renderer/manifest.json index fb1420c..02c926d 100644 --- a/packages/katex-renderer/manifest.json +++ b/packages/katex-renderer/manifest.json @@ -6,7 +6,7 @@ "tier": "browser", "author": "switchboard", "description": "Renders LaTeX math expressions: ```latex blocks and inline $...$ / $$...$$ syntax", - "requires": ["chat"], + "requires": [], "permissions": [], "tools": [], "surfaces": [], diff --git a/packages/mermaid-renderer/manifest.json b/packages/mermaid-renderer/manifest.json index 7353dbc..15b8d83 100644 --- a/packages/mermaid-renderer/manifest.json +++ b/packages/mermaid-renderer/manifest.json @@ -6,7 +6,7 @@ "tier": "browser", "author": "switchboard", "description": "Renders ```mermaid code blocks as interactive SVG diagrams with zoom/pan, SVG/PNG export, and source copy", - "requires": ["chat"], + "requires": [], "permissions": [], "tools": [], "surfaces": [], diff --git a/server/cluster/registry.go b/server/cluster/registry.go index 9d236ed..453cb23 100644 --- a/server/cluster/registry.go +++ b/server/cluster/registry.go @@ -34,6 +34,26 @@ type Registry struct { startTime time.Time stopCh chan struct{} wg sync.WaitGroup + + // Optional callbacks — set after construction via setters. + sandboxStats func() (exec, errors uint64, avgMs float64) + triggerFires func() int64 + extensionCount func() int +} + +// SetSandboxStats registers a callback for sandbox execution counters. +func (r *Registry) SetSandboxStats(fn func() (uint64, uint64, float64)) { + r.sandboxStats = fn +} + +// SetTriggerFireCount registers a callback for trigger fire count. +func (r *Registry) SetTriggerFireCount(fn func() int64) { + r.triggerFires = fn +} + +// SetExtensionCount registers a callback for active extension count. +func (r *Registry) SetExtensionCount(fn func() int) { + r.extensionCount = fn } // NewRegistry creates a cluster registry instance. @@ -129,13 +149,28 @@ func (r *Registry) collectStats() json.RawMessage { runtime.ReadMemStats(&m) stats := map[string]any{ - "goroutines": runtime.NumGoroutine(), - "heap_alloc": m.HeapAlloc, - "heap_sys": m.HeapSys, - "gc_cycles": m.NumGC, - "gc_pause_ns": m.PauseNs[(m.NumGC+255)%256], - "uptime_sec": time.Since(r.startTime).Seconds(), - "ws_clients": r.hub.ConnCount(), + "goroutines": runtime.NumGoroutine(), + "heap_alloc": m.HeapAlloc, + "heap_sys": m.HeapSys, + "stack_in_use": m.StackInuse, + "gc_cycles": m.NumGC, + "gc_pause_ns": m.PauseNs[(m.NumGC+255)%256], + "gc_cpu_pct": m.GCCPUFraction * 100, + "uptime_sec": time.Since(r.startTime).Seconds(), + "ws_clients": r.hub.ConnCount(), + } + + if r.extensionCount != nil { + stats["extensions_loaded"] = r.extensionCount() + } + if r.sandboxStats != nil { + exec, errors, avgMs := r.sandboxStats() + stats["starlark_exec_total"] = exec + stats["starlark_errors_total"] = errors + stats["starlark_avg_duration_ms"] = avgMs + } + if r.triggerFires != nil { + stats["trigger_fires_total"] = r.triggerFires() } data, _ := json.Marshal(stats) diff --git a/server/cluster/registry_test.go b/server/cluster/registry_test.go index 26d30d4..8f41ec0 100644 --- a/server/cluster/registry_test.go +++ b/server/cluster/registry_test.go @@ -75,6 +75,46 @@ func TestCollectStats(t *testing.T) { } } +func TestCollectStatsFattened(t *testing.T) { + hub := &mockHub{count: 3} + reg := NewRegistry("test-node", "", nil, hub, RegistryConfig{ + HeartbeatInterval: 10 * time.Second, + StaleThreshold: 30 * time.Second, + }) + reg.SetSandboxStats(func() (uint64, uint64, float64) { return 100, 5, 12.3 }) + reg.SetTriggerFireCount(func() int64 { return 42 }) + reg.SetExtensionCount(func() int { return 8 }) + + data := reg.collectStats() + + var stats map[string]any + if err := json.Unmarshal(data, &stats); err != nil { + t.Fatalf("collectStats returned invalid JSON: %v", err) + } + + // New fattened keys + fatKeys := []string{ + "stack_in_use", "gc_cpu_pct", + "extensions_loaded", "starlark_exec_total", "starlark_errors_total", + "starlark_avg_duration_ms", "trigger_fires_total", + } + for _, key := range fatKeys { + if _, ok := stats[key]; !ok { + t.Errorf("missing fattened stats key: %s", key) + } + } + + if v := stats["extensions_loaded"].(float64); int(v) != 8 { + t.Errorf("extensions_loaded = %v, want 8", v) + } + if v := stats["starlark_exec_total"].(float64); int(v) != 100 { + t.Errorf("starlark_exec_total = %v, want 100", v) + } + if v := stats["trigger_fires_total"].(float64); int(v) != 42 { + t.Errorf("trigger_fires_total = %v, want 42", v) + } +} + func TestRegistryStartStop(t *testing.T) { ms := &mockClusterStore{heartbeatRows: 1} hub := &mockHub{count: 0} diff --git a/server/events/bus.go b/server/events/bus.go index bb47ffa..1f868a8 100644 --- a/server/events/bus.go +++ b/server/events/bus.go @@ -3,6 +3,7 @@ package events import ( "strings" "sync" + "sync/atomic" "time" ) @@ -14,6 +15,8 @@ type Bus struct { subs map[string][]*subscription seq uint64 // subscription ID counter broadcastHook func(Event) // called after Publish for cross-pod fan-out; nil-safe + publishCount atomic.Int64 + deliverCount atomic.Int64 } type subscription struct { @@ -86,6 +89,8 @@ func (b *Bus) Publish(event Event) { // the broadcastHook. Used by the Postgres listener to re-publish remote // events without causing an infinite re-broadcast loop. func (b *Bus) publishLocal(event Event) { + b.publishCount.Add(1) + b.mu.RLock() var matched []Handler for pattern, subs := range b.subs { @@ -98,6 +103,7 @@ func (b *Bus) publishLocal(event Event) { b.mu.RUnlock() for _, h := range matched { + b.deliverCount.Add(1) h(event) } } @@ -126,6 +132,12 @@ func (b *Bus) PublishAsync(event Event) { } } +// PublishCount returns the cumulative number of events published. +func (b *Bus) PublishCount() int64 { return b.publishCount.Load() } + +// DeliverCount returns the cumulative number of subscriber deliveries. +func (b *Bus) DeliverCount() int64 { return b.deliverCount.Load() } + // match checks if a concrete label matches a subscription pattern. // // "chat.message.abc" matches "chat.message.abc" (exact) diff --git a/server/handlers/admin_metrics.go b/server/handlers/admin_metrics.go new file mode 100644 index 0000000..435401e --- /dev/null +++ b/server/handlers/admin_metrics.go @@ -0,0 +1,24 @@ +package handlers + +import ( + "github.com/gin-gonic/gin" + + "switchboard-core/metrics" +) + +// MetricsHandler serves the admin metrics endpoint. +type MetricsHandler struct { + collector *metrics.Collector +} + +// NewMetricsHandler creates a MetricsHandler. +func NewMetricsHandler(c *metrics.Collector) *MetricsHandler { + return &MetricsHandler{collector: c} +} + +// GetMetrics returns a full metrics snapshot. +// GET /api/v1/admin/metrics +func (h *MetricsHandler) GetMetrics(c *gin.Context) { + snap := h.collector.Collect(c.Request.Context()) + c.JSON(200, snap) +} diff --git a/server/handlers/admin_metrics_test.go b/server/handlers/admin_metrics_test.go new file mode 100644 index 0000000..268efa4 --- /dev/null +++ b/server/handlers/admin_metrics_test.go @@ -0,0 +1,179 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gin-gonic/gin" + + "switchboard-core/metrics" + "switchboard-core/store" +) + +// mockHub implements metrics.ConnCounter for tests. +type mockHub struct{ count int } + +func (m *mockHub) ConnCount() int { return m.count } + +// mockBus implements metrics.BusCounter for tests. +type mockBus struct{ pub, del int64 } + +func (m *mockBus) PublishCount() int64 { return m.pub } +func (m *mockBus) DeliverCount() int64 { return m.del } + +func TestMetrics_SQLiteShape(t *testing.T) { + gin.SetMode(gin.TestMode) + + collector := metrics.NewCollector( + nil, // no DB + &mockHub{count: 5}, + &mockBus{pub: 10, del: 20}, + store.Stores{}, // no cluster store + func() (uint64, uint64, float64) { return 42, 3, 12.5 }, + func() int64 { return 7 }, + time.Now().Add(-10*time.Minute), + ) + + h := NewMetricsHandler(collector) + r := gin.New() + r.GET("/api/v1/admin/metrics", h.GetMetrics) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/metrics", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != 200 { + t.Fatalf("status = %d, want 200", w.Code) + } + + var snap metrics.Snapshot + if err := json.Unmarshal(w.Body.Bytes(), &snap); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + // Runtime + if snap.Runtime.WSClients != 5 { + t.Errorf("ws_clients = %d, want 5", snap.Runtime.WSClients) + } + if snap.Runtime.UptimeSec < 600 { + t.Errorf("uptime_sec = %f, want >= 600", snap.Runtime.UptimeSec) + } + + // Cluster should be nil (SQLite) + if snap.Cluster != nil { + t.Errorf("cluster should be nil for SQLite, got %+v", snap.Cluster) + } + + // Extensions + if snap.Extensions.StarlarkExecTotal != 42 { + t.Errorf("starlark_exec_total = %d, want 42", snap.Extensions.StarlarkExecTotal) + } + if snap.Extensions.StarlarkErrorsTotal != 3 { + t.Errorf("starlark_errors_total = %d, want 3", snap.Extensions.StarlarkErrorsTotal) + } + if snap.Extensions.StarlarkAvgDuration != 12.5 { + t.Errorf("starlark_avg_duration_ms = %f, want 12.5", snap.Extensions.StarlarkAvgDuration) + } + if snap.Extensions.TriggerFiresTotal != 7 { + t.Errorf("trigger_fires_total = %d, want 7", snap.Extensions.TriggerFiresTotal) + } + if snap.Extensions.EventBusPublished != 10 { + t.Errorf("event_bus_published = %d, want 10", snap.Extensions.EventBusPublished) + } + if snap.Extensions.EventBusDelivered != 20 { + t.Errorf("event_bus_delivered = %d, want 20", snap.Extensions.EventBusDelivered) + } +} + +func TestMetrics_WithCluster(t *testing.T) { + gin.SetMode(gin.TestMode) + + mock := &mockClusterStore{ + nodes: []store.ClusterNode{ + { + NodeID: "node-1", + Endpoint: "http://node-1:8080", + Heartbeat: time.Now(), + Stats: json.RawMessage(`{"uptime_sec":120,"ws_clients":3}`), + }, + }, + } + + collector := metrics.NewCollector( + nil, + &mockHub{count: 3}, + &mockBus{}, + store.Stores{Cluster: mock}, + func() (uint64, uint64, float64) { return 0, 0, 0 }, + func() int64 { return 0 }, + time.Now(), + ) + + h := NewMetricsHandler(collector) + r := gin.New() + r.GET("/api/v1/admin/metrics", h.GetMetrics) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/metrics", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + if w.Code != 200 { + t.Fatalf("status = %d, want 200", w.Code) + } + + var snap metrics.Snapshot + if err := json.Unmarshal(w.Body.Bytes(), &snap); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if snap.Cluster == nil { + t.Fatal("cluster should not be nil with cluster store") + } + if snap.Cluster.Size != 1 { + t.Errorf("cluster.size = %d, want 1", snap.Cluster.Size) + } + if snap.Cluster.Nodes[0].NodeID != "node-1" { + t.Errorf("node_id = %q, want %q", snap.Cluster.Nodes[0].NodeID, "node-1") + } + if snap.Cluster.Nodes[0].UptimeSec != 120 { + t.Errorf("uptime_sec = %f, want 120", snap.Cluster.Nodes[0].UptimeSec) + } +} + +func TestMetrics_ExtensionCounters(t *testing.T) { + gin.SetMode(gin.TestMode) + + collector := metrics.NewCollector( + nil, + &mockHub{}, + &mockBus{pub: 100, del: 500}, + store.Stores{}, + func() (uint64, uint64, float64) { return 1000, 50, 8.3 }, + func() int64 { return 25 }, + time.Now(), + ) + + h := NewMetricsHandler(collector) + r := gin.New() + r.GET("/api/v1/admin/metrics", h.GetMetrics) + + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/metrics", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + + var snap metrics.Snapshot + _ = json.Unmarshal(w.Body.Bytes(), &snap) + + if snap.Extensions.StarlarkExecTotal != 1000 { + t.Errorf("exec total = %d, want 1000", snap.Extensions.StarlarkExecTotal) + } + if snap.Extensions.EventBusPublished != 100 { + t.Errorf("bus published = %d, want 100", snap.Extensions.EventBusPublished) + } + if snap.Extensions.EventBusDelivered != 500 { + t.Errorf("bus delivered = %d, want 500", snap.Extensions.EventBusDelivered) + } +} diff --git a/server/main.go b/server/main.go index 455d477..1a01ece 100644 --- a/server/main.go +++ b/server/main.go @@ -56,6 +56,7 @@ func main() { } // ── Server startup ────────────────────── + startTime := time.Now() cfg := config.Load() // Structured logging — must be first so all subsequent @@ -213,6 +214,24 @@ func main() { HeartbeatInterval: cfg.ClusterHeartbeatInterval, StaleThreshold: cfg.ClusterStaleThreshold, }) + clusterReg.SetSandboxStats(sandbox.SandboxStats) + clusterReg.SetTriggerFireCount(triggerEngine.FireCount) + clusterReg.SetExtensionCount(func() int { + if stores.Packages == nil { + return 0 + } + pkgs, err := stores.Packages.List(context.Background()) + if err != nil { + return 0 + } + count := 0 + for _, p := range pkgs { + if p.Status == "active" { + count++ + } + } + return count + }) if err := clusterReg.Start(); err != nil { log.Printf("⚠ Cluster registry failed to start: %v", err) clusterReg = nil @@ -255,7 +274,7 @@ func main() { } // Health check (k8s probes hit this directly) - base.GET("/health", func(c *gin.Context) { + buildHealthResponse := func() gin.H { info := gin.H{ "status": "ok", "version": Version, @@ -263,8 +282,15 @@ func main() { "database_name": database.Name(), "schema_version": database.SchemaVersion(), } + if database.IsConnected() { + info["registration_enabled"] = handlers.IsRegistrationEnabled(stores) + } appendClusterHealth(info, clusterReg, stores) - c.JSON(200, info) + return info + } + + base.GET("/health", func(c *gin.Context) { + c.JSON(200, buildHealthResponse()) }) // Liveness: process is alive and serving (no dependency checks). @@ -352,20 +378,9 @@ func main() { api := base.Group("/api/v1") { - // Health (routable through ingress) + // Health (routable through ingress — same shape as /health) api.GET("/health", func(c *gin.Context) { - info := gin.H{ - "status": "ok", - "version": Version, - "schema_version": database.SchemaVersion(), - "database": database.IsConnected(), - "database_name": database.Name(), - } - if database.IsConnected() { - info["registration_enabled"] = handlers.IsRegistrationEnabled(stores) - } - appendClusterHealth(info, clusterReg, stores) - c.JSON(200, info) + c.JSON(200, buildHealthResponse()) }) authGroup := api.Group("/auth") @@ -794,6 +809,16 @@ func main() { admin.GET("/cluster", clusterH.ListNodes) } + // ── Metrics ───────────────── + metricsCollector := metrics.NewCollector( + database.DB, hub, bus, stores, + sandbox.SandboxStats, + triggerEngine.FireCount, + startTime, + ) + metricsH := handlers.NewMetricsHandler(metricsCollector) + admin.GET("/metrics", metricsH.GetMetrics) + // ── Backup/Restore ───────── backupH := handlers.NewBackupHandler(stores, packagesDir, cfg.StoragePath) admin.POST("/backup", backupH.CreateBackup) diff --git a/server/metrics/collector.go b/server/metrics/collector.go new file mode 100644 index 0000000..e7896d6 --- /dev/null +++ b/server/metrics/collector.go @@ -0,0 +1,253 @@ +// Package metrics — collector.go +// +// On-demand metrics collector for the admin /api/v1/admin/metrics endpoint. +// Gathers runtime, database, cluster, and extension stats into a single JSON snapshot. +package metrics + +import ( + "context" + "database/sql" + "encoding/json" + "os" + "runtime" + "time" + + "switchboard-core/database" + "switchboard-core/store" +) + +// Snapshot is the top-level JSON response from GET /api/v1/admin/metrics. +type Snapshot struct { + Runtime RuntimeMetrics `json:"runtime"` + DB DBMetrics `json:"db"` + Cluster *ClusterMetrics `json:"cluster,omitempty"` + Extensions ExtensionMetrics `json:"extensions"` +} + +type RuntimeMetrics struct { + Goroutines int `json:"goroutines"` + HeapAlloc uint64 `json:"heap_alloc"` + HeapSys uint64 `json:"heap_sys"` + StackInUse uint64 `json:"stack_in_use"` + GCCycles uint32 `json:"gc_cycles"` + GCPauseNs uint64 `json:"gc_pause_ns"` + GCCPUPercent float64 `json:"gc_cpu_pct"` + UptimeSec float64 `json:"uptime_sec"` + WSClients int `json:"ws_clients"` + ExtensionsLoaded int `json:"extensions_loaded"` + OpenFDs int `json:"open_fds"` +} + +type DBMetrics struct { + LatencyMs float64 `json:"latency_ms"` + PoolActive int `json:"pool_active"` + PoolIdle int `json:"pool_idle"` + PoolMax int `json:"pool_max"` + WaitCount int64 `json:"wait_count"` + WaitDuration float64 `json:"wait_duration_ms"` + // PG-only fields (zero/omitted on SQLite) + DeadTuples *int64 `json:"dead_tuples,omitempty"` + ActiveBackends *int `json:"active_backends,omitempty"` +} + +type ClusterMetrics struct { + Size int `json:"size"` + Nodes []ClusterNode `json:"nodes"` +} + +type ClusterNode struct { + NodeID string `json:"node_id"` + Endpoint string `json:"endpoint"` + UptimeSec float64 `json:"uptime_sec"` + HeartbeatAge int64 `json:"heartbeat_age_ms"` + Stats json.RawMessage `json:"stats"` +} + +type ExtensionMetrics struct { + StarlarkExecTotal uint64 `json:"starlark_exec_total"` + StarlarkErrorsTotal uint64 `json:"starlark_errors_total"` + StarlarkAvgDuration float64 `json:"starlark_avg_duration_ms"` + TriggerFiresTotal int64 `json:"trigger_fires_total"` + EventBusPublished int64 `json:"event_bus_published"` + EventBusDelivered int64 `json:"event_bus_delivered"` +} + +// ConnCounter provides WebSocket connection count (satisfied by events.Hub). +type ConnCounter interface { + ConnCount() int +} + +// BusCounter provides publish/deliver counts (satisfied by events.Bus). +type BusCounter interface { + PublishCount() int64 + DeliverCount() int64 +} + +// SandboxStatsFunc returns cumulative sandbox execution counters. +type SandboxStatsFunc func() (execCount, errorCount uint64, avgDurationMs float64) + +// TriggerFireCountFunc returns cumulative trigger fire count. +type TriggerFireCountFunc func() int64 + +// Collector gathers metrics on demand for the admin endpoint. +type Collector struct { + db *sql.DB + hub ConnCounter + bus BusCounter + stores store.Stores + sandboxStats SandboxStatsFunc + triggerFireCount TriggerFireCountFunc + startTime time.Time +} + +// NewCollector creates a metrics collector with all required dependencies. +func NewCollector(db *sql.DB, hub ConnCounter, bus BusCounter, stores store.Stores, sandboxFn SandboxStatsFunc, triggerFn TriggerFireCountFunc, startTime time.Time) *Collector { + return &Collector{ + db: db, + hub: hub, + bus: bus, + stores: stores, + sandboxStats: sandboxFn, + triggerFireCount: triggerFn, + startTime: startTime, + } +} + +// Collect gathers all metrics synchronously and returns a snapshot. +func (c *Collector) Collect(ctx context.Context) *Snapshot { + snap := &Snapshot{} + snap.Runtime = c.collectRuntime() + snap.DB = c.collectDB(ctx) + snap.Cluster = c.collectCluster(ctx) + snap.Extensions = c.collectExtensions() + return snap +} + +func (c *Collector) collectRuntime() RuntimeMetrics { + var m runtime.MemStats + runtime.ReadMemStats(&m) + + extCount := 0 + if c.stores.Packages != nil { + if pkgs, err := c.stores.Packages.List(context.Background()); err == nil { + for _, p := range pkgs { + if p.Status == "active" { + extCount++ + } + } + } + } + + return RuntimeMetrics{ + Goroutines: runtime.NumGoroutine(), + HeapAlloc: m.HeapAlloc, + HeapSys: m.HeapSys, + StackInUse: m.StackInuse, + GCCycles: m.NumGC, + GCPauseNs: m.PauseNs[(m.NumGC+255)%256], + GCCPUPercent: m.GCCPUFraction * 100, + UptimeSec: time.Since(c.startTime).Seconds(), + WSClients: c.hub.ConnCount(), + ExtensionsLoaded: extCount, + OpenFDs: countOpenFDs(), + } +} + +func (c *Collector) collectDB(ctx context.Context) DBMetrics { + dm := DBMetrics{} + if c.db == nil { + return dm + } + + // Pool stats + stats := c.db.Stats() + dm.PoolActive = stats.InUse + dm.PoolIdle = stats.Idle + dm.PoolMax = stats.MaxOpenConnections + dm.WaitCount = stats.WaitCount + dm.WaitDuration = float64(stats.WaitDuration.Milliseconds()) + + // Latency probe + probeCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + start := time.Now() + if err := c.db.PingContext(probeCtx); err == nil { + dm.LatencyMs = float64(time.Since(start).Microseconds()) / 1000.0 + } + + // PG-only stats + if database.IsPostgres() { + var deadTuples int64 + if err := c.db.QueryRowContext(ctx, `SELECT COALESCE(SUM(n_dead_tup), 0) FROM pg_stat_user_tables`).Scan(&deadTuples); err == nil { + dm.DeadTuples = &deadTuples + } + var activeBackends int + if err := c.db.QueryRowContext(ctx, `SELECT count(*) FROM pg_stat_activity WHERE datname = current_database()`).Scan(&activeBackends); err == nil { + dm.ActiveBackends = &activeBackends + } + } + + return dm +} + +func (c *Collector) collectCluster(ctx context.Context) *ClusterMetrics { + if c.stores.Cluster == nil { + return nil + } + + nodes, err := c.stores.Cluster.ListNodes(ctx) + if err != nil { + return nil + } + if len(nodes) == 0 { + return nil + } + + cm := &ClusterMetrics{ + Size: len(nodes), + Nodes: make([]ClusterNode, len(nodes)), + } + now := time.Now() + for i, n := range nodes { + // Extract uptime from stats JSONB + var uptimeSec float64 + var statsMap map[string]any + if json.Unmarshal(n.Stats, &statsMap) == nil { + if u, ok := statsMap["uptime_sec"].(float64); ok { + uptimeSec = u + } + } + cm.Nodes[i] = ClusterNode{ + NodeID: n.NodeID, + Endpoint: n.Endpoint, + UptimeSec: uptimeSec, + HeartbeatAge: now.Sub(n.Heartbeat).Milliseconds(), + Stats: n.Stats, + } + } + return cm +} + +func (c *Collector) collectExtensions() ExtensionMetrics { + em := ExtensionMetrics{} + if c.sandboxStats != nil { + em.StarlarkExecTotal, em.StarlarkErrorsTotal, em.StarlarkAvgDuration = c.sandboxStats() + } + if c.triggerFireCount != nil { + em.TriggerFiresTotal = c.triggerFireCount() + } + if c.bus != nil { + em.EventBusPublished = c.bus.PublishCount() + em.EventBusDelivered = c.bus.DeliverCount() + } + return em +} + +// countOpenFDs counts open file descriptors via /proc/self/fd (Linux only). +func countOpenFDs() int { + entries, err := os.ReadDir("/proc/self/fd") + if err != nil { + return -1 + } + return len(entries) +} diff --git a/server/sandbox/runner.go b/server/sandbox/runner.go index eb39e98..08dfd25 100644 --- a/server/sandbox/runner.go +++ b/server/sandbox/runner.go @@ -23,16 +23,37 @@ import ( "os" "path/filepath" "strings" + "sync/atomic" + "time" "go.starlark.net/starlark" starlarkjson "go.starlark.net/lib/json" "switchboard-core/events" + "switchboard-core/metrics" "switchboard-core/models" "switchboard-core/store" ) +// sandboxStats tracks cumulative execution counters for the admin metrics endpoint. +var sandboxStats struct { + execCount atomic.Int64 + errorCount atomic.Int64 + totalDurationNs atomic.Int64 +} + +// SandboxStats returns cumulative execution counters. +func SandboxStats() (execCount, errorCount uint64, avgDurationMs float64) { + exec := sandboxStats.execCount.Load() + errs := sandboxStats.errorCount.Load() + totalNs := sandboxStats.totalDurationNs.Load() + if exec > 0 { + avgDurationMs = float64(totalNs) / float64(exec) / 1e6 + } + return uint64(exec), uint64(errs), avgDurationMs +} + // RunContext carries per-invocation state that modules need but which // varies per caller (API route vs filter vs task). Nil is safe — modules // that need RunContext fields gracefully degrade. @@ -142,7 +163,20 @@ func (r *Runner) ExecPackage(ctx context.Context, pkg *store.PackageRegistration log.Printf(" 🔧 runner: exec %s (%d modules granted)", pkg.ID, len(modules)) - return r.sandbox.ExecWithLoader(ctx, pkg.ID+"/script.star", script, modules, loader) + start := time.Now() + result, err := r.sandbox.ExecWithLoader(ctx, pkg.ID+"/script.star", script, modules, loader) + duration := time.Since(start) + + sandboxStats.execCount.Add(1) + sandboxStats.totalDurationNs.Add(int64(duration)) + status := "success" + if err != nil { + sandboxStats.errorCount.Add(1) + status = "error" + } + metrics.SandboxExecutionsTotal.WithLabelValues("exec", status).Inc() + + return result, err } // loadScript reads the entry point script from disk (primary path) @@ -266,8 +300,20 @@ func (r *Runner) CallEntryPoint(ctx context.Context, pkg *store.PackageRegistrat return nil, result.Output, fmt.Errorf("package %q: %s is not callable", pkg.ID, entryPoint) } - val, callOutput, err := r.sandbox.Call(ctx, callable, args, kwargs) - return val, result.Output + callOutput, err + start := time.Now() + val, callOutput, callErr := r.sandbox.Call(ctx, callable, args, kwargs) + duration := time.Since(start) + + sandboxStats.execCount.Add(1) + sandboxStats.totalDurationNs.Add(int64(duration)) + status := "success" + if callErr != nil { + sandboxStats.errorCount.Add(1) + status = "error" + } + metrics.SandboxExecutionsTotal.WithLabelValues(entryPoint, status).Inc() + + return val, result.Output + callOutput, callErr } // buildModules assembles the module map based on granted permissions. diff --git a/server/triggers/engine.go b/server/triggers/engine.go index 665f2c6..ececbc4 100644 --- a/server/triggers/engine.go +++ b/server/triggers/engine.go @@ -9,6 +9,7 @@ import ( "encoding/json" "log" "sync" + "sync/atomic" "time" "github.com/robfig/cron/v3" @@ -30,10 +31,15 @@ type Engine struct { unsubs map[string]func() // trigger_id → bus unsubscribe cronIDs map[string]cron.EntryID // scheduled_task_id → cron entry + fireCount atomic.Int64 // cumulative trigger fires + ctx context.Context cancel context.CancelFunc } +// FireCount returns the cumulative number of trigger fires. +func (e *Engine) FireCount() int64 { return e.fireCount.Load() } + // New creates a trigger engine. Call Start() to begin processing. func New(stores store.Stores, runner *sandbox.Runner, bus *events.Bus) *Engine { return &Engine{ @@ -201,8 +207,9 @@ func (e *Engine) logExecution(triggerID, scheduledTaskID string, firedAt time.Ti } } -// publishEvent emits a trigger lifecycle event on the bus. +// publishEvent emits a trigger lifecycle event on the bus and increments the fire counter. func (e *Engine) publishEvent(label string, triggerID string) { + e.fireCount.Add(1) if e.bus != nil { payload, _ := json.Marshal(map[string]any{"trigger_id": triggerID}) e.bus.Publish(events.Event{ diff --git a/src/js/sw/sdk/api-domains.js b/src/js/sw/sdk/api-domains.js index c863f8f..4ae3ade 100644 --- a/src/js/sw/sdk/api-domains.js +++ b/src/js/sw/sdk/api-domains.js @@ -163,6 +163,7 @@ export function createDomains(restClient) { // ── 18. Admin ────────────────────────── admin: { stats: () => rc.get('/api/v1/admin/stats'), + metrics: () => rc.get('/api/v1/admin/metrics'), users: { list: (opts) => rc.get('/api/v1/admin/users' + _qs(opts)), diff --git a/src/js/sw/surfaces/admin/health.js b/src/js/sw/surfaces/admin/health.js new file mode 100644 index 0000000..aedf441 --- /dev/null +++ b/src/js/sw/surfaces/admin/health.js @@ -0,0 +1,206 @@ +/** + * Admin > Monitoring > Health + * + * Polls GET /api/v1/admin/metrics and displays runtime, DB, cluster, + * and extension metrics. Cluster panel only shows on PG multi-node. + */ +const { html } = window; +const { useState, useEffect, useRef, useCallback } = hooks; + +// ── Formatting helpers ────────────────────────── + +function formatBytes(b) { + if (b == null) return '\u2014'; + if (b < 1024) return b + ' B'; + if (b < 1024 * 1024) return (b / 1024).toFixed(1) + ' KB'; + if (b < 1024 * 1024 * 1024) return (b / (1024 * 1024)).toFixed(1) + ' MB'; + return (b / (1024 * 1024 * 1024)).toFixed(2) + ' GB'; +} + +function formatDuration(sec) { + if (sec == null) return '\u2014'; + if (sec < 60) return Math.floor(sec) + 's'; + if (sec < 3600) return Math.floor(sec / 60) + 'm ' + Math.floor(sec % 60) + 's'; + const h = Math.floor(sec / 3600); + const m = Math.floor((sec % 3600) / 60); + if (h < 24) return h + 'h ' + m + 'm'; + const d = Math.floor(h / 24); + return d + 'd ' + (h % 24) + 'h'; +} + +function formatMs(ms) { + if (ms == null) return '\u2014'; + if (ms < 1) return '<1 ms'; + if (ms < 1000) return ms.toFixed(1) + ' ms'; + return (ms / 1000).toFixed(2) + ' s'; +} + +function formatPct(v) { + if (v == null) return '\u2014'; + return v.toFixed(2) + '%'; +} + +function formatNum(v) { + if (v == null) return '\u2014'; + if (typeof v === 'number') return v.toLocaleString(); + return String(v); +} + +// ── Stat card ─────────────────────────────────── + +function Stat({ label, value }) { + return html` +
+
${label}
+
${value}
+
+ `; +} + +// ── Panels ────────────────────────────────────── + +function PanelHeader({ title }) { + return html`

${title}

`; +} + +function RuntimePanel({ m }) { + return html` +
+ <${PanelHeader} title="Runtime" /> +
+ <${Stat} label="Uptime" value=${formatDuration(m.uptime_sec)} /> + <${Stat} label="Goroutines" value=${formatNum(m.goroutines)} /> + <${Stat} label="Heap Alloc" value=${formatBytes(m.heap_alloc)} /> + <${Stat} label="Heap Sys" value=${formatBytes(m.heap_sys)} /> + <${Stat} label="Stack In Use" value=${formatBytes(m.stack_in_use)} /> + <${Stat} label="GC Cycles" value=${formatNum(m.gc_cycles)} /> + <${Stat} label="Last GC Pause" value=${formatMs((m.gc_pause_ns || 0) / 1e6)} /> + <${Stat} label="GC CPU" value=${formatPct(m.gc_cpu_pct)} /> + <${Stat} label="WS Clients" value=${formatNum(m.ws_clients)} /> + <${Stat} label="Extensions" value=${formatNum(m.extensions_loaded)} /> + <${Stat} label="Open FDs" value=${m.open_fds >= 0 ? formatNum(m.open_fds) : '\u2014'} /> +
+
+ `; +} + +function DBPanel({ m }) { + return html` +
+ <${PanelHeader} title="Database" /> +
+ <${Stat} label="Latency" value=${formatMs(m.latency_ms)} /> + <${Stat} label="Pool Active" value=${formatNum(m.pool_active)} /> + <${Stat} label="Pool Idle" value=${formatNum(m.pool_idle)} /> + <${Stat} label="Pool Max" value=${formatNum(m.pool_max)} /> + <${Stat} label="Wait Count" value=${formatNum(m.wait_count)} /> + <${Stat} label="Wait Duration" value=${formatMs(m.wait_duration_ms)} /> + ${m.dead_tuples != null && html`<${Stat} label="Dead Tuples" value=${formatNum(m.dead_tuples)} />`} + ${m.active_backends != null && html`<${Stat} label="Active Backends" value=${formatNum(m.active_backends)} />`} +
+
+ `; +} + +function ClusterPanel({ cluster }) { + return html` +
+ <${PanelHeader} title=${'Cluster \u2014 ' + cluster.size + ' node' + (cluster.size !== 1 ? 's' : '')} /> +
+ ${cluster.nodes.map(n => html`<${NodeCard} key=${n.node_id} node=${n} />`)} +
+
+ `; +} + +function NodeCard({ node }) { + const stats = node.stats || {}; + const ageLabel = node.heartbeat_age_ms < 15000 + ? Math.floor(node.heartbeat_age_ms / 1000) + 's ago' + : Math.floor(node.heartbeat_age_ms / 60000) + 'm ago'; + const ageColor = node.heartbeat_age_ms < 15000 ? 'var(--accent, #2a6)' : 'var(--warning, #c80)'; + + return html` +
+
+ ${node.node_id} + ${ageLabel} +
+ ${node.endpoint && html`
${node.endpoint}
`} +
+ <${Stat} label="Uptime" value=${formatDuration(stats.uptime_sec)} /> + <${Stat} label="WS Clients" value=${formatNum(stats.ws_clients)} /> + <${Stat} label="Goroutines" value=${formatNum(stats.goroutines)} /> + <${Stat} label="Heap" value=${formatBytes(stats.heap_alloc)} /> + <${Stat} label="GC Pause" value=${formatMs((stats.gc_pause_ns || 0) / 1e6)} /> + <${Stat} label="GC Cycles" value=${formatNum(stats.gc_cycles)} /> +
+
+ `; +} + +function ExtensionPanel({ m }) { + return html` +
+ <${PanelHeader} title="Extensions" /> +
+ <${Stat} label="Starlark Execs" value=${formatNum(m.starlark_exec_total)} /> + <${Stat} label="Starlark Errors" value=${formatNum(m.starlark_errors_total)} /> + <${Stat} label="Avg Duration" value=${formatMs(m.starlark_avg_duration_ms)} /> + <${Stat} label="Trigger Fires" value=${formatNum(m.trigger_fires_total)} /> + <${Stat} label="Events Published" value=${formatNum(m.event_bus_published)} /> + <${Stat} label="Events Delivered" value=${formatNum(m.event_bus_delivered)} /> +
+
+ `; +} + +// ── Main component ────────────────────────────── + +export default function HealthSection() { + const [data, setData] = useState(null); + const [error, setError] = useState(null); + const [interval, setInterval_] = useState(10); + const timerRef = useRef(null); + + const load = useCallback(async () => { + try { + const resp = await sw.api.admin.metrics(); + setData(resp); + setError(null); + } catch (e) { + setError(e.message); + } + }, []); + + useEffect(() => { + load(); + timerRef.current = setInterval(load, interval * 1000); + return () => clearInterval(timerRef.current); + }, [load, interval]); + + if (error && !data) { + return html`
Failed to load metrics: ${error}
`; + } + if (!data) { + return html`
Loading metrics\u2026
`; + } + + return html` +
+
+ Refresh + +
+ <${RuntimePanel} m=${data.runtime} /> + <${DBPanel} m=${data.db} /> + ${data.cluster && html`<${ClusterPanel} cluster=${data.cluster} />`} + <${ExtensionPanel} m=${data.extensions} /> +
+ `; +} diff --git a/src/js/sw/surfaces/admin/index.js b/src/js/sw/surfaces/admin/index.js index 5ab1c5c..daedfbc 100644 --- a/src/js/sw/surfaces/admin/index.js +++ b/src/js/sw/surfaces/admin/index.js @@ -22,7 +22,7 @@ const ADMIN_SECTIONS = { people: ['users', 'teams', 'groups'], workflows: ['workflows'], system: ['settings', 'storage', 'packages', 'connections', 'broadcast', 'backup'], - monitoring: ['audit'], + monitoring: ['health', 'audit'], }; const ADMIN_LABELS = { @@ -31,7 +31,7 @@ const ADMIN_LABELS = { settings: 'Settings', storage: 'Storage', packages: 'Packages', connections: 'Connections', broadcast: 'Broadcast', backup: 'Backup', - audit: 'Audit', + health: 'Health', audit: 'Audit', }; // ── Extension config sections ────── @@ -52,7 +52,7 @@ const CATEGORY_META = { people: { label: 'People', first: 'users', icon: 'M17 21v-2a4 4 0 0 0-4-4H5a4 4 0 0 0-4 4v2|C9 7 4' }, workflows: { label: 'Workflows', first: 'workflows', icon: 'P16 3 21 3 21 8|L4 20 21 3|P21 16 21 21 16 21|L15 15 21 21|L4 4 9 9' }, system: { label: 'System', first: 'settings', icon: 'C12 12 3|M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1-2.83 2.83l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-4 0v-.09' }, - monitoring: { label: 'Monitoring', first: 'audit', icon: 'L18 20 18 10|L12 20 12 4|L6 20 6 14' }, + monitoring: { label: 'Monitoring', first: 'health', icon: 'L18 20 18 10|L12 20 12 4|L6 20 6 14' }, }; // ── Lazy section imports ──────────────────── @@ -69,6 +69,7 @@ const sectionModules = { connections: () => import(`./connections.js${_v}`), broadcast: () => import(`./broadcast.js${_v}`), backup: () => import(`./backup.js${_v}`), + health: () => import(`./health.js${_v}`), audit: () => import(`./audit.js${_v}`), }; -- 2.49.1 From 349ff5c80a7dd4b83122f0ca573e48c9de73862f Mon Sep 17 00:00:00 2001 From: Jeffrey Smith Date: Tue, 31 Mar 2026 13:40:18 +0000 Subject: [PATCH 2/2] Fix health tab: node identity indicator + cluster card overflow Add node_id to metrics snapshot so the health tab shows which instance the user is viewing from. Hoist nodeID computation so it works on both SQLite and Postgres deployments. Fix cluster node card overflow with text-overflow ellipsis and proper flex constraints. Co-Authored-By: Claude Opus 4.6 (1M context) --- server/handlers/admin_metrics_test.go | 8 ++++++++ server/main.go | 15 +++++++++------ server/metrics/collector.go | 7 +++++-- src/js/sw/surfaces/admin/health.js | 17 +++++++++-------- 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/server/handlers/admin_metrics_test.go b/server/handlers/admin_metrics_test.go index 268efa4..0285dbe 100644 --- a/server/handlers/admin_metrics_test.go +++ b/server/handlers/admin_metrics_test.go @@ -28,6 +28,7 @@ func TestMetrics_SQLiteShape(t *testing.T) { gin.SetMode(gin.TestMode) collector := metrics.NewCollector( + "test-node", nil, // no DB &mockHub{count: 5}, &mockBus{pub: 10, del: 20}, @@ -54,6 +55,11 @@ func TestMetrics_SQLiteShape(t *testing.T) { t.Fatalf("unmarshal: %v", err) } + // Node ID + if snap.NodeID != "test-node" { + t.Errorf("node_id = %q, want %q", snap.NodeID, "test-node") + } + // Runtime if snap.Runtime.WSClients != 5 { t.Errorf("ws_clients = %d, want 5", snap.Runtime.WSClients) @@ -103,6 +109,7 @@ func TestMetrics_WithCluster(t *testing.T) { } collector := metrics.NewCollector( + "test-node", nil, &mockHub{count: 3}, &mockBus{}, @@ -147,6 +154,7 @@ func TestMetrics_ExtensionCounters(t *testing.T) { gin.SetMode(gin.TestMode) collector := metrics.NewCollector( + "test-node", nil, &mockHub{}, &mockBus{pub: 100, del: 500}, diff --git a/server/main.go b/server/main.go index 1a01ece..150a85d 100644 --- a/server/main.go +++ b/server/main.go @@ -201,15 +201,18 @@ func main() { // ── WebSocket Hub ───────────────────────── hub := events.NewHub(bus, middleware.GetAllowedOrigins(cfg)) + // ── Node Identity ──────────────── + // Used by cluster registry (PG) and metrics endpoint (all deployments). + nodeID := cfg.ClusterNodeID + if nodeID == "" { + hostname, _ := os.Hostname() + nodeID = fmt.Sprintf("%s-%d", hostname, os.Getpid()) + } + // ── Cluster Registry ──────────── // PG-backed node self-registration + heartbeat. No-op on SQLite. var clusterReg *cluster.Registry if database.IsPostgres() && stores.Cluster != nil { - nodeID := cfg.ClusterNodeID - if nodeID == "" { - hostname, _ := os.Hostname() - nodeID = fmt.Sprintf("%s-%d", hostname, os.Getpid()) - } clusterReg = cluster.NewRegistry(nodeID, cfg.ClusterEndpoint, stores.Cluster, hub, cluster.RegistryConfig{ HeartbeatInterval: cfg.ClusterHeartbeatInterval, StaleThreshold: cfg.ClusterStaleThreshold, @@ -811,7 +814,7 @@ func main() { // ── Metrics ───────────────── metricsCollector := metrics.NewCollector( - database.DB, hub, bus, stores, + nodeID, database.DB, hub, bus, stores, sandbox.SandboxStats, triggerEngine.FireCount, startTime, diff --git a/server/metrics/collector.go b/server/metrics/collector.go index e7896d6..d9ffd03 100644 --- a/server/metrics/collector.go +++ b/server/metrics/collector.go @@ -18,6 +18,7 @@ import ( // Snapshot is the top-level JSON response from GET /api/v1/admin/metrics. type Snapshot struct { + NodeID string `json:"node_id"` Runtime RuntimeMetrics `json:"runtime"` DB DBMetrics `json:"db"` Cluster *ClusterMetrics `json:"cluster,omitempty"` @@ -91,6 +92,7 @@ type TriggerFireCountFunc func() int64 // Collector gathers metrics on demand for the admin endpoint. type Collector struct { + nodeID string db *sql.DB hub ConnCounter bus BusCounter @@ -101,8 +103,9 @@ type Collector struct { } // NewCollector creates a metrics collector with all required dependencies. -func NewCollector(db *sql.DB, hub ConnCounter, bus BusCounter, stores store.Stores, sandboxFn SandboxStatsFunc, triggerFn TriggerFireCountFunc, startTime time.Time) *Collector { +func NewCollector(nodeID string, db *sql.DB, hub ConnCounter, bus BusCounter, stores store.Stores, sandboxFn SandboxStatsFunc, triggerFn TriggerFireCountFunc, startTime time.Time) *Collector { return &Collector{ + nodeID: nodeID, db: db, hub: hub, bus: bus, @@ -115,7 +118,7 @@ func NewCollector(db *sql.DB, hub ConnCounter, bus BusCounter, stores store.Stor // Collect gathers all metrics synchronously and returns a snapshot. func (c *Collector) Collect(ctx context.Context) *Snapshot { - snap := &Snapshot{} + snap := &Snapshot{NodeID: c.nodeID} snap.Runtime = c.collectRuntime() snap.DB = c.collectDB(ctx) snap.Cluster = c.collectCluster(ctx) diff --git a/src/js/sw/surfaces/admin/health.js b/src/js/sw/surfaces/admin/health.js index aedf441..d340566 100644 --- a/src/js/sw/surfaces/admin/health.js +++ b/src/js/sw/surfaces/admin/health.js @@ -121,13 +121,13 @@ function NodeCard({ node }) { const ageColor = node.heartbeat_age_ms < 15000 ? 'var(--accent, #2a6)' : 'var(--warning, #c80)'; return html` -
-
- ${node.node_id} - ${ageLabel} +
+
+ ${node.node_id} + ${ageLabel}
- ${node.endpoint && html`
${node.endpoint}
`} -
+ ${node.endpoint && html`
${node.endpoint}
`} +
<${Stat} label="Uptime" value=${formatDuration(stats.uptime_sec)} /> <${Stat} label="WS Clients" value=${formatNum(stats.ws_clients)} /> <${Stat} label="Goroutines" value=${formatNum(stats.goroutines)} /> @@ -188,8 +188,9 @@ export default function HealthSection() { return html`
-
- Refresh +
+ ${data.node_id && html`Viewing from ${data.node_id}`} + Refresh