Jeffrey Smith
35d2309450
All checks were successful
CI/CD / detect-changes (pull_request) Successful in 21s
CI/CD / test-frontend (pull_request) Successful in 5s
CI/CD / test-go-pg (pull_request) Successful in 2m58s
CI/CD / test-sqlite (pull_request) Successful in 3m2s
CI/CD / build-and-deploy (pull_request) Successful in 1m15s
PG-backed cluster registry for horizontal scaling — zero new infrastructure (no etcd/Consul/Redis). MVP convergence point. - node_registry UNLOGGED table (migration 013) - Self-registration, heartbeat tick (10s), stale sweep (30s), self-eviction (os.Exit on 0 rows → K8s restarts) - GET /api/v1/admin/cluster returns nodes with runtime stats - Health endpoints include node_id + cluster size/peers - cluster-dashboard admin surface with auto-refresh - 3-replica E2E test (docker-compose-e2e.yml + ci/e2e-cluster-test.sh) - chat + cluster-dashboard added to curated default bundle - 5 new tests (3 unit + 2 handler), all passing Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
146 lines
4.5 KiB
Bash
Executable File
146 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# e2e-cluster-test.sh — Cluster registry E2E test (v0.6.0)
|
|
#
|
|
# Requires: docker-compose-e2e.yml running with 3 replicas.
|
|
# Usage:
|
|
# docker compose -f docker-compose-e2e.yml up --build -d
|
|
# ./ci/e2e-cluster-test.sh
|
|
# docker compose -f docker-compose-e2e.yml down -v
|
|
|
|
set -euo pipefail
|
|
|
|
LB="http://localhost:3000"
|
|
DIRECT_1="http://localhost:8081"
|
|
STALE_WAIT=20 # seconds — must exceed CLUSTER_STALE_THRESHOLD (15s)
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
|
|
pass() { PASS=$((PASS + 1)); echo " PASS: $1"; }
|
|
fail() { FAIL=$((FAIL + 1)); echo " FAIL: $1"; }
|
|
|
|
# ── Wait for all 3 replicas ──────────────────
|
|
echo "=== Waiting for replicas ==="
|
|
for port in 8081 8082 8083; do
|
|
for i in $(seq 1 30); do
|
|
if curl -sf "http://localhost:$port/health" > /dev/null 2>&1; then
|
|
echo " replica :$port ready"
|
|
break
|
|
fi
|
|
sleep 1
|
|
if [ "$i" -eq 30 ]; then
|
|
echo "FATAL: replica :$port did not become healthy"
|
|
exit 1
|
|
fi
|
|
done
|
|
done
|
|
|
|
# ── Login as admin ────────────────────────────
|
|
echo "=== Authenticating ==="
|
|
TOKEN=$(curl -sf "$LB/api/v1/auth/login" \
|
|
-H "Content-Type: application/json" \
|
|
-d '{"login":"admin","password":"admin"}' | jq -r '.access_token')
|
|
|
|
if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then
|
|
echo "FATAL: login failed"
|
|
exit 1
|
|
fi
|
|
echo " admin token acquired"
|
|
|
|
auth() { echo "Authorization: Bearer $TOKEN"; }
|
|
|
|
# Wait for heartbeat to propagate (at least 2 tick cycles at 5s each)
|
|
echo "=== Waiting for heartbeat convergence ==="
|
|
sleep 12
|
|
|
|
# ── Test 1: All 3 nodes registered ───────────
|
|
echo "=== Test 1: Cluster shows 3 nodes ==="
|
|
NODES=$(curl -sf "$LB/api/v1/admin/cluster" -H "$(auth)")
|
|
COUNT=$(echo "$NODES" | jq '.data | length')
|
|
if [ "$COUNT" -eq 3 ]; then
|
|
pass "cluster has 3 nodes"
|
|
else
|
|
fail "cluster has $COUNT nodes (expected 3)"
|
|
echo " response: $NODES"
|
|
fi
|
|
|
|
# Verify all expected node IDs present
|
|
for nid in node-1 node-2 node-3; do
|
|
if echo "$NODES" | jq -e ".data[] | select(.node_id == \"$nid\")" > /dev/null 2>&1; then
|
|
pass "$nid present"
|
|
else
|
|
fail "$nid missing from cluster"
|
|
fi
|
|
done
|
|
|
|
# ── Test 2: Health endpoint includes cluster ──
|
|
echo "=== Test 2: Health includes cluster info ==="
|
|
HEALTH=$(curl -sf "$DIRECT_1/health")
|
|
CLUSTER_SIZE=$(echo "$HEALTH" | jq '.cluster.size // 0')
|
|
if [ "$CLUSTER_SIZE" -eq 3 ]; then
|
|
pass "health shows cluster.size=3"
|
|
else
|
|
fail "health shows cluster.size=$CLUSTER_SIZE (expected 3)"
|
|
fi
|
|
|
|
NODE_ID=$(echo "$HEALTH" | jq -r '.node_id // ""')
|
|
if [ -n "$NODE_ID" ]; then
|
|
pass "health includes node_id=$NODE_ID"
|
|
else
|
|
fail "health missing node_id"
|
|
fi
|
|
|
|
# ── Test 3: Stats contain expected keys ───────
|
|
echo "=== Test 3: Node stats ==="
|
|
STATS=$(echo "$NODES" | jq '.data[0].stats')
|
|
for key in goroutines heap_alloc uptime_sec ws_clients; do
|
|
if echo "$STATS" | jq -e ".$key" > /dev/null 2>&1; then
|
|
pass "stats.$key present"
|
|
else
|
|
fail "stats.$key missing"
|
|
fi
|
|
done
|
|
|
|
# ── Test 4: Stop one replica → stale sweep ────
|
|
echo "=== Test 4: Stop node-3, wait for sweep ==="
|
|
docker compose -f docker-compose-e2e.yml stop switchboard-3
|
|
|
|
echo " waiting ${STALE_WAIT}s for stale sweep..."
|
|
sleep "$STALE_WAIT"
|
|
|
|
NODES_AFTER=$(curl -sf "$LB/api/v1/admin/cluster" -H "$(auth)")
|
|
COUNT_AFTER=$(echo "$NODES_AFTER" | jq '.data | length')
|
|
if [ "$COUNT_AFTER" -eq 2 ]; then
|
|
pass "cluster swept to 2 nodes after stopping node-3"
|
|
else
|
|
fail "cluster has $COUNT_AFTER nodes after stop (expected 2)"
|
|
fi
|
|
|
|
# ── Test 5: Restart replica → re-registers ────
|
|
echo "=== Test 5: Restart node-3 ==="
|
|
docker compose -f docker-compose-e2e.yml start switchboard-3
|
|
|
|
# Wait for startup + at least one heartbeat
|
|
for i in $(seq 1 30); do
|
|
if curl -sf "http://localhost:8083/health" > /dev/null 2>&1; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
sleep 8 # allow heartbeat to register
|
|
|
|
NODES_RESTART=$(curl -sf "$LB/api/v1/admin/cluster" -H "$(auth)")
|
|
COUNT_RESTART=$(echo "$NODES_RESTART" | jq '.data | length')
|
|
if [ "$COUNT_RESTART" -eq 3 ]; then
|
|
pass "cluster back to 3 nodes after restart"
|
|
else
|
|
fail "cluster has $COUNT_RESTART nodes after restart (expected 3)"
|
|
fi
|
|
|
|
# ── Summary ───────────────────────────────────
|
|
echo ""
|
|
echo "=== Results: $PASS passed, $FAIL failed ==="
|
|
if [ "$FAIL" -gt 0 ]; then
|
|
exit 1
|
|
fi
|