#!/usr/bin/env bash # e2e-cluster-test.sh — Cluster registry E2E test (v0.6.0) # # Requires: docker-compose-e2e.yml running with 3 replicas. # Usage: # docker compose -f docker-compose-e2e.yml up --build -d # ./ci/e2e-cluster-test.sh # docker compose -f docker-compose-e2e.yml down -v set -euo pipefail LB="http://localhost:3000" DIRECT_1="http://localhost:8081" STALE_WAIT=20 # seconds — must exceed CLUSTER_STALE_THRESHOLD (15s) PASS=0 FAIL=0 pass() { PASS=$((PASS + 1)); echo " PASS: $1"; } fail() { FAIL=$((FAIL + 1)); echo " FAIL: $1"; } # ── Wait for all 3 replicas ────────────────── echo "=== Waiting for replicas ===" for port in 8081 8082 8083; do for i in $(seq 1 30); do if curl -sf "http://localhost:$port/health" > /dev/null 2>&1; then echo " replica :$port ready" break fi sleep 1 if [ "$i" -eq 30 ]; then echo "FATAL: replica :$port did not become healthy" exit 1 fi done done # ── Login as admin ──────────────────────────── echo "=== Authenticating ===" TOKEN=$(curl -sf "$LB/api/v1/auth/login" \ -H "Content-Type: application/json" \ -d '{"login":"admin","password":"admin"}' | jq -r '.access_token') if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then echo "FATAL: login failed" exit 1 fi echo " admin token acquired" auth() { echo "Authorization: Bearer $TOKEN"; } # Wait for heartbeat to propagate (at least 2 tick cycles at 5s each) echo "=== Waiting for heartbeat convergence ===" sleep 12 # ── Test 1: All 3 nodes registered ─────────── echo "=== Test 1: Cluster shows 3 nodes ===" NODES=$(curl -sf "$LB/api/v1/admin/cluster" -H "$(auth)") COUNT=$(echo "$NODES" | jq '.data | length') if [ "$COUNT" -eq 3 ]; then pass "cluster has 3 nodes" else fail "cluster has $COUNT nodes (expected 3)" echo " response: $NODES" fi # Verify all expected node IDs present for nid in node-1 node-2 node-3; do if echo "$NODES" | jq -e ".data[] | select(.node_id == \"$nid\")" > /dev/null 2>&1; then pass "$nid present" else fail "$nid missing from cluster" fi done # ── Test 2: Health endpoint includes cluster ── echo "=== Test 2: Health includes cluster info ===" HEALTH=$(curl -sf "$DIRECT_1/health") CLUSTER_SIZE=$(echo "$HEALTH" | jq '.cluster.size // 0') if [ "$CLUSTER_SIZE" -eq 3 ]; then pass "health shows cluster.size=3" else fail "health shows cluster.size=$CLUSTER_SIZE (expected 3)" fi NODE_ID=$(echo "$HEALTH" | jq -r '.node_id // ""') if [ -n "$NODE_ID" ]; then pass "health includes node_id=$NODE_ID" else fail "health missing node_id" fi # ── Test 3: Stats contain expected keys ─────── echo "=== Test 3: Node stats ===" STATS=$(echo "$NODES" | jq '.data[0].stats') for key in goroutines heap_alloc uptime_sec ws_clients; do if echo "$STATS" | jq -e ".$key" > /dev/null 2>&1; then pass "stats.$key present" else fail "stats.$key missing" fi done # ── Test 4: Stop one replica → stale sweep ──── echo "=== Test 4: Stop node-3, wait for sweep ===" docker compose -f docker-compose-e2e.yml stop switchboard-3 echo " waiting ${STALE_WAIT}s for stale sweep..." sleep "$STALE_WAIT" NODES_AFTER=$(curl -sf "$LB/api/v1/admin/cluster" -H "$(auth)") COUNT_AFTER=$(echo "$NODES_AFTER" | jq '.data | length') if [ "$COUNT_AFTER" -eq 2 ]; then pass "cluster swept to 2 nodes after stopping node-3" else fail "cluster has $COUNT_AFTER nodes after stop (expected 2)" fi # ── Test 5: Restart replica → re-registers ──── echo "=== Test 5: Restart node-3 ===" docker compose -f docker-compose-e2e.yml start switchboard-3 # Wait for startup + at least one heartbeat for i in $(seq 1 30); do if curl -sf "http://localhost:8083/health" > /dev/null 2>&1; then break fi sleep 1 done sleep 8 # allow heartbeat to register NODES_RESTART=$(curl -sf "$LB/api/v1/admin/cluster" -H "$(auth)") COUNT_RESTART=$(echo "$NODES_RESTART" | jq '.data | length') if [ "$COUNT_RESTART" -eq 3 ]; then pass "cluster back to 3 nodes after restart" else fail "cluster has $COUNT_RESTART nodes after restart (expected 3)" fi # ── Summary ─────────────────────────────────── echo "" echo "=== Results: $PASS passed, $FAIL failed ===" if [ "$FAIL" -gt 0 ]; then exit 1 fi