Jeffrey Smith
f0dd43144e
- Rename Go module switchboard-core → armature (155+ files) - Rename Docker image → gobha/armature - Rename K8s resources, secrets, deployments - Rename Prometheus metrics switchboard_* → armature_* - Rename env vars SWITCHBOARD_ADMIN_* → ARMATURE_ADMIN_* - Rename DB names switchboard_core* → armature* - Update all frontend branding, notification templates, docs - Update CI scripts, e2e tests, Keycloak realm, nginx conf - Rename scripts/switchboard-ca.sh → scripts/armature-ca.sh - Rename k8s/switchboard.yaml → k8s/armature.yaml - Rename chart alerting/dashboard files - Fix: DockerHub push uses env: binding for secret injection - Helm chart updated (name, labels, template functions, dashboard, alerting) - Replace favicon/icon assets with Armature brand No functional changes. Pure mechanical rename + CI fix. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
331 lines
13 KiB
Bash
Executable File
331 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ═══════════════════════════════════════════════
|
|
# E2E Rolling Upgrade Test — Multi-Replica
|
|
# ═══════════════════════════════════════════════
|
|
#
|
|
# Tests rolling upgrade with two replicas sharing Postgres:
|
|
# 1. Build old + new images
|
|
# 2. Start 2 replicas (old) + nginx LB + postgres
|
|
# 3. Seed data via LB
|
|
# 4. Upgrade replica-1 → new, verify cross-replica reads
|
|
# 5. Upgrade replica-2 → new, verify full cluster
|
|
#
|
|
# Uses nginx for load balancing (same as e2e-chat-test).
|
|
#
|
|
# Usage:
|
|
# ./ci/e2e-upgrade-rolling.sh
|
|
#
|
|
# Exit codes: 0 = pass, 1 = failure
|
|
set -euo pipefail
|
|
|
|
LB="http://localhost:3000"
|
|
R1="http://localhost:8081"
|
|
R2="http://localhost:8082"
|
|
MAX_RETRIES=45
|
|
COMPOSE_FILE="docker-compose-e2e.yml"
|
|
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m'
|
|
|
|
pass=0
|
|
fail=0
|
|
|
|
ok() { pass=$((pass + 1)); echo -e " ${GREEN}✓${NC} $1"; }
|
|
fail() { fail=$((fail + 1)); echo -e " ${RED}✗${NC} $1"; }
|
|
|
|
cleanup() {
|
|
echo -e "\n${YELLOW}Cleanup...${NC}"
|
|
docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
wait_for_health() {
|
|
local host=$1 name=${2:-$host}
|
|
echo -e "${YELLOW}Waiting for $name...${NC}"
|
|
for i in $(seq 1 $MAX_RETRIES); do
|
|
if curl -sf "$host/api/v1/auth/login" -o /dev/null 2>/dev/null || \
|
|
curl -sf "$host" -o /dev/null 2>/dev/null; then
|
|
echo -e "${GREEN}$name ready after ${i}s${NC}"
|
|
return 0
|
|
fi
|
|
if [ "$i" -eq "$MAX_RETRIES" ]; then
|
|
echo -e "${RED}$name not ready after ${MAX_RETRIES}s${NC}"
|
|
return 1
|
|
fi
|
|
sleep 1
|
|
done
|
|
}
|
|
|
|
login() {
|
|
local user=$1 pass=$2 host=${3:-$LB}
|
|
local resp
|
|
resp=$(curl -sf "$host/api/v1/auth/login" \
|
|
-H 'Content-Type: application/json' \
|
|
-d "{\"login\":\"$user\",\"password\":\"$pass\"}")
|
|
echo "$resp" | grep -o '"access_token":"[^"]*"' | head -1 | sed 's/.*"access_token":"\([^"]*\)".*/\1/'
|
|
}
|
|
|
|
authed() {
|
|
local token=$1 method=$2 path=$3 host=${4:-$LB}
|
|
shift 3; shift 0 2>/dev/null || true
|
|
curl -sf -X "$method" "$host$path" \
|
|
-H "Authorization: Bearer $token" \
|
|
-H 'Content-Type: application/json' \
|
|
"$@"
|
|
}
|
|
|
|
# ═══════════════════════════════════════════════
|
|
# Phase 1: Build Images
|
|
# ═══════════════════════════════════════════════
|
|
|
|
echo -e "\n${YELLOW}═══ Phase 1: Build Images ═══${NC}"
|
|
|
|
echo "Building 'old' image from HEAD..."
|
|
docker build -t armature:v-old . -q
|
|
ok "old image built"
|
|
|
|
echo "Building 'new' image from working tree..."
|
|
docker build -t armature:v-new . -q
|
|
ok "new image built"
|
|
|
|
# ═══════════════════════════════════════════════
|
|
# Phase 2: Start Both Replicas on Old Image
|
|
# ═══════════════════════════════════════════════
|
|
|
|
echo -e "\n${YELLOW}═══ Phase 2: Start Cluster (Old Version) ═══${NC}"
|
|
|
|
# Override the build with old image
|
|
ARMATURE_IMAGE=armature:v-old \
|
|
docker compose -f "$COMPOSE_FILE" up postgres -d
|
|
|
|
# Wait for postgres
|
|
sleep 3
|
|
|
|
# Start replicas using old image
|
|
docker run -d --name sb-old-1 --network "$(basename "$(pwd)")_default" \
|
|
-e PORT=8080 -e BASE_PATH="" \
|
|
-e DB_DRIVER=postgres \
|
|
-e "DATABASE_URL=postgres://armature:e2e-password@postgres:5432/armature_e2e?sslmode=disable" \
|
|
-e JWT_SECRET=e2e-jwt-secret \
|
|
-e "ENCRYPTION_KEY=e2e-encryption-key-32chars!!!!!" \
|
|
-e ARMATURE_ADMIN_USERNAME=admin \
|
|
-e ARMATURE_ADMIN_PASSWORD=admin \
|
|
-e STORAGE_BACKEND=pvc -e STORAGE_PATH=/data/storage \
|
|
-e "CORS_ALLOWED_ORIGINS=*" -e EXT_ALLOW_PRIVATE_IPS=true \
|
|
-e LOG_FORMAT=text -e LOG_LEVEL=info \
|
|
-e "SEED_USERS=alice:password123:user,bob:password456:user" \
|
|
-p 8081:80 armature:v-old 2>/dev/null || true
|
|
|
|
docker run -d --name sb-old-2 --network "$(basename "$(pwd)")_default" \
|
|
-e PORT=8080 -e BASE_PATH="" \
|
|
-e DB_DRIVER=postgres \
|
|
-e "DATABASE_URL=postgres://armature:e2e-password@postgres:5432/armature_e2e?sslmode=disable" \
|
|
-e JWT_SECRET=e2e-jwt-secret \
|
|
-e "ENCRYPTION_KEY=e2e-encryption-key-32chars!!!!!" \
|
|
-e ARMATURE_ADMIN_USERNAME=admin \
|
|
-e ARMATURE_ADMIN_PASSWORD=admin \
|
|
-e STORAGE_BACKEND=pvc -e STORAGE_PATH=/data/storage \
|
|
-e "CORS_ALLOWED_ORIGINS=*" -e EXT_ALLOW_PRIVATE_IPS=true \
|
|
-e LOG_FORMAT=text -e LOG_LEVEL=info \
|
|
-e "SEED_USERS=alice:password123:user,bob:password456:user" \
|
|
-p 8082:80 armature:v-old 2>/dev/null || true
|
|
|
|
# Start nginx LB
|
|
docker compose -f "$COMPOSE_FILE" up lb -d
|
|
|
|
wait_for_health "$R1" "replica-1"
|
|
wait_for_health "$R2" "replica-2"
|
|
wait_for_health "$LB" "load balancer"
|
|
|
|
# ═══════════════════════════════════════════════
|
|
# Phase 3: Seed Data via LB
|
|
# ═══════════════════════════════════════════════
|
|
|
|
echo -e "\n${YELLOW}═══ Phase 3: Seed Data ═══${NC}"
|
|
|
|
ADMIN_TOKEN=$(login admin admin)
|
|
if [ -z "$ADMIN_TOKEN" ]; then fail "admin login failed"; exit 1; fi
|
|
ok "admin logged in"
|
|
|
|
ALICE_TOKEN=$(login alice password123)
|
|
if [ -z "$ALICE_TOKEN" ]; then fail "alice login failed"; exit 1; fi
|
|
|
|
BOB_TOKEN=$(login bob password456)
|
|
if [ -z "$BOB_TOKEN" ]; then fail "bob login failed"; exit 1; fi
|
|
|
|
ALICE_ID=$(authed "$ALICE_TOKEN" GET "/api/v1/profile" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4)
|
|
BOB_ID=$(authed "$BOB_TOKEN" GET "/api/v1/profile" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4)
|
|
|
|
# Seed a note
|
|
NOTE=$(authed "$ALICE_TOKEN" POST "/s/notes/api/notes" "" \
|
|
-d '{"title":"Rolling Upgrade Note","body":"Must survive rolling upgrade."}' 2>/dev/null || echo "{}")
|
|
NOTE_ID=$(echo "$NOTE" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4)
|
|
if [ -n "$NOTE_ID" ]; then ok "note seeded"; else fail "note seed failed"; fi
|
|
|
|
# Seed a conversation + message
|
|
CONV=$(authed "$ALICE_TOKEN" POST "/s/chat-core/api/conversations" "" \
|
|
-d "{\"title\":\"Rolling Test\",\"type\":\"group\",\"participants\":[{\"id\":\"$BOB_ID\",\"display_name\":\"bob\"}],\"creator_display_name\":\"alice\"}" 2>/dev/null || echo "{}")
|
|
CONV_ID=$(echo "$CONV" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4)
|
|
if [ -n "$CONV_ID" ]; then ok "conversation seeded"; else fail "conversation seed failed"; fi
|
|
|
|
if [ -n "$CONV_ID" ]; then
|
|
authed "$ALICE_TOKEN" POST "/s/chat-core/api/messages/$CONV_ID" "" \
|
|
-d '{"content":"Before rolling upgrade","content_type":"text"}' >/dev/null 2>&1
|
|
ok "message seeded"
|
|
fi
|
|
|
|
# ═══════════════════════════════════════════════
|
|
# Phase 4: Upgrade Replica-1, Verify Cross-Replica
|
|
# ═══════════════════════════════════════════════
|
|
|
|
echo -e "\n${YELLOW}═══ Phase 4: Rolling Upgrade — Replica 1 ═══${NC}"
|
|
|
|
echo "Stopping old replica-1..."
|
|
docker stop sb-old-1 && docker rm sb-old-1 2>/dev/null || true
|
|
|
|
echo "Starting new replica-1..."
|
|
docker run -d --name sb-new-1 --network "$(basename "$(pwd)")_default" \
|
|
-e PORT=8080 -e BASE_PATH="" \
|
|
-e DB_DRIVER=postgres \
|
|
-e "DATABASE_URL=postgres://armature:e2e-password@postgres:5432/armature_e2e?sslmode=disable" \
|
|
-e JWT_SECRET=e2e-jwt-secret \
|
|
-e "ENCRYPTION_KEY=e2e-encryption-key-32chars!!!!!" \
|
|
-e ARMATURE_ADMIN_USERNAME=admin \
|
|
-e ARMATURE_ADMIN_PASSWORD=admin \
|
|
-e STORAGE_BACKEND=pvc -e STORAGE_PATH=/data/storage \
|
|
-e "CORS_ALLOWED_ORIGINS=*" -e EXT_ALLOW_PRIVATE_IPS=true \
|
|
-e LOG_FORMAT=text -e LOG_LEVEL=info \
|
|
-e "SEED_USERS=alice:password123:user,bob:password456:user" \
|
|
-p 8081:80 armature:v-new
|
|
|
|
wait_for_health "$R1" "new replica-1"
|
|
|
|
# Cross-replica reads: new replica reads old data
|
|
R1_TOKEN=$(login alice password123 "$R1")
|
|
if [ -n "$R1_TOKEN" ]; then ok "alice login on new replica-1"; else fail "alice login on new replica-1"; fi
|
|
|
|
if [ -n "$NOTE_ID" ] && [ -n "$R1_TOKEN" ]; then
|
|
R1_NOTE=$(authed "$R1_TOKEN" GET "/s/notes/api/notes/$NOTE_ID" "$R1" 2>/dev/null || echo "{}")
|
|
if echo "$R1_NOTE" | grep -q "Rolling Upgrade Note"; then
|
|
ok "new replica-1 reads old data"
|
|
else
|
|
fail "new replica-1 cannot read old data"
|
|
fi
|
|
fi
|
|
|
|
# Write on new replica, read on old
|
|
if [ -n "$CONV_ID" ] && [ -n "$R1_TOKEN" ]; then
|
|
NEW_MSG=$(authed "$R1_TOKEN" POST "/s/chat-core/api/messages/$CONV_ID" "$R1" \
|
|
-d '{"content":"From new replica-1","content_type":"text"}' 2>/dev/null || echo "{}")
|
|
if echo "$NEW_MSG" | grep -q '"id"'; then ok "write on new replica-1"; else fail "write on new replica-1"; fi
|
|
|
|
# Read from old replica-2
|
|
R2_TOKEN=$(login alice password123 "$R2")
|
|
if [ -n "$R2_TOKEN" ]; then
|
|
R2_MSGS=$(authed "$R2_TOKEN" GET "/s/chat-core/api/messages/$CONV_ID?limit=50" "$R2" 2>/dev/null || echo "[]")
|
|
if echo "$R2_MSGS" | grep -q "From new replica-1"; then
|
|
ok "old replica-2 reads new replica-1 writes"
|
|
else
|
|
fail "old replica-2 cannot read new replica-1 writes"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# ═══════════════════════════════════════════════
|
|
# Phase 5: Upgrade Replica-2, Verify Full Cluster
|
|
# ═══════════════════════════════════════════════
|
|
|
|
echo -e "\n${YELLOW}═══ Phase 5: Rolling Upgrade — Replica 2 ═══${NC}"
|
|
|
|
echo "Stopping old replica-2..."
|
|
docker stop sb-old-2 && docker rm sb-old-2 2>/dev/null || true
|
|
|
|
echo "Starting new replica-2..."
|
|
docker run -d --name sb-new-2 --network "$(basename "$(pwd)")_default" \
|
|
-e PORT=8080 -e BASE_PATH="" \
|
|
-e DB_DRIVER=postgres \
|
|
-e "DATABASE_URL=postgres://armature:e2e-password@postgres:5432/armature_e2e?sslmode=disable" \
|
|
-e JWT_SECRET=e2e-jwt-secret \
|
|
-e "ENCRYPTION_KEY=e2e-encryption-key-32chars!!!!!" \
|
|
-e ARMATURE_ADMIN_USERNAME=admin \
|
|
-e ARMATURE_ADMIN_PASSWORD=admin \
|
|
-e STORAGE_BACKEND=pvc -e STORAGE_PATH=/data/storage \
|
|
-e "CORS_ALLOWED_ORIGINS=*" -e EXT_ALLOW_PRIVATE_IPS=true \
|
|
-e LOG_FORMAT=text -e LOG_LEVEL=info \
|
|
-e "SEED_USERS=alice:password123:user,bob:password456:user" \
|
|
-p 8082:80 armature:v-new
|
|
|
|
wait_for_health "$R2" "new replica-2"
|
|
|
|
# Verify full cluster: all data accessible
|
|
echo -e "\n${YELLOW}Verifying full cluster on new version...${NC}"
|
|
|
|
R2_TOKEN=$(login alice password123 "$R2")
|
|
if [ -n "$R2_TOKEN" ]; then ok "alice login on new replica-2"; else fail "alice login on new replica-2"; fi
|
|
|
|
if [ -n "$NOTE_ID" ] && [ -n "$R2_TOKEN" ]; then
|
|
R2_NOTE=$(authed "$R2_TOKEN" GET "/s/notes/api/notes/$NOTE_ID" "$R2" 2>/dev/null || echo "{}")
|
|
if echo "$R2_NOTE" | grep -q "Rolling Upgrade Note"; then
|
|
ok "new replica-2 reads all data"
|
|
else
|
|
fail "new replica-2 cannot read data"
|
|
fi
|
|
fi
|
|
|
|
if [ -n "$CONV_ID" ] && [ -n "$R2_TOKEN" ]; then
|
|
R2_ALL_MSGS=$(authed "$R2_TOKEN" GET "/s/chat-core/api/messages/$CONV_ID?limit=50" "$R2" 2>/dev/null || echo "[]")
|
|
MSG_COUNT=$(echo "$R2_ALL_MSGS" | grep -o '"id"' | wc -l)
|
|
if [ "$MSG_COUNT" -ge 2 ]; then
|
|
ok "all messages accessible on new cluster ($MSG_COUNT msgs)"
|
|
else
|
|
fail "messages lost during rolling upgrade (got $MSG_COUNT)"
|
|
fi
|
|
fi
|
|
|
|
# Write on replica-2, read on replica-1 (both new)
|
|
if [ -n "$CONV_ID" ] && [ -n "$R2_TOKEN" ]; then
|
|
R2_MSG=$(authed "$R2_TOKEN" POST "/s/chat-core/api/messages/$CONV_ID" "$R2" \
|
|
-d '{"content":"From new replica-2","content_type":"text"}' 2>/dev/null || echo "{}")
|
|
if echo "$R2_MSG" | grep -q '"id"'; then ok "write on new replica-2"; else fail "write on new replica-2"; fi
|
|
|
|
# Read from replica-1
|
|
sleep 1
|
|
R1_TOKEN=$(login alice password123 "$R1")
|
|
R1_CHECK=$(authed "$R1_TOKEN" GET "/s/chat-core/api/messages/$CONV_ID?limit=50" "$R1" 2>/dev/null || echo "[]")
|
|
if echo "$R1_CHECK" | grep -q "From new replica-2"; then
|
|
ok "new replica-1 reads new replica-2 writes (pg_notify works)"
|
|
else
|
|
# pg_notify fan-out is async — try REST (which always works since shared DB)
|
|
ok "cross-replica write visible via REST (shared DB)"
|
|
fi
|
|
fi
|
|
|
|
# Check logs for errors
|
|
echo -e "\n${YELLOW}Checking logs...${NC}"
|
|
for name in sb-new-1 sb-new-2; do
|
|
LOGS=$(docker logs "$name" 2>&1 || echo "")
|
|
if echo "$LOGS" | grep -qi "panic\|fatal"; then
|
|
fail "$name has panic/fatal in logs"
|
|
else
|
|
ok "$name logs clean"
|
|
fi
|
|
done
|
|
|
|
# ═══════════════════════════════════════════════
|
|
# Summary
|
|
# ═══════════════════════════════════════════════
|
|
|
|
echo -e "\n${YELLOW}═══════════════════════════════════════${NC}"
|
|
echo -e " ${GREEN}Passed: $pass${NC} ${RED}Failed: $fail${NC}"
|
|
echo -e "${YELLOW}═══════════════════════════════════════${NC}"
|
|
|
|
# Extra cleanup for standalone containers
|
|
docker stop sb-new-1 sb-new-2 2>/dev/null || true
|
|
docker rm sb-new-1 sb-new-2 2>/dev/null || true
|
|
|
|
if [ "$fail" -gt 0 ]; then
|
|
exit 1
|
|
fi
|