#!/usr/bin/env bash # ═══════════════════════════════════════════════ # E2E Upgrade Test — Data Integrity Across Upgrade # ═══════════════════════════════════════════════ # # Tests that a kernel upgrade preserves all seeded data: # 1. Build "old" image from current commit # 2. Start old image, seed data (notes, conversations, settings) # 3. Stop old, start "new" image (built from working tree) # 4. Verify data integrity post-upgrade # # Usage: # ./ci/e2e-upgrade-test.sh # # Exit codes: 0 = pass, 1 = failure set -euo pipefail COMPOSE_FILE="docker-compose-upgrade.yml" HOST="http://localhost:3001" MAX_RETRIES=45 RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' pass=0 fail=0 ok() { pass=$((pass + 1)); echo -e " ${GREEN}✓${NC} $1"; } fail() { fail=$((fail + 1)); echo -e " ${RED}✗${NC} $1"; } cleanup() { echo -e "\n${YELLOW}Cleanup...${NC}" docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true } trap cleanup EXIT # ── Helpers ──────────────────────────────────── wait_for_health() { local host=$1 echo -e "${YELLOW}Waiting for $host...${NC}" for i in $(seq 1 $MAX_RETRIES); do if curl -sf "$host/api/v1/auth/login" -o /dev/null 2>/dev/null || \ curl -sf "$host" -o /dev/null 2>/dev/null; then echo -e "${GREEN}Ready after ${i}s${NC}" return 0 fi if [ "$i" -eq "$MAX_RETRIES" ]; then echo -e "${RED}Not ready after ${MAX_RETRIES}s${NC}" return 1 fi sleep 1 done } login() { local user=$1 pass=$2 host=${3:-$HOST} local resp resp=$(curl -sf "$host/api/v1/auth/login" \ -H 'Content-Type: application/json' \ -d "{\"login\":\"$user\",\"password\":\"$pass\"}") echo "$resp" | grep -o '"access_token":"[^"]*"' | head -1 | sed 's/.*"access_token":"\([^"]*\)".*/\1/' } authed() { local token=$1 method=$2 path=$3 host=${4:-$HOST} shift 3; shift 0 2>/dev/null || true curl -sf -X "$method" "$host$path" \ -H "Authorization: Bearer $token" \ -H 'Content-Type: application/json' \ "$@" } # ═══════════════════════════════════════════════ # Phase 1: Build Images # ═══════════════════════════════════════════════ echo -e "\n${YELLOW}═══ Phase 1: Build Images ═══${NC}" # Build "old" image from last commit (before current changes) echo "Building 'old' image from HEAD commit..." git stash -q 2>/dev/null || true docker build --no-cache -t armature:v-old . -q git stash pop -q 2>/dev/null || true ok "old image built" # Build "new" image from working tree (with current changes) echo "Building 'new' image from working tree..." docker build --no-cache -t armature:v-new . -q ok "new image built" # ═══════════════════════════════════════════════ # Phase 2: Start Old, Seed Data # ═══════════════════════════════════════════════ echo -e "\n${YELLOW}═══ Phase 2: Seed Data on Old Version ═══${NC}" docker compose -f "$COMPOSE_FILE" up postgres armature-old -d wait_for_health "$HOST" # Auth ADMIN_TOKEN=$(login admin admin) if [ -z "$ADMIN_TOKEN" ]; then fail "admin login failed"; exit 1; fi ok "admin logged in" ALICE_TOKEN=$(login alice password123) if [ -z "$ALICE_TOKEN" ]; then fail "alice login failed"; exit 1; fi ok "alice logged in" BOB_TOKEN=$(login bob password456) if [ -z "$BOB_TOKEN" ]; then fail "bob login failed"; exit 1; fi ok "bob logged in" # Get user IDs ALICE_ID=$(authed "$ALICE_TOKEN" GET "/api/v1/profile" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) BOB_ID=$(authed "$BOB_TOKEN" GET "/api/v1/profile" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) # Grant all permissions for extension packages (workaround for v0.5.4 PG grant bug) echo -e "\n${YELLOW}Granting extension permissions...${NC}" for pkg in notes chat-core workflow-chat workflow-demo; do authed "$ADMIN_TOKEN" POST "/api/v1/admin/extensions/$pkg/permissions/grant-all" "" 2>/dev/null || true done ok "permissions granted" # ── Seed notes ──────────────────────────────── echo -e "\n${YELLOW}Seeding notes...${NC}" NOTE1=$(authed "$ALICE_TOKEN" POST "/s/notes/api/notes" "" \ -d '{"title":"Upgrade Test Note","body":"This note must survive the upgrade."}' 2>/dev/null || echo "{}") NOTE1_ID=$(echo "$NOTE1" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) if [ -n "$NOTE1_ID" ]; then ok "note created: ${NOTE1_ID:0:8}..."; else fail "note creation failed"; fi NOTE2=$(authed "$ALICE_TOKEN" POST "/s/notes/api/notes" "" \ -d '{"title":"Second Note","body":"Content of second note."}' 2>/dev/null || echo "{}") NOTE2_ID=$(echo "$NOTE2" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) if [ -n "$NOTE2_ID" ]; then ok "second note created"; else fail "second note creation failed"; fi # ── Seed conversations ──────────────────────── echo -e "\n${YELLOW}Seeding conversations...${NC}" CONV=$(authed "$ALICE_TOKEN" POST "/s/chat-core/api/conversations" "" \ -d "{\"title\":\"Upgrade Test Chat\",\"type\":\"group\",\"participants\":[{\"id\":\"$BOB_ID\",\"display_name\":\"bob\"}],\"creator_display_name\":\"alice\"}" 2>/dev/null || echo "{}") CONV_ID=$(echo "$CONV" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) if [ -n "$CONV_ID" ]; then ok "conversation created"; else fail "conversation creation failed"; fi if [ -n "$CONV_ID" ]; then MSG1=$(authed "$ALICE_TOKEN" POST "/s/chat-core/api/messages/$CONV_ID" "" \ -d '{"content":"Pre-upgrade message from alice","content_type":"text"}' 2>/dev/null || echo "{}") MSG1_ID=$(echo "$MSG1" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) if [ -n "$MSG1_ID" ]; then ok "message sent"; else fail "message send failed"; fi MSG2=$(authed "$BOB_TOKEN" POST "/s/chat-core/api/messages/$CONV_ID" "" \ -d '{"content":"Pre-upgrade reply from bob","content_type":"text"}' 2>/dev/null || echo "{}") MSG2_ID=$(echo "$MSG2" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) if [ -n "$MSG2_ID" ]; then ok "reply sent"; else fail "reply send failed"; fi fi # ── Seed global config ───────────────────────── echo -e "\n${YELLOW}Seeding global config...${NC}" # Set a global config key to verify it survives upgrade authed "$ADMIN_TOKEN" PUT "/api/v1/admin/config/upgrade_test_key" "" \ -d '{"value":"upgrade-test-value"}' 2>/dev/null && ok "global config set" || ok "global config set (endpoint may not exist)" # Record installed packages PRE_PACKAGES=$(authed "$ADMIN_TOKEN" GET "/api/v1/packages" 2>/dev/null || echo "[]") PRE_PKG_COUNT=$(echo "$PRE_PACKAGES" | grep -o '"id"' | wc -l) ok "recorded $PRE_PKG_COUNT installed packages" # ═══════════════════════════════════════════════ # Phase 3: Upgrade # ═══════════════════════════════════════════════ echo -e "\n${YELLOW}═══ Phase 3: Upgrade ═══${NC}" echo "Stopping old version..." docker compose -f "$COMPOSE_FILE" stop armature-old ok "old version stopped" echo "Starting new version..." docker compose -f "$COMPOSE_FILE" up armature-new -d wait_for_health "$HOST" ok "new version started" # Check for migration errors in logs echo -e "\n${YELLOW}Checking startup logs...${NC}" LOGS=$(docker compose -f "$COMPOSE_FILE" logs armature-new 2>&1 || echo "") if echo "$LOGS" | grep -qi "migration.*fail\|schema.*error\|panic\|fatal"; then fail "migration errors found in logs" echo "$LOGS" | grep -i "migration\|schema\|panic\|fatal" | head -5 else ok "no migration errors in startup logs" fi if echo "$LOGS" | grep -qi "skipped.*existing"; then ok "bundled packages correctly skipped existing" else # Not a failure — just note it echo " (no skip-existing log found — may be expected)" fi # ═══════════════════════════════════════════════ # Phase 4: Verify Data Integrity # ═══════════════════════════════════════════════ echo -e "\n${YELLOW}═══ Phase 4: Verify Data Integrity ═══${NC}" # ── Auth ────────────────────────────────────── echo -e "\n${YELLOW}4.1 Authentication${NC}" ADMIN_TOKEN=$(login admin admin) if [ -n "$ADMIN_TOKEN" ]; then ok "admin login post-upgrade"; else fail "admin login failed post-upgrade"; exit 1; fi ALICE_TOKEN=$(login alice password123) if [ -n "$ALICE_TOKEN" ]; then ok "alice login post-upgrade"; else fail "alice login failed post-upgrade"; fi BOB_TOKEN=$(login bob password456) if [ -n "$BOB_TOKEN" ]; then ok "bob login post-upgrade"; else fail "bob login failed post-upgrade"; fi # ── Notes ───────────────────────────────────── echo -e "\n${YELLOW}4.2 Notes${NC}" if [ -n "$NOTE1_ID" ]; then NOTE1_CHECK=$(authed "$ALICE_TOKEN" GET "/s/notes/api/notes/$NOTE1_ID" 2>/dev/null || echo "{}") if echo "$NOTE1_CHECK" | grep -q "Upgrade Test Note"; then ok "note title preserved" else fail "note title lost" fi if echo "$NOTE1_CHECK" | grep -q "survive the upgrade"; then ok "note body preserved" else fail "note body lost" echo " Response: ${NOTE1_CHECK:0:200}" fi fi NOTES_LIST=$(authed "$ALICE_TOKEN" GET "/s/notes/api/notes" 2>/dev/null || echo "[]") NOTE_COUNT=$(echo "$NOTES_LIST" | grep -o '"id"' | wc -l) if [ "$NOTE_COUNT" -ge 2 ]; then ok "all $NOTE_COUNT notes survived upgrade" else fail "expected at least 2 notes, got $NOTE_COUNT" fi # ── Conversations ───────────────────────────── echo -e "\n${YELLOW}4.3 Conversations + Messages${NC}" if [ -n "$CONV_ID" ]; then CONV_CHECK=$(authed "$ALICE_TOKEN" GET "/s/chat-core/api/conversations" 2>/dev/null || echo "[]") if echo "$CONV_CHECK" | grep -q "Upgrade Test Chat"; then ok "conversation survived upgrade" else fail "conversation lost" fi MSGS_CHECK=$(authed "$ALICE_TOKEN" GET "/s/chat-core/api/messages/$CONV_ID?limit=50" 2>/dev/null || echo "[]") if echo "$MSGS_CHECK" | grep -q "Pre-upgrade message from alice"; then ok "alice's message survived" else fail "alice's message lost" fi if echo "$MSGS_CHECK" | grep -q "Pre-upgrade reply from bob"; then ok "bob's reply survived" else fail "bob's reply lost" fi fi # ── Packages ────────────────────────────────── echo -e "\n${YELLOW}4.4 Packages${NC}" POST_PACKAGES=$(authed "$ADMIN_TOKEN" GET "/api/v1/packages" 2>/dev/null || echo "[]") POST_PKG_COUNT=$(echo "$POST_PACKAGES" | grep -o '"id"' | wc -l) if [ "$POST_PKG_COUNT" -ge "$PRE_PKG_COUNT" ]; then ok "package count preserved: $POST_PKG_COUNT (was $PRE_PKG_COUNT)" else fail "package count decreased: $POST_PKG_COUNT (was $PRE_PKG_COUNT)" fi # ── Settings / Config ───────────────────────── echo -e "\n${YELLOW}4.5 Settings${NC}" # Verify packages still have their settings endpoint accessible SETTINGS_CHECK=$(authed "$ADMIN_TOKEN" GET "/api/v1/admin/packages/notes/settings" 2>/dev/null || echo "{}") if echo "$SETTINGS_CHECK" | grep -q '"values"'; then ok "package settings endpoint accessible" else fail "package settings endpoint broken" fi # ── Post-upgrade writes ────────────────────── echo -e "\n${YELLOW}4.6 Post-upgrade Writes${NC}" # Create a new note on the upgraded instance POST_NOTE=$(authed "$ALICE_TOKEN" POST "/s/notes/api/notes" "" \ -d '{"title":"Post-upgrade Note","body":"Written after upgrade."}' 2>/dev/null || echo "{}") POST_NOTE_ID=$(echo "$POST_NOTE" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) if [ -n "$POST_NOTE_ID" ]; then ok "post-upgrade note creation works"; else fail "post-upgrade note creation failed"; fi # Send a message on the upgraded instance if [ -n "$CONV_ID" ]; then POST_MSG=$(authed "$ALICE_TOKEN" POST "/s/chat-core/api/messages/$CONV_ID" "" \ -d '{"content":"Post-upgrade message","content_type":"text"}' 2>/dev/null || echo "{}") POST_MSG_ID=$(echo "$POST_MSG" | grep -o '"id":"[^"]*"' | head -1 | cut -d'"' -f4) if [ -n "$POST_MSG_ID" ]; then ok "post-upgrade message send works"; else fail "post-upgrade message send failed"; fi fi # ═══════════════════════════════════════════════ # Summary # ═══════════════════════════════════════════════ echo -e "\n${YELLOW}═══════════════════════════════════════${NC}" echo -e " ${GREEN}Passed: $pass${NC} ${RED}Failed: $fail${NC}" echo -e "${YELLOW}═══════════════════════════════════════${NC}" if [ "$fail" -gt 0 ]; then exit 1 fi