From bde1113e03ebb633d05e82a14796a9240b7b5e55 Mon Sep 17 00:00:00 2001 From: Jeffrey Smith Date: Thu, 2 Apr 2026 15:21:21 +0000 Subject: [PATCH] Add E2E smoke test harness for surface navigation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New Playwright-based smoke test that visits every installed surface and asserts: shell topbar renders, no JS console errors, home link exists. Captures full-page screenshot + console log on failure. Baseline screenshots captured for all surfaces (informational). Follows the existing surface-test-driver.js/run-surface-tests.sh pattern: shell script authenticates, Node driver navigates. New files: - ci/e2e-smoke-test.sh — auth + invoke driver - ci/e2e-smoke-driver.js — Playwright navigation assertions Co-Authored-By: Claude Opus 4.6 (1M context) --- ci/e2e-smoke-driver.js | 218 +++++++++++++++++++++++++++++++++++++++++ ci/e2e-smoke-test.sh | 63 ++++++++++++ 2 files changed, 281 insertions(+) create mode 100644 ci/e2e-smoke-driver.js create mode 100755 ci/e2e-smoke-test.sh diff --git a/ci/e2e-smoke-driver.js b/ci/e2e-smoke-driver.js new file mode 100644 index 0000000..9f3613c --- /dev/null +++ b/ci/e2e-smoke-driver.js @@ -0,0 +1,218 @@ +#!/usr/bin/env node +/** + * E2E Smoke Driver — Playwright navigation smoke test + * + * Visits every installed surface and asserts: + * 1. The shell topbar (.sw-topbar) renders + * 2. No JS console errors + * 3. The home link exists + * + * On failure: captures full-page screenshot + console log. + * + * Usage: + * node ci/e2e-smoke-driver.js --server=http://localhost:3000 --token=TOKEN + * + * Exit codes: 0 = all passed, 1 = failures + */ +'use strict'; + +const { chromium } = require('playwright'); +const fs = require('fs'); +const path = require('path'); + +// Parse args +const args = {}; +process.argv.slice(2).forEach(a => { + const [k, v] = a.replace(/^--/, '').split('='); + args[k] = v; +}); + +const SERVER = args.server || 'http://localhost:3000'; +const TOKEN = args.token || ''; +const SCREENSHOT_DIR = args.screenshots || '/tmp/e2e-screenshots'; + +if (!TOKEN) { + console.error('ERROR: --token is required'); + process.exit(1); +} + +// Surfaces that require URL parameters or aren't navigable directly +const SKIP_SURFACES = new Set([ + 'workflow', // requires /w/:id + 'workflow-landing', // requires /w/:id/public + 'welcome', // redirect/onboarding flow + 'test-runners', // tested separately by surface-test-driver +]); + +/** + * Discover all surfaces: core + extension packages with surface_route. + */ +async function discoverSurfaces(page) { + const surfaces = []; + + // Core surfaces (always present) + const coreSurfaces = [ + { id: 'admin', route: '/admin' }, + { id: 'settings', route: '/settings' }, + { id: 'docs', route: '/docs' }, + { id: 'team-admin', route: '/team-admin' }, + ]; + + for (const s of coreSurfaces) { + if (!SKIP_SURFACES.has(s.id)) { + surfaces.push(s); + } + } + + // Extension surfaces from API + try { + const packages = await page.evaluate(async (serverUrl) => { + const r = await fetch(serverUrl + '/api/v1/admin/packages', { + credentials: 'include', + }); + if (r.status === 200) { + const body = await r.json(); + return body.data || body || []; + } + return []; + }, SERVER); + + for (const pkg of packages) { + if (!pkg.enabled) continue; + if (pkg.type !== 'surface' && pkg.type !== 'full') continue; + if (SKIP_SURFACES.has(pkg.id)) continue; + + // Extension surfaces live at /s/{id} + const route = pkg.surface_route || ('/s/' + pkg.id); + surfaces.push({ id: pkg.id, route }); + } + } catch (e) { + console.warn('WARN: Could not fetch package list:', e.message); + } + + return surfaces; +} + +(async () => { + const browser = await chromium.launch({ headless: true }); + const context = await browser.newContext(); + + // Set auth cookie + await context.addCookies([{ + name: 'arm_token', + value: TOKEN, + domain: new URL(SERVER).hostname, + path: '/', + }]); + + const page = await context.newPage(); + + // ── Discover surfaces ───────────────────── + console.log('Discovering surfaces...'); + + // Navigate to a page first so cookies are sent with API call + await page.goto(SERVER + '/admin', { waitUntil: 'networkidle', timeout: 30000 }); + + const surfaces = await discoverSurfaces(page); + console.log(`Found ${surfaces.length} surfaces: ${surfaces.map(s => s.id).join(', ')}`); + + if (surfaces.length === 0) { + console.error('ERROR: No surfaces discovered'); + await browser.close(); + process.exit(1); + } + + // ── Visit each surface ──────────────────── + const results = []; + let failures = 0; + + for (const surface of surfaces) { + const url = SERVER + surface.route; + const consoleErrors = []; + + // Fresh console error collector per surface + const onConsole = msg => { + if (msg.type() === 'error') { + const text = msg.text(); + // Ignore benign errors (favicon, service worker) + if (text.includes('favicon') || text.includes('service-worker')) return; + consoleErrors.push(text); + } + }; + page.on('console', onConsole); + + try { + process.stdout.write(` ${surface.id} (${surface.route}) ... `); + await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 }); + + // Assert 1: Shell topbar exists + const topbar = await page.$('.sw-topbar'); + if (!topbar) { + throw new Error('Shell topbar (.sw-topbar) not found'); + } + + // Assert 2: Home link exists (favicon link or any link to /) + const homeLink = await page.$('a[href="/"], a[href="./"], .sw-topbar__home'); + if (!homeLink) { + throw new Error('Home link not found'); + } + + // Assert 3: No JS console errors + // Give a moment for any async errors to settle + await page.waitForTimeout(500); + if (consoleErrors.length > 0) { + throw new Error(`JS console errors: ${consoleErrors.join('; ')}`); + } + + console.log('PASS'); + results.push({ surface: surface.id, status: 'pass' }); + + // Capture baseline screenshot (informational, not a gate) + const slug = surface.id.replace(/[^a-z0-9-]/g, '_'); + await page.screenshot({ + path: path.join(SCREENSHOT_DIR, `baseline-${slug}.png`), + fullPage: true, + }); + + } catch (err) { + console.log('FAIL — ' + err.message); + failures++; + results.push({ surface: surface.id, status: 'fail', error: err.message }); + + // Screenshot + console log on failure + const slug = surface.id.replace(/[^a-z0-9-]/g, '_'); + try { + await page.screenshot({ + path: path.join(SCREENSHOT_DIR, `fail-${slug}.png`), + fullPage: true, + }); + if (consoleErrors.length > 0) { + fs.writeFileSync( + path.join(SCREENSHOT_DIR, `fail-${slug}-console.log`), + consoleErrors.join('\n') + ); + } + } catch (screenshotErr) { + console.warn(' (could not capture screenshot:', screenshotErr.message + ')'); + } + } finally { + page.removeListener('console', onConsole); + } + } + + // ── Summary ─────────────────────────────── + console.log('\n═══ E2E Smoke Summary ═══'); + console.log(` Total: ${results.length}`); + console.log(` Passed: ${results.filter(r => r.status === 'pass').length}`); + console.log(` Failed: ${failures}`); + + if (failures > 0) { + console.log('\nFailed surfaces:'); + for (const r of results.filter(r => r.status === 'fail')) { + console.log(` ✗ ${r.surface}: ${r.error}`); + } + } + + await browser.close(); + process.exit(failures > 0 ? 1 : 0); +})(); diff --git a/ci/e2e-smoke-test.sh b/ci/e2e-smoke-test.sh new file mode 100755 index 0000000..88f1ca0 --- /dev/null +++ b/ci/e2e-smoke-test.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# ═══════════════════════════════════════════════ +# E2E Smoke Test — CI Entrypoint +# ═══════════════════════════════════════════════ +# +# Authenticates as admin, then runs a Playwright navigation smoke +# test against every installed surface. Asserts shell topbar +# renders, no JS console errors, and the home link works. +# +# Prerequisites: +# - Server running at $SERVER_URL (default: http://localhost:3000) +# - npx playwright install chromium +# - ADMIN_USER / ADMIN_PASS env vars (default: admin/admin) +# +# Exit codes: 0 = all surfaces passed, 1 = failures detected +set -euo pipefail + +SERVER_URL="${SERVER_URL:-http://localhost:3000}" +ADMIN_USER="${ADMIN_USER:-admin}" +ADMIN_PASS="${ADMIN_PASS:-admin}" +SCREENSHOT_DIR="${SCREENSHOT_DIR:-/tmp/e2e-screenshots}" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +echo -e "${YELLOW}═══ E2E Smoke Test ═══${NC}" +echo " Server: ${SERVER_URL}" + +# ── Ensure screenshot dir ──────────────────── +mkdir -p "${SCREENSHOT_DIR}" + +# ── Authenticate ───────────────────────────── +echo -e "${YELLOW}Authenticating...${NC}" +TOKEN=$(curl -sf -X POST "${SERVER_URL}/api/v1/auth/login" \ + -H "Content-Type: application/json" \ + -d "{\"login\":\"${ADMIN_USER}\",\"password\":\"${ADMIN_PASS}\"}" \ + | node -e "process.stdin.resume(); let d=''; process.stdin.on('data',c=>d+=c); process.stdin.on('end',()=>{try{console.log(JSON.parse(d).token)}catch(e){process.exit(1)}})") + +if [ -z "$TOKEN" ]; then + echo -e "${RED}Failed to authenticate${NC}" + exit 1 +fi +echo -e "${GREEN}Authenticated${NC}" + +# ── Run via Playwright ─────────────────────── +echo -e "${YELLOW}Running E2E smoke tests via Playwright...${NC}" +node "$(dirname "$0")/e2e-smoke-driver.js" \ + --server="${SERVER_URL}" \ + --token="${TOKEN}" \ + --screenshots="${SCREENSHOT_DIR}" + +EXIT_CODE=$? + +if [ $EXIT_CODE -eq 0 ]; then + echo -e "${GREEN}═══ All E2E smoke tests passed ═══${NC}" +else + echo -e "${RED}═══ E2E smoke tests FAILED ═══${NC}" + echo -e "${YELLOW}Screenshots saved to: ${SCREENSHOT_DIR}${NC}" +fi + +exit $EXIT_CODE