Add E2E smoke test harness for surface navigation
New Playwright-based smoke test that visits every installed surface and asserts: shell topbar renders, no JS console errors, home link exists. Captures full-page screenshot + console log on failure. Baseline screenshots captured for all surfaces (informational). Follows the existing surface-test-driver.js/run-surface-tests.sh pattern: shell script authenticates, Node driver navigates. New files: - ci/e2e-smoke-test.sh — auth + invoke driver - ci/e2e-smoke-driver.js — Playwright navigation assertions Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
218
ci/e2e-smoke-driver.js
Normal file
218
ci/e2e-smoke-driver.js
Normal file
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* E2E Smoke Driver — Playwright navigation smoke test
|
||||
*
|
||||
* Visits every installed surface and asserts:
|
||||
* 1. The shell topbar (.sw-topbar) renders
|
||||
* 2. No JS console errors
|
||||
* 3. The home link exists
|
||||
*
|
||||
* On failure: captures full-page screenshot + console log.
|
||||
*
|
||||
* Usage:
|
||||
* node ci/e2e-smoke-driver.js --server=http://localhost:3000 --token=TOKEN
|
||||
*
|
||||
* Exit codes: 0 = all passed, 1 = failures
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const { chromium } = require('playwright');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Parse args
|
||||
const args = {};
|
||||
process.argv.slice(2).forEach(a => {
|
||||
const [k, v] = a.replace(/^--/, '').split('=');
|
||||
args[k] = v;
|
||||
});
|
||||
|
||||
const SERVER = args.server || 'http://localhost:3000';
|
||||
const TOKEN = args.token || '';
|
||||
const SCREENSHOT_DIR = args.screenshots || '/tmp/e2e-screenshots';
|
||||
|
||||
if (!TOKEN) {
|
||||
console.error('ERROR: --token is required');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Surfaces that require URL parameters or aren't navigable directly
|
||||
const SKIP_SURFACES = new Set([
|
||||
'workflow', // requires /w/:id
|
||||
'workflow-landing', // requires /w/:id/public
|
||||
'welcome', // redirect/onboarding flow
|
||||
'test-runners', // tested separately by surface-test-driver
|
||||
]);
|
||||
|
||||
/**
|
||||
* Discover all surfaces: core + extension packages with surface_route.
|
||||
*/
|
||||
async function discoverSurfaces(page) {
|
||||
const surfaces = [];
|
||||
|
||||
// Core surfaces (always present)
|
||||
const coreSurfaces = [
|
||||
{ id: 'admin', route: '/admin' },
|
||||
{ id: 'settings', route: '/settings' },
|
||||
{ id: 'docs', route: '/docs' },
|
||||
{ id: 'team-admin', route: '/team-admin' },
|
||||
];
|
||||
|
||||
for (const s of coreSurfaces) {
|
||||
if (!SKIP_SURFACES.has(s.id)) {
|
||||
surfaces.push(s);
|
||||
}
|
||||
}
|
||||
|
||||
// Extension surfaces from API
|
||||
try {
|
||||
const packages = await page.evaluate(async (serverUrl) => {
|
||||
const r = await fetch(serverUrl + '/api/v1/admin/packages', {
|
||||
credentials: 'include',
|
||||
});
|
||||
if (r.status === 200) {
|
||||
const body = await r.json();
|
||||
return body.data || body || [];
|
||||
}
|
||||
return [];
|
||||
}, SERVER);
|
||||
|
||||
for (const pkg of packages) {
|
||||
if (!pkg.enabled) continue;
|
||||
if (pkg.type !== 'surface' && pkg.type !== 'full') continue;
|
||||
if (SKIP_SURFACES.has(pkg.id)) continue;
|
||||
|
||||
// Extension surfaces live at /s/{id}
|
||||
const route = pkg.surface_route || ('/s/' + pkg.id);
|
||||
surfaces.push({ id: pkg.id, route });
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn('WARN: Could not fetch package list:', e.message);
|
||||
}
|
||||
|
||||
return surfaces;
|
||||
}
|
||||
|
||||
(async () => {
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const context = await browser.newContext();
|
||||
|
||||
// Set auth cookie
|
||||
await context.addCookies([{
|
||||
name: 'arm_token',
|
||||
value: TOKEN,
|
||||
domain: new URL(SERVER).hostname,
|
||||
path: '/',
|
||||
}]);
|
||||
|
||||
const page = await context.newPage();
|
||||
|
||||
// ── Discover surfaces ─────────────────────
|
||||
console.log('Discovering surfaces...');
|
||||
|
||||
// Navigate to a page first so cookies are sent with API call
|
||||
await page.goto(SERVER + '/admin', { waitUntil: 'networkidle', timeout: 30000 });
|
||||
|
||||
const surfaces = await discoverSurfaces(page);
|
||||
console.log(`Found ${surfaces.length} surfaces: ${surfaces.map(s => s.id).join(', ')}`);
|
||||
|
||||
if (surfaces.length === 0) {
|
||||
console.error('ERROR: No surfaces discovered');
|
||||
await browser.close();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ── Visit each surface ────────────────────
|
||||
const results = [];
|
||||
let failures = 0;
|
||||
|
||||
for (const surface of surfaces) {
|
||||
const url = SERVER + surface.route;
|
||||
const consoleErrors = [];
|
||||
|
||||
// Fresh console error collector per surface
|
||||
const onConsole = msg => {
|
||||
if (msg.type() === 'error') {
|
||||
const text = msg.text();
|
||||
// Ignore benign errors (favicon, service worker)
|
||||
if (text.includes('favicon') || text.includes('service-worker')) return;
|
||||
consoleErrors.push(text);
|
||||
}
|
||||
};
|
||||
page.on('console', onConsole);
|
||||
|
||||
try {
|
||||
process.stdout.write(` ${surface.id} (${surface.route}) ... `);
|
||||
await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
|
||||
|
||||
// Assert 1: Shell topbar exists
|
||||
const topbar = await page.$('.sw-topbar');
|
||||
if (!topbar) {
|
||||
throw new Error('Shell topbar (.sw-topbar) not found');
|
||||
}
|
||||
|
||||
// Assert 2: Home link exists (favicon link or any link to /)
|
||||
const homeLink = await page.$('a[href="/"], a[href="./"], .sw-topbar__home');
|
||||
if (!homeLink) {
|
||||
throw new Error('Home link not found');
|
||||
}
|
||||
|
||||
// Assert 3: No JS console errors
|
||||
// Give a moment for any async errors to settle
|
||||
await page.waitForTimeout(500);
|
||||
if (consoleErrors.length > 0) {
|
||||
throw new Error(`JS console errors: ${consoleErrors.join('; ')}`);
|
||||
}
|
||||
|
||||
console.log('PASS');
|
||||
results.push({ surface: surface.id, status: 'pass' });
|
||||
|
||||
// Capture baseline screenshot (informational, not a gate)
|
||||
const slug = surface.id.replace(/[^a-z0-9-]/g, '_');
|
||||
await page.screenshot({
|
||||
path: path.join(SCREENSHOT_DIR, `baseline-${slug}.png`),
|
||||
fullPage: true,
|
||||
});
|
||||
|
||||
} catch (err) {
|
||||
console.log('FAIL — ' + err.message);
|
||||
failures++;
|
||||
results.push({ surface: surface.id, status: 'fail', error: err.message });
|
||||
|
||||
// Screenshot + console log on failure
|
||||
const slug = surface.id.replace(/[^a-z0-9-]/g, '_');
|
||||
try {
|
||||
await page.screenshot({
|
||||
path: path.join(SCREENSHOT_DIR, `fail-${slug}.png`),
|
||||
fullPage: true,
|
||||
});
|
||||
if (consoleErrors.length > 0) {
|
||||
fs.writeFileSync(
|
||||
path.join(SCREENSHOT_DIR, `fail-${slug}-console.log`),
|
||||
consoleErrors.join('\n')
|
||||
);
|
||||
}
|
||||
} catch (screenshotErr) {
|
||||
console.warn(' (could not capture screenshot:', screenshotErr.message + ')');
|
||||
}
|
||||
} finally {
|
||||
page.removeListener('console', onConsole);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Summary ───────────────────────────────
|
||||
console.log('\n═══ E2E Smoke Summary ═══');
|
||||
console.log(` Total: ${results.length}`);
|
||||
console.log(` Passed: ${results.filter(r => r.status === 'pass').length}`);
|
||||
console.log(` Failed: ${failures}`);
|
||||
|
||||
if (failures > 0) {
|
||||
console.log('\nFailed surfaces:');
|
||||
for (const r of results.filter(r => r.status === 'fail')) {
|
||||
console.log(` ✗ ${r.surface}: ${r.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
await browser.close();
|
||||
process.exit(failures > 0 ? 1 : 0);
|
||||
})();
|
||||
63
ci/e2e-smoke-test.sh
Executable file
63
ci/e2e-smoke-test.sh
Executable file
@@ -0,0 +1,63 @@
|
||||
#!/usr/bin/env bash
|
||||
# ═══════════════════════════════════════════════
|
||||
# E2E Smoke Test — CI Entrypoint
|
||||
# ═══════════════════════════════════════════════
|
||||
#
|
||||
# Authenticates as admin, then runs a Playwright navigation smoke
|
||||
# test against every installed surface. Asserts shell topbar
|
||||
# renders, no JS console errors, and the home link works.
|
||||
#
|
||||
# Prerequisites:
|
||||
# - Server running at $SERVER_URL (default: http://localhost:3000)
|
||||
# - npx playwright install chromium
|
||||
# - ADMIN_USER / ADMIN_PASS env vars (default: admin/admin)
|
||||
#
|
||||
# Exit codes: 0 = all surfaces passed, 1 = failures detected
|
||||
set -euo pipefail
|
||||
|
||||
SERVER_URL="${SERVER_URL:-http://localhost:3000}"
|
||||
ADMIN_USER="${ADMIN_USER:-admin}"
|
||||
ADMIN_PASS="${ADMIN_PASS:-admin}"
|
||||
SCREENSHOT_DIR="${SCREENSHOT_DIR:-/tmp/e2e-screenshots}"
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
echo -e "${YELLOW}═══ E2E Smoke Test ═══${NC}"
|
||||
echo " Server: ${SERVER_URL}"
|
||||
|
||||
# ── Ensure screenshot dir ────────────────────
|
||||
mkdir -p "${SCREENSHOT_DIR}"
|
||||
|
||||
# ── Authenticate ─────────────────────────────
|
||||
echo -e "${YELLOW}Authenticating...${NC}"
|
||||
TOKEN=$(curl -sf -X POST "${SERVER_URL}/api/v1/auth/login" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"login\":\"${ADMIN_USER}\",\"password\":\"${ADMIN_PASS}\"}" \
|
||||
| node -e "process.stdin.resume(); let d=''; process.stdin.on('data',c=>d+=c); process.stdin.on('end',()=>{try{console.log(JSON.parse(d).token)}catch(e){process.exit(1)}})")
|
||||
|
||||
if [ -z "$TOKEN" ]; then
|
||||
echo -e "${RED}Failed to authenticate${NC}"
|
||||
exit 1
|
||||
fi
|
||||
echo -e "${GREEN}Authenticated${NC}"
|
||||
|
||||
# ── Run via Playwright ───────────────────────
|
||||
echo -e "${YELLOW}Running E2E smoke tests via Playwright...${NC}"
|
||||
node "$(dirname "$0")/e2e-smoke-driver.js" \
|
||||
--server="${SERVER_URL}" \
|
||||
--token="${TOKEN}" \
|
||||
--screenshots="${SCREENSHOT_DIR}"
|
||||
|
||||
EXIT_CODE=$?
|
||||
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo -e "${GREEN}═══ All E2E smoke tests passed ═══${NC}"
|
||||
else
|
||||
echo -e "${RED}═══ E2E smoke tests FAILED ═══${NC}"
|
||||
echo -e "${YELLOW}Screenshots saved to: ${SCREENSHOT_DIR}${NC}"
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
Reference in New Issue
Block a user