diff --git a/CHANGELOG.md b/CHANGELOG.md index dc952f3..49902a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,39 @@ All notable changes to Switchboard Core are documented here. +## v0.3.8 — Distribution + +### Added + +- **Bundled packages**: Production Docker image now ships with 12 pre-built + `.pkg` archives (4 workflows, 3 surfaces, 1 full, 1 library, 2 test runners, + 1 extension). Auto-installed on first boot via `InstallBundledPackages()`. + Install-once, skip-if-present — admin uninstalls are respected on restart. +- **Package allowlist** (`BUNDLED_PACKAGES`): Comma-separated list of package + IDs to install. Empty (default) installs all. Useful for Helm/K8s where + different environments need different packages. +- **Skip bundled** (`SKIP_BUNDLED_PACKAGES=true`): Disables auto-install + entirely. Intended for production until post-MVP packages are ready. +- **Builder image** (`Dockerfile.builder`): Pre-caches Go modules, Node + dependencies, and vendor lib tarballs for faster custom builds. +- **Distribution docs** (`docs/DISTRIBUTION.md`): Quick start, bundled + packages, builder image usage, custom build guide, production deployment + reference. +- **Migration 012**: Adds `bundled` to `packages.source` CHECK constraint + (both Postgres and SQLite). +- **K8s manifest**: `SKIP_BUNDLED_PACKAGES` and `BUNDLED_PACKAGES` env vars + added to `k8s/switchboard.yaml`. +- **Tests**: 6 handler tests (fresh install, skip existing, missing dir, empty + dir, dormant handling, allowlist filtering). + +### Environment defaults + +| Environment | `SKIP_BUNDLED_PACKAGES` | `BUNDLED_PACKAGES` | +|-------------|------------------------|--------------------| +| Dev (compose) | `false` | (empty = all) | +| Test (K8s) | `false` | (empty = all) | +| Prod (K8s) | `true` | — | + ## v0.3.7 — Package Audit ### Added diff --git a/Dockerfile b/Dockerfile index ae22a02..8dab240 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,8 @@ # Stage 1: Build Go backend # Stage 2: Download JS vendor libs (marked, DOMPurify) # Stage 3: Build CM6 editor bundle (esbuild) -# Stage 4: Production image (nginx + backend) +# Stage 4: Build bundled packages (.pkg archives) +# Stage 5: Production image (nginx + backend) # # Vendor libs are baked in during build so the # app works in disconnected environments @@ -57,7 +58,13 @@ COPY scripts/build-editor.sh /build/scripts/build-editor.sh RUN cd /build/src/editor && npm ci --loglevel=warn RUN sh /build/scripts/build-editor.sh /build/dist -# ── Stage 4: Production ───────────────────── +# ── Stage 4: Build bundled packages ───────── +FROM alpine:3 AS packages +RUN apk add --no-cache zip bash +COPY packages/ /packages/ +RUN cd /packages && bash build.sh + +# ── Stage 5: Production ───────────────────── FROM nginx:1-alpine RUN apk add --no-cache bash git @@ -85,6 +92,9 @@ COPY --from=vendor /vendor/katex/katex.min.css /usr/share/nginx/html/vendor/kate COPY --from=vendor /vendor/katex/fonts/ /usr/share/nginx/html/vendor/katex/fonts/ COPY --from=cm6-build /build/dist/ /usr/share/nginx/html/vendor/codemirror/ +# Bundled packages (v0.3.8) — auto-installed on first run +COPY --from=packages /dist/ /app/bundled-packages/ + # nginx config (template — envsubst injects BASE_PATH at runtime) COPY nginx.conf.template /etc/nginx/templates/default.conf.template # Only substitute BASE_PATH — preserve nginx variables ($host, $uri, etc.) diff --git a/Dockerfile.builder b/Dockerfile.builder new file mode 100644 index 0000000..f26e4d8 --- /dev/null +++ b/Dockerfile.builder @@ -0,0 +1,65 @@ +# ============================================ +# Switchboard Core — Builder Image +# ============================================ +# Pre-caches Go modules and Node dependencies +# for faster custom builds. Use as a base in +# your own Dockerfile to skip dependency +# download on every build. +# +# Usage: +# FROM ghcr.io/switchboard-core/builder:latest AS go-builder +# COPY server/ /app/ +# RUN cd /app && go build -o /bin/switchboard . +# +# Or build this image locally: +# docker build -f Dockerfile.builder -t switchboard-builder . +# ============================================ + +# ── Go module cache ───────────────────────── +FROM golang:1.23-bookworm AS go-cache +WORKDIR /cache +COPY server/go.mod server/go.sum* ./ +RUN go mod download && go mod verify + +# ── Node dependency cache ─────────────────── +FROM node:20-alpine AS node-cache +WORKDIR /cache + +# Editor bundle dependencies +COPY src/editor/package*.json ./editor/ +RUN cd editor && npm ci --loglevel=warn + +# Vendor libs (same versions as production Dockerfile) +RUN npm pack marked@16.3.0 && \ + npm pack dompurify@3.2.4 && \ + npm pack mermaid@11.4.1 && \ + npm pack katex@0.16.11 && \ + mkdir -p /cache/vendor-tarballs && \ + mv *.tgz /cache/vendor-tarballs/ + +# ── Final builder image ──────────────────── +FROM golang:1.23-bookworm + +# Pre-install build tools +RUN apt-get update && apt-get install -y --no-install-recommends \ + zip \ + && rm -rf /var/lib/apt/lists/* + +# Go module cache (populated) +COPY --from=go-cache /go/pkg/mod /go/pkg/mod + +# Node dependencies (for editor bundle builds) +COPY --from=node-cache /cache/editor/node_modules /cache/editor/node_modules + +# Vendor lib tarballs (avoid re-download) +COPY --from=node-cache /cache/vendor-tarballs /cache/vendor-tarballs + +# Node.js for frontend builds +COPY --from=node-cache /usr/local/bin/node /usr/local/bin/node +COPY --from=node-cache /usr/local/lib/node_modules /usr/local/lib/node_modules +RUN ln -sf /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm + +WORKDIR /build + +LABEL org.opencontainers.image.title="Switchboard Core Builder" +LABEL org.opencontainers.image.description="Pre-cached build dependencies for faster custom Switchboard Core builds" diff --git a/README.md b/README.md index f52b946..89feba3 100644 --- a/README.md +++ b/README.md @@ -22,12 +22,21 @@ those are all extension packages. ## Quick Start ```bash +# Docker (recommended) +docker compose up --build +# → http://localhost:3000 (admin/admin) + +# Or from source git clone && cd switchboard-core cp server/.env.example server/.env # edit DB credentials cd server && go run . # → http://localhost:8080 ``` +Bundled packages (workflows, surfaces, task manager) are auto-installed on +first boot. See [Distribution Guide](docs/DISTRIBUTION.md) for production +deployment and customization. + ## Kernel Features - **Auth**: Builtin password, mTLS (client cert), OIDC (Keycloak et al.) @@ -43,15 +52,17 @@ cd server && go run . ## Documentation +- [Distribution Guide](docs/DISTRIBUTION.md) — Docker, bundled packages, builder image, production deployment - [Architecture](docs/ARCHITECTURE.md) — kernel components and design reasoning - [Roadmap](ROADMAP.md) — current status and planned milestones - [Changelog](CHANGELOG.md) — version history ## Project Status -**v0.1.0** (in progress) — kernel extracted from chat-switchboard v0.38.5. -~44K lines of chat/AI code removed, 27 kernel tables, 20 store interfaces. -See [ROADMAP.md](ROADMAP.md) for details. +**v0.3.8** — Distribution. Bundled packages auto-install on first boot, +builder image for faster custom builds, per-environment package allowlists. +See [ROADMAP.md](ROADMAP.md) for the full journey from v0.1.0 kernel +extraction through v0.3.x workflows to the upcoming v0.4.0 Notes surface. ## License diff --git a/ROADMAP.md b/ROADMAP.md index 6e1b38b..74c5fc7 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -220,9 +220,10 @@ Builder image for faster builds, bundled packages for zero-config first run. | Step | Status | Description | |------|--------|-------------| -| Builder image | | `Dockerfile.builder` with cached Go modules + Node vendor libs. Published to Docker Hub for faster custom builds. | -| Bundled packages | | Dockerfile stage builds example workflows + demo surface into production image. Auto-installs on first run (standard install path, admin can uninstall). `SKIP_BUNDLED_PACKAGES=true` to disable. | -| Distribution docs | | `docs/DISTRIBUTION.md` with builder image usage, custom build guide, production deployment recommendations. | +| Builder image | ✅ | `Dockerfile.builder` with cached Go modules + Node vendor libs for faster custom builds. | +| Bundled packages | ✅ | Dockerfile stage builds all 12 non-dormant packages into production image. `InstallBundledPackages` auto-installs on first run (skip-if-present). `SKIP_BUNDLED_PACKAGES=true` to disable. `BUNDLED_PACKAGES` allowlist for selective install (Helm/K8s). Migration 012 adds `bundled` source to CHECK constraint. | +| Distribution docs | ✅ | `docs/DISTRIBUTION.md` — quick start, bundled packages, builder image, custom builds, production deployment. | +| Tests | ✅ | 6 handler tests (fresh install, skip existing, missing dir, empty dir, dormant handling, allowlist filtering). All existing tests pass. | ## v0.4.0 — Notes Surface diff --git a/VERSION b/VERSION index 42045ac..6678432 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.4 +0.3.8 diff --git a/docker-compose.yml b/docker-compose.yml index 730edce..be37571 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,7 @@ services: EXT_ALLOW_PRIVATE_IPS: ${EXT_ALLOW_PRIVATE_IPS:-true} LOG_FORMAT: ${LOG_FORMAT:-text} LOG_LEVEL: ${LOG_LEVEL:-info} + BUNDLED_PACKAGES: ${BUNDLED_PACKAGES:-} # Dev seed users — ignored if ENVIRONMENT=production SEED_USERS: ${SEED_USERS:-alice:password123:user,bob:password456:user,charlie:password789:user} volumes: diff --git a/docs/DISTRIBUTION.md b/docs/DISTRIBUTION.md new file mode 100644 index 0000000..6cd8565 --- /dev/null +++ b/docs/DISTRIBUTION.md @@ -0,0 +1,202 @@ +# Distribution Guide + +## Quick Start + +```bash +docker pull ghcr.io/switchboard-core/switchboard-core:latest +docker run -p 8080:80 \ + -e SWITCHBOARD_ADMIN_USERNAME=admin \ + -e SWITCHBOARD_ADMIN_PASSWORD=changeme \ + ghcr.io/switchboard-core/switchboard-core:latest +``` + +On first run, bundled packages are automatically installed — workflows, surfaces, and extensions are ready to use immediately. + +## Bundled Packages + +The production Docker image ships with pre-built packages that are auto-installed on first boot: + +| Package | Type | Description | +|---------|------|-------------| +| bug-report-triage | workflow | Public entry, severity routing, SLA timers | +| content-approval | workflow | Multi-party signoff example | +| employee-onboarding | workflow | Automated provisioning + manager signoff | +| webhook-notifier | workflow | HTTP outbound via connections + Starlark | +| workflow-demo | surface | Interactive walkthrough with diagrams | +| hello-dashboard | surface | Welcome/getting started surface | +| schedules | surface | Cron task management UI | +| tasks | full | Kanban/list task manager with webhooks | +| team-activity-log | surface | Team activity feed | +| gitea-client | library | Gitea API integration library | +| icd-test-runner | surface | E2E API test suite | +| sdk-test-runner | surface | SDK feature test suite | + +### Behavior + +- **First boot**: All bundled packages are installed and enabled automatically. +- **Subsequent boots**: No-op — already-installed packages are skipped. +- **Admin uninstalls a package**: It stays uninstalled. Bundled packages are never force-reinstalled. +- **To re-install**: Delete the package from the database, then restart the container. + +### Selecting Packages (Allowlist) + +Set `BUNDLED_PACKAGES` to a comma-separated list of package IDs to install only a subset: + +```bash +# K8s / Helm — install only core surfaces, no demo/test packages +docker run -p 8080:80 \ + -e BUNDLED_PACKAGES="tasks,schedules,hello-dashboard" \ + ghcr.io/switchboard-core/switchboard-core:latest +``` + +Empty (default) means install all bundled packages. This is useful for Helm charts where different environments need different packages. + +### Disabling Auto-Install + +Set `SKIP_BUNDLED_PACKAGES=true` to prevent bundled packages from being installed: + +```bash +docker run -p 8080:80 \ + -e SKIP_BUNDLED_PACKAGES=true \ + ghcr.io/switchboard-core/switchboard-core:latest +``` + +### Custom Bundle Directory + +Override the default bundled packages location with `BUNDLED_PACKAGES_DIR`: + +```bash +docker run -p 8080:80 \ + -e BUNDLED_PACKAGES_DIR=/custom/packages \ + -v /host/packages:/custom/packages \ + ghcr.io/switchboard-core/switchboard-core:latest +``` + +## Builder Image + +The builder image pre-caches Go modules and Node dependencies for faster custom builds. + +```bash +docker pull ghcr.io/switchboard-core/builder:latest +``` + +### What It Caches + +- Go module cache (`go mod download` for all server dependencies) +- Node modules for the CM6 editor bundle +- Vendor library tarballs (marked, DOMPurify, mermaid, KaTeX) +- Build tools (zip, Node.js runtime) + +### Using in Custom Builds + +Reference the builder image as a base stage in your Dockerfile: + +```dockerfile +FROM ghcr.io/switchboard-core/builder:latest AS builder +WORKDIR /app +COPY server/ . +RUN go build -ldflags="-s -w" -o /bin/switchboard . +``` + +### Building Locally + +```bash +docker build -f Dockerfile.builder -t switchboard-builder . +``` + +## Custom Build Guide + +### Adding Custom Packages + +1. Create your package in `packages/your-package/` with a `manifest.json` +2. Build all packages: `cd packages && bash build.sh all` +3. Build the Docker image: `docker build -t my-switchboard .` + +The Dockerfile automatically builds all packages in the `packages/` directory and bundles them into the production image. + +### Removing Bundled Packages + +To exclude specific packages from the bundle, either: +- Remove them from `packages/` before building +- Set `SKIP_BUNDLED_PACKAGES=true` and install packages manually via the admin UI + +### Forking for Custom Builds + +```bash +git clone https://github.com/switchboard-core/switchboard-core.git +cd switchboard-core + +# Add/modify packages +cp -r my-extension packages/my-extension/ + +# Build with builder image for faster compilation +docker build -t my-switchboard . +``` + +## Production Deployment + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `PORT` | `8080` | Backend API port | +| `DB_DRIVER` | (auto) | `postgres` or `sqlite` | +| `DATABASE_URL` | | PostgreSQL connection string | +| `JWT_SECRET` | `dev-secret-change-me` | **Must change in production** | +| `ENCRYPTION_KEY` | | AES-256 key for credential encryption | +| `AUTH_MODE` | `builtin` | `builtin`, `mtls`, or `oidc` | +| `STORAGE_BACKEND` | (auto) | `pvc` or `s3` | +| `STORAGE_PATH` | `/data/storage` | PVC mount point | +| `BASE_PATH` | | URL prefix (e.g. `/switchboard`) | +| `SKIP_BUNDLED_PACKAGES` | `false` | Disable auto-install of bundled packages | +| `BUNDLED_PACKAGES` | (empty = all) | Comma-separated allowlist of package IDs to install | +| `BUNDLED_PACKAGES_DIR` | `/app/bundled-packages` | Override bundled packages location | +| `LOG_FORMAT` | `text` | `text` or `json` | +| `LOG_LEVEL` | `info` | `debug`, `info`, `warn`, `error` | + +### Database + +PostgreSQL is recommended for production. SQLite is suitable for single-instance evaluation. + +```bash +# PostgreSQL (recommended) +docker run -p 8080:80 \ + -e DATABASE_URL="postgres://user:pass@host:5432/switchboard?sslmode=require" \ + -e JWT_SECRET="$(openssl rand -hex 32)" \ + -e ENCRYPTION_KEY="$(openssl rand -hex 32)" \ + ghcr.io/switchboard-core/switchboard-core:latest + +# SQLite (evaluation only) +docker run -p 8080:80 \ + -e DB_DRIVER=sqlite \ + -v switchboard-data:/data \ + ghcr.io/switchboard-core/switchboard-core:latest +``` + +### Storage + +Object storage is required for file uploads and package asset extraction. + +```bash +# PVC (auto-detected if path is writable) +docker run -v switchboard-storage:/data/storage ... + +# S3-compatible (MinIO, AWS S3, Ceph) +docker run \ + -e STORAGE_BACKEND=s3 \ + -e S3_BUCKET=switchboard \ + -e S3_ENDPOINT=https://minio.corp:9000 \ + -e S3_ACCESS_KEY=... \ + -e S3_SECRET_KEY=... \ + -e S3_FORCE_PATH_STYLE=true \ + ... +``` + +### Kubernetes + +See the `k8s/` directory for example manifests. Key considerations: + +- Use `POSTGRES_HOST`, `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB` env vars (assembled into DSN automatically) +- Set liveness probe to `/healthz/live`, readiness probe to `/healthz/ready` +- Mount a PVC at `/data/storage` for file storage, or configure S3 +- Set `JWT_SECRET` and `ENCRYPTION_KEY` via Kubernetes Secrets diff --git a/k8s/switchboard.yaml b/k8s/switchboard.yaml index 9a4fd5a..094ebd6 100644 --- a/k8s/switchboard.yaml +++ b/k8s/switchboard.yaml @@ -155,6 +155,12 @@ spec: optional: true - name: S3_FORCE_PATH_STYLE value: "true" + # Bundled packages (v0.3.8) + # Dev: install all (empty = all). Test: install all. Prod: skip (nothing bundled needed yet). + - name: SKIP_BUNDLED_PACKAGES + value: "${SKIP_BUNDLED_PACKAGES}" + - name: BUNDLED_PACKAGES + value: "${BUNDLED_PACKAGES}" volumeMounts: - name: storage mountPath: /data/storage diff --git a/server/config/config.go b/server/config/config.go index ab67e5c..1b05acb 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -63,6 +63,15 @@ type Config struct { MTLSAutoActivate bool // auto-activate new users (default true) MTLSDefaultTeam string // team ID for auto-provisioned users (optional) + // Bundled packages (v0.3.8) + // SKIP_BUNDLED_PACKAGES: set true to disable auto-install of bundled packages on first run. + // BUNDLED_PACKAGES_DIR: directory containing pre-built .pkg archives (default /app/bundled-packages). + // BUNDLED_PACKAGES: comma-separated allowlist of package IDs to install (empty = all). + // e.g. "tasks,schedules,hello-dashboard" installs only those three. + SkipBundledPackages bool + BundledPackagesDir string + BundledPackages string + // OIDC (v0.24.1) OIDCIssuerURL string // e.g. "https://keycloak.corp/realms/switchboard" OIDCExternalIssuerURL string // OIDC_EXTERNAL_ISSUER_URL @@ -105,6 +114,10 @@ func Load() *Config { S3Prefix: getEnv("S3_PREFIX", ""), S3ForcePathStyle: getEnv("S3_FORCE_PATH_STYLE", "true") == "true", + SkipBundledPackages: getEnvBool("SKIP_BUNDLED_PACKAGES", false), + BundledPackagesDir: getEnv("BUNDLED_PACKAGES_DIR", "/app/bundled-packages"), + BundledPackages: getEnv("BUNDLED_PACKAGES", ""), + LogFormat: getEnv("LOG_FORMAT", "text"), LogLevel: getEnv("LOG_LEVEL", "info"), diff --git a/server/database/migrations/postgres/003_packages.sql b/server/database/migrations/postgres/003_packages.sql index dad0abc..3ecca0d 100644 --- a/server/database/migrations/postgres/003_packages.sql +++ b/server/database/migrations/postgres/003_packages.sql @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS packages ( schema_version INTEGER NOT NULL DEFAULT 0, package_settings JSONB NOT NULL DEFAULT '{}', source TEXT NOT NULL DEFAULT 'core' - CHECK (source IN ('core', 'builtin', 'extension', 'registry')), + CHECK (source IN ('core', 'builtin', 'extension', 'registry', 'bundled')), installed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); diff --git a/server/database/migrations/postgres/012_bundled_source.sql b/server/database/migrations/postgres/012_bundled_source.sql new file mode 100644 index 0000000..ceb58ef --- /dev/null +++ b/server/database/migrations/postgres/012_bundled_source.sql @@ -0,0 +1,6 @@ +-- 012_bundled_source.sql — v0.3.8 +-- Adds 'bundled' to the packages.source CHECK constraint for auto-installed packages. + +ALTER TABLE packages DROP CONSTRAINT IF EXISTS packages_source_check; +ALTER TABLE packages ADD CONSTRAINT packages_source_check + CHECK (source IN ('core', 'builtin', 'extension', 'registry', 'bundled')); diff --git a/server/database/migrations/sqlite/003_packages.sql b/server/database/migrations/sqlite/003_packages.sql index 16947db..96556dc 100644 --- a/server/database/migrations/sqlite/003_packages.sql +++ b/server/database/migrations/sqlite/003_packages.sql @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS packages ( schema_version INTEGER NOT NULL DEFAULT 0, package_settings TEXT NOT NULL DEFAULT '{}', source TEXT NOT NULL DEFAULT 'core' - CHECK (source IN ('core', 'builtin', 'extension', 'registry')), + CHECK (source IN ('core', 'builtin', 'extension', 'registry', 'bundled')), installed_at TEXT NOT NULL DEFAULT (datetime('now')), updated_at TEXT NOT NULL DEFAULT (datetime('now')) ); diff --git a/server/database/migrations/sqlite/012_bundled_source.sql b/server/database/migrations/sqlite/012_bundled_source.sql new file mode 100644 index 0000000..ba38af6 --- /dev/null +++ b/server/database/migrations/sqlite/012_bundled_source.sql @@ -0,0 +1,39 @@ +-- 012_bundled_source.sql — v0.3.8 +-- Adds 'bundled' to the packages.source CHECK constraint for auto-installed packages. +-- SQLite doesn't support ALTER CHECK — must recreate the table. + +CREATE TABLE IF NOT EXISTS packages_new ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + type TEXT NOT NULL DEFAULT 'surface' + CHECK (type IN ('surface', 'extension', 'full', 'workflow', 'library')), + version TEXT NOT NULL DEFAULT '0.0.0', + description TEXT NOT NULL DEFAULT '', + author TEXT NOT NULL DEFAULT '', + tier TEXT NOT NULL DEFAULT 'browser' + CHECK (tier IN ('browser', 'starlark', 'sidecar')), + is_system INTEGER NOT NULL DEFAULT 0, + scope TEXT NOT NULL DEFAULT 'global' + CHECK (scope IN ('global', 'team', 'personal')), + team_id TEXT REFERENCES teams(id) ON DELETE CASCADE, + installed_by TEXT REFERENCES users(id) ON DELETE SET NULL, + manifest TEXT NOT NULL DEFAULT '{}', + enabled INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'pending_review', 'suspended', 'dormant')), + schema_version INTEGER NOT NULL DEFAULT 0, + package_settings TEXT NOT NULL DEFAULT '{}', + source TEXT NOT NULL DEFAULT 'core' + CHECK (source IN ('core', 'builtin', 'extension', 'registry', 'bundled')), + installed_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +INSERT OR IGNORE INTO packages_new SELECT * FROM packages; +DROP TABLE packages; +ALTER TABLE packages_new RENAME TO packages; + +CREATE INDEX IF NOT EXISTS idx_packages_type ON packages(type); +CREATE INDEX IF NOT EXISTS idx_packages_enabled ON packages(enabled); +CREATE INDEX IF NOT EXISTS idx_packages_team ON packages(team_id); +CREATE INDEX IF NOT EXISTS idx_packages_source ON packages(source); diff --git a/server/handlers/packages_bundled.go b/server/handlers/packages_bundled.go new file mode 100644 index 0000000..5e9367f --- /dev/null +++ b/server/handlers/packages_bundled.go @@ -0,0 +1,309 @@ +package handlers + +import ( + "archive/zip" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/gin-gonic/gin" + + "switchboard-core/database" + "switchboard-core/models" + "switchboard-core/sandbox" + "switchboard-core/store" + "switchboard-core/triggers" +) + +// InstallBundledPackages scans bundledDir for .pkg archives and installs +// any that don't already exist in the database. Called once at startup +// (unless SKIP_BUNDLED_PACKAGES=true). +// +// allowlist is an optional comma-separated list of package IDs to install. +// Empty string means install all. This allows Helm/K8s deployments to +// select a subset: e.g. "tasks,schedules,hello-dashboard". +// +// Design: install-once, skip-if-present. If an admin uninstalls a bundled +// package, it won't be re-installed on the next restart. +func InstallBundledPackages(bundledDir, packagesDir, allowlist string, stores store.Stores, runner *sandbox.Runner) { + entries, err := os.ReadDir(bundledDir) + if err != nil { + if os.IsNotExist(err) { + log.Printf("[bundled] No bundled packages directory at %s — skipping", bundledDir) + return + } + log.Printf("[bundled] ⚠️ Cannot read bundled packages: %v", err) + return + } + + // Parse allowlist into a set (empty = allow all) + allowed := parseBundleAllowlist(allowlist) + if len(allowed) > 0 { + log.Printf("[bundled] Allowlist: %v", allowlist) + } + + var installed, skipped, filtered int + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".pkg") { + continue + } + + // Check allowlist by filename (strip .pkg extension = package ID) + if len(allowed) > 0 { + pkgName := strings.TrimSuffix(entry.Name(), ".pkg") + if !allowed[pkgName] { + filtered++ + continue + } + } + + pkgPath := filepath.Join(bundledDir, entry.Name()) + result, err := installBundledPackage(pkgPath, packagesDir, stores, runner) + if err != nil { + log.Printf("[bundled] ⚠️ Failed to install %s: %v", entry.Name(), err) + continue + } + if result == "skipped" { + skipped++ + } else { + installed++ + } + } + + if installed > 0 || skipped > 0 || filtered > 0 { + msg := fmt.Sprintf("[bundled] Installed %d, skipped %d existing", installed, skipped) + if filtered > 0 { + msg += fmt.Sprintf(", filtered %d by allowlist", filtered) + } + log.Println(msg) + } +} + +// parseBundleAllowlist parses a comma-separated string into a set of +// package IDs. Returns nil for empty input (meaning "allow all"). +func parseBundleAllowlist(s string) map[string]bool { + s = strings.TrimSpace(s) + if s == "" { + return nil + } + parts := strings.Split(s, ",") + m := make(map[string]bool, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + m[p] = true + } + } + return m +} + +// installBundledPackage installs a single .pkg archive if its package ID +// doesn't already exist in the database. Returns "installed" or "skipped". +func installBundledPackage(pkgPath, packagesDir string, stores store.Stores, runner *sandbox.Runner) (string, error) { + ctx := context.Background() + + // Open as zip + zr, err := zip.OpenReader(pkgPath) + if err != nil { + return "", fmt.Errorf("invalid zip archive: %w", err) + } + defer zr.Close() + + // Find and parse manifest.json + manifest, err := extractManifest(zr) + if err != nil { + return "", err + } + + pkgID, _ := manifest["id"].(string) + if pkgID == "" { + return "", fmt.Errorf("manifest missing 'id'") + } + + // Skip if already exists (admin uninstalled → stays uninstalled) + existing, _ := stores.Packages.Get(ctx, pkgID) + if existing != nil { + return "skipped", nil + } + + title, _ := manifest["title"].(string) + if title == "" { + return "", fmt.Errorf("manifest missing 'title'") + } + + pkgType, _ := manifest["type"].(string) + if pkgType == "" { + pkgType = "surface" + } + + // Extract static assets to packagesDir/{id}/ + if packagesDir != "" { + if err := extractPackageAssets(zr, packagesDir, pkgID); err != nil { + return "", fmt.Errorf("asset extraction: %w", err) + } + } + + // Extract manifest fields for DB columns + version, _ := manifest["version"].(string) + if version == "" { + version = "0.0.0" + } + description, _ := manifest["description"].(string) + author, _ := manifest["author"].(string) + tier, _ := manifest["tier"].(string) + if tier == "" { + tier = "browser" + } + + // Register in database via Seed (upsert) + if err := stores.Packages.Seed(ctx, pkgID, title, "bundled", manifest); err != nil { + return "", fmt.Errorf("seed failed: %w", err) + } + + // Update fields that Seed doesn't set + pkg := &store.PackageRegistration{ + Title: title, + Type: pkgType, + Version: version, + Description: description, + Author: author, + Tier: tier, + IsSystem: false, + Enabled: true, + Manifest: manifest, + } + if err := stores.Packages.Update(ctx, pkgID, pkg); err != nil { + log.Printf("[bundled] Failed to update package metadata for %s: %v", pkgID, err) + } + + // Create namespaced DB tables declared in the manifest + if tables, ok := ParseDBTables(manifest); ok { + if err := CreateExtTables(ctx, database.DB, database.IsPostgres(), stores, pkgID, tables); err != nil { + log.Printf("[bundled] [ext-db] schema create failed for %s: %v", pkgID, err) + } + } + + // Sync permissions — use a minimal gin.Context for the functions that need it + fakeCtx := newBackgroundGinContext() + SyncManifestPermissions(fakeCtx, stores, pkgID, manifest) + + // Sync triggers from manifest + SyncManifestTriggers(ctx, stores, triggers.GlobalEngine(), pkgID, manifest) + + // Install workflow definition if applicable + if pkgType == "workflow" { + if err := InstallWorkflowFromManifest(fakeCtx, stores, pkgID, manifest); err != nil { + log.Printf("[bundled] workflow install failed for %s: %v", pkgID, err) + } + } + + // Handle dormant status for packages with unmet requires + knownCaps := map[string]bool{} + if reqs, ok := manifest["requires"].([]any); ok && len(reqs) > 0 { + var unmetReqs []string + for _, r := range reqs { + req, _ := r.(string) + if req != "" && !knownCaps[req] { + unmetReqs = append(unmetReqs, req) + } + } + if len(unmetReqs) > 0 { + stores.Packages.SetStatus(ctx, pkgID, "dormant") + stores.Packages.SetEnabled(ctx, pkgID, false) + log.Printf("[bundled] %s: dormant (unmet requires: %v)", pkgID, unmetReqs) + } + } + + // Auto-set default_surface when the first surface is installed + if (pkgType == "surface" || pkgType == "full") { + if raw, err := stores.GlobalConfig.Get(ctx, "default_surface"); err != nil || raw == nil { + dflt := models.JSONMap{"id": pkgID} + if setErr := stores.GlobalConfig.Set(ctx, "default_surface", dflt, ""); setErr != nil { + log.Printf("[bundled] failed to auto-set default_surface to %s: %v", pkgID, setErr) + } + } + } + + log.Printf("[bundled] Installed %s (%s) v%s", pkgID, pkgType, version) + return "installed", nil +} + +// extractManifest reads and parses manifest.json from a zip archive. +func extractManifest(zr *zip.ReadCloser) (map[string]any, error) { + for _, f := range zr.File { + name := filepath.Base(f.Name) + if name == "manifest.json" && !f.FileInfo().IsDir() { + rc, err := f.Open() + if err != nil { + continue + } + data, err := io.ReadAll(rc) + rc.Close() + if err != nil { + continue + } + var manifest map[string]any + if err := json.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("invalid manifest.json: %w", err) + } + return manifest, nil + } + } + return nil, fmt.Errorf("archive missing manifest.json") +} + +// extractPackageAssets extracts static assets from a zip archive to packagesDir/{pkgID}/. +func extractPackageAssets(zr *zip.ReadCloser, packagesDir, pkgID string) error { + destDir := filepath.Join(packagesDir, pkgID) + os.MkdirAll(destDir, 0755) + + for _, f := range zr.File { + if f.FileInfo().IsDir() { + continue + } + + relPath := extractableRelPath(f.Name) + if relPath == "" { + continue + } + + destPath := filepath.Join(destDir, relPath) + + // Security: prevent path traversal + if !strings.HasPrefix(filepath.Clean(destPath), filepath.Clean(destDir)) { + continue + } + + os.MkdirAll(filepath.Dir(destPath), 0755) + + rc, err := f.Open() + if err != nil { + continue + } + out, err := os.Create(destPath) + if err != nil { + rc.Close() + continue + } + io.Copy(out, rc) + out.Close() + rc.Close() + } + return nil +} + +// newBackgroundGinContext creates a minimal gin.Context backed by a +// background context. Used by startup code that calls functions +// originally designed for HTTP handlers. +func newBackgroundGinContext() *gin.Context { + req, _ := http.NewRequestWithContext(context.Background(), "GET", "/", nil) + c := &gin.Context{} + c.Request = req + return c +} diff --git a/server/handlers/packages_bundled_test.go b/server/handlers/packages_bundled_test.go new file mode 100644 index 0000000..170ef80 --- /dev/null +++ b/server/handlers/packages_bundled_test.go @@ -0,0 +1,226 @@ +package handlers + +import ( + "archive/zip" + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "switchboard-core/database" + "switchboard-core/store" + postgres "switchboard-core/store/postgres" + sqlite "switchboard-core/store/sqlite" +) + +// ── Test helpers ────────────────────────────── + +func newTestStores(t *testing.T) store.Stores { + t.Helper() + database.RequireTestDB(t) + database.TruncateAll(t) + + if database.IsSQLite() { + return sqlite.NewStores(database.TestDB) + } + return postgres.NewStores(database.TestDB) +} + +// buildTestPkg creates a minimal .pkg archive in dir with the given manifest. +func buildTestPkg(t *testing.T, dir string, manifest map[string]any) string { + t.Helper() + + pkgID, _ := manifest["id"].(string) + if pkgID == "" { + t.Fatal("manifest must have 'id'") + } + + data, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + + pkgPath := filepath.Join(dir, pkgID+".pkg") + f, err := os.Create(pkgPath) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + zw := zip.NewWriter(f) + w, err := zw.Create("manifest.json") + if err != nil { + t.Fatal(err) + } + if _, err := w.Write(data); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + + return pkgPath +} + +// ═══════════════════════════════════════════════ +// Tests +// ═══════════════════════════════════════════════ + +// TestBundledInstall_FreshDB verifies that bundled packages are installed +// into an empty database on first run. +func TestBundledInstall_FreshDB(t *testing.T) { + stores := newTestStores(t) + ctx := context.Background() + + bundledDir := t.TempDir() + packagesDir := t.TempDir() + + buildTestPkg(t, bundledDir, map[string]any{ + "id": "test-surface-a", + "title": "Test Surface A", + "type": "surface", + }) + buildTestPkg(t, bundledDir, map[string]any{ + "id": "test-surface-b", + "title": "Test Surface B", + "type": "surface", + "route": "/s/test-b", + }) + + InstallBundledPackages(bundledDir, packagesDir, "", stores, nil) + + // Both packages should be installed and enabled + for _, id := range []string{"test-surface-a", "test-surface-b"} { + pkg, err := stores.Packages.Get(ctx, id) + if err != nil || pkg == nil { + t.Fatalf("package %s not found after install", id) + } + if !pkg.Enabled { + t.Errorf("package %s should be enabled", id) + } + if pkg.Source != "bundled" { + t.Errorf("package %s source = %q, want %q", id, pkg.Source, "bundled") + } + } +} + +// TestBundledInstall_SkipsExisting verifies that already-installed packages +// are not re-installed (preserves admin uninstall/reconfigure). +func TestBundledInstall_SkipsExisting(t *testing.T) { + stores := newTestStores(t) + ctx := context.Background() + + bundledDir := t.TempDir() + packagesDir := t.TempDir() + + // Pre-seed a package with same ID but different title (simulating existing install) + manifest := map[string]any{"id": "pre-existing", "title": "Original Title", "type": "surface"} + if err := stores.Packages.Seed(ctx, "pre-existing", "Original Title", "extension", manifest); err != nil { + t.Fatal(err) + } + + // Bundled package has same ID but different title + buildTestPkg(t, bundledDir, map[string]any{ + "id": "pre-existing", + "title": "Bundled Title", + "type": "surface", + }) + + InstallBundledPackages(bundledDir, packagesDir, "", stores, nil) + + // Package should retain original title (not overwritten) + pkg, err := stores.Packages.Get(ctx, "pre-existing") + if err != nil || pkg == nil { + t.Fatal("pre-existing package not found") + } + if pkg.Title != "Original Title" { + t.Errorf("title = %q, want %q (should not overwrite existing)", pkg.Title, "Original Title") + } +} + +// TestBundledInstall_MissingDir verifies graceful handling when bundled +// packages directory doesn't exist. +func TestBundledInstall_MissingDir(t *testing.T) { + stores := newTestStores(t) + + // Should not panic or error — just log and return + InstallBundledPackages("/nonexistent/path", "", "", stores, nil) +} + +// TestBundledInstall_EmptyDir verifies no-op when directory exists but is empty. +func TestBundledInstall_EmptyDir(t *testing.T) { + stores := newTestStores(t) + + bundledDir := t.TempDir() + + // Should not panic or error + InstallBundledPackages(bundledDir, "", "", stores, nil) +} + +// TestBundledInstall_DormantPackage verifies that bundled packages with +// unmet requires are marked dormant. +func TestBundledInstall_DormantPackage(t *testing.T) { + stores := newTestStores(t) + ctx := context.Background() + + bundledDir := t.TempDir() + packagesDir := t.TempDir() + + buildTestPkg(t, bundledDir, map[string]any{ + "id": "dormant-pkg", + "title": "Dormant Package", + "type": "extension", + "requires": []string{"chat"}, + }) + + InstallBundledPackages(bundledDir, packagesDir, "", stores, nil) + + pkg, err := stores.Packages.Get(ctx, "dormant-pkg") + if err != nil || pkg == nil { + t.Fatal("dormant package not found after install") + } + if pkg.Status != "dormant" { + t.Errorf("status = %q, want %q", pkg.Status, "dormant") + } + if pkg.Enabled { + t.Error("dormant package should not be enabled") + } +} + +// TestBundledInstall_Allowlist verifies that BUNDLED_PACKAGES allowlist +// filters which packages are installed. +func TestBundledInstall_Allowlist(t *testing.T) { + stores := newTestStores(t) + ctx := context.Background() + + bundledDir := t.TempDir() + packagesDir := t.TempDir() + + buildTestPkg(t, bundledDir, map[string]any{ + "id": "pkg-alpha", "title": "Alpha", "type": "surface", + }) + buildTestPkg(t, bundledDir, map[string]any{ + "id": "pkg-beta", "title": "Beta", "type": "surface", + }) + buildTestPkg(t, bundledDir, map[string]any{ + "id": "pkg-gamma", "title": "Gamma", "type": "surface", + }) + + // Only allow alpha and gamma + InstallBundledPackages(bundledDir, packagesDir, "pkg-alpha,pkg-gamma", stores, nil) + + // Alpha and gamma should be installed + for _, id := range []string{"pkg-alpha", "pkg-gamma"} { + pkg, err := stores.Packages.Get(ctx, id) + if err != nil || pkg == nil { + t.Fatalf("package %s should have been installed (in allowlist)", id) + } + } + + // Beta should NOT be installed + pkg, _ := stores.Packages.Get(ctx, "pkg-beta") + if pkg != nil { + t.Fatal("pkg-beta should NOT have been installed (not in allowlist)") + } +} diff --git a/server/main.go b/server/main.go index d14fee2..b2e6178 100644 --- a/server/main.go +++ b/server/main.go @@ -174,6 +174,17 @@ func main() { log.Printf(" ⚠️ Extension SSRF protection relaxed: private IPs allowed") } + // ── Bundled Packages (v0.3.8) ─────────────── + // Auto-install bundled .pkg archives on first run. + // Skipped if SKIP_BUNDLED_PACKAGES=true or packages already exist. + if !cfg.SkipBundledPackages && stores.Packages != nil { + bundledPkgDir := "" + if cfg.StoragePath != "" { + bundledPkgDir = cfg.StoragePath + "/packages" + } + handlers.InstallBundledPackages(cfg.BundledPackagesDir, bundledPkgDir, cfg.BundledPackages, stores, starlarkRunner) + } + // ── Trigger Engine (v0.2.2) ───────────────── triggerEngine := triggers.New(stores, starlarkRunner, bus) if err := triggerEngine.Start(context.Background()); err != nil {