7 Commits

Author SHA1 Message Date
5ad6d77c56 Feat v0.9.0 multi surface packages (#73)
All checks were successful
CI/CD / detect-changes (push) Successful in 4s
CI/CD / test-runners (push) Has been skipped
CI/CD / e2e-smoke (push) Has been skipped
CI/CD / test-frontend (push) Successful in 5s
CI/CD / test-go-pg (push) Successful in 2m46s
CI/CD / test-sqlite (push) Successful in 2m57s
CI/CD / build-and-deploy (push) Successful in 44s
Co-authored-by: Jeffrey Smith <jasafpro@gmail.com>
Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>
2026-04-03 12:40:47 +00:00
98fd3eb3e6 Feat v0.8.5 extension composability (#72)
All checks were successful
CI/CD / detect-changes (push) Successful in 4s
CI/CD / test-runners (push) Has been skipped
CI/CD / e2e-smoke (push) Has been skipped
CI/CD / test-frontend (push) Successful in 6s
CI/CD / test-go-pg (push) Successful in 2m44s
CI/CD / test-sqlite (push) Successful in 2m54s
CI/CD / build-and-deploy (push) Successful in 52s
Co-authored-by: Jeffrey Smith <jasafpro@gmail.com>
Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>
2026-04-03 11:33:47 +00:00
3c403dd884 Feat v0.8.4 docs surface fix (#71)
All checks were successful
CI/CD / detect-changes (push) Successful in 4s
CI/CD / test-runners (push) Has been skipped
CI/CD / e2e-smoke (push) Has been skipped
CI/CD / test-frontend (push) Successful in 6s
CI/CD / test-go-pg (push) Successful in 2m53s
CI/CD / test-sqlite (push) Successful in 2m56s
CI/CD / build-and-deploy (push) Successful in 1m18s
Co-authored-by: Jeffrey Smith <jasafpro@gmail.com>
Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>
2026-04-03 10:43:13 +00:00
190905b3e6 Feat v0.8.3 vector column (#70)
All checks were successful
CI/CD / detect-changes (push) Successful in 4s
CI/CD / test-runners (push) Has been skipped
CI/CD / test-frontend (push) Has been skipped
CI/CD / e2e-smoke (push) Has been skipped
CI/CD / test-sqlite (push) Successful in 2m59s
CI/CD / test-go-pg (push) Successful in 2m58s
CI/CD / build-and-deploy (push) Successful in 1m14s
Co-authored-by: Jeffrey Smith <jasafpro@gmail.com>
Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>
2026-04-03 09:41:32 +00:00
00ef970163 Feat v0.8.2 capability negotiation (#69)
All checks were successful
CI/CD / detect-changes (push) Successful in 4s
CI/CD / test-frontend (push) Has been skipped
CI/CD / test-runners (push) Has been skipped
CI/CD / e2e-smoke (push) Has been skipped
CI/CD / test-go-pg (push) Successful in 2m47s
CI/CD / test-sqlite (push) Successful in 2m52s
CI/CD / build-and-deploy (push) Successful in 30s
Co-authored-by: Jeffrey Smith <jasafpro@gmail.com>
Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>
2026-04-03 09:14:00 +00:00
435f972ded Feat v0.8.1 workspace module (#68)
All checks were successful
CI/CD / detect-changes (push) Successful in 4s
CI/CD / test-frontend (push) Has been skipped
CI/CD / test-runners (push) Has been skipped
CI/CD / e2e-smoke (push) Has been skipped
CI/CD / test-go-pg (push) Successful in 2m49s
CI/CD / test-sqlite (push) Successful in 2m54s
CI/CD / build-and-deploy (push) Successful in 36s
Co-authored-by: Jeffrey Smith <jasafpro@gmail.com>
Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>
2026-04-03 00:38:40 +00:00
694779fac6 Feat v0.8.0 files module (#67)
All checks were successful
CI/CD / detect-changes (push) Successful in 4s
CI/CD / test-frontend (push) Has been skipped
CI/CD / test-runners (push) Has been skipped
CI/CD / e2e-smoke (push) Has been skipped
CI/CD / test-go-pg (push) Successful in 2m45s
CI/CD / test-sqlite (push) Successful in 2m52s
CI/CD / build-and-deploy (push) Successful in 1m20s
Co-authored-by: Jeffrey Smith <jasafpro@gmail.com>
Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>
2026-04-03 00:18:05 +00:00
51 changed files with 5363 additions and 694 deletions

View File

@@ -2,6 +2,309 @@
All notable changes to Armature are documented here. All notable changes to Armature are documented here.
## v0.9.0 — Multi-Surface Packages
Packages can now declare multiple surfaces — each with its own path, access
level, title, and layout. A single package can serve a public submission
form, an authenticated dashboard, and an admin settings page. The kernel
resolves incoming requests to the correct surface and enforces access
server-side.
**Manifest: `surfaces` array**
- Packages declare a `surfaces` array with entries like
`{ "path": "/submit", "access": "public", "title": "Report a Bug" }`.
- Each surface has independent `path`, `access`, `title`, `layout`, and
`nav` fields. Package-level `auth` and `layout` serve as defaults.
- Supported access levels: `public`, `authenticated`, `admin`, `group:{name}`.
- Packages without `surfaces` get auto-synthesis from legacy `auth`/`layout`
fields — no existing packages break.
**Kernel: unified route tree**
- Surface handler and ext API handler share a single `/s/:slug` route tree
via `RegisterExtensionRoutes`. The dispatcher checks the path prefix:
`/api/*` → ext API handler with JWT auth; everything else → surface
handler with per-surface access checks.
- `matchSurface()` resolves request paths against surface patterns with
Gin-style `:param` support. Static segments preferred over params.
- `evaluateAccess()` checks access requirements per-surface, with login
redirect for unauthenticated users and 403 for insufficient permissions.
**Frontend: `__SURFACE_PATH__` + `sw.navigate()`**
- `window.__SURFACE_PATH__` and `window.__SURFACE_PARAMS__` injected into
every extension surface page. Packages use these to decide which view to
render.
- `sw.navigate(path, params)` for SPA-style intra-package routing via
`pushState`. Emits `surface.navigate` events. Handles back/forward via
`popstate`.
- SDK version bumped to `0.9.0`.
**Navigation**
- `extensionNavItems` reads the `surfaces` array to find the nav entry
(first `nav: true`, or root `/`).
- Fix: packages with `status: pending_review` no longer appear in the
sidebar navigation.
**Validation**
- `ValidateManifest` validates `surfaces` entries: path required, must
start with `/`, no duplicates, access level must be recognized.
- Empty `surfaces` array rejected. Non-array `surfaces` rejected.
**Tests: 22 new**
- 11 manifest validation tests (surfaces valid/invalid, auto-synthesis,
group access, duplicate paths).
- 11 route matching tests (static paths, param extraction, specificity
ordering, nav resolution).
**Modified files:**
- `server/handlers/package_validate.go` — surfaces validation + auto-synthesis
- `server/handlers/package_validate_test.go` — 11 new tests
- `server/main.go` — unified extension route registration
- `server/pages/pages.go` — matchSurface, evaluateAccess, findNavSurface,
RegisterExtensionRoutes, nav status filter
- `server/pages/pages_surface_match_test.go` — 11 new tests
- `server/pages/templates/base.html` — surface path/params injection
- `src/js/sw/sdk/index.js` — sw.navigate, popstate, version bump
- `docs/PACKAGE-FORMAT.md` — surfaces field documentation
- `docs/MULTI-SURFACE-GUIDE.md` — developer guide
## v0.8.5 — Extension Composability
Extensions can now compose with each other through declared slots, UI
contributions, and cross-package function calls. This is the last kernel
feature before 1.0 — the platform now supports the "extensions extending
extensions" pattern.
**Manifest declarations**
- Surfaces declare named `slots` in their manifest (e.g., `toolbar-actions`,
`note-footer`) with context documentation for extension authors.
- Extensions declare `contributes` entries targeting `{host}:{slot}` names
(e.g., `notes:toolbar-actions`). Coupling is soft — install order doesn't
matter.
- The kernel validates slot/contribution conventions at install time.
**Backend: `lib.require()` relaxation**
- `lib.require()` now works with any package that declares `exports`, not
just `type: "library"` packages. A full package (with surfaces, settings,
UI) can export callable functions for cross-package use.
- The existing `depends` and permission model is unchanged — the called
function runs with the target package's permissions.
**Admin slots endpoint**
- `GET /api/v1/admin/slots` returns an aggregated map of all declared slots
across installed packages with their contributors.
- Uninstalling a package that declares slots warns about orphaned
contributions in other packages.
**SDK additions**
- `sw.slots.renderAll(name, context)` — convenience helper for host surfaces
to render all components in a slot with error isolation.
- `sw.slots.declare(name, description)` — runtime slot declaration for
discoverability and debugging.
**Documentation**
- `PACKAGE-FORMAT.md` — added `slots`, `contributes`, `depends` field docs.
- `EXTENSION-GUIDE.md` — new "Extension Composability" section with slot
naming conventions, contribution patterns, and cross-package call examples.
- `STARLARK-REFERENCE.md` — updated `lib` module to reflect exports-based
calling (not library-type-only).
**Modified files:**
- `server/sandbox/lib_module.go` — type check → exports check
- `server/handlers/extensions.go` — composability field validation
- `server/handlers/packages.go` — orphaned contribution warning
- `server/main.go` — admin slots route registration
- `src/js/sw/sdk/slots.js``renderAll()`, `declare()`, `declarations()`
- `docs/PACKAGE-FORMAT.md` — slots, contributes, depends
- `docs/EXTENSION-GUIDE.md` — composability section
- `docs/STARLARK-REFERENCE.md` — lib module update
- `docs/DESIGN-extension-composability.md` — status Draft → Implemented
**New files:**
- `server/handlers/admin_slots.go` — admin slot aggregation endpoint
## v0.8.4 — Documentation Refresh + Surface Sizing Fix
Eight versions of module additions (v0.7.5v0.8.3) shipped without a
docs pass. This release brings the public-facing guides up to date and
fixes a CSS layout bug affecting all surfaces.
**Documentation refresh**
- `STARLARK-REFERENCE.md` — added `workspace` module section (5 builtins),
`permissions` module section, and `settings.has_capability()` documentation.
- `EXTENSION-GUIDE.md` — added `capabilities` manifest block, `vector(N)`
column type, `user_permissions` and `gate_permission` manifest fields,
updated sandbox permissions list with v0.8.0+ additions (`files.read`,
`files.write`, `workspace.manage`).
- `TUTORIAL-FIRST-EXTENSION.md` — reviewed for v0.7+ accuracy (no changes needed).
**Surface sizing fix**
All surfaces using the shell topbar had scroll content clipped at the
bottom by ~44px (the topbar height). Root cause: surface containers were
siblings of `#shell-topbar` inside `.surface-inner`, which used
`height: 100%` without flex layout — the surface div claimed the full
parent height, ignoring the topbar sibling.
- Fix: `.surface-inner` now uses `display: flex; flex-direction: column`
so the topbar and surface share vertical space via flex layout.
- All surface containers (`.surface-docs`, `.surface-admin`,
`.surface-settings`, `.surface-editor`, `.extension-surface`) changed
from `height: 100%` to `flex: 1; min-height: 0`.
- Inline styles on `surface-team-admin` and `welcome-mount` templates
updated to match.
**Modified files:**
- `server/pages/templates/base.html``.surface-inner` gains flex column layout
- `src/css/surfaces.css``.surface-docs`, `.surface-admin`, `.surface-settings`, `.surface-editor`
- `src/css/extension-surface.css``.extension-surface`
- `server/pages/templates/surfaces/team-admin.html` — inline style fix
- `server/pages/templates/surfaces/welcome.html` — inline style fix
- `docs/STARLARK-REFERENCE.md` — workspace, permissions, has_capability
- `docs/EXTENSION-GUIDE.md` — capabilities, vector, user_permissions, gate_permission
## v0.8.3 — Vector Column Type
Extensions can now declare vector columns and perform similarity search.
Three-tier progressive enhancement: native pgvector on Postgres, JSONB
fallback without pgvector, TEXT fallback on SQLite.
**Manifest: `db_tables` vector columns**
- Declare `"vector(N)"` as a column type (N = dimension, 14096).
- On Postgres with pgvector: native `vector(N)` type with auto-created
HNSW index (`vector_cosine_ops`).
- On Postgres without pgvector: `JSONB` column.
- On SQLite: `TEXT` column (JSON-encoded float arrays).
**Starlark API**
- `db.query_similar(table, column, vector=[], limit=10, filters={}, metric="cosine")`
— returns rows ordered by ascending cosine distance with injected `_distance` key.
- `db.insert()` now accepts list values (serialized as JSON strings) for
vector column storage.
**Internal**
- Modified: `handlers/ext_db_schema.go``parseVectorDim`, `mapColType`
gains `hasPgvector` parameter, HNSW index creation for vector columns.
- Modified: `sandbox/db_module.go``HasPgvector` in `DBModuleConfig`,
list support in `starlarkToGoValue`, `dbQuerySimilar` with pgvector and
fallback paths, `cosineDistance` helper.
- Modified: `sandbox/runner.go` — wire `HasPgvector` from capabilities.
- Modified: `handlers/extensions.go``SetCapabilities` on `ExtensionHandler`.
- Modified: `server/main.go` — wire capabilities to extension handler.
- Updated: `docs/STARLARK-REFERENCE.md` — vector similarity section.
- New tests: 5 schema tests + 8 db module tests (13 total).
## v0.8.2 — Capability Negotiation
Extensions declare environment requirements in their manifest. The kernel
validates at install time and exposes a runtime query for graceful degradation.
**Manifest: `capabilities` block**
- `capabilities.required` — array of capability names. Install is rejected
(HTTP 422) if any are unavailable. Rollback deletes the package row.
- `capabilities.optional` — array of capability names. Install succeeds
regardless; extensions query at runtime via `settings.has_capability()`.
**Detected capabilities:** `postgres`, `pgvector`, `object_storage`, `s3`,
`workspace`.
**Starlark API**
- `settings.has_capability(name)` — returns `True` or `False`. Always
available (no permission required).
**Admin API**
- `GET /api/v1/admin/capabilities` — re-probes and returns current state.
**Internal**
- New: `handlers/capabilities.go` (detection, parsing, validation, admin handler).
- New: `handlers/capabilities_test.go` (14 tests).
- New: `sandbox/settings_module_test.go` (4 tests).
- Modified: `handlers/packages.go` — replaced stub `checkCapabilities` with
real validation; `SetCapabilities` setter.
- Modified: `handlers/packages_bundled.go` — bundled packages with unmet
required capabilities are skipped on startup.
- Modified: `sandbox/settings_module.go``has_capability` builtin.
- Modified: `sandbox/runner.go``capabilities` field + `SetCapabilities` setter.
- Modified: `server/main.go``DetectCapabilities` at startup, wired to
runner and package handler, admin route registered.
---
## v0.8.1 — Workspace Module
New `workspace` sandbox module. Managed disk directories for extensions
that need a real filesystem (git, compilers, media tools).
**workspace module (permission: `workspace.manage`)**
- `workspace.create(name)` — create a workspace directory (idempotent). Returns absolute path. Enforces quota if `WORKSPACE_QUOTA_MB` > 0.
- `workspace.path(name)` — get the absolute path of an existing workspace. Returns None if not found.
- `workspace.list()` — list workspace names for this extension.
- `workspace.delete(name)` — recursively remove a workspace (idempotent).
- `workspace.usage(name)` — disk usage in bytes (10-second timeout on directory walk).
**Configuration**
- `WORKSPACE_ROOT` — mount point for extension workspaces (default `/data/workspaces`).
- `WORKSPACE_QUOTA_MB` — per-extension quota in MB (default `0` = unlimited).
**Internal**
- New file: `sandbox/workspace_module.go`.
- New permission constant: `ExtPermWorkspaceManage`.
- Config: `WorkspaceRoot`, `WorkspaceQuotaMB` fields.
- Runner wiring: `SetWorkspaceRoot()` setter, `buildModulesWithLibCtx` creates workspace module when permission granted.
- Startup: `main.go` creates workspace root directory if writable, graceful degradation if not.
- All directories scoped to `{WORKSPACE_ROOT}/{packageID}/{name}/`.
- Security: name regex (`^[a-z][a-z0-9_]{0,62}$`), `filepath.Clean` + prefix check, `filepath.EvalSymlinks` for symlink escape detection.
- 16 new tests (create, path, list, delete, usage, name validation, quota enforcement).
## v0.8.0 — Files Module
New `files` sandbox module. Bridges the existing ObjectStore (PVC/S3) into
the Starlark sandbox with extension-scoped key namespacing.
**files module (permissions: `files.read`, `files.write`)**
- `files.put(name, content, content_type, metadata)` — store a file with optional metadata companion. Accepts string or bytes content. 50 MB default limit (configurable via `EXT_FILES_MAX_SIZE`).
- `files.get(name)` — read a file. Returns dict with `content` (bytes), `content_type`, `size`, `metadata`. Returns None if not found.
- `files.meta(name)` — read metadata only (no content transfer).
- `files.list(prefix, limit)` — list files by prefix. Returns list of dicts. Filters out internal `_meta/` companions.
- `files.delete(name)` — delete a file and its metadata companion. Idempotent.
- `files.delete_prefix(prefix)` — delete all files under a prefix.
- `files.exists(name)` — check existence without reading.
**Internal**
- New file: `sandbox/files_module.go`.
- New permission constants: `ExtPermFilesRead`, `ExtPermFilesWrite`.
- `ObjectStore` interface gains `List(ctx, prefix, limit)` method; implemented for PVC and S3.
- Runner wiring: `SetObjectStore()` setter, `buildModulesWithLibCtx` creates files module when permission granted.
- All keys scoped to `ext/{packageID}/`. Metadata stored as companion JSON at `ext/{packageID}/_meta/{name}`.
- 16 new tests (15 files module + 1 PVC list).
## v0.7.12 — Concurrent Execution Primitive ## v0.7.12 — Concurrent Execution Primitive
New `batch` sandbox module. Enables extensions to parallelize arbitrary New `batch` sandbox module. Enables extensions to parallelize arbitrary

View File

@@ -1,6 +1,6 @@
# Armature — Roadmap # Armature — Roadmap
## Current: v0.7.12 — Concurrent Execution Primitive ## Current: v0.9.x — Workflow Redesign
Self-hosted extensible platform kernel. Auth, identity, packages, Starlark Self-hosted extensible platform kernel. Auth, identity, packages, Starlark
sandbox, storage, realtime, and ops are kernel primitives. Everything else sandbox, storage, realtime, and ops are kernel primitives. Everything else
@@ -9,544 +9,213 @@ is an extension.
**Kernel capabilities:** Auth (builtin/mTLS/OIDC) · Users/teams/groups/RBAC · **Kernel capabilities:** Auth (builtin/mTLS/OIDC) · Users/teams/groups/RBAC ·
Surfaces/extensions/libraries/workflows · Starlark sandbox (capability-gated) · Surfaces/extensions/libraries/workflows · Starlark sandbox (capability-gated) ·
Object storage (PVC/S3) + ext_data tables · WebSocket hub + realtime pub/sub · Object storage (PVC/S3) + ext_data tables · WebSocket hub + realtime pub/sub ·
Audit log · Notifications · Scheduled tasks · Cluster registry + HA Audit log · Notifications · Scheduled tasks · Cluster registry + HA ·
Extension composability (slots/contributes/cross-package calls)
**Completed history:** v0.2.xv0.5.x fully documented in `CHANGELOG.md`.
Highlights: RBAC + settings cascade, event bus + triggers, SDK stabilization,
workflow engine (multi-stage, team roles, signoff gate, public entry, SLA),
package distribution, Notes surface (CM6, folders, tags, backlinks, graph),
realtime primitive, Chat surface (chat-core library + surface + polish),
upgrade test harness, cluster registry + HA.
--- ---
## v0.6.x — Completed (MVP + Hardening) ## Completed — v0.6.x through v0.8.x
All v0.6.x work is shipped and documented in `CHANGELOG.md`. Summary: All completed work is documented in `CHANGELOG.md`.
| Version | Title | Key Deliverables | ### v0.6.x — MVP + Hardening
|---------|-------|-----------------|
| v0.6.0 | Cluster Registry + HA | PG-backed node registry, heartbeat sweep, LISTEN/NOTIFY routing, self-eviction | | Version | Title |
| v0.6.1 | Backup/Restore + Docs | `.swb` archive format, server-side backups, docs surface + 5 guides | |---------|-------|
| v0.6.2 | Docs Polish + OpenAPI | Dark mode fix, topbar nav, `api_schema` manifest field, dynamic spec builder | | v0.6.0 | Cluster Registry + HA |
| v0.6.3 | Dead Code Sweep | Registry install fix, dead Go/JS/HTML deletion, narrowed default bundle | | v0.6.1 | Backup/Restore + Docs |
| v0.6.4 | Admin Health/Metrics | Cluster dashboard merged into Admin tab, block renderer `requires` removed | | v0.6.2 | Docs Polish + Dynamic OpenAPI |
| v0.6.5 | Renderer Pipeline | `sw.renderers.register()` kernel primitive, unified markdown, docs rewrite | | v0.6.3 | Dead Code Sweep + Registry Fix |
| v0.6.6 | Final Hardening | Dependency auto-activation, `ValidateManifest()`, OIDC nonce, ICD/SDK update | | v0.6.4 | Admin Health/Metrics + Cluster Merge |
| v0.6.7 | Native mTLS | `TLS_MODE` config, `MTLSNativeProvider`, node-to-node mTLS, `armature-ca.sh` | | v0.6.5 | Renderer Pipeline + Docs Rewrite |
| v0.6.8 | Cookie Fix + UI Roadmap | Cookie SameSite fix, UI hardening roadmap published | | v0.6.6 | Final Hardening |
| v0.6.9 | Session Lifetime Config | Admin-configurable TTLs, idle timeout, "keep me logged in" | | v0.6.7 | Native mTLS |
| v0.6.10 | Viewport Foundation | Single layout model, CSS zoom, 100dvh, dead shell deprecated | | v0.6.8 | Rebrand + Cookie Fix |
| v0.6.11 | CSS Deduplication | Old primitive system retired, one class per concept | | v0.6.9 | Session Lifetime Config |
| v0.6.12 | Extension CSS Isolation | Prefix enforcement via linter, all 12 in-tree packages migrated | | v0.6.10 | Viewport Foundation |
| v0.6.13 | Responsive & Spacing | Spacing token scale (4px grid), tablet breakpoint | | v0.6.11 | CSS Deduplication |
| v0.6.14 | Visual Polish | Stale fallback colors purged, fonts self-hosted, radius tokens | | v0.6.12 | Extension CSS Isolation |
| v0.6.15 | User Display Audit | Batch user resolve API, `sw.users` SDK module | | v0.6.13 | Responsive & Spacing |
| v0.6.16 | Usability Survey Gate | Four audit scripts, contrast/touch-target fixes | | v0.6.14 | Visual Polish |
| v0.6.17 | Bug Fixes & Welcome | Notes folder fix, team member add fix, welcome auto-disable, zero default bundle | | v0.6.15 | User Display Audit |
| v0.6.18 | CI Bundle Wiring | `BUNDLED_PACKAGES` env var wired into Gitea CI pipeline | | v0.6.16 | Usability Survey Gate |
| v0.6.17 | Bug Fixes & Welcome |
--- | v0.6.18 | CI Bundle Wiring |
## v0.7.x — Test Infrastructure + Quality Gate ### v0.7.x — Test Infrastructure + Quality Gate
The v0.6.x series built the kernel. v0.7.x makes it provably correct. | Version | Title |
|---------|-------|
A full surface audit (docs/AUDIT-surfaces.md) found 7 cross-surface issues | v0.7.0 | Shell Contract + Surface Audit + Rebrand |
and 18 surface-specific issues across Settings, Admin, Team Admin, and Docs. | v0.7.1 | Surface Runner Framework |
Only Docs is properly built. The fix is three phases: (1) establish a shell | v0.7.2 | Package Runners + CI Gate |
contract and bring all four primary surfaces to parity, (2) build a runner | v0.7.3 | Extension Shell Migration |
framework for end-to-end browser tests, (3) automate those runners headlessly | v0.7.4 | Documentation + Surface Work |
in CI. | v0.7.5 | Headless E2E + CI Gate |
| v0.7.6 | Code Hygiene + Test Coverage |
Design docs: | v0.7.7 | API Tokens + Extension Permissions |
- `docs/DESIGN-shell-contract.md` — two-slot topbar, three navigation patterns, surface migrations | v0.7.8 | Bug Fixes & Admin Gaps |
- `docs/DESIGN-surface-runners.md` — runner framework, requires declarations, headless E2E | v0.7.9 | Workflow Independence Audit |
- `docs/AUDIT-surfaces.md` — full audit findings | v0.7.10 | Workflow Handoff + Assignment UI |
| v0.7.11 | Query & HTTP Ergonomics |
### v0.7.0 — Shell Contract + Surface Audit + Rebrand Cleanup | v0.7.12 | Concurrent Execution Primitive |
Design doc: `docs/DESIGN-shell-contract.md` ### v0.8.x — Storage Primitives + Composability (Kernel Complete)
**Shell Infrastructure** | Version | Title |
|---------|-------|
| Step | Status | Description | | v0.8.0 | `files` Module |
|------|--------|-------------| | v0.8.1 | `workspace` Module |
| Shell topbar (two-slot model) | done | Kernel injects topbar into `surface-extension` template. Two named slots: **left** (defaults to manifest title) and **center** (`flex: 1`, for tabs/pickers/search). Home link, notification bell, and user menu always present. | | v0.8.2 | Capability Negotiation |
| Topbar customization API | done | `sw.shell.topbar.setLeft(vnode)` overrides left slot. `sw.shell.topbar.setSlot(vnode)` sets center slot. `sw.shell.topbar.setTitle(str)` shorthand for text-only left. `sw.shell.topbar.hide()` / `.show()` for full-bleed surfaces. | | v0.8.3 | Vector Column Type |
| Kernel tab CSS | done | `.sw-topbar__tabs` and `.sw-topbar__tab` classes for consistent tab styling in the center slot. Surfaces use these for free or style their own slot content. | | v0.8.4 | Documentation Refresh + Surface Sizing Fix |
| Notification read broadcast | done | Backend emits `notification.read` and `notification.all_read` WS events. Bell listens for `.created`, `.read`, `.all_read`. Cross-surface sync without refetch. | | v0.8.5 | Extension Composability |
| User menu reactivity | done | Emit `package.changed` (install/uninstall/enable/disable) and `auth.changed` (role/membership) WS events. UserMenu listens and re-fetches surface list. Most impactful single fix. |
| Shell announcement global dismiss | done | Dismissed state persisted to localStorage keyed by content hash. Dismiss once, dismissed everywhere. |
**Surface Migrations**
| Step | Status | Description |
|------|--------|-------------|
| Settings → Pattern B (flat tabs) | done | Delete custom topbar + sidebar nav. 6 sections become flat tabs in topbar center slot. Content full-width. Fix Teams section: add team admin link, role display, leave action. Remove sessionStorage return URL logic. |
| Admin → Pattern C (category tabs + sidebar) | done | Delete custom `admin-topbar`. `setLeft()` for favicon + "Administration". `setSlot()` for category tabs (People / Workflows / System / Monitoring). Admin sidebar (sub-navigation) unchanged — surface-owned, below the topbar. Bell + user menu come free from shell. Delete bespoke CatIcon renderer if using standard SVGs. |
| Team Admin → Pattern B (flat tabs) | done | Delete custom topbar + sidebar nav. 5 sections (Members / Connections / Workflows / Settings / Activity — Groups removed) become flat tabs. Content full-width. `setTitle()` for team-specific name. Remove sessionStorage return URL. Fix signoff user display (`user_id``sw.users.displayName()`). |
| Team Admin: remove Groups tab | done | 37-line dead-end. Read-only "No groups" with no create/docs/link. Remove until team-scoped group management is properly designed. |
| Docs → Pattern A (default) | done | Delete explicit Topbar import. Shell topbar auto-renders with manifest title. Docs sidebar (document list) is in content area, unaffected. |
**Error Handling + UX Pass**
| Step | Status | Description |
|------|--------|-------------|
| Inline error states | done | Replace `catch { toast }` with inline error + retry on all list endpoints. New `.sw-inline-error` CSS primitive. Systematic pass across Settings, Admin, Team Admin. |
| Empty state guidance | done | Every "No X" message gets one-line explanation + primary action (create button or doc link). Admin Groups, Workflows, Teams; Team Admin Workflows; Settings Notifications. |
**Bug Fixes**
| Step | Status | Description |
|------|--------|-------------|
| evil-chat cleanup | done | ICD security tier: `finally` cleanup block + tighten `409` assertion. |
| Workflow demo error surfacing | done | Replace silent `catch` with inline error + retry. |
| Hello dashboard removal | done | Delete `packages/hello-dashboard/`. |
**Rebrand**
| Step | Status | Description |
|------|--------|-------------|
| Light-mode icon SVG | done | New `favicon-light.svg` — square icon, transparent bg, dark node fills. Rename current `favicon-light.svg` (wordmark) to `wordmark.svg`. |
| Dark-mode wordmark SVG | done | New `wordmark-dark.svg` — light text for dark backgrounds. |
| Light-mode raster assets | done | `favicon-light-32.png`, `favicon-light-256.png`. |
| PWA manifest description | done | "Self-hosted extension platform — build, compose, and run extensions." |
| REBRAND-SPEC.md | | Land into `docs/`. Find/replace patterns, validation checklist, asset inventory. |
| base.html favicon swap | done | Verify theme swap works with new square light icon. |
**Tests**
| Step | Status | Description |
|------|--------|-------------|
| Shell topbar renders for extensions | done | Home, left slot, center slot, bell, user menu present. |
| Topbar API (setLeft, setSlot, hide) | done | Custom content renders. Hide removes topbar. |
| All 4 surfaces use shell topbar | done | No double topbars. Each pattern (A/B/C) renders correctly. |
| User menu reactive to package install | | Install → menu updates without reload. |
| Notification bell cross-surface sync | | Dismiss on Notes → clears on Chat. |
| Inline error on API failure | | Error + retry shown, not empty list. |
### v0.7.1 — Surface Runner Framework
Design doc: `docs/DESIGN-surface-runners.md`
| Step | Status | Description |
|------|--------|-------------|
| Runner framework (`sw.testing`) | done | SDK module: suite/test/assert, lifecycle hooks, structured JSON results. |
| `requires` declarations | done | Runner manifests declare dependencies. Missing packages → clean skip. |
| Cleanup enforcement | done | `s.track(type, id)` auto-deletes in `afterAll`. No leaked state. |
| Warning tier | done | pass / fail / warning. No silent catch swallowing. |
| ICD runner migration | done | Refactor to `sw.testing`. Test logic preserved. |
| SDK runner migration | done | Refactor to `sw.testing`. Domain suites preserved. |
| Runner registry surface | done | `/s/test-runners` — list runners, run-all, results dashboard. |
### v0.7.2 — Package Runners + CI Gate
| Step | Status | Description |
|------|--------|-------------|
| Notes runner | done | `requires: ["notes"]`. 3 suites (crud, folders, tags-search), 12 tests. |
| Chat runner | done | `requires: ["chat", "chat-core"]`. 2 suites (conversations, messaging), 9 tests. |
| Schedules runner | done | `requires: ["schedules"]`. 1 suite (crud), 5 tests. |
| Workflow runner | done | `requires: ["content-approval"]`. 1 suite (lifecycle), 5 tests. |
| Renderer runner | done | `requires: ["mermaid-renderer"]`. 1 suite (contract), 4 tests. |
| Runner result API | done | `POST/GET /api/v1/admin/test-runners/results`. In-memory store, 4 Go tests. |
| CI integration | done | `test-runners` stage in Gitea CI. Playwright driver, `wait-for-healthy.sh`. |
| CI DinD networking fix | done | Resolve container IP via `docker inspect` — DinD port mapping doesn't expose to runner localhost. |
### v0.7.3 — Extension Shell Migration
Chat, Notes, and Schedules still use the old `sw.shell.Topbar` component,
producing a double topbar (shell-injected + surface-owned). Migrate all
three to the v0.7.0 shell contract (`sw.shell.topbar.setTitle/setSlot`),
same pattern as the kernel surface migrations.
| Step | Status | Description |
|------|--------|-------------|
| Chat → shell topbar | done | Delete `<Topbar>`, use `setTitle('Chat')` + `setSlot()` for thread title/people button. |
| Notes → shell topbar | done | Delete `<Topbar>`, use `setTitle('Notes')` + `setSlot()` for action buttons. |
| Schedules → shell topbar | done | Delete `<Topbar>`, use `setTitle('Schedules')` + `setSlot()` for count/new button. |
| Update package runner tests | done | Shell-topbar test suite in each runner — assert no legacy Topbar, uses shell API. |
### v0.7.4 — Documentation + Deferred Surface Work
| Step | Status | Description |
|------|--------|-------------|
| Docs category grouping | done | Backend `Category` field on doc entries. Frontend groups sidebar by category with headings. Four categories: Getting Started, Platform, Extension Development, Operations. |
| Permissions & Groups guide | done | RBAC model, 7 permission slugs, system/custom groups, settings cascade, extension permissions. |
| Workflows user guide | done | Entry modes, stages, team roles, signoff gates, SLA, public forms, branch rules, Starlark hooks. |
| Starlark Reference | done | Sandbox constraints, 10 modules with function signatures, permission gates, example hook script. |
| Frontend JS Guide | done | Preact+htm runtime, 16 SDK modules with API reference, shell topbar patterns, CSS contract. |
| Extension config_section docs | done | Manifest schema, backend discovery, frontend contract, example component. Added to Extension Guide. |
| Docs content refresh | done | All 10 user-facing docs reviewed for v0.7.x accuracy: rebrand volume names, stale CSS vars, TLS_MODE env, self-hosted font notes, Architecture frontend section. |
| Team Admin Workflows split | done | 722-line `workflows.js` split into 3 modules: `workflows.js` (router+CRUD), `workflow-editor.js` (editor+stages), `workflow-monitor.js` (assignments+monitor+signoff). |
| Stale CSS variable fix | done | `--bg-2` references in `sw-shell.css` and `sw-primitives.css` replaced with `--bg-secondary`. |
### v0.7.5 — Headless E2E + CI Gate
| Step | Status | Description |
|------|--------|-------------|
| CI gating redesign | done | Fix path rules: `VERSION`/`scripts/*` deploy-only, `docs/*` no longer skips deploy, test-runners trigger on any code change. |
| E2E smoke test harness | done | `ci/e2e-smoke-test.sh` + `ci/e2e-smoke-driver.js` — Playwright visits every surface, asserts topbar, no JS errors, home link. |
| Screenshot-on-failure | done | Full-page screenshot + console log saved as CI artifacts. |
| CI pipeline integration | done | Re-enable `test-runners` stage, add `e2e-smoke` stage. Failure blocks merge. |
| Design docs landed | done | `docs/DESIGN-storage-primitives.md`, `docs/DESIGN-extension-composability.md`. Roadmap expanded through v1.0. |
### v0.7.6 — Code Hygiene + Test Coverage
Codebase analysis (April 2026) found 9 code smells from the chat gut and
a test coverage gap in the store layer. Fix before v0.8.x kernel expansion.
**Critical Fixes**
| Step | Status | Description |
|------|--------|-------------|
| Remove channels from allowedViews | done | `db_module.go``ext_view_channels` does not exist. `db.view("channels")` crashes. |
| Fix workflow.html dead routes | done | Template calls `/api/v1/channels/{id}/workflow/*` — routes moved to `/api/v1/public/workflows/`. Update template to match. Also fixed `RenderWorkflow` handler to use route param as entry token. |
**Dead Code Removal**
| Step | Status | Description |
|------|--------|-------------|
| Delete SeedTestChannel() | done | `database/testhelper.go` — inserts into nonexistent channels table. |
| Remove RunContext.ChannelID | done | `sandbox/runner.go` — vestigial field, unused by any module. |
| Remove webhook.ChannelID | done | `webhook/webhook.go` — vestigial struct field from chat era. |
| Fix stale comments | done | `storage.go` key format, `prometheus.go` route pattern, `workflow_module.go` param name — reference dead `/channels` paths. |
**Migration Hygiene**
| Step | Status | Description |
|------|--------|-------------|
| Align SQLite migration numbering | done | SQLite 013 placeholder + renumber test_runner_type to 014. Compat rename in migrate.go. |
| Document missing 008 | done | Comment in both 009 files explaining the gap (merged into 007). |
**Test Gap Closure**
| Step | Status | Description |
|------|--------|-------------|
| store/ unit tests | done | Moved to v0.7.9. 17 SQLite store tests: workflow CRUD, stages, instances, lifecycle, tokens, users, groups. |
| workflow/ routing unit tests | done | 82 tests: ResolveNextStage, ResolveStageByName, ParseStageConfig, evaluateCondition (all operators). |
| middleware/ unit tests | done | 10 tests: RequirePermission, RequireAdmin, permission caching, RateLimiter (allow/deny/fail-open). |
| InstallPackage decomposition | done | Moved to v0.7.9. Split into 7 phases: receiveUpload, parseAndValidateArchive, extractPackageAssets, registerPackage, applySchemaAndPermissions, resolveDependencies, checkCapabilities. |
**Bug Fixes**
| Step | Status | Description |
|------|--------|-------------|
| Remove Admin Storage tab | done | Dead weight under System — backend endpoint preserved for future Monitoring use. |
| Fix backup download | done | `sw.auth.token()``sw.auth._getToken()`. Both "Download Backup" and server backup download buttons now work. |
### v0.7.7 — API Tokens + Extension Permissions
Personal access tokens (PATs) for programmatic API access, plus
extension-declared user permissions for backend RBAC enforcement.
Unblocks: E2E test auth, CI automation, extension permission gating,
sidecar auth (v0.10.x).
**API Tokens (PATs)**
| Step | Status | Description |
|------|--------|-------------|
| `api_tokens` table + migration | done | `id`, `user_id`, `name`, `token_hash`, `permissions` (JSON array), `expires_at`, `last_used_at`, `created_at`. PG + SQLite. |
| Token store interface | done | `Create`, `GetByHash`, `ListForUser`, `Revoke`, `CleanExpired`, `UpdateLastUsed`. |
| Token generation endpoint | done | `POST /api/v1/auth/tokens` — name, permissions (subset of user's), expires_at. Returns token once (plain text), stored as SHA-256 hash. |
| Token management endpoints | done | `GET /api/v1/auth/tokens` (list mine), `DELETE /api/v1/auth/tokens/:id` (revoke). |
| Admin token endpoint | done | `POST /api/v1/admin/tokens` — creates token for any user. Explicit admin action, audit logged. |
| Auth middleware update | done | Accept `Bearer arm_pat_...` tokens alongside JWTs. Resolve user + scoped permissions from token. PAT requests skip refresh token flow. |
| Permission scoping | done | Token permissions ⊆ creating user's resolved permissions at creation time. Validated on create. If user loses a permission later, token retains it until revoked (git model). |
| Settings UI | done | Settings → API Tokens tab. Create, list, revoke. Show permissions, expiry, last used. Token value shown once on creation. |
| Admin UI | done | Admin → People → user detail → "Create API Token" action. |
| E2E test migration | done | Replace `ci/e2e-smoke-test.sh` login flow with PAT-based auth. Seed token via `ARMATURE_BOOTSTRAP_PAT=true`. |
**Extension-Declared User Permissions**
| Step | Status | Description |
|------|--------|-------------|
| `user_permissions` manifest field | done | Extensions declare permissions users need: `"user_permissions": ["image-gen.use", "image-gen.admin"]`. |
| Dynamic permission registry | done | On install: merge into valid permission set. On uninstall: remove. `AllPermissions` becomes `KernelPermissions + ExtensionPermissions`. |
| Group UI update | done | Admin → Groups → permission checkboxes include extension-registered permissions, grouped by source package. |
| `permissions` Starlark module | done | `permissions.check(user_id, "image-gen.use")` → resolves user's groups, returns bool. New permission: none required (read-only check against kernel data). |
| `gate_permission` manifest field | done | Optional. If set, `ext_api.go` checks this permission before calling `on_request`. Extension doesn't execute for unauthorized users. |
| `req["permissions"]` in request dict | done | `ext_api.go` resolves user permissions and includes them in the request dict. Extensions can check inline without the module. |
### v0.7.8 — Bug Fixes & Admin Gaps
| Step | Status | Description |
|------|--------|-------------|
| Workflow landing page Start | done | Added `StartBySlug` handler + `/api/v1/workflow-entry/:scope/:slug` route. Landing page Start button now resolves scope/slug → workflow ID and creates instance. |
| Workflow delete guard | done | Admin `DELETE /api/v1/workflows/:id` rejects team-scoped workflows (403). Team workflows must use team endpoint. Prevents accidental global template deletion. |
| Package button cleanup | done | Delete button hidden for `bundled` packages via `IMMUTABLE_SOURCES` guard, matching Export/Update. |
| Package export fetch | done | Export uses `fetch()` with auth token instead of `window.open()`. Handles errors, shows toast on missing assets. |
| Settings CSS fix | done | Bottom padding increased to `--sp-12` on `.admin-settings-form`. Save button no longer cut off. |
| Admin stage builder + links | done | Shared `StageForm` component between admin and team-admin. Public entry URL with copy button in admin workflow editor. |
### v0.7.9 — Workflow Independence Audit
Workflows must work end-to-end without chat or any 3rd-party package
dependency. Chat, notifications, and other packages should *enhance*
workflows but never be required for core operation.
**RenderWorkflow Fix (critical)**
| Step | Status | Description |
|------|--------|-------------|
| Fix RenderWorkflow() handler | done | Was a stub (always `stageMode = "custom"`, no data loaded). Now loads instance by token/ID, resolves current stage, populates all template data. |
| Rename WorkflowPageData fields | done | `ChannelID``EntryToken`, `ChannelTitle``WorkflowTitle`, `ChannelDescription``WorkflowDescription`. Vestigial chat-era names removed. |
| Align stage mode values | done | Template now uses Go constants (`form`, `review`, `delegated`, `automated`) instead of legacy names (`form_only`, `form_chat`). |
| Remove dead chat UI | done | Chat CSS, split layout, `sendMessage()` function, and fallback chat branch all removed from `workflow.html`. Replaced with clean form/review/unsupported modes. |
| Fix landing page mode mapping | done | `workflow-landing.html` conditionals updated to match Go stage mode constants. |
| Update OpenAPI spec | done | `stage_mode` enum changed from `[custom, form_only, form_chat, review]` to `[form, review, delegated, automated]`. |
**Verification & Testing**
| Step | Status | Description |
|------|--------|-------------|
| Admin/team-admin UI audit | done | All workflow CRUD, monitoring, stage builder verified clean — zero chat/chat-core references. |
| E2E workflow test | done | `ci/e2e-workflow-nochat.sh` — creates workflow via API, adds form+review stages, starts instance via public entry, verifies landing + execution pages. |
| Store unit tests (SQLite) | done | 17 tests: workflow CRUD, stages, instances, lifecycle (advance/complete/cancel/stale), team scope, API tokens, users, groups. Carried from v0.7.6. |
| InstallPackage decomposition | done | Split 400-line handler into 7 private methods: `receiveUpload`, `parseAndValidateArchive`, `extractPackageAssets`, `registerPackage`, `applySchemaAndPermissions`, `resolveDependencies`, `checkCapabilities`. Carried from v0.7.6. |
**Bug Fixes (found during verification)**
| Step | Status | Description |
|------|--------|-------------|
| Entry token resolution | done | `RenderWorkflow()` passed route param (instance ID) as entry token. JS then sent instance ID to the advance API which expects entry token. Fixed: resolve actual token from `inst.EntryToken` + check `?token=` query param. |
| Fieldset submit guard | done | `submitForm()` checked `FORM_TPL.fields` but not `.fieldsets` — progressive forms silently no-op'd on submit. Fixed: accept either. |
| Stage advance status check | done | Template JS checked `result.status === 'advanced'` but API returns instance status (`active`/`completed`). Fixed: on 200 OK with `active` status, reload to render next stage. |
**Discovered issues (deferred)**
The audit uncovered deeper workflow UX gaps that are out of scope for v0.7.9:
1. **Public→authenticated stage handoff** — After a public visitor completes their stages, the workflow advances to an authenticated stage (e.g. "classify"). The visitor sees the next stage's form + an auth error instead of a "thank you, we'll take it from here" message. The page should detect audience mismatch and show a completion/waiting screen.
2. **Team member pickup UI** — No surface exists for team members to see workflow instances assigned to their team and claim/work on them. Currently only visible in team-admin monitoring.
3. **Assignment flow** — No mechanism for team admins to assign a specific instance to a specific team member. The `workflow_assignments` table exists but has no admin UI for manual assignment.
These will be addressed in a future version (see v0.7.10 below).
### v0.7.10 — Workflow Handoff + Assignment UI
Addresses the three UX gaps found during the v0.7.9 independence audit.
The workflow engine and data model already support these flows — the
missing pieces are page-level audience detection and team-facing UI.
| Step | Status | Description |
|------|--------|-------------|
| Public stage completion screen | done | `RenderWorkflow()` detects audience mismatch (team/system stage + unauthenticated visitor) and renders "Submitted Successfully" screen with reference ID. |
| Team workflow inbox | done | Enhanced `AssignmentsTab` in team-admin: claim/unclaim/work actions, time-ago display, WS live updates. Fixed vestigial `channel_id` references. |
| Manual assignment UI | done | "Assign" button on unassigned rows with team member dropdown. New `POST /api/v1/assignments/:id/assign` endpoint (requires `workflow.create` permission). |
| Assignment notifications | done | Engine calls `notifyAssignment()` on assignment creation — notifies specific user or all team members via notification system. |
| Team workflow instances endpoint | done | `GET /api/v1/teams/:teamId/workflow-instances` + cancel endpoint. `ListInstancesByTeam` store method (SQLite + PG). SDK methods added. |
| API response enrichment | done | `ListByTeam` and `ListMine` handlers enrich assignments with `workflow_name`, `stage_name`, `sla_breached` by joining instance → workflow → version snapshot. Cached per-request. Same for `ListTeamInstances` with `instanceView` struct. |
| Team middleware fix | done | `RequireTeamAdmin` / `RequireTeamMember` system-admin bypass was dead code (`c.Get("role")` never set). Fixed to check `PermSurfaceAdminAccess` via `resolveAndCachePerms`. |
| SDK gap closure | done | Added `workflowAssignments` domain (claim/unclaim/complete/cancel/assign/mine), `teams.assignments`, `teams.workflowInstances`, `teams.cancelWorkflowInstance`. Fixed dead `workflows.cancel` route. |
| E2E multi-stage test | done | `ci/e2e-workflow-handoff.sh`: public form → team review → claim → complete. Verifies audience mismatch screen, assignment creation, full lifecycle. |
### v0.7.11 — Query & HTTP Ergonomics (was v0.7.10)
Four additions to existing sandbox modules. No new files beyond tests,
no schema changes. Motivated by real-world extension porting feedback:
complex SQL decomposition and synchronous HTTP fan-out are the two
remaining friction points for extensions migrating from native Go/Python.
| Step | Status | Description |
|------|--------|-------------|
| `db.count()` | done | `db.count(table, filters={})` → int. `SELECT count(*) FROM ext_{pkg}_{table} WHERE ...`. Uses existing `starlarkFiltersToSQL` builder. Permission: `db.read`. |
| `db.aggregate()` | done | `db.aggregate(table, column, op, filters={})` → single value. `op` ∈ {`count`, `sum`, `avg`, `min`, `max`}. Column name validated same as filter keys. Returns int/float/None. Permission: `db.read`. |
| `db.query_batch()` | done | `db.query_batch(queries)` → list of result sets. Each query spec is a dict with `table` (required), `filters`, `order`, `limit`, `before`, `after`, `search_like` (all optional, same as `db.query`). Loops extracted `buildSelectQuery` helper internally. Permission: `db.read`. |
| `http.batch()` | done | `http.batch(requests)` → list of response dicts (ordered). Each request dict: `method`, `url`, `body`, `headers`. Concurrent dispatch via goroutines capped at 10, `sync.WaitGroup` collection. Reuses existing `executeHTTPRequest` plumbing (SSRF checks, domain allowlist, body cap, timeout). Individual failures return error response dicts, don't abort batch. Permission: `api.http`. |
| Tests | done | 21 new tests (14 db, 7 http). `db.count`/`db.aggregate` test empty table, filtered, type coercion, invalid op/column. `db.query_batch` test mixed specs, empty list, cap, missing table. `http.batch` test blocked domains, partial failure, mixed results, input validation. |
### v0.7.12 — Concurrent Execution Primitive (was v0.7.11)
New `batch` sandbox module. Enables extensions to parallelize arbitrary
Starlark callables — including library function calls via `lib.require()`.
The kernel manages concurrency (multiple `starlark.Thread` instances in
goroutines); extensions stay single-threaded and synchronous from the
author's perspective.
Key insight: library exports loaded via `lib.require()` are frozen structs
(immutable, safe to share across threads). Each concurrent branch gets
fresh module instances (`db`, `http`, etc.) — same pattern as
`buildRestrictedModules` in `triggers/schedule.go`.
| Step | Status | Description |
|------|--------|-------------|
| Design doc | done | `docs/DESIGN-batch-exec.md`. Thread isolation model, module construction, error semantics (partial results on failure), permission model, concurrency cap. |
| `batch.exec()` | done | `batch.exec(callables)``(results, errors)` tuple. List of Starlark callables (lambdas, function refs). Each runs in its own `starlark.Thread` with fresh modules. Concurrent goroutines capped at 8. Ordered results. Errors are per-callable (None on success, string on failure). Permission: `batch.exec` (new). New file: `sandbox/batch_module.go`. |
| Runner wiring | done | `buildModulesWithLibCtx` creates `batch` module when `batch.exec` permission granted. Module receives runner reference for per-branch module construction. |
| Tests | done | 12 unit tests: parallel execution ordering, partial failure, timeout per-branch, parent cancellation, cap enforcement, empty list, non-callable, permission gating, frozen struct sharing, nested batch.exec, default timeout, timeout clamping. All pass with `-race`. |
--- ---
## Planned ## Planned
### v0.8.x — Storage Primitives ### v0.9.x — Multi-Surface Packages + Workflow Redesign
Design doc: `docs/DESIGN-storage-primitives.md` **v0.9.0 — Multi-Surface Packages** *(completed)*
The last major kernel expansion. Completes the primitive set that the Packages declare a `surfaces` array with per-path access controls,
entire extension ecosystem builds on. After v0.8.x, the kernel is titles, and layouts. Unified route tree dispatches between surface
feature-complete for 1.0 — all new capabilities are extensions. rendering and ext API calls. `sw.navigate()` for client-side sub-path
routing. Design doc: `docs/DESIGN-multi-surface.md`.
**v0.8.0 — `files` Module** **v0.9.1 — Server-Side Sub-Path Routing**
Bridge `ObjectStore` (PVC/S3) into the Starlark sandbox. Extension-scoped Full-page refresh on sub-paths (e.g. `/s/my-pkg/monitor`) currently
key namespacing (`ext/{packageID}/`). Permissions: `files.read`, requires client-side routing. This version restructures the Gin route
`files.write`. Metadata companions stored as sibling objects. tree so the kernel resolves sub-paths server-side, including proper
`OptionalAuth` for packages with mixed public/authenticated surfaces.
New: `sandbox/files_module.go`. Modified: runner wiring, permission **v0.9.2 — Starlark Converter Consolidation + Snapshot Cleanup**
constants.
**v0.8.1 — `workspace` Module** Design doc: `docs/DESIGN-workflow-redesign.md`
Managed disk directories for tools that need a real filesystem (git, Three files contain near-identical Go↔Starlark converters; three copies
compilers, media tools). `workspace.create(name)` → scoped path. of the snapshot parser exist. Consolidate into `sandbox/convert.go` and
Permission: `workspace.manage`. Quota enforcement via `WORKSPACE_QUOTA_MB`. one exported snapshot function. Standardize on wrapped snapshot format.
New: `sandbox/workspace_module.go`. Modified: runner wiring, permission **v0.9.3 — Team User Roles**
constants.
**v0.8.2 — Capability Negotiation** Promote the buried role system to a kernel primitive. New
`team_user_roles` table (many-to-many). Manifest `requires_roles` field.
Team admin UI for role assignment. Kernel middleware `RequireRole()`.
Starlark SDK: `teams.get_member_roles()`, `teams.has_role()`.
Extensions declare environment requirements in manifest (`capabilities.required`, **v0.9.4 — Package Adoption + Roles**
`capabilities.optional`). Kernel validates at install time. Runtime query
via `settings.has_capability()`. Admin endpoint: `GET /admin/capabilities`.
Detected capabilities: `pgvector`, `workspace`, `object_storage`, `s3`, `scope: adoptable` manifest field. When a team adopts an adoptable
`postgres`. package, the package's `requires_roles` auto-populate into the team's
role slots. Replaces `AdoptTeamWorkflow` clone mechanism.
New: `handlers/capabilities.go`. Modified: install handler, settings module. **v0.9.5 — Typed Forms → SDK Primitive**
**v0.8.3 — Vector Column Type** Extract `TypedFormTemplate`, `FormField`, `FormFieldset`, etc. from
`models/workflow.go` into a `forms` package. FE SDK: `sw.forms.render()`
and `sw.forms.validate()`. Starlark: `forms.validate()`. Any package
can declare forms, not just workflow stages.
`"vector(N)"` in `db_tables` manifest. Maps to native `vector(N)` on **v0.9.6 — Deprecate `stage_type`, Collapse `stage_mode`**
PG+pgvector, `JSONB` on PG without, `TEXT` on SQLite. New `db.query_similar()`
with dual-path dispatch (native operator vs Go-side brute force).
Modified: `ext_db_schema.go`, `db_module.go`. `stage_type` (simple/dynamic/automated) is redundant with `starlark_hook`
presence. Remove from new manifests, keep parsing for backward compat.
Collapse `stage_mode` from 4 to 3 values: form / delegated / automated.
**v0.8.4 — Extension Composability** **v0.9.7 — Full Read/Write Workflow Starlark Module**
Design doc: `docs/DESIGN-extension-composability.md` Add `workflow.start()`, `workflow.advance()`, `workflow.cancel()`,
`workflow.submit_signoff()` to the Starlark module. Extract engine
interface to break circular import.
Manifest declarations for `slots` (host surfaces declare injection points) **v0.9.8 — Conditional Routing → SDK Primitive**
and `contributes` (extensions declare UI contributions to other packages'
slots). `lib.require()` relaxed from library-only to any package with
`exports` — enables full packages (with surfaces, settings, UI) to also
expose callable backend functions. `sw.slots.renderAll()` SDK helper.
Admin slot aggregation endpoint.
The composability primitive that enables: LLM tool registration, UI Expose `routing.evaluate(rules, data)` as a Starlark SDK function.
contribution (toolbar buttons, image actions), and cross-extension Branch rules become a reusable decision engine for any extension.
function calls. No new tables, migrations, or permissions.
Modified: `sandbox/lib_module.go` (~1 line), `handlers/extensions.go`, **v0.9.9 — Surface Access via Roles**
`src/js/sw/sdk/slots.js`. New: `handlers/admin_slots.go`.
Wire team roles (v0.9.3) into surface access declarations:
`access: role:approver`. Kernel middleware checks role membership.
Completes the workflow→package access story.
--- ---
### v0.9.x — Reference Extensions ### v0.10.x — Reference Extensions
The kernel is complete. This series proves the platform by shipping The kernel is complete. This series proves the platform by shipping
first-party extensions that exercise every primitive. These are first-party extensions that exercise every primitive. These are
**extensions, not kernel code** — they ship as `.pkg` files and can be **extensions, not kernel code** — they ship as `.pkg` files and can be
uninstalled. uninstalled.
**v0.9.0 — `vector-store` Library** **v0.10.0 — `vector-store` Library**
Library extension: document ingestion, chunk storage, embedding persistence, Document ingestion, chunk storage, embedding persistence, semantic
semantic similarity search. Three-tier vector strategy (pgvector → JSONB similarity search. Consumes: `db.write`, `files.read`.
fallback → TEXT fallback). Consumes: `db.write`, `files.read`.
**v0.9.1 — `llm-bridge` Library** **v0.10.1 — `llm-bridge` Library**
Library extension: model abstraction, tool-use routing, provider BYOK via Model abstraction, tool-use routing, provider BYOK via connections.
connections. Exposes `complete()`, `embed()`, `classify()`, `register_tool()` Exposes `complete()`, `embed()`, `classify()`, `register_tool()`.
to consumer extensions. Tool extensions (image-gen, code-exec, web-search)
register at load time — LLM discovers available tools automatically.
Consumes: `connections.read`, `api.http`, `db.write`. Consumes: `connections.read`, `api.http`, `db.write`.
**v0.9.2 — `file-share` Extension** **v0.10.2 — `file-share` Extension**
File upload via `api_routes`, storage via `files` module, download links, File upload, storage via `files` module, download links, team/group
optional team/group ACLs via resource grants. First extension to exercise ACLs via resource grants.
the full `files` primitive end-to-end.
**v0.9.3 — `code-workspace` Extension** **v0.10.3 — `code-workspace` Extension**
Managed code repositories via `workspace` module. Git clone/pull via Managed code repos via `workspace` module. Git operations, file browser
`api.http` to Gitea/GitHub API. File browser surface. Structural indexing surface, structural indexing. Consumes: `workspace.manage`, `files.write`.
(AST-aware) for code navigation. Consumes: `workspace.manage`, `files.write`,
`api.http`.
**v0.9.4 — `image-gen` + `image-edit` Extensions** **v0.10.4 — `image-gen` + `image-edit` Extensions**
Image generation via external APIs (DALL-E, Stable Diffusion). Registers Image generation/editing via external APIs. Composability demo:
as LLM tool with `llm-bridge`. Contributes regen/edit buttons to contributes to `chat:image-actions` slot.
`chat:image-actions` slot. Image-edit extends with inpaint, outpaint,
upscale, and style transfer — each contributing action buttons to the
same slot independently. First full demonstration of the composability
pattern: three extensions contributing UI to a fourth's surface.
**v0.9.5 — Chat System** **v0.10.5 — Chat System**
Generic 1-to-N human messaging. `chat-core` library (conversation/message Generic 1-to-N messaging with slots. `chat-core` library + `chat`
CRUD) + `chat` surface. Declares `chat:message-actions`, `chat:image-actions`, surface. Declares `chat:message-actions`, `chat:image-actions`,
`chat:composer-tools` slots for extension contribution. LLM participation `chat:composer-tools` slots. Consumes: `db.write`, `realtime.publish`,
as a follow-on consuming `llm-bridge`. Consumes: `db.write`, `files.write`.
`realtime.publish`, `files.write` (attachments).
--- ---
### v0.10.x — Sidecar Tier + Polish ### v0.11.x — Sidecar Tier + Polish
**v0.10.0 — Sidecar Tier** **v0.11.0 — Sidecar Tier**
Autonomous out-of-process extensions for workloads that can't run in Out-of-process extensions for workloads that can't run in Starlark (ML
Starlark: ML inference, media transcoding, language servers, local git inference, media transcoding, language servers, local git). Sidecars are
operations. The kernel does NOT manage container lifecycle — sidecars independent processes that connect inward to the kernel, following the
are independent processes (Docker, systemd, bare binary) that connect cluster registry pattern. Authentication via mTLS or registration tokens.
inward to the kernel, following the cluster registry pattern:
- Sidecar boots with cluster config (instance URLs, credentials). **v0.11.1 — Native Dialog Audit**
- Connects and self-registers, declaring capabilities ("sklearn",
"gpu", "ffmpeg", "sentence-transformers").
- Goes inert until an admin authorizes (same flow as extension permissions).
- Kernel dispatches work; sidecar returns results.
- Heartbeat keeps registration alive; missed heartbeat = eviction.
Communication channel TBD (WebSocket over existing event bus, gRPC, or
HTTP callback model — design doc required). Authentication via mTLS
certs or registration tokens.
This model eliminates k8s RBAC dependency for local deployments. The
kernel stays thin — it accepts registrations and dispatches work. The
sidecar brings its own compute and manages its own lifecycle.
**v0.10.1 — Native Dialog Audit**
Replace `prompt()`/`confirm()`/`alert()` with `sw.dialog` SDK primitives. Replace `prompt()`/`confirm()`/`alert()` with `sw.dialog` SDK primitives.
Accessible, themed, promise-based. Small kernel SDK change that improves
every surface.
**v0.10.2 — Stability + Migration Tooling** **v0.11.2 — Stability + Migration Tooling**
Proper versioned migrations (post-MVP discipline). `armature migrate` Versioned migrations, `armature migrate` CLI, backup/restore validation,
CLI command. Backup/restore validation against reference dataset. pre-1.0 schema freeze.
Pre-1.0 schema freeze.
--- ---
### v1.0.0 — Stable Release ### v1.0.0 — Stable Release
The kernel API surface is frozen. Extensions are the product. Kernel API surface frozen. Extensions are the product.
Gate criteria: Gate criteria:
- All kernel Starlark modules documented with examples - All kernel Starlark modules documented with examples
- All `api_routes` covered by OpenAPI spec - All `api_routes` covered by OpenAPI spec
- Upgrade path tested from v0.8.0 → v1.0.0 - Upgrade path tested from v0.8.0 → v1.0.0
- At least 3 reference extensions shipped and tested (vector-store, llm-bridge, chat) - At least 3 reference extensions shipped (vector-store, llm-bridge, chat)
- At least 1 cross-extension composability demo (image-gen → chat:image-actions) - At least 1 cross-extension composability demo (image-gen → chat:image-actions)
- Headless E2E green on PG + SQLite - Headless E2E green on PG + SQLite
- `armature-ca.sh` + mTLS deployment guide - `armature-ca.sh` + mTLS deployment guide
@@ -592,28 +261,14 @@ These are candidates, not commitments. Each requires a design doc.
| Single Docker image | Go binary + assets + migrations. | | Single Docker image | Go binary + assets + migrations. |
| Admin → RBAC group | Grant check replaces role check. | | Admin → RBAC group | Grant check replaces role check. |
| Settings cascade | Scope auth + `user_overridable`. Two orthogonal axes. | | Settings cascade | Scope auth + `user_overridable`. Two orthogonal axes. |
| No new migrations pre-MVP | Proper versioned migrations post-MVP. |
| Chat as extension, not kernel | Zero kernel awareness. Proves extensibility thesis. | | Chat as extension, not kernel | Zero kernel awareness. Proves extensibility thesis. |
| PG as consensus layer | UNLOGGED node_registry + LISTEN/NOTIFY. No etcd/Consul/Redis. | | PG as consensus layer | UNLOGGED node_registry + LISTEN/NOTIFY. No etcd/Consul/Redis. |
| Two trigger tiers | Extension-declared (full sandbox) vs user ad-hoc (restricted). | | Two trigger tiers | Extension-declared (full sandbox) vs user ad-hoc (restricted). |
| Builtin package rationale | Must enhance kernel surfaces or demonstrate platform capabilities. | | Two-slot topbar model | Left slot (title/branding) + center slot (`flex: 1`, tabs/pickers). |
| Two-slot topbar model | Left slot (title/branding) + center slot (`flex: 1`, tabs/pickers). Two named slots cover every navigation pattern: simple title (Pattern A), flat tabs full-width (Pattern B), category tabs + surface-owned sidebar (Pattern C). Surfaces without sub-items get full content width; surfaces with hierarchical navigation add their own sidebar below the topbar. Shell provides the stage; surface decides the theater. | | `db` module is the structured store | No separate KV primitive. Extensions declare tables. |
| All four primary surfaces migrate to shell topbar | Admin was "keep custom topbar" initially. The two-slot model makes it unnecessary — category tabs fit in the center slot, sidebar is surface-owned below. One topbar implementation replaces four. Bell + user menu + reactivity come free on every surface. | | `files` module rides ObjectStore | No new kernel tables. Metadata as companion objects. |
| Settings / Team Admin → flat tabs (Pattern B) | Both had thin sidebars (~140px) that consumed width without justification. 56 sections fit cleanly in topbar tabs. Full-width content is a better use of space for these surfaces. | | `workspace` for real filesystem | Flat blob store can't serve git/compilers/ffmpeg. Managed disk paths. |
| Team Admin Groups removed | 37-line read-only dead-end. Admin Groups has full CRUD. Restore when properly designed. |
| Docs is the reference surface | Only surface with shell Topbar, bell, user menu, inline errors. Others converge. |
| Surface runners over expanding ICD | Different test tier, different failure class. |
| Headless E2E via Playwright | Runners produce structured JSON; Playwright navigates and reads output. |
| `db` module is the structured store | No separate KV primitive. Extensions declare tables — infrastructure exists. |
| `files` module rides ObjectStore | No new kernel tables. Metadata as companion objects. Works on PVC and S3 identically. |
| `workspace` for real filesystem | Flat blob store can't serve git/compilers/ffmpeg. Managed disk paths are the explicit escape hatch. |
| Capability negotiation at install | Fail loud with actionable message, not silently at runtime. | | Capability negotiation at install | Fail loud with actionable message, not silently at runtime. |
| Vector column with three-tier fallback | Works everywhere, works fast with pgvector. Documentation > hard gates. | | Vector column with three-tier fallback | Works everywhere, works fast with pgvector. |
| Sidecar deferred to v0.10.x | HTTP module covers external APIs. Local compute is a v0.10.x concern. Sidecars connect inward (like cluster nodes), not spawned outward (no k8s RBAC needed). | | Sidecar deferred to v0.11.x | HTTP module covers external APIs. Sidecars connect inward (no k8s RBAC needed). |
| No second scripting runtime | Starlark for glue, HTTP for external compute, sidecars for local compute. Three tiers with clear boundaries. Lua/WASM middle tier rejected — Turing-complete enough to be dangerous, not isolated enough to be trustworthy. | | Workflow redesign before reference extensions | Clean up debt and promote primitives before building on top. |
| Reference extensions prove the platform | First-party `.pkg` files, not kernel code. Uninstallable. Dogfooding. |
| Composability via slots + exports, not kernel | `sw.slots` + `sw.actions` + `lib.require()` are the composition primitives. Manifest declarations add admin visibility. No inter-extension message bus — too complex, too fragile. |
| Gut cleanup before kernel expansion | Codebase analysis found 9 smells from the chat gut. Fix before adding new primitives — dead views and broken templates compound when new code builds on them. |
| PATs over OAuth client credentials | PATs are simpler (one table, one middleware check), cover all use cases (E2E tests, CI, sidecars, user automation), and follow the git hosting convention users already understand. OAuth adds client registration, token exchange, scopes — complexity without benefit for a self-hosted platform. |
| Extension-declared user permissions | Extensions register custom permissions in the kernel's group system. Admin assigns them to groups. Backend enforces via `permissions.check()` or `gate_permission`. Frontend hides via `sw.can()`. The kernel owns the permission registry; extensions contribute to it. |
| Admin token as explicit action | Following Venice.ai pattern — admin token creation is a separate endpoint, audit logged, requires `surface.admin.access`. No ambiguity about the privilege level. |

View File

@@ -1 +1 @@
0.7.11 0.9.0

View File

@@ -1,7 +1,7 @@
# DESIGN — Extension Composability # DESIGN — Extension Composability
**Version:** v0.8.4 **Version:** v0.8.5
**Status:** Draft **Status:** Implemented
**Author:** Jeff / Claude session 2026-04-02 **Author:** Jeff / Claude session 2026-04-02
--- ---

View File

@@ -0,0 +1,97 @@
# Design: Vector Column Type (v0.8.3)
## Problem
Extensions building semantic search, RAG, or recommendation features need to
store and query high-dimensional vectors (embeddings). Without kernel-level
support, each extension would need to reinvent storage, serialization, and
similarity search — duplicating effort and missing the pgvector optimization
path.
## Design
### Three-tier progressive enhancement
| Backend | Column DDL | Storage | Search |
|---------|-----------|---------|--------|
| Postgres + pgvector | `vector(N)` + HNSW index | Native vector type | `<=>` operator (index-backed) |
| Postgres (no pgvector) | `JSONB` | JSON array | Go-side cosine distance |
| SQLite | `TEXT` | JSON string | Go-side cosine distance |
Extensions declare `"vector(N)"` in their manifest `db_tables` block.
The kernel maps this to the appropriate SQL type at install time based on
detected capabilities.
### Manifest example
```json
{
"db_tables": {
"documents": {
"columns": {
"title": "text",
"embedding": "vector(384)"
}
}
},
"capabilities": {
"optional": ["pgvector"]
}
}
```
### API surface
```python
# Insert (vector as list)
db.insert("documents", {"title": "hello", "embedding": [0.1, 0.2, ...]})
# Similarity search
rows = db.query_similar(
"documents", "embedding",
vector=[0.1, 0.2, ...],
limit=10,
filters={"active": True},
metric="cosine"
)
# → [{..., "_distance": 0.023}, ...]
```
### Dispatch paths
**pgvector path** — SQL-side computation with index:
```sql
SELECT *, (embedding <=> $1::vector) AS _distance
FROM ext_pkg_documents
WHERE ... ORDER BY embedding <=> $1::vector LIMIT $2
```
**Fallback path** — Go-side computation:
1. `SELECT * FROM table WHERE filters LIMIT 1000`
2. Parse each row's vector column from JSON
3. Compute cosine distance in Go
4. Sort by distance, return top N with `_distance` injected
### Dimension validation
- Manifest: 1 ≤ N ≤ 4096 (validated by `parseVectorDim`)
- Insert-time: no dimension validation (store whatever list is given)
- Query-time: dimension mismatches produce distance = 1.0 (treated as unrelated)
### Performance characteristics
| Path | 1K rows | 10K rows | 100K rows |
|------|---------|----------|-----------|
| pgvector (HNSW) | <1ms | <5ms | <10ms |
| Fallback (Go) | <10ms | ~100ms | Not recommended |
Fallback caps at 1000 rows fetched. Extensions needing large-scale similarity
search should declare `capabilities.optional: ["pgvector"]` and degrade
gracefully.
## Limitations
- Only cosine distance metric (v0.8.3). L2 / inner product can be added later.
- No dimension validation at insert time.
- Fallback path fetches at most 1000 rows — not suitable for large datasets.
- HNSW index is only created for pgvector backends.

View File

@@ -0,0 +1,126 @@
# DESIGN — Workflow System Redesign (0.9.x)
**Version:** v0.9.0
**Status:** Draft
**Author:** Jeff / Claude session 2026-04-03
---
## Problem
The workflow system spans ~7,600 lines across 25+ files and was built
incrementally from v0.3.x through v0.7.10. It works, but several
concepts are redundant, primitives are buried, and the Starlark module
is read-only. Before shipping reference extensions (v0.10.x) that
build on workflows, the system needs cleanup and promotion of reusable
primitives.
## Audit Summary
Full audit in `/config/Downloads/AUDIT-workflow-0.9.md`. Classification:
### KEEP — Core primitives that survive
| Component | File(s) | Lines | Rationale |
|-----------|---------|-------|-----------|
| Engine core | `workflow/engine.go` | ~400 | Stateless state machine — Start, Advance, Cancel, version-pinned execution |
| Conditional routing | `workflow/routing.go` | ~500 | Pure functions, 8 operators, well-tested (497 lines of tests) |
| Automated processing | `workflow/automated.go` | ~300 | Starlark hook execution, cycle guard (max 10) |
| Scanner | `workflow/scanner.go` | ~200 | Background SLA + staleness checks, 5-minute interval |
| Signoff system | engine.go + handlers | ~400 | Multi-party approve/reject with quorum, role-gated |
| Assignment queue | handlers | ~300 | Claim/unclaim/complete/cancel lifecycle |
| Version snapshots | store + models | ~200 | Immutable snapshots, version-pinned instances |
| Typed form system | `models/workflow.go` | ~300 | 8 field types, fieldsets, conditional visibility |
| Store interface | `store/workflow_iface.go` | 61 methods | Complete lifecycle coverage |
### OBE — Superseded by new design
| Concept | Superseded by |
|---------|---------------|
| Per-stage `audience` field | Multi-page packages with mixed access declarations |
| `entry_mode` (public_link/team_only) | Package-level `scope: adoptable` + per-page access |
| `stage_mode` proliferation (4 values) | Collapse to 3: form / delegated / automated |
| `stage_type` (simple/dynamic/automated) | Redundant with `starlark_hook` presence check |
| `AdoptTeamWorkflow` clone | Package adoption model |
| `ExportWorkflowPackage` endpoint | Standard package export |
### PROMOTE — Buried features to expose as kernel primitives
1. **Roles → kernel primitive**: `team_user_roles` table, manifest `requires_roles`,
team admin UI, kernel middleware, Starlark SDK
2. **Typed forms → SDK primitive**: Move from workflow models to `forms` package,
FE SDK `sw.forms.render()` / `sw.forms.validate()`
3. **Conditional routing → SDK primitive**: Starlark `routing.evaluate(rules, data)`
4. **Workflow Starlark module → full read/write**: `workflow.start()`, `.advance()`,
`.cancel()`, `.submit_signoff()`
---
## Technical Debt
### Starlark Converter Duplication
Three files contain nearly identical Go↔Starlark conversion functions:
| File | Functions |
|------|-----------|
| `workflow/automated.go` | `goToStarlark`, `starlarkDictToMap`, `starlarkToGo` |
| `handlers/workflow_hooks.go` | `starlarkDictToMap`, `starlarkToGo`, `jsonToStarlark` |
| `sandbox/workflow_module.go` | `goValToStarlark` |
**Fix:** Consolidate into `sandbox/convert.go`.
### Snapshot Format Inconsistency
Two formats exist (wrapped `{stages, workflow}` vs legacy flat array).
Three copies of the parser across engine, instance handlers, and
assignment handlers.
**Fix:** Consolidate into one exported function. Standardize on wrapped format.
---
## Sequencing
| Version | Feature | Dependencies |
|---------|---------|-------------|
| **v0.9.0** | Starlark converter consolidation + snapshot format cleanup | None |
| **v0.9.1** | `team_user_roles` table + management API + admin UI | None |
| **v0.9.2** | `scope: adoptable` manifest field + adoption flow with role auto-populate | v0.9.1 |
| **v0.9.3** | Promote typed forms to SDK primitive (`sw.forms`) | None |
| **v0.9.4** | Deprecate `stage_type`, collapse `stage_mode` to 3 values | None |
| **v0.9.5** | Full read/write workflow Starlark module | v0.9.0 |
| **v0.9.6** | Conditional routing as SDK primitive | v0.9.5 |
| **v0.9.7** | Multi-surface manifest + kernel route resolution | None |
| **v0.9.8** | Wire roles into surface access (`access: role:X`) + kernel middleware | v0.9.1, v0.9.7 |
Each step is CI-green independently. The first four are the high-value items.
---
## Files Affected (by component)
### Converter Consolidation (v0.9.0)
- New: `sandbox/convert.go`
- Modified: `workflow/automated.go`, `handlers/workflow_hooks.go`, `sandbox/workflow_module.go`
### Team Roles (v0.9.1)
- New: migration (PG + SQLite), `store/team_roles.go`, `handlers/team_roles.go`
- Modified: manifest validation, adoption flow, team admin UI
### Typed Forms (v0.9.3)
- New: `forms/` package (extracted from `models/workflow.go`)
- New: `sw.forms` SDK module
- Modified: workflow engine to import from `forms/` instead of inline
### Workflow Starlark Module (v0.9.5)
- Modified: `sandbox/workflow_module.go` — add start/advance/cancel/signoff
- Modified: `workflow/engine.go` — extract interface to break circular import
---
## Non-Goals
- Rewriting the workflow engine. The state machine is correct and stays.
- Adding a visual workflow builder. That's an extension concern, not kernel.
- Multi-tenancy changes. Team scoping works as designed.

View File

@@ -72,19 +72,34 @@ Every package has a `manifest.json` at its root. Example for a surface:
| `icon` | no | Emoji icon for sidebar/menu | | `icon` | no | Emoji icon for sidebar/menu |
| `route` | surfaces | URL path (e.g., `/s/my-surface`) | | `route` | surfaces | URL path (e.g., `/s/my-surface`) |
| `auth` | no | `authenticated` (default) or `public` | | `auth` | no | `authenticated` (default) or `public` |
| `permissions` | no | Capabilities requested: `db.write`, `http`, `notifications`, `secrets`, `realtime.publish` | | `permissions` | no | Sandbox capabilities: `db.write`, `db.read`, `api.http`, `notifications`, `secrets`, `realtime.publish`, `connections.read`, `workflow.access`, `batch.exec`, `files.read`, `files.write`, `workspace.manage` |
| `api_routes` | no | Array of `{method, path}` for extension HTTP endpoints | | `api_routes` | no | Array of `{method, path}` for extension HTTP endpoints |
| `api_schema` | no | OpenAPI documentation for extension API routes (see below) | | `api_schema` | no | OpenAPI documentation for extension API routes (see below) |
| `db_tables` | no | Table definitions (see below) | | `db_tables` | no | Table definitions (see below) |
| `settings` | no | User-configurable settings schema | | `settings` | no | User-configurable settings schema |
| `exports` | libraries | Functions exported for other packages | | `exports` | no | Functions exported for cross-package calls via `lib.require()` |
| `depends` | no | Array of package IDs this package depends on |
| `slots` | no | Named UI injection points for host surfaces |
| `contributes` | no | Slot contributions into other surfaces |
| `hooks` | no | Event bus subscriptions | | `hooks` | no | Event bus subscriptions |
| `config_section` | no | Settings/Admin panel injection (see below) | | `config_section` | no | Settings/Admin panel injection (see below) |
| `capabilities` | no | Environment requirements (see below) |
| `user_permissions` | no | Permissions this extension registers for users (see below) |
| `gate_permission` | no | Permission checked before `on_request` executes |
| `schema_version` | no | Integer for additive schema migrations | | `schema_version` | no | Integer for additive schema migrations |
## db_tables Schema ## db_tables Schema
Tables are automatically namespaced as `ext_{package_id}_{table_name}`. Column types: `text`, `int`. Every table gets an auto-generated `id` primary key and `created_at` timestamp. Tables are automatically namespaced as `ext_{package_id}_{table_name}`.
Every table gets an auto-generated `id` primary key and `created_at` timestamp.
Column types: `text`, `int`, `vector(N)`.
The `vector(N)` type stores N-dimensional float vectors for similarity
search via `db.query_similar()`. Storage adapts to the backend:
Postgres + pgvector uses native `vector(N)` with HNSW indexes,
Postgres without pgvector uses `JSONB`, SQLite uses `TEXT`. N can be
14096. See the [Starlark Reference](STARLARK-REFERENCE) for query API.
```json ```json
"db_tables": { "db_tables": {
@@ -137,6 +152,162 @@ Extensions can optionally declare an `api_schema` array in their manifest to pro
Only `path` and `method` are required. All other fields are optional. Malformed entries are logged and skipped without blocking extension loading. Only `path` and `method` are required. All other fields are optional. Malformed entries are logged and skipped without blocking extension loading.
## Capabilities
Extensions can declare environment requirements via the `capabilities`
manifest field. The kernel validates these at install time.
```json
{
"capabilities": {
"required": ["postgres"],
"optional": ["pgvector", "workspace"]
}
}
```
- **required** — install is rejected (HTTP 422) if any capability is missing.
- **optional** — install succeeds with a logged warning. Query at runtime
with `settings.has_capability("pgvector")` to adapt behavior.
Detected capabilities: `pgvector`, `workspace`, `object_storage`, `s3`,
`postgres`. The admin can view detected capabilities at
**Admin > System > Capabilities** or via `GET /admin/capabilities`.
## User Permissions
Extensions can register custom permissions that the admin assigns to
user groups. This controls access to extension features beyond the
sandbox permission model.
```json
{
"user_permissions": ["image-gen.use", "image-gen.admin"],
"gate_permission": "image-gen.use"
}
```
- **user_permissions** — on install, these are merged into the kernel's
permission registry. On uninstall, they are removed. The admin assigns
them to groups in **Admin > Groups**.
- **gate_permission** — if set, the kernel checks this permission before
calling `on_request`. Unauthorized users get a 403 without the
extension code executing.
In Starlark, check permissions inline via `req["permissions"]` or call
`permissions.check(user_id, "image-gen.use")`.
## Extension Composability
Extensions compose with each other through three mechanisms: manifest-declared
**slots** (UI injection points), **contributions** (UI components injected into
those slots), and cross-package **function calls** via `lib.require()`.
### Slots — Host Surfaces Declare Injection Points
A surface declares named slots in its manifest where other extensions can
inject UI components:
```json
{
"id": "notes",
"type": "surface",
"slots": {
"toolbar-actions": {
"description": "Toolbar action buttons",
"context": {
"noteId": "string",
"getContent": "function — returns note body",
"setContent": "function — replaces note body"
}
}
}
}
```
The surface renders slot contents using the SDK helper:
```javascript
html`<div class="toolbar">
${sw.slots.renderAll('notes:toolbar-actions', {
noteId: note.id,
getContent: () => editor.getValue(),
setContent: (text) => editor.setValue(text),
})}
</div>`
```
### Contributions — Extensions Inject UI
An extension declares which slots it contributes to:
```json
{
"id": "note-dictate",
"type": "extension",
"contributes": {
"notes:toolbar-actions": {
"label": "Dictate",
"icon": "🎤",
"description": "Voice-to-text dictation"
}
}
}
```
In its JavaScript, it registers the component:
```javascript
sw.slots.register('notes:toolbar-actions', {
id: 'note-dictate',
priority: 200,
component: ({ noteId, setContent, getContent }) => {
// ... component implementation
return html`<button class="sw-btn sw-btn--ghost">🎤</button>`;
}
});
```
Contributions are soft-coupled — install order doesn't matter. The admin
can view all slots and contributors at **Admin > Packages** or via
`GET /api/v1/admin/slots`.
### Cross-Package Function Calls
Any package that declares `exports` in its manifest can be called by other
packages via `lib.require()`. The caller declares the dependency:
```json
{
"id": "note-ai",
"depends": ["llm-bridge"],
"permissions": ["api.http"]
}
```
In Starlark:
```python
llm = lib.require("llm-bridge")
result = llm.complete([{"role": "user", "content": prompt}])
```
The called function runs with the *target* package's permissions, not the
caller's. This is the same security model as library packages.
### Slot Naming Convention
Slot names follow `{host-package-id}:{slot-name}`. The colon separates the
namespace from the slot. Standard slots for first-party packages:
| Slot | Host | Use Case |
|------|------|----------|
| `notes:toolbar-actions` | notes | Dictation, AI tools, formatting |
| `notes:note-footer` | notes | Related items, AI summary |
| `chat:composer-tools` | chat | Image gen, file attach |
| `chat:message-actions` | chat | Reactions, translate, bookmark |
| `chat:image-actions` | chat | Regen, edit, upscale |
## Starlark Sandbox API ## Starlark Sandbox API
Starlark scripts run server-side with a 1M operation budget and no Starlark scripts run server-side with a 1M operation budget and no

201
docs/MULTI-SURFACE-GUIDE.md Normal file
View File

@@ -0,0 +1,201 @@
# Multi-Surface Packages
A multi-surface package serves multiple pages from a single package, each
with its own URL path, access level, and layout. Before v0.9.0, a package
got one route (`/s/{id}`), one auth posture, and one layout. Now a single
package can serve a public submission form alongside an authenticated
dashboard and an admin settings page.
## When to Use Multi-Surface
Use multi-surface when your package has logically related pages that share
the same backend (Starlark hooks, ext API, database tables) but need:
- **Different access levels** — public intake form + authenticated dashboard
- **Multiple views** — list view, detail view, edit view, monitor view
- **Sub-pages** — settings, admin, or debug pages within the package
If your pages don't share backend state, use separate packages instead.
## Manifest Setup
Add a `surfaces` array to your manifest. Each entry declares a page:
```json
{
"id": "workflow-builder",
"title": "Workflow Builder",
"type": "full",
"version": "0.1.0",
"icon": "🔧",
"auth": "authenticated",
"layout": "single",
"surfaces": [
{ "path": "/", "title": "Workflows" },
{ "path": "/new", "title": "New Workflow", "nav": false },
{ "path": "/:id/edit", "title": "Edit Workflow", "nav": false },
{ "path": "/monitor", "title": "Monitor", "nav": true },
{ "path": "/monitor/:id", "title": "Instance Detail", "nav": false }
],
"hooks": ["surface"],
"permissions": ["workflow.access"]
}
```
This generates five server routes, all under `/s/workflow-builder/`:
| Route | Access | Nav |
|-------|--------|-----|
| `/s/workflow-builder/` | authenticated | yes |
| `/s/workflow-builder/new` | authenticated | no |
| `/s/workflow-builder/:id/edit` | authenticated | no |
| `/s/workflow-builder/monitor` | authenticated | yes |
| `/s/workflow-builder/monitor/:id` | authenticated | no |
### Surface Entry Fields
| Field | Default | Description |
|----------|----------------|-------------|
| `path` | **required** | URL path relative to `/s/{id}`. Supports `:param` segments. |
| `access` | package `auth` | `public`, `authenticated`, `admin`, or `group:{name}`. |
| `title` | package `title`| Label shown in nav and page title. |
| `layout` | package `layout`| `single` or `editor`. |
| `nav` | `true` for `/`, `false` otherwise | Whether this surface appears in the sidebar. |
### Mixed Access Levels
A package can mix public and authenticated surfaces:
```json
{
"id": "bug-tracker",
"auth": "authenticated",
"surfaces": [
{ "path": "/submit", "access": "public", "title": "Report a Bug" },
{ "path": "/", "title": "Dashboard" },
{ "path": "/:id", "title": "Bug Detail", "nav": false },
{ "path": "/admin", "access": "admin", "title": "Settings", "nav": false }
]
}
```
The kernel enforces access per-surface. Unauthenticated visitors can reach
`/submit` but are redirected to login if they try `/` or `/:id`.
## Frontend: Routing Within Your Package
### Reading the Current Surface
When your package JS loads, two globals tell you which surface was matched:
```js
const path = window.__SURFACE_PATH__ || '/';
const params = window.__SURFACE_PARAMS__ || {};
```
Use these to decide which view to render:
```js
function render() {
const path = window.__SURFACE_PATH__ || '/';
const params = window.__SURFACE_PARAMS__ || {};
const root = document.getElementById('surface-root');
switch (path) {
case '/': return renderList(root);
case '/new': return renderEditor(root, null);
case '/:id/edit': return renderEditor(root, params.id);
case '/monitor': return renderMonitor(root);
case '/monitor/:id':return renderDetail(root, params.id);
default: return render404(root);
}
}
render();
```
### SPA Navigation with `sw.navigate()`
Navigate between surfaces without a full page reload:
```js
// Navigate to a static path
sw.navigate('/new');
// Navigate with params
sw.navigate('/:id/edit', { id: 'wf-42' });
// Navigate to monitor sub-page
sw.navigate('/monitor/:id', { id: 'inst-7' });
```
`sw.navigate()` does three things:
1. Updates `window.__SURFACE_PATH__` and `window.__SURFACE_PARAMS__`
2. Calls `history.pushState()` to update the URL
3. Emits a `surface.navigate` event
### Listening for Navigation Events
Re-render when the user navigates (including browser back/forward):
```js
sw.on('surface.navigate', ({ path, params }) => {
window.__SURFACE_PATH__ = path;
window.__SURFACE_PARAMS__ = params;
render();
});
```
### Links Between Surfaces
For simple `<a>` links that do full page loads:
```html
<a href="/s/workflow-builder/monitor">Monitor</a>
```
For SPA-style navigation:
```js
button.onclick = () => sw.navigate('/monitor');
```
## API Routes
All surfaces in a package share the same ext API. API calls go through
`/s/{id}/api/*` regardless of which surface is active:
```js
// These work from any surface in the package
const items = await sw.api.get('/items');
const item = await sw.api.get(`/items/${id}`);
await sw.api.post('/items', { title: 'New item' });
```
The ext API handler checks `package.status == "active"` — if the package
is in `pending_review`, all API calls return 403.
## Backward Compatibility
Packages without a `surfaces` array continue to work. The kernel
synthesizes a single-entry array from the legacy `auth` and `layout`
fields:
```
auth: "authenticated" + layout: "single"
→ surfaces: [{ "path": "/", "access": "authenticated", "layout": "single" }]
```
No migration is required for existing packages.
## Constraints
- **No cross-package routing.** Surface paths are relative to the package
mount point. A package cannot claim arbitrary top-level routes.
- **No per-surface Starlark hooks.** All surfaces share the same backend
hooks. Use the surface path in your hook logic to differentiate.
- **No SSR.** Surface rendering is client-side. The kernel serves the
shell template; your JS renders the content.

View File

@@ -60,17 +60,140 @@ Only `manifest.json` is required. All other directories are optional and include
| `description` | Short description | | `description` | Short description |
| `icon` | Emoji for sidebar/menu display | | `icon` | Emoji for sidebar/menu display |
| `author` | Package author | | `author` | Package author |
| `route` | URL path for surfaces (e.g., `/s/my-package`) | | `surfaces` | Array of surface entries with per-path access, title, layout (see below) |
| `auth` | `authenticated` or `public` | | `route` | *(deprecated — use `surfaces`)* URL path for surfaces |
| `layout` | Surface layout mode (e.g., `single`) | | `auth` | Default access level for all surfaces: `authenticated`, `public`, `admin` |
| `layout` | Default layout for all surfaces: `single`, `editor` |
| `permissions` | Array of required capabilities (`db.write`, `http`, `notifications`, `secrets`, `realtime.publish`) | | `permissions` | Array of required capabilities (`db.write`, `http`, `notifications`, `secrets`, `realtime.publish`) |
| `api_routes` | Array of `{"method": "GET", "path": "/items"}` | | `api_routes` | Array of `{"method": "GET", "path": "/items"}` |
| `db_tables` | Table definitions with columns and indexes | | `db_tables` | Table definitions with columns and indexes |
| `settings` | User-configurable settings with type, label, description, default | | `settings` | User-configurable settings with type, label, description, default |
| `hooks` | Event bus subscription patterns | | `hooks` | Event bus subscription patterns |
| `exports` | Functions exported by library packages | | `exports` | Functions exported for cross-package calls via `lib.require()` |
| `depends` | Array of package IDs this package depends on |
| `slots` | Named UI injection points (host surfaces declare these) |
| `contributes` | Slot contributions this package injects into other surfaces |
| `capabilities` | Environment requirements: `{"required": [...], "optional": [...]}` |
| `schema_version` | Integer for additive schema migrations | | `schema_version` | Integer for additive schema migrations |
## Multi-Surface Packages
A package can serve multiple pages, each with its own path, access level,
title, and layout. Declare a `surfaces` array in the manifest:
```json
{
"id": "bug-tracker",
"title": "Bug Tracker",
"type": "full",
"auth": "authenticated",
"layout": "single",
"surfaces": [
{ "path": "/", "title": "Dashboard" },
{ "path": "/submit", "access": "public", "title": "Report a Bug" },
{ "path": "/:id", "title": "Bug Detail", "nav": false },
{ "path": "/admin", "access": "admin", "title": "Settings", "nav": false }
]
}
```
### Surface Entry Fields
| Field | Type | Default | Description |
|----------|--------|----------------|-------------|
| `path` | string | **required** | Relative to `/s/{pkg-id}`. Supports `:param` segments. |
| `access` | string | package `auth` | `public`, `authenticated`, `admin`, `group:{name}`. |
| `title` | string | package `title`| Human label. Used in nav if `nav: true`. |
| `layout` | string | package `layout`| `single`, `editor`, or future layouts. |
| `nav` | bool | see rules | Show in sidebar navigation. |
### Nav Visibility Rules
- `path: "/"` defaults to `nav: true` (primary entry point)
- All others default to `nav: false` (sub-pages)
- Explicit `"nav": true` overrides — a package can put multiple entries in nav
- The sidebar links to the first surface with `nav: true`
### Backward Compatibility
If `surfaces` is absent, the kernel synthesizes one entry from the legacy
`auth` and `layout` fields. No existing packages break.
### Client-Side Navigation
Within a multi-surface package, use `sw.navigate()` for SPA-style routing:
```js
const path = window.__SURFACE_PATH__ || '/';
const params = window.__SURFACE_PARAMS__ || {};
if (path === '/') renderDashboard();
if (path === '/submit') renderSubmitForm();
if (path === '/:id') renderDetail(params.id);
// Navigate to another surface within the package
sw.navigate('/submit');
sw.navigate('/:id', { id: 'bug-42' });
// Listen for navigation events (including back/forward)
sw.on('surface.navigate', ({ path, params }) => {
renderView(path, params);
});
```
## Composability: Slots and Contributions
Packages compose with each other through named UI injection points (**slots**) and **contributions**.
### Declaring Slots (Host Surfaces)
Surfaces declare slots where other extensions can inject UI:
```json
{
"slots": {
"toolbar-actions": {
"description": "Toolbar action buttons",
"context": {
"noteId": "string — current note ID",
"getContent": "function — returns note body text"
}
}
}
}
```
Slot names are namespaced at runtime as `{package-id}:{slot-name}` (e.g., `notes:toolbar-actions`). The `context` field documents what data the slot provides — it is not enforced at runtime.
### Contributing to Slots
Extensions declare which slots they inject into:
```json
{
"contributes": {
"notes:toolbar-actions": {
"label": "Dictate",
"icon": "🎤",
"description": "Voice-to-text dictation"
}
}
}
```
Contributing extensions can be installed before or after their host surface — the coupling is soft. The admin can view all slots and their contributors at `GET /api/v1/admin/slots`.
### Cross-Package Function Calls
Any package that declares `exports` can be called via `lib.require()`, not just library-type packages. The caller must declare the target in its `depends` array:
```json
{
"depends": ["image-gen"],
"permissions": ["api.http"]
}
```
## Package Lifecycle ## Package Lifecycle
1. **Install**: Upload a `.pkg` file via Admin > Packages or `POST /api/v1/admin/packages/install`. The kernel extracts the archive, creates database tables, and registers routes. 1. **Install**: Upload a `.pkg` file via Admin > Packages or `POST /api/v1/admin/packages/install`. The kernel extracts the archive, creates database tables, and registers routes.

View File

@@ -37,9 +37,20 @@ val = settings.get("theme", "light")
The cascade respects the `user_overridable` flag from the package manifest. The cascade respects the `user_overridable` flag from the package manifest.
See [Permissions & Groups](PERMISSIONS-AND-GROUPS) for details. See [Permissions & Groups](PERMISSIONS-AND-GROUPS) for details.
```python
# Check if a runtime capability is available
if settings.has_capability("pgvector"):
# Use native vector search
...
```
`has_capability(name)` returns `True` if the named environment capability
is detected by the kernel. Detected capabilities: `pgvector`, `workspace`,
`object_storage`, `s3`, `postgres`.
### lib ### lib
Load exported functions from library packages. Load exported functions from other packages.
```python ```python
helpers = lib.require("my-utils") helpers = lib.require("my-utils")
@@ -47,11 +58,31 @@ result = helpers.format_date("2026-01-15")
``` ```
Requirements: Requirements:
- The library must be declared in your package manifest's `dependencies`. - The target package must be declared in your manifest's `depends` array.
- The library must be type `library`, status `active`, tier `starlark`. - The target package must declare `exports` in its manifest.
- The target package must be status `active`, tier `starlark`.
- Any package type (`library`, `extension`, `full`) can be called as long
as it declares exports. This enables full packages to expose callable
functions alongside their UI and API routes.
- Circular dependencies are detected and rejected. - Circular dependencies are detected and rejected.
- Results are cached per execution (calling `require` twice returns the - Results are cached per execution (calling `require` twice returns the
same object). same object).
- The called function runs with the *target* package's permissions, not
the caller's.
### permissions
Check whether a user has a specific permission.
```python
if permissions.check(user_id, "image-gen.use"):
# User is authorized
...
```
Returns `True` if the user has the permission, `False` otherwise (including
when the user is not found). Resolves the user's groups and merges granted
permissions — works for both kernel and extension-declared permissions.
## Permission-gated modules ## Permission-gated modules
@@ -136,6 +167,33 @@ results = db.query_batch([
# Each query spec supports: table (required), filters, order, limit, before, after, search_like # Each query spec supports: table (required), filters, order, limit, before, after, search_like
``` ```
#### Vector similarity search
```python
# Find rows with the most similar embeddings (cosine distance)
rows = db.query_similar(
"documents", # table name
"embedding", # vector column name
vector=[0.1, 0.2, ...], # query vector (list of floats)
limit=10, # max results (default 10, max 100)
filters={"active": True}, # optional equality filters
metric="cosine", # only "cosine" supported
)
# Returns rows ordered by ascending _distance (0.0 = identical, 1.0 = orthogonal)
# Each row dict includes an injected "_distance" float key.
```
Vector columns are declared as `"vector(N)"` in the manifest `db_tables` block
(N = dimension, 14096). Storage varies by backend:
| Backend | Column type | Search |
|---------|-------------|--------|
| Postgres + pgvector | `vector(N)` with HNSW index | Native `<=>` operator |
| Postgres (no pgvector) | `JSONB` | Go-side cosine computation |
| SQLite | `TEXT` | Go-side cosine computation |
Insert vectors as lists: `db.insert("docs", {"embedding": [0.1, 0.2, 0.3]})`.
#### Write operations #### Write operations
```python ```python
@@ -275,6 +333,83 @@ results, errors = batch.exec([
--- ---
### files
**Permissions:** `files.read`, `files.write`
Store and retrieve files via the kernel ObjectStore (PVC or S3).
All keys are scoped to `ext/{packageID}/` — extensions cannot access
each other's files.
```python
# Store a file with optional metadata
files.put("reports/q1.pdf", pdf_bytes,
content_type="application/pdf",
metadata={"quarter": "Q1", "year": 2026})
# Read a file
result = files.get("reports/q1.pdf")
# result = {"content": b"...", "content_type": "application/pdf",
# "size": 12345, "metadata": {"quarter": "Q1", "year": 2026}}
# Metadata only (no content transfer)
meta = files.meta("reports/q1.pdf")
# List files by prefix
entries = files.list(prefix="reports/", limit=50)
# entries = [{"name": "reports/q1.pdf", "size": 12345, "content_type": "..."}]
# Check existence
if files.exists("reports/q1.pdf"):
files.delete("reports/q1.pdf")
# Bulk delete
files.delete_prefix("temp/")
```
**Notes:**
- `content` accepts string or bytes; `get()` returns bytes.
- Maximum file size: 50 MB (configurable via `EXT_FILES_MAX_SIZE`).
- Metadata is stored as a companion JSON object, not in the file itself.
- `files.list()` automatically filters out internal metadata companions.
---
### workspace
**Permission:** `workspace.manage`
Managed disk directories for extensions that need a real filesystem
(git clones, compilers, media tools). Each workspace is scoped to
`{WORKSPACE_ROOT}/{packageID}/{name}/`.
```python
# Create a workspace (idempotent)
path = workspace.create("my-repo")
# Returns the absolute path to the directory
# Get the path (None if workspace doesn't exist)
path = workspace.path("my-repo")
# List all workspaces owned by this extension
names = workspace.list() # ["my-repo", "cache"]
# Delete a workspace and all its contents
workspace.delete("my-repo")
# Get disk usage in bytes (10-second timeout)
size = workspace.usage("my-repo") # 1048576
```
**Constraints:**
- Names must match `^[a-z][a-z0-9_]{0,62}$` (lowercase, no spaces or separators).
- Path traversal and symlink escape are blocked.
- Quota enforcement via `WORKSPACE_QUOTA_MB` env var (0 = unlimited).
- Module not available if `WORKSPACE_ROOT` is unset or not writable.
Use `settings.has_capability("workspace")` to check availability.
---
## Example: automated stage hook ## Example: automated stage hook
A simple hook that reads a setting, queries data, and advances: A simple hook that reads a setting, queries data, and advances:

View File

@@ -47,6 +47,12 @@ type Config struct {
S3Prefix string // optional key prefix within bucket (e.g. "armature/") S3Prefix string // optional key prefix within bucket (e.g. "armature/")
S3ForcePathStyle bool // use path-style URLs (required for MinIO, most self-hosted) S3ForcePathStyle bool // use path-style URLs (required for MinIO, most self-hosted)
// Extension workspaces
// WORKSPACE_ROOT: mount point for extension-managed directories (default /data/workspaces).
// WORKSPACE_QUOTA_MB: per-extension quota in MB (default 0 = unlimited).
WorkspaceRoot string
WorkspaceQuotaMB int
// Structured logging // Structured logging
// LOG_FORMAT: "text" (default, backward-compatible) or "json" (structured). // LOG_FORMAT: "text" (default, backward-compatible) or "json" (structured).
// LOG_LEVEL: "debug", "info" (default), "warn", "error". // LOG_LEVEL: "debug", "info" (default), "warn", "error".
@@ -143,6 +149,9 @@ func Load() *Config {
S3Prefix: getEnv("S3_PREFIX", ""), S3Prefix: getEnv("S3_PREFIX", ""),
S3ForcePathStyle: getEnv("S3_FORCE_PATH_STYLE", "true") == "true", S3ForcePathStyle: getEnv("S3_FORCE_PATH_STYLE", "true") == "true",
WorkspaceRoot: getEnv("WORKSPACE_ROOT", "/data/workspaces"),
WorkspaceQuotaMB: getEnvInt("WORKSPACE_QUOTA_MB", 0),
SkipBundledPackages: getEnvBool("SKIP_BUNDLED_PACKAGES", false), SkipBundledPackages: getEnvBool("SKIP_BUNDLED_PACKAGES", false),
BundledPackagesDir: getEnv("BUNDLED_PACKAGES_DIR", "/app/bundled-packages"), BundledPackagesDir: getEnv("BUNDLED_PACKAGES_DIR", "/app/bundled-packages"),
BundledPackages: getEnv("BUNDLED_PACKAGES", ""), BundledPackages: getEnv("BUNDLED_PACKAGES", ""),

View File

@@ -0,0 +1,96 @@
package handlers
import (
"net/http"
"strings"
"github.com/gin-gonic/gin"
)
// slotInfo describes a declared slot and its contributors.
type slotInfo struct {
Host string `json:"host"`
Description string `json:"description"`
Context map[string]any `json:"context,omitempty"`
Contributors []slotContributor `json:"contributors"`
}
// slotContributor describes one package's contribution to a slot.
type slotContributor struct {
Package string `json:"package"`
Label string `json:"label,omitempty"`
Icon string `json:"icon,omitempty"`
Description string `json:"description,omitempty"`
}
// ListSlots returns an aggregated map of all declared slots across installed
// packages together with their contributors. Built on-the-fly from manifests.
// GET /api/v1/admin/slots
func (h *PackageHandler) ListSlots(c *gin.Context) {
pkgs, err := h.stores.Packages.List(c.Request.Context())
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list packages"})
return
}
slots := map[string]*slotInfo{}
// Pass 1: collect declared slots from host surfaces
for _, pkg := range pkgs {
slotsRaw, ok := pkg.Manifest["slots"].(map[string]any)
if !ok {
continue
}
for name, v := range slotsRaw {
fullName := pkg.ID + ":" + name
entry, ok := v.(map[string]any)
if !ok {
continue
}
desc, _ := entry["description"].(string)
var ctx map[string]any
if c, ok := entry["context"].(map[string]any); ok {
ctx = c
}
slots[fullName] = &slotInfo{
Host: pkg.ID,
Description: desc,
Context: ctx,
Contributors: []slotContributor{},
}
}
}
// Pass 2: collect contributions
for _, pkg := range pkgs {
if !pkg.Enabled {
continue
}
contribs, ok := pkg.Manifest["contributes"].(map[string]any)
if !ok {
continue
}
for slotKey, v := range contribs {
// Ensure the slot entry exists (contributor can arrive before host)
if _, exists := slots[slotKey]; !exists {
host := slotKey
if idx := strings.Index(slotKey, ":"); idx >= 0 {
host = slotKey[:idx]
}
slots[slotKey] = &slotInfo{
Host: host,
Contributors: []slotContributor{},
}
}
contrib := slotContributor{Package: pkg.ID}
if m, ok := v.(map[string]any); ok {
contrib.Label, _ = m["label"].(string)
contrib.Icon, _ = m["icon"].(string)
contrib.Description, _ = m["description"].(string)
}
slots[slotKey].Contributors = append(slots[slotKey].Contributors, contrib)
}
}
c.JSON(http.StatusOK, gin.H{"data": slots})
}

View File

@@ -0,0 +1,146 @@
package handlers
import (
"context"
"database/sql"
"log"
"github.com/gin-gonic/gin"
"armature/storage"
)
// ── Capability detection ────────────────────────────────────────────
// DetectCapabilities probes the runtime environment and returns a map
// of capability name → available. Called once at startup and again on
// each admin GET /capabilities request.
func DetectCapabilities(db *sql.DB, isPostgres bool, objStore storage.ObjectStore, workspaceRoot string) map[string]bool {
caps := map[string]bool{
"postgres": false,
"pgvector": false,
"object_storage": false,
"s3": false,
"workspace": false,
}
// postgres: dialect check
if isPostgres {
caps["postgres"] = true
}
// pgvector: query pg_extension (PG only)
if isPostgres && db != nil {
var one int
err := db.QueryRow("SELECT 1 FROM pg_extension WHERE extname = 'vector'").Scan(&one)
caps["pgvector"] = err == nil
}
// object_storage: configured and healthy
if objStore != nil {
if err := objStore.Healthy(context.Background()); err == nil {
caps["object_storage"] = true
}
}
// s3: specific backend check
if objStore != nil && objStore.Backend() == "s3" {
caps["s3"] = true
}
// workspace: root exists and is writable
if workspaceRoot != "" && storage.IsPathWritable(workspaceRoot) {
caps["workspace"] = true
}
return caps
}
// ── Manifest parsing ────────────────────────────────────────────────
// Capabilities holds the parsed capabilities block from a package manifest.
type Capabilities struct {
Required []string
Optional []string
}
// ParseCapabilities extracts capabilities.required and capabilities.optional
// from a manifest map. Returns zero value + false if no capabilities block.
func ParseCapabilities(manifest map[string]any) (Capabilities, bool) {
capsRaw, ok := manifest["capabilities"].(map[string]any)
if !ok {
return Capabilities{}, false
}
var caps Capabilities
if req, ok := capsRaw["required"].([]any); ok {
for _, v := range req {
if s, ok := v.(string); ok && s != "" {
caps.Required = append(caps.Required, s)
}
}
}
if opt, ok := capsRaw["optional"].([]any); ok {
for _, v := range opt {
if s, ok := v.(string); ok && s != "" {
caps.Optional = append(caps.Optional, s)
}
}
}
return caps, true
}
// CheckRequiredCapabilities returns the subset of required capabilities
// that are not present in the detected set.
func CheckRequiredCapabilities(required []string, detected map[string]bool) []string {
var missing []string
for _, r := range required {
if !detected[r] {
missing = append(missing, r)
}
}
return missing
}
// capabilityHelpText returns operator-facing remediation hints keyed by
// capability name.
func capabilityHelpText(missing []string) map[string]string {
hints := map[string]string{
"postgres": "Switch DB_DRIVER to postgres and configure DATABASE_URL.",
"pgvector": "Run `CREATE EXTENSION vector;` in your PostgreSQL database.",
"object_storage": "Configure STORAGE_BACKEND (pvc or s3) and ensure the backend is healthy.",
"s3": "Set STORAGE_BACKEND=s3 and configure S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY, S3_SECRET_KEY.",
"workspace": "Set WORKSPACE_ROOT to a writable directory path.",
}
result := make(map[string]string, len(missing))
for _, m := range missing {
if h, ok := hints[m]; ok {
result[m] = h
} else {
result[m] = "Unknown capability — check package documentation."
}
}
return result
}
// ── Admin endpoint ──────────────────────────────────────────────────
// CapabilitiesHandler serves the admin capabilities endpoint.
type CapabilitiesHandler struct {
detect func() map[string]bool
}
// NewCapabilitiesHandler creates a handler that re-probes capabilities on
// each request so the response reflects current state.
func NewCapabilitiesHandler(detect func() map[string]bool) *CapabilitiesHandler {
return &CapabilitiesHandler{detect: detect}
}
// GetCapabilities returns detected environment capabilities.
// GET /api/v1/admin/capabilities
func (h *CapabilitiesHandler) GetCapabilities(c *gin.Context) {
caps := h.detect()
log.Printf("[capabilities] probed: %v", caps)
c.JSON(200, caps)
}

View File

@@ -0,0 +1,205 @@
package handlers
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
)
// ── ParseCapabilities ───────────────────────────────────────────────
func TestParseCapabilities_Full(t *testing.T) {
manifest := map[string]any{
"id": "test-pkg",
"capabilities": map[string]any{
"required": []any{"postgres", "pgvector"},
"optional": []any{"s3"},
},
}
caps, ok := ParseCapabilities(manifest)
if !ok {
t.Fatal("expected ok=true")
}
if len(caps.Required) != 2 || caps.Required[0] != "postgres" || caps.Required[1] != "pgvector" {
t.Errorf("required = %v, want [postgres pgvector]", caps.Required)
}
if len(caps.Optional) != 1 || caps.Optional[0] != "s3" {
t.Errorf("optional = %v, want [s3]", caps.Optional)
}
}
func TestParseCapabilities_Missing(t *testing.T) {
manifest := map[string]any{"id": "test-pkg"}
_, ok := ParseCapabilities(manifest)
if ok {
t.Error("expected ok=false when no capabilities block")
}
}
func TestParseCapabilities_EmptyArrays(t *testing.T) {
manifest := map[string]any{
"capabilities": map[string]any{
"required": []any{},
"optional": []any{},
},
}
caps, ok := ParseCapabilities(manifest)
if !ok {
t.Fatal("expected ok=true")
}
if len(caps.Required) != 0 {
t.Errorf("required should be empty, got %v", caps.Required)
}
if len(caps.Optional) != 0 {
t.Errorf("optional should be empty, got %v", caps.Optional)
}
}
func TestParseCapabilities_IgnoresEmpty(t *testing.T) {
manifest := map[string]any{
"capabilities": map[string]any{
"required": []any{"pgvector", "", "postgres"},
},
}
caps, _ := ParseCapabilities(manifest)
if len(caps.Required) != 2 {
t.Errorf("expected 2 entries (empty filtered), got %v", caps.Required)
}
}
// ── CheckRequiredCapabilities ───────────────────────────────────────
func TestCheckRequired_AllMet(t *testing.T) {
detected := map[string]bool{"postgres": true, "pgvector": true, "s3": true}
missing := CheckRequiredCapabilities([]string{"postgres", "pgvector"}, detected)
if len(missing) != 0 {
t.Errorf("expected no missing, got %v", missing)
}
}
func TestCheckRequired_SomeUnmet(t *testing.T) {
detected := map[string]bool{"postgres": true, "pgvector": false, "s3": false}
missing := CheckRequiredCapabilities([]string{"postgres", "pgvector", "s3"}, detected)
if len(missing) != 2 {
t.Errorf("expected 2 missing, got %v", missing)
}
}
func TestCheckRequired_EmptyRequired(t *testing.T) {
detected := map[string]bool{"postgres": true}
missing := CheckRequiredCapabilities(nil, detected)
if len(missing) != 0 {
t.Errorf("expected no missing for nil required, got %v", missing)
}
}
func TestCheckRequired_UnknownCap(t *testing.T) {
detected := map[string]bool{"postgres": true}
missing := CheckRequiredCapabilities([]string{"gpu"}, detected)
if len(missing) != 1 || missing[0] != "gpu" {
t.Errorf("expected [gpu], got %v", missing)
}
}
// ── capabilityHelpText ──────────────────────────────────────────────
func TestCapabilityHelpText_KnownCaps(t *testing.T) {
hints := capabilityHelpText([]string{"pgvector", "s3"})
if len(hints) != 2 {
t.Fatalf("expected 2 hints, got %d", len(hints))
}
if hints["pgvector"] == "" {
t.Error("pgvector hint is empty")
}
if hints["s3"] == "" {
t.Error("s3 hint is empty")
}
}
func TestCapabilityHelpText_UnknownCap(t *testing.T) {
hints := capabilityHelpText([]string{"gpu"})
if hints["gpu"] == "" {
t.Error("expected fallback hint for unknown cap")
}
}
// ── DetectCapabilities (SQLite path — no real DB) ───────────────────
func TestDetectCapabilities_SQLite_NilStore(t *testing.T) {
caps := DetectCapabilities(nil, false, nil, "")
if caps["postgres"] {
t.Error("postgres should be false on SQLite")
}
if caps["pgvector"] {
t.Error("pgvector should be false on SQLite")
}
if caps["object_storage"] {
t.Error("object_storage should be false with nil store")
}
if caps["s3"] {
t.Error("s3 should be false with nil store")
}
if caps["workspace"] {
t.Error("workspace should be false with empty root")
}
}
func TestDetectCapabilities_WorkspaceWritable(t *testing.T) {
tmpDir := t.TempDir()
caps := DetectCapabilities(nil, false, nil, tmpDir)
if !caps["workspace"] {
t.Error("workspace should be true for writable temp dir")
}
}
func TestDetectCapabilities_WorkspaceUnwritable(t *testing.T) {
caps := DetectCapabilities(nil, false, nil, "/nonexistent/path/xyz")
if caps["workspace"] {
t.Error("workspace should be false for nonexistent path")
}
}
// ── Admin endpoint ──────────────────────────────────────────────────
func TestGetCapabilities_HTTP(t *testing.T) {
gin.SetMode(gin.TestMode)
h := NewCapabilitiesHandler(func() map[string]bool {
return map[string]bool{
"postgres": false,
"pgvector": false,
"object_storage": true,
"s3": false,
"workspace": true,
}
})
r := gin.New()
r.GET("/api/v1/admin/capabilities", h.GetCapabilities)
req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/capabilities", nil)
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
if w.Code != 200 {
t.Fatalf("status = %d, want 200", w.Code)
}
var result map[string]bool
if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil {
t.Fatalf("unmarshal: %v", err)
}
if result["postgres"] {
t.Error("postgres should be false")
}
if !result["object_storage"] {
t.Error("object_storage should be true")
}
if !result["workspace"] {
t.Error("workspace should be true")
}
}

View File

@@ -0,0 +1,97 @@
package handlers
import (
"testing"
)
func TestValidateComposabilityFields_ValidSlots(t *testing.T) {
m := map[string]any{
"slots": map[string]any{
"toolbar-actions": map[string]any{
"description": "Toolbar action buttons",
"context": map[string]any{
"noteId": "string",
},
},
},
}
if err := ValidateComposabilityFields(m); err != "" {
t.Errorf("expected valid, got: %s", err)
}
}
func TestValidateComposabilityFields_SlotMissingDescription(t *testing.T) {
m := map[string]any{
"slots": map[string]any{
"toolbar": map[string]any{
"context": map[string]any{},
},
},
}
if err := ValidateComposabilityFields(m); err == "" {
t.Error("expected error for slot missing description")
}
}
func TestValidateComposabilityFields_SlotsNotObject(t *testing.T) {
m := map[string]any{
"slots": "bad",
}
if err := ValidateComposabilityFields(m); err == "" {
t.Error("expected error for non-object slots")
}
}
func TestValidateComposabilityFields_ValidContributes(t *testing.T) {
m := map[string]any{
"contributes": map[string]any{
"notes:toolbar-actions": map[string]any{
"label": "Dictate",
"icon": "🎤",
},
},
}
if err := ValidateComposabilityFields(m); err != "" {
t.Errorf("expected valid, got: %s", err)
}
}
func TestValidateComposabilityFields_ContributesMissingColon(t *testing.T) {
m := map[string]any{
"contributes": map[string]any{
"toolbar-actions": map[string]any{
"label": "Bad",
},
},
}
if err := ValidateComposabilityFields(m); err == "" {
t.Error("expected error for contributes key without colon")
}
}
func TestValidateComposabilityFields_Empty(t *testing.T) {
m := map[string]any{
"id": "test",
}
if err := ValidateComposabilityFields(m); err != "" {
t.Errorf("expected valid for manifest without slots/contributes, got: %s", err)
}
}
func TestValidateComposabilityFields_BothValid(t *testing.T) {
m := map[string]any{
"slots": map[string]any{
"my-slot": map[string]any{
"description": "A slot",
},
},
"contributes": map[string]any{
"other:their-slot": map[string]any{
"label": "Hello",
},
},
}
if err := ValidateComposabilityFields(m); err != "" {
t.Errorf("expected valid, got: %s", err)
}
}

View File

@@ -13,20 +13,38 @@ import (
"fmt" "fmt"
"log" "log"
"regexp" "regexp"
"strconv"
"strings" "strings"
"armature/store" "armature/store"
) )
// vectorDimRegex matches manifest type strings like "vector(384)".
var vectorDimRegex = regexp.MustCompile(`^vector\((\d+)\)$`)
// validSchemaIdentifier matches safe SQL identifiers for table and column names. // validSchemaIdentifier matches safe SQL identifiers for table and column names.
var validSchemaIdentifier = regexp.MustCompile(`^[a-z][a-z0-9_]{0,62}$`) var validSchemaIdentifier = regexp.MustCompile(`^[a-z][a-z0-9_]{0,62}$`)
// TableDef describes a single extension-owned table as declared in a manifest. // TableDef describes a single extension-owned table as declared in a manifest.
type TableDef struct { type TableDef struct {
Columns map[string]string // colName → manifest type ("text","int","real","bool","timestamp") Columns map[string]string // colName → manifest type ("text","int","real","bool","timestamp","vector(N)")
Indexes [][]string // each inner slice is an ordered list of column names for one index Indexes [][]string // each inner slice is an ordered list of column names for one index
} }
// parseVectorDim extracts the dimension N from a "vector(N)" type string.
// Returns (N, true) for valid dimensions 1..4096, or (0, false) otherwise.
func parseVectorDim(typStr string) (int, bool) {
m := vectorDimRegex.FindStringSubmatch(strings.ToLower(typStr))
if m == nil {
return 0, false
}
n, err := strconv.Atoi(m[1])
if err != nil || n < 1 || n > 4096 {
return 0, false
}
return n, true
}
// ParseDBTables extracts the "db_tables" key from a package manifest map. // ParseDBTables extracts the "db_tables" key from a package manifest map.
// Returns the map of logicalName→TableDef and ok=true when the key is present // Returns the map of logicalName→TableDef and ok=true when the key is present
// and produces at least one valid table definition. // and produces at least one valid table definition.
@@ -93,8 +111,22 @@ func ParseDBTables(manifest map[string]any) (map[string]TableDef, bool) {
} }
// mapColType maps a manifest type string to a SQL column type for the given dialect. // mapColType maps a manifest type string to a SQL column type for the given dialect.
func mapColType(typStr string, isPostgres bool) string { // hasPgvector indicates whether the pgvector extension is available on Postgres.
switch strings.ToLower(typStr) { func mapColType(typStr string, isPostgres bool, hasPgvector bool) string {
lower := strings.ToLower(typStr)
// Check for vector type first.
if _, ok := parseVectorDim(lower); ok {
if isPostgres && hasPgvector {
return lower // native pgvector type, e.g. "vector(384)"
}
if isPostgres {
return "JSONB" // structured fallback without pgvector
}
return "TEXT" // SQLite fallback — JSON string
}
switch lower {
case "int", "integer": case "int", "integer":
return "INTEGER" return "INTEGER"
case "real", "float": case "real", "float":
@@ -133,6 +165,8 @@ func createdAtColDef(isPostgres bool) string {
// in the provided map. Each successfully created table is registered in the // in the provided map. Each successfully created table is registered in the
// ext_data_tables catalog via stores.ExtData. // ext_data_tables catalog via stores.ExtData.
// //
// hasPgvector enables native pgvector column types and HNSW indexes when true.
//
// Errors from individual table creation are returned immediately (fail-fast). // Errors from individual table creation are returned immediately (fail-fast).
// Index creation failures are logged but non-fatal. // Index creation failures are logged but non-fatal.
// Catalog registration failures are logged but non-fatal. // Catalog registration failures are logged but non-fatal.
@@ -143,6 +177,7 @@ func CreateExtTables(
stores store.Stores, stores store.Stores,
packageID string, packageID string,
tables map[string]TableDef, tables map[string]TableDef,
hasPgvector bool,
) error { ) error {
if db == nil { if db == nil {
return nil return nil
@@ -153,7 +188,7 @@ func CreateExtTables(
// Build column list: id first, user columns, created_at last. // Build column list: id first, user columns, created_at last.
cols := []string{"id TEXT PRIMARY KEY"} cols := []string{"id TEXT PRIMARY KEY"}
for col, typ := range td.Columns { for col, typ := range td.Columns {
cols = append(cols, col+" "+mapColType(typ, isPostgres)) cols = append(cols, col+" "+mapColType(typ, isPostgres, hasPgvector))
} }
cols = append(cols, createdAtColDef(isPostgres)) cols = append(cols, createdAtColDef(isPostgres))
@@ -174,6 +209,21 @@ func CreateExtTables(
} }
} }
// Create HNSW indexes for vector columns when pgvector is available.
if isPostgres && hasPgvector {
for col, typ := range td.Columns {
if _, ok := parseVectorDim(strings.ToLower(typ)); ok {
hnswName := fmt.Sprintf("idx_%s_%s_hnsw", physical, col)
hnswDDL := fmt.Sprintf(
"CREATE INDEX IF NOT EXISTS %s ON %s USING hnsw (%s vector_cosine_ops)",
hnswName, physical, col)
if _, err := db.ExecContext(ctx, hnswDDL); err != nil {
log.Printf("[ext-db] hnsw index create failed (%s): %v", hnswName, err)
}
}
}
}
// Record logical name in catalog. // Record logical name in catalog.
if stores.ExtData != nil { if stores.ExtData != nil {
if err := stores.ExtData.RegisterTable(ctx, packageID, logicalName); err != nil { if err := stores.ExtData.RegisterTable(ctx, packageID, logicalName); err != nil {
@@ -245,6 +295,7 @@ func MigrateExtTables(
stores store.Stores, stores store.Stores,
packageID string, packageID string,
newTables map[string]TableDef, newTables map[string]TableDef,
hasPgvector bool,
) ([]string, error) { ) ([]string, error) {
if db == nil { if db == nil {
return nil, nil return nil, nil
@@ -270,7 +321,7 @@ func MigrateExtTables(
if !existingTableNames[logicalName] { if !existingTableNames[logicalName] {
// New table — full creation // New table — full creation
if err := CreateExtTables(ctx, db, isPostgres, stores, packageID, if err := CreateExtTables(ctx, db, isPostgres, stores, packageID,
map[string]TableDef{logicalName: td}); err != nil { map[string]TableDef{logicalName: td}, hasPgvector); err != nil {
return changes, fmt.Errorf("create new table %s: %w", physical, err) return changes, fmt.Errorf("create new table %s: %w", physical, err)
} }
changes = append(changes, fmt.Sprintf("created table %s", physical)) changes = append(changes, fmt.Sprintf("created table %s", physical))
@@ -287,7 +338,7 @@ func MigrateExtTables(
if existingCols[col] { if existingCols[col] {
continue continue
} }
sqlType := mapColType(typ, isPostgres) sqlType := mapColType(typ, isPostgres, hasPgvector)
alterDDL := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", physical, col, sqlType) alterDDL := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", physical, col, sqlType)
if _, err := db.ExecContext(ctx, alterDDL); err != nil { if _, err := db.ExecContext(ctx, alterDDL); err != nil {
return changes, fmt.Errorf("add column %s.%s: %w", physical, col, err) return changes, fmt.Errorf("add column %s.%s: %w", physical, col, err)

View File

@@ -145,7 +145,7 @@ func TestMapColType_SQLite(t *testing.T) {
{"unknown", "TEXT"}, {"unknown", "TEXT"},
} }
for _, c := range cases { for _, c := range cases {
got := mapColType(c.in, false) got := mapColType(c.in, false, false)
if got != c.want { if got != c.want {
t.Errorf("mapColType(%q, sqlite) = %q, want %q", c.in, got, c.want) t.Errorf("mapColType(%q, sqlite) = %q, want %q", c.in, got, c.want)
} }
@@ -153,13 +153,13 @@ func TestMapColType_SQLite(t *testing.T) {
} }
func TestMapColType_Postgres(t *testing.T) { func TestMapColType_Postgres(t *testing.T) {
if mapColType("bool", true) != "BOOLEAN" { if mapColType("bool", true, false) != "BOOLEAN" {
t.Error("expected BOOLEAN for bool in postgres") t.Error("expected BOOLEAN for bool in postgres")
} }
if mapColType("timestamp", true) != "TIMESTAMPTZ" { if mapColType("timestamp", true, false) != "TIMESTAMPTZ" {
t.Error("expected TIMESTAMPTZ for timestamp in postgres") t.Error("expected TIMESTAMPTZ for timestamp in postgres")
} }
if mapColType("int", true) != "INTEGER" { if mapColType("int", true, false) != "INTEGER" {
t.Error("expected INTEGER for int in postgres") t.Error("expected INTEGER for int in postgres")
} }
} }
@@ -189,7 +189,7 @@ func TestCreateExtTables_CreatesTable(t *testing.T) {
Columns: map[string]string{"message": "text", "count": "int"}, Columns: map[string]string{"message": "text", "count": "int"},
}, },
} }
if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "test-pkg", tables); err != nil { if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "test-pkg", tables, false); err != nil {
t.Fatalf("CreateExtTables: %v", err) t.Fatalf("CreateExtTables: %v", err)
} }
@@ -212,7 +212,7 @@ func TestCreateExtTables_WithIndexes(t *testing.T) {
Indexes: [][]string{{"user_id"}, {"user_id", "kind"}}, Indexes: [][]string{{"user_id"}, {"user_id", "kind"}},
}, },
} }
if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "idx-pkg", tables); err != nil { if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "idx-pkg", tables, false); err != nil {
t.Fatalf("CreateExtTables: %v", err) t.Fatalf("CreateExtTables: %v", err)
} }
@@ -243,7 +243,7 @@ func TestCreateExtTables_CatalogRegistration(t *testing.T) {
"logs": {Columns: map[string]string{"msg": "text"}}, "logs": {Columns: map[string]string{"msg": "text"}},
"errors": {Columns: map[string]string{"detail": "text"}}, "errors": {Columns: map[string]string{"detail": "text"}},
} }
if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "cat-pkg", tables); err != nil { if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "cat-pkg", tables, false); err != nil {
t.Fatalf("CreateExtTables: %v", err) t.Fatalf("CreateExtTables: %v", err)
} }
@@ -259,7 +259,7 @@ func TestCreateExtTables_NilDBIsNoop(t *testing.T) {
// Should not panic or error. // Should not panic or error.
err := CreateExtTables(ctx, nil, false, storesWithExtData(m), "pkg", map[string]TableDef{ err := CreateExtTables(ctx, nil, false, storesWithExtData(m), "pkg", map[string]TableDef{
"t": {Columns: map[string]string{"x": "text"}}, "t": {Columns: map[string]string{"x": "text"}},
}) }, false)
if err != nil { if err != nil {
t.Errorf("expected nil error for nil db, got: %v", err) t.Errorf("expected nil error for nil db, got: %v", err)
} }
@@ -276,7 +276,7 @@ func TestDropExtTables_DropsAndClearsCatalog(t *testing.T) {
tables := map[string]TableDef{ tables := map[string]TableDef{
"items": {Columns: map[string]string{"name": "text"}}, "items": {Columns: map[string]string{"name": "text"}},
} }
if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "drop-pkg", tables); err != nil { if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "drop-pkg", tables, false); err != nil {
t.Fatalf("CreateExtTables: %v", err) t.Fatalf("CreateExtTables: %v", err)
} }
@@ -309,7 +309,7 @@ func TestDropExtTables_MultipleTables(t *testing.T) {
"alpha": {Columns: map[string]string{"v": "text"}}, "alpha": {Columns: map[string]string{"v": "text"}},
"beta": {Columns: map[string]string{"v": "text"}}, "beta": {Columns: map[string]string{"v": "text"}},
} }
CreateExtTables(ctx, db, false, storesWithExtData(m), "multi-pkg", tables) CreateExtTables(ctx, db, false, storesWithExtData(m), "multi-pkg", tables, false)
DropExtTables(ctx, db, storesWithExtData(m), "multi-pkg") DropExtTables(ctx, db, storesWithExtData(m), "multi-pkg")
for _, name := range []string{"ext_multi_pkg_alpha", "ext_multi_pkg_beta"} { for _, name := range []string{"ext_multi_pkg_alpha", "ext_multi_pkg_beta"} {
@@ -331,7 +331,7 @@ func TestCreateExtTables_DDLContainsAutoColumns(t *testing.T) {
ctx := context.Background() ctx := context.Background()
tables := map[string]TableDef{"empty": {Columns: map[string]string{}}} tables := map[string]TableDef{"empty": {Columns: map[string]string{}}}
if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "auto-pkg", tables); err != nil { if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "auto-pkg", tables, false); err != nil {
t.Fatalf("CreateExtTables: %v", err) t.Fatalf("CreateExtTables: %v", err)
} }
@@ -348,3 +348,77 @@ func TestCreateExtTables_DDLContainsAutoColumns(t *testing.T) {
t.Error("expected created_at to be populated by default") t.Error("expected created_at to be populated by default")
} }
} }
// ── parseVectorDim ──────────────────────────────────────────────────────────
func TestParseVectorDim(t *testing.T) {
cases := []struct {
input string
dim int
ok bool
}{
{"vector(384)", 384, true},
{"vector(1)", 1, true},
{"vector(4096)", 4096, true},
{"Vector(128)", 128, true}, // case insensitive
{"vector(0)", 0, false},
{"vector(5000)", 0, false}, // exceeds 4096
{"vector()", 0, false},
{"vector(abc)", 0, false},
{"text", 0, false},
{"vector", 0, false},
}
for _, c := range cases {
dim, ok := parseVectorDim(c.input)
if ok != c.ok || dim != c.dim {
t.Errorf("parseVectorDim(%q) = (%d, %v), want (%d, %v)", c.input, dim, ok, c.dim, c.ok)
}
}
}
// ── mapColType vector ───────────────────────────────────────────────────────
func TestMapColType_VectorSQLite(t *testing.T) {
got := mapColType("vector(384)", false, false)
if got != "TEXT" {
t.Errorf("vector on SQLite: got %q, want TEXT", got)
}
}
func TestMapColType_VectorPgNoPgvector(t *testing.T) {
got := mapColType("vector(384)", true, false)
if got != "JSONB" {
t.Errorf("vector on PG without pgvector: got %q, want JSONB", got)
}
}
func TestMapColType_VectorPgvector(t *testing.T) {
got := mapColType("vector(384)", true, true)
if got != "vector(384)" {
t.Errorf("vector on PG with pgvector: got %q, want vector(384)", got)
}
}
// ── CreateExtTables with vector column ──────────────────────────────────────
func TestCreateExtTables_VectorColumn(t *testing.T) {
db := newSchemaTestDB(t)
m := newMemExtDataStore()
ctx := context.Background()
tables := map[string]TableDef{
"docs": {
Columns: map[string]string{"title": "text", "embedding": "vector(384)"},
},
}
if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "vec-pkg", tables, false); err != nil {
t.Fatalf("CreateExtTables: %v", err)
}
// Verify table exists and embedding is TEXT (SQLite fallback).
_, err := db.ExecContext(ctx,
`INSERT INTO ext_vec_pkg_docs (id, title, embedding) VALUES ('1', 'test', '[0.1,0.2]')`)
if err != nil {
t.Errorf("table not created or wrong schema: %v", err)
}
}

View File

@@ -17,13 +17,19 @@ import (
// ExtensionHandler serves extension management endpoints. // ExtensionHandler serves extension management endpoints.
type ExtensionHandler struct { type ExtensionHandler struct {
stores store.Stores stores store.Stores
capabilities map[string]bool
} }
func NewExtensionHandler(stores store.Stores) *ExtensionHandler { func NewExtensionHandler(stores store.Stores) *ExtensionHandler {
return &ExtensionHandler{stores: stores} return &ExtensionHandler{stores: stores}
} }
// SetCapabilities stores the detected environment capabilities map.
func (h *ExtensionHandler) SetCapabilities(caps map[string]bool) {
h.capabilities = caps
}
// validTiers is the set of accepted extension tier values. // validTiers is the set of accepted extension tier values.
var validTiers = map[string]bool{ var validTiers = map[string]bool{
models.ExtTierBrowser: true, models.ExtTierBrowser: true,
@@ -31,6 +37,44 @@ var validTiers = map[string]bool{
models.ExtTierSidecar: true, models.ExtTierSidecar: true,
} }
// ValidateComposabilityFields checks that manifest slots and contributes
// fields follow the expected conventions. Slots values must have a
// "description" string. Contributes keys must follow "pkg:slot" naming.
// Returns nil if valid or missing; returns an error string otherwise.
func ValidateComposabilityFields(manifest map[string]any) string {
// Validate slots: map of name → {description, ...}
if slotsRaw, ok := manifest["slots"]; ok {
slots, ok := slotsRaw.(map[string]any)
if !ok {
return "slots must be an object"
}
for name, v := range slots {
entry, ok := v.(map[string]any)
if !ok {
return "slots." + name + " must be an object"
}
if _, ok := entry["description"]; !ok {
return "slots." + name + " must have a description"
}
}
}
// Validate contributes: map of "pkg:slot" → {label, ...}
if contribRaw, ok := manifest["contributes"]; ok {
contribs, ok := contribRaw.(map[string]any)
if !ok {
return "contributes must be an object"
}
for key := range contribs {
if !strings.Contains(key, ":") {
return "contributes key " + key + " must follow pkg:slot convention"
}
}
}
return ""
}
// ── User endpoints ────────────────────────────── // ── User endpoints ──────────────────────────────
// ListUserExtensions returns all enabled extensions for the current user, // ListUserExtensions returns all enabled extensions for the current user,
@@ -202,6 +246,12 @@ func (h *ExtensionHandler) AdminInstallExtension(c *gin.Context) {
manifestMap = map[string]any{} manifestMap = map[string]any{}
} }
// Validate composability fields (slots / contributes)
if errMsg := ValidateComposabilityFields(manifestMap); errMsg != "" {
c.JSON(400, gin.H{"error": "manifest: " + errMsg})
return
}
pkg := &store.PackageRegistration{ pkg := &store.PackageRegistration{
ID: body.ExtID, ID: body.ExtID,
Title: body.Name, Title: body.Name,
@@ -231,7 +281,7 @@ func (h *ExtensionHandler) AdminInstallExtension(c *gin.Context) {
SyncManifestPermissions(c, h.stores, pkg.ID, manifestMap) SyncManifestPermissions(c, h.stores, pkg.ID, manifestMap)
if tables, ok := ParseDBTables(manifestMap); ok { if tables, ok := ParseDBTables(manifestMap); ok {
if err := CreateExtTables(c.Request.Context(), database.DB, database.IsPostgres(), h.stores, pkg.ID, tables); err != nil { if err := CreateExtTables(c.Request.Context(), database.DB, database.IsPostgres(), h.stores, pkg.ID, tables, h.capabilities["pgvector"]); err != nil {
log.Printf("[ext-db] schema create failed for %s: %v", pkg.ID, err) log.Printf("[ext-db] schema create failed for %s: %v", pkg.ID, err)
} }
} }

View File

@@ -4,6 +4,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"regexp" "regexp"
"strings"
) )
// validManifestID matches lowercase alphanumeric slugs with optional hyphens. // validManifestID matches lowercase alphanumeric slugs with optional hyphens.
@@ -21,6 +22,7 @@ type ManifestInfo struct {
Tier string Tier string
SchemaVersion int SchemaVersion int
HasRoute bool HasRoute bool
HasSurfaces bool
HasTools bool HasTools bool
HasPipes bool HasPipes bool
HasHooks bool HasHooks bool
@@ -81,7 +83,64 @@ func ValidateManifest(manifest map[string]any) (*ManifestInfo, error) {
} }
info.Signature, _ = manifest["signature"].(string) info.Signature, _ = manifest["signature"].(string)
info.HasRoute = manifest["route"] != nil || manifest["routes"] != nil // ── Surfaces validation ─────────────────────────────────────
if rawSurfaces, ok := manifest["surfaces"].([]any); ok {
if len(rawSurfaces) == 0 {
return nil, fmt.Errorf("surfaces array cannot be empty")
}
seen := map[string]bool{}
for i, raw := range rawSurfaces {
s, ok := raw.(map[string]any)
if !ok {
return nil, fmt.Errorf("surfaces[%d] must be an object", i)
}
path, _ := s["path"].(string)
if path == "" {
return nil, fmt.Errorf("surfaces[%d] requires a path", i)
}
if !strings.HasPrefix(path, "/") {
return nil, fmt.Errorf("surfaces[%d] path must start with /", i)
}
if seen[path] {
return nil, fmt.Errorf("duplicate surface path: %s", path)
}
seen[path] = true
if access, ok := s["access"].(string); ok {
if !validAccessLevels(access) {
return nil, fmt.Errorf("surfaces[%d] invalid access: %s", i, access)
}
}
}
info.HasSurfaces = true
info.HasRoute = true
} else if manifest["surfaces"] != nil {
// surfaces key present but not an array
return nil, fmt.Errorf("surfaces must be an array")
} else {
// No surfaces declared — synthesize from legacy fields.
// This ensures every routable package always has a surfaces array.
if info.Type == "surface" || info.Type == "full" {
auth, _ := manifest["auth"].(string)
if auth == "" {
auth = "authenticated"
}
layout, _ := manifest["layout"].(string)
if layout == "" {
layout = "single"
}
manifest["surfaces"] = []any{
map[string]any{
"path": "/",
"access": auth,
"layout": layout,
"title": info.Title,
},
}
info.HasSurfaces = true
}
info.HasRoute = manifest["route"] != nil || manifest["routes"] != nil || info.HasSurfaces
}
info.HasTools = manifest["tools"] != nil info.HasTools = manifest["tools"] != nil
info.HasPipes = manifest["pipes"] != nil info.HasPipes = manifest["pipes"] != nil
info.HasHooks = manifest["hooks"] != nil info.HasHooks = manifest["hooks"] != nil
@@ -156,6 +215,18 @@ func ValidateManifest(manifest map[string]any) (*ManifestInfo, error) {
return info, nil return info, nil
} }
// validAccessLevels checks whether an access string is a recognized value.
func validAccessLevels(access string) bool {
switch {
case access == "public", access == "authenticated", access == "admin":
return true
case strings.HasPrefix(access, "group:"):
return strings.TrimPrefix(access, "group:") != ""
default:
return false
}
}
// ValidateManifestJSON parses raw JSON bytes into a manifest map and validates. // ValidateManifestJSON parses raw JSON bytes into a manifest map and validates.
func ValidateManifestJSON(data []byte) (map[string]any, *ManifestInfo, error) { func ValidateManifestJSON(data []byte) (map[string]any, *ManifestInfo, error) {
var manifest map[string]any var manifest map[string]any

View File

@@ -198,3 +198,154 @@ func TestValidateManifest_WorkflowNoDef(t *testing.T) {
t.Fatal("expected error for workflow without workflow_definition") t.Fatal("expected error for workflow without workflow_definition")
} }
} }
// ── Surfaces validation ─────────────────────────────────────
func TestValidateManifest_ValidSurfaces(t *testing.T) {
m := map[string]any{
"id": "bug-tracker",
"title": "Bug Tracker",
"type": "full",
"hooks": []any{"surface"},
"surfaces": []any{
map[string]any{"path": "/", "title": "Dashboard"},
map[string]any{"path": "/submit", "access": "public", "title": "Report a Bug"},
map[string]any{"path": "/:id", "title": "Bug Detail", "nav": false},
map[string]any{"path": "/admin", "access": "admin", "title": "Settings"},
},
}
info, err := ValidateManifest(m)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !info.HasSurfaces {
t.Error("expected HasSurfaces to be true")
}
if !info.HasRoute {
t.Error("expected HasRoute to be true")
}
}
func TestValidateManifest_EmptySurfaces(t *testing.T) {
m := map[string]any{
"id": "my-pkg",
"title": "My Package",
"type": "surface",
"surfaces": []any{},
}
_, err := ValidateManifest(m)
if err == nil {
t.Fatal("expected error for empty surfaces array")
}
}
func TestValidateManifest_SurfaceMissingPath(t *testing.T) {
m := map[string]any{
"id": "my-pkg",
"title": "My Package",
"type": "surface",
"surfaces": []any{
map[string]any{"title": "No Path"},
},
}
_, err := ValidateManifest(m)
if err == nil {
t.Fatal("expected error for surface without path")
}
}
func TestValidateManifest_SurfacePathNoSlash(t *testing.T) {
m := map[string]any{
"id": "my-pkg",
"title": "My Package",
"type": "surface",
"surfaces": []any{
map[string]any{"path": "submit"},
},
}
_, err := ValidateManifest(m)
if err == nil {
t.Fatal("expected error for path not starting with /")
}
}
func TestValidateManifest_SurfaceDuplicatePath(t *testing.T) {
m := map[string]any{
"id": "my-pkg",
"title": "My Package",
"type": "surface",
"surfaces": []any{
map[string]any{"path": "/"},
map[string]any{"path": "/"},
},
}
_, err := ValidateManifest(m)
if err == nil {
t.Fatal("expected error for duplicate surface path")
}
}
func TestValidateManifest_SurfaceInvalidAccess(t *testing.T) {
m := map[string]any{
"id": "my-pkg",
"title": "My Package",
"type": "surface",
"surfaces": []any{
map[string]any{"path": "/", "access": "superuser"},
},
}
_, err := ValidateManifest(m)
if err == nil {
t.Fatal("expected error for invalid access level")
}
}
func TestValidateManifest_SurfaceGroupAccess(t *testing.T) {
m := map[string]any{
"id": "my-pkg",
"title": "My Package",
"type": "surface",
"surfaces": []any{
map[string]any{"path": "/", "access": "group:engineering"},
},
}
info, err := ValidateManifest(m)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !info.HasSurfaces {
t.Error("expected HasSurfaces to be true")
}
}
func TestValidateManifest_AutoSynthesizeSurfaces(t *testing.T) {
m := map[string]any{
"id": "legacy-pkg",
"title": "Legacy Package",
"type": "surface",
"auth": "public",
"layout": "editor",
}
info, err := ValidateManifest(m)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !info.HasSurfaces {
t.Error("expected HasSurfaces after auto-synthesis")
}
// Verify the synthesized surfaces array was injected into the manifest
surfaces, ok := m["surfaces"].([]any)
if !ok || len(surfaces) != 1 {
t.Fatalf("expected 1 synthesized surface, got %v", m["surfaces"])
}
s := surfaces[0].(map[string]any)
if s["path"] != "/" {
t.Errorf("expected path '/', got %q", s["path"])
}
if s["access"] != "public" {
t.Errorf("expected access 'public', got %q", s["access"])
}
if s["layout"] != "editor" {
t.Errorf("expected layout 'editor', got %q", s["layout"])
}
}

View File

@@ -38,6 +38,7 @@ type PackageHandler struct {
sandbox *sandbox.Sandbox sandbox *sandbox.Sandbox
runner *sandbox.Runner runner *sandbox.Runner
hub *events.Hub hub *events.Hub
capabilities map[string]bool // detected environment capabilities
} }
func NewPackageHandler(s store.Stores, packagesDir ...string) *PackageHandler { func NewPackageHandler(s store.Stores, packagesDir ...string) *PackageHandler {
@@ -69,6 +70,12 @@ func (h *PackageHandler) SetHub(hub *events.Hub) {
h.hub = hub h.hub = hub
} }
// SetCapabilities stores the detected environment capabilities map.
// Used at install time to validate manifest capabilities.required.
func (h *PackageHandler) SetCapabilities(caps map[string]bool) {
h.capabilities = caps
}
// broadcastPackageChanged emits a package.changed event to all clients. // broadcastPackageChanged emits a package.changed event to all clients.
func (h *PackageHandler) broadcastPackageChanged(action, id string) { func (h *PackageHandler) broadcastPackageChanged(action, id string) {
if h.hub == nil { if h.hub == nil {
@@ -205,10 +212,48 @@ func (h *PackageHandler) DeletePackage(c *gin.Context) {
} }
} }
c.JSON(http.StatusOK, gin.H{"id": id, "deleted": true}) resp := gin.H{"id": id, "deleted": true}
// Warn about orphaned slot contributions
if orphaned := findOrphanedContributors(c, h.stores, id); len(orphaned) > 0 {
resp["warning"] = "slot contributions orphaned in: " + strings.Join(orphaned, ", ")
}
c.JSON(http.StatusOK, resp)
h.broadcastPackageChanged("deleted", id) h.broadcastPackageChanged("deleted", id)
} }
// findOrphanedContributors returns package IDs that declare contributes
// targeting slots owned by the given package (e.g. "notes:toolbar-actions"
// targets the "notes" package). This is a warning, not a blocker.
func findOrphanedContributors(c *gin.Context, stores store.Stores, deletedPkgID string) []string {
allPkgs, err := stores.Packages.List(c.Request.Context())
if err != nil {
return nil
}
prefix := deletedPkgID + ":"
seen := map[string]bool{}
for _, p := range allPkgs {
if p.ID == deletedPkgID || !p.Enabled {
continue
}
contribs, ok := p.Manifest["contributes"].(map[string]any)
if !ok {
continue
}
for key := range contribs {
if strings.HasPrefix(key, prefix) {
seen[p.ID] = true
}
}
}
result := make([]string, 0, len(seen))
for id := range seen {
result = append(result, id)
}
return result
}
// InstallPackage uploads and installs a .pkg/.surface archive. // InstallPackage uploads and installs a .pkg/.surface archive.
// POST /api/v1/admin/packages/install // POST /api/v1/admin/packages/install
// //
@@ -286,8 +331,23 @@ func (h *PackageHandler) InstallPackage(c *gin.Context) {
} }
} }
// Phase 10: Check capabilities → mark dormant if unmet // Phase 10: Validate required capabilities
isDormant := h.checkCapabilities(c, pkgID, manifest, &preservedEnabled) if caps, ok := ParseCapabilities(manifest); ok {
missing := CheckRequiredCapabilities(caps.Required, h.capabilities)
if len(missing) > 0 {
h.stores.Packages.Delete(c.Request.Context(), pkgID)
c.JSON(http.StatusUnprocessableEntity, gin.H{
"error": "missing required capabilities",
"missing": missing,
"help": capabilityHelpText(missing),
})
return
}
unmetOpt := CheckRequiredCapabilities(caps.Optional, h.capabilities)
if len(unmetOpt) > 0 {
log.Printf("[packages] %s: optional capabilities unavailable: %v", pkgID, unmetOpt)
}
}
resp := gin.H{ resp := gin.H{
"id": pkgID, "id": pkgID,
@@ -297,9 +357,6 @@ func (h *PackageHandler) InstallPackage(c *gin.Context) {
"source": pkgSource, "source": pkgSource,
"enabled": preservedEnabled, "enabled": preservedEnabled,
} }
if isDormant {
resp["status"] = "dormant"
}
c.JSON(http.StatusOK, resp) c.JSON(http.StatusOK, resp)
h.broadcastPackageChanged("installed", pkgID) h.broadcastPackageChanged("installed", pkgID)
} }
@@ -447,6 +504,13 @@ func (h *PackageHandler) parseAndValidateArchive(c *gin.Context, tmpPath string)
return nil, nil, nil, err return nil, nil, nil, err
} }
// Validate composability fields (slots / contributes)
if errMsg := ValidateComposabilityFields(manifest); errMsg != "" {
zr.Close()
c.JSON(http.StatusBadRequest, gin.H{"error": "manifest: " + errMsg})
return nil, nil, nil, fmt.Errorf("invalid composability fields")
}
// Signing hook (reserved) // Signing hook (reserved)
if os.Getenv("PACKAGE_VERIFY_SIGNATURES") == "true" { if os.Getenv("PACKAGE_VERIFY_SIGNATURES") == "true" {
if mInfo.Signature != "" { if mInfo.Signature != "" {
@@ -534,7 +598,7 @@ func (h *PackageHandler) registerPackage(c *gin.Context, pkgID string, mInfo *Ma
// and runs schema migrations. // and runs schema migrations.
func (h *PackageHandler) applySchemaAndPermissions(c *gin.Context, pkgID string, mInfo *ManifestInfo, manifest map[string]any, existing *store.PackageRegistration) error { func (h *PackageHandler) applySchemaAndPermissions(c *gin.Context, pkgID string, mInfo *ManifestInfo, manifest map[string]any, existing *store.PackageRegistration) error {
if tables, ok := ParseDBTables(manifest); ok { if tables, ok := ParseDBTables(manifest); ok {
if err := CreateExtTables(c.Request.Context(), database.DB, database.IsPostgres(), h.stores, pkgID, tables); err != nil { if err := CreateExtTables(c.Request.Context(), database.DB, database.IsPostgres(), h.stores, pkgID, tables, h.capabilities["pgvector"]); err != nil {
log.Printf("[ext-db] schema create failed for %s: %v", pkgID, err) log.Printf("[ext-db] schema create failed for %s: %v", pkgID, err)
} }
} }
@@ -591,7 +655,7 @@ func (h *PackageHandler) resolveDependencies(c *gin.Context, pkgID string, manif
pkgPath := filepath.Join(h.bundledDir, libID+".pkg") pkgPath := filepath.Join(h.bundledDir, libID+".pkg")
if _, statErr := os.Stat(pkgPath); statErr == nil { if _, statErr := os.Stat(pkgPath); statErr == nil {
log.Printf("[packages] auto-installing bundled dependency %s for %s", libID, pkgID) log.Printf("[packages] auto-installing bundled dependency %s for %s", libID, pkgID)
if _, installErr := installBundledPackage(pkgPath, h.packagesDir, h.stores, h.runner, c.GetString("user_id")); installErr != nil { if _, installErr := installBundledPackage(pkgPath, h.packagesDir, h.stores, h.runner, c.GetString("user_id"), h.capabilities); installErr != nil {
c.JSON(http.StatusBadRequest, gin.H{ c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("dependency %s auto-install failed: %v", libID, installErr), "error": fmt.Sprintf("dependency %s auto-install failed: %v", libID, installErr),
}) })
@@ -634,27 +698,6 @@ func (h *PackageHandler) resolveDependencies(c *gin.Context, pkgID string, manif
return nil return nil
} }
// checkCapabilities marks a package dormant if it declares unmet `requires` entries.
func (h *PackageHandler) checkCapabilities(c *gin.Context, pkgID string, manifest map[string]any, preservedEnabled *bool) bool {
knownCaps := map[string]bool{}
var unmetReqs []string
if reqs, ok := manifest["requires"].([]any); ok && len(reqs) > 0 {
for _, r := range reqs {
req, _ := r.(string)
if req != "" && !knownCaps[req] {
unmetReqs = append(unmetReqs, req)
}
}
}
if len(unmetReqs) == 0 {
return false
}
h.stores.Packages.SetStatus(c.Request.Context(), pkgID, "dormant")
h.stores.Packages.SetEnabled(c.Request.Context(), pkgID, false)
*preservedEnabled = false
log.Printf("[packages] %s: dormant (unmet requires: %v)", pkgID, unmetReqs)
return true
}
// extractableRelPath returns the relative path for a zip entry if it // extractableRelPath returns the relative path for a zip entry if it
// belongs to an extractable directory (js/, css/, assets/, star/) or // belongs to an extractable directory (js/, css/, assets/, star/) or
@@ -1049,7 +1092,7 @@ func (h *PackageHandler) UpdatePackage(c *gin.Context) {
// 6. Additive schema migration // 6. Additive schema migration
var schemaChanges []string var schemaChanges []string
if tables, ok := ParseDBTables(manifest); ok { if tables, ok := ParseDBTables(manifest); ok {
changes, err := MigrateExtTables(ctx, database.DB, database.IsPostgres(), h.stores, pkgID, tables) changes, err := MigrateExtTables(ctx, database.DB, database.IsPostgres(), h.stores, pkgID, tables, h.capabilities["pgvector"])
if err != nil { if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "schema migration failed: " + err.Error()}) c.JSON(http.StatusInternalServerError, gin.H{"error": "schema migration failed: " + err.Error()})
return return

View File

@@ -40,7 +40,7 @@ var defaultBundledPackages = map[string]bool{}
// //
// Design: install-once, skip-if-present. If an admin uninstalls a bundled // Design: install-once, skip-if-present. If an admin uninstalls a bundled
// package, it won't be re-installed on the next restart. // package, it won't be re-installed on the next restart.
func InstallBundledPackages(bundledDir, packagesDir, allowlist string, stores store.Stores, runner *sandbox.Runner) { func InstallBundledPackages(bundledDir, packagesDir, allowlist string, stores store.Stores, runner *sandbox.Runner, detectedCaps map[string]bool) {
entries, err := os.ReadDir(bundledDir) entries, err := os.ReadDir(bundledDir)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
@@ -82,7 +82,7 @@ func InstallBundledPackages(bundledDir, packagesDir, allowlist string, stores st
} }
pkgPath := filepath.Join(bundledDir, entry.Name()) pkgPath := filepath.Join(bundledDir, entry.Name())
result, err := installBundledPackage(pkgPath, packagesDir, stores, runner, adminUserID) result, err := installBundledPackage(pkgPath, packagesDir, stores, runner, adminUserID, detectedCaps)
if err != nil { if err != nil {
log.Printf("[bundled] ⚠️ Failed to install %s: %v", entry.Name(), err) log.Printf("[bundled] ⚠️ Failed to install %s: %v", entry.Name(), err)
continue continue
@@ -129,7 +129,7 @@ func parseBundleAllowlist(s string) map[string]bool {
// installBundledPackage installs a single .pkg archive if its package ID // installBundledPackage installs a single .pkg archive if its package ID
// doesn't already exist in the database. Returns "installed" or "skipped". // doesn't already exist in the database. Returns "installed" or "skipped".
func installBundledPackage(pkgPath, packagesDir string, stores store.Stores, runner *sandbox.Runner, adminUserID string) (string, error) { func installBundledPackage(pkgPath, packagesDir string, stores store.Stores, runner *sandbox.Runner, adminUserID string, detectedCaps map[string]bool) (string, error) {
ctx := context.Background() ctx := context.Background()
// Open as zip // Open as zip
@@ -196,7 +196,7 @@ func installBundledPackage(pkgPath, packagesDir string, stores store.Stores, run
// Create namespaced DB tables declared in the manifest // Create namespaced DB tables declared in the manifest
if tables, ok := ParseDBTables(manifest); ok { if tables, ok := ParseDBTables(manifest); ok {
if err := CreateExtTables(ctx, database.DB, database.IsPostgres(), stores, pkgID, tables); err != nil { if err := CreateExtTables(ctx, database.DB, database.IsPostgres(), stores, pkgID, tables, detectedCaps["pgvector"]); err != nil {
log.Printf("[bundled] [ext-db] schema create failed for %s: %v", pkgID, err) log.Printf("[bundled] [ext-db] schema create failed for %s: %v", pkgID, err)
} }
} }
@@ -237,20 +237,17 @@ func installBundledPackage(pkgPath, packagesDir string, stores store.Stores, run
} }
} }
// Handle dormant status for packages with unmet requires // Validate required capabilities — skip package if unmet
knownCaps := map[string]bool{} if caps, ok := ParseCapabilities(manifest); ok {
if reqs, ok := manifest["requires"].([]any); ok && len(reqs) > 0 { missing := CheckRequiredCapabilities(caps.Required, detectedCaps)
var unmetReqs []string if len(missing) > 0 {
for _, r := range reqs { stores.Packages.Delete(ctx, pkgID)
req, _ := r.(string) log.Printf("[bundled] %s: skipped (missing required capabilities: %v)", pkgID, missing)
if req != "" && !knownCaps[req] { return "skipped", nil
unmetReqs = append(unmetReqs, req)
}
} }
if len(unmetReqs) > 0 { unmetOpt := CheckRequiredCapabilities(caps.Optional, detectedCaps)
stores.Packages.SetStatus(ctx, pkgID, "dormant") if len(unmetOpt) > 0 {
stores.Packages.SetEnabled(ctx, pkgID, false) log.Printf("[bundled] %s: optional capabilities unavailable: %v", pkgID, unmetOpt)
log.Printf("[bundled] %s: dormant (unmet requires: %v)", pkgID, unmetReqs)
} }
} }

View File

@@ -88,7 +88,7 @@ func TestBundledInstall_FreshDB(t *testing.T) {
"route": "/s/test-b", "route": "/s/test-b",
}) })
InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil) InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil, nil)
// Both packages should be installed and enabled // Both packages should be installed and enabled
for _, id := range []string{"test-surface-a", "test-surface-b"} { for _, id := range []string{"test-surface-a", "test-surface-b"} {
@@ -127,7 +127,7 @@ func TestBundledInstall_SkipsExisting(t *testing.T) {
"type": "surface", "type": "surface",
}) })
InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil) InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil, nil)
// Package should retain original title (not overwritten) // Package should retain original title (not overwritten)
pkg, err := stores.Packages.Get(ctx, "pre-existing") pkg, err := stores.Packages.Get(ctx, "pre-existing")
@@ -145,7 +145,7 @@ func TestBundledInstall_MissingDir(t *testing.T) {
stores := newTestStores(t) stores := newTestStores(t)
// Should not panic or error — just log and return // Should not panic or error — just log and return
InstallBundledPackages("/nonexistent/path", "", "", stores, nil) InstallBundledPackages("/nonexistent/path", "", "", stores, nil, nil)
} }
// TestBundledInstall_EmptyDir verifies no-op when directory exists but is empty. // TestBundledInstall_EmptyDir verifies no-op when directory exists but is empty.
@@ -155,12 +155,12 @@ func TestBundledInstall_EmptyDir(t *testing.T) {
bundledDir := t.TempDir() bundledDir := t.TempDir()
// Should not panic or error // Should not panic or error
InstallBundledPackages(bundledDir, "", "", stores, nil) InstallBundledPackages(bundledDir, "", "", stores, nil, nil)
} }
// TestBundledInstall_DormantPackage verifies that bundled packages with // TestBundledInstall_SkippedOnUnmetRequiredCaps verifies that bundled
// unmet requires are marked dormant. // packages with unmet capabilities.required are skipped (not registered).
func TestBundledInstall_DormantPackage(t *testing.T) { func TestBundledInstall_SkippedOnUnmetRequiredCaps(t *testing.T) {
stores := newTestStores(t) stores := newTestStores(t)
ctx := context.Background() ctx := context.Background()
@@ -168,24 +168,21 @@ func TestBundledInstall_DormantPackage(t *testing.T) {
packagesDir := t.TempDir() packagesDir := t.TempDir()
buildTestPkg(t, bundledDir, map[string]any{ buildTestPkg(t, bundledDir, map[string]any{
"id": "dormant-pkg", "id": "needs-pgvector",
"title": "Dormant Package", "title": "Needs pgvector",
"type": "extension", "type": "extension",
"hooks": []string{"on_install"}, "hooks": []string{"on_install"},
"requires": []string{"chat"}, "capabilities": map[string]any{
"required": []string{"pgvector"},
},
}) })
InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil) // No capabilities detected → pgvector is unavailable
InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil, nil)
pkg, err := stores.Packages.Get(ctx, "dormant-pkg") pkg, _ := stores.Packages.Get(ctx, "needs-pgvector")
if err != nil || pkg == nil { if pkg != nil {
t.Fatal("dormant package not found after install") t.Error("package with unmet required capabilities should not be registered")
}
if pkg.Status != "dormant" {
t.Errorf("status = %q, want %q", pkg.Status, "dormant")
}
if pkg.Enabled {
t.Error("dormant package should not be enabled")
} }
} }
@@ -209,7 +206,7 @@ func TestBundledInstall_Allowlist(t *testing.T) {
}) })
// Only allow alpha and gamma // Only allow alpha and gamma
InstallBundledPackages(bundledDir, packagesDir, "pkg-alpha,pkg-gamma", stores, nil) InstallBundledPackages(bundledDir, packagesDir, "pkg-alpha,pkg-gamma", stores, nil, nil)
// Alpha and gamma should be installed // Alpha and gamma should be installed
for _, id := range []string{"pkg-alpha", "pkg-gamma"} { for _, id := range []string{"pkg-alpha", "pkg-gamma"} {

View File

@@ -267,7 +267,7 @@ func TestUpdatePackage_SchemaAddColumn(t *testing.T) {
}, },
}, },
}) })
CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "schema-pkg", tables) CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "schema-pkg", tables, false)
// Insert a row to verify data preservation // Insert a row to verify data preservation
physical := extPhysicalTable("schema-pkg", "items") physical := extPhysicalTable("schema-pkg", "items")
@@ -459,7 +459,7 @@ func TestBundledInstall_DefaultAllowlist(t *testing.T) {
}) })
// Empty allowlist → empty default set → nothing installed // Empty allowlist → empty default set → nothing installed
InstallBundledPackages(bundledDir, packagesDir, "", stores, nil) InstallBundledPackages(bundledDir, packagesDir, "", stores, nil, nil)
pkg, _ := stores.Packages.Get(ctx, "notes") pkg, _ := stores.Packages.Get(ctx, "notes")
if pkg != nil { if pkg != nil {
@@ -484,7 +484,7 @@ func TestBundledInstall_WildcardAllowlist(t *testing.T) {
"id": "any-pkg", "title": "Any", "type": "surface", "version": "1.0.0", "id": "any-pkg", "title": "Any", "type": "surface", "version": "1.0.0",
}) })
InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil) InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil, nil)
pkg, err := stores.Packages.Get(ctx, "any-pkg") pkg, err := stores.Packages.Get(ctx, "any-pkg")
if err != nil || pkg == nil { if err != nil || pkg == nil {

View File

@@ -42,7 +42,7 @@ func TestUpgrade_SchemaAddIndex(t *testing.T) {
}, },
}, },
}) })
CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "idx-pkg", tables) CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "idx-pkg", tables, false)
// Insert a row // Insert a row
physical := extPhysicalTable("idx-pkg", "events") physical := extPhysicalTable("idx-pkg", "events")
@@ -116,7 +116,7 @@ func TestUpgrade_SchemaAddColumnIdempotent(t *testing.T) {
}, },
}, },
}) })
CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "idem-pkg", tables) CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "idem-pkg", tables, false)
// Insert a row // Insert a row
physical := extPhysicalTable("idem-pkg", "items") physical := extPhysicalTable("idem-pkg", "items")
@@ -183,7 +183,7 @@ func TestUpgrade_SchemaMultiColumnAdd(t *testing.T) {
}, },
}, },
}) })
CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "multi-pkg", tables) CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "multi-pkg", tables, false)
// Insert a row // Insert a row
physical := extPhysicalTable("multi-pkg", "records") physical := extPhysicalTable("multi-pkg", "records")
@@ -411,7 +411,7 @@ func TestUpgrade_BundledSkipExistingOnRestart(t *testing.T) {
}) })
// First install // First install
InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil) InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil, nil)
pkg, _ := stores.Packages.Get(ctx, "restart-pkg") pkg, _ := stores.Packages.Get(ctx, "restart-pkg")
if pkg == nil { if pkg == nil {
@@ -427,7 +427,7 @@ func TestUpgrade_BundledSkipExistingOnRestart(t *testing.T) {
}) })
// Second install — should skip because package exists // Second install — should skip because package exists
InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil) InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil, nil)
pkg, _ = stores.Packages.Get(ctx, "restart-pkg") pkg, _ = stores.Packages.Get(ctx, "restart-pkg")
if pkg.Version != "1.0.0" { if pkg.Version != "1.0.0" {
@@ -438,7 +438,7 @@ func TestUpgrade_BundledSkipExistingOnRestart(t *testing.T) {
} }
} }
func TestUpgrade_PackageDormantOnUnmetRequires(t *testing.T) { func TestUpgrade_PackageSkippedOnUnmetRequiredCaps(t *testing.T) {
stores := newTestStores(t) stores := newTestStores(t)
ctx := context.Background() ctx := context.Background()
@@ -447,25 +447,22 @@ func TestUpgrade_PackageDormantOnUnmetRequires(t *testing.T) {
// Package requires a capability that doesn't exist // Package requires a capability that doesn't exist
buildTestPkg(t, bundledDir, map[string]any{ buildTestPkg(t, bundledDir, map[string]any{
"id": "future-pkg", "id": "future-pkg",
"title": "Future", "title": "Future",
"type": "extension", "type": "extension",
"version": "1.0.0", "version": "1.0.0",
"hooks": []string{"on_install"}, "hooks": []string{"on_install"},
"requires": []string{"kernel>=99.0.0"}, "capabilities": map[string]any{
"required": []string{"pgvector"},
},
}) })
InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil) // No capabilities detected → pgvector is unavailable
InstallBundledPackages(bundledDir, packagesDir, "*", stores, nil, nil)
pkg, _ := stores.Packages.Get(ctx, "future-pkg") pkg, _ := stores.Packages.Get(ctx, "future-pkg")
if pkg == nil { if pkg != nil {
t.Fatal("package should still be registered") t.Error("package with unmet required capabilities should not be registered")
}
if pkg.Status != "dormant" {
t.Errorf("status = %q, want %q", pkg.Status, "dormant")
}
if pkg.Enabled {
t.Error("dormant package should not be enabled")
} }
} }
@@ -524,7 +521,7 @@ func TestUpgrade_MigrateExtTables_NewTable(t *testing.T) {
}, },
} }
changes, err := MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-new", newTables) changes, err := MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-new", newTables, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -562,7 +559,7 @@ func TestUpgrade_MigrateExtTables_AddColumnPreservesRows(t *testing.T) {
}, },
}, },
}) })
CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-col", tables) CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-col", tables, false)
physical := extPhysicalTable("migrate-col", "records") physical := extPhysicalTable("migrate-col", "records")
_, err := database.TestDB.ExecContext(ctx, _, err := database.TestDB.ExecContext(ctx,
@@ -577,7 +574,7 @@ func TestUpgrade_MigrateExtTables_AddColumnPreservesRows(t *testing.T) {
Columns: map[string]string{"title": "text", "status": "text"}, Columns: map[string]string{"title": "text", "status": "text"},
}, },
} }
changes, err := MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-col", newTables) changes, err := MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-col", newTables, false)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -629,7 +626,7 @@ func TestUpgrade_MigrateExtTables_IndexIdempotent(t *testing.T) {
}, },
}, },
}) })
CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-idx", tables) CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-idx", tables, false)
// Migrate with same index — should not fail // Migrate with same index — should not fail
newTables := map[string]TableDef{ newTables := map[string]TableDef{
@@ -638,13 +635,13 @@ func TestUpgrade_MigrateExtTables_IndexIdempotent(t *testing.T) {
Indexes: [][]string{{"category"}}, Indexes: [][]string{{"category"}},
}, },
} }
_, err := MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-idx", newTables) _, err := MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-idx", newTables, false)
if err != nil { if err != nil {
t.Fatal("idempotent index migration should not fail:", err) t.Fatal("idempotent index migration should not fail:", err)
} }
// Run again — still no error // Run again — still no error
_, err = MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-idx", newTables) _, err = MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-idx", newTables, false)
if err != nil { if err != nil {
t.Fatal("second idempotent index migration should not fail:", err) t.Fatal("second idempotent index migration should not fail:", err)
} }

View File

@@ -169,10 +169,28 @@ func main() {
starlarkRunner.SetPackagesDir(cfg.StoragePath + "/packages") starlarkRunner.SetPackagesDir(cfg.StoragePath + "/packages")
} }
starlarkRunner.SetBus(bus) starlarkRunner.SetBus(bus)
if objStore != nil {
starlarkRunner.SetObjectStore(objStore)
}
if os.Getenv("EXT_ALLOW_PRIVATE_IPS") == "true" { if os.Getenv("EXT_ALLOW_PRIVATE_IPS") == "true" {
starlarkRunner.SetAllowPrivateIPs(true) starlarkRunner.SetAllowPrivateIPs(true)
log.Printf(" ⚠️ Extension SSRF protection relaxed: private IPs allowed") log.Printf(" ⚠️ Extension SSRF protection relaxed: private IPs allowed")
} }
if cfg.WorkspaceRoot != "" {
if err := os.MkdirAll(cfg.WorkspaceRoot, 0755); err == nil {
starlarkRunner.SetWorkspaceRoot(cfg.WorkspaceRoot, cfg.WorkspaceQuotaMB)
log.Printf(" workspace root: %s", cfg.WorkspaceRoot)
} else {
log.Printf(" workspace root %s not writable, workspace module disabled", cfg.WorkspaceRoot)
}
}
// ── Capability Detection ──────────
// Probe the runtime environment to discover available capabilities.
// Used by install validation and the settings.has_capability() Starlark API.
detectedCaps := handlers.DetectCapabilities(database.DB, database.IsPostgres(), objStore, cfg.WorkspaceRoot)
starlarkRunner.SetCapabilities(detectedCaps)
log.Printf(" capabilities: %v", detectedCaps)
// ── Bundled Packages ─────────────── // ── Bundled Packages ───────────────
// Auto-install bundled .pkg archives on first run. // Auto-install bundled .pkg archives on first run.
@@ -182,7 +200,7 @@ func main() {
if cfg.StoragePath != "" { if cfg.StoragePath != "" {
bundledPkgDir = cfg.StoragePath + "/packages" bundledPkgDir = cfg.StoragePath + "/packages"
} }
handlers.InstallBundledPackages(cfg.BundledPackagesDir, bundledPkgDir, cfg.BundledPackages, stores, starlarkRunner) handlers.InstallBundledPackages(cfg.BundledPackagesDir, bundledPkgDir, cfg.BundledPackages, stores, starlarkRunner, detectedCaps)
} }
// ── Register extension user permissions ───── // ── Register extension user permissions ─────
@@ -791,6 +809,7 @@ func main() {
// Extensions (admin) // Extensions (admin)
extAdm := handlers.NewExtensionHandler(stores) extAdm := handlers.NewExtensionHandler(stores)
extAdm.SetCapabilities(detectedCaps)
admin.GET("/extensions", extAdm.AdminListExtensions) admin.GET("/extensions", extAdm.AdminListExtensions)
admin.POST("/extensions", extAdm.AdminInstallExtension) admin.POST("/extensions", extAdm.AdminInstallExtension)
admin.PUT("/extensions/:id", extAdm.AdminUpdateExtension) admin.PUT("/extensions/:id", extAdm.AdminUpdateExtension)
@@ -819,6 +838,7 @@ func main() {
pkgAdm.SetSandbox(sandbox.New(sandbox.DefaultConfig())) pkgAdm.SetSandbox(sandbox.New(sandbox.DefaultConfig()))
pkgAdm.SetBundledDir(cfg.BundledPackagesDir) pkgAdm.SetBundledDir(cfg.BundledPackagesDir)
pkgAdm.SetHub(hub) pkgAdm.SetHub(hub)
pkgAdm.SetCapabilities(detectedCaps)
// Package registry — must be registered before /packages/:id // Package registry — must be registered before /packages/:id
registryH := handlers.NewRegistryHandler(stores, packagesDir, pkgAdm) registryH := handlers.NewRegistryHandler(stores, packagesDir, pkgAdm)
@@ -839,6 +859,7 @@ func main() {
admin.POST("/packages/:id/update", pkgAdm.UpdatePackage) admin.POST("/packages/:id/update", pkgAdm.UpdatePackage)
admin.GET("/packages/:id/export", pkgAdm.ExportPackage) admin.GET("/packages/:id/export", pkgAdm.ExportPackage)
admin.GET("/dependencies", pkgAdm.ListAllDependencies) admin.GET("/dependencies", pkgAdm.ListAllDependencies)
admin.GET("/slots", pkgAdm.ListSlots)
// Workflow package export // Workflow package export
wfPkgH := handlers.NewWorkflowPackageHandler(stores) wfPkgH := handlers.NewWorkflowPackageHandler(stores)
@@ -883,6 +904,12 @@ func main() {
metricsH := handlers.NewMetricsHandler(metricsCollector) metricsH := handlers.NewMetricsHandler(metricsCollector)
admin.GET("/metrics", metricsH.GetMetrics) admin.GET("/metrics", metricsH.GetMetrics)
// ── Capabilities ──────────
capsH := handlers.NewCapabilitiesHandler(func() map[string]bool {
return handlers.DetectCapabilities(database.DB, database.IsPostgres(), objStore, cfg.WorkspaceRoot)
})
admin.GET("/capabilities", capsH.GetCapabilities)
// ── Backup/Restore ───────── // ── Backup/Restore ─────────
backupH := handlers.NewBackupHandler(stores, packagesDir, cfg.StoragePath) backupH := handlers.NewBackupHandler(stores, packagesDir, cfg.StoragePath)
admin.POST("/backup", backupH.CreateBackup) admin.POST("/backup", backupH.CreateBackup)
@@ -931,12 +958,16 @@ func main() {
Session: middleware.OptionalAuth(cfg, stores.Users, userCache), Session: middleware.OptionalAuth(cfg, stores.Users, userCache),
}) })
// Mounted at /s/:slug/api/* with JWT auth (returns 401, not redirect). // Extension surface + API routes: /s/:slug/* dispatches between
// surface rendering (GET) and ext API calls (all methods, /api/* prefix).
{ {
extAPIH := handlers.NewExtAPIHandler(stores, starlarkRunner) extAPIH := handlers.NewExtAPIHandler(stores, starlarkRunner)
extAPI := base.Group("/s/:slug/api") pageEngine.RegisterExtensionRoutes(
extAPI.Use(middleware.Auth(cfg, stores.Users, userCache, stores.APITokens)) base,
extAPI.Any("/*path", extAPIH.Handle) middleware.OptionalAuth(cfg, stores.Users, userCache),
middleware.Auth(cfg, stores.Users, userCache, stores.APITokens),
extAPIH.Handle,
)
} }
bp := cfg.BasePath bp := cfg.BasePath

View File

@@ -37,6 +37,9 @@ const (
ExtPermTriggersRegister = "triggers.register" ExtPermTriggersRegister = "triggers.register"
ExtPermRealtimePublish = "realtime.publish" ExtPermRealtimePublish = "realtime.publish"
ExtPermBatchExec = "batch.exec" ExtPermBatchExec = "batch.exec"
ExtPermFilesRead = "files.read"
ExtPermFilesWrite = "files.write"
ExtPermWorkspaceManage = "workspace.manage"
) )
// ValidExtensionPermissions is the set of recognized permission keys. // ValidExtensionPermissions is the set of recognized permission keys.
@@ -52,6 +55,9 @@ var ValidExtensionPermissions = map[string]bool{
ExtPermTriggersRegister: true, ExtPermTriggersRegister: true,
ExtPermRealtimePublish: true, ExtPermRealtimePublish: true,
ExtPermBatchExec: true, ExtPermBatchExec: true,
ExtPermFilesRead: true,
ExtPermFilesWrite: true,
ExtPermWorkspaceManage: true,
} }
// ── Extension Permission Model ─────────────── // ── Extension Permission Model ───────────────

View File

@@ -99,7 +99,9 @@ type PageData struct {
Theme string // "dark", "light", or "" (= use default) Theme string // "dark", "light", or "" (= use default)
SurfaceSettings any // JSON-serialized to window.__SETTINGS__ SurfaceSettings any // JSON-serialized to window.__SETTINGS__
Manifest *SurfaceManifest `json:"manifest,omitempty"` Manifest *SurfaceManifest `json:"manifest,omitempty"`
SurfacePath string `json:"-"` // matched surface path pattern, e.g. "/monitor"
SurfaceParams map[string]string `json:"-"` // extracted params, e.g. {"id": "abc"}
EnabledSurfaces []string `json:"-"` EnabledSurfaces []string `json:"-"`
@@ -158,29 +160,60 @@ func (e *Engine) extensionNavItems() []ExtensionNavItem {
var items []ExtensionNavItem var items []ExtensionNavItem
for _, sr := range surfaces { for _, sr := range surfaces {
if sr.Source == "core" || !sr.Enabled { if sr.Source == "core" || !sr.Enabled || sr.Status != "active" {
continue continue
} }
// Only surface and full types have routes — extensions are headless // Only surface and full types have routes — extensions are headless
if sr.Type != "surface" && sr.Type != "full" { if sr.Type != "surface" && sr.Type != "full" {
continue continue
} }
route := ""
basePath := "/s/" + sr.ID
title := sr.Title
// Multi-surface: find the nav entry from surfaces array
if sr.Manifest != nil { if sr.Manifest != nil {
route, _ = sr.Manifest["route"].(string) if rawSurfaces, ok := sr.Manifest["surfaces"].([]any); ok {
} navEntry := findNavSurface(rawSurfaces)
if route == "" { if navEntry != nil {
route = "/s/" + sr.ID if p, ok := navEntry["path"].(string); ok && p != "/" {
basePath = "/s/" + sr.ID + p
}
if t, ok := navEntry["title"].(string); ok && t != "" {
title = t
}
}
}
} }
items = append(items, ExtensionNavItem{ items = append(items, ExtensionNavItem{
ID: sr.ID, ID: sr.ID,
Title: sr.Title, Title: title,
Route: route, Route: basePath,
}) })
} }
return items return items
} }
// findNavSurface returns the surface entry to use for navigation.
// Returns the first entry with nav:true, falling back to the entry with path "/".
func findNavSurface(surfaces []any) map[string]any {
var root map[string]any
for _, raw := range surfaces {
s, ok := raw.(map[string]any)
if !ok {
continue
}
if nav, ok := s["nav"].(bool); ok && nav {
return s
}
if p, ok := s["path"].(string); ok && p == "/" {
root = s
}
}
return root
}
// browserExtensionIDs returns IDs of enabled browser-tier extensions. // browserExtensionIDs returns IDs of enabled browser-tier extensions.
// These extensions have JS scripts that should be injected into every page // These extensions have JS scripts that should be injected into every page
// so they can register renderers with the SDK. // so they can register renderers with the SDK.
@@ -401,8 +434,10 @@ func (e *Engine) RenderSurface(surfaceID string) gin.HandlerFunc {
} }
} }
// RenderExtensionSurface handles /s/:slug — dynamic DB lookup for extension surfaces. // RenderExtensionSurface handles extension surface rendering with multi-surface
// No restart required after installing a surface via admin API. // support. Each surface entry in the manifest can have its own path, access
// level, title, and layout. Called by RegisterExtensionRoutes for both root
// requests (/s/:slug) and sub-path requests (/s/:slug/monitor).
func (e *Engine) RenderExtensionSurface() gin.HandlerFunc { func (e *Engine) RenderExtensionSurface() gin.HandlerFunc {
return func(c *gin.Context) { return func(c *gin.Context) {
slug := c.Param("slug") slug := c.Param("slug")
@@ -410,6 +445,10 @@ func (e *Engine) RenderExtensionSurface() gin.HandlerFunc {
c.String(http.StatusNotFound, "Surface not found") c.String(http.StatusNotFound, "Surface not found")
return return
} }
reqPath := c.Param("path")
if reqPath == "" {
reqPath = "/"
}
if e.stores.Packages == nil { if e.stores.Packages == nil {
c.String(http.StatusNotFound, "Surface registry not available") c.String(http.StatusNotFound, "Surface registry not available")
@@ -427,28 +466,70 @@ func (e *Engine) RenderExtensionSurface() gin.HandlerFunc {
return return
} }
if sr.Source == "core" { if sr.Source == "core" {
// Core surfaces have their own routes — don't serve them here
c.String(http.StatusNotFound, "Surface not found: "+slug) c.String(http.StatusNotFound, "Surface not found: "+slug)
return return
} }
// ── Multi-surface resolution ─────────────────────────────
var surfacePath string
var surfaceParams map[string]string
surfaceTitle := sr.Title
surfaceAccess := "authenticated"
surfaceLayout := "single"
// Read package-level defaults
if pkgAuth, ok := sr.Manifest["auth"].(string); ok && pkgAuth != "" {
surfaceAccess = pkgAuth
}
if pkgLayout, ok := sr.Manifest["layout"].(string); ok && pkgLayout != "" {
surfaceLayout = pkgLayout
}
if rawSurfaces, ok := sr.Manifest["surfaces"].([]any); ok && len(rawSurfaces) > 0 {
matched, params := matchSurface(rawSurfaces, reqPath)
if matched == nil {
c.String(http.StatusNotFound, "Page not found")
return
}
surfacePath, _ = matched["path"].(string)
surfaceParams = params
if t, ok := matched["title"].(string); ok && t != "" {
surfaceTitle = t
}
if a, ok := matched["access"].(string); ok && a != "" {
surfaceAccess = a
}
if l, ok := matched["layout"].(string); ok && l != "" {
surfaceLayout = l
}
} else {
// Legacy package without surfaces — only serve root path
if reqPath != "/" {
c.String(http.StatusNotFound, "Page not found")
return
}
surfacePath = "/"
}
// ── Access check ─────────────────────────────────────────
if !evaluateAccess(c, surfaceAccess) {
// Redirect unauthenticated users to login; deny others with 403
if c.GetString("user_id") == "" {
c.Redirect(http.StatusTemporaryRedirect, e.cfg.BasePath+"/login")
} else {
c.String(http.StatusForbidden, "Access denied")
}
return
}
user := e.getUserContext(c) user := e.getUserContext(c)
// Build manifest from DB record
route, _ := sr.Manifest["route"].(string)
if route == "" {
route = "/s/" + sr.ID
}
layout, _ := sr.Manifest["layout"].(string)
if layout == "" {
layout = "single"
}
manifest := &SurfaceManifest{ manifest := &SurfaceManifest{
ID: sr.ID, ID: sr.ID,
Route: route, Route: "/s/" + sr.ID,
Title: sr.Title, Title: surfaceTitle,
Template: "surface-extension", Template: "surface-extension",
Layout: layout, Layout: surfaceLayout,
Source: "extension", Source: "extension",
} }
@@ -456,6 +537,8 @@ func (e *Engine) RenderExtensionSurface() gin.HandlerFunc {
Surface: sr.ID, Surface: sr.ID,
User: user, User: user,
Manifest: manifest, Manifest: manifest,
SurfacePath: surfacePath,
SurfaceParams: surfaceParams,
EnabledSurfaces: e.EnabledSurfaceIDs(), EnabledSurfaces: e.EnabledSurfaceIDs(),
ExtensionSurfaces: e.extensionNavItems(), ExtensionSurfaces: e.extensionNavItems(),
BrowserExtensions: e.browserExtensionIDs(), BrowserExtensions: e.browserExtensionIDs(),
@@ -463,6 +546,98 @@ func (e *Engine) RenderExtensionSurface() gin.HandlerFunc {
} }
} }
// matchSurface finds the best-matching surface entry for a request path.
// Supports Gin-style param segments (/:id matches /abc). Static segments
// are preferred over param segments. Returns nil if no surface matches.
func matchSurface(surfaces []any, reqPath string) (map[string]any, map[string]string) {
reqParts := splitPath(reqPath)
type candidate struct {
surface map[string]any
params map[string]string
score int // higher = more static segments = better match
}
var best *candidate
for _, raw := range surfaces {
s, ok := raw.(map[string]any)
if !ok {
continue
}
pattern, _ := s["path"].(string)
if pattern == "" {
continue
}
patParts := splitPath(pattern)
if len(patParts) != len(reqParts) {
continue
}
params := map[string]string{}
score := 0
matched := true
for i, pp := range patParts {
if strings.HasPrefix(pp, ":") {
params[pp[1:]] = reqParts[i]
} else if pp == reqParts[i] {
score++
} else {
matched = false
break
}
}
if !matched {
continue
}
if best == nil || score > best.score {
best = &candidate{surface: s, params: params, score: score}
}
}
if best == nil {
return nil, nil
}
return best.surface, best.params
}
// splitPath splits a URL path into non-empty segments.
// "/" → [], "/monitor" → ["monitor"], "/monitor/:id" → ["monitor", ":id"]
func splitPath(p string) []string {
var parts []string
for _, s := range strings.Split(p, "/") {
if s != "" {
parts = append(parts, s)
}
}
return parts
}
// evaluateAccess checks whether the current request meets an access requirement.
func evaluateAccess(c *gin.Context, access string) bool {
switch {
case access == "public":
return true
case access == "authenticated":
return c.GetString("user_id") != ""
case access == "admin":
return c.GetBool("is_admin")
case strings.HasPrefix(access, "group:"):
// Group membership check — look for groups set by auth middleware
group := strings.TrimPrefix(access, "group:")
if groups, exists := c.Get("user_groups"); exists {
if gs, ok := groups.([]string); ok {
for _, g := range gs {
if g == group {
return true
}
}
}
}
return false
default:
return false
}
}
// PageRouteMiddleware holds middleware handlers for each auth level. // PageRouteMiddleware holds middleware handlers for each auth level.
// Passed to RegisterPageRoutes by main.go. // Passed to RegisterPageRoutes by main.go.
type PageRouteMiddleware struct { type PageRouteMiddleware struct {
@@ -502,8 +677,9 @@ func (e *Engine) RegisterPageRoutes(base *gin.RouterGroup, mw PageRouteMiddlewar
registerRoutes(group, s, handler) registerRoutes(group, s, handler)
} }
// No restart needed after installing new surfaces via admin API. // Extension surfaces are registered separately via
group.GET("/s/:slug", e.RenderExtensionSurface()) // RegisterExtensionRoutes (called from main.go) to unify
// the /s/:slug route tree with the ext API dispatcher.
} }
// Admin surfaces — auth + admin role middleware // Admin surfaces — auth + admin role middleware
@@ -553,6 +729,63 @@ func (e *Engine) RegisterPageRoutes(base *gin.RouterGroup, mw PageRouteMiddlewar
log.Printf("[pages] Registered routes for %d surfaces", len(e.surfaces)) log.Printf("[pages] Registered routes for %d surfaces", len(e.surfaces))
} }
// RegisterExtensionRoutes registers the combined extension surface and ext API
// routes. A single /s/:slug/*path catch-all dispatches between:
// - /s/:slug/api/* → apiAuth middleware + extAPIHandler (JSON API)
// - /s/:slug/* → surface handler with per-surface access checks (HTML)
//
// This avoids Gin route conflicts between the catch-all and the API sub-path.
// Must be called AFTER RegisterPageRoutes (which handles core surfaces).
func (e *Engine) RegisterExtensionRoutes(
base *gin.RouterGroup,
optAuth gin.HandlerFunc,
apiAuth gin.HandlerFunc,
extAPIHandler gin.HandlerFunc,
) {
surfaceHandler := e.RenderExtensionSurface()
sGroup := base.Group("/s/:slug")
sGroup.Use(optAuth) // populate user context if token present; don't require it
// Root path: /s/:slug (no sub-path) — surface only
sGroup.GET("", surfaceHandler)
// Sub-paths: /s/:slug/* — dispatch between API and surface
sGroup.Any("/*path", func(c *gin.Context) {
subPath := c.Param("path")
// API requests: /s/:slug/api/*
if strings.HasPrefix(subPath, "/api/") || subPath == "/api" {
apiAuth(c)
if c.IsAborted() {
return
}
// Rewrite the path param so the ext API handler sees the
// API-relative path, e.g. "/api/items" → "/items"
apiPath := strings.TrimPrefix(subPath, "/api")
if apiPath == "" {
apiPath = "/"
}
for i := range c.Params {
if c.Params[i].Key == "path" {
c.Params[i].Value = apiPath
break
}
}
extAPIHandler(c)
return
}
// Surface requests: GET only
if c.Request.Method != http.MethodGet {
c.String(http.StatusMethodNotAllowed, "Method not allowed")
return
}
surfaceHandler(c)
})
}
// surfaceHandler returns the appropriate Gin handler for a surface. // surfaceHandler returns the appropriate Gin handler for a surface.
func (e *Engine) surfaceHandler(s SurfaceManifest) gin.HandlerFunc { func (e *Engine) surfaceHandler(s SurfaceManifest) gin.HandlerFunc {
if s.ID == "workflow" { if s.ID == "workflow" {

View File

@@ -0,0 +1,175 @@
package pages
import (
"testing"
)
// ── matchSurface tests ──────────────────────────────────────
func TestMatchSurface_StaticRoot(t *testing.T) {
surfaces := []any{
map[string]any{"path": "/", "title": "Dashboard"},
map[string]any{"path": "/submit", "title": "Submit"},
}
s, params := matchSurface(surfaces, "/")
if s == nil {
t.Fatal("expected match for /")
}
if s["title"] != "Dashboard" {
t.Errorf("expected Dashboard, got %v", s["title"])
}
if len(params) != 0 {
t.Errorf("expected no params, got %v", params)
}
}
func TestMatchSurface_StaticPath(t *testing.T) {
surfaces := []any{
map[string]any{"path": "/", "title": "Dashboard"},
map[string]any{"path": "/submit", "title": "Submit"},
map[string]any{"path": "/monitor", "title": "Monitor"},
}
s, _ := matchSurface(surfaces, "/submit")
if s == nil {
t.Fatal("expected match for /submit")
}
if s["title"] != "Submit" {
t.Errorf("expected Submit, got %v", s["title"])
}
}
func TestMatchSurface_ParamPath(t *testing.T) {
surfaces := []any{
map[string]any{"path": "/", "title": "Dashboard"},
map[string]any{"path": "/:id", "title": "Detail"},
}
s, params := matchSurface(surfaces, "/abc123")
if s == nil {
t.Fatal("expected match for /:id")
}
if s["title"] != "Detail" {
t.Errorf("expected Detail, got %v", s["title"])
}
if params["id"] != "abc123" {
t.Errorf("expected id=abc123, got %v", params["id"])
}
}
func TestMatchSurface_MultiSegmentParam(t *testing.T) {
surfaces := []any{
map[string]any{"path": "/monitor", "title": "Monitor"},
map[string]any{"path": "/monitor/:id", "title": "Instance Detail"},
}
s, params := matchSurface(surfaces, "/monitor/inst-42")
if s == nil {
t.Fatal("expected match for /monitor/:id")
}
if s["title"] != "Instance Detail" {
t.Errorf("expected Instance Detail, got %v", s["title"])
}
if params["id"] != "inst-42" {
t.Errorf("expected id=inst-42, got %v", params["id"])
}
}
func TestMatchSurface_NoMatch(t *testing.T) {
surfaces := []any{
map[string]any{"path": "/", "title": "Dashboard"},
map[string]any{"path": "/submit", "title": "Submit"},
}
s, _ := matchSurface(surfaces, "/nonexistent/deep")
if s != nil {
t.Errorf("expected no match, got %v", s)
}
}
func TestMatchSurface_StaticPreferredOverParam(t *testing.T) {
surfaces := []any{
map[string]any{"path": "/:id", "title": "Detail"},
map[string]any{"path": "/new", "title": "New"},
}
s, params := matchSurface(surfaces, "/new")
if s == nil {
t.Fatal("expected match for /new")
}
if s["title"] != "New" {
t.Errorf("expected static match 'New', got %v", s["title"])
}
if len(params) != 0 {
t.Errorf("expected no params for static match, got %v", params)
}
}
func TestMatchSurface_EmptySurfaces(t *testing.T) {
s, _ := matchSurface([]any{}, "/")
if s != nil {
t.Errorf("expected no match for empty surfaces, got %v", s)
}
}
// ── splitPath tests ─────────────────────────────────────────
func TestSplitPath(t *testing.T) {
tests := []struct {
input string
want int
}{
{"/", 0},
{"/submit", 1},
{"/monitor/:id", 2},
{"/:id/edit", 2},
}
for _, tt := range tests {
parts := splitPath(tt.input)
if len(parts) != tt.want {
t.Errorf("splitPath(%q) = %d parts, want %d", tt.input, len(parts), tt.want)
}
}
}
// ── findNavSurface tests ────────────────────────────────────
func TestFindNavSurface_ExplicitNav(t *testing.T) {
surfaces := []any{
map[string]any{"path": "/", "title": "Dashboard"},
map[string]any{"path": "/monitor", "title": "Monitor", "nav": true},
}
s := findNavSurface(surfaces)
if s == nil {
t.Fatal("expected nav surface")
}
if s["title"] != "Monitor" {
t.Errorf("expected Monitor, got %v", s["title"])
}
}
func TestFindNavSurface_FallbackRoot(t *testing.T) {
surfaces := []any{
map[string]any{"path": "/submit", "title": "Submit"},
map[string]any{"path": "/", "title": "Dashboard"},
}
s := findNavSurface(surfaces)
if s == nil {
t.Fatal("expected nav surface")
}
if s["title"] != "Dashboard" {
t.Errorf("expected Dashboard, got %v", s["title"])
}
}
func TestFindNavSurface_NoMatch(t *testing.T) {
surfaces := []any{
map[string]any{"path": "/submit", "title": "Submit"},
map[string]any{"path": "/admin", "title": "Admin"},
}
s := findNavSurface(surfaces)
if s != nil {
t.Errorf("expected nil, got %v", s)
}
}
// ── evaluateAccess tests ────────────────────────────────────
// Note: evaluateAccess depends on gin.Context which is hard to unit test
// without a full Gin setup. These are covered via handler integration tests.
// The matchSurface + splitPath + findNavSurface functions above are the
// pure-logic components that benefit most from unit testing.

View File

@@ -29,7 +29,7 @@
<style> <style>
body { margin: 0; background: var(--bg); color: var(--text); display: flex; flex-direction: column; height: 100vh; height: 100dvh; overflow: hidden; padding: env(safe-area-inset-top, 0) env(safe-area-inset-right, 0) env(safe-area-inset-bottom, 0) env(safe-area-inset-left, 0); } body { margin: 0; background: var(--bg); color: var(--text); display: flex; flex-direction: column; height: 100vh; height: 100dvh; overflow: hidden; padding: env(safe-area-inset-top, 0) env(safe-area-inset-right, 0) env(safe-area-inset-bottom, 0) env(safe-area-inset-left, 0); }
.surface { flex: 1; min-height: 0; overflow: hidden; } .surface { flex: 1; min-height: 0; overflow: hidden; }
.surface-inner { width: 100%; height: 100%; overflow: hidden; } .surface-inner { width: 100%; height: 100%; overflow: hidden; display: flex; flex-direction: column; }
.banner { flex-shrink: 0; } .banner { flex-shrink: 0; }
</style> </style>
<meta name="theme-color" content="#0e0e10" id="metaThemeColor"> <meta name="theme-color" content="#0e0e10" id="metaThemeColor">
@@ -122,6 +122,8 @@
{{/* __PAGE_DATA__ and __USER__ removed in v0.37.19 — SDK boot {{/* __PAGE_DATA__ and __USER__ removed in v0.37.19 — SDK boot
populates sw.auth.user and sw.auth.policies from API. */}} populates sw.auth.user and sw.auth.policies from API. */}}
{{if .Manifest}}window.__MANIFEST__ = {{.Manifest | toJSON}};{{end}} {{if .Manifest}}window.__MANIFEST__ = {{.Manifest | toJSON}};{{end}}
{{if .SurfacePath}}window.__SURFACE_PATH__ = '{{.SurfacePath}}';{{end}}
{{if .SurfaceParams}}window.__SURFACE_PARAMS__ = {{.SurfaceParams | toJSON}};{{end}}
</script> </script>
{{/* All surfaces use Preact SDK boot(). Legacy script includes removed. {{/* All surfaces use Preact SDK boot(). Legacy script includes removed.

View File

@@ -6,7 +6,7 @@
{{define "surface-team-admin"}} {{define "surface-team-admin"}}
<div id="shell-topbar"></div> <div id="shell-topbar"></div>
<div id="team-admin-mount" class="surface-team-admin" style="height:100%;overflow:hidden;"> <div id="team-admin-mount" class="surface-team-admin" style="flex:1;min-height:0;overflow:hidden;">
<div class="settings-placeholder" style="padding:40px;text-align:center;">Loading team admin&hellip;</div> <div class="settings-placeholder" style="padding:40px;text-align:center;">Loading team admin&hellip;</div>
</div> </div>
{{end}} {{end}}

View File

@@ -4,7 +4,7 @@
*/}} */}}
{{define "surface-welcome"}} {{define "surface-welcome"}}
<div id="welcome-mount" style="display:flex;flex-direction:column;height:100%;"></div> <div id="welcome-mount" style="display:flex;flex-direction:column;flex:1;min-height:0;"></div>
{{end}} {{end}}
{{define "scripts-welcome"}} {{define "scripts-welcome"}}

View File

@@ -30,7 +30,10 @@ import (
"crypto/rand" "crypto/rand"
"database/sql" "database/sql"
"encoding/hex" "encoding/hex"
"encoding/json"
"fmt" "fmt"
"math"
"sort"
"strings" "strings"
"go.starlark.net/starlark" "go.starlark.net/starlark"
@@ -39,10 +42,11 @@ import (
// DBModuleConfig holds configuration for a db module instance. // DBModuleConfig holds configuration for a db module instance.
type DBModuleConfig struct { type DBModuleConfig struct {
PackageID string PackageID string
CanWrite bool // true when db.write permission is granted CanWrite bool // true when db.write permission is granted
DB *sql.DB DB *sql.DB
IsPostgres bool IsPostgres bool
HasPgvector bool // true when pgvector extension is available on Postgres
} }
// allowedViews is the set of platform views extensions may query via db.view(). // allowedViews is the set of platform views extensions may query via db.view().
@@ -55,12 +59,13 @@ var allowedViews = map[string]bool{
// prefixed with ext_{packageID}_. Write operations are guarded by CanWrite. // prefixed with ext_{packageID}_. Write operations are guarded by CanWrite.
func BuildDBModule(ctx context.Context, cfg DBModuleConfig) *starlarkstruct.Module { func BuildDBModule(ctx context.Context, cfg DBModuleConfig) *starlarkstruct.Module {
fns := starlark.StringDict{ fns := starlark.StringDict{
"query": starlark.NewBuiltin("db.query", dbQuery(ctx, cfg)), "query": starlark.NewBuiltin("db.query", dbQuery(ctx, cfg)),
"count": starlark.NewBuiltin("db.count", dbCount(ctx, cfg)), "count": starlark.NewBuiltin("db.count", dbCount(ctx, cfg)),
"aggregate": starlark.NewBuiltin("db.aggregate", dbAggregate(ctx, cfg)), "aggregate": starlark.NewBuiltin("db.aggregate", dbAggregate(ctx, cfg)),
"query_batch": starlark.NewBuiltin("db.query_batch", dbQueryBatch(ctx, cfg)), "query_batch": starlark.NewBuiltin("db.query_batch", dbQueryBatch(ctx, cfg)),
"view": starlark.NewBuiltin("db.view", dbView(ctx, cfg)), "query_similar": starlark.NewBuiltin("db.query_similar", dbQuerySimilar(ctx, cfg)),
"list_tables": starlark.NewBuiltin("db.list_tables", dbListTables(ctx, cfg)), "view": starlark.NewBuiltin("db.view", dbView(ctx, cfg)),
"list_tables": starlark.NewBuiltin("db.list_tables", dbListTables(ctx, cfg)),
} }
if cfg.CanWrite { if cfg.CanWrite {
@@ -160,6 +165,20 @@ func starlarkToGoValue(v starlark.Value) (any, error) {
return bool(val), nil return bool(val), nil
case starlark.NoneType: case starlark.NoneType:
return nil, nil return nil, nil
case *starlark.List:
goSlice := make([]any, val.Len())
for i := 0; i < val.Len(); i++ {
elem, err := starlarkToGoValue(val.Index(i))
if err != nil {
return nil, fmt.Errorf("list[%d]: %w", i, err)
}
goSlice[i] = elem
}
b, err := json.Marshal(goSlice)
if err != nil {
return nil, fmt.Errorf("list marshal: %w", err)
}
return string(b), nil
default: default:
return nil, fmt.Errorf("unsupported type %s", v.Type()) return nil, fmt.Errorf("unsupported type %s", v.Type())
} }
@@ -926,3 +945,271 @@ func dbDelete(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *s
return starlark.True, nil return starlark.True, nil
} }
} }
// ── Vector Similarity ─────────────────────────
// starlarkListToFloats converts a Starlark list to a Go float64 slice.
func starlarkListToFloats(list *starlark.List) ([]float64, error) {
n := list.Len()
if n == 0 {
return nil, fmt.Errorf("vector must not be empty")
}
out := make([]float64, n)
for i := 0; i < n; i++ {
switch v := list.Index(i).(type) {
case starlark.Float:
out[i] = float64(v)
case starlark.Int:
i64, _ := v.Int64()
out[i] = float64(i64)
default:
return nil, fmt.Errorf("vector[%d]: expected number, got %s", i, list.Index(i).Type())
}
}
return out, nil
}
// floatsToVectorString serializes a float64 slice as a JSON array string.
// e.g. [0.1, 0.2, 0.3] → "[0.1,0.2,0.3]"
func floatsToVectorString(v []float64) string {
b, _ := json.Marshal(v)
return string(b)
}
// parseVectorJSON parses a JSON array string (or pgvector text) into float64 slice.
func parseVectorJSON(s string) ([]float64, error) {
var out []float64
if err := json.Unmarshal([]byte(s), &out); err != nil {
return nil, fmt.Errorf("parse vector: %w", err)
}
return out, nil
}
// cosineDistance computes cosine distance between two vectors.
// Returns 0.0 for identical vectors, 1.0 for orthogonal, 2.0 for opposite.
func cosineDistance(a, b []float64) float64 {
if len(a) != len(b) || len(a) == 0 {
return 1.0
}
var dot, normA, normB float64
for i := range a {
dot += a[i] * b[i]
normA += a[i] * a[i]
normB += b[i] * b[i]
}
if normA == 0 || normB == 0 {
return 1.0
}
return 1.0 - (dot / (math.Sqrt(normA) * math.Sqrt(normB)))
}
// dbQuerySimilar implements db.query_similar(table, column, vector=[], limit=10, filters=None, metric="cosine").
// Returns rows ordered by distance with an injected _distance float.
//
// Three dispatch paths:
// - pgvector: native <=> operator with HNSW index
// - fallback: fetch rows, compute cosine distance in Go, sort, return top N
func dbQuerySimilar(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var table, column string
var vectorVal *starlark.List
var limit starlark.Int = starlark.MakeInt(10)
var filters starlark.Value = starlark.None
var metric starlark.String = "cosine"
if err := starlark.UnpackArgs(b.Name(), args, kwargs,
"table", &table,
"column", &column,
"vector", &vectorVal,
"limit?", &limit,
"filters?", &filters,
"metric?", &metric,
); err != nil {
return nil, err
}
if string(metric) != "cosine" {
return nil, fmt.Errorf("db.query_similar: unsupported metric %q (only \"cosine\" is supported)", string(metric))
}
queryVec, err := starlarkListToFloats(vectorVal)
if err != nil {
return nil, fmt.Errorf("db.query_similar: %w", err)
}
lim, ok := limit.Int64()
if !ok || lim < 1 {
lim = 10
}
if lim > 100 {
lim = 100
}
if strings.ContainsAny(column, " \t\n\"';-") {
return nil, fmt.Errorf("db.query_similar: invalid column name %q", column)
}
physTable, err := cfg.physicalTable(table)
if err != nil {
return nil, err
}
if cfg.HasPgvector {
return querySimilarPgvector(ctx, cfg, physTable, column, queryVec, int(lim), filters)
}
return querySimilarFallback(ctx, cfg, physTable, column, queryVec, int(lim), filters)
}
}
// querySimilarPgvector uses the native pgvector <=> operator.
func querySimilarPgvector(ctx context.Context, cfg DBModuleConfig, physTable, column string, queryVec []float64, limit int, filters starlark.Value) (starlark.Value, error) {
vecStr := floatsToVectorString(queryVec)
whereClause, whereArgs, err := cfg.starlarkFiltersToSQL(filters, 1)
if err != nil {
return nil, err
}
nextIdx := len(whereArgs) + 1
vecPH := cfg.ph(nextIdx)
limitPH := cfg.ph(nextIdx + 1)
query := fmt.Sprintf("SELECT *, (%s <=> %s::vector) AS _distance FROM %s",
column, vecPH, physTable)
if whereClause != "" {
query += " " + whereClause
}
query += fmt.Sprintf(" ORDER BY %s <=> %s::vector LIMIT %s", column, vecPH, limitPH)
allArgs := append(whereArgs, vecStr, limit)
rows, err := cfg.DB.QueryContext(ctx, query, allArgs...)
if err != nil {
return nil, fmt.Errorf("db.query_similar: %w", err)
}
defer rows.Close()
return rowsToStarlark(rows)
}
// querySimilarFallback fetches rows and computes cosine distance in Go.
// Used for SQLite (TEXT) and Postgres without pgvector (JSONB).
func querySimilarFallback(ctx context.Context, cfg DBModuleConfig, physTable, column string, queryVec []float64, limit int, filters starlark.Value) (starlark.Value, error) {
whereClause, whereArgs, err := cfg.starlarkFiltersToSQL(filters, 1)
if err != nil {
return nil, err
}
// Fetch up to 1000 rows for distance computation.
fetchLimit := limit * 10
if fetchLimit > 1000 {
fetchLimit = 1000
}
if fetchLimit < limit {
fetchLimit = limit
}
limitPH := cfg.ph(len(whereArgs) + 1)
query := fmt.Sprintf("SELECT * FROM %s", physTable)
if whereClause != "" {
query += " " + whereClause
}
query += " LIMIT " + limitPH
allArgs := append(whereArgs, fetchLimit)
rows, err := cfg.DB.QueryContext(ctx, query, allArgs...)
if err != nil {
return nil, fmt.Errorf("db.query_similar: %w", err)
}
defer rows.Close()
cols, err := rows.Columns()
if err != nil {
return nil, err
}
// Find the vector column index.
vecColIdx := -1
for i, c := range cols {
if c == column {
vecColIdx = i
break
}
}
if vecColIdx == -1 {
return nil, fmt.Errorf("db.query_similar: column %q not found in table", column)
}
type scoredRow struct {
vals []any
distance float64
}
var scored []scoredRow
for rows.Next() {
vals := make([]any, len(cols))
ptrs := make([]any, len(cols))
for i := range vals {
ptrs[i] = &vals[i]
}
if err := rows.Scan(ptrs...); err != nil {
return nil, err
}
// Parse the vector column value.
var vecStr string
switch v := vals[vecColIdx].(type) {
case string:
vecStr = v
case []byte:
vecStr = string(v)
default:
continue // skip rows with unparseable vectors
}
rowVec, err := parseVectorJSON(vecStr)
if err != nil {
continue // skip malformed vectors
}
dist := cosineDistance(queryVec, rowVec)
scored = append(scored, scoredRow{vals: vals, distance: dist})
}
if err := rows.Err(); err != nil {
return nil, err
}
// Sort by distance ascending.
sort.Slice(scored, func(i, j int) bool {
return scored[i].distance < scored[j].distance
})
// Take top N and build Starlark dicts.
if len(scored) > limit {
scored = scored[:limit]
}
var result []starlark.Value
for _, sr := range scored {
d := starlark.NewDict(len(cols) + 1)
for i, col := range cols {
sv, err := goToStarlark(sr.vals[i])
if err != nil {
return nil, fmt.Errorf("column %q: %w", col, err)
}
if err := d.SetKey(starlark.String(col), sv); err != nil {
return nil, err
}
}
// Inject _distance.
if err := d.SetKey(starlark.String("_distance"), starlark.Float(sr.distance)); err != nil {
return nil, err
}
result = append(result, d)
}
if result == nil {
result = []starlark.Value{}
}
return starlark.NewList(result), nil
}

View File

@@ -3,6 +3,7 @@ package sandbox
import ( import (
"context" "context"
"database/sql" "database/sql"
"math"
"strings" "strings"
"testing" "testing"
@@ -684,3 +685,252 @@ func TestDBQueryBatch_MissingTable(t *testing.T) {
} }
} }
// ── starlarkToGoValue list ──────────────────
func TestStarlarkToGoValue_List(t *testing.T) {
list := starlark.NewList([]starlark.Value{
starlark.Float(0.1), starlark.Float(0.2), starlark.Float(0.3),
})
got, err := starlarkToGoValue(list)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
s, ok := got.(string)
if !ok {
t.Fatalf("expected string, got %T", got)
}
if s != "[0.1,0.2,0.3]" {
t.Errorf("got %q, want %q", s, "[0.1,0.2,0.3]")
}
}
// ── cosineDistance ───────────────────────────
func TestCosineDistance(t *testing.T) {
// Identical vectors → distance 0
d := cosineDistance([]float64{1, 0, 0}, []float64{1, 0, 0})
if math.Abs(d) > 1e-9 {
t.Errorf("identical vectors: got %f, want 0", d)
}
// Orthogonal vectors → distance 1
d = cosineDistance([]float64{1, 0}, []float64{0, 1})
if math.Abs(d-1.0) > 1e-9 {
t.Errorf("orthogonal vectors: got %f, want 1", d)
}
// Opposite vectors → distance 2
d = cosineDistance([]float64{1, 0}, []float64{-1, 0})
if math.Abs(d-2.0) > 1e-9 {
t.Errorf("opposite vectors: got %f, want 2", d)
}
// Empty/mismatched → distance 1
d = cosineDistance([]float64{}, []float64{})
if d != 1.0 {
t.Errorf("empty vectors: got %f, want 1", d)
}
d = cosineDistance([]float64{1}, []float64{1, 2})
if d != 1.0 {
t.Errorf("mismatched vectors: got %f, want 1", d)
}
}
// ── vector helper: newVectorTestDB ──────────
func newVectorTestDB(t *testing.T) (*sql.DB, DBModuleConfig) {
t.Helper()
db, err := sql.Open("sqlite", ":memory:")
if err != nil {
t.Fatalf("open db: %v", err)
}
t.Cleanup(func() { db.Close() })
// Create a table with a TEXT column for vector storage (SQLite fallback).
_, err = db.Exec(`CREATE TABLE ext_test_ext_embeddings (
id TEXT PRIMARY KEY,
embedding TEXT,
category TEXT,
created_at TEXT DEFAULT (datetime('now'))
)`)
if err != nil {
t.Fatalf("create test table: %v", err)
}
cfg := DBModuleConfig{
PackageID: "test-ext",
CanWrite: true,
DB: db,
IsPostgres: false,
HasPgvector: false,
}
return db, cfg
}
// ── db.query_similar ────────────────────────
func TestDBQuerySimilar_Basic(t *testing.T) {
db, cfg := newVectorTestDB(t)
// Insert 5 rows with known vectors.
db.Exec(`INSERT INTO ext_test_ext_embeddings (id, embedding, category) VALUES ('a', '[1,0,0]', 'x')`)
db.Exec(`INSERT INTO ext_test_ext_embeddings (id, embedding, category) VALUES ('b', '[0,1,0]', 'x')`)
db.Exec(`INSERT INTO ext_test_ext_embeddings (id, embedding, category) VALUES ('c', '[0,0,1]', 'x')`)
db.Exec(`INSERT INTO ext_test_ext_embeddings (id, embedding, category) VALUES ('d', '[0.9,0.1,0]', 'x')`)
db.Exec(`INSERT INTO ext_test_ext_embeddings (id, embedding, category) VALUES ('e', '[0,0.9,0.1]', 'x')`)
// Query similar to [1,0,0] — should return 'a' first (identical), then 'd' (close).
result, err := execScript(t, cfg, `
rows = db.query_similar("embeddings", "embedding", vector=[1.0, 0.0, 0.0], limit=3)
`)
if err != nil {
t.Fatalf("script error: %v", err)
}
rows, ok := result.Globals["rows"].(*starlark.List)
if !ok {
t.Fatalf("rows is %T, want *starlark.List", result.Globals["rows"])
}
if rows.Len() != 3 {
t.Fatalf("got %d rows, want 3", rows.Len())
}
// First row should be 'a' (distance ≈ 0).
first := rows.Index(0).(*starlark.Dict)
idVal, _, _ := first.Get(starlark.String("id"))
if string(idVal.(starlark.String)) != "a" {
t.Errorf("first row id = %s, want 'a'", idVal)
}
// Verify _distance is present and near 0.
distVal, _, _ := first.Get(starlark.String("_distance"))
dist := float64(distVal.(starlark.Float))
if dist > 0.01 {
t.Errorf("first row _distance = %f, want ≈ 0", dist)
}
// Second row should be 'd' (closest after identical).
second := rows.Index(1).(*starlark.Dict)
idVal2, _, _ := second.Get(starlark.String("id"))
if string(idVal2.(starlark.String)) != "d" {
t.Errorf("second row id = %s, want 'd'", idVal2)
}
}
func TestDBQuerySimilar_WithFilters(t *testing.T) {
db, cfg := newVectorTestDB(t)
db.Exec(`INSERT INTO ext_test_ext_embeddings (id, embedding, category) VALUES ('a', '[1,0,0]', 'alpha')`)
db.Exec(`INSERT INTO ext_test_ext_embeddings (id, embedding, category) VALUES ('b', '[0.9,0.1,0]', 'beta')`)
db.Exec(`INSERT INTO ext_test_ext_embeddings (id, embedding, category) VALUES ('c', '[0,1,0]', 'alpha')`)
// Filter to alpha only — should exclude 'b' even though it's close.
result, err := execScript(t, cfg, `
rows = db.query_similar("embeddings", "embedding", vector=[1.0, 0.0, 0.0], filters={"category": "alpha"})
`)
if err != nil {
t.Fatalf("script error: %v", err)
}
rows := result.Globals["rows"].(*starlark.List)
for i := 0; i < rows.Len(); i++ {
d := rows.Index(i).(*starlark.Dict)
idVal, _, _ := d.Get(starlark.String("id"))
if string(idVal.(starlark.String)) == "b" {
t.Error("filtered query should not include row 'b' (category=beta)")
}
}
}
func TestDBQuerySimilar_EmptyTable(t *testing.T) {
_, cfg := newVectorTestDB(t)
result, err := execScript(t, cfg, `
rows = db.query_similar("embeddings", "embedding", vector=[1.0, 0.0, 0.0])
`)
if err != nil {
t.Fatalf("script error: %v", err)
}
rows := result.Globals["rows"].(*starlark.List)
if rows.Len() != 0 {
t.Errorf("expected 0 rows for empty table, got %d", rows.Len())
}
}
func TestDBQuerySimilar_InvalidMetric(t *testing.T) {
_, cfg := newVectorTestDB(t)
_, err := execScript(t, cfg, `
rows = db.query_similar("embeddings", "embedding", vector=[1.0], metric="l2")
`)
if err == nil {
t.Fatal("expected error for unsupported metric")
}
if !strings.Contains(err.Error(), "unsupported metric") {
t.Errorf("unexpected error: %v", err)
}
}
func TestDBQuerySimilar_LimitCap(t *testing.T) {
db, cfg := newVectorTestDB(t)
// Insert 5 rows
for i := 0; i < 5; i++ {
db.Exec(`INSERT INTO ext_test_ext_embeddings (id, embedding) VALUES (?, '[1,0,0]')`,
generateID())
}
result, err := execScript(t, cfg, `
rows = db.query_similar("embeddings", "embedding", vector=[1.0, 0.0, 0.0], limit=2)
`)
if err != nil {
t.Fatalf("script error: %v", err)
}
rows := result.Globals["rows"].(*starlark.List)
if rows.Len() != 2 {
t.Errorf("expected 2 rows with limit=2, got %d", rows.Len())
}
}
func TestDBInsert_VectorColumn(t *testing.T) {
db, cfg := newVectorTestDB(t)
_, err := execScript(t, cfg, `
row = db.insert("embeddings", {"embedding": [0.1, 0.2, 0.3], "category": "test"})
`)
if err != nil {
t.Fatalf("script error: %v", err)
}
// Verify the stored value is valid JSON.
var stored string
db.QueryRow(`SELECT embedding FROM ext_test_ext_embeddings WHERE category = 'test'`).Scan(&stored)
if stored != "[0.1,0.2,0.3]" {
t.Errorf("stored vector = %q, want %q", stored, "[0.1,0.2,0.3]")
}
}
func TestDBQuerySimilar_InsertThenQuery(t *testing.T) {
_, cfg := newVectorTestDB(t)
// Full round-trip: insert via db.insert, then query_similar.
result, err := execScript(t, cfg, `
db.insert("embeddings", {"embedding": [1.0, 0.0, 0.0], "category": "a"})
db.insert("embeddings", {"embedding": [0.0, 1.0, 0.0], "category": "b"})
rows = db.query_similar("embeddings", "embedding", vector=[1.0, 0.0, 0.0], limit=1)
`)
if err != nil {
t.Fatalf("script error: %v", err)
}
rows := result.Globals["rows"].(*starlark.List)
if rows.Len() != 1 {
t.Fatalf("expected 1 row, got %d", rows.Len())
}
first := rows.Index(0).(*starlark.Dict)
catVal, _, _ := first.Get(starlark.String("category"))
if string(catVal.(starlark.String)) != "a" {
t.Errorf("closest match should be category=a, got %s", catVal)
}
}

View File

@@ -0,0 +1,472 @@
// Package sandbox — files_module.go
//
// Permissions: files.read, files.write
//
// Starlark API:
// files.put(name, content, content_type="application/octet-stream", metadata={})
// result = files.get(name) → dict or None
// meta = files.meta(name) → dict or None
// entries = files.list(prefix="", limit=100)
// files.delete(name)
// files.delete_prefix(prefix)
// exists = files.exists(name)
//
// Bridges the ObjectStore (PVC/S3) into Starlark. All keys are scoped
// to ext/{packageID}/ — extensions cannot access each other's files.
// Metadata is stored as companion JSON at ext/{packageID}/_meta/{name}.
package sandbox
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
"strconv"
"strings"
"unicode"
"go.starlark.net/starlark"
"go.starlark.net/starlarkstruct"
"armature/storage"
)
const (
filesDefaultMaxSize = 50 * 1024 * 1024 // 50 MB
filesDefaultLimit = 100
filesMaxNameLen = 512
filesMetaPrefix = "_meta/"
)
// FilesModuleConfig holds the configuration for the files module.
type FilesModuleConfig struct {
PackageID string
CanWrite bool
Store storage.ObjectStore
}
// BuildFilesModule creates the "files" module.
func BuildFilesModule(ctx context.Context, cfg FilesModuleConfig) *starlarkstruct.Module {
fns := starlark.StringDict{
"get": starlark.NewBuiltin("files.get", filesGet(ctx, cfg)),
"meta": starlark.NewBuiltin("files.meta", filesMeta(ctx, cfg)),
"list": starlark.NewBuiltin("files.list", filesList(ctx, cfg)),
"exists": starlark.NewBuiltin("files.exists", filesExists(ctx, cfg)),
}
if cfg.CanWrite {
fns["put"] = starlark.NewBuiltin("files.put", filesPut(ctx, cfg))
fns["delete"] = starlark.NewBuiltin("files.delete", filesDelete(ctx, cfg))
fns["delete_prefix"] = starlark.NewBuiltin("files.delete_prefix", filesDeletePrefix(ctx, cfg))
}
return MakeModule("files", fns)
}
// ── Builtins ──────────────────────────────────
func filesPut(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var name string
var content starlark.Value
var contentType string = "application/octet-stream"
var metadata *starlark.Dict
if err := starlark.UnpackArgs(b.Name(), args, kwargs,
"name", &name,
"content", &content,
"content_type?", &contentType,
"metadata?", &metadata,
); err != nil {
return nil, err
}
if err := validateFileName(name); err != nil {
return nil, fmt.Errorf("files.put: %w", err)
}
// Extract raw bytes from content (accept String or Bytes).
var raw []byte
switch v := content.(type) {
case starlark.String:
raw = []byte(string(v))
case starlark.Bytes:
raw = []byte(v)
default:
return nil, fmt.Errorf("files.put: content must be string or bytes, got %s", content.Type())
}
// Enforce size limit.
maxSize := filesMaxSize()
if int64(len(raw)) > maxSize {
return nil, fmt.Errorf("files.put: content size %d exceeds limit %d", len(raw), maxSize)
}
// Write main object.
key := cfg.scopedKey(name)
if err := cfg.Store.Put(ctx, key, bytes.NewReader(raw), int64(len(raw)), contentType); err != nil {
return nil, fmt.Errorf("files.put: %w", err)
}
// Write metadata companion if provided.
if metadata != nil && metadata.Len() > 0 {
metaMap, err := starlarkDictToMap(metadata)
if err != nil {
return nil, fmt.Errorf("files.put: metadata: %w", err)
}
metaJSON, err := json.Marshal(metaMap)
if err != nil {
return nil, fmt.Errorf("files.put: marshal metadata: %w", err)
}
metaKey := cfg.metaKey(name)
if err := cfg.Store.Put(ctx, metaKey, bytes.NewReader(metaJSON), int64(len(metaJSON)), "application/json"); err != nil {
return nil, fmt.Errorf("files.put: write metadata: %w", err)
}
}
return starlark.True, nil
}
}
func filesGet(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var name string
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil {
return nil, err
}
if err := validateFileName(name); err != nil {
return nil, fmt.Errorf("files.get: %w", err)
}
key := cfg.scopedKey(name)
rc, size, ct, err := cfg.Store.Get(ctx, key)
if err == storage.ErrNotFound {
return starlark.None, nil
}
if err != nil {
return nil, fmt.Errorf("files.get: %w", err)
}
defer rc.Close()
data, err := io.ReadAll(rc)
if err != nil {
return nil, fmt.Errorf("files.get: read: %w", err)
}
// Load metadata companion.
metaDict := starlark.NewDict(0)
metaKey := cfg.metaKey(name)
if metaRC, _, _, metaErr := cfg.Store.Get(ctx, metaKey); metaErr == nil {
metaBytes, _ := io.ReadAll(metaRC)
metaRC.Close()
var m map[string]any
if json.Unmarshal(metaBytes, &m) == nil {
metaDict = mapToStarlarkDict(m)
}
}
result := starlark.NewDict(4)
result.SetKey(starlark.String("content"), starlark.Bytes(data))
result.SetKey(starlark.String("content_type"), starlark.String(ct))
result.SetKey(starlark.String("size"), starlark.MakeInt64(size))
result.SetKey(starlark.String("metadata"), metaDict)
return result, nil
}
}
func filesMeta(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var name string
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil {
return nil, err
}
if err := validateFileName(name); err != nil {
return nil, fmt.Errorf("files.meta: %w", err)
}
// Check the main object exists.
key := cfg.scopedKey(name)
exists, err := cfg.Store.Exists(ctx, key)
if err != nil {
return nil, fmt.Errorf("files.meta: %w", err)
}
if !exists {
return starlark.None, nil
}
// Load metadata companion.
metaDict := starlark.NewDict(0)
metaKey := cfg.metaKey(name)
if metaRC, _, _, metaErr := cfg.Store.Get(ctx, metaKey); metaErr == nil {
metaBytes, _ := io.ReadAll(metaRC)
metaRC.Close()
var m map[string]any
if json.Unmarshal(metaBytes, &m) == nil {
metaDict = mapToStarlarkDict(m)
}
}
return metaDict, nil
}
}
func filesList(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var prefix string
var limit int = filesDefaultLimit
if err := starlark.UnpackArgs(b.Name(), args, kwargs,
"prefix?", &prefix,
"limit?", &limit,
); err != nil {
return nil, err
}
if limit <= 0 || limit > 1000 {
limit = filesDefaultLimit
}
// Build the full scoped prefix.
scopedPrefix := cfg.scopedPrefix()
if prefix != "" {
// Validate the user-supplied prefix portion.
if strings.Contains(prefix, "..") {
return nil, fmt.Errorf("files.list: path traversal not allowed")
}
scopedPrefix += prefix
}
// Request more than limit to account for _meta/ entries we'll filter out.
entries, err := cfg.Store.List(ctx, scopedPrefix, limit*2)
if err != nil {
return nil, fmt.Errorf("files.list: %w", err)
}
// Filter out _meta/ companions and strip the package prefix.
pkgPrefix := cfg.scopedPrefix()
var results []starlark.Value
for _, e := range entries {
// Strip package prefix to get the extension-relative key.
relKey := strings.TrimPrefix(e.Key, pkgPrefix)
// Skip metadata companions.
if strings.HasPrefix(relKey, filesMetaPrefix) {
continue
}
d := starlark.NewDict(3)
d.SetKey(starlark.String("name"), starlark.String(relKey))
d.SetKey(starlark.String("size"), starlark.MakeInt64(e.Size))
d.SetKey(starlark.String("content_type"), starlark.String(e.ContentType))
results = append(results, d)
if len(results) >= limit {
break
}
}
return starlark.NewList(results), nil
}
}
func filesDelete(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var name string
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil {
return nil, err
}
if err := validateFileName(name); err != nil {
return nil, fmt.Errorf("files.delete: %w", err)
}
// Delete main object.
key := cfg.scopedKey(name)
if err := cfg.Store.Delete(ctx, key); err != nil {
return nil, fmt.Errorf("files.delete: %w", err)
}
// Delete metadata companion (best-effort, idempotent).
metaKey := cfg.metaKey(name)
cfg.Store.Delete(ctx, metaKey)
return starlark.True, nil
}
}
func filesDeletePrefix(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var prefix string
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &prefix); err != nil {
return nil, err
}
if strings.Contains(prefix, "..") {
return nil, fmt.Errorf("files.delete_prefix: path traversal not allowed")
}
scopedPrefix := cfg.scopedPrefix() + prefix
if err := cfg.Store.DeletePrefix(ctx, scopedPrefix); err != nil {
return nil, fmt.Errorf("files.delete_prefix: %w", err)
}
// Also delete metadata companions under this prefix.
metaPrefix := cfg.scopedPrefix() + filesMetaPrefix + prefix
cfg.Store.DeletePrefix(ctx, metaPrefix)
return starlark.True, nil
}
}
func filesExists(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var name string
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil {
return nil, err
}
if err := validateFileName(name); err != nil {
return nil, fmt.Errorf("files.exists: %w", err)
}
key := cfg.scopedKey(name)
exists, err := cfg.Store.Exists(ctx, key)
if err != nil {
return nil, fmt.Errorf("files.exists: %w", err)
}
return starlark.Bool(exists), nil
}
}
// ── Helpers ──────────────────────────────────
// scopedKey returns the full ObjectStore key for a file name.
func (cfg FilesModuleConfig) scopedKey(name string) string {
return fmt.Sprintf("ext/%s/%s", cfg.PackageID, name)
}
// metaKey returns the ObjectStore key for a file's metadata companion.
func (cfg FilesModuleConfig) metaKey(name string) string {
return fmt.Sprintf("ext/%s/%s%s", cfg.PackageID, filesMetaPrefix, name)
}
// scopedPrefix returns the ObjectStore key prefix for this package.
func (cfg FilesModuleConfig) scopedPrefix() string {
return fmt.Sprintf("ext/%s/", cfg.PackageID)
}
// validateFileName checks that a file name is safe.
func validateFileName(name string) error {
if name == "" {
return fmt.Errorf("file name is required")
}
if len(name) > filesMaxNameLen {
return fmt.Errorf("file name exceeds %d characters", filesMaxNameLen)
}
if strings.HasPrefix(name, "/") || strings.HasPrefix(name, "\\") {
return fmt.Errorf("absolute paths not allowed: %q", name)
}
if strings.Contains(name, "..") {
return fmt.Errorf("path traversal not allowed: %q", name)
}
if strings.HasPrefix(name, filesMetaPrefix) {
return fmt.Errorf("names starting with %q are reserved", filesMetaPrefix)
}
for _, r := range name {
if unicode.IsControl(r) {
return fmt.Errorf("control characters not allowed in file name")
}
}
return nil
}
// filesMaxSize returns the maximum file size from env or default.
func filesMaxSize() int64 {
if v := os.Getenv("EXT_FILES_MAX_SIZE"); v != "" {
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
return n
}
}
return filesDefaultMaxSize
}
// starlarkDictToMap converts a *starlark.Dict to map[string]any.
func starlarkDictToMap(d *starlark.Dict) (map[string]any, error) {
m := make(map[string]any, d.Len())
for _, item := range d.Items() {
k, ok := starlark.AsString(item[0])
if !ok {
return nil, fmt.Errorf("metadata keys must be strings, got %s", item[0].Type())
}
m[k] = starlarkValueToGo(item[1])
}
return m, nil
}
// starlarkValueToGo converts a starlark.Value to a Go value for JSON serialization.
func starlarkValueToGo(v starlark.Value) any {
switch x := v.(type) {
case starlark.String:
return string(x)
case starlark.Int:
if i, ok := x.Int64(); ok {
return i
}
return x.String()
case starlark.Float:
return float64(x)
case starlark.Bool:
return bool(x)
case *starlark.List:
out := make([]any, x.Len())
for i := 0; i < x.Len(); i++ {
out[i] = starlarkValueToGo(x.Index(i))
}
return out
case *starlark.Dict:
m, _ := starlarkDictToMap(x)
return m
default:
return v.String()
}
}
// mapToStarlarkDict converts a map[string]any to *starlark.Dict.
func mapToStarlarkDict(m map[string]any) *starlark.Dict {
d := starlark.NewDict(len(m))
for k, v := range m {
d.SetKey(starlark.String(k), goValueToStarlark(v))
}
return d
}
// goValueToStarlark converts a Go value (from JSON) to starlark.Value.
func goValueToStarlark(v any) starlark.Value {
switch x := v.(type) {
case string:
return starlark.String(x)
case float64:
if x == float64(int64(x)) {
return starlark.MakeInt64(int64(x))
}
return starlark.Float(x)
case bool:
return starlark.Bool(x)
case []any:
elems := make([]starlark.Value, len(x))
for i, e := range x {
elems[i] = goValueToStarlark(e)
}
return starlark.NewList(elems)
case map[string]any:
return mapToStarlarkDict(x)
default:
return starlark.None
}
}

View File

@@ -0,0 +1,410 @@
package sandbox
import (
"context"
"fmt"
"io"
"strings"
"testing"
"go.starlark.net/starlark"
"armature/storage"
)
// memStore is an in-memory ObjectStore for testing.
type memStore struct {
objects map[string]memObject
}
type memObject struct {
data []byte
contentType string
}
func newMemStore() *memStore {
return &memStore{objects: make(map[string]memObject)}
}
func (m *memStore) Put(_ context.Context, key string, r io.Reader, size int64, contentType string) error {
data, err := io.ReadAll(r)
if err != nil {
return err
}
m.objects[key] = memObject{data: data, contentType: contentType}
return nil
}
func (m *memStore) Get(_ context.Context, key string) (io.ReadCloser, int64, string, error) {
obj, ok := m.objects[key]
if !ok {
return nil, 0, "", storage.ErrNotFound
}
return io.NopCloser(&memReader{data: obj.data}), int64(len(obj.data)), obj.contentType, nil
}
func (m *memStore) Delete(_ context.Context, key string) error {
delete(m.objects, key)
return nil
}
func (m *memStore) DeletePrefix(_ context.Context, prefix string) error {
for k := range m.objects {
if strings.HasPrefix(k, prefix) {
delete(m.objects, k)
}
}
return nil
}
func (m *memStore) Exists(_ context.Context, key string) (bool, error) {
_, ok := m.objects[key]
return ok, nil
}
func (m *memStore) Healthy(_ context.Context) error { return nil }
func (m *memStore) Stats(_ context.Context) (*storage.StorageStats, error) {
return &storage.StorageStats{Backend: "mem", Configured: true, Healthy: true}, nil
}
func (m *memStore) Backend() string { return "mem" }
func (m *memStore) List(_ context.Context, prefix string, limit int) ([]storage.ObjectEntry, error) {
if limit <= 0 {
limit = 100
}
var entries []storage.ObjectEntry
for k, obj := range m.objects {
if strings.HasPrefix(k, prefix) {
entries = append(entries, storage.ObjectEntry{
Key: k,
Size: int64(len(obj.data)),
ContentType: obj.contentType,
})
if len(entries) >= limit {
break
}
}
}
return entries, nil
}
// memReader wraps a byte slice as io.ReadCloser.
type memReader struct {
data []byte
pos int
}
func (r *memReader) Read(p []byte) (int, error) {
if r.pos >= len(r.data) {
return 0, io.EOF
}
n := copy(p, r.data[r.pos:])
r.pos += n
if r.pos >= len(r.data) {
return n, io.EOF
}
return n, nil
}
func (r *memReader) Close() error { return nil }
// ── Test helpers ──────────────────────────────
func execWithFiles(t *testing.T, canWrite bool, script string) (*Result, error) {
t.Helper()
ctx := context.Background()
store := newMemStore()
mod := BuildFilesModule(ctx, FilesModuleConfig{
PackageID: "test-pkg",
CanWrite: canWrite,
Store: store,
})
sb := New(DefaultConfig())
return sb.Exec(ctx, "test.star", script, map[string]starlark.Value{
"files": mod,
})
}
func execWithFilesStore(t *testing.T, store *memStore, canWrite bool, script string) (*Result, error) {
t.Helper()
ctx := context.Background()
mod := BuildFilesModule(ctx, FilesModuleConfig{
PackageID: "test-pkg",
CanWrite: canWrite,
Store: store,
})
sb := New(DefaultConfig())
return sb.Exec(ctx, "test.star", script, map[string]starlark.Value{
"files": mod,
})
}
// ── Put + Get round-trip ──────────────────────
func TestFilesPut_StringContent(t *testing.T) {
_, err := execWithFiles(t, true, `
files.put("hello.txt", "hello world", content_type="text/plain")
result = files.get("hello.txt")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
}
func TestFilesPut_BytesContent(t *testing.T) {
result, err := execWithFiles(t, true, `
files.put("bin.dat", b"\x00\x01\x02\xff")
got = files.get("bin.dat")
size = got["size"]
ct = got["content_type"]
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
size, _ := starlark.AsInt32(result.Globals["size"])
if size != 4 {
t.Errorf("size = %d, want 4", size)
}
ct := string(result.Globals["ct"].(starlark.String))
if ct != "application/octet-stream" {
t.Errorf("content_type = %q, want application/octet-stream", ct)
}
}
func TestFilesPut_WithMetadata(t *testing.T) {
result, err := execWithFiles(t, true, `
files.put("doc.pdf", "pdf content", content_type="application/pdf", metadata={"author": "test", "pages": 5})
meta = files.meta("doc.pdf")
author = meta["author"]
pages = meta["pages"]
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
author := string(result.Globals["author"].(starlark.String))
if author != "test" {
t.Errorf("author = %q, want %q", author, "test")
}
pages, _ := starlark.AsInt32(result.Globals["pages"])
if pages != 5 {
t.Errorf("pages = %d, want 5", pages)
}
}
func TestFilesPut_SizeLimit(t *testing.T) {
// Set a small limit for testing.
t.Setenv("EXT_FILES_MAX_SIZE", "10")
_, err := execWithFiles(t, true, `
files.put("big.bin", "this content is longer than ten bytes")
`)
if err == nil {
t.Fatal("expected size limit error")
}
if !strings.Contains(err.Error(), "exceeds limit") {
t.Errorf("unexpected error: %v", err)
}
}
// ── Get ───────────────────────────────────────
func TestFilesGet_NotFound(t *testing.T) {
result, err := execWithFiles(t, false, `
got = files.get("nonexistent.txt")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
if result.Globals["got"] != starlark.None {
t.Errorf("expected None, got %v", result.Globals["got"])
}
}
func TestFilesGet_ReturnsAllFields(t *testing.T) {
result, err := execWithFiles(t, true, `
files.put("test.txt", "data", content_type="text/plain", metadata={"k": "v"})
got = files.get("test.txt")
has_content = "content" in got
has_ct = "content_type" in got
has_size = "size" in got
has_meta = "metadata" in got
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
for _, key := range []string{"has_content", "has_ct", "has_size", "has_meta"} {
if result.Globals[key] != starlark.True {
t.Errorf("%s = False, want True", key)
}
}
}
// ── Meta ──────────────────────────────────────
func TestFilesMeta_NotFound(t *testing.T) {
result, err := execWithFiles(t, false, `
got = files.meta("nonexistent.txt")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
if result.Globals["got"] != starlark.None {
t.Errorf("expected None, got %v", result.Globals["got"])
}
}
func TestFilesMeta_NoCompanion(t *testing.T) {
result, err := execWithFiles(t, true, `
files.put("plain.txt", "data")
meta = files.meta("plain.txt")
count = len(meta)
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
count, _ := starlark.AsInt32(result.Globals["count"])
if count != 0 {
t.Errorf("meta length = %d, want 0", count)
}
}
// ── List ──────────────────────────────────────
func TestFilesList_PrefixFilter(t *testing.T) {
result, err := execWithFiles(t, true, `
files.put("images/a.png", "a")
files.put("images/b.png", "b")
files.put("docs/c.pdf", "c")
all_entries = files.list()
img_entries = files.list(prefix="images/")
all_count = len(all_entries)
img_count = len(img_entries)
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
allCount, _ := starlark.AsInt32(result.Globals["all_count"])
imgCount, _ := starlark.AsInt32(result.Globals["img_count"])
if allCount != 3 {
t.Errorf("all_count = %d, want 3", allCount)
}
if imgCount != 2 {
t.Errorf("img_count = %d, want 2", imgCount)
}
}
func TestFilesList_ExcludesMeta(t *testing.T) {
result, err := execWithFiles(t, true, `
files.put("file.txt", "data", metadata={"key": "val"})
entries = files.list()
count = len(entries)
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
count, _ := starlark.AsInt32(result.Globals["count"])
if count != 1 {
t.Errorf("count = %d, want 1 (should exclude _meta/ companion)", count)
}
}
// ── Delete ────────────────────────────────────
func TestFilesDelete_Idempotent(t *testing.T) {
_, err := execWithFiles(t, true, `
files.put("temp.txt", "temp")
files.delete("temp.txt")
files.delete("temp.txt")
exists = files.exists("temp.txt")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
}
func TestFilesDeletePrefix(t *testing.T) {
result, err := execWithFiles(t, true, `
files.put("uploads/a.txt", "a")
files.put("uploads/b.txt", "b")
files.put("keep.txt", "keep")
files.delete_prefix("uploads/")
a_exists = files.exists("uploads/a.txt")
b_exists = files.exists("uploads/b.txt")
keep_exists = files.exists("keep.txt")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
if result.Globals["a_exists"] != starlark.False {
t.Error("uploads/a.txt should be deleted")
}
if result.Globals["b_exists"] != starlark.False {
t.Error("uploads/b.txt should be deleted")
}
if result.Globals["keep_exists"] != starlark.True {
t.Error("keep.txt should be preserved")
}
}
// ── Exists ────────────────────────────────────
func TestFilesExists(t *testing.T) {
result, err := execWithFiles(t, true, `
files.put("exists.txt", "yes")
yes = files.exists("exists.txt")
no = files.exists("nope.txt")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
if result.Globals["yes"] != starlark.True {
t.Error("exists should be True")
}
if result.Globals["no"] != starlark.False {
t.Error("nope should be False")
}
}
// ── Name Validation ──────────────────────────
func TestFilesNameValidation(t *testing.T) {
badNames := []string{
"../escape",
"/absolute",
"_meta/reserved",
"",
}
for _, name := range badNames {
t.Run(name, func(t *testing.T) {
script := fmt.Sprintf(`files.exists(%q)`, name)
_, err := execWithFiles(t, false, script)
if err == nil {
t.Errorf("expected error for name %q", name)
}
})
}
}
// ── Write Permission Gate ────────────────────
func TestFilesWritePermission(t *testing.T) {
// Read-only mode: put/delete/delete_prefix should not be available.
_, err := execWithFiles(t, false, `
files.put("test.txt", "data")
`)
if err == nil {
t.Fatal("expected error: put should not be available in read-only mode")
}
if !strings.Contains(err.Error(), "no .put field") && !strings.Contains(err.Error(), "has no .put") {
t.Errorf("unexpected error: %v", err)
}
}

View File

@@ -79,19 +79,21 @@ func BuildLibModule(ctx context.Context, r *Runner, consumerPkgID string, rc *Ru
return nil, fmt.Errorf("lib.require: package %q does not declare dependency on %q", consumerPkgID, libraryID) return nil, fmt.Errorf("lib.require: package %q does not declare dependency on %q", consumerPkgID, libraryID)
} }
// 4. Load library's PackageRegistration // 4. Load target PackageRegistration
libPkg, err := r.stores.Packages.Get(ctx, libraryID) libPkg, err := r.stores.Packages.Get(ctx, libraryID)
if err != nil { if err != nil {
return nil, fmt.Errorf("lib.require: library %q not found: %w", libraryID, err) return nil, fmt.Errorf("lib.require: package %q not found: %w", libraryID, err)
} }
if libPkg.Type != "library" { // Any package with exports is callable — not just type "library"
return nil, fmt.Errorf("lib.require: package %q is type %q, not library", libraryID, libPkg.Type) exports := extractExports(libPkg.Manifest)
if len(exports) == 0 {
return nil, fmt.Errorf("lib.require: package %q declares no exports", libraryID)
} }
if libPkg.Status != models.PackageStatusActive { if libPkg.Status != models.PackageStatusActive {
return nil, fmt.Errorf("lib.require: library %q is %s, not active", libraryID, libPkg.Status) return nil, fmt.Errorf("lib.require: package %q is %s, not active", libraryID, libPkg.Status)
} }
if libPkg.Tier != models.ExtTierStarlark { if libPkg.Tier != models.ExtTierStarlark {
return nil, fmt.Errorf("lib.require: library %q is tier %s, not starlark", libraryID, libPkg.Tier) return nil, fmt.Errorf("lib.require: package %q is tier %s, not starlark", libraryID, libPkg.Tier)
} }
// 5. Build library's module set (library's own permissions) // 5. Build library's module set (library's own permissions)
@@ -113,12 +115,7 @@ func BuildLibModule(ctx context.Context, r *Runner, consumerPkgID string, rc *Ru
return nil, fmt.Errorf("lib.require: error executing library %q: %w", libraryID, err) return nil, fmt.Errorf("lib.require: error executing library %q: %w", libraryID, err)
} }
// 8. Extract exports from manifest // 8. Build exported members (exports already extracted above)
exports := extractExports(libPkg.Manifest)
if len(exports) == 0 {
return nil, fmt.Errorf("lib.require: library %q declares no exports", libraryID)
}
members := make(starlark.StringDict, len(exports)) members := make(starlark.StringDict, len(exports))
for _, name := range exports { for _, name := range exports {
val, ok := result.Globals[name] val, ok := result.Globals[name]

View File

@@ -33,6 +33,7 @@ import (
"armature/events" "armature/events"
"armature/metrics" "armature/metrics"
"armature/models" "armature/models"
"armature/storage"
"armature/store" "armature/store"
) )
@@ -77,6 +78,10 @@ type Runner struct {
dbPostgres bool // true = use $N placeholders; false = use ? dbPostgres bool // true = use $N placeholders; false = use ?
allowPrivateIPs bool allowPrivateIPs bool
bus *events.Bus bus *events.Bus
objectStore storage.ObjectStore // nil = files module unavailable
workspaceRoot string // empty = workspace module unavailable
workspaceQuota int // MB, 0 = unlimited
capabilities map[string]bool // detected environment capabilities
} }
// NewRunner creates a runner with the given sandbox and dependencies. // NewRunner creates a runner with the given sandbox and dependencies.
@@ -116,6 +121,24 @@ func (r *Runner) SetBus(bus *events.Bus) {
r.bus = bus r.bus = bus
} }
// SetObjectStore attaches the blob storage backend for the files module.
func (r *Runner) SetObjectStore(s storage.ObjectStore) {
r.objectStore = s
}
// SetWorkspaceRoot sets the base directory for extension workspaces.
// quotaMB is the per-extension quota in MB (0 = unlimited).
func (r *Runner) SetWorkspaceRoot(root string, quotaMB int) {
r.workspaceRoot = root
r.workspaceQuota = quotaMB
}
// SetCapabilities stores the detected environment capabilities map.
// Passed to the settings module so extensions can call has_capability().
func (r *Runner) SetCapabilities(caps map[string]bool) {
r.capabilities = caps
}
// SetAllowPrivateIPs disables the SSRF check that blocks connections to // SetAllowPrivateIPs disables the SSRF check that blocks connections to
// private/loopback IPs. For self-hosted environments where extensions // private/loopback IPs. For self-hosted environments where extensions
// reach internal services. Controlled by EXT_ALLOW_PRIVATE_IPS env var. // reach internal services. Controlled by EXT_ALLOW_PRIVATE_IPS env var.
@@ -336,8 +359,9 @@ func (r *Runner) buildModulesWithLibCtx(ctx context.Context, packageID string, m
modules := make(map[string]starlark.Value) modules := make(map[string]starlark.Value)
// Track db permission level: 0=none, 1=read, 2=write // Track tiered permission levels: 0=none, 1=read, 2=write
dbLevel := 0 dbLevel := 0
filesLevel := 0
hasBatchExec := false hasBatchExec := false
for _, perm := range granted { for _, perm := range granted {
@@ -363,6 +387,14 @@ func (r *Runner) buildModulesWithLibCtx(ctx context.Context, packageID string, m
case models.ExtPermDBWrite: case models.ExtPermDBWrite:
dbLevel = 2 dbLevel = 2
case models.ExtPermFilesRead:
if filesLevel < 1 {
filesLevel = 1
}
case models.ExtPermFilesWrite:
filesLevel = 2
case models.ExtPermWorkflowAccess: case models.ExtPermWorkflowAccess:
modules["workflow"] = BuildWorkflowModule(ctx, r.stores) modules["workflow"] = BuildWorkflowModule(ctx, r.stores)
@@ -376,6 +408,15 @@ func (r *Runner) buildModulesWithLibCtx(ctx context.Context, packageID string, m
modules["realtime"] = BuildRealtimeModule(ctx, r.bus, packageID) modules["realtime"] = BuildRealtimeModule(ctx, r.bus, packageID)
} }
case models.ExtPermWorkspaceManage:
if r.workspaceRoot != "" {
modules["workspace"] = BuildWorkspaceModule(ctx, WorkspaceModuleConfig{
PackageID: packageID,
WorkspaceRoot: r.workspaceRoot,
QuotaMB: r.workspaceQuota,
})
}
case models.ExtPermBatchExec: case models.ExtPermBatchExec:
hasBatchExec = true hasBatchExec = true
} }
@@ -384,10 +425,20 @@ func (r *Runner) buildModulesWithLibCtx(ctx context.Context, packageID string, m
// Wire db module at the highest granted level. // Wire db module at the highest granted level.
if dbLevel > 0 && r.db != nil { if dbLevel > 0 && r.db != nil {
modules["db"] = BuildDBModule(ctx, DBModuleConfig{ modules["db"] = BuildDBModule(ctx, DBModuleConfig{
PackageID: packageID, PackageID: packageID,
CanWrite: dbLevel == 2, CanWrite: dbLevel == 2,
DB: r.db, DB: r.db,
IsPostgres: r.dbPostgres, IsPostgres: r.dbPostgres,
HasPgvector: r.capabilities["pgvector"],
})
}
// Wire files module at the highest granted level.
if filesLevel > 0 && r.objectStore != nil {
modules["files"] = BuildFilesModule(ctx, FilesModuleConfig{
PackageID: packageID,
CanWrite: filesLevel == 2,
Store: r.objectStore,
}) })
} }
@@ -398,7 +449,7 @@ func (r *Runner) buildModulesWithLibCtx(ctx context.Context, packageID string, m
userID = rc.UserID userID = rc.UserID
teamID = rc.TeamID teamID = rc.TeamID
} }
modules["settings"] = BuildSettingsModule(ctx, r.stores, packageID, userID, teamID) modules["settings"] = BuildSettingsModule(ctx, r.stores, packageID, userID, teamID, r.capabilities)
// Always available — read-only check against kernel permission data // Always available — read-only check against kernel permission data
modules["permissions"] = BuildPermissionsModule(ctx, r.stores) modules["permissions"] = BuildPermissionsModule(ctx, r.stores)

View File

@@ -11,6 +11,7 @@ package sandbox
// Starlark API: // Starlark API:
// val = settings.get("key") # returns string, number, bool, or None // val = settings.get("key") # returns string, number, bool, or None
// val = settings.get("key", "default") # returns default if key not set // val = settings.get("key", "default") # returns default if key not set
// ok = settings.has_capability("pgvector") # returns True or False
import ( import (
"context" "context"
@@ -24,12 +25,28 @@ import (
// BuildSettingsModule creates the "settings" Starlark module for a package. // BuildSettingsModule creates the "settings" Starlark module for a package.
// It resolves the three-tier cascade (global → team → user) respecting // It resolves the three-tier cascade (global → team → user) respecting
// the user_overridable flag from the package manifest. // the user_overridable flag from the package manifest.
func BuildSettingsModule(ctx context.Context, stores store.Stores, packageID, userID, teamID string) *starlarkstruct.Module { func BuildSettingsModule(ctx context.Context, stores store.Stores, packageID, userID, teamID string, capabilities map[string]bool) *starlarkstruct.Module {
return MakeModule("settings", starlark.StringDict{ return MakeModule("settings", starlark.StringDict{
"get": starlark.NewBuiltin("settings.get", settingsGet(ctx, stores, packageID, userID, teamID)), "get": starlark.NewBuiltin("settings.get", settingsGet(ctx, stores, packageID, userID, teamID)),
"has_capability": starlark.NewBuiltin("settings.has_capability", hasCapability(capabilities)),
}) })
} }
// hasCapability returns a Starlark builtin that checks whether a named
// environment capability is available. Read-only, no permission needed.
func hasCapability(caps map[string]bool) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var name string
if err := starlark.UnpackPositionalArgs("settings.has_capability", args, kwargs, 1, &name); err != nil {
return nil, err
}
if caps == nil {
return starlark.False, nil
}
return starlark.Bool(caps[name]), nil
}
}
func settingsGet(ctx context.Context, stores store.Stores, packageID, userID, teamID string) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { func settingsGet(ctx context.Context, stores store.Stores, packageID, userID, teamID string) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { return func(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var key string var key string

View File

@@ -0,0 +1,75 @@
package sandbox
import (
"context"
"testing"
"go.starlark.net/starlark"
"armature/store"
)
func execWithSettings(t *testing.T, caps map[string]bool, script string) (*Result, error) {
t.Helper()
ctx := context.Background()
mod := BuildSettingsModule(ctx, store.Stores{}, "", "", "", caps)
sb := New(DefaultConfig())
return sb.Exec(ctx, "test.star", script, map[string]starlark.Value{
"settings": mod,
})
}
func TestHasCapability_True(t *testing.T) {
caps := map[string]bool{"postgres": true, "pgvector": true}
result, err := execWithSettings(t, caps, `
result = settings.has_capability("pgvector")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
v := result.Globals["result"]
if v != starlark.True {
t.Errorf("has_capability('pgvector') = %v, want True", v)
}
}
func TestHasCapability_False(t *testing.T) {
caps := map[string]bool{"postgres": true, "pgvector": false}
result, err := execWithSettings(t, caps, `
result = settings.has_capability("pgvector")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
v := result.Globals["result"]
if v != starlark.False {
t.Errorf("has_capability('pgvector') = %v, want False", v)
}
}
func TestHasCapability_UnknownCap(t *testing.T) {
caps := map[string]bool{"postgres": true}
result, err := execWithSettings(t, caps, `
result = settings.has_capability("gpu")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
v := result.Globals["result"]
if v != starlark.False {
t.Errorf("has_capability('gpu') = %v, want False", v)
}
}
func TestHasCapability_NilCaps(t *testing.T) {
result, err := execWithSettings(t, nil, `
result = settings.has_capability("postgres")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
v := result.Globals["result"]
if v != starlark.False {
t.Errorf("has_capability with nil caps = %v, want False", v)
}
}

View File

@@ -0,0 +1,261 @@
// Package sandbox — workspace_module.go
//
// Permission: workspace.manage
//
// Starlark API:
// path = workspace.create(name) → absolute path string (idempotent)
// path = workspace.path(name) → path string or None
// names = workspace.list() → list of workspace name strings
// workspace.delete(name) → True (idempotent, destructive)
// bytes = workspace.usage(name) → int (disk usage in bytes)
//
// Managed disk directories for extensions that need a real filesystem
// (git, compilers, media tools). All directories are scoped to
// {WORKSPACE_ROOT}/{packageID}/{name}/ — extensions cannot access each
// other's workspaces. Optional quota enforcement via WORKSPACE_QUOTA_MB.
package sandbox
import (
"context"
"fmt"
"io/fs"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"go.starlark.net/starlark"
"go.starlark.net/starlarkstruct"
)
var workspaceNameRe = regexp.MustCompile(`^[a-z][a-z0-9_]{0,62}$`)
// WorkspaceModuleConfig holds the configuration for the workspace module.
type WorkspaceModuleConfig struct {
PackageID string
WorkspaceRoot string
QuotaMB int // 0 = unlimited
}
// BuildWorkspaceModule creates the "workspace" module.
func BuildWorkspaceModule(ctx context.Context, cfg WorkspaceModuleConfig) *starlarkstruct.Module {
fns := starlark.StringDict{
"create": starlark.NewBuiltin("workspace.create", workspaceCreate(ctx, cfg)),
"path": starlark.NewBuiltin("workspace.path", workspacePath(ctx, cfg)),
"list": starlark.NewBuiltin("workspace.list", workspaceList(ctx, cfg)),
"delete": starlark.NewBuiltin("workspace.delete", workspaceDelete(ctx, cfg)),
"usage": starlark.NewBuiltin("workspace.usage", workspaceUsage(ctx, cfg)),
}
return MakeModule("workspace", fns)
}
// ── Builtins ──────────────────────────────────
func workspaceCreate(ctx context.Context, cfg WorkspaceModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var name string
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil {
return nil, err
}
dir, err := cfg.safePath(name)
if err != nil {
return nil, fmt.Errorf("workspace.create: %w", err)
}
// Enforce quota before creating.
if cfg.QuotaMB > 0 {
quotaCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
used, err := dirUsage(quotaCtx, cfg.packageDir())
if err != nil && !os.IsNotExist(err) {
return nil, fmt.Errorf("workspace.create: quota check: %w", err)
}
limit := int64(cfg.QuotaMB) * 1024 * 1024
if used >= limit {
return nil, fmt.Errorf("workspace.create: quota exceeded (%d bytes used, limit %d)", used, limit)
}
}
if err := os.MkdirAll(dir, 0755); err != nil {
return nil, fmt.Errorf("workspace.create: %w", err)
}
return starlark.String(dir), nil
}
}
func workspacePath(ctx context.Context, cfg WorkspaceModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var name string
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil {
return nil, err
}
dir, err := cfg.safePath(name)
if err != nil {
return nil, fmt.Errorf("workspace.path: %w", err)
}
info, err := os.Stat(dir)
if err != nil || !info.IsDir() {
return starlark.None, nil
}
return starlark.String(dir), nil
}
}
func workspaceList(ctx context.Context, cfg WorkspaceModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 0); err != nil {
return nil, err
}
entries, err := os.ReadDir(cfg.packageDir())
if os.IsNotExist(err) {
return starlark.NewList(nil), nil
}
if err != nil {
return nil, fmt.Errorf("workspace.list: %w", err)
}
var names []starlark.Value
for _, e := range entries {
if e.IsDir() && workspaceNameRe.MatchString(e.Name()) {
names = append(names, starlark.String(e.Name()))
}
}
return starlark.NewList(names), nil
}
}
func workspaceDelete(ctx context.Context, cfg WorkspaceModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var name string
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil {
return nil, err
}
dir, err := cfg.safePath(name)
if err != nil {
return nil, fmt.Errorf("workspace.delete: %w", err)
}
if err := os.RemoveAll(dir); err != nil {
return nil, fmt.Errorf("workspace.delete: %w", err)
}
return starlark.True, nil
}
}
func workspaceUsage(ctx context.Context, cfg WorkspaceModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var name string
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil {
return nil, err
}
dir, err := cfg.safePath(name)
if err != nil {
return nil, fmt.Errorf("workspace.usage: %w", err)
}
info, statErr := os.Stat(dir)
if statErr != nil || !info.IsDir() {
return nil, fmt.Errorf("workspace.usage: workspace %q does not exist", name)
}
usageCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
total, err := dirUsage(usageCtx, dir)
if err != nil {
return nil, fmt.Errorf("workspace.usage: %w", err)
}
return starlark.MakeInt64(total), nil
}
}
// ── Helpers ──────────────────────────────────
// packageDir returns the root directory for this extension's workspaces.
func (cfg WorkspaceModuleConfig) packageDir() string {
return filepath.Join(cfg.WorkspaceRoot, cfg.PackageID)
}
// safePath validates a workspace name and returns the absolute directory
// path, guarding against path traversal and symlink escape.
func (cfg WorkspaceModuleConfig) safePath(name string) (string, error) {
if err := validateWorkspaceName(name); err != nil {
return "", err
}
dir := filepath.Join(cfg.WorkspaceRoot, cfg.PackageID, name)
cleaned := filepath.Clean(dir)
// Ensure the cleaned path is under the package directory.
pkgDir := filepath.Clean(cfg.packageDir())
if !strings.HasPrefix(cleaned, pkgDir+string(filepath.Separator)) {
return "", fmt.Errorf("path escapes workspace root")
}
// If the parent directory exists, resolve symlinks to detect escape.
parent := filepath.Dir(cleaned)
if _, err := os.Stat(parent); err == nil {
resolved, err := filepath.EvalSymlinks(parent)
if err != nil {
return "", fmt.Errorf("symlink resolution failed: %w", err)
}
resolvedRoot := filepath.Clean(cfg.WorkspaceRoot)
if realRoot, err := filepath.EvalSymlinks(resolvedRoot); err == nil {
resolvedRoot = realRoot
}
if !strings.HasPrefix(resolved, resolvedRoot) {
return "", fmt.Errorf("path escapes workspace root via symlink")
}
}
return cleaned, nil
}
// validateWorkspaceName checks that a workspace name matches the allowed pattern.
func validateWorkspaceName(name string) error {
if name == "" {
return fmt.Errorf("workspace name is required")
}
if !workspaceNameRe.MatchString(name) {
return fmt.Errorf("workspace name %q must match %s", name, workspaceNameRe.String())
}
return nil
}
// dirUsage walks a directory tree and returns the total size in bytes.
// Respects context cancellation for timeout enforcement.
func dirUsage(ctx context.Context, root string) (int64, error) {
var total int64
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if ctx.Err() != nil {
return ctx.Err()
}
if !d.IsDir() {
info, err := d.Info()
if err != nil {
return nil // skip unreadable files
}
total += info.Size()
}
return nil
})
if os.IsNotExist(err) {
return 0, nil
}
return total, err
}

View File

@@ -0,0 +1,344 @@
package sandbox
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"go.starlark.net/starlark"
)
// ── Test helpers ──────────────────────────────
func execWithWorkspace(t *testing.T, script string) (*Result, string, error) {
t.Helper()
tmpDir := t.TempDir()
ctx := context.Background()
mod := BuildWorkspaceModule(ctx, WorkspaceModuleConfig{
PackageID: "test-pkg",
WorkspaceRoot: tmpDir,
})
sb := New(DefaultConfig())
result, err := sb.Exec(ctx, "test.star", script, map[string]starlark.Value{
"workspace": mod,
})
return result, tmpDir, err
}
func execWithWorkspaceQuota(t *testing.T, quotaMB int, script string) (*Result, string, error) {
t.Helper()
tmpDir := t.TempDir()
ctx := context.Background()
mod := BuildWorkspaceModule(ctx, WorkspaceModuleConfig{
PackageID: "test-pkg",
WorkspaceRoot: tmpDir,
QuotaMB: quotaMB,
})
sb := New(DefaultConfig())
result, err := sb.Exec(ctx, "test.star", script, map[string]starlark.Value{
"workspace": mod,
})
return result, tmpDir, err
}
// ── Create ────────────────────────────────────
func TestWorkspaceCreate_ReturnsPath(t *testing.T) {
result, tmpDir, err := execWithWorkspace(t, `
path = workspace.create("myrepo")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
got := string(result.Globals["path"].(starlark.String))
expected := filepath.Join(tmpDir, "test-pkg", "myrepo")
if got != expected {
t.Errorf("path = %q, want %q", got, expected)
}
}
func TestWorkspaceCreate_Idempotent(t *testing.T) {
result, _, err := execWithWorkspace(t, `
path1 = workspace.create("myrepo")
path2 = workspace.create("myrepo")
same = (path1 == path2)
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
if result.Globals["same"] != starlark.True {
t.Error("expected same path on second create")
}
}
func TestWorkspaceCreate_DirectoryExists(t *testing.T) {
result, _, err := execWithWorkspace(t, `
path = workspace.create("build_cache")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
dir := string(result.Globals["path"].(starlark.String))
info, statErr := os.Stat(dir)
if statErr != nil {
t.Fatalf("directory does not exist: %v", statErr)
}
if !info.IsDir() {
t.Error("expected a directory")
}
}
// ── Path ──────────────────────────────────────
func TestWorkspacePath_Exists(t *testing.T) {
result, tmpDir, err := execWithWorkspace(t, `
workspace.create("myrepo")
path = workspace.path("myrepo")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
got := string(result.Globals["path"].(starlark.String))
expected := filepath.Join(tmpDir, "test-pkg", "myrepo")
if got != expected {
t.Errorf("path = %q, want %q", got, expected)
}
}
func TestWorkspacePath_NotFound(t *testing.T) {
result, _, err := execWithWorkspace(t, `
path = workspace.path("nonexistent")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
if result.Globals["path"] != starlark.None {
t.Errorf("expected None, got %v", result.Globals["path"])
}
}
// ── List ──────────────────────────────────────
func TestWorkspaceList_Empty(t *testing.T) {
result, _, err := execWithWorkspace(t, `
names = workspace.list()
count = len(names)
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
count, _ := starlark.AsInt32(result.Globals["count"])
if count != 0 {
t.Errorf("count = %d, want 0", count)
}
}
func TestWorkspaceList_Multiple(t *testing.T) {
result, _, err := execWithWorkspace(t, `
workspace.create("alpha")
workspace.create("beta")
workspace.create("gamma")
names = workspace.list()
count = len(names)
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
count, _ := starlark.AsInt32(result.Globals["count"])
if count != 3 {
t.Errorf("count = %d, want 3", count)
}
}
// ── Delete ────────────────────────────────────
func TestWorkspaceDelete_Removes(t *testing.T) {
result, _, err := execWithWorkspace(t, `
workspace.create("temp")
workspace.delete("temp")
path = workspace.path("temp")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
if result.Globals["path"] != starlark.None {
t.Error("expected None after delete")
}
}
func TestWorkspaceDelete_Idempotent(t *testing.T) {
_, _, err := execWithWorkspace(t, `
workspace.delete("nonexistent")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
}
// ── Usage ─────────────────────────────────────
func TestWorkspaceUsage_EmptyDir(t *testing.T) {
result, _, err := execWithWorkspace(t, `
workspace.create("empty")
bytes = workspace.usage("empty")
`)
if err != nil {
t.Fatalf("exec error: %v", err)
}
usage, _ := starlark.AsInt32(result.Globals["bytes"])
if usage != 0 {
t.Errorf("usage = %d, want 0", usage)
}
}
func TestWorkspaceUsage_WithFiles(t *testing.T) {
tmpDir := t.TempDir()
ctx := context.Background()
// Pre-create workspace directory with a file.
wsDir := filepath.Join(tmpDir, "test-pkg", "data")
if err := os.MkdirAll(wsDir, 0755); err != nil {
t.Fatal(err)
}
// Write a 1024-byte file.
if err := os.WriteFile(filepath.Join(wsDir, "file.bin"), make([]byte, 1024), 0644); err != nil {
t.Fatal(err)
}
mod := BuildWorkspaceModule(ctx, WorkspaceModuleConfig{
PackageID: "test-pkg",
WorkspaceRoot: tmpDir,
})
sb := New(DefaultConfig())
result, err := sb.Exec(ctx, "test.star", `bytes = workspace.usage("data")`, map[string]starlark.Value{
"workspace": mod,
})
if err != nil {
t.Fatalf("exec error: %v", err)
}
usage, _ := starlark.AsInt32(result.Globals["bytes"])
if usage != 1024 {
t.Errorf("usage = %d, want 1024", usage)
}
}
func TestWorkspaceUsage_NonExistent(t *testing.T) {
_, _, err := execWithWorkspace(t, `
bytes = workspace.usage("nope")
`)
if err == nil {
t.Fatal("expected error for nonexistent workspace")
}
if !strings.Contains(err.Error(), "does not exist") {
t.Errorf("unexpected error: %v", err)
}
}
// ── Name Validation ──────────────────────────
func TestWorkspaceNameValidation_BadNames(t *testing.T) {
badNames := []struct {
name string
desc string
}{
{"../escape", "traversal"},
{"UPPER", "uppercase"},
{"", "empty"},
{"has space", "space"},
{"has/slash", "slash"},
{".dotstart", "dot prefix"},
{"0numstart", "digit prefix"},
{strings.Repeat("a", 64), "too long"},
}
for _, tc := range badNames {
t.Run(tc.desc, func(t *testing.T) {
script := fmt.Sprintf(`workspace.path(%q)`, tc.name)
_, _, err := execWithWorkspace(t, script)
if err == nil {
t.Errorf("expected error for name %q", tc.name)
}
})
}
}
func TestWorkspaceNameValidation_GoodNames(t *testing.T) {
goodNames := []string{"myrepo", "build_cache", "a", "a0_b1_c2"}
for _, name := range goodNames {
t.Run(name, func(t *testing.T) {
script := fmt.Sprintf(`path = workspace.path(%q)`, name)
_, _, err := execWithWorkspace(t, script)
if err != nil {
t.Errorf("unexpected error for name %q: %v", name, err)
}
})
}
}
// ── Quota ─────────────────────────────────────
func TestWorkspaceQuota_Enforced(t *testing.T) {
tmpDir := t.TempDir()
ctx := context.Background()
// Pre-fill the package directory with >1 MB of data.
pkgDir := filepath.Join(tmpDir, "test-pkg", "existing")
if err := os.MkdirAll(pkgDir, 0755); err != nil {
t.Fatal(err)
}
bigFile := make([]byte, 1100*1024) // ~1.07 MB
if err := os.WriteFile(filepath.Join(pkgDir, "big.bin"), bigFile, 0644); err != nil {
t.Fatal(err)
}
mod := BuildWorkspaceModule(ctx, WorkspaceModuleConfig{
PackageID: "test-pkg",
WorkspaceRoot: tmpDir,
QuotaMB: 1,
})
sb := New(DefaultConfig())
_, err := sb.Exec(ctx, "test.star", `workspace.create("newone")`, map[string]starlark.Value{
"workspace": mod,
})
if err == nil {
t.Fatal("expected quota exceeded error")
}
if !strings.Contains(err.Error(), "quota exceeded") {
t.Errorf("unexpected error: %v", err)
}
}
func TestWorkspaceQuota_Unlimited(t *testing.T) {
tmpDir := t.TempDir()
ctx := context.Background()
// Pre-fill with data — should not matter with quota=0.
pkgDir := filepath.Join(tmpDir, "test-pkg", "existing")
if err := os.MkdirAll(pkgDir, 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(pkgDir, "big.bin"), make([]byte, 2*1024*1024), 0644); err != nil {
t.Fatal(err)
}
mod := BuildWorkspaceModule(ctx, WorkspaceModuleConfig{
PackageID: "test-pkg",
WorkspaceRoot: tmpDir,
QuotaMB: 0, // unlimited
})
sb := New(DefaultConfig())
_, err := sb.Exec(ctx, "test.star", `workspace.create("newone")`, map[string]starlark.Value{
"workspace": mod,
})
if err != nil {
t.Fatalf("expected success with unlimited quota: %v", err)
}
}

View File

@@ -211,6 +211,78 @@ func (s *PVCStore) Exists(ctx context.Context, key string) (bool, error) {
return false, fmt.Errorf("storage/pvc: exists %q: %w", key, err) return false, fmt.Errorf("storage/pvc: exists %q: %w", key, err)
} }
// List returns objects under the given prefix, up to limit entries.
func (s *PVCStore) List(ctx context.Context, prefix string, limit int) ([]ObjectEntry, error) {
if prefix != "" {
if err := validateKey(prefix); err != nil {
return nil, err
}
}
if limit <= 0 {
limit = 100
}
absPrefix := s.resolve(prefix)
// If the prefix path doesn't exist, return empty.
info, err := os.Stat(absPrefix)
if os.IsNotExist(err) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("storage/pvc: list %q: %w", prefix, err)
}
var entries []ObjectEntry
if !info.IsDir() {
// Prefix points to a single file — return it.
s.mu.RLock()
ct := s.mimeMap[prefix]
s.mu.RUnlock()
if ct == "" {
ct = "application/octet-stream"
}
return []ObjectEntry{{Key: prefix, Size: info.Size(), ContentType: ct}}, nil
}
// Walk the directory tree under the prefix.
walkRoot := absPrefix
err = filepath.Walk(walkRoot, func(path string, fi os.FileInfo, walkErr error) error {
if walkErr != nil {
return nil // skip inaccessible entries
}
if fi.IsDir() {
return nil
}
if len(entries) >= limit {
return filepath.SkipAll
}
// Recover the storage key by stripping basePath + "/".
rel, relErr := filepath.Rel(s.basePath, path)
if relErr != nil {
return nil
}
key := filepath.ToSlash(rel)
s.mu.RLock()
ct := s.mimeMap[key]
s.mu.RUnlock()
if ct == "" {
ct = "application/octet-stream"
}
entries = append(entries, ObjectEntry{Key: key, Size: fi.Size(), ContentType: ct})
return nil
})
if err != nil {
return entries, fmt.Errorf("storage/pvc: list walk %q: %w", prefix, err)
}
return entries, nil
}
// Healthy verifies the storage path is writable. // Healthy verifies the storage path is writable.
func (s *PVCStore) Healthy(ctx context.Context) error { func (s *PVCStore) Healthy(ctx context.Context) error {
probe := filepath.Join(s.basePath, ".health-probe") probe := filepath.Join(s.basePath, ".health-probe")

View File

@@ -282,6 +282,64 @@ func TestPVC_SubdirCreation(t *testing.T) {
} }
} }
func TestPVC_List(t *testing.T) {
s := tempStore(t)
ctx := context.Background()
// Empty prefix — no files.
entries, err := s.List(ctx, "ext/pkg1/", 100)
if err != nil {
t.Fatalf("List empty: %v", err)
}
if len(entries) != 0 {
t.Errorf("expected 0 entries, got %d", len(entries))
}
// Add some files under a prefix.
for _, name := range []string{"a.txt", "b.png", "sub/c.pdf"} {
key := "ext/pkg1/" + name
_ = s.Put(ctx, key, bytes.NewReader([]byte("x")), 1, "text/plain")
}
// Add a file under a different prefix.
_ = s.Put(ctx, "ext/pkg2/other.txt", bytes.NewReader([]byte("y")), 1, "text/plain")
// List all under pkg1.
entries, err = s.List(ctx, "ext/pkg1/", 100)
if err != nil {
t.Fatalf("List: %v", err)
}
if len(entries) != 3 {
t.Errorf("expected 3 entries, got %d", len(entries))
}
// List with limit.
entries, err = s.List(ctx, "ext/pkg1/", 2)
if err != nil {
t.Fatalf("List limited: %v", err)
}
if len(entries) != 2 {
t.Errorf("expected 2 entries (limit), got %d", len(entries))
}
// List with sub-prefix.
entries, err = s.List(ctx, "ext/pkg1/sub/", 100)
if err != nil {
t.Fatalf("List sub: %v", err)
}
if len(entries) != 1 {
t.Errorf("expected 1 entry in sub/, got %d", len(entries))
}
// pkg2 prefix should only find 1.
entries, err = s.List(ctx, "ext/pkg2/", 100)
if err != nil {
t.Fatalf("List pkg2: %v", err)
}
if len(entries) != 1 {
t.Errorf("expected 1 entry in pkg2, got %d", len(entries))
}
}
func TestNewPVC_InvalidPath(t *testing.T) { func TestNewPVC_InvalidPath(t *testing.T) {
// Path that can't be created (nested under a file) // Path that can't be created (nested under a file)
tmp := t.TempDir() tmp := t.TempDir()

View File

@@ -232,6 +232,49 @@ func (s *S3Store) Exists(ctx context.Context, key string) (bool, error) {
return true, nil return true, nil
} }
// List returns objects under the given prefix, up to limit entries.
func (s *S3Store) List(ctx context.Context, prefix string, limit int) ([]ObjectEntry, error) {
if prefix != "" {
if err := validateKey(prefix); err != nil {
return nil, err
}
}
if limit <= 0 {
limit = 100
}
fullPrefix := s.fullKey(prefix)
objectsCh := s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{
Prefix: fullPrefix,
Recursive: true,
})
var entries []ObjectEntry
for obj := range objectsCh {
if obj.Err != nil {
return entries, fmt.Errorf("storage/s3: list %q: %w", prefix, obj.Err)
}
if len(entries) >= limit {
// Drain the remaining channel entries (minio requires it).
for range objectsCh {
}
break
}
// Strip the s3 prefix to recover the storage key.
key := strings.TrimPrefix(obj.Key, s.prefix)
ct := obj.ContentType
if ct == "" {
ct = "application/octet-stream"
}
entries = append(entries, ObjectEntry{Key: key, Size: obj.Size, ContentType: ct})
}
return entries, nil
}
// Healthy checks connectivity by performing a HeadBucket. // Healthy checks connectivity by performing a HeadBucket.
func (s *S3Store) Healthy(ctx context.Context) error { func (s *S3Store) Healthy(ctx context.Context) error {
exists, err := s.client.BucketExists(ctx, s.bucket) exists, err := s.client.BucketExists(ctx, s.bucket)

View File

@@ -56,10 +56,21 @@ type ObjectStore interface {
// Stats returns aggregate storage statistics. // Stats returns aggregate storage statistics.
Stats(ctx context.Context) (*StorageStats, error) Stats(ctx context.Context) (*StorageStats, error)
// List returns objects under the given prefix, up to limit entries.
// Returns an empty slice (not error) if no objects match.
List(ctx context.Context, prefix string, limit int) ([]ObjectEntry, error)
// Backend returns the backend type identifier ("pvc", "s3"). // Backend returns the backend type identifier ("pvc", "s3").
Backend() string Backend() string
} }
// ObjectEntry represents a single object returned by List.
type ObjectEntry struct {
Key string `json:"key"`
Size int64 `json:"size"`
ContentType string `json:"content_type"`
}
// StorageStats holds aggregate storage metrics. // StorageStats holds aggregate storage metrics.
type StorageStats struct { type StorageStats struct {
Backend string `json:"backend"` Backend string `json:"backend"`

View File

@@ -134,7 +134,7 @@ func (e *Engine) buildRestrictedModules(ctx context.Context, creatorID, runAs st
// Settings module — read-only access to platform settings // Settings module — read-only access to platform settings
// (no package context for scheduled tasks; they read global settings) // (no package context for scheduled tasks; they read global settings)
if e.stores.Packages != nil { if e.stores.Packages != nil {
modules["settings"] = sandbox.BuildSettingsModule(ctx, e.stores, "", creatorID, "") modules["settings"] = sandbox.BuildSettingsModule(ctx, e.stores, "", creatorID, "", nil)
} }
// Notifications module — if available // Notifications module — if available

View File

@@ -5,7 +5,8 @@
.extension-surface { .extension-surface {
display: flex; display: flex;
flex-direction: column; flex-direction: column;
height: 100%; flex: 1;
min-height: 0;
overflow: hidden; overflow: hidden;
} }

View File

@@ -22,13 +22,13 @@
/* ── Docs Surface ────────────────────────── */ /* ── Docs Surface ────────────────────────── */
.surface-docs { .surface-docs {
display: flex; flex-direction: column; height: 100%; overflow: hidden; display: flex; flex-direction: column; flex: 1; min-height: 0; overflow: hidden;
} }
/* ── Settings Surface ────────────────────── */ /* ── Settings Surface ────────────────────── */
.surface-settings { .surface-settings {
display: flex; height: 100%; overflow: hidden; display: flex; flex: 1; min-height: 0; overflow: hidden;
} }
.settings-topbar { .settings-topbar {
display: flex; align-items: center; gap: var(--sp-3); display: flex; align-items: center; gap: var(--sp-3);
@@ -89,7 +89,7 @@
/* ── Admin Surface ───────────────────────── */ /* ── Admin Surface ───────────────────────── */
.surface-admin { .surface-admin {
display: flex; flex-direction: column; height: 100%; overflow: hidden; display: flex; flex-direction: column; flex: 1; min-height: 0; overflow: hidden;
} }
.admin-topbar { .admin-topbar {
display: flex; align-items: center; gap: var(--sp-3); display: flex; align-items: center; gap: var(--sp-3);
@@ -168,7 +168,7 @@
/* ── Editor Surface ──────────────────────── */ /* ── Editor Surface ──────────────────────── */
.surface-editor { .surface-editor {
display: flex; flex-direction: column; height: 100%; overflow: hidden; display: flex; flex-direction: column; flex: 1; min-height: 0; overflow: hidden;
} }
.editor-topbar { .editor-topbar {
display: flex; align-items: center; gap: var(--sp-2); display: flex; align-items: center; gap: var(--sp-2);

View File

@@ -186,8 +186,46 @@ export async function boot() {
}); });
}; };
// Navigate — multi-surface intra-package routing
sw.navigate = function (path, params) {
const manifest = window.__MANIFEST__;
if (!manifest?.id) {
console.warn('[sw] navigate: no manifest — not inside an extension surface');
return;
}
// Interpolate :param placeholders
let resolved = path;
if (params) {
for (const [k, v] of Object.entries(params)) {
resolved = resolved.replace(':' + k, encodeURIComponent(v));
}
}
const base = window.__BASE__ || '';
const url = base + '/s/' + manifest.id + resolved;
// Update globals so the package can re-render
window.__SURFACE_PATH__ = path;
window.__SURFACE_PARAMS__ = params || {};
history.pushState({ surfacePath: path, surfaceParams: params }, '', url);
events.emit('surface.navigate', { path, params: params || {}, url }, { localOnly: true });
};
// Listen for popstate (back/forward) to emit navigation events
window.addEventListener('popstate', (e) => {
if (e.state?.surfacePath != null) {
window.__SURFACE_PATH__ = e.state.surfacePath;
window.__SURFACE_PARAMS__ = e.state.surfaceParams || {};
events.emit('surface.navigate', {
path: e.state.surfacePath,
params: e.state.surfaceParams || {},
url: location.pathname,
}, { localOnly: true });
}
});
// Marker for idempotency // Marker for idempotency
sw._sdk = '0.7.1'; sw._sdk = '0.9.0';
// 8. Expose globally // 8. Expose globally
window.sw = sw; window.sw = sw;

View File

@@ -15,6 +15,8 @@
export function createSlots(emitFn) { export function createSlots(emitFn) {
/** @type {Map<string, Array<{id:string, component:Function, priority:number}>>} */ /** @type {Map<string, Array<{id:string, component:Function, priority:number}>>} */
const _registry = new Map(); const _registry = new Map();
/** @type {Map<string, string>|undefined} */
let _declarations;
function _sort(arr) { function _sort(arr) {
arr.sort((a, b) => a.priority - b.priority); arr.sort((a, b) => a.priority - b.priority);
@@ -73,5 +75,43 @@ export function createSlots(emitFn) {
names() { names() {
return [..._registry.keys()]; return [..._registry.keys()];
}, },
/**
* Render all components registered in a slot.
* Each component is called with the context object. Errors are
* caught so one broken contributor doesn't break the host surface.
* @param {string} name — slot name
* @param {object} context — props passed to each component
* @returns {Array} — array of rendered vnodes (nulls filtered)
*/
renderAll(name, context = {}) {
return this.get(name).map(e => {
try {
return e.component(context);
} catch (err) {
console.error(`[sw.slots] Error rendering "${e.id}" in slot "${name}":`, err);
return null;
}
}).filter(Boolean);
},
/**
* Declare a slot for discoverability. Runtime-only metadata —
* does not affect registration or rendering.
* @param {string} name — slot name
* @param {string} description — human-readable purpose
*/
declare(name, description) {
if (!_declarations) _declarations = new Map();
_declarations.set(name, description);
},
/**
* List declared slot metadata (from declare()).
* @returns {Map<string, string>}
*/
declarations() {
return _declarations ? new Map(_declarations) : new Map();
},
}; };
} }