From 115004a3abeecb745b81633520b6bc289f268909 Mon Sep 17 00:00:00 2001 From: Jeffrey Smith Date: Tue, 17 Mar 2026 22:31:34 +0000 Subject: [PATCH] Changeset 0.29.2 (#197) Co-authored-by: Jeffrey Smith Co-committed-by: Jeffrey Smith --- .gitignore | 3 + CHANGELOG.md | 109 ++++ VERSION | 2 +- docker-compose.yml | 2 + docs/ICD/enums.md | 6 +- docs/ICD/extensions.md | 108 +++- docs/ROADMAP.md | 18 +- nginx.conf | 5 +- server/database/migrations/016_packages.sql | 37 ++ .../migrations/sqlite/016_packages.sql | 26 + server/go.mod | 3 +- server/go.sum | 16 +- server/handlers/completion.go | 21 +- server/handlers/ext_db_schema.go | 215 +++++++ server/handlers/ext_db_schema_test.go | 350 +++++++++++ server/handlers/ext_tools_test.go | 166 ++++++ server/handlers/extensions.go | 12 + server/handlers/messages.go | 2 +- server/handlers/packages.go | 13 + server/handlers/resolve.go | 50 +- server/handlers/stream_loop.go | 10 + server/handlers/tool_loop.go | 138 +++++ server/main.go | 6 +- server/middleware/page_auth.go | 25 +- server/sandbox/db_module.go | 550 ++++++++++++++++++ server/sandbox/db_module_test.go | 254 ++++++++ server/sandbox/runner.go | 52 +- server/scheduler/executor.go | 3 + server/store/ext_data_iface.go | 20 + server/store/interfaces.go | 1 + server/store/postgres/ext_data.go | 55 ++ server/store/postgres/stores.go | 1 + server/store/sqlite/ext_data.go | 49 ++ server/store/sqlite/stores.go | 1 + src/js/extensions.js | 4 +- 35 files changed, 2285 insertions(+), 48 deletions(-) create mode 100644 server/handlers/ext_db_schema.go create mode 100644 server/handlers/ext_db_schema_test.go create mode 100644 server/handlers/ext_tools_test.go create mode 100644 server/sandbox/db_module.go create mode 100644 server/sandbox/db_module_test.go create mode 100644 server/store/ext_data_iface.go create mode 100644 server/store/postgres/ext_data.go create mode 100644 server/store/sqlite/ext_data.go diff --git a/.gitignore b/.gitignore index 453e3fa..3b9b116 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,6 @@ secrets/ dist/ build/ *.surface + +.claude/ +data/ diff --git a/CHANGELOG.md b/CHANGELOG.md index c088068..d848435 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,114 @@ # Changelog +## [0.29.2] — 2026-03-17 + +### Summary + +DB extensions and server-side tool execution. Starlark packages declare +namespaced database tables via `db_tables` in the manifest — created on +install, dropped on uninstall. A structured Starlark `db` module +(`query/insert/update/delete/view/list_tables`) provides namespaced +access to extension-owned tables and column-allowlisted platform views. +Server-side tool execution wires the `on_tool_call` entry point into the +completion tool loop: extensions declare `tools` in their manifest, the +tool loop dispatches matched calls to the sandbox. Three changesets +(CS0–CS3) plus docs (CS4). + +### New + +- **`db` Starlark module** — structured API for extension-owned tables. + Requires `db.read` permission (query/view/list_tables) or `db.write` + (insert/update/delete). Tables namespaced as `ext_{pkg_slug}_{name}`. + - `db.query(table, filters=None, order=None, limit=100)` → list of dicts + - `db.insert(table, row_dict)` → inserted row dict (auto-generates `id`) + - `db.update(table, id, partial_dict)` → True + - `db.delete(table, id)` → True + - `db.list_tables()` → list of logical table names for this package + - `db.view(view_name, filters=None, limit=100)` → list of dicts + (allowed views: `"users"`, `"channels"`) +- **`db_tables` manifest key** — declare extension-owned tables with + typed columns and indexes. Tables created on install, dropped on + uninstall. + ```json + { + "db_tables": { + "logs": { + "columns": {"message": "text", "user_id": "text", "count": "int"}, + "indexes": [["user_id"]] + } + } + } + ``` + Supported column types: `text`, `int`/`integer`, `real`/`float`, + `bool`/`boolean`, `timestamp`. Auto-columns: `id TEXT PRIMARY KEY`, + `created_at` (dialect-correct default). Dialect-correct DDL for both + PG (`TIMESTAMPTZ`, `BOOLEAN`) and SQLite (`TEXT`, `INTEGER`). +- **Platform views** — column-allowlisted read-only views over platform + tables accessible via `db.view(...)`: + - `ext_view_users`: `id`, `display_name`, `email` + - `ext_view_channels`: `id`, `name`, `type`, `team_id` +- **`ext_data_tables` catalog** — tracks logical→physical table mappings + per package. Powers `db.list_tables()` and uninstall cleanup. +- **Server-side tool execution** — starlark extensions declare `tools` + in the manifest. `BuildToolDefs` includes starlark-tier active package + tools alongside server tools. `CoreToolLoop` dispatches matched calls + to `executeExtensionTool` which calls the `on_tool_call` entry point. + ```json + { + "tools": [{ + "name": "search_logs", + "description": "Search extension log entries", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"] + } + }] + } + ``` + Entry point: + ```python + def on_tool_call(call): + # call = {"tool_name": "...", "tool_call_id": "...", "arguments": {...}} + if call["tool_name"] == "search_logs": + rows = db.query("logs", filters={"user_id": call["arguments"]["user_id"]}) + return {"results": rows} + return {"error": "unknown tool"} + ``` +- **`BuildExtToolMap`** — pre-loads `map[toolName]*PackageRegistration` + per request to avoid DB lookup on each tool dispatch. +- **`ExtDataStore`** interface + implementations (`postgres`, `sqlite`) — + `RegisterTable`, `ListTables`, `DeletePackageTables`. +- **`db.read` and `db.write` permissions** — active in this release + (previously listed as "future"). + +### Changed + +- `LoopConfig` gains `Runner *sandbox.Runner` and + `ExtTools map[string]*store.PackageRegistration` for extension tool + dispatch. All callers updated: `streamCompletion`, `syncCompletion`, + multi-model path, `messages.go` regen, scheduler executor. +- `BuildToolDefs` (resolve.go) now includes starlark-tier active package + tools in addition to browser-tier tools. +- `streamWithToolLoop` and `streamModelResponse` accept runner and + extTools parameters. +- `CompletionHandler` gains `SetRunner(r *sandbox.Runner)` and + `buildExtToolMap` helpers. +- `AdminInstallExtension` and `AdminUninstallExtension` call + `CreateExtTables`/`DropExtTables` for schema lifecycle. +- `InstallPackage` and `DeletePackage` (packages.go) call the same + schema lifecycle hooks. +- Migration `016_packages.sql` extended with `ext_data_tables` catalog + table, `idx_ext_data_tables_pkg` index, and `ext_view_users`/ + `ext_view_channels` platform views. +- `Runner.SetDB(db, isPostgres)` wires the DB handle for `buildModules`. + `main.go` calls `starlarkRunner.SetDB` at startup. +- Extension permissions table in docs updated: `db.read` and `db.write` + now listed as `v0.29.2` (previously "future"). +- Extension tiers in enums.md updated: `starlark` now implemented. + +--- + ## [0.29.1] — 2026-03-17 ### Summary diff --git a/VERSION b/VERSION index 25939d3..20f0687 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.29.1 +0.29.2 diff --git a/docker-compose.yml b/docker-compose.yml index 70fb8ef..91c2e39 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,10 +24,12 @@ services: DB_DRIVER: sqlite DATABASE_URL: /data/switchboard.db JWT_SECRET: ${JWT_SECRET:-change-me-for-production} + ENCRYPTION_KEY: ${ENCRYPTION_KEY:-change-me-for-production} SWITCHBOARD_ADMIN_USERNAME: ${ADMIN_USERNAME:-admin} SWITCHBOARD_ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin} STORAGE_BACKEND: pvc STORAGE_PATH: /data/storage + CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-http://localhost:3000} volumes: - ./data:/data ports: diff --git a/docs/ICD/enums.md b/docs/ICD/enums.md index 1527e6c..f5b447c 100644 --- a/docs/ICD/enums.md +++ b/docs/ICD/enums.md @@ -110,7 +110,7 @@ All enum values used across the API. Definitive source of truth. ## Extension Tiers -`browser` (implemented), `starlark` (future), `sidecar` (future) +`browser` (implemented), `starlark` (implemented, v0.29.0), `sidecar` (future) ## Grant Types @@ -266,8 +266,8 @@ Declared in package manifests, granted by admin review. | `filters.pre_completion` | — | v0.29.0 | | `api.http` | `http` | v0.29.0 | | `provider.complete` | `provider` | v0.29.1 | -| `db.read` | `db` | future | -| `db.write` | `db` | future | +| `db.read` | `db` | v0.29.2 | +| `db.write` | `db` | v0.29.2 | ## Policies diff --git a/docs/ICD/extensions.md b/docs/ICD/extensions.md index bdb92f4..0268642 100644 --- a/docs/ICD/extensions.md +++ b/docs/ICD/extensions.md @@ -143,8 +143,8 @@ Admin must grant each before the package activates. | `filters.pre_completion` | — | Register pre-completion filter | | `api.http` | `http` | Outbound HTTP requests (v0.29.0: module, v0.29.1: also required for API routes) | | `provider.complete` | `provider` | LLM completion calls via BYOK chain (v0.29.1) | -| `db.read` | `db` | Read extension tables (future) | -| `db.write` | `db` | Write extension tables (future) | +| `db.read` | `db` | Read extension tables and platform views (v0.29.2) | +| `db.write` | `db` | Write extension tables (v0.29.2) | ### Starlark Modules (v0.29.0+) @@ -171,6 +171,110 @@ Modules injected into the script namespace based on granted permissions: - Response: `{"content", "model", "finish_reason", "input_tokens", "output_tokens"}` - Provider resolved via BYOK chain; pinnable via `requires_provider.provider_config_id` +**`db`** (requires `db.read` or `db.write`, v0.29.2): +- `db.query(table, filters=None, order=None, limit=100)` → list of dicts + - `table`: logical name (physical: `ext_{pkg_slug}_{table}`) + - `filters`: dict of `{column: value}` equality filters (optional) + - `order`: `"col"` or `"-col"` for descending (optional) + - `limit`: max rows (default 100) +- `db.insert(table, row_dict)` → inserted row dict with auto-generated `id` (`db.write`) +- `db.update(table, id, partial_dict)` → True (`db.write`) +- `db.delete(table, id)` → True (`db.write`) +- `db.list_tables()` → list of logical table names for this package +- `db.view(view_name, filters=None, limit=100)` → list of dicts + - Allowed view names: `"users"` → `ext_view_users`, `"channels"` → `ext_view_channels` + - `ext_view_users`: `id`, `display_name`, `email` + - `ext_view_channels`: `id`, `title`, `type`, `team_id` + +### Extension Database Tables (v0.29.2) + +Starlark packages declare owned tables in `manifest.db_tables`. Tables +are created on install and dropped on uninstall. + +**Manifest `db_tables` field:** + +```json +{ + "db_tables": { + "logs": { + "columns": { + "message": "text", + "user_id": "text", + "count": "int", + "score": "real", + "active": "bool", + "created_at": "timestamp" + }, + "indexes": [ + ["user_id"], + ["user_id", "created_at"] + ] + } + } +} +``` + +- **Physical name**: `ext_{pkg_slug}_{logical_name}` (hyphens → underscores) +- **Auto-columns**: `id TEXT PRIMARY KEY` (UUID generated on insert), + `created_at` (dialect-correct timestamp default) +- **Column types**: `text`, `int`/`integer`, `real`/`float`, + `bool`/`boolean`, `timestamp` — mapped to dialect-correct SQL +- **Indexes**: each entry is a list of columns for a composite index +- **Dialect**: PG uses `TIMESTAMPTZ`/`BOOLEAN`; SQLite uses `TEXT`/`INTEGER` +- **Catalog**: tracked in `ext_data_tables` per package for lifecycle management + +### Extension Tools (v0.29.2) + +Starlark packages declare server-side tools in `manifest.tools`. These +are included in `BuildToolDefs` alongside server tools. The completion +tool loop dispatches matched calls to the `on_tool_call` entry point. + +**Manifest `tools` field (starlark tier only):** + +```json +{ + "tier": "starlark", + "permissions": ["db.read"], + "tools": [ + { + "name": "search_logs", + "description": "Search extension log entries", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "user_id": {"type": "string"} + }, + "required": ["query"] + } + } + ] +} +``` + +**`on_tool_call(call)` entry point:** + +Called by the completion tool loop when a tool declared in `tools` is +invoked by the LLM. + +```python +def on_tool_call(call): + # call dict: + # { + # "tool_name": "search_logs", + # "tool_call_id": "call_abc123", + # "arguments": {"query": "hello", "user_id": "u1"} + # } + if call["tool_name"] == "search_logs": + rows = db.query("logs", filters={"user_id": call["arguments"]["user_id"]}) + return {"results": rows} + return {"error": "unknown tool"} +``` + +- Return value is serialized to JSON and returned as the tool result +- All sandbox modules (including `db`) are available per granted permissions +- No additional permission is required beyond package being `active` + ### Builtin Seeding On startup, `SeedBuiltinExtensions()` scans `extensions/builtin/` diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 54fc1c3..deeb591 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -32,7 +32,7 @@ v0.9.x–v0.28.7 Foundation through Platform Polish ✅ │ │ v0.29.0 Starlark Sandbox ✅ v0.32.0 Multi-Replica HA v0.29.1 API Extensions ✅ v0.33.0 Observability - v0.29.2 DB Extensions v0.34.0 Data Portability + v0.29.2 DB Extensions ✅ v0.34.0 Data Portability v0.29.3 Workflow Forms │ v0.30.0 Package Lifecycle │ v0.30.1 SDK Adoption │ @@ -188,17 +188,19 @@ Depends on: v0.29.0. - Server-side tool execution in completion handler (requires tool registry integration with sandbox; aligns with DB extensions scope) -### v0.29.2 — DB Extensions +### v0.29.2 — DB Extensions ✅ -Namespaced tables for extension data. Create-only (no migrations yet). +Namespaced tables for extension data. Structured API, not raw SQL. +Server-side tool execution (deferred from v0.29.1) included. Depends on: v0.29.1. -- [ ] `ext_{id}_*` tables, dialect-correct DDL (PG + SQLite) -- [ ] `db` Starlark module: `query()`, `exec()` scoped to extension tables -- [ ] Views as read contract over platform tables (column allowlist) -- [ ] Schema creation on install, drop on uninstall -- [ ] Server-side tool execution in completion handler (deferred from v0.29.1) +- [x] `ext_{id}_*` tables, dialect-correct DDL (PG + SQLite) +- [x] `db` Starlark module: structured `query/insert/update/delete/list_tables/view` + (structured API instead of raw `exec()` — prevents SQL injection) +- [x] Views as read contract over platform tables (`ext_view_users`, `ext_view_channels`) +- [x] Schema creation on install, drop on uninstall +- [x] Server-side tool execution in completion handler (deferred from v0.29.1) ### v0.29.3 — Workflow Forms diff --git a/nginx.conf b/nginx.conf index bc43e55..5499f10 100644 --- a/nginx.conf +++ b/nginx.conf @@ -86,8 +86,9 @@ server { # v0.27.0: Extension surface static assets (JS, CSS, images) # v0.28.7: On-disk path moved to /data/packages/, URL retained for stability. - location /surfaces/ { - alias /data/packages/; + # v0.29.2: ^~ prefix beats regex; corrected alias to /data/storage/packages/. + location ^~ /surfaces/ { + alias /data/storage/packages/; expires 1h; add_header Cache-Control "public"; } diff --git a/server/database/migrations/016_packages.sql b/server/database/migrations/016_packages.sql index ee31681..c1e97d0 100644 --- a/server/database/migrations/016_packages.sql +++ b/server/database/migrations/016_packages.sql @@ -7,6 +7,7 @@ -- -- v0.28.7: packages table + package_user_settings in 012. -- v0.29.0: status column + extension_permissions table. +-- v0.29.2: ext_data_tables catalog + platform read views. -- ========================================== CREATE TABLE IF NOT EXISTS packages ( @@ -88,3 +89,39 @@ CREATE INDEX IF NOT EXISTS idx_ext_perm_granted ON extension_permissions(granted COMMENT ON TABLE extension_permissions IS 'Declared permissions from package manifests. Admin grants control runtime module injection.'; COMMENT ON COLUMN extension_permissions.permission IS 'Capability key: secrets.read, notifications.send, filters.pre_completion, db.read, db.write, api.http'; COMMENT ON COLUMN extension_permissions.granted IS 'Admin has reviewed and approved this capability for the package'; + + +-- ========================================= +-- EXTENSION DATA TABLES CATALOG +-- ========================================= +-- Tracks namespaced tables created for each extension on install. +-- Used by uninstall hooks (DROP TABLE) and db.list_tables(). +-- Table names stored without the ext_{id}_ prefix. +-- v0.29.2: Extension data storage. + +CREATE TABLE IF NOT EXISTS ext_data_tables ( + package_id TEXT NOT NULL REFERENCES packages(id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (package_id, table_name) +); + +CREATE INDEX IF NOT EXISTS idx_ext_data_tables_pkg ON ext_data_tables(package_id); + +COMMENT ON TABLE ext_data_tables IS 'Catalog of namespaced tables created by extension install hooks. Drives uninstall cleanup and db.list_tables().'; +COMMENT ON COLUMN ext_data_tables.table_name IS 'Logical name (without ext_{id}_ prefix). Physical table is ext_{package_id}_{table_name}.'; + + +-- ========================================= +-- PLATFORM VIEWS (extension read access) +-- ========================================= +-- Column-allowlisted views over core platform tables. +-- Extensions query these via db.view("users"), db.view("channels"). +-- Only safe, non-sensitive columns are exposed. +-- v0.29.2: Read contract for cross-platform data access. + +CREATE OR REPLACE VIEW ext_view_users AS + SELECT id, display_name, email FROM users; + +CREATE OR REPLACE VIEW ext_view_channels AS + SELECT id, title, type, team_id FROM channels; diff --git a/server/database/migrations/sqlite/016_packages.sql b/server/database/migrations/sqlite/016_packages.sql index 75068dc..f861ae6 100644 --- a/server/database/migrations/sqlite/016_packages.sql +++ b/server/database/migrations/sqlite/016_packages.sql @@ -2,6 +2,7 @@ -- Unified package registry. Replaces surface_registry + extensions. -- v0.28.7: packages + package_user_settings -- v0.29.0: status column + extension_permissions table +-- v0.29.2: ext_data_tables catalog + platform read views CREATE TABLE IF NOT EXISTS packages ( id TEXT PRIMARY KEY, @@ -61,3 +62,28 @@ CREATE TABLE IF NOT EXISTS extension_permissions ( CREATE INDEX IF NOT EXISTS idx_ext_perm_package ON extension_permissions(package_id); CREATE INDEX IF NOT EXISTS idx_ext_perm_granted ON extension_permissions(granted); + + +-- Extension data tables catalog (v0.29.2) +-- Tracks namespaced tables created by extension install hooks. + +CREATE TABLE IF NOT EXISTS ext_data_tables ( + package_id TEXT NOT NULL REFERENCES packages(id) ON DELETE CASCADE, + table_name TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (package_id, table_name) +); + +CREATE INDEX IF NOT EXISTS idx_ext_data_tables_pkg ON ext_data_tables(package_id); + + +-- Platform views — column-allowlisted read access for extensions (v0.29.2) +-- Extensions query these via db.view("users"), db.view("channels"). + +DROP VIEW IF EXISTS ext_view_users; +CREATE VIEW ext_view_users AS + SELECT id, display_name, email FROM users; + +DROP VIEW IF EXISTS ext_view_channels; +CREATE VIEW ext_view_channels AS + SELECT id, title, type, team_id FROM channels; diff --git a/server/go.mod b/server/go.mod index 2d9ae87..1721abe 100644 --- a/server/go.mod +++ b/server/go.mod @@ -11,6 +11,7 @@ require ( github.com/lib/pq v1.10.9 github.com/minio/minio-go/v7 v7.0.82 github.com/robfig/cron/v3 v3.0.1 + go.starlark.net v0.0.0-20260210143700-b62fd896b91b golang.org/x/crypto v0.28.0 golang.org/x/net v0.30.0 modernc.org/sqlite v1.34.5 @@ -44,7 +45,7 @@ require ( golang.org/x/arch v0.3.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect diff --git a/server/go.sum b/server/go.sum index de7c2b3..97f2a0a 100644 --- a/server/go.sum +++ b/server/go.sum @@ -29,7 +29,6 @@ github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -72,6 +71,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -90,6 +91,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +go.starlark.net v0.0.0-20260210143700-b62fd896b91b h1:mDO9/2PuBcapqFbhiCmFcEQZvlQnk3ILEZR+a8NL1z4= +go.starlark.net v0.0.0-20260210143700-b62fd896b91b/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= @@ -105,15 +108,16 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/server/handlers/completion.go b/server/handlers/completion.go index 04b670c..bc7af31 100644 --- a/server/handlers/completion.go +++ b/server/handlers/completion.go @@ -25,6 +25,7 @@ import ( "git.gobha.me/xcaliber/chat-switchboard/notifications" "git.gobha.me/xcaliber/chat-switchboard/providers" "git.gobha.me/xcaliber/chat-switchboard/routing" + "git.gobha.me/xcaliber/chat-switchboard/sandbox" "git.gobha.me/xcaliber/chat-switchboard/storage" "git.gobha.me/xcaliber/chat-switchboard/store" "git.gobha.me/xcaliber/chat-switchboard/tools" @@ -57,6 +58,7 @@ type CompletionHandler struct { healthStore HealthStatusQuerier // health status queries for routing (v0.22.2, nil = disabled) router *routing.Evaluator // routing policy evaluator (v0.22.2, nil = disabled) filterChain *filters.Chain // pre-completion filter chain (v0.29.0, nil = disabled) + runner *sandbox.Runner // v0.29.2: Starlark extension tool dispatch (nil = disabled) } // HealthRecorder is the interface for recording provider call outcomes. @@ -101,6 +103,16 @@ func (h *CompletionHandler) SetFilterChain(fc *filters.Chain) { h.filterChain = fc } +// SetRunner attaches the Starlark sandbox runner for extension tool dispatch (v0.29.2). +func (h *CompletionHandler) SetRunner(r *sandbox.Runner) { + h.runner = r +} + +// buildExtToolMap returns the extension tool map for the current user. +func (h *CompletionHandler) buildExtToolMap(ctx context.Context, userID string) map[string]*store.PackageRegistration { + return BuildExtToolMap(ctx, h.stores, userID) +} + // evaluateRouting applies routing policies to select the best provider config // for this request. Returns the winning config details and a routing decision // for observability. If routing is disabled or no policies match, returns @@ -846,7 +858,8 @@ func (h *CompletionHandler) multiModelStream( } // Stream this model's response using the shared streaming core - result := streamModelResponse(c, provider, providerCfg, &provReq, model, providerID, target.DisplayName, userID, channelID, personaID, workspaceID, configID, teamID, h.hub, h.health) + multiExtTools := h.buildExtToolMap(c.Request.Context(), userID) + result := streamModelResponse(c, provider, providerCfg, &provReq, model, providerID, target.DisplayName, userID, channelID, personaID, workspaceID, configID, teamID, h.hub, h.health, h.runner, multiExtTools) // Persist assistant message with model attribution if result.Content != "" { @@ -955,7 +968,8 @@ func (h *CompletionHandler) streamCompletion( hooks.PreRequest(cfg, &req) } - result := streamWithToolLoop(c, provider, cfg, &req, model, providerID, userID, channelID, personaID, workspaceID, configID, teamID, h.hub, h.health) + extTools := h.buildExtToolMap(c.Request.Context(), userID) + result := streamWithToolLoop(c, provider, cfg, &req, model, providerID, userID, channelID, personaID, workspaceID, configID, teamID, h.hub, h.health, h.runner, extTools) // Persist assistant response if result.Content != "" { @@ -995,6 +1009,7 @@ func (h *CompletionHandler) syncCompletion( hooks.PreRequest(cfg, &req) } + extTools := h.buildExtToolMap(c.Request.Context(), userID) result := CoreToolLoop(c.Request.Context(), LoopConfig{ Provider: provider, Cfg: cfg, @@ -1013,6 +1028,8 @@ func (h *CompletionHandler) syncCompletion( ConfigID: configID, Budget: LoopBudget{}, Streaming: false, + Runner: h.runner, + ExtTools: extTools, }, accumSink{}) // Provider error diff --git a/server/handlers/ext_db_schema.go b/server/handlers/ext_db_schema.go new file mode 100644 index 0000000..709b1b1 --- /dev/null +++ b/server/handlers/ext_db_schema.go @@ -0,0 +1,215 @@ +package handlers + +// ext_db_schema.go — v0.29.2 +// +// DDL generation and lifecycle management for extension-owned database tables. +// Tables are namespaced as ext_{pkg_slug}_{logical_name} and tracked in the +// ext_data_tables catalog. CreateExtTables is called on install; +// DropExtTables is called on uninstall. + +import ( + "context" + "database/sql" + "fmt" + "log" + "regexp" + "strings" + + "git.gobha.me/xcaliber/chat-switchboard/store" +) + +// validSchemaIdentifier matches safe SQL identifiers for table and column names. +var validSchemaIdentifier = regexp.MustCompile(`^[a-z][a-z0-9_]{0,62}$`) + +// TableDef describes a single extension-owned table as declared in a manifest. +type TableDef struct { + Columns map[string]string // colName → manifest type ("text","int","real","bool","timestamp") + Indexes [][]string // each inner slice is an ordered list of column names for one index +} + +// ParseDBTables extracts the "db_tables" key from a package manifest map. +// Returns the map of logicalName→TableDef and ok=true when the key is present +// and produces at least one valid table definition. +func ParseDBTables(manifest map[string]any) (map[string]TableDef, bool) { + raw, ok := manifest["db_tables"] + if !ok { + return nil, false + } + tablesRaw, ok := raw.(map[string]any) + if !ok || len(tablesRaw) == 0 { + return nil, false + } + + tables := make(map[string]TableDef, len(tablesRaw)) + for name, defRaw := range tablesRaw { + if !validSchemaIdentifier.MatchString(name) { + log.Printf("[ext-db] skipping invalid table name %q", name) + continue + } + defMap, ok := defRaw.(map[string]any) + if !ok { + continue + } + + td := TableDef{Columns: map[string]string{}} + + // Parse columns map. + if colsRaw, ok := defMap["columns"].(map[string]any); ok { + for col, typ := range colsRaw { + if !validSchemaIdentifier.MatchString(col) { + log.Printf("[ext-db] skipping invalid column name %q in table %q", col, name) + continue + } + if t, ok := typ.(string); ok { + td.Columns[col] = t + } + } + } + + // Parse indexes array. + if idxsRaw, ok := defMap["indexes"].([]any); ok { + for _, idxRaw := range idxsRaw { + if idxArr, ok := idxRaw.([]any); ok { + var cols []string + for _, c := range idxArr { + if s, ok := c.(string); ok && validSchemaIdentifier.MatchString(s) { + cols = append(cols, s) + } + } + if len(cols) > 0 { + td.Indexes = append(td.Indexes, cols) + } + } + } + } + + tables[name] = td + } + + if len(tables) == 0 { + return nil, false + } + return tables, true +} + +// mapColType maps a manifest type string to a SQL column type for the given dialect. +func mapColType(typStr string, isPostgres bool) string { + switch strings.ToLower(typStr) { + case "int", "integer": + return "INTEGER" + case "real", "float": + return "REAL" + case "bool", "boolean": + if isPostgres { + return "BOOLEAN" + } + return "INTEGER" + case "timestamp": + if isPostgres { + return "TIMESTAMPTZ" + } + return "TEXT" + default: // "text" and anything unrecognized + return "TEXT" + } +} + +// extPhysicalTable builds the namespaced physical table name for a package's logical table. +// e.g. packageID="my-ext", logicalName="logs" → "ext_my_ext_logs" +func extPhysicalTable(packageID, logicalName string) string { + slug := strings.ReplaceAll(packageID, "-", "_") + return "ext_" + slug + "_" + logicalName +} + +// createdAtColDef returns the dialect-correct DDL fragment for the created_at column. +func createdAtColDef(isPostgres bool) string { + if isPostgres { + return "created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()" + } + return "created_at TEXT NOT NULL DEFAULT (datetime('now'))" +} + +// CreateExtTables generates dialect-correct DDL and executes it for all tables +// in the provided map. Each successfully created table is registered in the +// ext_data_tables catalog via stores.ExtData. +// +// Errors from individual table creation are returned immediately (fail-fast). +// Index creation failures are logged but non-fatal. +// Catalog registration failures are logged but non-fatal. +func CreateExtTables( + ctx context.Context, + db *sql.DB, + isPostgres bool, + stores store.Stores, + packageID string, + tables map[string]TableDef, +) error { + if db == nil { + return nil + } + for logicalName, td := range tables { + physical := extPhysicalTable(packageID, logicalName) + + // Build column list: id first, user columns, created_at last. + cols := []string{"id TEXT PRIMARY KEY"} + for col, typ := range td.Columns { + cols = append(cols, col+" "+mapColType(typ, isPostgres)) + } + cols = append(cols, createdAtColDef(isPostgres)) + + ddl := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n\t%s\n)", + physical, strings.Join(cols, ",\n\t")) + + if _, err := db.ExecContext(ctx, ddl); err != nil { + return fmt.Errorf("create ext table %s: %w", physical, err) + } + + // Create declared indexes. + for i, idxCols := range td.Indexes { + idxName := fmt.Sprintf("idx_%s_%d", physical, i) + idxDDL := fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (%s)", + idxName, physical, strings.Join(idxCols, ", ")) + if _, err := db.ExecContext(ctx, idxDDL); err != nil { + log.Printf("[ext-db] index create failed (%s): %v", idxName, err) + } + } + + // Record logical name in catalog. + if stores.ExtData != nil { + if err := stores.ExtData.RegisterTable(ctx, packageID, logicalName); err != nil { + log.Printf("[ext-db] catalog register failed (%s/%s): %v", packageID, logicalName, err) + } + } + + log.Printf("[ext-db] created table %s for package %s", physical, packageID) + } + return nil +} + +// DropExtTables drops all physical tables registered for a package and removes +// their entries from the ext_data_tables catalog. +// +// Table drop errors are logged but non-fatal; the catalog is cleared regardless. +func DropExtTables( + ctx context.Context, + db *sql.DB, + stores store.Stores, + packageID string, +) error { + if db == nil || stores.ExtData == nil { + return nil + } + tables, err := stores.ExtData.ListTables(ctx, packageID) + if err != nil { + return fmt.Errorf("list ext tables for %s: %w", packageID, err) + } + for _, logicalName := range tables { + physical := extPhysicalTable(packageID, logicalName) + if _, err := db.ExecContext(ctx, "DROP TABLE IF EXISTS "+physical); err != nil { + log.Printf("[ext-db] drop table %s failed: %v", physical, err) + } else { + log.Printf("[ext-db] dropped table %s for package %s", physical, packageID) + } + } + return stores.ExtData.DeletePackageTables(ctx, packageID) +} diff --git a/server/handlers/ext_db_schema_test.go b/server/handlers/ext_db_schema_test.go new file mode 100644 index 0000000..7be3d55 --- /dev/null +++ b/server/handlers/ext_db_schema_test.go @@ -0,0 +1,350 @@ +package handlers + +import ( + "context" + "database/sql" + "strings" + "testing" + + _ "modernc.org/sqlite" + + "git.gobha.me/xcaliber/chat-switchboard/store" +) + +// ── mock ExtDataStore ──────────────────────────────────────────────────────── + +// memExtDataStore is an in-memory ExtDataStore for testing. +type memExtDataStore struct { + tables map[string][]string // packageID → []logicalName +} + +func newMemExtDataStore() *memExtDataStore { + return &memExtDataStore{tables: map[string][]string{}} +} + +func (m *memExtDataStore) RegisterTable(_ context.Context, packageID, tableName string) error { + m.tables[packageID] = append(m.tables[packageID], tableName) + return nil +} + +func (m *memExtDataStore) ListTables(_ context.Context, packageID string) ([]string, error) { + return m.tables[packageID], nil +} + +func (m *memExtDataStore) DeletePackageTables(_ context.Context, packageID string) error { + delete(m.tables, packageID) + return nil +} + +// ── helpers ────────────────────────────────────────────────────────────────── + +func newSchemaTestDB(t *testing.T) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open db: %v", err) + } + t.Cleanup(func() { db.Close() }) + return db +} + +func storesWithExtData(m *memExtDataStore) store.Stores { + return store.Stores{ExtData: m} +} + +// ── ParseDBTables ──────────────────────────────────────────────────────────── + +func TestParseDBTables_Valid(t *testing.T) { + manifest := map[string]any{ + "db_tables": map[string]any{ + "logs": map[string]any{ + "columns": map[string]any{ + "message": "text", + "count": "int", + }, + "indexes": []any{[]any{"message"}}, + }, + }, + } + tables, ok := ParseDBTables(manifest) + if !ok { + t.Fatal("expected ok=true") + } + if _, has := tables["logs"]; !has { + t.Fatal("expected 'logs' in parsed tables") + } + if tables["logs"].Columns["message"] != "text" { + t.Errorf("expected column message=text, got %q", tables["logs"].Columns["message"]) + } + if len(tables["logs"].Indexes) != 1 { + t.Errorf("expected 1 index, got %d", len(tables["logs"].Indexes)) + } +} + +func TestParseDBTables_Missing(t *testing.T) { + _, ok := ParseDBTables(map[string]any{"title": "hi"}) + if ok { + t.Fatal("expected ok=false when db_tables absent") + } +} + +func TestParseDBTables_InvalidTableName(t *testing.T) { + manifest := map[string]any{ + "db_tables": map[string]any{ + "bad name": map[string]any{}, + "good_name": map[string]any{"columns": map[string]any{"x": "text"}}, + }, + } + tables, ok := ParseDBTables(manifest) + if !ok { + t.Fatal("expected ok=true (good_name is valid)") + } + if _, has := tables["bad name"]; has { + t.Error("invalid table name should be skipped") + } + if _, has := tables["good_name"]; !has { + t.Error("valid table name should be present") + } +} + +func TestParseDBTables_InvalidColumnName(t *testing.T) { + manifest := map[string]any{ + "db_tables": map[string]any{ + "events": map[string]any{ + "columns": map[string]any{ + "bad col": "text", + "ok_col": "text", + }, + }, + }, + } + tables, ok := ParseDBTables(manifest) + if !ok { + t.Fatal("expected ok=true") + } + if _, has := tables["events"].Columns["bad col"]; has { + t.Error("invalid column name should be skipped") + } + if _, has := tables["events"].Columns["ok_col"]; !has { + t.Error("valid column name should be present") + } +} + +// ── mapColType ─────────────────────────────────────────────────────────────── + +func TestMapColType_SQLite(t *testing.T) { + cases := []struct{ in, want string }{ + {"text", "TEXT"}, + {"int", "INTEGER"}, + {"integer", "INTEGER"}, + {"real", "REAL"}, + {"float", "REAL"}, + {"bool", "INTEGER"}, + {"boolean", "INTEGER"}, + {"timestamp", "TEXT"}, + {"unknown", "TEXT"}, + } + for _, c := range cases { + got := mapColType(c.in, false) + if got != c.want { + t.Errorf("mapColType(%q, sqlite) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestMapColType_Postgres(t *testing.T) { + if mapColType("bool", true) != "BOOLEAN" { + t.Error("expected BOOLEAN for bool in postgres") + } + if mapColType("timestamp", true) != "TIMESTAMPTZ" { + t.Error("expected TIMESTAMPTZ for timestamp in postgres") + } + if mapColType("int", true) != "INTEGER" { + t.Error("expected INTEGER for int in postgres") + } +} + +// ── extPhysicalTable ───────────────────────────────────────────────────────── + +func TestExtPhysicalTable(t *testing.T) { + got := extPhysicalTable("my-ext", "logs") + if got != "ext_my_ext_logs" { + t.Errorf("got %q, want ext_my_ext_logs", got) + } + got = extPhysicalTable("simple", "events") + if got != "ext_simple_events" { + t.Errorf("got %q, want ext_simple_events", got) + } +} + +// ── CreateExtTables ────────────────────────────────────────────────────────── + +func TestCreateExtTables_CreatesTable(t *testing.T) { + db := newSchemaTestDB(t) + m := newMemExtDataStore() + ctx := context.Background() + + tables := map[string]TableDef{ + "logs": { + Columns: map[string]string{"message": "text", "count": "int"}, + }, + } + if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "test-pkg", tables); err != nil { + t.Fatalf("CreateExtTables: %v", err) + } + + // Verify table exists by inserting a row. + _, err := db.ExecContext(ctx, + `INSERT INTO ext_test_pkg_logs (id, message, count) VALUES ('1', 'hello', 42)`) + if err != nil { + t.Errorf("table not created or wrong schema: %v", err) + } +} + +func TestCreateExtTables_WithIndexes(t *testing.T) { + db := newSchemaTestDB(t) + m := newMemExtDataStore() + ctx := context.Background() + + tables := map[string]TableDef{ + "events": { + Columns: map[string]string{"user_id": "text", "kind": "text"}, + Indexes: [][]string{{"user_id"}, {"user_id", "kind"}}, + }, + } + if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "idx-pkg", tables); err != nil { + t.Fatalf("CreateExtTables: %v", err) + } + + // Verify indexes exist via sqlite_master. + rows, err := db.QueryContext(ctx, + `SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='ext_idx_pkg_events'`) + if err != nil { + t.Fatalf("query indexes: %v", err) + } + defer rows.Close() + var idxNames []string + for rows.Next() { + var name string + rows.Scan(&name) + idxNames = append(idxNames, name) + } + if len(idxNames) < 2 { + t.Errorf("expected at least 2 indexes, got %d: %v", len(idxNames), idxNames) + } +} + +func TestCreateExtTables_CatalogRegistration(t *testing.T) { + db := newSchemaTestDB(t) + m := newMemExtDataStore() + ctx := context.Background() + + tables := map[string]TableDef{ + "logs": {Columns: map[string]string{"msg": "text"}}, + "errors": {Columns: map[string]string{"detail": "text"}}, + } + if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "cat-pkg", tables); err != nil { + t.Fatalf("CreateExtTables: %v", err) + } + + registered := m.tables["cat-pkg"] + if len(registered) != 2 { + t.Errorf("expected 2 registered tables, got %d: %v", len(registered), registered) + } +} + +func TestCreateExtTables_NilDBIsNoop(t *testing.T) { + m := newMemExtDataStore() + ctx := context.Background() + // Should not panic or error. + err := CreateExtTables(ctx, nil, false, storesWithExtData(m), "pkg", map[string]TableDef{ + "t": {Columns: map[string]string{"x": "text"}}, + }) + if err != nil { + t.Errorf("expected nil error for nil db, got: %v", err) + } +} + +// ── DropExtTables ───────────────────────────────────────────────────────────── + +func TestDropExtTables_DropsAndClearsCatalog(t *testing.T) { + db := newSchemaTestDB(t) + m := newMemExtDataStore() + ctx := context.Background() + + // Create table first. + tables := map[string]TableDef{ + "items": {Columns: map[string]string{"name": "text"}}, + } + if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "drop-pkg", tables); err != nil { + t.Fatalf("CreateExtTables: %v", err) + } + + // Drop. + if err := DropExtTables(ctx, db, storesWithExtData(m), "drop-pkg"); err != nil { + t.Fatalf("DropExtTables: %v", err) + } + + // Table should be gone. + var count int + db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='ext_drop_pkg_items'`, + ).Scan(&count) + if count != 0 { + t.Error("expected table to be dropped, still present") + } + + // Catalog should be empty. + if len(m.tables["drop-pkg"]) != 0 { + t.Errorf("expected empty catalog, got: %v", m.tables["drop-pkg"]) + } +} + +func TestDropExtTables_MultipleTables(t *testing.T) { + db := newSchemaTestDB(t) + m := newMemExtDataStore() + ctx := context.Background() + + tables := map[string]TableDef{ + "alpha": {Columns: map[string]string{"v": "text"}}, + "beta": {Columns: map[string]string{"v": "text"}}, + } + CreateExtTables(ctx, db, false, storesWithExtData(m), "multi-pkg", tables) + DropExtTables(ctx, db, storesWithExtData(m), "multi-pkg") + + for _, name := range []string{"ext_multi_pkg_alpha", "ext_multi_pkg_beta"} { + var count int + db.QueryRowContext(ctx, + `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?`, name, + ).Scan(&count) + if count != 0 { + t.Errorf("expected %s to be dropped", name) + } + } +} + +func TestCreateExtTables_DDLContainsAutoColumns(t *testing.T) { + // Verify that the DDL-generated table has 'id' and 'created_at' columns + // even when no columns are declared in the manifest. + db := newSchemaTestDB(t) + m := newMemExtDataStore() + ctx := context.Background() + + tables := map[string]TableDef{"empty": {Columns: map[string]string{}}} + if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "auto-pkg", tables); err != nil { + t.Fatalf("CreateExtTables: %v", err) + } + + // Insert with just id — should work (created_at has default). + _, err := db.ExecContext(ctx, `INSERT INTO ext_auto_pkg_empty (id) VALUES ('x')`) + if err != nil { + t.Errorf("expected auto-columns (id, created_at) to exist: %v", err) + } + + // Verify created_at was populated. + var ca string + db.QueryRowContext(ctx, `SELECT created_at FROM ext_auto_pkg_empty WHERE id='x'`).Scan(&ca) + if strings.TrimSpace(ca) == "" { + t.Error("expected created_at to be populated by default") + } +} diff --git a/server/handlers/ext_tools_test.go b/server/handlers/ext_tools_test.go new file mode 100644 index 0000000..b4638af --- /dev/null +++ b/server/handlers/ext_tools_test.go @@ -0,0 +1,166 @@ +package handlers + +import ( + "encoding/json" + "testing" + + "go.starlark.net/starlark" +) + +// ── jsonToStarlark ──────────────────────────────────────────────────────────── + +func TestJSONToStarlark_String(t *testing.T) { + v := jsonToStarlark("hello") + s, ok := v.(starlark.String) + if !ok || string(s) != "hello" { + t.Errorf("expected starlark.String(hello), got %v (%T)", v, v) + } +} + +func TestJSONToStarlark_Int(t *testing.T) { + v := jsonToStarlark(float64(42)) + i, ok := v.(starlark.Int) + if !ok { + t.Fatalf("expected starlark.Int, got %T", v) + } + n, _ := i.Int64() + if n != 42 { + t.Errorf("expected 42, got %d", n) + } +} + +func TestJSONToStarlark_Float(t *testing.T) { + v := jsonToStarlark(3.14) + if _, ok := v.(starlark.Float); !ok { + t.Errorf("expected starlark.Float, got %T", v) + } +} + +func TestJSONToStarlark_Bool(t *testing.T) { + if jsonToStarlark(true) != starlark.True { + t.Error("expected starlark.True") + } + if jsonToStarlark(false) != starlark.False { + t.Error("expected starlark.False") + } +} + +func TestJSONToStarlark_Nil(t *testing.T) { + if jsonToStarlark(nil) != starlark.None { + t.Error("expected starlark.None for nil") + } +} + +func TestJSONToStarlark_Dict(t *testing.T) { + v := jsonToStarlark(map[string]interface{}{"key": "val"}) + d, ok := v.(*starlark.Dict) + if !ok { + t.Fatalf("expected *starlark.Dict, got %T", v) + } + got, found, _ := d.Get(starlark.String("key")) + if !found || string(got.(starlark.String)) != "val" { + t.Errorf("expected key=val in dict, got %v", got) + } +} + +func TestJSONToStarlark_List(t *testing.T) { + v := jsonToStarlark([]interface{}{"a", "b"}) + l, ok := v.(*starlark.List) + if !ok { + t.Fatalf("expected *starlark.List, got %T", v) + } + if l.Len() != 2 { + t.Errorf("expected 2 elements, got %d", l.Len()) + } +} + +// ── starlarkValueToGo ───────────────────────────────────────────────────────── + +func TestStarlarkValueToGo_String(t *testing.T) { + v := starlarkValueToGo(starlark.String("hi")) + if v != "hi" { + t.Errorf("expected 'hi', got %v", v) + } +} + +func TestStarlarkValueToGo_Int(t *testing.T) { + v := starlarkValueToGo(starlark.MakeInt(7)) + if v != int64(7) { + t.Errorf("expected int64(7), got %v (%T)", v, v) + } +} + +func TestStarlarkValueToGo_Bool(t *testing.T) { + if starlarkValueToGo(starlark.True) != true { + t.Error("expected true") + } +} + +func TestStarlarkValueToGo_None(t *testing.T) { + if starlarkValueToGo(starlark.None) != nil { + t.Error("expected nil for starlark.None") + } +} + +func TestStarlarkValueToGo_Dict(t *testing.T) { + d := starlark.NewDict(1) + _ = d.SetKey(starlark.String("x"), starlark.MakeInt(1)) + v := starlarkValueToGo(d) + m, ok := v.(map[string]interface{}) + if !ok { + t.Fatalf("expected map, got %T", v) + } + if m["x"] != int64(1) { + t.Errorf("expected x=1, got %v", m["x"]) + } +} + +func TestStarlarkValueToGo_List(t *testing.T) { + l := starlark.NewList([]starlark.Value{starlark.String("a"), starlark.String("b")}) + v := starlarkValueToGo(l) + arr, ok := v.([]interface{}) + if !ok { + t.Fatalf("expected []interface{}, got %T", v) + } + if len(arr) != 2 || arr[0] != "a" { + t.Errorf("unexpected list: %v", arr) + } +} + +// ── RoundTrip: JSON → Starlark → Go → JSON ─────────────────────────────────── + +func TestRoundTrip_JSONToStarlarkToJSON(t *testing.T) { + // Simulate what executeExtensionTool does: parse args, pass as Starlark, convert result back. + argsJSON := `{"query": "hello", "limit": 10, "active": true}` + var raw map[string]interface{} + if err := json.Unmarshal([]byte(argsJSON), &raw); err != nil { + t.Fatalf("unmarshal: %v", err) + } + sv := jsonToStarlark(raw) + got := starlarkValueToGo(sv) + out, err := json.Marshal(got) + if err != nil { + t.Fatalf("marshal: %v", err) + } + // Verify key fields present + var result map[string]interface{} + if err := json.Unmarshal(out, &result); err != nil { + t.Fatalf("unmarshal result: %v", err) + } + if result["query"] != "hello" { + t.Errorf("expected query=hello, got %v", result["query"]) + } + if result["active"] != true { + t.Errorf("expected active=true, got %v", result["active"]) + } +} + +// ── BuildExtToolMap ─────────────────────────────────────────────────────────── + +func TestBuildExtToolMap_Empty(t *testing.T) { + // nil Packages store → returns nil (no panic) + result := BuildExtToolMap(nil, storesWithExtData(newMemExtDataStore()), "user1") + if result != nil && len(result) != 0 { + t.Errorf("expected empty map for stores without Packages, got %v", result) + } +} diff --git a/server/handlers/extensions.go b/server/handlers/extensions.go index 5b5cdeb..61be842 100644 --- a/server/handlers/extensions.go +++ b/server/handlers/extensions.go @@ -210,6 +210,13 @@ func (h *ExtensionHandler) AdminInstallExtension(c *gin.Context) { // If permissions are declared, package moves to pending_review. SyncManifestPermissions(c, h.stores, pkg.ID, manifestMap) + // v0.29.2: Create namespaced DB tables declared in the manifest. + if tables, ok := ParseDBTables(manifestMap); ok { + if err := CreateExtTables(c.Request.Context(), database.DB, database.IsPostgres(), h.stores, pkg.ID, tables); err != nil { + log.Printf("[ext-db] schema create failed for %s: %v", pkg.ID, err) + } + } + c.JSON(201, gin.H{"data": pkg}) } @@ -370,6 +377,11 @@ func (h *ExtensionHandler) AdminUninstallExtension(c *gin.Context) { return } + // v0.29.2: Drop namespaced DB tables before removing the package record. + if err := DropExtTables(c.Request.Context(), database.DB, h.stores, id); err != nil { + log.Printf("[ext-db] schema drop failed for %s: %v", id, err) + } + if err := h.stores.Packages.Delete(c.Request.Context(), id); err != nil { if err == sql.ErrNoRows { c.JSON(400, gin.H{"error": "core packages cannot be deleted"}) diff --git a/server/handlers/messages.go b/server/handlers/messages.go index b2deb87..26c6e07 100644 --- a/server/handlers/messages.go +++ b/server/handlers/messages.go @@ -478,7 +478,7 @@ func (h *MessageHandler) Regenerate(c *gin.Context) { hooks.PreRequest(providerCfg, &provReq) } - result := streamWithToolLoop(c, provider, providerCfg, &provReq, model, providerID, userID, channelID, personaID, workspaceID, configID, msgTeamID, h.hub, comp.health) + result := streamWithToolLoop(c, provider, providerCfg, &provReq, model, providerID, userID, channelID, personaID, workspaceID, configID, msgTeamID, h.hub, comp.health, comp.runner, comp.buildExtToolMap(c.Request.Context(), userID)) // Persist as sibling (regen) with tool activity if result.Content != "" { diff --git a/server/handlers/packages.go b/server/handlers/packages.go index c96984f..e5da854 100644 --- a/server/handlers/packages.go +++ b/server/handlers/packages.go @@ -13,6 +13,7 @@ import ( "github.com/gin-gonic/gin" + "git.gobha.me/xcaliber/chat-switchboard/database" "git.gobha.me/xcaliber/chat-switchboard/store" ) @@ -117,6 +118,11 @@ func (h *PackageHandler) DeletePackage(c *gin.Context) { return } + // v0.29.2: Drop namespaced DB tables before removing the package record. + if err := DropExtTables(c.Request.Context(), database.DB, h.stores, id); err != nil { + log.Printf("[ext-db] schema drop failed for %s: %v", id, err) + } + if err := h.stores.Packages.Delete(c.Request.Context(), id); err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete package"}) return @@ -358,6 +364,13 @@ func (h *PackageHandler) InstallPackage(c *gin.Context) { log.Printf("[packages] Failed to update package metadata for %s: %v", pkgID, err) } + // v0.29.2: Create namespaced DB tables declared in the manifest. + if tables, ok := ParseDBTables(manifest); ok { + if err := CreateExtTables(c.Request.Context(), database.DB, database.IsPostgres(), h.stores, pkgID, tables); err != nil { + log.Printf("[ext-db] schema create failed for %s: %v", pkgID, err) + } + } + c.JSON(http.StatusOK, gin.H{ "id": pkgID, "title": title, diff --git a/server/handlers/resolve.go b/server/handlers/resolve.go index 0824155..e58727b 100644 --- a/server/handlers/resolve.go +++ b/server/handlers/resolve.go @@ -180,15 +180,25 @@ func BuildToolDefs( }) } - // Append browser-defined tool schemas from extensions - if includeBrowser && stores.Packages != nil { + // Append browser-defined and starlark-tier tool schemas from extensions. + // v0.29.2: starlark tools are always included (not gated on browser connection). + if stores.Packages != nil { pkgs, err := stores.Packages.ListForUser(ctx, userID) if err != nil { log.Printf("⚠️ Failed to load extensions for tools: %v", err) return defs } for _, pkg := range pkgs { - if pkg.Tier != "browser" { + // Browser tools only when a browser client is connected. + if pkg.Tier == "browser" && !includeBrowser { + continue + } + // Only browser and starlark tiers expose tools this way. + if pkg.Tier != "browser" && pkg.Tier != "starlark" { + continue + } + // Starlark packages must be active to expose tools. + if pkg.Tier == "starlark" && pkg.Status != "active" { continue } var manifest struct { @@ -196,7 +206,6 @@ func BuildToolDefs( Name string `json:"name"` Description string `json:"description"` Parameters json.RawMessage `json:"parameters"` - Tier string `json:"tier"` } `json:"tools"` } if err := json.Unmarshal(marshalManifest(pkg.Manifest), &manifest); err != nil { @@ -243,6 +252,39 @@ func BuildToolDefs( return defs } +// BuildExtToolMap returns a map of toolName → PackageRegistration for all +// active starlark-tier extensions that declare tools in their manifest. +// Used by the tool loop to dispatch matched calls to on_tool_call. +func BuildExtToolMap(ctx context.Context, stores store.Stores, userID string) map[string]*store.PackageRegistration { + if stores.Packages == nil { + return nil + } + pkgs, err := stores.Packages.ListForUser(ctx, userID) + if err != nil { + return nil + } + result := make(map[string]*store.PackageRegistration) + for i, pkg := range pkgs { + if pkg.Tier != "starlark" || pkg.Status != "active" { + continue + } + var manifest struct { + Tools []struct { + Name string `json:"name"` + } `json:"tools"` + } + if json.Unmarshal(marshalManifest(pkg.Manifest), &manifest) != nil { + continue + } + for _, t := range manifest.Tools { + if t.Name != "" { + result[t.Name] = &pkgs[i].PackageRegistration + } + } + } + return result +} + // FilterToolDefsByGrants applies an additional allowlist to tool defs. // Used by the task scheduler to enforce task-level tool grants on top // of persona-level grants. Passing nil or empty grants returns defs unchanged. diff --git a/server/handlers/stream_loop.go b/server/handlers/stream_loop.go index 14e691d..5b4448c 100644 --- a/server/handlers/stream_loop.go +++ b/server/handlers/stream_loop.go @@ -13,6 +13,8 @@ import ( "git.gobha.me/xcaliber/chat-switchboard/events" "git.gobha.me/xcaliber/chat-switchboard/providers" + "git.gobha.me/xcaliber/chat-switchboard/sandbox" + "git.gobha.me/xcaliber/chat-switchboard/store" "git.gobha.me/xcaliber/chat-switchboard/tools" ) @@ -49,6 +51,8 @@ func streamWithToolLoop( model, providerType, userID, channelID, personaID, workspaceID, configID, teamID string, hub *events.Hub, health HealthRecorder, + runner *sandbox.Runner, + extTools map[string]*store.PackageRegistration, ) LoopResult { // Set SSE headers c.Header("Content-Type", "text/event-stream") @@ -77,6 +81,8 @@ func streamWithToolLoop( ConfigID: configID, Budget: LoopBudget{}, Streaming: true, + Runner: runner, + ExtTools: extTools, }, sink) } @@ -93,6 +99,8 @@ func streamModelResponse( model, providerType, displayName, userID, channelID, personaID, workspaceID, configID, teamID string, hub *events.Hub, health HealthRecorder, + runner *sandbox.Runner, + extTools map[string]*store.PackageRegistration, ) LoopResult { sink := newSSEModelSink(c, model, displayName) @@ -114,6 +122,8 @@ func streamModelResponse( ConfigID: configID, Budget: LoopBudget{}, Streaming: true, + Runner: runner, + ExtTools: extTools, }, sink) } diff --git a/server/handlers/tool_loop.go b/server/handlers/tool_loop.go index 115e4d8..29b7fc7 100644 --- a/server/handlers/tool_loop.go +++ b/server/handlers/tool_loop.go @@ -25,9 +25,12 @@ import ( "time" "github.com/gin-gonic/gin" + "go.starlark.net/starlark" "git.gobha.me/xcaliber/chat-switchboard/events" "git.gobha.me/xcaliber/chat-switchboard/providers" + "git.gobha.me/xcaliber/chat-switchboard/sandbox" + "git.gobha.me/xcaliber/chat-switchboard/store" "git.gobha.me/xcaliber/chat-switchboard/tools" ) @@ -79,6 +82,9 @@ type LoopConfig struct { ConfigID string Budget LoopBudget Streaming bool // true = StreamCompletion, false = ChatCompletion + // v0.29.2: extension tool dispatch (nil = no extension tools) + Runner *sandbox.Runner + ExtTools map[string]*store.PackageRegistration // toolName → package } // ── Sink Interface ───────────────────────────── @@ -398,6 +404,12 @@ func executeToolCall(ctx context.Context, lcfg LoopConfig, call tools.ToolCall) return result } + // v0.29.2: Starlark extension tool + if pkg, ok := lcfg.ExtTools[call.Name]; ok && lcfg.Runner != nil { + log.Printf("🔧 Executing tool (extension): %s (call %s, pkg %s)", call.Name, call.ID, pkg.ID) + return executeExtensionTool(ctx, lcfg, pkg, call) + } + // Browser bridge (only available when hub is connected) if lcfg.Hub != nil && lcfg.Hub.IsConnected(lcfg.ExecCtx.UserID) { log.Printf("🔧 Executing tool (browser): %s (call %s)", call.Name, call.ID) @@ -414,6 +426,132 @@ func executeToolCall(ctx context.Context, lcfg LoopConfig, call tools.ToolCall) } } +// executeExtensionTool calls a starlark extension's on_tool_call entry point +// and serializes the return value to a JSON string for the tool result. +func executeExtensionTool(ctx context.Context, lcfg LoopConfig, pkg *store.PackageRegistration, call tools.ToolCall) tools.ToolResult { + // Parse JSON arguments into a Starlark dict. + var rawArgs map[string]interface{} + if call.Arguments != "" { + if err := json.Unmarshal([]byte(call.Arguments), &rawArgs); err != nil { + return tools.ToolResult{ + ToolCallID: call.ID, + Name: call.Name, + Content: `{"error":"invalid tool arguments JSON"}`, + IsError: true, + } + } + } + + // Build the call dict passed to on_tool_call(call). + callDict := starlark.NewDict(3) + _ = callDict.SetKey(starlark.String("tool_name"), starlark.String(call.Name)) + _ = callDict.SetKey(starlark.String("tool_call_id"), starlark.String(call.ID)) + _ = callDict.SetKey(starlark.String("arguments"), jsonToStarlark(rawArgs)) + + rc := &sandbox.RunContext{ + UserID: lcfg.ExecCtx.UserID, + ChannelID: lcfg.ExecCtx.ChannelID, + } + val, output, err := lcfg.Runner.CallEntryPoint(ctx, pkg, "on_tool_call", + starlark.Tuple{callDict}, nil, rc) + if output != "" { + log.Printf(" 🔧 ext tool %s print: %s", pkg.ID, output) + } + if err != nil { + log.Printf("⚠️ ext tool %s on_tool_call error: %v", pkg.ID, err) + return tools.ToolResult{ + ToolCallID: call.ID, + Name: call.Name, + Content: fmt.Sprintf(`{"error":%q}`, err.Error()), + IsError: true, + } + } + + // Serialize the Starlark return value to JSON. + content, jsonErr := json.Marshal(starlarkValueToGo(val)) + if jsonErr != nil { + content = []byte(fmt.Sprintf(`{"error":%q}`, jsonErr.Error())) + } + return tools.ToolResult{ + ToolCallID: call.ID, + Name: call.Name, + Content: string(content), + } +} + +// jsonToStarlark recursively converts a decoded-JSON value (map/slice/scalar) +// to its Starlark equivalent. +func jsonToStarlark(v interface{}) starlark.Value { + if v == nil { + return starlark.None + } + switch val := v.(type) { + case map[string]interface{}: + d := starlark.NewDict(len(val)) + for k, v := range val { + _ = d.SetKey(starlark.String(k), jsonToStarlark(v)) + } + return d + case []interface{}: + elems := make([]starlark.Value, len(val)) + for i, v := range val { + elems[i] = jsonToStarlark(v) + } + return starlark.NewList(elems) + case string: + return starlark.String(val) + case float64: + if val == float64(int64(val)) { + return starlark.MakeInt64(int64(val)) + } + return starlark.Float(val) + case bool: + return starlark.Bool(val) + default: + return starlark.String(fmt.Sprintf("%v", val)) + } +} + +// starlarkValueToGo recursively converts a Starlark value to a Go value +// suitable for json.Marshal. +func starlarkValueToGo(v starlark.Value) interface{} { + if v == nil || v == starlark.None { + return nil + } + switch val := v.(type) { + case starlark.String: + return string(val) + case starlark.Int: + i64, ok := val.Int64() + if ok { + return i64 + } + return val.String() + case starlark.Float: + return float64(val) + case starlark.Bool: + return bool(val) + case *starlark.Dict: + m := make(map[string]interface{}, val.Len()) + for _, item := range val.Items() { + k, ok := starlark.AsString(item[0]) + if !ok { + k = item[0].String() + } + m[k] = starlarkValueToGo(item[1]) + } + return m + case *starlark.List: + list := make([]interface{}, val.Len()) + for i := 0; i < val.Len(); i++ { + list[i] = starlarkValueToGo(val.Index(i)) + } + return list + default: + return v.String() + } +} + // emitWorkspaceEvent sends workspace.file.changed when a write tool succeeds. func emitWorkspaceEvent(lcfg LoopConfig, call tools.ToolCall, result tools.ToolResult) { if lcfg.Hub == nil || result.IsError || lcfg.ExecCtx.WorkspaceID == "" { diff --git a/server/main.go b/server/main.go index 2a3bbdd..25cd575 100644 --- a/server/main.go +++ b/server/main.go @@ -381,6 +381,8 @@ func main() { ) // v0.29.1: provider module adapter — bridges sandbox.ProviderResolver to handlers.ResolveProviderConfig starlarkRunner.SetProviderResolver(handlers.NewProviderResolverAdapter(stores, keyResolver)) + // v0.29.2: db module — extension namespaced table access + starlarkRunner.SetDB(database.DB, database.IsPostgres()) // Discover and register active Starlark pre-completion filters filters.DiscoverStarlarkFilters(context.Background(), filterChain, stores, starlarkRunner) @@ -497,7 +499,7 @@ func main() { log.Printf(" 🔑 Auth mode: %s", authMode) authH := handlers.NewAuthHandler(cfg, stores, uekCache, authProvider) - authLimiter := middleware.NewRateLimiter(1, 5) + authLimiter := middleware.NewRateLimiter(5, 30) api := base.Group("/api/v1") { @@ -697,6 +699,7 @@ func main() { } comp.SetRoutingEvaluator(routing.NewEvaluator()) comp.SetFilterChain(filterChain) + comp.SetRunner(starlarkRunner) // v0.29.2: extension tool dispatch protected.POST("/chat/completions", comp.Complete) protected.GET("/tools", comp.ListTools) @@ -1267,6 +1270,7 @@ func main() { wfComp := handlers.NewCompletionHandler(keyResolver, stores, hub, objStore, kbEmbedder) wfComp.SetFilterChain(filterChain) + wfComp.SetRunner(starlarkRunner) // v0.29.2: extension tool dispatch wfAPI.POST("/:id/completions", wfComp.Complete) } diff --git a/server/middleware/page_auth.go b/server/middleware/page_auth.go index 7b04a3c..b46c699 100644 --- a/server/middleware/page_auth.go +++ b/server/middleware/page_auth.go @@ -55,14 +55,29 @@ func AuthOrRedirect(cfg *config.Config, users store.UserStore, cache *UserStatus return } - // Verify user is active and resolve current role from DB - role, valid := verifyUser(c, claims, users, cache) - if !valid { - // verifyUser sent a JSON error, but for page routes we want a redirect. - // The abort already happened, so just return. + if claims.UserID == "" { + redirectToLogin(c, loginPath) return } + // Resolve user role — redirect (not JSON) on any failure. + var role string + if entry, hit := cache.get(claims.UserID); hit { + if !entry.isActive { + redirectToLogin(c, loginPath) + return + } + role = entry.role + } else { + user, err := users.GetByID(c.Request.Context(), claims.UserID) + if err != nil || !user.IsActive { + redirectToLogin(c, loginPath) + return + } + cache.set(claims.UserID, user.IsActive, user.Role) + role = user.Role + } + c.Set("user_id", claims.UserID) c.Set("email", claims.Email) c.Set("role", role) diff --git a/server/sandbox/db_module.go b/server/sandbox/db_module.go new file mode 100644 index 0000000..d22f875 --- /dev/null +++ b/server/sandbox/db_module.go @@ -0,0 +1,550 @@ +// Package sandbox — db_module.go +// +// v0.29.2 CS1: Structured db module for Starlark extensions. +// +// Permissions: +// db.read → db.query(), db.view(), db.list_tables() +// db.write → all of the above + db.insert(), db.update(), db.delete() +// +// Starlark API: +// +// rows = db.query("logs", filters={"user_id": "abc"}, order="created_at", limit=50) +// row = db.insert("logs", {"message": "hello"}) +// ok = db.update("logs", row_id, {"message": "updated"}) +// ok = db.delete("logs", row_id) +// names = db.list_tables() +// rows = db.view("users", filters={"id": "abc"}) +// +// All table access is scoped to ext_{package_id}_{table_name}. +// Platform views (ext_view_users, ext_view_channels) are accessible +// via db.view() and are read-only regardless of permission level. +// +// Queries are parameterized — no raw SQL is ever accepted from scripts. +// Dialect differences (placeholder style) are handled internally. +package sandbox + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "fmt" + "strings" + + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" +) + +// DBModuleConfig holds configuration for a db module instance. +type DBModuleConfig struct { + PackageID string + CanWrite bool // true when db.write permission is granted + DB *sql.DB + IsPostgres bool +} + +// allowedViews is the set of platform views extensions may query via db.view(). +var allowedViews = map[string]bool{ + "users": true, + "channels": true, +} + +// BuildDBModule creates the "db" starlark module for an extension. +// The module enforces namespace isolation — all physical table names are +// prefixed with ext_{packageID}_. Write operations are guarded by CanWrite. +func BuildDBModule(ctx context.Context, cfg DBModuleConfig) *starlarkstruct.Module { + fns := starlark.StringDict{ + "query": starlark.NewBuiltin("db.query", dbQuery(ctx, cfg)), + "view": starlark.NewBuiltin("db.view", dbView(ctx, cfg)), + "list_tables": starlark.NewBuiltin("db.list_tables", dbListTables(ctx, cfg)), + } + + if cfg.CanWrite { + fns["insert"] = starlark.NewBuiltin("db.insert", dbInsert(ctx, cfg)) + fns["update"] = starlark.NewBuiltin("db.update", dbUpdate(ctx, cfg)) + fns["delete"] = starlark.NewBuiltin("db.delete", dbDelete(ctx, cfg)) + } + + return MakeModule("db", fns) +} + +// ── Helpers ────────────────────────────────── + +// physicalTable returns the fully-qualified table name for an extension table. +func (cfg DBModuleConfig) physicalTable(logicalName string) (string, error) { + if logicalName == "" { + return "", fmt.Errorf("db: table name must not be empty") + } + if strings.ContainsAny(logicalName, " \t\n\"';-") { + return "", fmt.Errorf("db: invalid table name %q", logicalName) + } + // Sanitize package ID for use in identifiers (replace - with _) + pkgSlug := strings.ReplaceAll(cfg.PackageID, "-", "_") + return fmt.Sprintf("ext_%s_%s", pkgSlug, logicalName), nil +} + +// ph returns the parameter placeholder for the nth argument (1-based). +func (cfg DBModuleConfig) ph(n int) string { + if cfg.IsPostgres { + return fmt.Sprintf("$%d", n) + } + return "?" +} + +// generateID produces a random hex ID for new rows. +func generateID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +// starlarkFiltersToSQL converts a Starlark dict of {col: val} into a +// WHERE clause and argument slice. Returns empty string if filters is None. +func (cfg DBModuleConfig) starlarkFiltersToSQL(filters starlark.Value, startIdx int) (string, []any, error) { + if filters == starlark.None || filters == nil { + return "", nil, nil + } + d, ok := filters.(*starlark.Dict) + if !ok { + return "", nil, fmt.Errorf("db: filters must be a dict, got %s", filters.Type()) + } + if d.Len() == 0 { + return "", nil, nil + } + + var parts []string + var args []any + idx := startIdx + + for _, item := range d.Items() { + col, ok := item[0].(starlark.String) + if !ok { + return "", nil, fmt.Errorf("db: filter key must be a string, got %s", item[0].Type()) + } + colStr := string(col) + if strings.ContainsAny(colStr, " \t\n\"';-") { + return "", nil, fmt.Errorf("db: invalid column name %q", colStr) + } + + val, err := starlarkToGoValue(item[1]) + if err != nil { + return "", nil, fmt.Errorf("db: filter value for %q: %w", colStr, err) + } + + parts = append(parts, fmt.Sprintf("%s = %s", colStr, cfg.ph(idx))) + args = append(args, val) + idx++ + } + + return "WHERE " + strings.Join(parts, " AND "), args, nil +} + +// starlarkToGoValue converts a Starlark value to a Go value suitable for SQL. +func starlarkToGoValue(v starlark.Value) (any, error) { + switch val := v.(type) { + case starlark.String: + return string(val), nil + case starlark.Int: + i, ok := val.Int64() + if !ok { + return nil, fmt.Errorf("integer overflow") + } + return i, nil + case starlark.Float: + return float64(val), nil + case starlark.Bool: + return bool(val), nil + case starlark.NoneType: + return nil, nil + default: + return nil, fmt.Errorf("unsupported type %s", v.Type()) + } +} + +// rowsToStarlark converts sql.Rows to a Starlark list of dicts. +func rowsToStarlark(rows *sql.Rows) (*starlark.List, error) { + cols, err := rows.Columns() + if err != nil { + return nil, err + } + + var result []starlark.Value + for rows.Next() { + vals := make([]any, len(cols)) + ptrs := make([]any, len(cols)) + for i := range vals { + ptrs[i] = &vals[i] + } + if err := rows.Scan(ptrs...); err != nil { + return nil, err + } + + d := starlark.NewDict(len(cols)) + for i, col := range cols { + sv, err := goToStarlark(vals[i]) + if err != nil { + return nil, fmt.Errorf("column %q: %w", col, err) + } + if err := d.SetKey(starlark.String(col), sv); err != nil { + return nil, err + } + } + result = append(result, d) + } + if err := rows.Err(); err != nil { + return nil, err + } + return starlark.NewList(result), nil +} + +// goToStarlark converts a Go value (from sql.Scan) to a Starlark value. +func goToStarlark(v any) (starlark.Value, error) { + if v == nil { + return starlark.None, nil + } + switch val := v.(type) { + case string: + return starlark.String(val), nil + case []byte: + return starlark.String(string(val)), nil + case int64: + return starlark.MakeInt64(val), nil + case float64: + return starlark.Float(val), nil + case bool: + return starlark.Bool(val), nil + default: + // Fallback: stringify + return starlark.String(fmt.Sprintf("%v", val)), nil + } +} + +// ── Builtins ───────────────────────────────── + +// dbQuery implements db.query(table, filters=None, order=None, limit=100). +func dbQuery(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var table string + var filters starlark.Value = starlark.None + var order starlark.Value = starlark.None + var limit starlark.Int = starlark.MakeInt(100) + + if err := starlark.UnpackArgs(b.Name(), args, kwargs, + "table", &table, + "filters?", &filters, + "order?", &order, + "limit?", &limit, + ); err != nil { + return nil, err + } + + physTable, err := cfg.physicalTable(table) + if err != nil { + return nil, err + } + + whereClause, whereArgs, err := cfg.starlarkFiltersToSQL(filters, 1) + if err != nil { + return nil, err + } + + lim, ok := limit.Int64() + if !ok || lim < 1 || lim > 1000 { + lim = 100 + } + + query := fmt.Sprintf("SELECT * FROM %s", physTable) + if whereClause != "" { + query += " " + whereClause + } + + if order != starlark.None { + col, ok := order.(starlark.String) + if !ok { + return nil, fmt.Errorf("db.query: order must be a string column name") + } + colStr := string(col) + dir := "ASC" + if strings.HasPrefix(colStr, "-") { + colStr = colStr[1:] + dir = "DESC" + } + if strings.ContainsAny(colStr, " \t\n\"';") { + return nil, fmt.Errorf("db.query: invalid order column %q", colStr) + } + query += fmt.Sprintf(" ORDER BY %s %s", colStr, dir) + } + + limitPH := cfg.ph(len(whereArgs) + 1) + query += " LIMIT " + limitPH + whereArgs = append(whereArgs, lim) + + rows, err := cfg.DB.QueryContext(ctx, query, whereArgs...) + if err != nil { + return nil, fmt.Errorf("db.query: %w", err) + } + defer rows.Close() + + return rowsToStarlark(rows) + } +} + +// dbView implements db.view(view_name, filters=None, limit=100). +// Queries ext_view_{view_name} — only allowedViews are permitted. +func dbView(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var viewName string + var filters starlark.Value = starlark.None + var limit starlark.Int = starlark.MakeInt(100) + + if err := starlark.UnpackArgs(b.Name(), args, kwargs, + "view_name", &viewName, + "filters?", &filters, + "limit?", &limit, + ); err != nil { + return nil, err + } + + if !allowedViews[viewName] { + return nil, fmt.Errorf("db.view: unknown view %q (allowed: users, channels)", viewName) + } + + physView := "ext_view_" + viewName + + whereClause, whereArgs, err := cfg.starlarkFiltersToSQL(filters, 1) + if err != nil { + return nil, err + } + + lim, ok := limit.Int64() + if !ok || lim < 1 || lim > 1000 { + lim = 100 + } + + query := fmt.Sprintf("SELECT * FROM %s", physView) + if whereClause != "" { + query += " " + whereClause + } + limitPH := cfg.ph(len(whereArgs) + 1) + query += " LIMIT " + limitPH + whereArgs = append(whereArgs, lim) + + rows, err := cfg.DB.QueryContext(ctx, query, whereArgs...) + if err != nil { + return nil, fmt.Errorf("db.view: %w", err) + } + defer rows.Close() + + return rowsToStarlark(rows) + } +} + +// dbListTables implements db.list_tables() → list of logical table names. +func dbListTables(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 0); err != nil { + return nil, err + } + + prefix := "ext_" + strings.ReplaceAll(cfg.PackageID, "-", "_") + "_" + var query string + var queryArgs []any + + if cfg.IsPostgres { + query = `SELECT table_name FROM information_schema.tables + WHERE table_schema = 'public' AND table_name LIKE $1 + ORDER BY table_name` + queryArgs = []any{prefix + "%"} + } else { + query = `SELECT name FROM sqlite_master + WHERE type = 'table' AND name LIKE ? + ORDER BY name` + queryArgs = []any{prefix + "%"} + } + + rows, err := cfg.DB.QueryContext(ctx, query, queryArgs...) + if err != nil { + return nil, fmt.Errorf("db.list_tables: %w", err) + } + defer rows.Close() + + var names []starlark.Value + for rows.Next() { + var physName string + if err := rows.Scan(&physName); err != nil { + return nil, err + } + // Strip prefix to return logical name + logical := strings.TrimPrefix(physName, prefix) + names = append(names, starlark.String(logical)) + } + if names == nil { + names = []starlark.Value{} + } + return starlark.NewList(names), nil + } +} + +// dbInsert implements db.insert(table, row_dict) → inserted row dict with id. +func dbInsert(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var table string + var rowVal starlark.Value + + if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 2, &table, &rowVal); err != nil { + return nil, err + } + + row, ok := rowVal.(*starlark.Dict) + if !ok { + return nil, fmt.Errorf("db.insert: row must be a dict, got %s", rowVal.Type()) + } + + physTable, err := cfg.physicalTable(table) + if err != nil { + return nil, err + } + + id := generateID() + cols := []string{"id"} + placeholders := []string{cfg.ph(1)} + vals := []any{id} + idx := 2 + + for _, item := range row.Items() { + col, ok := item[0].(starlark.String) + if !ok { + return nil, fmt.Errorf("db.insert: column key must be a string") + } + colStr := string(col) + if colStr == "id" || colStr == "created_at" { + continue // auto-managed + } + if strings.ContainsAny(colStr, " \t\n\"';-") { + return nil, fmt.Errorf("db.insert: invalid column name %q", colStr) + } + val, err := starlarkToGoValue(item[1]) + if err != nil { + return nil, fmt.Errorf("db.insert: column %q: %w", colStr, err) + } + cols = append(cols, colStr) + placeholders = append(placeholders, cfg.ph(idx)) + vals = append(vals, val) + idx++ + } + + query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", + physTable, + strings.Join(cols, ", "), + strings.Join(placeholders, ", "), + ) + + if _, err := cfg.DB.ExecContext(ctx, query, vals...); err != nil { + return nil, fmt.Errorf("db.insert: %w", err) + } + + // Return the inserted row (re-query by id) + selectQ := fmt.Sprintf("SELECT * FROM %s WHERE id = %s", physTable, cfg.ph(1)) + rows, err := cfg.DB.QueryContext(ctx, selectQ, id) + if err != nil { + return nil, fmt.Errorf("db.insert: re-query failed: %w", err) + } + defer rows.Close() + + list, err := rowsToStarlark(rows) + if err != nil { + return nil, err + } + if list.Len() == 0 { + return starlark.None, nil + } + return list.Index(0), nil + } +} + +// dbUpdate implements db.update(table, id, partial_dict) → True. +func dbUpdate(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var table, rowID string + var patchVal starlark.Value + + if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 3, &table, &rowID, &patchVal); err != nil { + return nil, err + } + + patch, ok := patchVal.(*starlark.Dict) + if !ok { + return nil, fmt.Errorf("db.update: patch must be a dict, got %s", patchVal.Type()) + } + if patch.Len() == 0 { + return starlark.True, nil // nothing to do + } + + physTable, err := cfg.physicalTable(table) + if err != nil { + return nil, err + } + + var setParts []string + var vals []any + idx := 1 + + for _, item := range patch.Items() { + col, ok := item[0].(starlark.String) + if !ok { + return nil, fmt.Errorf("db.update: column key must be a string") + } + colStr := string(col) + if colStr == "id" || colStr == "created_at" { + continue + } + if strings.ContainsAny(colStr, " \t\n\"';-") { + return nil, fmt.Errorf("db.update: invalid column name %q", colStr) + } + val, err := starlarkToGoValue(item[1]) + if err != nil { + return nil, fmt.Errorf("db.update: column %q: %w", colStr, err) + } + setParts = append(setParts, fmt.Sprintf("%s = %s", colStr, cfg.ph(idx))) + vals = append(vals, val) + idx++ + } + + if len(setParts) == 0 { + return starlark.True, nil + } + + vals = append(vals, rowID) + query := fmt.Sprintf("UPDATE %s SET %s WHERE id = %s", + physTable, + strings.Join(setParts, ", "), + cfg.ph(idx), + ) + + if _, err := cfg.DB.ExecContext(ctx, query, vals...); err != nil { + return nil, fmt.Errorf("db.update: %w", err) + } + return starlark.True, nil + } +} + +// dbDelete implements db.delete(table, id) → True. +func dbDelete(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var table, rowID string + if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 2, &table, &rowID); err != nil { + return nil, err + } + + physTable, err := cfg.physicalTable(table) + if err != nil { + return nil, err + } + + query := fmt.Sprintf("DELETE FROM %s WHERE id = %s", physTable, cfg.ph(1)) + if _, err := cfg.DB.ExecContext(ctx, query, rowID); err != nil { + return nil, fmt.Errorf("db.delete: %w", err) + } + return starlark.True, nil + } +} diff --git a/server/sandbox/db_module_test.go b/server/sandbox/db_module_test.go new file mode 100644 index 0000000..743c182 --- /dev/null +++ b/server/sandbox/db_module_test.go @@ -0,0 +1,254 @@ +package sandbox + +import ( + "context" + "database/sql" + "strings" + "testing" + + "go.starlark.net/starlark" + _ "modernc.org/sqlite" +) + +// newTestDB creates an in-memory SQLite database with a test extension table. +func newTestDB(t *testing.T) (*sql.DB, DBModuleConfig) { + t.Helper() + db, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("open db: %v", err) + } + t.Cleanup(func() { db.Close() }) + + // Create the extension table the tests will use. + // physicalTable("logs") for package "test-ext" → ext_test_ext_logs + _, err = db.Exec(`CREATE TABLE ext_test_ext_logs ( + id TEXT PRIMARY KEY, + message TEXT, + user_id TEXT, + count INTEGER, + created_at TEXT DEFAULT (datetime('now')) + )`) + if err != nil { + t.Fatalf("create test table: %v", err) + } + + cfg := DBModuleConfig{ + PackageID: "test-ext", + CanWrite: true, + DB: db, + IsPostgres: false, + } + return db, cfg +} + +// execScript runs a Starlark snippet with the db module injected. +func execScript(t *testing.T, cfg DBModuleConfig, script string) (*Result, error) { + t.Helper() + sb := New(DefaultConfig()) + ctx := context.Background() + modules := map[string]starlark.Value{ + "db": BuildDBModule(ctx, cfg), + } + return sb.Exec(ctx, "test.star", script, modules) +} + +// ── physicalTable ─────────────────────────── + +func TestPhysicalTable_ValidName(t *testing.T) { + cfg := DBModuleConfig{PackageID: "my-ext"} + got, err := cfg.physicalTable("logs") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != "ext_my_ext_logs" { + t.Errorf("got %q, want ext_my_ext_logs", got) + } +} + +func TestPhysicalTable_EmptyName(t *testing.T) { + cfg := DBModuleConfig{PackageID: "my-ext"} + _, err := cfg.physicalTable("") + if err == nil { + t.Fatal("expected error for empty table name") + } +} + +func TestPhysicalTable_InvalidChars(t *testing.T) { + cfg := DBModuleConfig{PackageID: "my-ext"} + _, err := cfg.physicalTable("bad name") + if err == nil { + t.Fatal("expected error for table name with space") + } + _, err = cfg.physicalTable("drop;table") + if err == nil { + t.Fatal("expected error for table name with semicolon") + } +} + +// ── db.insert / db.query ──────────────────── + +func TestDBInsertAndQuery(t *testing.T) { + _, cfg := newTestDB(t) + + result, err := execScript(t, cfg, ` +row = db.insert("logs", {"message": "hello", "user_id": "u1"}) +rows = db.query("logs") +`) + if err != nil { + t.Fatalf("script error: %v", err) + } + + rowsVal, ok := result.Globals["rows"] + if !ok { + t.Fatal("rows not in globals") + } + if _, ok := rowsVal.(*starlark.List); !ok { + t.Fatalf("rows is %T, want *starlark.List", rowsVal) + } +} + +func TestDBQueryFilters(t *testing.T) { + db, cfg := newTestDB(t) + + // Pre-populate with two rows + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id) VALUES ('a', 'one', 'u1')`) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id) VALUES ('b', 'two', 'u2')`) + + result, err := execScript(t, cfg, ` +rows = db.query("logs", filters={"user_id": "u1"}) +`) + if err != nil { + t.Fatalf("script error: %v", err) + } + + rowsStr := result.Globals["rows"].String() + if !strings.Contains(rowsStr, "one") { + t.Errorf("expected row with message=one, got: %s", rowsStr) + } + if strings.Contains(rowsStr, "two") { + t.Errorf("unexpected row with message=two: %s", rowsStr) + } +} + +func TestDBQueryLimit(t *testing.T) { + db, cfg := newTestDB(t) + for i := 0; i < 5; i++ { + db.Exec(`INSERT INTO ext_test_ext_logs (id, message) VALUES (?, ?)`, + generateID(), "msg") + } + + result, err := execScript(t, cfg, `rows = db.query("logs", limit=2)`) + if err != nil { + t.Fatalf("script error: %v", err) + } + // String representation of a 2-element list has exactly 2 entries + rowsStr := result.Globals["rows"].String() + // Count occurrences of "id" key — each row has one + count := strings.Count(rowsStr, `"id"`) + if count != 2 { + t.Errorf("expected 2 rows with limit=2, got repr: %s", rowsStr) + } +} + +// ── db.update ─────────────────────────────── + +func TestDBUpdate(t *testing.T) { + db, cfg := newTestDB(t) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message) VALUES ('r1', 'original')`) + + _, err := execScript(t, cfg, `db.update("logs", "r1", {"message": "updated"})`) + if err != nil { + t.Fatalf("script error: %v", err) + } + + var msg string + db.QueryRow(`SELECT message FROM ext_test_ext_logs WHERE id = 'r1'`).Scan(&msg) + if msg != "updated" { + t.Errorf("expected message=updated, got %q", msg) + } +} + +// ── db.delete ─────────────────────────────── + +func TestDBDelete(t *testing.T) { + db, cfg := newTestDB(t) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message) VALUES ('r1', 'to-delete')`) + + _, err := execScript(t, cfg, `db.delete("logs", "r1")`) + if err != nil { + t.Fatalf("script error: %v", err) + } + + var count int + db.QueryRow(`SELECT COUNT(*) FROM ext_test_ext_logs WHERE id = 'r1'`).Scan(&count) + if count != 0 { + t.Errorf("expected row deleted, count = %d", count) + } +} + +// ── write guard ───────────────────────────── + +func TestDBReadOnlyModuleNoWriteFunctions(t *testing.T) { + _, cfg := newTestDB(t) + cfg.CanWrite = false + + mod := BuildDBModule(context.Background(), cfg) + modStr := mod.String() + + // insert/update/delete should not be present in a read-only module + for _, fn := range []string{"insert", "update", "delete"} { + if strings.Contains(modStr, fn) { + t.Errorf("read-only module exposes write function %q", fn) + } + } +} + +func TestDBWriteFunctionBlockedByReadOnly(t *testing.T) { + _, cfg := newTestDB(t) + cfg.CanWrite = false + + _, err := execScript(t, cfg, `db.insert("logs", {"message": "x"})`) + if err == nil { + t.Fatal("expected error calling insert on read-only module") + } +} + +// ── view allowlist ─────────────────────────── + +func TestDBViewRejectsUnknownView(t *testing.T) { + _, cfg := newTestDB(t) + _, err := execScript(t, cfg, `db.view("secrets")`) + if err == nil { + t.Fatal("expected error for unknown view") + } + if !strings.Contains(err.Error(), "unknown view") { + t.Errorf("unexpected error: %v", err) + } +} + +// ── namespace isolation ────────────────────── + +func TestDBQueryRejectsOtherExtensionTable(t *testing.T) { + _, cfg := newTestDB(t) + // Attempt to query a table that belongs to a different extension + _, err := execScript(t, cfg, `db.query("../../etc/passwd")`) + if err == nil { + t.Fatal("expected error for path traversal attempt") + } +} + +// ── list_tables ───────────────────────────── + +func TestDBListTables(t *testing.T) { + _, cfg := newTestDB(t) + result, err := execScript(t, cfg, `names = db.list_tables()`) + if err != nil { + t.Fatalf("script error: %v", err) + } + namesStr := result.Globals["names"].String() + // The test table ext_test_ext_logs should appear as "logs" + if !strings.Contains(namesStr, "logs") { + t.Errorf("expected 'logs' in list_tables output, got: %s", namesStr) + } +} + diff --git a/server/sandbox/runner.go b/server/sandbox/runner.go index e9023a5..7d1be17 100644 --- a/server/sandbox/runner.go +++ b/server/sandbox/runner.go @@ -9,12 +9,17 @@ // v0.29.1 CS2: Adds RunContext for per-invocation state (user_id), // vault for provider key decryption, and provider.complete module. // +// v0.29.2 CS1: Adds db *sql.DB for the db module. SetDB() wires it +// at startup. db.read grants query/view/list_tables; db.write adds +// insert/update/delete. If both are granted, db.write wins (superset). +// // The runner is the bridge between the package/permission system // and the sandboxed Starlark interpreter. package sandbox import ( "context" + "database/sql" "fmt" "log" @@ -38,10 +43,12 @@ type RunContext struct { // Runner executes Starlark package scripts with permission-gated modules. type Runner struct { - sandbox *Sandbox - stores store.Stores - notifier NotificationSender // nil = notifications module unavailable - resolver ProviderResolver // nil = provider module unavailable + sandbox *Sandbox + stores store.Stores + notifier NotificationSender // nil = notifications module unavailable + resolver ProviderResolver // nil = provider module unavailable + db *sql.DB // nil = db module unavailable + dbPostgres bool // true = use $N placeholders; false = use ? } // NewRunner creates a runner with the given sandbox and dependencies. @@ -62,6 +69,14 @@ func (r *Runner) SetProviderResolver(pr ProviderResolver) { r.resolver = pr } +// SetDB attaches the raw database connection for the db module. +// isPostgres controls whether $N or ? placeholders are used. +// Call this at startup after database.Connect(). +func (r *Runner) SetDB(db *sql.DB, isPostgres bool) { + r.db = db + r.dbPostgres = isPostgres +} + // ExecPackage loads a package's script from its manifest and executes it // with modules gated by the package's granted permissions. // @@ -98,9 +113,9 @@ func (r *Runner) ExecPackage(ctx context.Context, pkg *store.PackageRegistration // CallEntryPoint executes a package script and calls a named function. // This is the standard pattern for event-driven extensions: -// 1. Exec the script (defines functions) -// 2. Find the named entry point in globals -// 3. Call it with the provided arguments +// 1. Exec the script (defines functions) +// 2. Find the named entry point in globals +// 3. Call it with the provided arguments // // The RunContext carries per-invocation state. Pass nil if not needed. // @@ -141,6 +156,9 @@ func (r *Runner) buildModules(ctx context.Context, packageID string, manifest ma modules := make(map[string]starlark.Value) + // Track db permission level: 0=none, 1=read, 2=write + dbLevel := 0 + for _, perm := range granted { switch perm { case models.ExtPermSecretsRead: @@ -163,11 +181,25 @@ func (r *Runner) buildModules(ctx context.Context, packageID string, manifest ma } } - // Future permissions: - // case models.ExtPermDBRead, models.ExtPermDBWrite: - // modules["db"] = BuildDBModule(...) + case models.ExtPermDBRead: + if dbLevel < 1 { + dbLevel = 1 + } + + case models.ExtPermDBWrite: + dbLevel = 2 } } + // Wire db module at the highest granted level. + if dbLevel > 0 && r.db != nil { + modules["db"] = BuildDBModule(ctx, DBModuleConfig{ + PackageID: packageID, + CanWrite: dbLevel == 2, + DB: r.db, + IsPostgres: r.dbPostgres, + }) + } + return modules, nil } diff --git a/server/scheduler/executor.go b/server/scheduler/executor.go index 6695d6f..e5cda43 100644 --- a/server/scheduler/executor.go +++ b/server/scheduler/executor.go @@ -182,6 +182,7 @@ func (e *Executor) Execute(ctx context.Context, task models.Task, run *models.Ta personaID = *task.PersonaID } + extTools := handlers.BuildExtToolMap(ctx, e.stores, task.OwnerID) sink := handlers.NewHeadlessSink(task.ID) result := handlers.CoreToolLoop(ctx, handlers.LoopConfig{ Provider: provider, @@ -203,6 +204,8 @@ func (e *Executor) Execute(ctx context.Context, task models.Task, run *models.Ta MaxTokens: task.MaxTokens, }, Streaming: false, // headless — use ChatCompletion + Runner: e.runner, + ExtTools: extTools, }, sink) // ── 6. Persist output based on output_mode ── diff --git a/server/store/ext_data_iface.go b/server/store/ext_data_iface.go new file mode 100644 index 0000000..077a8f4 --- /dev/null +++ b/server/store/ext_data_iface.go @@ -0,0 +1,20 @@ +package store + +import "context" + +// ExtDataStore tracks the namespaced tables created for each extension. +// It is the catalog backing db.list_tables() and the install/uninstall lifecycle. +// The actual extension data lives in tables named ext_{package_id}_{table_name}; +// this store only records the logical (unprefixed) names. +type ExtDataStore interface { + // RegisterTable records that a table was created for a package. + // Safe to call multiple times — idempotent. + RegisterTable(ctx context.Context, packageID, tableName string) error + + // ListTables returns all logical table names registered for a package. + ListTables(ctx context.Context, packageID string) ([]string, error) + + // DeletePackageTables removes all catalog entries for a package. + // Called after the physical tables have been dropped on uninstall. + DeletePackageTables(ctx context.Context, packageID string) error +} diff --git a/server/store/interfaces.go b/server/store/interfaces.go index 55e26cc..e81fc70 100644 --- a/server/store/interfaces.go +++ b/server/store/interfaces.go @@ -61,6 +61,7 @@ type Stores struct { Folders FolderStore // v0.29.0: Chat folder CRUD Health HealthStore // v0.29.0: Provider health window management ExtPermissions ExtensionPermissionStore // v0.29.0: Extension declared/granted capabilities + ExtData ExtDataStore // v0.29.2: Extension namespaced table catalog } // ========================================= diff --git a/server/store/postgres/ext_data.go b/server/store/postgres/ext_data.go new file mode 100644 index 0000000..95c74b4 --- /dev/null +++ b/server/store/postgres/ext_data.go @@ -0,0 +1,55 @@ +package postgres + +import ( + "context" + "database/sql" +) + +// ExtDataStore implements store.ExtDataStore for Postgres. +type ExtDataStore struct { + db *sql.DB +} + +func NewExtDataStore(db *sql.DB) *ExtDataStore { + return &ExtDataStore{db: db} +} + +func (s *ExtDataStore) RegisterTable(ctx context.Context, packageID, tableName string) error { + _, err := s.db.ExecContext(ctx, + `INSERT INTO ext_data_tables (package_id, table_name) + VALUES ($1, $2) + ON CONFLICT (package_id, table_name) DO NOTHING`, + packageID, tableName) + return err +} + +func (s *ExtDataStore) ListTables(ctx context.Context, packageID string) ([]string, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT table_name FROM ext_data_tables + WHERE package_id = $1 + ORDER BY table_name`, + packageID) + if err != nil { + return nil, err + } + defer rows.Close() + + var names []string + for rows.Next() { + var n string + if err := rows.Scan(&n); err != nil { + return nil, err + } + names = append(names, n) + } + if names == nil { + names = []string{} + } + return names, nil +} + +func (s *ExtDataStore) DeletePackageTables(ctx context.Context, packageID string) error { + _, err := s.db.ExecContext(ctx, + `DELETE FROM ext_data_tables WHERE package_id = $1`, packageID) + return err +} diff --git a/server/store/postgres/stores.go b/server/store/postgres/stores.go index a6e4d22..e4da4f9 100644 --- a/server/store/postgres/stores.go +++ b/server/store/postgres/stores.go @@ -47,5 +47,6 @@ func NewStores(db *sql.DB) store.Stores { Folders: NewFolderStore(), Health: NewHealthStore(db), ExtPermissions: NewExtensionPermissionStore(db), + ExtData: NewExtDataStore(db), } } diff --git a/server/store/sqlite/ext_data.go b/server/store/sqlite/ext_data.go new file mode 100644 index 0000000..7589af6 --- /dev/null +++ b/server/store/sqlite/ext_data.go @@ -0,0 +1,49 @@ +package sqlite + +import "context" + +// ExtDataStore implements store.ExtDataStore for SQLite. +type ExtDataStore struct{} + +func NewExtDataStore() *ExtDataStore { + return &ExtDataStore{} +} + +func (s *ExtDataStore) RegisterTable(ctx context.Context, packageID, tableName string) error { + _, err := DB.ExecContext(ctx, + `INSERT OR IGNORE INTO ext_data_tables (package_id, table_name) + VALUES (?, ?)`, + packageID, tableName) + return err +} + +func (s *ExtDataStore) ListTables(ctx context.Context, packageID string) ([]string, error) { + rows, err := DB.QueryContext(ctx, + `SELECT table_name FROM ext_data_tables + WHERE package_id = ? + ORDER BY table_name`, + packageID) + if err != nil { + return nil, err + } + defer rows.Close() + + var names []string + for rows.Next() { + var n string + if err := rows.Scan(&n); err != nil { + return nil, err + } + names = append(names, n) + } + if names == nil { + names = []string{} + } + return names, nil +} + +func (s *ExtDataStore) DeletePackageTables(ctx context.Context, packageID string) error { + _, err := DB.ExecContext(ctx, + `DELETE FROM ext_data_tables WHERE package_id = ?`, packageID) + return err +} diff --git a/server/store/sqlite/stores.go b/server/store/sqlite/stores.go index dd34278..1ac96e1 100644 --- a/server/store/sqlite/stores.go +++ b/server/store/sqlite/stores.go @@ -47,5 +47,6 @@ func NewStores(db *sql.DB) store.Stores { Folders: NewFolderStore(), Health: NewHealthStore(), ExtPermissions: NewExtensionPermissionStore(), + ExtData: NewExtDataStore(), } } diff --git a/src/js/extensions.js b/src/js/extensions.js index 4e46afb..fd5d369 100644 --- a/src/js/extensions.js +++ b/src/js/extensions.js @@ -42,7 +42,7 @@ const Extensions = { for (const ext of exts) { // Parse manifest to check for _script (inline) or entry (file) const manifest = ext.manifest || {}; - const extId = ext.ext_id; + const extId = ext.id; if (manifest._script) { // Inject script tag pointing at the asset endpoint @@ -431,7 +431,7 @@ const Extensions = { _getUserSettings(extId) { // Find user settings from the manifest data loaded from API - const manifest = this._manifests.find(m => m.ext_id === extId); + const manifest = this._manifests.find(m => m.id === extId); if (manifest?.user_settings) { try { return typeof manifest.user_settings === 'string'