Changeset 0.29.2 (#197)
Co-authored-by: Jeffrey Smith <jasafpro@gmail.com> Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -51,3 +51,6 @@ secrets/
|
||||
dist/
|
||||
build/
|
||||
*.surface
|
||||
|
||||
.claude/
|
||||
data/
|
||||
|
||||
109
CHANGELOG.md
109
CHANGELOG.md
@@ -1,5 +1,114 @@
|
||||
# Changelog
|
||||
|
||||
## [0.29.2] — 2026-03-17
|
||||
|
||||
### Summary
|
||||
|
||||
DB extensions and server-side tool execution. Starlark packages declare
|
||||
namespaced database tables via `db_tables` in the manifest — created on
|
||||
install, dropped on uninstall. A structured Starlark `db` module
|
||||
(`query/insert/update/delete/view/list_tables`) provides namespaced
|
||||
access to extension-owned tables and column-allowlisted platform views.
|
||||
Server-side tool execution wires the `on_tool_call` entry point into the
|
||||
completion tool loop: extensions declare `tools` in their manifest, the
|
||||
tool loop dispatches matched calls to the sandbox. Three changesets
|
||||
(CS0–CS3) plus docs (CS4).
|
||||
|
||||
### New
|
||||
|
||||
- **`db` Starlark module** — structured API for extension-owned tables.
|
||||
Requires `db.read` permission (query/view/list_tables) or `db.write`
|
||||
(insert/update/delete). Tables namespaced as `ext_{pkg_slug}_{name}`.
|
||||
- `db.query(table, filters=None, order=None, limit=100)` → list of dicts
|
||||
- `db.insert(table, row_dict)` → inserted row dict (auto-generates `id`)
|
||||
- `db.update(table, id, partial_dict)` → True
|
||||
- `db.delete(table, id)` → True
|
||||
- `db.list_tables()` → list of logical table names for this package
|
||||
- `db.view(view_name, filters=None, limit=100)` → list of dicts
|
||||
(allowed views: `"users"`, `"channels"`)
|
||||
- **`db_tables` manifest key** — declare extension-owned tables with
|
||||
typed columns and indexes. Tables created on install, dropped on
|
||||
uninstall.
|
||||
```json
|
||||
{
|
||||
"db_tables": {
|
||||
"logs": {
|
||||
"columns": {"message": "text", "user_id": "text", "count": "int"},
|
||||
"indexes": [["user_id"]]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
Supported column types: `text`, `int`/`integer`, `real`/`float`,
|
||||
`bool`/`boolean`, `timestamp`. Auto-columns: `id TEXT PRIMARY KEY`,
|
||||
`created_at` (dialect-correct default). Dialect-correct DDL for both
|
||||
PG (`TIMESTAMPTZ`, `BOOLEAN`) and SQLite (`TEXT`, `INTEGER`).
|
||||
- **Platform views** — column-allowlisted read-only views over platform
|
||||
tables accessible via `db.view(...)`:
|
||||
- `ext_view_users`: `id`, `display_name`, `email`
|
||||
- `ext_view_channels`: `id`, `name`, `type`, `team_id`
|
||||
- **`ext_data_tables` catalog** — tracks logical→physical table mappings
|
||||
per package. Powers `db.list_tables()` and uninstall cleanup.
|
||||
- **Server-side tool execution** — starlark extensions declare `tools`
|
||||
in the manifest. `BuildToolDefs` includes starlark-tier active package
|
||||
tools alongside server tools. `CoreToolLoop` dispatches matched calls
|
||||
to `executeExtensionTool` which calls the `on_tool_call` entry point.
|
||||
```json
|
||||
{
|
||||
"tools": [{
|
||||
"name": "search_logs",
|
||||
"description": "Search extension log entries",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"]
|
||||
}
|
||||
}]
|
||||
}
|
||||
```
|
||||
Entry point:
|
||||
```python
|
||||
def on_tool_call(call):
|
||||
# call = {"tool_name": "...", "tool_call_id": "...", "arguments": {...}}
|
||||
if call["tool_name"] == "search_logs":
|
||||
rows = db.query("logs", filters={"user_id": call["arguments"]["user_id"]})
|
||||
return {"results": rows}
|
||||
return {"error": "unknown tool"}
|
||||
```
|
||||
- **`BuildExtToolMap`** — pre-loads `map[toolName]*PackageRegistration`
|
||||
per request to avoid DB lookup on each tool dispatch.
|
||||
- **`ExtDataStore`** interface + implementations (`postgres`, `sqlite`) —
|
||||
`RegisterTable`, `ListTables`, `DeletePackageTables`.
|
||||
- **`db.read` and `db.write` permissions** — active in this release
|
||||
(previously listed as "future").
|
||||
|
||||
### Changed
|
||||
|
||||
- `LoopConfig` gains `Runner *sandbox.Runner` and
|
||||
`ExtTools map[string]*store.PackageRegistration` for extension tool
|
||||
dispatch. All callers updated: `streamCompletion`, `syncCompletion`,
|
||||
multi-model path, `messages.go` regen, scheduler executor.
|
||||
- `BuildToolDefs` (resolve.go) now includes starlark-tier active package
|
||||
tools in addition to browser-tier tools.
|
||||
- `streamWithToolLoop` and `streamModelResponse` accept runner and
|
||||
extTools parameters.
|
||||
- `CompletionHandler` gains `SetRunner(r *sandbox.Runner)` and
|
||||
`buildExtToolMap` helpers.
|
||||
- `AdminInstallExtension` and `AdminUninstallExtension` call
|
||||
`CreateExtTables`/`DropExtTables` for schema lifecycle.
|
||||
- `InstallPackage` and `DeletePackage` (packages.go) call the same
|
||||
schema lifecycle hooks.
|
||||
- Migration `016_packages.sql` extended with `ext_data_tables` catalog
|
||||
table, `idx_ext_data_tables_pkg` index, and `ext_view_users`/
|
||||
`ext_view_channels` platform views.
|
||||
- `Runner.SetDB(db, isPostgres)` wires the DB handle for `buildModules`.
|
||||
`main.go` calls `starlarkRunner.SetDB` at startup.
|
||||
- Extension permissions table in docs updated: `db.read` and `db.write`
|
||||
now listed as `v0.29.2` (previously "future").
|
||||
- Extension tiers in enums.md updated: `starlark` now implemented.
|
||||
|
||||
---
|
||||
|
||||
## [0.29.1] — 2026-03-17
|
||||
|
||||
### Summary
|
||||
|
||||
@@ -24,10 +24,12 @@ services:
|
||||
DB_DRIVER: sqlite
|
||||
DATABASE_URL: /data/switchboard.db
|
||||
JWT_SECRET: ${JWT_SECRET:-change-me-for-production}
|
||||
ENCRYPTION_KEY: ${ENCRYPTION_KEY:-change-me-for-production}
|
||||
SWITCHBOARD_ADMIN_USERNAME: ${ADMIN_USERNAME:-admin}
|
||||
SWITCHBOARD_ADMIN_PASSWORD: ${ADMIN_PASSWORD:-admin}
|
||||
STORAGE_BACKEND: pvc
|
||||
STORAGE_PATH: /data/storage
|
||||
CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-http://localhost:3000}
|
||||
volumes:
|
||||
- ./data:/data
|
||||
ports:
|
||||
|
||||
@@ -110,7 +110,7 @@ All enum values used across the API. Definitive source of truth.
|
||||
|
||||
## Extension Tiers
|
||||
|
||||
`browser` (implemented), `starlark` (future), `sidecar` (future)
|
||||
`browser` (implemented), `starlark` (implemented, v0.29.0), `sidecar` (future)
|
||||
|
||||
## Grant Types
|
||||
|
||||
@@ -266,8 +266,8 @@ Declared in package manifests, granted by admin review.
|
||||
| `filters.pre_completion` | — | v0.29.0 |
|
||||
| `api.http` | `http` | v0.29.0 |
|
||||
| `provider.complete` | `provider` | v0.29.1 |
|
||||
| `db.read` | `db` | future |
|
||||
| `db.write` | `db` | future |
|
||||
| `db.read` | `db` | v0.29.2 |
|
||||
| `db.write` | `db` | v0.29.2 |
|
||||
|
||||
## Policies
|
||||
|
||||
|
||||
@@ -143,8 +143,8 @@ Admin must grant each before the package activates.
|
||||
| `filters.pre_completion` | — | Register pre-completion filter |
|
||||
| `api.http` | `http` | Outbound HTTP requests (v0.29.0: module, v0.29.1: also required for API routes) |
|
||||
| `provider.complete` | `provider` | LLM completion calls via BYOK chain (v0.29.1) |
|
||||
| `db.read` | `db` | Read extension tables (future) |
|
||||
| `db.write` | `db` | Write extension tables (future) |
|
||||
| `db.read` | `db` | Read extension tables and platform views (v0.29.2) |
|
||||
| `db.write` | `db` | Write extension tables (v0.29.2) |
|
||||
|
||||
### Starlark Modules (v0.29.0+)
|
||||
|
||||
@@ -171,6 +171,110 @@ Modules injected into the script namespace based on granted permissions:
|
||||
- Response: `{"content", "model", "finish_reason", "input_tokens", "output_tokens"}`
|
||||
- Provider resolved via BYOK chain; pinnable via `requires_provider.provider_config_id`
|
||||
|
||||
**`db`** (requires `db.read` or `db.write`, v0.29.2):
|
||||
- `db.query(table, filters=None, order=None, limit=100)` → list of dicts
|
||||
- `table`: logical name (physical: `ext_{pkg_slug}_{table}`)
|
||||
- `filters`: dict of `{column: value}` equality filters (optional)
|
||||
- `order`: `"col"` or `"-col"` for descending (optional)
|
||||
- `limit`: max rows (default 100)
|
||||
- `db.insert(table, row_dict)` → inserted row dict with auto-generated `id` (`db.write`)
|
||||
- `db.update(table, id, partial_dict)` → True (`db.write`)
|
||||
- `db.delete(table, id)` → True (`db.write`)
|
||||
- `db.list_tables()` → list of logical table names for this package
|
||||
- `db.view(view_name, filters=None, limit=100)` → list of dicts
|
||||
- Allowed view names: `"users"` → `ext_view_users`, `"channels"` → `ext_view_channels`
|
||||
- `ext_view_users`: `id`, `display_name`, `email`
|
||||
- `ext_view_channels`: `id`, `title`, `type`, `team_id`
|
||||
|
||||
### Extension Database Tables (v0.29.2)
|
||||
|
||||
Starlark packages declare owned tables in `manifest.db_tables`. Tables
|
||||
are created on install and dropped on uninstall.
|
||||
|
||||
**Manifest `db_tables` field:**
|
||||
|
||||
```json
|
||||
{
|
||||
"db_tables": {
|
||||
"logs": {
|
||||
"columns": {
|
||||
"message": "text",
|
||||
"user_id": "text",
|
||||
"count": "int",
|
||||
"score": "real",
|
||||
"active": "bool",
|
||||
"created_at": "timestamp"
|
||||
},
|
||||
"indexes": [
|
||||
["user_id"],
|
||||
["user_id", "created_at"]
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- **Physical name**: `ext_{pkg_slug}_{logical_name}` (hyphens → underscores)
|
||||
- **Auto-columns**: `id TEXT PRIMARY KEY` (UUID generated on insert),
|
||||
`created_at` (dialect-correct timestamp default)
|
||||
- **Column types**: `text`, `int`/`integer`, `real`/`float`,
|
||||
`bool`/`boolean`, `timestamp` — mapped to dialect-correct SQL
|
||||
- **Indexes**: each entry is a list of columns for a composite index
|
||||
- **Dialect**: PG uses `TIMESTAMPTZ`/`BOOLEAN`; SQLite uses `TEXT`/`INTEGER`
|
||||
- **Catalog**: tracked in `ext_data_tables` per package for lifecycle management
|
||||
|
||||
### Extension Tools (v0.29.2)
|
||||
|
||||
Starlark packages declare server-side tools in `manifest.tools`. These
|
||||
are included in `BuildToolDefs` alongside server tools. The completion
|
||||
tool loop dispatches matched calls to the `on_tool_call` entry point.
|
||||
|
||||
**Manifest `tools` field (starlark tier only):**
|
||||
|
||||
```json
|
||||
{
|
||||
"tier": "starlark",
|
||||
"permissions": ["db.read"],
|
||||
"tools": [
|
||||
{
|
||||
"name": "search_logs",
|
||||
"description": "Search extension log entries",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"},
|
||||
"user_id": {"type": "string"}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
**`on_tool_call(call)` entry point:**
|
||||
|
||||
Called by the completion tool loop when a tool declared in `tools` is
|
||||
invoked by the LLM.
|
||||
|
||||
```python
|
||||
def on_tool_call(call):
|
||||
# call dict:
|
||||
# {
|
||||
# "tool_name": "search_logs",
|
||||
# "tool_call_id": "call_abc123",
|
||||
# "arguments": {"query": "hello", "user_id": "u1"}
|
||||
# }
|
||||
if call["tool_name"] == "search_logs":
|
||||
rows = db.query("logs", filters={"user_id": call["arguments"]["user_id"]})
|
||||
return {"results": rows}
|
||||
return {"error": "unknown tool"}
|
||||
```
|
||||
|
||||
- Return value is serialized to JSON and returned as the tool result
|
||||
- All sandbox modules (including `db`) are available per granted permissions
|
||||
- No additional permission is required beyond package being `active`
|
||||
|
||||
### Builtin Seeding
|
||||
|
||||
On startup, `SeedBuiltinExtensions()` scans `extensions/builtin/`
|
||||
|
||||
@@ -32,7 +32,7 @@ v0.9.x–v0.28.7 Foundation through Platform Polish ✅
|
||||
│ │
|
||||
v0.29.0 Starlark Sandbox ✅ v0.32.0 Multi-Replica HA
|
||||
v0.29.1 API Extensions ✅ v0.33.0 Observability
|
||||
v0.29.2 DB Extensions v0.34.0 Data Portability
|
||||
v0.29.2 DB Extensions ✅ v0.34.0 Data Portability
|
||||
v0.29.3 Workflow Forms │
|
||||
v0.30.0 Package Lifecycle │
|
||||
v0.30.1 SDK Adoption │
|
||||
@@ -188,17 +188,19 @@ Depends on: v0.29.0.
|
||||
- Server-side tool execution in completion handler (requires tool
|
||||
registry integration with sandbox; aligns with DB extensions scope)
|
||||
|
||||
### v0.29.2 — DB Extensions
|
||||
### v0.29.2 — DB Extensions ✅
|
||||
|
||||
Namespaced tables for extension data. Create-only (no migrations yet).
|
||||
Namespaced tables for extension data. Structured API, not raw SQL.
|
||||
Server-side tool execution (deferred from v0.29.1) included.
|
||||
|
||||
Depends on: v0.29.1.
|
||||
|
||||
- [ ] `ext_{id}_*` tables, dialect-correct DDL (PG + SQLite)
|
||||
- [ ] `db` Starlark module: `query()`, `exec()` scoped to extension tables
|
||||
- [ ] Views as read contract over platform tables (column allowlist)
|
||||
- [ ] Schema creation on install, drop on uninstall
|
||||
- [ ] Server-side tool execution in completion handler (deferred from v0.29.1)
|
||||
- [x] `ext_{id}_*` tables, dialect-correct DDL (PG + SQLite)
|
||||
- [x] `db` Starlark module: structured `query/insert/update/delete/list_tables/view`
|
||||
(structured API instead of raw `exec()` — prevents SQL injection)
|
||||
- [x] Views as read contract over platform tables (`ext_view_users`, `ext_view_channels`)
|
||||
- [x] Schema creation on install, drop on uninstall
|
||||
- [x] Server-side tool execution in completion handler (deferred from v0.29.1)
|
||||
|
||||
### v0.29.3 — Workflow Forms
|
||||
|
||||
|
||||
@@ -86,8 +86,9 @@ server {
|
||||
|
||||
# v0.27.0: Extension surface static assets (JS, CSS, images)
|
||||
# v0.28.7: On-disk path moved to /data/packages/, URL retained for stability.
|
||||
location /surfaces/ {
|
||||
alias /data/packages/;
|
||||
# v0.29.2: ^~ prefix beats regex; corrected alias to /data/storage/packages/.
|
||||
location ^~ /surfaces/ {
|
||||
alias /data/storage/packages/;
|
||||
expires 1h;
|
||||
add_header Cache-Control "public";
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
--
|
||||
-- v0.28.7: packages table + package_user_settings in 012.
|
||||
-- v0.29.0: status column + extension_permissions table.
|
||||
-- v0.29.2: ext_data_tables catalog + platform read views.
|
||||
-- ==========================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS packages (
|
||||
@@ -88,3 +89,39 @@ CREATE INDEX IF NOT EXISTS idx_ext_perm_granted ON extension_permissions(granted
|
||||
COMMENT ON TABLE extension_permissions IS 'Declared permissions from package manifests. Admin grants control runtime module injection.';
|
||||
COMMENT ON COLUMN extension_permissions.permission IS 'Capability key: secrets.read, notifications.send, filters.pre_completion, db.read, db.write, api.http';
|
||||
COMMENT ON COLUMN extension_permissions.granted IS 'Admin has reviewed and approved this capability for the package';
|
||||
|
||||
|
||||
-- =========================================
|
||||
-- EXTENSION DATA TABLES CATALOG
|
||||
-- =========================================
|
||||
-- Tracks namespaced tables created for each extension on install.
|
||||
-- Used by uninstall hooks (DROP TABLE) and db.list_tables().
|
||||
-- Table names stored without the ext_{id}_ prefix.
|
||||
-- v0.29.2: Extension data storage.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ext_data_tables (
|
||||
package_id TEXT NOT NULL REFERENCES packages(id) ON DELETE CASCADE,
|
||||
table_name TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (package_id, table_name)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_ext_data_tables_pkg ON ext_data_tables(package_id);
|
||||
|
||||
COMMENT ON TABLE ext_data_tables IS 'Catalog of namespaced tables created by extension install hooks. Drives uninstall cleanup and db.list_tables().';
|
||||
COMMENT ON COLUMN ext_data_tables.table_name IS 'Logical name (without ext_{id}_ prefix). Physical table is ext_{package_id}_{table_name}.';
|
||||
|
||||
|
||||
-- =========================================
|
||||
-- PLATFORM VIEWS (extension read access)
|
||||
-- =========================================
|
||||
-- Column-allowlisted views over core platform tables.
|
||||
-- Extensions query these via db.view("users"), db.view("channels").
|
||||
-- Only safe, non-sensitive columns are exposed.
|
||||
-- v0.29.2: Read contract for cross-platform data access.
|
||||
|
||||
CREATE OR REPLACE VIEW ext_view_users AS
|
||||
SELECT id, display_name, email FROM users;
|
||||
|
||||
CREATE OR REPLACE VIEW ext_view_channels AS
|
||||
SELECT id, title, type, team_id FROM channels;
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
-- Unified package registry. Replaces surface_registry + extensions.
|
||||
-- v0.28.7: packages + package_user_settings
|
||||
-- v0.29.0: status column + extension_permissions table
|
||||
-- v0.29.2: ext_data_tables catalog + platform read views
|
||||
|
||||
CREATE TABLE IF NOT EXISTS packages (
|
||||
id TEXT PRIMARY KEY,
|
||||
@@ -61,3 +62,28 @@ CREATE TABLE IF NOT EXISTS extension_permissions (
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_ext_perm_package ON extension_permissions(package_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_ext_perm_granted ON extension_permissions(granted);
|
||||
|
||||
|
||||
-- Extension data tables catalog (v0.29.2)
|
||||
-- Tracks namespaced tables created by extension install hooks.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ext_data_tables (
|
||||
package_id TEXT NOT NULL REFERENCES packages(id) ON DELETE CASCADE,
|
||||
table_name TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
PRIMARY KEY (package_id, table_name)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_ext_data_tables_pkg ON ext_data_tables(package_id);
|
||||
|
||||
|
||||
-- Platform views — column-allowlisted read access for extensions (v0.29.2)
|
||||
-- Extensions query these via db.view("users"), db.view("channels").
|
||||
|
||||
DROP VIEW IF EXISTS ext_view_users;
|
||||
CREATE VIEW ext_view_users AS
|
||||
SELECT id, display_name, email FROM users;
|
||||
|
||||
DROP VIEW IF EXISTS ext_view_channels;
|
||||
CREATE VIEW ext_view_channels AS
|
||||
SELECT id, title, type, team_id FROM channels;
|
||||
|
||||
@@ -11,6 +11,7 @@ require (
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/minio/minio-go/v7 v7.0.82
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
go.starlark.net v0.0.0-20260210143700-b62fd896b91b
|
||||
golang.org/x/crypto v0.28.0
|
||||
golang.org/x/net v0.30.0
|
||||
modernc.org/sqlite v1.34.5
|
||||
@@ -44,7 +45,7 @@ require (
|
||||
golang.org/x/arch v0.3.0 // indirect
|
||||
golang.org/x/sys v0.26.0 // indirect
|
||||
golang.org/x/text v0.19.0 // indirect
|
||||
google.golang.org/protobuf v1.30.0 // indirect
|
||||
google.golang.org/protobuf v1.33.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/libc v1.55.3 // indirect
|
||||
modernc.org/mathutil v1.6.0 // indirect
|
||||
|
||||
@@ -29,7 +29,6 @@ github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
|
||||
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
|
||||
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
@@ -72,6 +71,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
@@ -90,6 +91,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
|
||||
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
go.starlark.net v0.0.0-20260210143700-b62fd896b91b h1:mDO9/2PuBcapqFbhiCmFcEQZvlQnk3ILEZR+a8NL1z4=
|
||||
go.starlark.net v0.0.0-20260210143700-b62fd896b91b/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8=
|
||||
golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
|
||||
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
|
||||
@@ -105,15 +108,16 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
|
||||
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
|
||||
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
|
||||
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
|
||||
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
|
||||
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
|
||||
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"git.gobha.me/xcaliber/chat-switchboard/notifications"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/providers"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/routing"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/sandbox"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/storage"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/tools"
|
||||
@@ -57,6 +58,7 @@ type CompletionHandler struct {
|
||||
healthStore HealthStatusQuerier // health status queries for routing (v0.22.2, nil = disabled)
|
||||
router *routing.Evaluator // routing policy evaluator (v0.22.2, nil = disabled)
|
||||
filterChain *filters.Chain // pre-completion filter chain (v0.29.0, nil = disabled)
|
||||
runner *sandbox.Runner // v0.29.2: Starlark extension tool dispatch (nil = disabled)
|
||||
}
|
||||
|
||||
// HealthRecorder is the interface for recording provider call outcomes.
|
||||
@@ -101,6 +103,16 @@ func (h *CompletionHandler) SetFilterChain(fc *filters.Chain) {
|
||||
h.filterChain = fc
|
||||
}
|
||||
|
||||
// SetRunner attaches the Starlark sandbox runner for extension tool dispatch (v0.29.2).
|
||||
func (h *CompletionHandler) SetRunner(r *sandbox.Runner) {
|
||||
h.runner = r
|
||||
}
|
||||
|
||||
// buildExtToolMap returns the extension tool map for the current user.
|
||||
func (h *CompletionHandler) buildExtToolMap(ctx context.Context, userID string) map[string]*store.PackageRegistration {
|
||||
return BuildExtToolMap(ctx, h.stores, userID)
|
||||
}
|
||||
|
||||
// evaluateRouting applies routing policies to select the best provider config
|
||||
// for this request. Returns the winning config details and a routing decision
|
||||
// for observability. If routing is disabled or no policies match, returns
|
||||
@@ -846,7 +858,8 @@ func (h *CompletionHandler) multiModelStream(
|
||||
}
|
||||
|
||||
// Stream this model's response using the shared streaming core
|
||||
result := streamModelResponse(c, provider, providerCfg, &provReq, model, providerID, target.DisplayName, userID, channelID, personaID, workspaceID, configID, teamID, h.hub, h.health)
|
||||
multiExtTools := h.buildExtToolMap(c.Request.Context(), userID)
|
||||
result := streamModelResponse(c, provider, providerCfg, &provReq, model, providerID, target.DisplayName, userID, channelID, personaID, workspaceID, configID, teamID, h.hub, h.health, h.runner, multiExtTools)
|
||||
|
||||
// Persist assistant message with model attribution
|
||||
if result.Content != "" {
|
||||
@@ -955,7 +968,8 @@ func (h *CompletionHandler) streamCompletion(
|
||||
hooks.PreRequest(cfg, &req)
|
||||
}
|
||||
|
||||
result := streamWithToolLoop(c, provider, cfg, &req, model, providerID, userID, channelID, personaID, workspaceID, configID, teamID, h.hub, h.health)
|
||||
extTools := h.buildExtToolMap(c.Request.Context(), userID)
|
||||
result := streamWithToolLoop(c, provider, cfg, &req, model, providerID, userID, channelID, personaID, workspaceID, configID, teamID, h.hub, h.health, h.runner, extTools)
|
||||
|
||||
// Persist assistant response
|
||||
if result.Content != "" {
|
||||
@@ -995,6 +1009,7 @@ func (h *CompletionHandler) syncCompletion(
|
||||
hooks.PreRequest(cfg, &req)
|
||||
}
|
||||
|
||||
extTools := h.buildExtToolMap(c.Request.Context(), userID)
|
||||
result := CoreToolLoop(c.Request.Context(), LoopConfig{
|
||||
Provider: provider,
|
||||
Cfg: cfg,
|
||||
@@ -1013,6 +1028,8 @@ func (h *CompletionHandler) syncCompletion(
|
||||
ConfigID: configID,
|
||||
Budget: LoopBudget{},
|
||||
Streaming: false,
|
||||
Runner: h.runner,
|
||||
ExtTools: extTools,
|
||||
}, accumSink{})
|
||||
|
||||
// Provider error
|
||||
|
||||
215
server/handlers/ext_db_schema.go
Normal file
215
server/handlers/ext_db_schema.go
Normal file
@@ -0,0 +1,215 @@
|
||||
package handlers
|
||||
|
||||
// ext_db_schema.go — v0.29.2
|
||||
//
|
||||
// DDL generation and lifecycle management for extension-owned database tables.
|
||||
// Tables are namespaced as ext_{pkg_slug}_{logical_name} and tracked in the
|
||||
// ext_data_tables catalog. CreateExtTables is called on install;
|
||||
// DropExtTables is called on uninstall.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
// validSchemaIdentifier matches safe SQL identifiers for table and column names.
|
||||
var validSchemaIdentifier = regexp.MustCompile(`^[a-z][a-z0-9_]{0,62}$`)
|
||||
|
||||
// TableDef describes a single extension-owned table as declared in a manifest.
|
||||
type TableDef struct {
|
||||
Columns map[string]string // colName → manifest type ("text","int","real","bool","timestamp")
|
||||
Indexes [][]string // each inner slice is an ordered list of column names for one index
|
||||
}
|
||||
|
||||
// ParseDBTables extracts the "db_tables" key from a package manifest map.
|
||||
// Returns the map of logicalName→TableDef and ok=true when the key is present
|
||||
// and produces at least one valid table definition.
|
||||
func ParseDBTables(manifest map[string]any) (map[string]TableDef, bool) {
|
||||
raw, ok := manifest["db_tables"]
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
tablesRaw, ok := raw.(map[string]any)
|
||||
if !ok || len(tablesRaw) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
tables := make(map[string]TableDef, len(tablesRaw))
|
||||
for name, defRaw := range tablesRaw {
|
||||
if !validSchemaIdentifier.MatchString(name) {
|
||||
log.Printf("[ext-db] skipping invalid table name %q", name)
|
||||
continue
|
||||
}
|
||||
defMap, ok := defRaw.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
td := TableDef{Columns: map[string]string{}}
|
||||
|
||||
// Parse columns map.
|
||||
if colsRaw, ok := defMap["columns"].(map[string]any); ok {
|
||||
for col, typ := range colsRaw {
|
||||
if !validSchemaIdentifier.MatchString(col) {
|
||||
log.Printf("[ext-db] skipping invalid column name %q in table %q", col, name)
|
||||
continue
|
||||
}
|
||||
if t, ok := typ.(string); ok {
|
||||
td.Columns[col] = t
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Parse indexes array.
|
||||
if idxsRaw, ok := defMap["indexes"].([]any); ok {
|
||||
for _, idxRaw := range idxsRaw {
|
||||
if idxArr, ok := idxRaw.([]any); ok {
|
||||
var cols []string
|
||||
for _, c := range idxArr {
|
||||
if s, ok := c.(string); ok && validSchemaIdentifier.MatchString(s) {
|
||||
cols = append(cols, s)
|
||||
}
|
||||
}
|
||||
if len(cols) > 0 {
|
||||
td.Indexes = append(td.Indexes, cols)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tables[name] = td
|
||||
}
|
||||
|
||||
if len(tables) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
return tables, true
|
||||
}
|
||||
|
||||
// mapColType maps a manifest type string to a SQL column type for the given dialect.
|
||||
func mapColType(typStr string, isPostgres bool) string {
|
||||
switch strings.ToLower(typStr) {
|
||||
case "int", "integer":
|
||||
return "INTEGER"
|
||||
case "real", "float":
|
||||
return "REAL"
|
||||
case "bool", "boolean":
|
||||
if isPostgres {
|
||||
return "BOOLEAN"
|
||||
}
|
||||
return "INTEGER"
|
||||
case "timestamp":
|
||||
if isPostgres {
|
||||
return "TIMESTAMPTZ"
|
||||
}
|
||||
return "TEXT"
|
||||
default: // "text" and anything unrecognized
|
||||
return "TEXT"
|
||||
}
|
||||
}
|
||||
|
||||
// extPhysicalTable builds the namespaced physical table name for a package's logical table.
|
||||
// e.g. packageID="my-ext", logicalName="logs" → "ext_my_ext_logs"
|
||||
func extPhysicalTable(packageID, logicalName string) string {
|
||||
slug := strings.ReplaceAll(packageID, "-", "_")
|
||||
return "ext_" + slug + "_" + logicalName
|
||||
}
|
||||
|
||||
// createdAtColDef returns the dialect-correct DDL fragment for the created_at column.
|
||||
func createdAtColDef(isPostgres bool) string {
|
||||
if isPostgres {
|
||||
return "created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()"
|
||||
}
|
||||
return "created_at TEXT NOT NULL DEFAULT (datetime('now'))"
|
||||
}
|
||||
|
||||
// CreateExtTables generates dialect-correct DDL and executes it for all tables
|
||||
// in the provided map. Each successfully created table is registered in the
|
||||
// ext_data_tables catalog via stores.ExtData.
|
||||
//
|
||||
// Errors from individual table creation are returned immediately (fail-fast).
|
||||
// Index creation failures are logged but non-fatal.
|
||||
// Catalog registration failures are logged but non-fatal.
|
||||
func CreateExtTables(
|
||||
ctx context.Context,
|
||||
db *sql.DB,
|
||||
isPostgres bool,
|
||||
stores store.Stores,
|
||||
packageID string,
|
||||
tables map[string]TableDef,
|
||||
) error {
|
||||
if db == nil {
|
||||
return nil
|
||||
}
|
||||
for logicalName, td := range tables {
|
||||
physical := extPhysicalTable(packageID, logicalName)
|
||||
|
||||
// Build column list: id first, user columns, created_at last.
|
||||
cols := []string{"id TEXT PRIMARY KEY"}
|
||||
for col, typ := range td.Columns {
|
||||
cols = append(cols, col+" "+mapColType(typ, isPostgres))
|
||||
}
|
||||
cols = append(cols, createdAtColDef(isPostgres))
|
||||
|
||||
ddl := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n\t%s\n)",
|
||||
physical, strings.Join(cols, ",\n\t"))
|
||||
|
||||
if _, err := db.ExecContext(ctx, ddl); err != nil {
|
||||
return fmt.Errorf("create ext table %s: %w", physical, err)
|
||||
}
|
||||
|
||||
// Create declared indexes.
|
||||
for i, idxCols := range td.Indexes {
|
||||
idxName := fmt.Sprintf("idx_%s_%d", physical, i)
|
||||
idxDDL := fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (%s)",
|
||||
idxName, physical, strings.Join(idxCols, ", "))
|
||||
if _, err := db.ExecContext(ctx, idxDDL); err != nil {
|
||||
log.Printf("[ext-db] index create failed (%s): %v", idxName, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Record logical name in catalog.
|
||||
if stores.ExtData != nil {
|
||||
if err := stores.ExtData.RegisterTable(ctx, packageID, logicalName); err != nil {
|
||||
log.Printf("[ext-db] catalog register failed (%s/%s): %v", packageID, logicalName, err)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("[ext-db] created table %s for package %s", physical, packageID)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DropExtTables drops all physical tables registered for a package and removes
|
||||
// their entries from the ext_data_tables catalog.
|
||||
//
|
||||
// Table drop errors are logged but non-fatal; the catalog is cleared regardless.
|
||||
func DropExtTables(
|
||||
ctx context.Context,
|
||||
db *sql.DB,
|
||||
stores store.Stores,
|
||||
packageID string,
|
||||
) error {
|
||||
if db == nil || stores.ExtData == nil {
|
||||
return nil
|
||||
}
|
||||
tables, err := stores.ExtData.ListTables(ctx, packageID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list ext tables for %s: %w", packageID, err)
|
||||
}
|
||||
for _, logicalName := range tables {
|
||||
physical := extPhysicalTable(packageID, logicalName)
|
||||
if _, err := db.ExecContext(ctx, "DROP TABLE IF EXISTS "+physical); err != nil {
|
||||
log.Printf("[ext-db] drop table %s failed: %v", physical, err)
|
||||
} else {
|
||||
log.Printf("[ext-db] dropped table %s for package %s", physical, packageID)
|
||||
}
|
||||
}
|
||||
return stores.ExtData.DeletePackageTables(ctx, packageID)
|
||||
}
|
||||
350
server/handlers/ext_db_schema_test.go
Normal file
350
server/handlers/ext_db_schema_test.go
Normal file
@@ -0,0 +1,350 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
// ── mock ExtDataStore ────────────────────────────────────────────────────────
|
||||
|
||||
// memExtDataStore is an in-memory ExtDataStore for testing.
|
||||
type memExtDataStore struct {
|
||||
tables map[string][]string // packageID → []logicalName
|
||||
}
|
||||
|
||||
func newMemExtDataStore() *memExtDataStore {
|
||||
return &memExtDataStore{tables: map[string][]string{}}
|
||||
}
|
||||
|
||||
func (m *memExtDataStore) RegisterTable(_ context.Context, packageID, tableName string) error {
|
||||
m.tables[packageID] = append(m.tables[packageID], tableName)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memExtDataStore) ListTables(_ context.Context, packageID string) ([]string, error) {
|
||||
return m.tables[packageID], nil
|
||||
}
|
||||
|
||||
func (m *memExtDataStore) DeletePackageTables(_ context.Context, packageID string) error {
|
||||
delete(m.tables, packageID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
func newSchemaTestDB(t *testing.T) *sql.DB {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return db
|
||||
}
|
||||
|
||||
func storesWithExtData(m *memExtDataStore) store.Stores {
|
||||
return store.Stores{ExtData: m}
|
||||
}
|
||||
|
||||
// ── ParseDBTables ────────────────────────────────────────────────────────────
|
||||
|
||||
func TestParseDBTables_Valid(t *testing.T) {
|
||||
manifest := map[string]any{
|
||||
"db_tables": map[string]any{
|
||||
"logs": map[string]any{
|
||||
"columns": map[string]any{
|
||||
"message": "text",
|
||||
"count": "int",
|
||||
},
|
||||
"indexes": []any{[]any{"message"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
tables, ok := ParseDBTables(manifest)
|
||||
if !ok {
|
||||
t.Fatal("expected ok=true")
|
||||
}
|
||||
if _, has := tables["logs"]; !has {
|
||||
t.Fatal("expected 'logs' in parsed tables")
|
||||
}
|
||||
if tables["logs"].Columns["message"] != "text" {
|
||||
t.Errorf("expected column message=text, got %q", tables["logs"].Columns["message"])
|
||||
}
|
||||
if len(tables["logs"].Indexes) != 1 {
|
||||
t.Errorf("expected 1 index, got %d", len(tables["logs"].Indexes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDBTables_Missing(t *testing.T) {
|
||||
_, ok := ParseDBTables(map[string]any{"title": "hi"})
|
||||
if ok {
|
||||
t.Fatal("expected ok=false when db_tables absent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDBTables_InvalidTableName(t *testing.T) {
|
||||
manifest := map[string]any{
|
||||
"db_tables": map[string]any{
|
||||
"bad name": map[string]any{},
|
||||
"good_name": map[string]any{"columns": map[string]any{"x": "text"}},
|
||||
},
|
||||
}
|
||||
tables, ok := ParseDBTables(manifest)
|
||||
if !ok {
|
||||
t.Fatal("expected ok=true (good_name is valid)")
|
||||
}
|
||||
if _, has := tables["bad name"]; has {
|
||||
t.Error("invalid table name should be skipped")
|
||||
}
|
||||
if _, has := tables["good_name"]; !has {
|
||||
t.Error("valid table name should be present")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDBTables_InvalidColumnName(t *testing.T) {
|
||||
manifest := map[string]any{
|
||||
"db_tables": map[string]any{
|
||||
"events": map[string]any{
|
||||
"columns": map[string]any{
|
||||
"bad col": "text",
|
||||
"ok_col": "text",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
tables, ok := ParseDBTables(manifest)
|
||||
if !ok {
|
||||
t.Fatal("expected ok=true")
|
||||
}
|
||||
if _, has := tables["events"].Columns["bad col"]; has {
|
||||
t.Error("invalid column name should be skipped")
|
||||
}
|
||||
if _, has := tables["events"].Columns["ok_col"]; !has {
|
||||
t.Error("valid column name should be present")
|
||||
}
|
||||
}
|
||||
|
||||
// ── mapColType ───────────────────────────────────────────────────────────────
|
||||
|
||||
func TestMapColType_SQLite(t *testing.T) {
|
||||
cases := []struct{ in, want string }{
|
||||
{"text", "TEXT"},
|
||||
{"int", "INTEGER"},
|
||||
{"integer", "INTEGER"},
|
||||
{"real", "REAL"},
|
||||
{"float", "REAL"},
|
||||
{"bool", "INTEGER"},
|
||||
{"boolean", "INTEGER"},
|
||||
{"timestamp", "TEXT"},
|
||||
{"unknown", "TEXT"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got := mapColType(c.in, false)
|
||||
if got != c.want {
|
||||
t.Errorf("mapColType(%q, sqlite) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapColType_Postgres(t *testing.T) {
|
||||
if mapColType("bool", true) != "BOOLEAN" {
|
||||
t.Error("expected BOOLEAN for bool in postgres")
|
||||
}
|
||||
if mapColType("timestamp", true) != "TIMESTAMPTZ" {
|
||||
t.Error("expected TIMESTAMPTZ for timestamp in postgres")
|
||||
}
|
||||
if mapColType("int", true) != "INTEGER" {
|
||||
t.Error("expected INTEGER for int in postgres")
|
||||
}
|
||||
}
|
||||
|
||||
// ── extPhysicalTable ─────────────────────────────────────────────────────────
|
||||
|
||||
func TestExtPhysicalTable(t *testing.T) {
|
||||
got := extPhysicalTable("my-ext", "logs")
|
||||
if got != "ext_my_ext_logs" {
|
||||
t.Errorf("got %q, want ext_my_ext_logs", got)
|
||||
}
|
||||
got = extPhysicalTable("simple", "events")
|
||||
if got != "ext_simple_events" {
|
||||
t.Errorf("got %q, want ext_simple_events", got)
|
||||
}
|
||||
}
|
||||
|
||||
// ── CreateExtTables ──────────────────────────────────────────────────────────
|
||||
|
||||
func TestCreateExtTables_CreatesTable(t *testing.T) {
|
||||
db := newSchemaTestDB(t)
|
||||
m := newMemExtDataStore()
|
||||
ctx := context.Background()
|
||||
|
||||
tables := map[string]TableDef{
|
||||
"logs": {
|
||||
Columns: map[string]string{"message": "text", "count": "int"},
|
||||
},
|
||||
}
|
||||
if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "test-pkg", tables); err != nil {
|
||||
t.Fatalf("CreateExtTables: %v", err)
|
||||
}
|
||||
|
||||
// Verify table exists by inserting a row.
|
||||
_, err := db.ExecContext(ctx,
|
||||
`INSERT INTO ext_test_pkg_logs (id, message, count) VALUES ('1', 'hello', 42)`)
|
||||
if err != nil {
|
||||
t.Errorf("table not created or wrong schema: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateExtTables_WithIndexes(t *testing.T) {
|
||||
db := newSchemaTestDB(t)
|
||||
m := newMemExtDataStore()
|
||||
ctx := context.Background()
|
||||
|
||||
tables := map[string]TableDef{
|
||||
"events": {
|
||||
Columns: map[string]string{"user_id": "text", "kind": "text"},
|
||||
Indexes: [][]string{{"user_id"}, {"user_id", "kind"}},
|
||||
},
|
||||
}
|
||||
if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "idx-pkg", tables); err != nil {
|
||||
t.Fatalf("CreateExtTables: %v", err)
|
||||
}
|
||||
|
||||
// Verify indexes exist via sqlite_master.
|
||||
rows, err := db.QueryContext(ctx,
|
||||
`SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='ext_idx_pkg_events'`)
|
||||
if err != nil {
|
||||
t.Fatalf("query indexes: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var idxNames []string
|
||||
for rows.Next() {
|
||||
var name string
|
||||
rows.Scan(&name)
|
||||
idxNames = append(idxNames, name)
|
||||
}
|
||||
if len(idxNames) < 2 {
|
||||
t.Errorf("expected at least 2 indexes, got %d: %v", len(idxNames), idxNames)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateExtTables_CatalogRegistration(t *testing.T) {
|
||||
db := newSchemaTestDB(t)
|
||||
m := newMemExtDataStore()
|
||||
ctx := context.Background()
|
||||
|
||||
tables := map[string]TableDef{
|
||||
"logs": {Columns: map[string]string{"msg": "text"}},
|
||||
"errors": {Columns: map[string]string{"detail": "text"}},
|
||||
}
|
||||
if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "cat-pkg", tables); err != nil {
|
||||
t.Fatalf("CreateExtTables: %v", err)
|
||||
}
|
||||
|
||||
registered := m.tables["cat-pkg"]
|
||||
if len(registered) != 2 {
|
||||
t.Errorf("expected 2 registered tables, got %d: %v", len(registered), registered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateExtTables_NilDBIsNoop(t *testing.T) {
|
||||
m := newMemExtDataStore()
|
||||
ctx := context.Background()
|
||||
// Should not panic or error.
|
||||
err := CreateExtTables(ctx, nil, false, storesWithExtData(m), "pkg", map[string]TableDef{
|
||||
"t": {Columns: map[string]string{"x": "text"}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Errorf("expected nil error for nil db, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ── DropExtTables ─────────────────────────────────────────────────────────────
|
||||
|
||||
func TestDropExtTables_DropsAndClearsCatalog(t *testing.T) {
|
||||
db := newSchemaTestDB(t)
|
||||
m := newMemExtDataStore()
|
||||
ctx := context.Background()
|
||||
|
||||
// Create table first.
|
||||
tables := map[string]TableDef{
|
||||
"items": {Columns: map[string]string{"name": "text"}},
|
||||
}
|
||||
if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "drop-pkg", tables); err != nil {
|
||||
t.Fatalf("CreateExtTables: %v", err)
|
||||
}
|
||||
|
||||
// Drop.
|
||||
if err := DropExtTables(ctx, db, storesWithExtData(m), "drop-pkg"); err != nil {
|
||||
t.Fatalf("DropExtTables: %v", err)
|
||||
}
|
||||
|
||||
// Table should be gone.
|
||||
var count int
|
||||
db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='ext_drop_pkg_items'`,
|
||||
).Scan(&count)
|
||||
if count != 0 {
|
||||
t.Error("expected table to be dropped, still present")
|
||||
}
|
||||
|
||||
// Catalog should be empty.
|
||||
if len(m.tables["drop-pkg"]) != 0 {
|
||||
t.Errorf("expected empty catalog, got: %v", m.tables["drop-pkg"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDropExtTables_MultipleTables(t *testing.T) {
|
||||
db := newSchemaTestDB(t)
|
||||
m := newMemExtDataStore()
|
||||
ctx := context.Background()
|
||||
|
||||
tables := map[string]TableDef{
|
||||
"alpha": {Columns: map[string]string{"v": "text"}},
|
||||
"beta": {Columns: map[string]string{"v": "text"}},
|
||||
}
|
||||
CreateExtTables(ctx, db, false, storesWithExtData(m), "multi-pkg", tables)
|
||||
DropExtTables(ctx, db, storesWithExtData(m), "multi-pkg")
|
||||
|
||||
for _, name := range []string{"ext_multi_pkg_alpha", "ext_multi_pkg_beta"} {
|
||||
var count int
|
||||
db.QueryRowContext(ctx,
|
||||
`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?`, name,
|
||||
).Scan(&count)
|
||||
if count != 0 {
|
||||
t.Errorf("expected %s to be dropped", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateExtTables_DDLContainsAutoColumns(t *testing.T) {
|
||||
// Verify that the DDL-generated table has 'id' and 'created_at' columns
|
||||
// even when no columns are declared in the manifest.
|
||||
db := newSchemaTestDB(t)
|
||||
m := newMemExtDataStore()
|
||||
ctx := context.Background()
|
||||
|
||||
tables := map[string]TableDef{"empty": {Columns: map[string]string{}}}
|
||||
if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "auto-pkg", tables); err != nil {
|
||||
t.Fatalf("CreateExtTables: %v", err)
|
||||
}
|
||||
|
||||
// Insert with just id — should work (created_at has default).
|
||||
_, err := db.ExecContext(ctx, `INSERT INTO ext_auto_pkg_empty (id) VALUES ('x')`)
|
||||
if err != nil {
|
||||
t.Errorf("expected auto-columns (id, created_at) to exist: %v", err)
|
||||
}
|
||||
|
||||
// Verify created_at was populated.
|
||||
var ca string
|
||||
db.QueryRowContext(ctx, `SELECT created_at FROM ext_auto_pkg_empty WHERE id='x'`).Scan(&ca)
|
||||
if strings.TrimSpace(ca) == "" {
|
||||
t.Error("expected created_at to be populated by default")
|
||||
}
|
||||
}
|
||||
166
server/handlers/ext_tools_test.go
Normal file
166
server/handlers/ext_tools_test.go
Normal file
@@ -0,0 +1,166 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"go.starlark.net/starlark"
|
||||
)
|
||||
|
||||
// ── jsonToStarlark ────────────────────────────────────────────────────────────
|
||||
|
||||
func TestJSONToStarlark_String(t *testing.T) {
|
||||
v := jsonToStarlark("hello")
|
||||
s, ok := v.(starlark.String)
|
||||
if !ok || string(s) != "hello" {
|
||||
t.Errorf("expected starlark.String(hello), got %v (%T)", v, v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONToStarlark_Int(t *testing.T) {
|
||||
v := jsonToStarlark(float64(42))
|
||||
i, ok := v.(starlark.Int)
|
||||
if !ok {
|
||||
t.Fatalf("expected starlark.Int, got %T", v)
|
||||
}
|
||||
n, _ := i.Int64()
|
||||
if n != 42 {
|
||||
t.Errorf("expected 42, got %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONToStarlark_Float(t *testing.T) {
|
||||
v := jsonToStarlark(3.14)
|
||||
if _, ok := v.(starlark.Float); !ok {
|
||||
t.Errorf("expected starlark.Float, got %T", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONToStarlark_Bool(t *testing.T) {
|
||||
if jsonToStarlark(true) != starlark.True {
|
||||
t.Error("expected starlark.True")
|
||||
}
|
||||
if jsonToStarlark(false) != starlark.False {
|
||||
t.Error("expected starlark.False")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONToStarlark_Nil(t *testing.T) {
|
||||
if jsonToStarlark(nil) != starlark.None {
|
||||
t.Error("expected starlark.None for nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONToStarlark_Dict(t *testing.T) {
|
||||
v := jsonToStarlark(map[string]interface{}{"key": "val"})
|
||||
d, ok := v.(*starlark.Dict)
|
||||
if !ok {
|
||||
t.Fatalf("expected *starlark.Dict, got %T", v)
|
||||
}
|
||||
got, found, _ := d.Get(starlark.String("key"))
|
||||
if !found || string(got.(starlark.String)) != "val" {
|
||||
t.Errorf("expected key=val in dict, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONToStarlark_List(t *testing.T) {
|
||||
v := jsonToStarlark([]interface{}{"a", "b"})
|
||||
l, ok := v.(*starlark.List)
|
||||
if !ok {
|
||||
t.Fatalf("expected *starlark.List, got %T", v)
|
||||
}
|
||||
if l.Len() != 2 {
|
||||
t.Errorf("expected 2 elements, got %d", l.Len())
|
||||
}
|
||||
}
|
||||
|
||||
// ── starlarkValueToGo ─────────────────────────────────────────────────────────
|
||||
|
||||
func TestStarlarkValueToGo_String(t *testing.T) {
|
||||
v := starlarkValueToGo(starlark.String("hi"))
|
||||
if v != "hi" {
|
||||
t.Errorf("expected 'hi', got %v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStarlarkValueToGo_Int(t *testing.T) {
|
||||
v := starlarkValueToGo(starlark.MakeInt(7))
|
||||
if v != int64(7) {
|
||||
t.Errorf("expected int64(7), got %v (%T)", v, v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStarlarkValueToGo_Bool(t *testing.T) {
|
||||
if starlarkValueToGo(starlark.True) != true {
|
||||
t.Error("expected true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStarlarkValueToGo_None(t *testing.T) {
|
||||
if starlarkValueToGo(starlark.None) != nil {
|
||||
t.Error("expected nil for starlark.None")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStarlarkValueToGo_Dict(t *testing.T) {
|
||||
d := starlark.NewDict(1)
|
||||
_ = d.SetKey(starlark.String("x"), starlark.MakeInt(1))
|
||||
v := starlarkValueToGo(d)
|
||||
m, ok := v.(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("expected map, got %T", v)
|
||||
}
|
||||
if m["x"] != int64(1) {
|
||||
t.Errorf("expected x=1, got %v", m["x"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestStarlarkValueToGo_List(t *testing.T) {
|
||||
l := starlark.NewList([]starlark.Value{starlark.String("a"), starlark.String("b")})
|
||||
v := starlarkValueToGo(l)
|
||||
arr, ok := v.([]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("expected []interface{}, got %T", v)
|
||||
}
|
||||
if len(arr) != 2 || arr[0] != "a" {
|
||||
t.Errorf("unexpected list: %v", arr)
|
||||
}
|
||||
}
|
||||
|
||||
// ── RoundTrip: JSON → Starlark → Go → JSON ───────────────────────────────────
|
||||
|
||||
func TestRoundTrip_JSONToStarlarkToJSON(t *testing.T) {
|
||||
// Simulate what executeExtensionTool does: parse args, pass as Starlark, convert result back.
|
||||
argsJSON := `{"query": "hello", "limit": 10, "active": true}`
|
||||
var raw map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(argsJSON), &raw); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
sv := jsonToStarlark(raw)
|
||||
got := starlarkValueToGo(sv)
|
||||
out, err := json.Marshal(got)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal: %v", err)
|
||||
}
|
||||
// Verify key fields present
|
||||
var result map[string]interface{}
|
||||
if err := json.Unmarshal(out, &result); err != nil {
|
||||
t.Fatalf("unmarshal result: %v", err)
|
||||
}
|
||||
if result["query"] != "hello" {
|
||||
t.Errorf("expected query=hello, got %v", result["query"])
|
||||
}
|
||||
if result["active"] != true {
|
||||
t.Errorf("expected active=true, got %v", result["active"])
|
||||
}
|
||||
}
|
||||
|
||||
// ── BuildExtToolMap ───────────────────────────────────────────────────────────
|
||||
|
||||
func TestBuildExtToolMap_Empty(t *testing.T) {
|
||||
// nil Packages store → returns nil (no panic)
|
||||
result := BuildExtToolMap(nil, storesWithExtData(newMemExtDataStore()), "user1")
|
||||
if result != nil && len(result) != 0 {
|
||||
t.Errorf("expected empty map for stores without Packages, got %v", result)
|
||||
}
|
||||
}
|
||||
@@ -210,6 +210,13 @@ func (h *ExtensionHandler) AdminInstallExtension(c *gin.Context) {
|
||||
// If permissions are declared, package moves to pending_review.
|
||||
SyncManifestPermissions(c, h.stores, pkg.ID, manifestMap)
|
||||
|
||||
// v0.29.2: Create namespaced DB tables declared in the manifest.
|
||||
if tables, ok := ParseDBTables(manifestMap); ok {
|
||||
if err := CreateExtTables(c.Request.Context(), database.DB, database.IsPostgres(), h.stores, pkg.ID, tables); err != nil {
|
||||
log.Printf("[ext-db] schema create failed for %s: %v", pkg.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
c.JSON(201, gin.H{"data": pkg})
|
||||
}
|
||||
|
||||
@@ -370,6 +377,11 @@ func (h *ExtensionHandler) AdminUninstallExtension(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// v0.29.2: Drop namespaced DB tables before removing the package record.
|
||||
if err := DropExtTables(c.Request.Context(), database.DB, h.stores, id); err != nil {
|
||||
log.Printf("[ext-db] schema drop failed for %s: %v", id, err)
|
||||
}
|
||||
|
||||
if err := h.stores.Packages.Delete(c.Request.Context(), id); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
c.JSON(400, gin.H{"error": "core packages cannot be deleted"})
|
||||
|
||||
@@ -478,7 +478,7 @@ func (h *MessageHandler) Regenerate(c *gin.Context) {
|
||||
hooks.PreRequest(providerCfg, &provReq)
|
||||
}
|
||||
|
||||
result := streamWithToolLoop(c, provider, providerCfg, &provReq, model, providerID, userID, channelID, personaID, workspaceID, configID, msgTeamID, h.hub, comp.health)
|
||||
result := streamWithToolLoop(c, provider, providerCfg, &provReq, model, providerID, userID, channelID, personaID, workspaceID, configID, msgTeamID, h.hub, comp.health, comp.runner, comp.buildExtToolMap(c.Request.Context(), userID))
|
||||
|
||||
// Persist as sibling (regen) with tool activity
|
||||
if result.Content != "" {
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/database"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
@@ -117,6 +118,11 @@ func (h *PackageHandler) DeletePackage(c *gin.Context) {
|
||||
return
|
||||
}
|
||||
|
||||
// v0.29.2: Drop namespaced DB tables before removing the package record.
|
||||
if err := DropExtTables(c.Request.Context(), database.DB, h.stores, id); err != nil {
|
||||
log.Printf("[ext-db] schema drop failed for %s: %v", id, err)
|
||||
}
|
||||
|
||||
if err := h.stores.Packages.Delete(c.Request.Context(), id); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete package"})
|
||||
return
|
||||
@@ -358,6 +364,13 @@ func (h *PackageHandler) InstallPackage(c *gin.Context) {
|
||||
log.Printf("[packages] Failed to update package metadata for %s: %v", pkgID, err)
|
||||
}
|
||||
|
||||
// v0.29.2: Create namespaced DB tables declared in the manifest.
|
||||
if tables, ok := ParseDBTables(manifest); ok {
|
||||
if err := CreateExtTables(c.Request.Context(), database.DB, database.IsPostgres(), h.stores, pkgID, tables); err != nil {
|
||||
log.Printf("[ext-db] schema create failed for %s: %v", pkgID, err)
|
||||
}
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"id": pkgID,
|
||||
"title": title,
|
||||
|
||||
@@ -180,15 +180,25 @@ func BuildToolDefs(
|
||||
})
|
||||
}
|
||||
|
||||
// Append browser-defined tool schemas from extensions
|
||||
if includeBrowser && stores.Packages != nil {
|
||||
// Append browser-defined and starlark-tier tool schemas from extensions.
|
||||
// v0.29.2: starlark tools are always included (not gated on browser connection).
|
||||
if stores.Packages != nil {
|
||||
pkgs, err := stores.Packages.ListForUser(ctx, userID)
|
||||
if err != nil {
|
||||
log.Printf("⚠️ Failed to load extensions for tools: %v", err)
|
||||
return defs
|
||||
}
|
||||
for _, pkg := range pkgs {
|
||||
if pkg.Tier != "browser" {
|
||||
// Browser tools only when a browser client is connected.
|
||||
if pkg.Tier == "browser" && !includeBrowser {
|
||||
continue
|
||||
}
|
||||
// Only browser and starlark tiers expose tools this way.
|
||||
if pkg.Tier != "browser" && pkg.Tier != "starlark" {
|
||||
continue
|
||||
}
|
||||
// Starlark packages must be active to expose tools.
|
||||
if pkg.Tier == "starlark" && pkg.Status != "active" {
|
||||
continue
|
||||
}
|
||||
var manifest struct {
|
||||
@@ -196,7 +206,6 @@ func BuildToolDefs(
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Parameters json.RawMessage `json:"parameters"`
|
||||
Tier string `json:"tier"`
|
||||
} `json:"tools"`
|
||||
}
|
||||
if err := json.Unmarshal(marshalManifest(pkg.Manifest), &manifest); err != nil {
|
||||
@@ -243,6 +252,39 @@ func BuildToolDefs(
|
||||
return defs
|
||||
}
|
||||
|
||||
// BuildExtToolMap returns a map of toolName → PackageRegistration for all
|
||||
// active starlark-tier extensions that declare tools in their manifest.
|
||||
// Used by the tool loop to dispatch matched calls to on_tool_call.
|
||||
func BuildExtToolMap(ctx context.Context, stores store.Stores, userID string) map[string]*store.PackageRegistration {
|
||||
if stores.Packages == nil {
|
||||
return nil
|
||||
}
|
||||
pkgs, err := stores.Packages.ListForUser(ctx, userID)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
result := make(map[string]*store.PackageRegistration)
|
||||
for i, pkg := range pkgs {
|
||||
if pkg.Tier != "starlark" || pkg.Status != "active" {
|
||||
continue
|
||||
}
|
||||
var manifest struct {
|
||||
Tools []struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"tools"`
|
||||
}
|
||||
if json.Unmarshal(marshalManifest(pkg.Manifest), &manifest) != nil {
|
||||
continue
|
||||
}
|
||||
for _, t := range manifest.Tools {
|
||||
if t.Name != "" {
|
||||
result[t.Name] = &pkgs[i].PackageRegistration
|
||||
}
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// FilterToolDefsByGrants applies an additional allowlist to tool defs.
|
||||
// Used by the task scheduler to enforce task-level tool grants on top
|
||||
// of persona-level grants. Passing nil or empty grants returns defs unchanged.
|
||||
|
||||
@@ -13,6 +13,8 @@ import (
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/events"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/providers"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/sandbox"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/tools"
|
||||
)
|
||||
|
||||
@@ -49,6 +51,8 @@ func streamWithToolLoop(
|
||||
model, providerType, userID, channelID, personaID, workspaceID, configID, teamID string,
|
||||
hub *events.Hub,
|
||||
health HealthRecorder,
|
||||
runner *sandbox.Runner,
|
||||
extTools map[string]*store.PackageRegistration,
|
||||
) LoopResult {
|
||||
// Set SSE headers
|
||||
c.Header("Content-Type", "text/event-stream")
|
||||
@@ -77,6 +81,8 @@ func streamWithToolLoop(
|
||||
ConfigID: configID,
|
||||
Budget: LoopBudget{},
|
||||
Streaming: true,
|
||||
Runner: runner,
|
||||
ExtTools: extTools,
|
||||
}, sink)
|
||||
}
|
||||
|
||||
@@ -93,6 +99,8 @@ func streamModelResponse(
|
||||
model, providerType, displayName, userID, channelID, personaID, workspaceID, configID, teamID string,
|
||||
hub *events.Hub,
|
||||
health HealthRecorder,
|
||||
runner *sandbox.Runner,
|
||||
extTools map[string]*store.PackageRegistration,
|
||||
) LoopResult {
|
||||
sink := newSSEModelSink(c, model, displayName)
|
||||
|
||||
@@ -114,6 +122,8 @@ func streamModelResponse(
|
||||
ConfigID: configID,
|
||||
Budget: LoopBudget{},
|
||||
Streaming: true,
|
||||
Runner: runner,
|
||||
ExtTools: extTools,
|
||||
}, sink)
|
||||
}
|
||||
|
||||
|
||||
@@ -25,9 +25,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.starlark.net/starlark"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/events"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/providers"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/sandbox"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/tools"
|
||||
)
|
||||
|
||||
@@ -79,6 +82,9 @@ type LoopConfig struct {
|
||||
ConfigID string
|
||||
Budget LoopBudget
|
||||
Streaming bool // true = StreamCompletion, false = ChatCompletion
|
||||
// v0.29.2: extension tool dispatch (nil = no extension tools)
|
||||
Runner *sandbox.Runner
|
||||
ExtTools map[string]*store.PackageRegistration // toolName → package
|
||||
}
|
||||
|
||||
// ── Sink Interface ─────────────────────────────
|
||||
@@ -398,6 +404,12 @@ func executeToolCall(ctx context.Context, lcfg LoopConfig, call tools.ToolCall)
|
||||
return result
|
||||
}
|
||||
|
||||
// v0.29.2: Starlark extension tool
|
||||
if pkg, ok := lcfg.ExtTools[call.Name]; ok && lcfg.Runner != nil {
|
||||
log.Printf("🔧 Executing tool (extension): %s (call %s, pkg %s)", call.Name, call.ID, pkg.ID)
|
||||
return executeExtensionTool(ctx, lcfg, pkg, call)
|
||||
}
|
||||
|
||||
// Browser bridge (only available when hub is connected)
|
||||
if lcfg.Hub != nil && lcfg.Hub.IsConnected(lcfg.ExecCtx.UserID) {
|
||||
log.Printf("🔧 Executing tool (browser): %s (call %s)", call.Name, call.ID)
|
||||
@@ -414,6 +426,132 @@ func executeToolCall(ctx context.Context, lcfg LoopConfig, call tools.ToolCall)
|
||||
}
|
||||
}
|
||||
|
||||
// executeExtensionTool calls a starlark extension's on_tool_call entry point
|
||||
// and serializes the return value to a JSON string for the tool result.
|
||||
func executeExtensionTool(ctx context.Context, lcfg LoopConfig, pkg *store.PackageRegistration, call tools.ToolCall) tools.ToolResult {
|
||||
// Parse JSON arguments into a Starlark dict.
|
||||
var rawArgs map[string]interface{}
|
||||
if call.Arguments != "" {
|
||||
if err := json.Unmarshal([]byte(call.Arguments), &rawArgs); err != nil {
|
||||
return tools.ToolResult{
|
||||
ToolCallID: call.ID,
|
||||
Name: call.Name,
|
||||
Content: `{"error":"invalid tool arguments JSON"}`,
|
||||
IsError: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build the call dict passed to on_tool_call(call).
|
||||
callDict := starlark.NewDict(3)
|
||||
_ = callDict.SetKey(starlark.String("tool_name"), starlark.String(call.Name))
|
||||
_ = callDict.SetKey(starlark.String("tool_call_id"), starlark.String(call.ID))
|
||||
_ = callDict.SetKey(starlark.String("arguments"), jsonToStarlark(rawArgs))
|
||||
|
||||
rc := &sandbox.RunContext{
|
||||
UserID: lcfg.ExecCtx.UserID,
|
||||
ChannelID: lcfg.ExecCtx.ChannelID,
|
||||
}
|
||||
val, output, err := lcfg.Runner.CallEntryPoint(ctx, pkg, "on_tool_call",
|
||||
starlark.Tuple{callDict}, nil, rc)
|
||||
if output != "" {
|
||||
log.Printf(" 🔧 ext tool %s print: %s", pkg.ID, output)
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("⚠️ ext tool %s on_tool_call error: %v", pkg.ID, err)
|
||||
return tools.ToolResult{
|
||||
ToolCallID: call.ID,
|
||||
Name: call.Name,
|
||||
Content: fmt.Sprintf(`{"error":%q}`, err.Error()),
|
||||
IsError: true,
|
||||
}
|
||||
}
|
||||
|
||||
// Serialize the Starlark return value to JSON.
|
||||
content, jsonErr := json.Marshal(starlarkValueToGo(val))
|
||||
if jsonErr != nil {
|
||||
content = []byte(fmt.Sprintf(`{"error":%q}`, jsonErr.Error()))
|
||||
}
|
||||
return tools.ToolResult{
|
||||
ToolCallID: call.ID,
|
||||
Name: call.Name,
|
||||
Content: string(content),
|
||||
}
|
||||
}
|
||||
|
||||
// jsonToStarlark recursively converts a decoded-JSON value (map/slice/scalar)
|
||||
// to its Starlark equivalent.
|
||||
func jsonToStarlark(v interface{}) starlark.Value {
|
||||
if v == nil {
|
||||
return starlark.None
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case map[string]interface{}:
|
||||
d := starlark.NewDict(len(val))
|
||||
for k, v := range val {
|
||||
_ = d.SetKey(starlark.String(k), jsonToStarlark(v))
|
||||
}
|
||||
return d
|
||||
case []interface{}:
|
||||
elems := make([]starlark.Value, len(val))
|
||||
for i, v := range val {
|
||||
elems[i] = jsonToStarlark(v)
|
||||
}
|
||||
return starlark.NewList(elems)
|
||||
case string:
|
||||
return starlark.String(val)
|
||||
case float64:
|
||||
if val == float64(int64(val)) {
|
||||
return starlark.MakeInt64(int64(val))
|
||||
}
|
||||
return starlark.Float(val)
|
||||
case bool:
|
||||
return starlark.Bool(val)
|
||||
default:
|
||||
return starlark.String(fmt.Sprintf("%v", val))
|
||||
}
|
||||
}
|
||||
|
||||
// starlarkValueToGo recursively converts a Starlark value to a Go value
|
||||
// suitable for json.Marshal.
|
||||
func starlarkValueToGo(v starlark.Value) interface{} {
|
||||
if v == nil || v == starlark.None {
|
||||
return nil
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case starlark.String:
|
||||
return string(val)
|
||||
case starlark.Int:
|
||||
i64, ok := val.Int64()
|
||||
if ok {
|
||||
return i64
|
||||
}
|
||||
return val.String()
|
||||
case starlark.Float:
|
||||
return float64(val)
|
||||
case starlark.Bool:
|
||||
return bool(val)
|
||||
case *starlark.Dict:
|
||||
m := make(map[string]interface{}, val.Len())
|
||||
for _, item := range val.Items() {
|
||||
k, ok := starlark.AsString(item[0])
|
||||
if !ok {
|
||||
k = item[0].String()
|
||||
}
|
||||
m[k] = starlarkValueToGo(item[1])
|
||||
}
|
||||
return m
|
||||
case *starlark.List:
|
||||
list := make([]interface{}, val.Len())
|
||||
for i := 0; i < val.Len(); i++ {
|
||||
list[i] = starlarkValueToGo(val.Index(i))
|
||||
}
|
||||
return list
|
||||
default:
|
||||
return v.String()
|
||||
}
|
||||
}
|
||||
|
||||
// emitWorkspaceEvent sends workspace.file.changed when a write tool succeeds.
|
||||
func emitWorkspaceEvent(lcfg LoopConfig, call tools.ToolCall, result tools.ToolResult) {
|
||||
if lcfg.Hub == nil || result.IsError || lcfg.ExecCtx.WorkspaceID == "" {
|
||||
|
||||
@@ -381,6 +381,8 @@ func main() {
|
||||
)
|
||||
// v0.29.1: provider module adapter — bridges sandbox.ProviderResolver to handlers.ResolveProviderConfig
|
||||
starlarkRunner.SetProviderResolver(handlers.NewProviderResolverAdapter(stores, keyResolver))
|
||||
// v0.29.2: db module — extension namespaced table access
|
||||
starlarkRunner.SetDB(database.DB, database.IsPostgres())
|
||||
|
||||
// Discover and register active Starlark pre-completion filters
|
||||
filters.DiscoverStarlarkFilters(context.Background(), filterChain, stores, starlarkRunner)
|
||||
@@ -497,7 +499,7 @@ func main() {
|
||||
log.Printf(" 🔑 Auth mode: %s", authMode)
|
||||
|
||||
authH := handlers.NewAuthHandler(cfg, stores, uekCache, authProvider)
|
||||
authLimiter := middleware.NewRateLimiter(1, 5)
|
||||
authLimiter := middleware.NewRateLimiter(5, 30)
|
||||
|
||||
api := base.Group("/api/v1")
|
||||
{
|
||||
@@ -697,6 +699,7 @@ func main() {
|
||||
}
|
||||
comp.SetRoutingEvaluator(routing.NewEvaluator())
|
||||
comp.SetFilterChain(filterChain)
|
||||
comp.SetRunner(starlarkRunner) // v0.29.2: extension tool dispatch
|
||||
protected.POST("/chat/completions", comp.Complete)
|
||||
protected.GET("/tools", comp.ListTools)
|
||||
|
||||
@@ -1267,6 +1270,7 @@ func main() {
|
||||
|
||||
wfComp := handlers.NewCompletionHandler(keyResolver, stores, hub, objStore, kbEmbedder)
|
||||
wfComp.SetFilterChain(filterChain)
|
||||
wfComp.SetRunner(starlarkRunner) // v0.29.2: extension tool dispatch
|
||||
wfAPI.POST("/:id/completions", wfComp.Complete)
|
||||
}
|
||||
|
||||
|
||||
@@ -55,14 +55,29 @@ func AuthOrRedirect(cfg *config.Config, users store.UserStore, cache *UserStatus
|
||||
return
|
||||
}
|
||||
|
||||
// Verify user is active and resolve current role from DB
|
||||
role, valid := verifyUser(c, claims, users, cache)
|
||||
if !valid {
|
||||
// verifyUser sent a JSON error, but for page routes we want a redirect.
|
||||
// The abort already happened, so just return.
|
||||
if claims.UserID == "" {
|
||||
redirectToLogin(c, loginPath)
|
||||
return
|
||||
}
|
||||
|
||||
// Resolve user role — redirect (not JSON) on any failure.
|
||||
var role string
|
||||
if entry, hit := cache.get(claims.UserID); hit {
|
||||
if !entry.isActive {
|
||||
redirectToLogin(c, loginPath)
|
||||
return
|
||||
}
|
||||
role = entry.role
|
||||
} else {
|
||||
user, err := users.GetByID(c.Request.Context(), claims.UserID)
|
||||
if err != nil || !user.IsActive {
|
||||
redirectToLogin(c, loginPath)
|
||||
return
|
||||
}
|
||||
cache.set(claims.UserID, user.IsActive, user.Role)
|
||||
role = user.Role
|
||||
}
|
||||
|
||||
c.Set("user_id", claims.UserID)
|
||||
c.Set("email", claims.Email)
|
||||
c.Set("role", role)
|
||||
|
||||
550
server/sandbox/db_module.go
Normal file
550
server/sandbox/db_module.go
Normal file
@@ -0,0 +1,550 @@
|
||||
// Package sandbox — db_module.go
|
||||
//
|
||||
// v0.29.2 CS1: Structured db module for Starlark extensions.
|
||||
//
|
||||
// Permissions:
|
||||
// db.read → db.query(), db.view(), db.list_tables()
|
||||
// db.write → all of the above + db.insert(), db.update(), db.delete()
|
||||
//
|
||||
// Starlark API:
|
||||
//
|
||||
// rows = db.query("logs", filters={"user_id": "abc"}, order="created_at", limit=50)
|
||||
// row = db.insert("logs", {"message": "hello"})
|
||||
// ok = db.update("logs", row_id, {"message": "updated"})
|
||||
// ok = db.delete("logs", row_id)
|
||||
// names = db.list_tables()
|
||||
// rows = db.view("users", filters={"id": "abc"})
|
||||
//
|
||||
// All table access is scoped to ext_{package_id}_{table_name}.
|
||||
// Platform views (ext_view_users, ext_view_channels) are accessible
|
||||
// via db.view() and are read-only regardless of permission level.
|
||||
//
|
||||
// Queries are parameterized — no raw SQL is ever accepted from scripts.
|
||||
// Dialect differences (placeholder style) are handled internally.
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"go.starlark.net/starlark"
|
||||
"go.starlark.net/starlarkstruct"
|
||||
)
|
||||
|
||||
// DBModuleConfig holds configuration for a db module instance.
|
||||
type DBModuleConfig struct {
|
||||
PackageID string
|
||||
CanWrite bool // true when db.write permission is granted
|
||||
DB *sql.DB
|
||||
IsPostgres bool
|
||||
}
|
||||
|
||||
// allowedViews is the set of platform views extensions may query via db.view().
|
||||
var allowedViews = map[string]bool{
|
||||
"users": true,
|
||||
"channels": true,
|
||||
}
|
||||
|
||||
// BuildDBModule creates the "db" starlark module for an extension.
|
||||
// The module enforces namespace isolation — all physical table names are
|
||||
// prefixed with ext_{packageID}_. Write operations are guarded by CanWrite.
|
||||
func BuildDBModule(ctx context.Context, cfg DBModuleConfig) *starlarkstruct.Module {
|
||||
fns := starlark.StringDict{
|
||||
"query": starlark.NewBuiltin("db.query", dbQuery(ctx, cfg)),
|
||||
"view": starlark.NewBuiltin("db.view", dbView(ctx, cfg)),
|
||||
"list_tables": starlark.NewBuiltin("db.list_tables", dbListTables(ctx, cfg)),
|
||||
}
|
||||
|
||||
if cfg.CanWrite {
|
||||
fns["insert"] = starlark.NewBuiltin("db.insert", dbInsert(ctx, cfg))
|
||||
fns["update"] = starlark.NewBuiltin("db.update", dbUpdate(ctx, cfg))
|
||||
fns["delete"] = starlark.NewBuiltin("db.delete", dbDelete(ctx, cfg))
|
||||
}
|
||||
|
||||
return MakeModule("db", fns)
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────
|
||||
|
||||
// physicalTable returns the fully-qualified table name for an extension table.
|
||||
func (cfg DBModuleConfig) physicalTable(logicalName string) (string, error) {
|
||||
if logicalName == "" {
|
||||
return "", fmt.Errorf("db: table name must not be empty")
|
||||
}
|
||||
if strings.ContainsAny(logicalName, " \t\n\"';-") {
|
||||
return "", fmt.Errorf("db: invalid table name %q", logicalName)
|
||||
}
|
||||
// Sanitize package ID for use in identifiers (replace - with _)
|
||||
pkgSlug := strings.ReplaceAll(cfg.PackageID, "-", "_")
|
||||
return fmt.Sprintf("ext_%s_%s", pkgSlug, logicalName), nil
|
||||
}
|
||||
|
||||
// ph returns the parameter placeholder for the nth argument (1-based).
|
||||
func (cfg DBModuleConfig) ph(n int) string {
|
||||
if cfg.IsPostgres {
|
||||
return fmt.Sprintf("$%d", n)
|
||||
}
|
||||
return "?"
|
||||
}
|
||||
|
||||
// generateID produces a random hex ID for new rows.
|
||||
func generateID() string {
|
||||
b := make([]byte, 16)
|
||||
_, _ = rand.Read(b)
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
// starlarkFiltersToSQL converts a Starlark dict of {col: val} into a
|
||||
// WHERE clause and argument slice. Returns empty string if filters is None.
|
||||
func (cfg DBModuleConfig) starlarkFiltersToSQL(filters starlark.Value, startIdx int) (string, []any, error) {
|
||||
if filters == starlark.None || filters == nil {
|
||||
return "", nil, nil
|
||||
}
|
||||
d, ok := filters.(*starlark.Dict)
|
||||
if !ok {
|
||||
return "", nil, fmt.Errorf("db: filters must be a dict, got %s", filters.Type())
|
||||
}
|
||||
if d.Len() == 0 {
|
||||
return "", nil, nil
|
||||
}
|
||||
|
||||
var parts []string
|
||||
var args []any
|
||||
idx := startIdx
|
||||
|
||||
for _, item := range d.Items() {
|
||||
col, ok := item[0].(starlark.String)
|
||||
if !ok {
|
||||
return "", nil, fmt.Errorf("db: filter key must be a string, got %s", item[0].Type())
|
||||
}
|
||||
colStr := string(col)
|
||||
if strings.ContainsAny(colStr, " \t\n\"';-") {
|
||||
return "", nil, fmt.Errorf("db: invalid column name %q", colStr)
|
||||
}
|
||||
|
||||
val, err := starlarkToGoValue(item[1])
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("db: filter value for %q: %w", colStr, err)
|
||||
}
|
||||
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", colStr, cfg.ph(idx)))
|
||||
args = append(args, val)
|
||||
idx++
|
||||
}
|
||||
|
||||
return "WHERE " + strings.Join(parts, " AND "), args, nil
|
||||
}
|
||||
|
||||
// starlarkToGoValue converts a Starlark value to a Go value suitable for SQL.
|
||||
func starlarkToGoValue(v starlark.Value) (any, error) {
|
||||
switch val := v.(type) {
|
||||
case starlark.String:
|
||||
return string(val), nil
|
||||
case starlark.Int:
|
||||
i, ok := val.Int64()
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("integer overflow")
|
||||
}
|
||||
return i, nil
|
||||
case starlark.Float:
|
||||
return float64(val), nil
|
||||
case starlark.Bool:
|
||||
return bool(val), nil
|
||||
case starlark.NoneType:
|
||||
return nil, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported type %s", v.Type())
|
||||
}
|
||||
}
|
||||
|
||||
// rowsToStarlark converts sql.Rows to a Starlark list of dicts.
|
||||
func rowsToStarlark(rows *sql.Rows) (*starlark.List, error) {
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var result []starlark.Value
|
||||
for rows.Next() {
|
||||
vals := make([]any, len(cols))
|
||||
ptrs := make([]any, len(cols))
|
||||
for i := range vals {
|
||||
ptrs[i] = &vals[i]
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
d := starlark.NewDict(len(cols))
|
||||
for i, col := range cols {
|
||||
sv, err := goToStarlark(vals[i])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("column %q: %w", col, err)
|
||||
}
|
||||
if err := d.SetKey(starlark.String(col), sv); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
result = append(result, d)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return starlark.NewList(result), nil
|
||||
}
|
||||
|
||||
// goToStarlark converts a Go value (from sql.Scan) to a Starlark value.
|
||||
func goToStarlark(v any) (starlark.Value, error) {
|
||||
if v == nil {
|
||||
return starlark.None, nil
|
||||
}
|
||||
switch val := v.(type) {
|
||||
case string:
|
||||
return starlark.String(val), nil
|
||||
case []byte:
|
||||
return starlark.String(string(val)), nil
|
||||
case int64:
|
||||
return starlark.MakeInt64(val), nil
|
||||
case float64:
|
||||
return starlark.Float(val), nil
|
||||
case bool:
|
||||
return starlark.Bool(val), nil
|
||||
default:
|
||||
// Fallback: stringify
|
||||
return starlark.String(fmt.Sprintf("%v", val)), nil
|
||||
}
|
||||
}
|
||||
|
||||
// ── Builtins ─────────────────────────────────
|
||||
|
||||
// dbQuery implements db.query(table, filters=None, order=None, limit=100).
|
||||
func dbQuery(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
|
||||
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
|
||||
var table string
|
||||
var filters starlark.Value = starlark.None
|
||||
var order starlark.Value = starlark.None
|
||||
var limit starlark.Int = starlark.MakeInt(100)
|
||||
|
||||
if err := starlark.UnpackArgs(b.Name(), args, kwargs,
|
||||
"table", &table,
|
||||
"filters?", &filters,
|
||||
"order?", &order,
|
||||
"limit?", &limit,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
physTable, err := cfg.physicalTable(table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
whereClause, whereArgs, err := cfg.starlarkFiltersToSQL(filters, 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
lim, ok := limit.Int64()
|
||||
if !ok || lim < 1 || lim > 1000 {
|
||||
lim = 100
|
||||
}
|
||||
|
||||
query := fmt.Sprintf("SELECT * FROM %s", physTable)
|
||||
if whereClause != "" {
|
||||
query += " " + whereClause
|
||||
}
|
||||
|
||||
if order != starlark.None {
|
||||
col, ok := order.(starlark.String)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("db.query: order must be a string column name")
|
||||
}
|
||||
colStr := string(col)
|
||||
dir := "ASC"
|
||||
if strings.HasPrefix(colStr, "-") {
|
||||
colStr = colStr[1:]
|
||||
dir = "DESC"
|
||||
}
|
||||
if strings.ContainsAny(colStr, " \t\n\"';") {
|
||||
return nil, fmt.Errorf("db.query: invalid order column %q", colStr)
|
||||
}
|
||||
query += fmt.Sprintf(" ORDER BY %s %s", colStr, dir)
|
||||
}
|
||||
|
||||
limitPH := cfg.ph(len(whereArgs) + 1)
|
||||
query += " LIMIT " + limitPH
|
||||
whereArgs = append(whereArgs, lim)
|
||||
|
||||
rows, err := cfg.DB.QueryContext(ctx, query, whereArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("db.query: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return rowsToStarlark(rows)
|
||||
}
|
||||
}
|
||||
|
||||
// dbView implements db.view(view_name, filters=None, limit=100).
|
||||
// Queries ext_view_{view_name} — only allowedViews are permitted.
|
||||
func dbView(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
|
||||
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
|
||||
var viewName string
|
||||
var filters starlark.Value = starlark.None
|
||||
var limit starlark.Int = starlark.MakeInt(100)
|
||||
|
||||
if err := starlark.UnpackArgs(b.Name(), args, kwargs,
|
||||
"view_name", &viewName,
|
||||
"filters?", &filters,
|
||||
"limit?", &limit,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !allowedViews[viewName] {
|
||||
return nil, fmt.Errorf("db.view: unknown view %q (allowed: users, channels)", viewName)
|
||||
}
|
||||
|
||||
physView := "ext_view_" + viewName
|
||||
|
||||
whereClause, whereArgs, err := cfg.starlarkFiltersToSQL(filters, 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
lim, ok := limit.Int64()
|
||||
if !ok || lim < 1 || lim > 1000 {
|
||||
lim = 100
|
||||
}
|
||||
|
||||
query := fmt.Sprintf("SELECT * FROM %s", physView)
|
||||
if whereClause != "" {
|
||||
query += " " + whereClause
|
||||
}
|
||||
limitPH := cfg.ph(len(whereArgs) + 1)
|
||||
query += " LIMIT " + limitPH
|
||||
whereArgs = append(whereArgs, lim)
|
||||
|
||||
rows, err := cfg.DB.QueryContext(ctx, query, whereArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("db.view: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return rowsToStarlark(rows)
|
||||
}
|
||||
}
|
||||
|
||||
// dbListTables implements db.list_tables() → list of logical table names.
|
||||
func dbListTables(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
|
||||
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
|
||||
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 0); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
prefix := "ext_" + strings.ReplaceAll(cfg.PackageID, "-", "_") + "_"
|
||||
var query string
|
||||
var queryArgs []any
|
||||
|
||||
if cfg.IsPostgres {
|
||||
query = `SELECT table_name FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name LIKE $1
|
||||
ORDER BY table_name`
|
||||
queryArgs = []any{prefix + "%"}
|
||||
} else {
|
||||
query = `SELECT name FROM sqlite_master
|
||||
WHERE type = 'table' AND name LIKE ?
|
||||
ORDER BY name`
|
||||
queryArgs = []any{prefix + "%"}
|
||||
}
|
||||
|
||||
rows, err := cfg.DB.QueryContext(ctx, query, queryArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("db.list_tables: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var names []starlark.Value
|
||||
for rows.Next() {
|
||||
var physName string
|
||||
if err := rows.Scan(&physName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Strip prefix to return logical name
|
||||
logical := strings.TrimPrefix(physName, prefix)
|
||||
names = append(names, starlark.String(logical))
|
||||
}
|
||||
if names == nil {
|
||||
names = []starlark.Value{}
|
||||
}
|
||||
return starlark.NewList(names), nil
|
||||
}
|
||||
}
|
||||
|
||||
// dbInsert implements db.insert(table, row_dict) → inserted row dict with id.
|
||||
func dbInsert(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
|
||||
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
|
||||
var table string
|
||||
var rowVal starlark.Value
|
||||
|
||||
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 2, &table, &rowVal); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
row, ok := rowVal.(*starlark.Dict)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("db.insert: row must be a dict, got %s", rowVal.Type())
|
||||
}
|
||||
|
||||
physTable, err := cfg.physicalTable(table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
id := generateID()
|
||||
cols := []string{"id"}
|
||||
placeholders := []string{cfg.ph(1)}
|
||||
vals := []any{id}
|
||||
idx := 2
|
||||
|
||||
for _, item := range row.Items() {
|
||||
col, ok := item[0].(starlark.String)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("db.insert: column key must be a string")
|
||||
}
|
||||
colStr := string(col)
|
||||
if colStr == "id" || colStr == "created_at" {
|
||||
continue // auto-managed
|
||||
}
|
||||
if strings.ContainsAny(colStr, " \t\n\"';-") {
|
||||
return nil, fmt.Errorf("db.insert: invalid column name %q", colStr)
|
||||
}
|
||||
val, err := starlarkToGoValue(item[1])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("db.insert: column %q: %w", colStr, err)
|
||||
}
|
||||
cols = append(cols, colStr)
|
||||
placeholders = append(placeholders, cfg.ph(idx))
|
||||
vals = append(vals, val)
|
||||
idx++
|
||||
}
|
||||
|
||||
query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)",
|
||||
physTable,
|
||||
strings.Join(cols, ", "),
|
||||
strings.Join(placeholders, ", "),
|
||||
)
|
||||
|
||||
if _, err := cfg.DB.ExecContext(ctx, query, vals...); err != nil {
|
||||
return nil, fmt.Errorf("db.insert: %w", err)
|
||||
}
|
||||
|
||||
// Return the inserted row (re-query by id)
|
||||
selectQ := fmt.Sprintf("SELECT * FROM %s WHERE id = %s", physTable, cfg.ph(1))
|
||||
rows, err := cfg.DB.QueryContext(ctx, selectQ, id)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("db.insert: re-query failed: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
list, err := rowsToStarlark(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if list.Len() == 0 {
|
||||
return starlark.None, nil
|
||||
}
|
||||
return list.Index(0), nil
|
||||
}
|
||||
}
|
||||
|
||||
// dbUpdate implements db.update(table, id, partial_dict) → True.
|
||||
func dbUpdate(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
|
||||
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
|
||||
var table, rowID string
|
||||
var patchVal starlark.Value
|
||||
|
||||
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 3, &table, &rowID, &patchVal); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
patch, ok := patchVal.(*starlark.Dict)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("db.update: patch must be a dict, got %s", patchVal.Type())
|
||||
}
|
||||
if patch.Len() == 0 {
|
||||
return starlark.True, nil // nothing to do
|
||||
}
|
||||
|
||||
physTable, err := cfg.physicalTable(table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var setParts []string
|
||||
var vals []any
|
||||
idx := 1
|
||||
|
||||
for _, item := range patch.Items() {
|
||||
col, ok := item[0].(starlark.String)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("db.update: column key must be a string")
|
||||
}
|
||||
colStr := string(col)
|
||||
if colStr == "id" || colStr == "created_at" {
|
||||
continue
|
||||
}
|
||||
if strings.ContainsAny(colStr, " \t\n\"';-") {
|
||||
return nil, fmt.Errorf("db.update: invalid column name %q", colStr)
|
||||
}
|
||||
val, err := starlarkToGoValue(item[1])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("db.update: column %q: %w", colStr, err)
|
||||
}
|
||||
setParts = append(setParts, fmt.Sprintf("%s = %s", colStr, cfg.ph(idx)))
|
||||
vals = append(vals, val)
|
||||
idx++
|
||||
}
|
||||
|
||||
if len(setParts) == 0 {
|
||||
return starlark.True, nil
|
||||
}
|
||||
|
||||
vals = append(vals, rowID)
|
||||
query := fmt.Sprintf("UPDATE %s SET %s WHERE id = %s",
|
||||
physTable,
|
||||
strings.Join(setParts, ", "),
|
||||
cfg.ph(idx),
|
||||
)
|
||||
|
||||
if _, err := cfg.DB.ExecContext(ctx, query, vals...); err != nil {
|
||||
return nil, fmt.Errorf("db.update: %w", err)
|
||||
}
|
||||
return starlark.True, nil
|
||||
}
|
||||
}
|
||||
|
||||
// dbDelete implements db.delete(table, id) → True.
|
||||
func dbDelete(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
|
||||
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
|
||||
var table, rowID string
|
||||
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 2, &table, &rowID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
physTable, err := cfg.physicalTable(table)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
query := fmt.Sprintf("DELETE FROM %s WHERE id = %s", physTable, cfg.ph(1))
|
||||
if _, err := cfg.DB.ExecContext(ctx, query, rowID); err != nil {
|
||||
return nil, fmt.Errorf("db.delete: %w", err)
|
||||
}
|
||||
return starlark.True, nil
|
||||
}
|
||||
}
|
||||
254
server/sandbox/db_module_test.go
Normal file
254
server/sandbox/db_module_test.go
Normal file
@@ -0,0 +1,254 @@
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.starlark.net/starlark"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// newTestDB creates an in-memory SQLite database with a test extension table.
|
||||
func newTestDB(t *testing.T) (*sql.DB, DBModuleConfig) {
|
||||
t.Helper()
|
||||
db, err := sql.Open("sqlite", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("open db: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { db.Close() })
|
||||
|
||||
// Create the extension table the tests will use.
|
||||
// physicalTable("logs") for package "test-ext" → ext_test_ext_logs
|
||||
_, err = db.Exec(`CREATE TABLE ext_test_ext_logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
message TEXT,
|
||||
user_id TEXT,
|
||||
count INTEGER,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
)`)
|
||||
if err != nil {
|
||||
t.Fatalf("create test table: %v", err)
|
||||
}
|
||||
|
||||
cfg := DBModuleConfig{
|
||||
PackageID: "test-ext",
|
||||
CanWrite: true,
|
||||
DB: db,
|
||||
IsPostgres: false,
|
||||
}
|
||||
return db, cfg
|
||||
}
|
||||
|
||||
// execScript runs a Starlark snippet with the db module injected.
|
||||
func execScript(t *testing.T, cfg DBModuleConfig, script string) (*Result, error) {
|
||||
t.Helper()
|
||||
sb := New(DefaultConfig())
|
||||
ctx := context.Background()
|
||||
modules := map[string]starlark.Value{
|
||||
"db": BuildDBModule(ctx, cfg),
|
||||
}
|
||||
return sb.Exec(ctx, "test.star", script, modules)
|
||||
}
|
||||
|
||||
// ── physicalTable ───────────────────────────
|
||||
|
||||
func TestPhysicalTable_ValidName(t *testing.T) {
|
||||
cfg := DBModuleConfig{PackageID: "my-ext"}
|
||||
got, err := cfg.physicalTable("logs")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got != "ext_my_ext_logs" {
|
||||
t.Errorf("got %q, want ext_my_ext_logs", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhysicalTable_EmptyName(t *testing.T) {
|
||||
cfg := DBModuleConfig{PackageID: "my-ext"}
|
||||
_, err := cfg.physicalTable("")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty table name")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPhysicalTable_InvalidChars(t *testing.T) {
|
||||
cfg := DBModuleConfig{PackageID: "my-ext"}
|
||||
_, err := cfg.physicalTable("bad name")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for table name with space")
|
||||
}
|
||||
_, err = cfg.physicalTable("drop;table")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for table name with semicolon")
|
||||
}
|
||||
}
|
||||
|
||||
// ── db.insert / db.query ────────────────────
|
||||
|
||||
func TestDBInsertAndQuery(t *testing.T) {
|
||||
_, cfg := newTestDB(t)
|
||||
|
||||
result, err := execScript(t, cfg, `
|
||||
row = db.insert("logs", {"message": "hello", "user_id": "u1"})
|
||||
rows = db.query("logs")
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("script error: %v", err)
|
||||
}
|
||||
|
||||
rowsVal, ok := result.Globals["rows"]
|
||||
if !ok {
|
||||
t.Fatal("rows not in globals")
|
||||
}
|
||||
if _, ok := rowsVal.(*starlark.List); !ok {
|
||||
t.Fatalf("rows is %T, want *starlark.List", rowsVal)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBQueryFilters(t *testing.T) {
|
||||
db, cfg := newTestDB(t)
|
||||
|
||||
// Pre-populate with two rows
|
||||
db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id) VALUES ('a', 'one', 'u1')`)
|
||||
db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id) VALUES ('b', 'two', 'u2')`)
|
||||
|
||||
result, err := execScript(t, cfg, `
|
||||
rows = db.query("logs", filters={"user_id": "u1"})
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("script error: %v", err)
|
||||
}
|
||||
|
||||
rowsStr := result.Globals["rows"].String()
|
||||
if !strings.Contains(rowsStr, "one") {
|
||||
t.Errorf("expected row with message=one, got: %s", rowsStr)
|
||||
}
|
||||
if strings.Contains(rowsStr, "two") {
|
||||
t.Errorf("unexpected row with message=two: %s", rowsStr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBQueryLimit(t *testing.T) {
|
||||
db, cfg := newTestDB(t)
|
||||
for i := 0; i < 5; i++ {
|
||||
db.Exec(`INSERT INTO ext_test_ext_logs (id, message) VALUES (?, ?)`,
|
||||
generateID(), "msg")
|
||||
}
|
||||
|
||||
result, err := execScript(t, cfg, `rows = db.query("logs", limit=2)`)
|
||||
if err != nil {
|
||||
t.Fatalf("script error: %v", err)
|
||||
}
|
||||
// String representation of a 2-element list has exactly 2 entries
|
||||
rowsStr := result.Globals["rows"].String()
|
||||
// Count occurrences of "id" key — each row has one
|
||||
count := strings.Count(rowsStr, `"id"`)
|
||||
if count != 2 {
|
||||
t.Errorf("expected 2 rows with limit=2, got repr: %s", rowsStr)
|
||||
}
|
||||
}
|
||||
|
||||
// ── db.update ───────────────────────────────
|
||||
|
||||
func TestDBUpdate(t *testing.T) {
|
||||
db, cfg := newTestDB(t)
|
||||
db.Exec(`INSERT INTO ext_test_ext_logs (id, message) VALUES ('r1', 'original')`)
|
||||
|
||||
_, err := execScript(t, cfg, `db.update("logs", "r1", {"message": "updated"})`)
|
||||
if err != nil {
|
||||
t.Fatalf("script error: %v", err)
|
||||
}
|
||||
|
||||
var msg string
|
||||
db.QueryRow(`SELECT message FROM ext_test_ext_logs WHERE id = 'r1'`).Scan(&msg)
|
||||
if msg != "updated" {
|
||||
t.Errorf("expected message=updated, got %q", msg)
|
||||
}
|
||||
}
|
||||
|
||||
// ── db.delete ───────────────────────────────
|
||||
|
||||
func TestDBDelete(t *testing.T) {
|
||||
db, cfg := newTestDB(t)
|
||||
db.Exec(`INSERT INTO ext_test_ext_logs (id, message) VALUES ('r1', 'to-delete')`)
|
||||
|
||||
_, err := execScript(t, cfg, `db.delete("logs", "r1")`)
|
||||
if err != nil {
|
||||
t.Fatalf("script error: %v", err)
|
||||
}
|
||||
|
||||
var count int
|
||||
db.QueryRow(`SELECT COUNT(*) FROM ext_test_ext_logs WHERE id = 'r1'`).Scan(&count)
|
||||
if count != 0 {
|
||||
t.Errorf("expected row deleted, count = %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
// ── write guard ─────────────────────────────
|
||||
|
||||
func TestDBReadOnlyModuleNoWriteFunctions(t *testing.T) {
|
||||
_, cfg := newTestDB(t)
|
||||
cfg.CanWrite = false
|
||||
|
||||
mod := BuildDBModule(context.Background(), cfg)
|
||||
modStr := mod.String()
|
||||
|
||||
// insert/update/delete should not be present in a read-only module
|
||||
for _, fn := range []string{"insert", "update", "delete"} {
|
||||
if strings.Contains(modStr, fn) {
|
||||
t.Errorf("read-only module exposes write function %q", fn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBWriteFunctionBlockedByReadOnly(t *testing.T) {
|
||||
_, cfg := newTestDB(t)
|
||||
cfg.CanWrite = false
|
||||
|
||||
_, err := execScript(t, cfg, `db.insert("logs", {"message": "x"})`)
|
||||
if err == nil {
|
||||
t.Fatal("expected error calling insert on read-only module")
|
||||
}
|
||||
}
|
||||
|
||||
// ── view allowlist ───────────────────────────
|
||||
|
||||
func TestDBViewRejectsUnknownView(t *testing.T) {
|
||||
_, cfg := newTestDB(t)
|
||||
_, err := execScript(t, cfg, `db.view("secrets")`)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for unknown view")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "unknown view") {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ── namespace isolation ──────────────────────
|
||||
|
||||
func TestDBQueryRejectsOtherExtensionTable(t *testing.T) {
|
||||
_, cfg := newTestDB(t)
|
||||
// Attempt to query a table that belongs to a different extension
|
||||
_, err := execScript(t, cfg, `db.query("../../etc/passwd")`)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for path traversal attempt")
|
||||
}
|
||||
}
|
||||
|
||||
// ── list_tables ─────────────────────────────
|
||||
|
||||
func TestDBListTables(t *testing.T) {
|
||||
_, cfg := newTestDB(t)
|
||||
result, err := execScript(t, cfg, `names = db.list_tables()`)
|
||||
if err != nil {
|
||||
t.Fatalf("script error: %v", err)
|
||||
}
|
||||
namesStr := result.Globals["names"].String()
|
||||
// The test table ext_test_ext_logs should appear as "logs"
|
||||
if !strings.Contains(namesStr, "logs") {
|
||||
t.Errorf("expected 'logs' in list_tables output, got: %s", namesStr)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,12 +9,17 @@
|
||||
// v0.29.1 CS2: Adds RunContext for per-invocation state (user_id),
|
||||
// vault for provider key decryption, and provider.complete module.
|
||||
//
|
||||
// v0.29.2 CS1: Adds db *sql.DB for the db module. SetDB() wires it
|
||||
// at startup. db.read grants query/view/list_tables; db.write adds
|
||||
// insert/update/delete. If both are granted, db.write wins (superset).
|
||||
//
|
||||
// The runner is the bridge between the package/permission system
|
||||
// and the sandboxed Starlark interpreter.
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
@@ -38,10 +43,12 @@ type RunContext struct {
|
||||
|
||||
// Runner executes Starlark package scripts with permission-gated modules.
|
||||
type Runner struct {
|
||||
sandbox *Sandbox
|
||||
stores store.Stores
|
||||
notifier NotificationSender // nil = notifications module unavailable
|
||||
resolver ProviderResolver // nil = provider module unavailable
|
||||
sandbox *Sandbox
|
||||
stores store.Stores
|
||||
notifier NotificationSender // nil = notifications module unavailable
|
||||
resolver ProviderResolver // nil = provider module unavailable
|
||||
db *sql.DB // nil = db module unavailable
|
||||
dbPostgres bool // true = use $N placeholders; false = use ?
|
||||
}
|
||||
|
||||
// NewRunner creates a runner with the given sandbox and dependencies.
|
||||
@@ -62,6 +69,14 @@ func (r *Runner) SetProviderResolver(pr ProviderResolver) {
|
||||
r.resolver = pr
|
||||
}
|
||||
|
||||
// SetDB attaches the raw database connection for the db module.
|
||||
// isPostgres controls whether $N or ? placeholders are used.
|
||||
// Call this at startup after database.Connect().
|
||||
func (r *Runner) SetDB(db *sql.DB, isPostgres bool) {
|
||||
r.db = db
|
||||
r.dbPostgres = isPostgres
|
||||
}
|
||||
|
||||
// ExecPackage loads a package's script from its manifest and executes it
|
||||
// with modules gated by the package's granted permissions.
|
||||
//
|
||||
@@ -98,9 +113,9 @@ func (r *Runner) ExecPackage(ctx context.Context, pkg *store.PackageRegistration
|
||||
|
||||
// CallEntryPoint executes a package script and calls a named function.
|
||||
// This is the standard pattern for event-driven extensions:
|
||||
// 1. Exec the script (defines functions)
|
||||
// 2. Find the named entry point in globals
|
||||
// 3. Call it with the provided arguments
|
||||
// 1. Exec the script (defines functions)
|
||||
// 2. Find the named entry point in globals
|
||||
// 3. Call it with the provided arguments
|
||||
//
|
||||
// The RunContext carries per-invocation state. Pass nil if not needed.
|
||||
//
|
||||
@@ -141,6 +156,9 @@ func (r *Runner) buildModules(ctx context.Context, packageID string, manifest ma
|
||||
|
||||
modules := make(map[string]starlark.Value)
|
||||
|
||||
// Track db permission level: 0=none, 1=read, 2=write
|
||||
dbLevel := 0
|
||||
|
||||
for _, perm := range granted {
|
||||
switch perm {
|
||||
case models.ExtPermSecretsRead:
|
||||
@@ -163,11 +181,25 @@ func (r *Runner) buildModules(ctx context.Context, packageID string, manifest ma
|
||||
}
|
||||
}
|
||||
|
||||
// Future permissions:
|
||||
// case models.ExtPermDBRead, models.ExtPermDBWrite:
|
||||
// modules["db"] = BuildDBModule(...)
|
||||
case models.ExtPermDBRead:
|
||||
if dbLevel < 1 {
|
||||
dbLevel = 1
|
||||
}
|
||||
|
||||
case models.ExtPermDBWrite:
|
||||
dbLevel = 2
|
||||
}
|
||||
}
|
||||
|
||||
// Wire db module at the highest granted level.
|
||||
if dbLevel > 0 && r.db != nil {
|
||||
modules["db"] = BuildDBModule(ctx, DBModuleConfig{
|
||||
PackageID: packageID,
|
||||
CanWrite: dbLevel == 2,
|
||||
DB: r.db,
|
||||
IsPostgres: r.dbPostgres,
|
||||
})
|
||||
}
|
||||
|
||||
return modules, nil
|
||||
}
|
||||
|
||||
@@ -182,6 +182,7 @@ func (e *Executor) Execute(ctx context.Context, task models.Task, run *models.Ta
|
||||
personaID = *task.PersonaID
|
||||
}
|
||||
|
||||
extTools := handlers.BuildExtToolMap(ctx, e.stores, task.OwnerID)
|
||||
sink := handlers.NewHeadlessSink(task.ID)
|
||||
result := handlers.CoreToolLoop(ctx, handlers.LoopConfig{
|
||||
Provider: provider,
|
||||
@@ -203,6 +204,8 @@ func (e *Executor) Execute(ctx context.Context, task models.Task, run *models.Ta
|
||||
MaxTokens: task.MaxTokens,
|
||||
},
|
||||
Streaming: false, // headless — use ChatCompletion
|
||||
Runner: e.runner,
|
||||
ExtTools: extTools,
|
||||
}, sink)
|
||||
|
||||
// ── 6. Persist output based on output_mode ──
|
||||
|
||||
20
server/store/ext_data_iface.go
Normal file
20
server/store/ext_data_iface.go
Normal file
@@ -0,0 +1,20 @@
|
||||
package store
|
||||
|
||||
import "context"
|
||||
|
||||
// ExtDataStore tracks the namespaced tables created for each extension.
|
||||
// It is the catalog backing db.list_tables() and the install/uninstall lifecycle.
|
||||
// The actual extension data lives in tables named ext_{package_id}_{table_name};
|
||||
// this store only records the logical (unprefixed) names.
|
||||
type ExtDataStore interface {
|
||||
// RegisterTable records that a table was created for a package.
|
||||
// Safe to call multiple times — idempotent.
|
||||
RegisterTable(ctx context.Context, packageID, tableName string) error
|
||||
|
||||
// ListTables returns all logical table names registered for a package.
|
||||
ListTables(ctx context.Context, packageID string) ([]string, error)
|
||||
|
||||
// DeletePackageTables removes all catalog entries for a package.
|
||||
// Called after the physical tables have been dropped on uninstall.
|
||||
DeletePackageTables(ctx context.Context, packageID string) error
|
||||
}
|
||||
@@ -61,6 +61,7 @@ type Stores struct {
|
||||
Folders FolderStore // v0.29.0: Chat folder CRUD
|
||||
Health HealthStore // v0.29.0: Provider health window management
|
||||
ExtPermissions ExtensionPermissionStore // v0.29.0: Extension declared/granted capabilities
|
||||
ExtData ExtDataStore // v0.29.2: Extension namespaced table catalog
|
||||
}
|
||||
|
||||
// =========================================
|
||||
|
||||
55
server/store/postgres/ext_data.go
Normal file
55
server/store/postgres/ext_data.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
)
|
||||
|
||||
// ExtDataStore implements store.ExtDataStore for Postgres.
|
||||
type ExtDataStore struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
func NewExtDataStore(db *sql.DB) *ExtDataStore {
|
||||
return &ExtDataStore{db: db}
|
||||
}
|
||||
|
||||
func (s *ExtDataStore) RegisterTable(ctx context.Context, packageID, tableName string) error {
|
||||
_, err := s.db.ExecContext(ctx,
|
||||
`INSERT INTO ext_data_tables (package_id, table_name)
|
||||
VALUES ($1, $2)
|
||||
ON CONFLICT (package_id, table_name) DO NOTHING`,
|
||||
packageID, tableName)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *ExtDataStore) ListTables(ctx context.Context, packageID string) ([]string, error) {
|
||||
rows, err := s.db.QueryContext(ctx,
|
||||
`SELECT table_name FROM ext_data_tables
|
||||
WHERE package_id = $1
|
||||
ORDER BY table_name`,
|
||||
packageID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var names []string
|
||||
for rows.Next() {
|
||||
var n string
|
||||
if err := rows.Scan(&n); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names = append(names, n)
|
||||
}
|
||||
if names == nil {
|
||||
names = []string{}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func (s *ExtDataStore) DeletePackageTables(ctx context.Context, packageID string) error {
|
||||
_, err := s.db.ExecContext(ctx,
|
||||
`DELETE FROM ext_data_tables WHERE package_id = $1`, packageID)
|
||||
return err
|
||||
}
|
||||
@@ -47,5 +47,6 @@ func NewStores(db *sql.DB) store.Stores {
|
||||
Folders: NewFolderStore(),
|
||||
Health: NewHealthStore(db),
|
||||
ExtPermissions: NewExtensionPermissionStore(db),
|
||||
ExtData: NewExtDataStore(db),
|
||||
}
|
||||
}
|
||||
|
||||
49
server/store/sqlite/ext_data.go
Normal file
49
server/store/sqlite/ext_data.go
Normal file
@@ -0,0 +1,49 @@
|
||||
package sqlite
|
||||
|
||||
import "context"
|
||||
|
||||
// ExtDataStore implements store.ExtDataStore for SQLite.
|
||||
type ExtDataStore struct{}
|
||||
|
||||
func NewExtDataStore() *ExtDataStore {
|
||||
return &ExtDataStore{}
|
||||
}
|
||||
|
||||
func (s *ExtDataStore) RegisterTable(ctx context.Context, packageID, tableName string) error {
|
||||
_, err := DB.ExecContext(ctx,
|
||||
`INSERT OR IGNORE INTO ext_data_tables (package_id, table_name)
|
||||
VALUES (?, ?)`,
|
||||
packageID, tableName)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *ExtDataStore) ListTables(ctx context.Context, packageID string) ([]string, error) {
|
||||
rows, err := DB.QueryContext(ctx,
|
||||
`SELECT table_name FROM ext_data_tables
|
||||
WHERE package_id = ?
|
||||
ORDER BY table_name`,
|
||||
packageID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var names []string
|
||||
for rows.Next() {
|
||||
var n string
|
||||
if err := rows.Scan(&n); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names = append(names, n)
|
||||
}
|
||||
if names == nil {
|
||||
names = []string{}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func (s *ExtDataStore) DeletePackageTables(ctx context.Context, packageID string) error {
|
||||
_, err := DB.ExecContext(ctx,
|
||||
`DELETE FROM ext_data_tables WHERE package_id = ?`, packageID)
|
||||
return err
|
||||
}
|
||||
@@ -47,5 +47,6 @@ func NewStores(db *sql.DB) store.Stores {
|
||||
Folders: NewFolderStore(),
|
||||
Health: NewHealthStore(),
|
||||
ExtPermissions: NewExtensionPermissionStore(),
|
||||
ExtData: NewExtDataStore(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ const Extensions = {
|
||||
for (const ext of exts) {
|
||||
// Parse manifest to check for _script (inline) or entry (file)
|
||||
const manifest = ext.manifest || {};
|
||||
const extId = ext.ext_id;
|
||||
const extId = ext.id;
|
||||
|
||||
if (manifest._script) {
|
||||
// Inject script tag pointing at the asset endpoint
|
||||
@@ -431,7 +431,7 @@ const Extensions = {
|
||||
|
||||
_getUserSettings(extId) {
|
||||
// Find user settings from the manifest data loaded from API
|
||||
const manifest = this._manifests.find(m => m.ext_id === extId);
|
||||
const manifest = this._manifests.find(m => m.id === extId);
|
||||
if (manifest?.user_settings) {
|
||||
try {
|
||||
return typeof manifest.user_settings === 'string'
|
||||
|
||||
Reference in New Issue
Block a user