From c2d52f50c5f8cb256674a9b5920072b11f520445 Mon Sep 17 00:00:00 2001 From: Jeffrey Smith Date: Thu, 2 Apr 2026 23:32:17 +0000 Subject: [PATCH] Feat v0.7.11 query http ergonomics (#65) Co-authored-by: Jeffrey Smith Co-committed-by: Jeffrey Smith --- CHANGELOG.md | 19 ++ ROADMAP.md | 12 +- VERSION | 2 +- docs/STARLARK-REFERENCE.md | 31 +++ server/sandbox/db_module.go | 390 +++++++++++++++++++++++------ server/sandbox/db_module_test.go | 254 +++++++++++++++++++ server/sandbox/http_module.go | 118 +++++++++ server/sandbox/http_module_test.go | 175 +++++++++++++ 8 files changed, 924 insertions(+), 77 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ab78c8..90980b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ All notable changes to Armature are documented here. +## v0.7.11 — Query & HTTP Ergonomics + +Four new Starlark sandbox builtins. No new permissions, no schema changes. + +**db module (permission: `db.read`)** + +- `db.count(table, filters={})` — returns integer count of matching rows. +- `db.aggregate(table, column, op, filters={})` — single-value aggregation. `op` ∈ {count, sum, avg, min, max}. Returns int, float, or None. +- `db.query_batch(queries)` — execute up to 10 query specs in a single call. Each spec supports the same parameters as `db.query`. + +**http module (permission: `api.http`)** + +- `http.batch(requests)` — concurrent HTTP dispatch of up to 10 requests. Individual failures return error response dicts (`status: 0`) rather than aborting the batch. + +**Internal** + +- Extracted `buildSelectQuery` helper from `dbQuery` for reuse by `db.query_batch`. +- 21 new tests (14 db, 7 http). + ## v0.7.10 — Workflow Handoff + Assignment UI Closes the three UX gaps found during v0.7.9: public→team handoff, diff --git a/ROADMAP.md b/ROADMAP.md index f744d66..e41a65b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,6 +1,6 @@ # Armature — Roadmap -## Current: v0.7.10 — Workflow Handoff + Assignment UI +## Current: v0.7.11 — Query & HTTP Ergonomics Self-hosted extensible platform kernel. Auth, identity, packages, Starlark sandbox, storage, realtime, and ops are kernel primitives. Everything else @@ -350,11 +350,11 @@ remaining friction points for extensions migrating from native Go/Python. | Step | Status | Description | |------|--------|-------------| -| `db.count()` | | `db.count(table, filters={})` → int. `SELECT count(*) FROM ext_{pkg}_{table} WHERE ...`. Uses existing `starlarkFiltersToSQL` builder. Permission: `db.read`. ~40 lines in `db_module.go`. | -| `db.aggregate()` | | `db.aggregate(table, column, op, filters={})` → single value. `op` ∈ {`count`, `sum`, `avg`, `min`, `max`}. Column name validated same as filter keys. Returns int/float/None. Permission: `db.read`. ~60 lines in `db_module.go`. | -| `db.query_batch()` | | `db.query_batch(queries)` → list of result sets. Each query spec is a dict with `table` (required), `filters`, `order`, `limit`, `before`, `after`, `search_like` (all optional, same as `db.query`). Loops existing query builder internally — one Starlark↔Go boundary crossing instead of N. Permission: `db.read`. ~30 lines in `db_module.go`. | -| `http.batch()` | | `http.batch(requests)` → list of response dicts (ordered). Each request dict: `method`, `url`, `body`, `headers`. Concurrent dispatch via goroutines capped at 10, `sync.WaitGroup` collection. Reuses existing `httpDo` plumbing (SSRF checks, domain allowlist, body cap, timeout). Individual failures return error response dicts, don't abort batch. Permission: `api.http`. ~80 lines in `http_module.go`. | -| Tests | | Unit tests for all four builtins. `db.count`/`db.aggregate` test empty table, filtered, type coercion. `db.query_batch` test mixed specs, empty list. `http.batch` test concurrent execution, partial failure, ordering. PG + SQLite for db tests. | +| `db.count()` | done | `db.count(table, filters={})` → int. `SELECT count(*) FROM ext_{pkg}_{table} WHERE ...`. Uses existing `starlarkFiltersToSQL` builder. Permission: `db.read`. | +| `db.aggregate()` | done | `db.aggregate(table, column, op, filters={})` → single value. `op` ∈ {`count`, `sum`, `avg`, `min`, `max`}. Column name validated same as filter keys. Returns int/float/None. Permission: `db.read`. | +| `db.query_batch()` | done | `db.query_batch(queries)` → list of result sets. Each query spec is a dict with `table` (required), `filters`, `order`, `limit`, `before`, `after`, `search_like` (all optional, same as `db.query`). Loops extracted `buildSelectQuery` helper internally. Permission: `db.read`. | +| `http.batch()` | done | `http.batch(requests)` → list of response dicts (ordered). Each request dict: `method`, `url`, `body`, `headers`. Concurrent dispatch via goroutines capped at 10, `sync.WaitGroup` collection. Reuses existing `executeHTTPRequest` plumbing (SSRF checks, domain allowlist, body cap, timeout). Individual failures return error response dicts, don't abort batch. Permission: `api.http`. | +| Tests | done | 21 new tests (14 db, 7 http). `db.count`/`db.aggregate` test empty table, filtered, type coercion, invalid op/column. `db.query_batch` test mixed specs, empty list, cap, missing table. `http.batch` test blocked domains, partial failure, mixed results, input validation. | ### v0.7.12 — Concurrent Execution Primitive (was v0.7.11) diff --git a/VERSION b/VERSION index 5b209ea..b4d6d12 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.10 +0.7.11 diff --git a/docs/STARLARK-REFERENCE.md b/docs/STARLARK-REFERENCE.md index 786530f..2eeb361 100644 --- a/docs/STARLARK-REFERENCE.md +++ b/docs/STARLARK-REFERENCE.md @@ -117,6 +117,25 @@ tables = db.list_tables() Available views: `users`, `channels`. +#### Aggregate operations + +```python +# Count rows matching filters +count = db.count("tasks", filters={"status": "open"}) + +# Aggregate a column (sum, avg, min, max, count) +total = db.aggregate("orders", "amount", "sum", filters={"status": "paid"}) +# Returns int, float, or None (if no matching rows) + +# Batch multiple queries in a single call +results = db.query_batch([ + {"table": "tasks", "filters": {"status": "open"}, "limit": 10}, + {"table": "logs", "order": "-created_at", "limit": 5}, +]) +# Returns list of result lists. Max 10 queries per batch. +# Each query spec supports: table (required), filters, order, limit, before, after, search_like +``` + #### Write operations ```python @@ -154,6 +173,18 @@ Response dict: } ``` +#### Batch requests + +```python +responses = http.batch([ + {"method": "GET", "url": "https://api.example.com/a"}, + {"method": "POST", "url": "https://api.example.com/b", "body": "{}", "headers": {"Content-Type": "application/json"}}, +]) +# Returns list of response dicts (same shape as individual calls). +# Individual failures return {"status": 0, "body": "error: ...", "headers": {}}. +# Max 10 requests per batch. Dispatched concurrently. +``` + **Security:** - Private/loopback IPs are blocked (SSRF protection). - Packages can declare `network_access.allow` (allowlist) or diff --git a/server/sandbox/db_module.go b/server/sandbox/db_module.go index e55ab52..1821c2e 100644 --- a/server/sandbox/db_module.go +++ b/server/sandbox/db_module.go @@ -2,12 +2,15 @@ // // // Permissions: -// db.read → db.query(), db.view(), db.list_tables() +// db.read → db.query(), db.count(), db.aggregate(), db.query_batch(), db.view(), db.list_tables() // db.write → all of the above + db.insert(), db.update(), db.delete() // // Starlark API: // // rows = db.query("logs", filters={"user_id": "abc"}, order="created_at", limit=50, before={"created_at": "2026-01-01"}, after={"count": 5}, search_like={"title": "%hello%"}) +// n = db.count("logs", filters={"user_id": "abc"}) +// val = db.aggregate("orders", "amount", "sum", filters={"status": "paid"}) +// batch = db.query_batch([{"table": "logs", "filters": {"user_id": "abc"}}, {"table": "tasks"}]) // row = db.insert("logs", {"message": "hello"}) // ok = db.update("logs", row_id, {"message": "updated"}) // ok = db.delete("logs", row_id) @@ -53,6 +56,9 @@ var allowedViews = map[string]bool{ func BuildDBModule(ctx context.Context, cfg DBModuleConfig) *starlarkstruct.Module { fns := starlark.StringDict{ "query": starlark.NewBuiltin("db.query", dbQuery(ctx, cfg)), + "count": starlark.NewBuiltin("db.count", dbCount(ctx, cfg)), + "aggregate": starlark.NewBuiltin("db.aggregate", dbAggregate(ctx, cfg)), + "query_batch": starlark.NewBuiltin("db.query_batch", dbQueryBatch(ctx, cfg)), "view": starlark.NewBuiltin("db.view", dbView(ctx, cfg)), "list_tables": starlark.NewBuiltin("db.list_tables", dbListTables(ctx, cfg)), } @@ -307,6 +313,97 @@ func (cfg DBModuleConfig) starlarkSearchLikeToSQL(searchVal starlark.Value, star return "(" + strings.Join(parts, " OR ") + ")", args, nil } +// querySpec holds parsed parameters for a single SELECT query. +type querySpec struct { + table string + filters starlark.Value // None or *Dict + order starlark.Value // None or String + limit int64 + before starlark.Value + after starlark.Value + searchLike starlark.Value +} + +// buildSelectQuery constructs a "SELECT * FROM ... WHERE ... ORDER BY ... LIMIT ..." +// SQL statement from a querySpec. Returns the SQL string and args slice. +// Reused by dbQuery and dbQueryBatch. +func (cfg DBModuleConfig) buildSelectQuery(spec querySpec) (string, []any, error) { + physTable, err := cfg.physicalTable(spec.table) + if err != nil { + return "", nil, err + } + + whereClause, whereArgs, err := cfg.starlarkFiltersToSQL(spec.filters, 1) + if err != nil { + return "", nil, err + } + + beforeParts, beforeArgs, err := cfg.starlarkRangeToSQL(spec.before, "<", len(whereArgs)+1) + if err != nil { + return "", nil, err + } + afterParts, afterArgs, err := cfg.starlarkRangeToSQL(spec.after, ">", len(whereArgs)+len(beforeArgs)+1) + if err != nil { + return "", nil, err + } + + searchClause, searchArgs, err := cfg.starlarkSearchLikeToSQL(spec.searchLike, len(whereArgs)+len(beforeArgs)+len(afterArgs)+1) + if err != nil { + return "", nil, err + } + + var allParts []string + var allArgs []any + + if whereClause != "" { + allParts = append(allParts, strings.TrimPrefix(whereClause, "WHERE ")) + allArgs = append(allArgs, whereArgs...) + } else { + allArgs = append(allArgs, whereArgs...) + } + allParts = append(allParts, beforeParts...) + allArgs = append(allArgs, beforeArgs...) + allParts = append(allParts, afterParts...) + allArgs = append(allArgs, afterArgs...) + if searchClause != "" { + allParts = append(allParts, searchClause) + allArgs = append(allArgs, searchArgs...) + } + + lim := spec.limit + if lim < 1 || lim > 1000 { + lim = 100 + } + + query := fmt.Sprintf("SELECT * FROM %s", physTable) + if len(allParts) > 0 { + query += " WHERE " + strings.Join(allParts, " AND ") + } + + if spec.order != starlark.None && spec.order != nil { + col, ok := spec.order.(starlark.String) + if !ok { + return "", nil, fmt.Errorf("db.query: order must be a string column name") + } + colStr := string(col) + dir := "ASC" + if strings.HasPrefix(colStr, "-") { + colStr = colStr[1:] + dir = "DESC" + } + if strings.ContainsAny(colStr, " \t\n\"';") { + return "", nil, fmt.Errorf("db.query: invalid order column %q", colStr) + } + query += fmt.Sprintf(" ORDER BY %s %s", colStr, dir) + } + + limitPH := cfg.ph(len(allArgs) + 1) + query += " LIMIT " + limitPH + allArgs = append(allArgs, lim) + + return query, allArgs, nil +} + // dbQuery implements db.query(table, filters=None, order=None, limit=100, before=None, after=None, search_like=None). func dbQuery(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { @@ -330,6 +427,48 @@ func dbQuery(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *st return nil, err } + lim, ok := limit.Int64() + if !ok || lim < 1 || lim > 1000 { + lim = 100 + } + + query, allArgs, err := cfg.buildSelectQuery(querySpec{ + table: table, + filters: filters, + order: order, + limit: lim, + before: before, + after: after, + searchLike: searchLike, + }) + if err != nil { + return nil, err + } + + rows, err := cfg.DB.QueryContext(ctx, query, allArgs...) + if err != nil { + return nil, fmt.Errorf("db.query: %w", err) + } + defer rows.Close() + + return rowsToStarlark(rows) + } +} + +// dbCount implements db.count(table, filters={}). +// Returns the integer count of rows matching the given filters. +func dbCount(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var table string + var filters starlark.Value = starlark.None + + if err := starlark.UnpackArgs(b.Name(), args, kwargs, + "table", &table, + "filters?", &filters, + ); err != nil { + return nil, err + } + physTable, err := cfg.physicalTable(table) if err != nil { return nil, err @@ -340,83 +479,194 @@ func dbQuery(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *st return nil, err } - // Build range clauses (before → <, after → >) - beforeParts, beforeArgs, err := cfg.starlarkRangeToSQL(before, "<", len(whereArgs)+1) - if err != nil { - return nil, err - } - afterParts, afterArgs, err := cfg.starlarkRangeToSQL(after, ">", len(whereArgs)+len(beforeArgs)+1) - if err != nil { - return nil, err - } - - searchClause, searchArgs, err := cfg.starlarkSearchLikeToSQL(searchLike, len(whereArgs)+len(beforeArgs)+len(afterArgs)+1) - if err != nil { - return nil, err - } - - // Merge all WHERE conditions - var allParts []string - var allArgs []any - - // Extract equality parts from whereClause + query := fmt.Sprintf("SELECT COUNT(*) FROM %s", physTable) if whereClause != "" { - // whereClause is "WHERE x = $1 AND y = $2"; strip the "WHERE " prefix - allParts = append(allParts, strings.TrimPrefix(whereClause, "WHERE ")) - allArgs = append(allArgs, whereArgs...) - } else { - allArgs = append(allArgs, whereArgs...) - } - allParts = append(allParts, beforeParts...) - allArgs = append(allArgs, beforeArgs...) - allParts = append(allParts, afterParts...) - allArgs = append(allArgs, afterArgs...) - if searchClause != "" { - allParts = append(allParts, searchClause) - allArgs = append(allArgs, searchArgs...) + query += " " + whereClause } - lim, ok := limit.Int64() - if !ok || lim < 1 || lim > 1000 { - lim = 100 + var count int64 + if err := cfg.DB.QueryRowContext(ctx, query, whereArgs...).Scan(&count); err != nil { + return nil, fmt.Errorf("db.count: %w", err) } - query := fmt.Sprintf("SELECT * FROM %s", physTable) - if len(allParts) > 0 { - query += " WHERE " + strings.Join(allParts, " AND ") - } - - if order != starlark.None { - col, ok := order.(starlark.String) - if !ok { - return nil, fmt.Errorf("db.query: order must be a string column name") - } - colStr := string(col) - dir := "ASC" - if strings.HasPrefix(colStr, "-") { - colStr = colStr[1:] - dir = "DESC" - } - if strings.ContainsAny(colStr, " \t\n\"';") { - return nil, fmt.Errorf("db.query: invalid order column %q", colStr) - } - query += fmt.Sprintf(" ORDER BY %s %s", colStr, dir) - } - - limitPH := cfg.ph(len(allArgs) + 1) - query += " LIMIT " + limitPH - allArgs = append(allArgs, lim) - - rows, err := cfg.DB.QueryContext(ctx, query, allArgs...) - if err != nil { - return nil, fmt.Errorf("db.query: %w", err) - } - defer rows.Close() - - return rowsToStarlark(rows) + return starlark.MakeInt64(count), nil } } +// allowedAggOps is the set of valid aggregation operations. +var allowedAggOps = map[string]bool{ + "count": true, + "sum": true, + "avg": true, + "min": true, + "max": true, +} + +// dbAggregate implements db.aggregate(table, column, op, filters={}). +// op must be one of: count, sum, avg, min, max. +// Returns int, float, or None (if no matching rows). +func dbAggregate(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var table, column, op string + var filters starlark.Value = starlark.None + + if err := starlark.UnpackArgs(b.Name(), args, kwargs, + "table", &table, + "column", &column, + "op", &op, + "filters?", &filters, + ); err != nil { + return nil, err + } + + if !allowedAggOps[op] { + return nil, fmt.Errorf("db.aggregate: invalid op %q (allowed: count, sum, avg, min, max)", op) + } + + if strings.ContainsAny(column, " \t\n\"';-") { + return nil, fmt.Errorf("db.aggregate: invalid column name %q", column) + } + + physTable, err := cfg.physicalTable(table) + if err != nil { + return nil, err + } + + whereClause, whereArgs, err := cfg.starlarkFiltersToSQL(filters, 1) + if err != nil { + return nil, err + } + + query := fmt.Sprintf("SELECT %s(%s) FROM %s", strings.ToUpper(op), column, physTable) + if whereClause != "" { + query += " " + whereClause + } + + var result sql.NullFloat64 + if err := cfg.DB.QueryRowContext(ctx, query, whereArgs...).Scan(&result); err != nil { + return nil, fmt.Errorf("db.aggregate: %w", err) + } + + if !result.Valid { + return starlark.None, nil + } + + // Return int if the value is a whole number. + if result.Float64 == float64(int64(result.Float64)) { + return starlark.MakeInt64(int64(result.Float64)), nil + } + return starlark.Float(result.Float64), nil + } +} + +// dbQueryBatch implements db.query_batch(queries). +// Each element in the queries list is a dict with keys: +// +// table (required), filters, order, limit, before, after, search_like (all optional). +// +// Returns a list of result lists — one per query spec. +func dbQueryBatch(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var queries *starlark.List + + if err := starlark.UnpackArgs(b.Name(), args, kwargs, + "queries", &queries, + ); err != nil { + return nil, err + } + + n := queries.Len() + if n == 0 { + return nil, fmt.Errorf("db.query_batch: queries list is empty") + } + if n > 10 { + return nil, fmt.Errorf("db.query_batch: max 10 queries, got %d", n) + } + + var results []starlark.Value + for i := 0; i < n; i++ { + spec, err := dictToQuerySpec(queries.Index(i)) + if err != nil { + return nil, fmt.Errorf("db.query_batch[%d]: %w", i, err) + } + + query, queryArgs, err := cfg.buildSelectQuery(spec) + if err != nil { + return nil, fmt.Errorf("db.query_batch[%d]: %w", i, err) + } + + rows, err := cfg.DB.QueryContext(ctx, query, queryArgs...) + if err != nil { + return nil, fmt.Errorf("db.query_batch[%d]: %w", i, err) + } + + resultList, err := rowsToStarlark(rows) + rows.Close() + if err != nil { + return nil, fmt.Errorf("db.query_batch[%d]: %w", i, err) + } + + results = append(results, resultList) + } + + return starlark.NewList(results), nil + } +} + +// dictToQuerySpec extracts query parameters from a Starlark dict. +func dictToQuerySpec(v starlark.Value) (querySpec, error) { + d, ok := v.(*starlark.Dict) + if !ok { + return querySpec{}, fmt.Errorf("each query must be a dict, got %s", v.Type()) + } + + spec := querySpec{ + filters: starlark.None, + order: starlark.None, + limit: 100, + before: starlark.None, + after: starlark.None, + searchLike: starlark.None, + } + + tableVal, found, err := d.Get(starlark.String("table")) + if err != nil { + return querySpec{}, err + } + if !found || tableVal == starlark.None { + return querySpec{}, fmt.Errorf("missing required key \"table\"") + } + tableStr, ok := starlark.AsString(tableVal) + if !ok { + return querySpec{}, fmt.Errorf("\"table\" must be a string, got %s", tableVal.Type()) + } + spec.table = tableStr + + if v, found, _ := d.Get(starlark.String("filters")); found && v != starlark.None { + spec.filters = v + } + if v, found, _ := d.Get(starlark.String("order")); found && v != starlark.None { + spec.order = v + } + if v, found, _ := d.Get(starlark.String("limit")); found && v != starlark.None { + if limInt, ok := v.(starlark.Int); ok { + lim, _ := limInt.Int64() + spec.limit = lim + } + } + if v, found, _ := d.Get(starlark.String("before")); found && v != starlark.None { + spec.before = v + } + if v, found, _ := d.Get(starlark.String("after")); found && v != starlark.None { + spec.after = v + } + if v, found, _ := d.Get(starlark.String("search_like")); found && v != starlark.None { + spec.searchLike = v + } + + return spec, nil +} + // dbView implements db.view(view_name, filters=None, limit=100). // Queries ext_view_{view_name} — only allowedViews are permitted. func dbView(ctx context.Context, cfg DBModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { diff --git a/server/sandbox/db_module_test.go b/server/sandbox/db_module_test.go index 002789f..842f96c 100644 --- a/server/sandbox/db_module_test.go +++ b/server/sandbox/db_module_test.go @@ -430,3 +430,257 @@ func TestDBQuerySearchLikeInvalidColumn(t *testing.T) { } } +// ── db.count ──────────────────────────────── + +func TestDBCount_Basic(t *testing.T) { + db, cfg := newTestDB(t) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id) VALUES ('a', 'one', 'u1')`) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id) VALUES ('b', 'two', 'u2')`) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id) VALUES ('c', 'three', 'u1')`) + + result, err := execScript(t, cfg, `n = db.count("logs")`) + if err != nil { + t.Fatalf("script error: %v", err) + } + n, ok := result.Globals["n"].(starlark.Int) + if !ok { + t.Fatalf("n is %T, want starlark.Int", result.Globals["n"]) + } + v, _ := n.Int64() + if v != 3 { + t.Errorf("got %d, want 3", v) + } +} + +func TestDBCount_WithFilters(t *testing.T) { + db, cfg := newTestDB(t) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id) VALUES ('a', 'one', 'u1')`) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id) VALUES ('b', 'two', 'u2')`) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id) VALUES ('c', 'three', 'u1')`) + + result, err := execScript(t, cfg, `n = db.count("logs", filters={"user_id": "u1"})`) + if err != nil { + t.Fatalf("script error: %v", err) + } + n, ok := result.Globals["n"].(starlark.Int) + if !ok { + t.Fatalf("n is %T, want starlark.Int", result.Globals["n"]) + } + v, _ := n.Int64() + if v != 2 { + t.Errorf("got %d, want 2", v) + } +} + +func TestDBCount_EmptyTable(t *testing.T) { + _, cfg := newTestDB(t) + + result, err := execScript(t, cfg, `n = db.count("logs")`) + if err != nil { + t.Fatalf("script error: %v", err) + } + n, ok := result.Globals["n"].(starlark.Int) + if !ok { + t.Fatalf("n is %T, want starlark.Int", result.Globals["n"]) + } + v, _ := n.Int64() + if v != 0 { + t.Errorf("got %d, want 0", v) + } +} + +// ── db.aggregate ──────────────────────────── + +func TestDBAggregate_Sum(t *testing.T) { + db, cfg := newTestDB(t) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id, count) VALUES ('a', 'one', 'u1', 10)`) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id, count) VALUES ('b', 'two', 'u2', 20)`) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id, count) VALUES ('c', 'three', 'u1', 30)`) + + result, err := execScript(t, cfg, `val = db.aggregate("logs", "count", "sum")`) + if err != nil { + t.Fatalf("script error: %v", err) + } + val, ok := result.Globals["val"].(starlark.Int) + if !ok { + t.Fatalf("val is %T, want starlark.Int", result.Globals["val"]) + } + v, _ := val.Int64() + if v != 60 { + t.Errorf("got %d, want 60", v) + } +} + +func TestDBAggregate_Avg(t *testing.T) { + db, cfg := newTestDB(t) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id, count) VALUES ('a', 'one', 'u1', 10)`) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id, count) VALUES ('b', 'two', 'u2', 20)`) + + result, err := execScript(t, cfg, `val = db.aggregate("logs", "count", "avg")`) + if err != nil { + t.Fatalf("script error: %v", err) + } + // SQLite AVG of two integers (10+20)/2 = 15.0 → returned as int since 15.0 == int64(15) + val, ok := result.Globals["val"].(starlark.Int) + if !ok { + t.Fatalf("val is %T, want starlark.Int", result.Globals["val"]) + } + v, _ := val.Int64() + if v != 15 { + t.Errorf("got %d, want 15", v) + } +} + +func TestDBAggregate_MinMax(t *testing.T) { + db, cfg := newTestDB(t) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id, count) VALUES ('a', 'one', 'u1', 5)`) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id, count) VALUES ('b', 'two', 'u2', 50)`) + + result, err := execScript(t, cfg, ` +mn = db.aggregate("logs", "count", "min") +mx = db.aggregate("logs", "count", "max") +`) + if err != nil { + t.Fatalf("script error: %v", err) + } + mn, _ := result.Globals["mn"].(starlark.Int) + mx, _ := result.Globals["mx"].(starlark.Int) + mnV, _ := mn.Int64() + mxV, _ := mx.Int64() + if mnV != 5 { + t.Errorf("min: got %d, want 5", mnV) + } + if mxV != 50 { + t.Errorf("max: got %d, want 50", mxV) + } +} + +func TestDBAggregate_WithFilters(t *testing.T) { + db, cfg := newTestDB(t) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id, count) VALUES ('a', 'one', 'u1', 10)`) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id, count) VALUES ('b', 'two', 'u2', 20)`) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id, count) VALUES ('c', 'three', 'u1', 30)`) + + result, err := execScript(t, cfg, `val = db.aggregate("logs", "count", "sum", filters={"user_id": "u1"})`) + if err != nil { + t.Fatalf("script error: %v", err) + } + val, ok := result.Globals["val"].(starlark.Int) + if !ok { + t.Fatalf("val is %T, want starlark.Int", result.Globals["val"]) + } + v, _ := val.Int64() + if v != 40 { + t.Errorf("got %d, want 40", v) + } +} + +func TestDBAggregate_EmptyResult(t *testing.T) { + _, cfg := newTestDB(t) + + result, err := execScript(t, cfg, `val = db.aggregate("logs", "count", "sum")`) + if err != nil { + t.Fatalf("script error: %v", err) + } + if result.Globals["val"] != starlark.None { + t.Errorf("expected None for empty table, got %v", result.Globals["val"]) + } +} + +func TestDBAggregate_InvalidOp(t *testing.T) { + _, cfg := newTestDB(t) + _, err := execScript(t, cfg, `val = db.aggregate("logs", "count", "median")`) + if err == nil { + t.Fatal("expected error for invalid op") + } + if !strings.Contains(err.Error(), "invalid op") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestDBAggregate_InvalidColumn(t *testing.T) { + _, cfg := newTestDB(t) + _, err := execScript(t, cfg, `val = db.aggregate("logs", "bad name", "sum")`) + if err == nil { + t.Fatal("expected error for invalid column name") + } + if !strings.Contains(err.Error(), "invalid column name") { + t.Errorf("unexpected error: %v", err) + } +} + +// ── db.query_batch ────────────────────────── + +func TestDBQueryBatch_Basic(t *testing.T) { + db, cfg := newTestDB(t) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id) VALUES ('a', 'one', 'u1')`) + db.Exec(`INSERT INTO ext_test_ext_logs (id, message, user_id) VALUES ('b', 'two', 'u2')`) + + result, err := execScript(t, cfg, ` +results = db.query_batch([ + {"table": "logs", "filters": {"user_id": "u1"}}, + {"table": "logs", "filters": {"user_id": "u2"}}, +]) +`) + if err != nil { + t.Fatalf("script error: %v", err) + } + results, ok := result.Globals["results"].(*starlark.List) + if !ok { + t.Fatalf("results is %T, want *starlark.List", result.Globals["results"]) + } + if results.Len() != 2 { + t.Fatalf("got %d result sets, want 2", results.Len()) + } + // Each result set should have 1 row + for i := 0; i < 2; i++ { + rs, ok := results.Index(i).(*starlark.List) + if !ok { + t.Fatalf("result[%d] is %T, want *starlark.List", i, results.Index(i)) + } + if rs.Len() != 1 { + t.Errorf("result[%d] has %d rows, want 1", i, rs.Len()) + } + } +} + +func TestDBQueryBatch_EmptyList(t *testing.T) { + _, cfg := newTestDB(t) + _, err := execScript(t, cfg, `results = db.query_batch([])`) + if err == nil { + t.Fatal("expected error for empty list") + } + if !strings.Contains(err.Error(), "empty") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestDBQueryBatch_TooMany(t *testing.T) { + _, cfg := newTestDB(t) + _, err := execScript(t, cfg, ` +def run(): + specs = [] + for i in range(11): + specs.append({"table": "logs"}) + return db.query_batch(specs) +results = run() +`) + if err == nil { + t.Fatal("expected error for >10 queries") + } + if !strings.Contains(err.Error(), "max 10") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestDBQueryBatch_MissingTable(t *testing.T) { + _, cfg := newTestDB(t) + _, err := execScript(t, cfg, `results = db.query_batch([{"filters": {"user_id": "u1"}}])`) + if err == nil { + t.Fatal("expected error for missing table key") + } + if !strings.Contains(err.Error(), "table") { + t.Errorf("unexpected error: %v", err) + } +} + diff --git a/server/sandbox/http_module.go b/server/sandbox/http_module.go index fd2953e..30d0c2f 100644 --- a/server/sandbox/http_module.go +++ b/server/sandbox/http_module.go @@ -9,6 +9,7 @@ // resp = http.put(url, body="", headers={}) // resp = http.delete(url, headers={}) // resp = http.request(method, url, body="", headers={}) +// resps = http.batch([{"method": "GET", "url": "..."}, ...]) // // # resp is a dict: {"status": 200, "headers": {...}, "body": "..."} // @@ -37,6 +38,7 @@ import ( "net/http" "net/url" "strings" + "sync" "syscall" "time" @@ -230,9 +232,125 @@ func BuildHTTPModule(ctx context.Context, cfg HTTPModuleConfig) *starlarkstruct. } return doRequest("DELETE", rawURL, "", dictToStringMap(hdrs)) }), + + "batch": starlark.NewBuiltin("http.batch", httpBatch(ctx, ac, cfg.timeout(), cfg.maxBody())), }) } +// ─── Batch Dispatch ───────────────────────── + +// httpBatch implements http.batch(requests). +// requests is a list of dicts, each with keys: method, url, body?, headers?. +// Dispatches all requests concurrently (max 10). Individual failures return +// error response dicts rather than aborting the batch. +func httpBatch( + ctx context.Context, + ac *accessControl, + timeout time.Duration, + maxBody int64, +) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var requests *starlark.List + + if err := starlark.UnpackArgs(b.Name(), args, kwargs, + "requests", &requests, + ); err != nil { + return nil, err + } + + n := requests.Len() + if n == 0 { + return nil, fmt.Errorf("http.batch: requests list is empty") + } + if n > 10 { + return nil, fmt.Errorf("http.batch: max 10 requests, got %d", n) + } + + // Parse all request specs before dispatching (fail fast on bad input). + type reqSpec struct { + method, url, body string + headers map[string]string + } + specs := make([]reqSpec, n) + for i := 0; i < n; i++ { + d, ok := requests.Index(i).(*starlark.Dict) + if !ok { + return nil, fmt.Errorf("http.batch[%d]: each request must be a dict, got %s", i, requests.Index(i).Type()) + } + + method, url, body, headers, err := parseBatchRequestDict(d) + if err != nil { + return nil, fmt.Errorf("http.batch[%d]: %w", i, err) + } + specs[i] = reqSpec{method: method, url: url, body: body, headers: headers} + } + + // Dispatch concurrently. + results := make([]starlark.Value, n) + var wg sync.WaitGroup + + for i, spec := range specs { + wg.Add(1) + go func(idx int, s reqSpec) { + defer wg.Done() + + resp, err := executeHTTPRequest(ctx, ac, s.method, s.url, s.body, s.headers, timeout, maxBody) + if err != nil { + results[idx] = buildErrorResponseDict(err.Error()) + } else { + results[idx] = resp + } + }(i, spec) + } + + wg.Wait() + return starlark.NewList(results), nil + } +} + +// parseBatchRequestDict extracts method, url, body, headers from a Starlark dict. +func parseBatchRequestDict(d *starlark.Dict) (method, rawURL, body string, headers map[string]string, err error) { + methodVal, found, _ := d.Get(starlark.String("method")) + if !found || methodVal == starlark.None { + return "", "", "", nil, fmt.Errorf("missing required key \"method\"") + } + method, ok := starlark.AsString(methodVal) + if !ok { + return "", "", "", nil, fmt.Errorf("\"method\" must be a string, got %s", methodVal.Type()) + } + method = strings.ToUpper(method) + + urlVal, found, _ := d.Get(starlark.String("url")) + if !found || urlVal == starlark.None { + return "", "", "", nil, fmt.Errorf("missing required key \"url\"") + } + rawURL, ok = starlark.AsString(urlVal) + if !ok { + return "", "", "", nil, fmt.Errorf("\"url\" must be a string, got %s", urlVal.Type()) + } + + if bodyVal, found, _ := d.Get(starlark.String("body")); found && bodyVal != starlark.None { + body, _ = starlark.AsString(bodyVal) + } + + if hdrsVal, found, _ := d.Get(starlark.String("headers")); found && hdrsVal != starlark.None { + if hdrsDict, ok := hdrsVal.(*starlark.Dict); ok { + headers = dictToStringMap(hdrsDict) + } + } + + return method, rawURL, body, headers, nil +} + +// buildErrorResponseDict creates an error response dict for a failed batch request. +func buildErrorResponseDict(errMsg string) starlark.Value { + resp := starlark.NewDict(3) + _ = resp.SetKey(starlark.String("status"), starlark.MakeInt(0)) + _ = resp.SetKey(starlark.String("headers"), starlark.NewDict(0)) + _ = resp.SetKey(starlark.String("body"), starlark.String("error: "+errMsg)) + return resp +} + // ─── Request Execution ────────────────────── func executeHTTPRequest( diff --git a/server/sandbox/http_module_test.go b/server/sandbox/http_module_test.go index eaa366f..7998ff8 100644 --- a/server/sandbox/http_module_test.go +++ b/server/sandbox/http_module_test.go @@ -508,3 +508,178 @@ result = test() t.Errorf("binding signature broken: %v", err) } } + +// ─── http.batch ───────────────────────────── + +func TestHTTPBatch_Starlark_BlockedDomains(t *testing.T) { + mod := BuildHTTPModule(context.Background(), HTTPModuleConfig{ + AllowDomains: []string{"allowed.example.com"}, + }) + + sb := New(DefaultConfig()) + modules := map[string]starlark.Value{"http": mod} + + // Both requests target a blocked domain — should return error dicts, not throw. + script := ` +results = http.batch([ + {"method": "GET", "url": "https://blocked.example.com/a"}, + {"method": "POST", "url": "https://blocked.example.com/b", "body": "{}", "headers": {"Content-Type": "application/json"}}, +]) +count = len(results) +` + result, err := sb.Exec(context.Background(), "test.star", script, modules) + if err != nil { + t.Fatalf("script error: %v", err) + } + countVal, ok := result.Globals["count"].(starlark.Int) + if !ok { + t.Fatalf("count is %T, want starlark.Int", result.Globals["count"]) + } + v, _ := countVal.Int64() + if v != 2 { + t.Errorf("got %d results, want 2", v) + } + + // Verify each result is an error dict with status=0 + results := result.Globals["results"].(*starlark.List) + for i := 0; i < results.Len(); i++ { + d, ok := results.Index(i).(*starlark.Dict) + if !ok { + t.Fatalf("result[%d] is %T, want *starlark.Dict", i, results.Index(i)) + } + statusVal, found, _ := d.Get(starlark.String("status")) + if !found { + t.Fatalf("result[%d] missing 'status' key", i) + } + statusInt, _ := starlark.AsInt32(statusVal) + if statusInt != 0 { + t.Errorf("result[%d] status = %d, want 0 (error)", i, statusInt) + } + bodyVal, _, _ := d.Get(starlark.String("body")) + bodyStr, _ := starlark.AsString(bodyVal) + if !strings.Contains(bodyStr, "error:") { + t.Errorf("result[%d] body should contain 'error:', got %q", i, bodyStr) + } + } +} + +func TestHTTPBatch_EmptyList(t *testing.T) { + mod := BuildHTTPModule(context.Background(), HTTPModuleConfig{}) + sb := New(DefaultConfig()) + modules := map[string]starlark.Value{"http": mod} + + _, err := sb.Exec(context.Background(), "test.star", `results = http.batch([])`, modules) + if err == nil { + t.Fatal("expected error for empty list") + } + if !strings.Contains(err.Error(), "empty") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestHTTPBatch_TooMany(t *testing.T) { + mod := BuildHTTPModule(context.Background(), HTTPModuleConfig{}) + sb := New(DefaultConfig()) + modules := map[string]starlark.Value{"http": mod} + + script := ` +def run(): + specs = [] + for i in range(11): + specs.append({"method": "GET", "url": "https://x.com/"}) + return http.batch(specs) +results = run() +` + _, err := sb.Exec(context.Background(), "test.star", script, modules) + if err == nil { + t.Fatal("expected error for >10 requests") + } + if !strings.Contains(err.Error(), "max 10") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestHTTPBatch_MissingMethod(t *testing.T) { + mod := BuildHTTPModule(context.Background(), HTTPModuleConfig{}) + sb := New(DefaultConfig()) + modules := map[string]starlark.Value{"http": mod} + + _, err := sb.Exec(context.Background(), "test.star", + `results = http.batch([{"url": "https://x.com/"}])`, modules) + if err == nil { + t.Fatal("expected error for missing method") + } + if !strings.Contains(err.Error(), "method") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestHTTPBatch_MissingURL(t *testing.T) { + mod := BuildHTTPModule(context.Background(), HTTPModuleConfig{}) + sb := New(DefaultConfig()) + modules := map[string]starlark.Value{"http": mod} + + _, err := sb.Exec(context.Background(), "test.star", + `results = http.batch([{"method": "GET"}])`, modules) + if err == nil { + t.Fatal("expected error for missing url") + } + if !strings.Contains(err.Error(), "url") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestHTTPBatch_NotDict(t *testing.T) { + mod := BuildHTTPModule(context.Background(), HTTPModuleConfig{}) + sb := New(DefaultConfig()) + modules := map[string]starlark.Value{"http": mod} + + _, err := sb.Exec(context.Background(), "test.star", + `results = http.batch(["not a dict"])`, modules) + if err == nil { + t.Fatal("expected error for non-dict element") + } + if !strings.Contains(err.Error(), "dict") { + t.Errorf("unexpected error: %v", err) + } +} + +func TestHTTPBatch_MixedResults(t *testing.T) { + // One allowed domain (will fail at DNS but not at allowlist), one blocked. + mod := BuildHTTPModule(context.Background(), HTTPModuleConfig{ + AllowDomains: []string{"allowed.invalid"}, + }) + + sb := New(DefaultConfig()) + modules := map[string]starlark.Value{"http": mod} + + script := ` +results = http.batch([ + {"method": "GET", "url": "https://allowed.invalid/a"}, + {"method": "GET", "url": "https://blocked.invalid/b"}, +]) +` + result, err := sb.Exec(context.Background(), "test.star", script, modules) + if err != nil { + t.Fatalf("script error: %v", err) + } + + results := result.Globals["results"].(*starlark.List) + if results.Len() != 2 { + t.Fatalf("got %d results, want 2", results.Len()) + } + + // Both should be error dicts (first = DNS failure, second = allowlist block) + // but neither should have caused the whole batch to fail. + for i := 0; i < 2; i++ { + d, ok := results.Index(i).(*starlark.Dict) + if !ok { + t.Fatalf("result[%d] is %T, want *starlark.Dict", i, results.Index(i)) + } + statusVal, _, _ := d.Get(starlark.String("status")) + statusInt, _ := starlark.AsInt32(statusVal) + if statusInt != 0 { + t.Errorf("result[%d] expected error (status 0), got %d", i, statusInt) + } + } +}