From 5b72ed254f00f472d0ecce187978d870bd246b58 Mon Sep 17 00:00:00 2001 From: Jeffrey Smith Date: Fri, 3 Apr 2026 00:14:28 +0000 Subject: [PATCH] Feat v0.8.0 files sandbox module Bridge ObjectStore (PVC/S3) into the Starlark sandbox with extension-scoped key namespacing. New files module with put/get/meta/list/delete/exists builtins gated by files.read and files.write permissions. Co-Authored-By: Claude Opus 4.6 (1M context) --- CHANGELOG.md | 24 ++ ROADMAP.md | 6 +- VERSION | 2 +- docs/STARLARK-REFERENCE.md | 42 +++ server/main.go | 3 + server/models/models_extension_perm.go | 4 + server/sandbox/files_module.go | 472 +++++++++++++++++++++++++ server/sandbox/files_module_test.go | 410 +++++++++++++++++++++ server/sandbox/runner.go | 27 +- server/storage/pvc.go | 72 ++++ server/storage/pvc_test.go | 58 +++ server/storage/s3.go | 43 +++ server/storage/storage.go | 11 + 13 files changed, 1169 insertions(+), 5 deletions(-) create mode 100644 server/sandbox/files_module.go create mode 100644 server/sandbox/files_module_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 75a5519..c6d9abe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,30 @@ All notable changes to Armature are documented here. +## v0.8.0 — Files Module + +New `files` sandbox module. Bridges the existing ObjectStore (PVC/S3) into +the Starlark sandbox with extension-scoped key namespacing. + +**files module (permissions: `files.read`, `files.write`)** + +- `files.put(name, content, content_type, metadata)` — store a file with optional metadata companion. Accepts string or bytes content. 50 MB default limit (configurable via `EXT_FILES_MAX_SIZE`). +- `files.get(name)` — read a file. Returns dict with `content` (bytes), `content_type`, `size`, `metadata`. Returns None if not found. +- `files.meta(name)` — read metadata only (no content transfer). +- `files.list(prefix, limit)` — list files by prefix. Returns list of dicts. Filters out internal `_meta/` companions. +- `files.delete(name)` — delete a file and its metadata companion. Idempotent. +- `files.delete_prefix(prefix)` — delete all files under a prefix. +- `files.exists(name)` — check existence without reading. + +**Internal** + +- New file: `sandbox/files_module.go`. +- New permission constants: `ExtPermFilesRead`, `ExtPermFilesWrite`. +- `ObjectStore` interface gains `List(ctx, prefix, limit)` method; implemented for PVC and S3. +- Runner wiring: `SetObjectStore()` setter, `buildModulesWithLibCtx` creates files module when permission granted. +- All keys scoped to `ext/{packageID}/`. Metadata stored as companion JSON at `ext/{packageID}/_meta/{name}`. +- 16 new tests (15 files module + 1 PVC list). + ## v0.7.12 — Concurrent Execution Primitive New `batch` sandbox module. Enables extensions to parallelize arbitrary diff --git a/ROADMAP.md b/ROADMAP.md index 53a5374..495c0f9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,6 +1,6 @@ # Armature — Roadmap -## Current: v0.7.12 — Concurrent Execution Primitive +## Current: v0.8.0 — Files Module Self-hosted extensible platform kernel. Auth, identity, packages, Starlark sandbox, storage, realtime, and ops are kernel primitives. Everything else @@ -388,14 +388,14 @@ The last major kernel expansion. Completes the primitive set that the entire extension ecosystem builds on. After v0.8.x, the kernel is feature-complete for 1.0 — all new capabilities are extensions. -**v0.8.0 — `files` Module** +**v0.8.0 — `files` Module** ✅ Bridge `ObjectStore` (PVC/S3) into the Starlark sandbox. Extension-scoped key namespacing (`ext/{packageID}/`). Permissions: `files.read`, `files.write`. Metadata companions stored as sibling objects. New: `sandbox/files_module.go`. Modified: runner wiring, permission -constants. +constants. `ObjectStore` interface gained `List()` method. 16 new tests. **v0.8.1 — `workspace` Module** diff --git a/VERSION b/VERSION index b4d6d12..a3df0a6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.11 +0.8.0 diff --git a/docs/STARLARK-REFERENCE.md b/docs/STARLARK-REFERENCE.md index a5b82d8..1fc317d 100644 --- a/docs/STARLARK-REFERENCE.md +++ b/docs/STARLARK-REFERENCE.md @@ -275,6 +275,48 @@ results, errors = batch.exec([ --- +### files + +**Permissions:** `files.read`, `files.write` + +Store and retrieve files via the kernel ObjectStore (PVC or S3). +All keys are scoped to `ext/{packageID}/` — extensions cannot access +each other's files. + +```python +# Store a file with optional metadata +files.put("reports/q1.pdf", pdf_bytes, + content_type="application/pdf", + metadata={"quarter": "Q1", "year": 2026}) + +# Read a file +result = files.get("reports/q1.pdf") +# result = {"content": b"...", "content_type": "application/pdf", +# "size": 12345, "metadata": {"quarter": "Q1", "year": 2026}} + +# Metadata only (no content transfer) +meta = files.meta("reports/q1.pdf") + +# List files by prefix +entries = files.list(prefix="reports/", limit=50) +# entries = [{"name": "reports/q1.pdf", "size": 12345, "content_type": "..."}] + +# Check existence +if files.exists("reports/q1.pdf"): + files.delete("reports/q1.pdf") + +# Bulk delete +files.delete_prefix("temp/") +``` + +**Notes:** +- `content` accepts string or bytes; `get()` returns bytes. +- Maximum file size: 50 MB (configurable via `EXT_FILES_MAX_SIZE`). +- Metadata is stored as a companion JSON object, not in the file itself. +- `files.list()` automatically filters out internal metadata companions. + +--- + ## Example: automated stage hook A simple hook that reads a setting, queries data, and advances: diff --git a/server/main.go b/server/main.go index a01d06d..b7d1702 100644 --- a/server/main.go +++ b/server/main.go @@ -169,6 +169,9 @@ func main() { starlarkRunner.SetPackagesDir(cfg.StoragePath + "/packages") } starlarkRunner.SetBus(bus) + if objStore != nil { + starlarkRunner.SetObjectStore(objStore) + } if os.Getenv("EXT_ALLOW_PRIVATE_IPS") == "true" { starlarkRunner.SetAllowPrivateIPs(true) log.Printf(" ⚠️ Extension SSRF protection relaxed: private IPs allowed") diff --git a/server/models/models_extension_perm.go b/server/models/models_extension_perm.go index 592a4b0..a85e94d 100644 --- a/server/models/models_extension_perm.go +++ b/server/models/models_extension_perm.go @@ -37,6 +37,8 @@ const ( ExtPermTriggersRegister = "triggers.register" ExtPermRealtimePublish = "realtime.publish" ExtPermBatchExec = "batch.exec" + ExtPermFilesRead = "files.read" + ExtPermFilesWrite = "files.write" ) // ValidExtensionPermissions is the set of recognized permission keys. @@ -52,6 +54,8 @@ var ValidExtensionPermissions = map[string]bool{ ExtPermTriggersRegister: true, ExtPermRealtimePublish: true, ExtPermBatchExec: true, + ExtPermFilesRead: true, + ExtPermFilesWrite: true, } // ── Extension Permission Model ─────────────── diff --git a/server/sandbox/files_module.go b/server/sandbox/files_module.go new file mode 100644 index 0000000..58960f4 --- /dev/null +++ b/server/sandbox/files_module.go @@ -0,0 +1,472 @@ +// Package sandbox — files_module.go +// +// Permissions: files.read, files.write +// +// Starlark API: +// files.put(name, content, content_type="application/octet-stream", metadata={}) +// result = files.get(name) → dict or None +// meta = files.meta(name) → dict or None +// entries = files.list(prefix="", limit=100) +// files.delete(name) +// files.delete_prefix(prefix) +// exists = files.exists(name) +// +// Bridges the ObjectStore (PVC/S3) into Starlark. All keys are scoped +// to ext/{packageID}/ — extensions cannot access each other's files. +// Metadata is stored as companion JSON at ext/{packageID}/_meta/{name}. +package sandbox + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "strconv" + "strings" + "unicode" + + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" + + "armature/storage" +) + +const ( + filesDefaultMaxSize = 50 * 1024 * 1024 // 50 MB + filesDefaultLimit = 100 + filesMaxNameLen = 512 + filesMetaPrefix = "_meta/" +) + +// FilesModuleConfig holds the configuration for the files module. +type FilesModuleConfig struct { + PackageID string + CanWrite bool + Store storage.ObjectStore +} + +// BuildFilesModule creates the "files" module. +func BuildFilesModule(ctx context.Context, cfg FilesModuleConfig) *starlarkstruct.Module { + fns := starlark.StringDict{ + "get": starlark.NewBuiltin("files.get", filesGet(ctx, cfg)), + "meta": starlark.NewBuiltin("files.meta", filesMeta(ctx, cfg)), + "list": starlark.NewBuiltin("files.list", filesList(ctx, cfg)), + "exists": starlark.NewBuiltin("files.exists", filesExists(ctx, cfg)), + } + + if cfg.CanWrite { + fns["put"] = starlark.NewBuiltin("files.put", filesPut(ctx, cfg)) + fns["delete"] = starlark.NewBuiltin("files.delete", filesDelete(ctx, cfg)) + fns["delete_prefix"] = starlark.NewBuiltin("files.delete_prefix", filesDeletePrefix(ctx, cfg)) + } + + return MakeModule("files", fns) +} + +// ── Builtins ────────────────────────────────── + +func filesPut(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var name string + var content starlark.Value + var contentType string = "application/octet-stream" + var metadata *starlark.Dict + + if err := starlark.UnpackArgs(b.Name(), args, kwargs, + "name", &name, + "content", &content, + "content_type?", &contentType, + "metadata?", &metadata, + ); err != nil { + return nil, err + } + + if err := validateFileName(name); err != nil { + return nil, fmt.Errorf("files.put: %w", err) + } + + // Extract raw bytes from content (accept String or Bytes). + var raw []byte + switch v := content.(type) { + case starlark.String: + raw = []byte(string(v)) + case starlark.Bytes: + raw = []byte(v) + default: + return nil, fmt.Errorf("files.put: content must be string or bytes, got %s", content.Type()) + } + + // Enforce size limit. + maxSize := filesMaxSize() + if int64(len(raw)) > maxSize { + return nil, fmt.Errorf("files.put: content size %d exceeds limit %d", len(raw), maxSize) + } + + // Write main object. + key := cfg.scopedKey(name) + if err := cfg.Store.Put(ctx, key, bytes.NewReader(raw), int64(len(raw)), contentType); err != nil { + return nil, fmt.Errorf("files.put: %w", err) + } + + // Write metadata companion if provided. + if metadata != nil && metadata.Len() > 0 { + metaMap, err := starlarkDictToMap(metadata) + if err != nil { + return nil, fmt.Errorf("files.put: metadata: %w", err) + } + metaJSON, err := json.Marshal(metaMap) + if err != nil { + return nil, fmt.Errorf("files.put: marshal metadata: %w", err) + } + metaKey := cfg.metaKey(name) + if err := cfg.Store.Put(ctx, metaKey, bytes.NewReader(metaJSON), int64(len(metaJSON)), "application/json"); err != nil { + return nil, fmt.Errorf("files.put: write metadata: %w", err) + } + } + + return starlark.True, nil + } +} + +func filesGet(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var name string + if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil { + return nil, err + } + + if err := validateFileName(name); err != nil { + return nil, fmt.Errorf("files.get: %w", err) + } + + key := cfg.scopedKey(name) + rc, size, ct, err := cfg.Store.Get(ctx, key) + if err == storage.ErrNotFound { + return starlark.None, nil + } + if err != nil { + return nil, fmt.Errorf("files.get: %w", err) + } + defer rc.Close() + + data, err := io.ReadAll(rc) + if err != nil { + return nil, fmt.Errorf("files.get: read: %w", err) + } + + // Load metadata companion. + metaDict := starlark.NewDict(0) + metaKey := cfg.metaKey(name) + if metaRC, _, _, metaErr := cfg.Store.Get(ctx, metaKey); metaErr == nil { + metaBytes, _ := io.ReadAll(metaRC) + metaRC.Close() + var m map[string]any + if json.Unmarshal(metaBytes, &m) == nil { + metaDict = mapToStarlarkDict(m) + } + } + + result := starlark.NewDict(4) + result.SetKey(starlark.String("content"), starlark.Bytes(data)) + result.SetKey(starlark.String("content_type"), starlark.String(ct)) + result.SetKey(starlark.String("size"), starlark.MakeInt64(size)) + result.SetKey(starlark.String("metadata"), metaDict) + + return result, nil + } +} + +func filesMeta(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var name string + if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil { + return nil, err + } + + if err := validateFileName(name); err != nil { + return nil, fmt.Errorf("files.meta: %w", err) + } + + // Check the main object exists. + key := cfg.scopedKey(name) + exists, err := cfg.Store.Exists(ctx, key) + if err != nil { + return nil, fmt.Errorf("files.meta: %w", err) + } + if !exists { + return starlark.None, nil + } + + // Load metadata companion. + metaDict := starlark.NewDict(0) + metaKey := cfg.metaKey(name) + if metaRC, _, _, metaErr := cfg.Store.Get(ctx, metaKey); metaErr == nil { + metaBytes, _ := io.ReadAll(metaRC) + metaRC.Close() + var m map[string]any + if json.Unmarshal(metaBytes, &m) == nil { + metaDict = mapToStarlarkDict(m) + } + } + + return metaDict, nil + } +} + +func filesList(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var prefix string + var limit int = filesDefaultLimit + + if err := starlark.UnpackArgs(b.Name(), args, kwargs, + "prefix?", &prefix, + "limit?", &limit, + ); err != nil { + return nil, err + } + + if limit <= 0 || limit > 1000 { + limit = filesDefaultLimit + } + + // Build the full scoped prefix. + scopedPrefix := cfg.scopedPrefix() + if prefix != "" { + // Validate the user-supplied prefix portion. + if strings.Contains(prefix, "..") { + return nil, fmt.Errorf("files.list: path traversal not allowed") + } + scopedPrefix += prefix + } + + // Request more than limit to account for _meta/ entries we'll filter out. + entries, err := cfg.Store.List(ctx, scopedPrefix, limit*2) + if err != nil { + return nil, fmt.Errorf("files.list: %w", err) + } + + // Filter out _meta/ companions and strip the package prefix. + pkgPrefix := cfg.scopedPrefix() + var results []starlark.Value + for _, e := range entries { + // Strip package prefix to get the extension-relative key. + relKey := strings.TrimPrefix(e.Key, pkgPrefix) + + // Skip metadata companions. + if strings.HasPrefix(relKey, filesMetaPrefix) { + continue + } + + d := starlark.NewDict(3) + d.SetKey(starlark.String("name"), starlark.String(relKey)) + d.SetKey(starlark.String("size"), starlark.MakeInt64(e.Size)) + d.SetKey(starlark.String("content_type"), starlark.String(e.ContentType)) + results = append(results, d) + + if len(results) >= limit { + break + } + } + + return starlark.NewList(results), nil + } +} + +func filesDelete(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var name string + if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil { + return nil, err + } + + if err := validateFileName(name); err != nil { + return nil, fmt.Errorf("files.delete: %w", err) + } + + // Delete main object. + key := cfg.scopedKey(name) + if err := cfg.Store.Delete(ctx, key); err != nil { + return nil, fmt.Errorf("files.delete: %w", err) + } + + // Delete metadata companion (best-effort, idempotent). + metaKey := cfg.metaKey(name) + cfg.Store.Delete(ctx, metaKey) + + return starlark.True, nil + } +} + +func filesDeletePrefix(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var prefix string + if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &prefix); err != nil { + return nil, err + } + + if strings.Contains(prefix, "..") { + return nil, fmt.Errorf("files.delete_prefix: path traversal not allowed") + } + + scopedPrefix := cfg.scopedPrefix() + prefix + if err := cfg.Store.DeletePrefix(ctx, scopedPrefix); err != nil { + return nil, fmt.Errorf("files.delete_prefix: %w", err) + } + + // Also delete metadata companions under this prefix. + metaPrefix := cfg.scopedPrefix() + filesMetaPrefix + prefix + cfg.Store.DeletePrefix(ctx, metaPrefix) + + return starlark.True, nil + } +} + +func filesExists(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var name string + if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil { + return nil, err + } + + if err := validateFileName(name); err != nil { + return nil, fmt.Errorf("files.exists: %w", err) + } + + key := cfg.scopedKey(name) + exists, err := cfg.Store.Exists(ctx, key) + if err != nil { + return nil, fmt.Errorf("files.exists: %w", err) + } + + return starlark.Bool(exists), nil + } +} + +// ── Helpers ────────────────────────────────── + +// scopedKey returns the full ObjectStore key for a file name. +func (cfg FilesModuleConfig) scopedKey(name string) string { + return fmt.Sprintf("ext/%s/%s", cfg.PackageID, name) +} + +// metaKey returns the ObjectStore key for a file's metadata companion. +func (cfg FilesModuleConfig) metaKey(name string) string { + return fmt.Sprintf("ext/%s/%s%s", cfg.PackageID, filesMetaPrefix, name) +} + +// scopedPrefix returns the ObjectStore key prefix for this package. +func (cfg FilesModuleConfig) scopedPrefix() string { + return fmt.Sprintf("ext/%s/", cfg.PackageID) +} + +// validateFileName checks that a file name is safe. +func validateFileName(name string) error { + if name == "" { + return fmt.Errorf("file name is required") + } + if len(name) > filesMaxNameLen { + return fmt.Errorf("file name exceeds %d characters", filesMaxNameLen) + } + if strings.HasPrefix(name, "/") || strings.HasPrefix(name, "\\") { + return fmt.Errorf("absolute paths not allowed: %q", name) + } + if strings.Contains(name, "..") { + return fmt.Errorf("path traversal not allowed: %q", name) + } + if strings.HasPrefix(name, filesMetaPrefix) { + return fmt.Errorf("names starting with %q are reserved", filesMetaPrefix) + } + for _, r := range name { + if unicode.IsControl(r) { + return fmt.Errorf("control characters not allowed in file name") + } + } + return nil +} + +// filesMaxSize returns the maximum file size from env or default. +func filesMaxSize() int64 { + if v := os.Getenv("EXT_FILES_MAX_SIZE"); v != "" { + if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 { + return n + } + } + return filesDefaultMaxSize +} + +// starlarkDictToMap converts a *starlark.Dict to map[string]any. +func starlarkDictToMap(d *starlark.Dict) (map[string]any, error) { + m := make(map[string]any, d.Len()) + for _, item := range d.Items() { + k, ok := starlark.AsString(item[0]) + if !ok { + return nil, fmt.Errorf("metadata keys must be strings, got %s", item[0].Type()) + } + m[k] = starlarkValueToGo(item[1]) + } + return m, nil +} + +// starlarkValueToGo converts a starlark.Value to a Go value for JSON serialization. +func starlarkValueToGo(v starlark.Value) any { + switch x := v.(type) { + case starlark.String: + return string(x) + case starlark.Int: + if i, ok := x.Int64(); ok { + return i + } + return x.String() + case starlark.Float: + return float64(x) + case starlark.Bool: + return bool(x) + case *starlark.List: + out := make([]any, x.Len()) + for i := 0; i < x.Len(); i++ { + out[i] = starlarkValueToGo(x.Index(i)) + } + return out + case *starlark.Dict: + m, _ := starlarkDictToMap(x) + return m + default: + return v.String() + } +} + +// mapToStarlarkDict converts a map[string]any to *starlark.Dict. +func mapToStarlarkDict(m map[string]any) *starlark.Dict { + d := starlark.NewDict(len(m)) + for k, v := range m { + d.SetKey(starlark.String(k), goValueToStarlark(v)) + } + return d +} + +// goValueToStarlark converts a Go value (from JSON) to starlark.Value. +func goValueToStarlark(v any) starlark.Value { + switch x := v.(type) { + case string: + return starlark.String(x) + case float64: + if x == float64(int64(x)) { + return starlark.MakeInt64(int64(x)) + } + return starlark.Float(x) + case bool: + return starlark.Bool(x) + case []any: + elems := make([]starlark.Value, len(x)) + for i, e := range x { + elems[i] = goValueToStarlark(e) + } + return starlark.NewList(elems) + case map[string]any: + return mapToStarlarkDict(x) + default: + return starlark.None + } +} diff --git a/server/sandbox/files_module_test.go b/server/sandbox/files_module_test.go new file mode 100644 index 0000000..52ca1ff --- /dev/null +++ b/server/sandbox/files_module_test.go @@ -0,0 +1,410 @@ +package sandbox + +import ( + "context" + "fmt" + "io" + "strings" + "testing" + + "go.starlark.net/starlark" + + "armature/storage" +) + +// memStore is an in-memory ObjectStore for testing. +type memStore struct { + objects map[string]memObject +} + +type memObject struct { + data []byte + contentType string +} + +func newMemStore() *memStore { + return &memStore{objects: make(map[string]memObject)} +} + +func (m *memStore) Put(_ context.Context, key string, r io.Reader, size int64, contentType string) error { + data, err := io.ReadAll(r) + if err != nil { + return err + } + m.objects[key] = memObject{data: data, contentType: contentType} + return nil +} + +func (m *memStore) Get(_ context.Context, key string) (io.ReadCloser, int64, string, error) { + obj, ok := m.objects[key] + if !ok { + return nil, 0, "", storage.ErrNotFound + } + return io.NopCloser(&memReader{data: obj.data}), int64(len(obj.data)), obj.contentType, nil +} + +func (m *memStore) Delete(_ context.Context, key string) error { + delete(m.objects, key) + return nil +} + +func (m *memStore) DeletePrefix(_ context.Context, prefix string) error { + for k := range m.objects { + if strings.HasPrefix(k, prefix) { + delete(m.objects, k) + } + } + return nil +} + +func (m *memStore) Exists(_ context.Context, key string) (bool, error) { + _, ok := m.objects[key] + return ok, nil +} + +func (m *memStore) Healthy(_ context.Context) error { return nil } + +func (m *memStore) Stats(_ context.Context) (*storage.StorageStats, error) { + return &storage.StorageStats{Backend: "mem", Configured: true, Healthy: true}, nil +} + +func (m *memStore) Backend() string { return "mem" } + +func (m *memStore) List(_ context.Context, prefix string, limit int) ([]storage.ObjectEntry, error) { + if limit <= 0 { + limit = 100 + } + var entries []storage.ObjectEntry + for k, obj := range m.objects { + if strings.HasPrefix(k, prefix) { + entries = append(entries, storage.ObjectEntry{ + Key: k, + Size: int64(len(obj.data)), + ContentType: obj.contentType, + }) + if len(entries) >= limit { + break + } + } + } + return entries, nil +} + +// memReader wraps a byte slice as io.ReadCloser. +type memReader struct { + data []byte + pos int +} + +func (r *memReader) Read(p []byte) (int, error) { + if r.pos >= len(r.data) { + return 0, io.EOF + } + n := copy(p, r.data[r.pos:]) + r.pos += n + if r.pos >= len(r.data) { + return n, io.EOF + } + return n, nil +} + +func (r *memReader) Close() error { return nil } + +// ── Test helpers ────────────────────────────── + +func execWithFiles(t *testing.T, canWrite bool, script string) (*Result, error) { + t.Helper() + ctx := context.Background() + store := newMemStore() + mod := BuildFilesModule(ctx, FilesModuleConfig{ + PackageID: "test-pkg", + CanWrite: canWrite, + Store: store, + }) + sb := New(DefaultConfig()) + return sb.Exec(ctx, "test.star", script, map[string]starlark.Value{ + "files": mod, + }) +} + +func execWithFilesStore(t *testing.T, store *memStore, canWrite bool, script string) (*Result, error) { + t.Helper() + ctx := context.Background() + mod := BuildFilesModule(ctx, FilesModuleConfig{ + PackageID: "test-pkg", + CanWrite: canWrite, + Store: store, + }) + sb := New(DefaultConfig()) + return sb.Exec(ctx, "test.star", script, map[string]starlark.Value{ + "files": mod, + }) +} + +// ── Put + Get round-trip ────────────────────── + +func TestFilesPut_StringContent(t *testing.T) { + _, err := execWithFiles(t, true, ` +files.put("hello.txt", "hello world", content_type="text/plain") +result = files.get("hello.txt") +`) + if err != nil { + t.Fatalf("exec error: %v", err) + } +} + +func TestFilesPut_BytesContent(t *testing.T) { + result, err := execWithFiles(t, true, ` +files.put("bin.dat", b"\x00\x01\x02\xff") +got = files.get("bin.dat") +size = got["size"] +ct = got["content_type"] +`) + if err != nil { + t.Fatalf("exec error: %v", err) + } + + size, _ := starlark.AsInt32(result.Globals["size"]) + if size != 4 { + t.Errorf("size = %d, want 4", size) + } + ct := string(result.Globals["ct"].(starlark.String)) + if ct != "application/octet-stream" { + t.Errorf("content_type = %q, want application/octet-stream", ct) + } +} + +func TestFilesPut_WithMetadata(t *testing.T) { + result, err := execWithFiles(t, true, ` +files.put("doc.pdf", "pdf content", content_type="application/pdf", metadata={"author": "test", "pages": 5}) +meta = files.meta("doc.pdf") +author = meta["author"] +pages = meta["pages"] +`) + if err != nil { + t.Fatalf("exec error: %v", err) + } + + author := string(result.Globals["author"].(starlark.String)) + if author != "test" { + t.Errorf("author = %q, want %q", author, "test") + } + pages, _ := starlark.AsInt32(result.Globals["pages"]) + if pages != 5 { + t.Errorf("pages = %d, want 5", pages) + } +} + +func TestFilesPut_SizeLimit(t *testing.T) { + // Set a small limit for testing. + t.Setenv("EXT_FILES_MAX_SIZE", "10") + _, err := execWithFiles(t, true, ` +files.put("big.bin", "this content is longer than ten bytes") +`) + if err == nil { + t.Fatal("expected size limit error") + } + if !strings.Contains(err.Error(), "exceeds limit") { + t.Errorf("unexpected error: %v", err) + } +} + +// ── Get ─────────────────────────────────────── + +func TestFilesGet_NotFound(t *testing.T) { + result, err := execWithFiles(t, false, ` +got = files.get("nonexistent.txt") +`) + if err != nil { + t.Fatalf("exec error: %v", err) + } + if result.Globals["got"] != starlark.None { + t.Errorf("expected None, got %v", result.Globals["got"]) + } +} + +func TestFilesGet_ReturnsAllFields(t *testing.T) { + result, err := execWithFiles(t, true, ` +files.put("test.txt", "data", content_type="text/plain", metadata={"k": "v"}) +got = files.get("test.txt") +has_content = "content" in got +has_ct = "content_type" in got +has_size = "size" in got +has_meta = "metadata" in got +`) + if err != nil { + t.Fatalf("exec error: %v", err) + } + for _, key := range []string{"has_content", "has_ct", "has_size", "has_meta"} { + if result.Globals[key] != starlark.True { + t.Errorf("%s = False, want True", key) + } + } +} + +// ── Meta ────────────────────────────────────── + +func TestFilesMeta_NotFound(t *testing.T) { + result, err := execWithFiles(t, false, ` +got = files.meta("nonexistent.txt") +`) + if err != nil { + t.Fatalf("exec error: %v", err) + } + if result.Globals["got"] != starlark.None { + t.Errorf("expected None, got %v", result.Globals["got"]) + } +} + +func TestFilesMeta_NoCompanion(t *testing.T) { + result, err := execWithFiles(t, true, ` +files.put("plain.txt", "data") +meta = files.meta("plain.txt") +count = len(meta) +`) + if err != nil { + t.Fatalf("exec error: %v", err) + } + count, _ := starlark.AsInt32(result.Globals["count"]) + if count != 0 { + t.Errorf("meta length = %d, want 0", count) + } +} + +// ── List ────────────────────────────────────── + +func TestFilesList_PrefixFilter(t *testing.T) { + result, err := execWithFiles(t, true, ` +files.put("images/a.png", "a") +files.put("images/b.png", "b") +files.put("docs/c.pdf", "c") +all_entries = files.list() +img_entries = files.list(prefix="images/") +all_count = len(all_entries) +img_count = len(img_entries) +`) + if err != nil { + t.Fatalf("exec error: %v", err) + } + + allCount, _ := starlark.AsInt32(result.Globals["all_count"]) + imgCount, _ := starlark.AsInt32(result.Globals["img_count"]) + + if allCount != 3 { + t.Errorf("all_count = %d, want 3", allCount) + } + if imgCount != 2 { + t.Errorf("img_count = %d, want 2", imgCount) + } +} + +func TestFilesList_ExcludesMeta(t *testing.T) { + result, err := execWithFiles(t, true, ` +files.put("file.txt", "data", metadata={"key": "val"}) +entries = files.list() +count = len(entries) +`) + if err != nil { + t.Fatalf("exec error: %v", err) + } + + count, _ := starlark.AsInt32(result.Globals["count"]) + if count != 1 { + t.Errorf("count = %d, want 1 (should exclude _meta/ companion)", count) + } +} + +// ── Delete ──────────────────────────────────── + +func TestFilesDelete_Idempotent(t *testing.T) { + _, err := execWithFiles(t, true, ` +files.put("temp.txt", "temp") +files.delete("temp.txt") +files.delete("temp.txt") +exists = files.exists("temp.txt") +`) + if err != nil { + t.Fatalf("exec error: %v", err) + } +} + +func TestFilesDeletePrefix(t *testing.T) { + result, err := execWithFiles(t, true, ` +files.put("uploads/a.txt", "a") +files.put("uploads/b.txt", "b") +files.put("keep.txt", "keep") +files.delete_prefix("uploads/") +a_exists = files.exists("uploads/a.txt") +b_exists = files.exists("uploads/b.txt") +keep_exists = files.exists("keep.txt") +`) + if err != nil { + t.Fatalf("exec error: %v", err) + } + + if result.Globals["a_exists"] != starlark.False { + t.Error("uploads/a.txt should be deleted") + } + if result.Globals["b_exists"] != starlark.False { + t.Error("uploads/b.txt should be deleted") + } + if result.Globals["keep_exists"] != starlark.True { + t.Error("keep.txt should be preserved") + } +} + +// ── Exists ──────────────────────────────────── + +func TestFilesExists(t *testing.T) { + result, err := execWithFiles(t, true, ` +files.put("exists.txt", "yes") +yes = files.exists("exists.txt") +no = files.exists("nope.txt") +`) + if err != nil { + t.Fatalf("exec error: %v", err) + } + + if result.Globals["yes"] != starlark.True { + t.Error("exists should be True") + } + if result.Globals["no"] != starlark.False { + t.Error("nope should be False") + } +} + +// ── Name Validation ────────────────────────── + +func TestFilesNameValidation(t *testing.T) { + badNames := []string{ + "../escape", + "/absolute", + "_meta/reserved", + "", + } + + for _, name := range badNames { + t.Run(name, func(t *testing.T) { + script := fmt.Sprintf(`files.exists(%q)`, name) + _, err := execWithFiles(t, false, script) + if err == nil { + t.Errorf("expected error for name %q", name) + } + }) + } +} + +// ── Write Permission Gate ──────────────────── + +func TestFilesWritePermission(t *testing.T) { + // Read-only mode: put/delete/delete_prefix should not be available. + _, err := execWithFiles(t, false, ` +files.put("test.txt", "data") +`) + if err == nil { + t.Fatal("expected error: put should not be available in read-only mode") + } + if !strings.Contains(err.Error(), "no .put field") && !strings.Contains(err.Error(), "has no .put") { + t.Errorf("unexpected error: %v", err) + } +} diff --git a/server/sandbox/runner.go b/server/sandbox/runner.go index ad98042..7a65dd0 100644 --- a/server/sandbox/runner.go +++ b/server/sandbox/runner.go @@ -33,6 +33,7 @@ import ( "armature/events" "armature/metrics" "armature/models" + "armature/storage" "armature/store" ) @@ -77,6 +78,7 @@ type Runner struct { dbPostgres bool // true = use $N placeholders; false = use ? allowPrivateIPs bool bus *events.Bus + objectStore storage.ObjectStore // nil = files module unavailable } // NewRunner creates a runner with the given sandbox and dependencies. @@ -116,6 +118,11 @@ func (r *Runner) SetBus(bus *events.Bus) { r.bus = bus } +// SetObjectStore attaches the blob storage backend for the files module. +func (r *Runner) SetObjectStore(s storage.ObjectStore) { + r.objectStore = s +} + // SetAllowPrivateIPs disables the SSRF check that blocks connections to // private/loopback IPs. For self-hosted environments where extensions // reach internal services. Controlled by EXT_ALLOW_PRIVATE_IPS env var. @@ -336,8 +343,9 @@ func (r *Runner) buildModulesWithLibCtx(ctx context.Context, packageID string, m modules := make(map[string]starlark.Value) - // Track db permission level: 0=none, 1=read, 2=write + // Track tiered permission levels: 0=none, 1=read, 2=write dbLevel := 0 + filesLevel := 0 hasBatchExec := false for _, perm := range granted { @@ -363,6 +371,14 @@ func (r *Runner) buildModulesWithLibCtx(ctx context.Context, packageID string, m case models.ExtPermDBWrite: dbLevel = 2 + case models.ExtPermFilesRead: + if filesLevel < 1 { + filesLevel = 1 + } + + case models.ExtPermFilesWrite: + filesLevel = 2 + case models.ExtPermWorkflowAccess: modules["workflow"] = BuildWorkflowModule(ctx, r.stores) @@ -391,6 +407,15 @@ func (r *Runner) buildModulesWithLibCtx(ctx context.Context, packageID string, m }) } + // Wire files module at the highest granted level. + if filesLevel > 0 && r.objectStore != nil { + modules["files"] = BuildFilesModule(ctx, FilesModuleConfig{ + PackageID: packageID, + CanWrite: filesLevel == 2, + Store: r.objectStore, + }) + } + // A package reads its own admin + team + user settings. userID := "" teamID := "" diff --git a/server/storage/pvc.go b/server/storage/pvc.go index 9a29967..d8e5268 100644 --- a/server/storage/pvc.go +++ b/server/storage/pvc.go @@ -211,6 +211,78 @@ func (s *PVCStore) Exists(ctx context.Context, key string) (bool, error) { return false, fmt.Errorf("storage/pvc: exists %q: %w", key, err) } +// List returns objects under the given prefix, up to limit entries. +func (s *PVCStore) List(ctx context.Context, prefix string, limit int) ([]ObjectEntry, error) { + if prefix != "" { + if err := validateKey(prefix); err != nil { + return nil, err + } + } + if limit <= 0 { + limit = 100 + } + + absPrefix := s.resolve(prefix) + + // If the prefix path doesn't exist, return empty. + info, err := os.Stat(absPrefix) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("storage/pvc: list %q: %w", prefix, err) + } + + var entries []ObjectEntry + + if !info.IsDir() { + // Prefix points to a single file — return it. + s.mu.RLock() + ct := s.mimeMap[prefix] + s.mu.RUnlock() + if ct == "" { + ct = "application/octet-stream" + } + return []ObjectEntry{{Key: prefix, Size: info.Size(), ContentType: ct}}, nil + } + + // Walk the directory tree under the prefix. + walkRoot := absPrefix + err = filepath.Walk(walkRoot, func(path string, fi os.FileInfo, walkErr error) error { + if walkErr != nil { + return nil // skip inaccessible entries + } + if fi.IsDir() { + return nil + } + if len(entries) >= limit { + return filepath.SkipAll + } + + // Recover the storage key by stripping basePath + "/". + rel, relErr := filepath.Rel(s.basePath, path) + if relErr != nil { + return nil + } + key := filepath.ToSlash(rel) + + s.mu.RLock() + ct := s.mimeMap[key] + s.mu.RUnlock() + if ct == "" { + ct = "application/octet-stream" + } + + entries = append(entries, ObjectEntry{Key: key, Size: fi.Size(), ContentType: ct}) + return nil + }) + if err != nil { + return entries, fmt.Errorf("storage/pvc: list walk %q: %w", prefix, err) + } + + return entries, nil +} + // Healthy verifies the storage path is writable. func (s *PVCStore) Healthy(ctx context.Context) error { probe := filepath.Join(s.basePath, ".health-probe") diff --git a/server/storage/pvc_test.go b/server/storage/pvc_test.go index 999a3a8..03058b1 100644 --- a/server/storage/pvc_test.go +++ b/server/storage/pvc_test.go @@ -282,6 +282,64 @@ func TestPVC_SubdirCreation(t *testing.T) { } } +func TestPVC_List(t *testing.T) { + s := tempStore(t) + ctx := context.Background() + + // Empty prefix — no files. + entries, err := s.List(ctx, "ext/pkg1/", 100) + if err != nil { + t.Fatalf("List empty: %v", err) + } + if len(entries) != 0 { + t.Errorf("expected 0 entries, got %d", len(entries)) + } + + // Add some files under a prefix. + for _, name := range []string{"a.txt", "b.png", "sub/c.pdf"} { + key := "ext/pkg1/" + name + _ = s.Put(ctx, key, bytes.NewReader([]byte("x")), 1, "text/plain") + } + // Add a file under a different prefix. + _ = s.Put(ctx, "ext/pkg2/other.txt", bytes.NewReader([]byte("y")), 1, "text/plain") + + // List all under pkg1. + entries, err = s.List(ctx, "ext/pkg1/", 100) + if err != nil { + t.Fatalf("List: %v", err) + } + if len(entries) != 3 { + t.Errorf("expected 3 entries, got %d", len(entries)) + } + + // List with limit. + entries, err = s.List(ctx, "ext/pkg1/", 2) + if err != nil { + t.Fatalf("List limited: %v", err) + } + if len(entries) != 2 { + t.Errorf("expected 2 entries (limit), got %d", len(entries)) + } + + // List with sub-prefix. + entries, err = s.List(ctx, "ext/pkg1/sub/", 100) + if err != nil { + t.Fatalf("List sub: %v", err) + } + if len(entries) != 1 { + t.Errorf("expected 1 entry in sub/, got %d", len(entries)) + } + + // pkg2 prefix should only find 1. + entries, err = s.List(ctx, "ext/pkg2/", 100) + if err != nil { + t.Fatalf("List pkg2: %v", err) + } + if len(entries) != 1 { + t.Errorf("expected 1 entry in pkg2, got %d", len(entries)) + } +} + func TestNewPVC_InvalidPath(t *testing.T) { // Path that can't be created (nested under a file) tmp := t.TempDir() diff --git a/server/storage/s3.go b/server/storage/s3.go index 042e0b3..ac4d424 100644 --- a/server/storage/s3.go +++ b/server/storage/s3.go @@ -232,6 +232,49 @@ func (s *S3Store) Exists(ctx context.Context, key string) (bool, error) { return true, nil } +// List returns objects under the given prefix, up to limit entries. +func (s *S3Store) List(ctx context.Context, prefix string, limit int) ([]ObjectEntry, error) { + if prefix != "" { + if err := validateKey(prefix); err != nil { + return nil, err + } + } + if limit <= 0 { + limit = 100 + } + + fullPrefix := s.fullKey(prefix) + objectsCh := s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{ + Prefix: fullPrefix, + Recursive: true, + }) + + var entries []ObjectEntry + for obj := range objectsCh { + if obj.Err != nil { + return entries, fmt.Errorf("storage/s3: list %q: %w", prefix, obj.Err) + } + if len(entries) >= limit { + // Drain the remaining channel entries (minio requires it). + for range objectsCh { + } + break + } + + // Strip the s3 prefix to recover the storage key. + key := strings.TrimPrefix(obj.Key, s.prefix) + + ct := obj.ContentType + if ct == "" { + ct = "application/octet-stream" + } + + entries = append(entries, ObjectEntry{Key: key, Size: obj.Size, ContentType: ct}) + } + + return entries, nil +} + // Healthy checks connectivity by performing a HeadBucket. func (s *S3Store) Healthy(ctx context.Context) error { exists, err := s.client.BucketExists(ctx, s.bucket) diff --git a/server/storage/storage.go b/server/storage/storage.go index 59ccc43..76afbc2 100644 --- a/server/storage/storage.go +++ b/server/storage/storage.go @@ -56,10 +56,21 @@ type ObjectStore interface { // Stats returns aggregate storage statistics. Stats(ctx context.Context) (*StorageStats, error) + // List returns objects under the given prefix, up to limit entries. + // Returns an empty slice (not error) if no objects match. + List(ctx context.Context, prefix string, limit int) ([]ObjectEntry, error) + // Backend returns the backend type identifier ("pvc", "s3"). Backend() string } +// ObjectEntry represents a single object returned by List. +type ObjectEntry struct { + Key string `json:"key"` + Size int64 `json:"size"` + ContentType string `json:"content_type"` +} + // StorageStats holds aggregate storage metrics. type StorageStats struct { Backend string `json:"backend"` -- 2.49.1