Feat v0.8.0 files module (#67)
All checks were successful
CI/CD / detect-changes (push) Successful in 4s
CI/CD / test-frontend (push) Has been skipped
CI/CD / test-runners (push) Has been skipped
CI/CD / e2e-smoke (push) Has been skipped
CI/CD / test-go-pg (push) Successful in 2m45s
CI/CD / test-sqlite (push) Successful in 2m52s
CI/CD / build-and-deploy (push) Successful in 1m20s
All checks were successful
CI/CD / detect-changes (push) Successful in 4s
CI/CD / test-frontend (push) Has been skipped
CI/CD / test-runners (push) Has been skipped
CI/CD / e2e-smoke (push) Has been skipped
CI/CD / test-go-pg (push) Successful in 2m45s
CI/CD / test-sqlite (push) Successful in 2m52s
CI/CD / build-and-deploy (push) Successful in 1m20s
Co-authored-by: Jeffrey Smith <jasafpro@gmail.com> Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>
This commit was merged in pull request #67.
This commit is contained in:
472
server/sandbox/files_module.go
Normal file
472
server/sandbox/files_module.go
Normal file
@@ -0,0 +1,472 @@
|
||||
// Package sandbox — files_module.go
|
||||
//
|
||||
// Permissions: files.read, files.write
|
||||
//
|
||||
// Starlark API:
|
||||
// files.put(name, content, content_type="application/octet-stream", metadata={})
|
||||
// result = files.get(name) → dict or None
|
||||
// meta = files.meta(name) → dict or None
|
||||
// entries = files.list(prefix="", limit=100)
|
||||
// files.delete(name)
|
||||
// files.delete_prefix(prefix)
|
||||
// exists = files.exists(name)
|
||||
//
|
||||
// Bridges the ObjectStore (PVC/S3) into Starlark. All keys are scoped
|
||||
// to ext/{packageID}/ — extensions cannot access each other's files.
|
||||
// Metadata is stored as companion JSON at ext/{packageID}/_meta/{name}.
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"go.starlark.net/starlark"
|
||||
"go.starlark.net/starlarkstruct"
|
||||
|
||||
"armature/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
filesDefaultMaxSize = 50 * 1024 * 1024 // 50 MB
|
||||
filesDefaultLimit = 100
|
||||
filesMaxNameLen = 512
|
||||
filesMetaPrefix = "_meta/"
|
||||
)
|
||||
|
||||
// FilesModuleConfig holds the configuration for the files module.
|
||||
type FilesModuleConfig struct {
|
||||
PackageID string
|
||||
CanWrite bool
|
||||
Store storage.ObjectStore
|
||||
}
|
||||
|
||||
// BuildFilesModule creates the "files" module.
|
||||
func BuildFilesModule(ctx context.Context, cfg FilesModuleConfig) *starlarkstruct.Module {
|
||||
fns := starlark.StringDict{
|
||||
"get": starlark.NewBuiltin("files.get", filesGet(ctx, cfg)),
|
||||
"meta": starlark.NewBuiltin("files.meta", filesMeta(ctx, cfg)),
|
||||
"list": starlark.NewBuiltin("files.list", filesList(ctx, cfg)),
|
||||
"exists": starlark.NewBuiltin("files.exists", filesExists(ctx, cfg)),
|
||||
}
|
||||
|
||||
if cfg.CanWrite {
|
||||
fns["put"] = starlark.NewBuiltin("files.put", filesPut(ctx, cfg))
|
||||
fns["delete"] = starlark.NewBuiltin("files.delete", filesDelete(ctx, cfg))
|
||||
fns["delete_prefix"] = starlark.NewBuiltin("files.delete_prefix", filesDeletePrefix(ctx, cfg))
|
||||
}
|
||||
|
||||
return MakeModule("files", fns)
|
||||
}
|
||||
|
||||
// ── Builtins ──────────────────────────────────
|
||||
|
||||
func filesPut(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
|
||||
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
|
||||
var name string
|
||||
var content starlark.Value
|
||||
var contentType string = "application/octet-stream"
|
||||
var metadata *starlark.Dict
|
||||
|
||||
if err := starlark.UnpackArgs(b.Name(), args, kwargs,
|
||||
"name", &name,
|
||||
"content", &content,
|
||||
"content_type?", &contentType,
|
||||
"metadata?", &metadata,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := validateFileName(name); err != nil {
|
||||
return nil, fmt.Errorf("files.put: %w", err)
|
||||
}
|
||||
|
||||
// Extract raw bytes from content (accept String or Bytes).
|
||||
var raw []byte
|
||||
switch v := content.(type) {
|
||||
case starlark.String:
|
||||
raw = []byte(string(v))
|
||||
case starlark.Bytes:
|
||||
raw = []byte(v)
|
||||
default:
|
||||
return nil, fmt.Errorf("files.put: content must be string or bytes, got %s", content.Type())
|
||||
}
|
||||
|
||||
// Enforce size limit.
|
||||
maxSize := filesMaxSize()
|
||||
if int64(len(raw)) > maxSize {
|
||||
return nil, fmt.Errorf("files.put: content size %d exceeds limit %d", len(raw), maxSize)
|
||||
}
|
||||
|
||||
// Write main object.
|
||||
key := cfg.scopedKey(name)
|
||||
if err := cfg.Store.Put(ctx, key, bytes.NewReader(raw), int64(len(raw)), contentType); err != nil {
|
||||
return nil, fmt.Errorf("files.put: %w", err)
|
||||
}
|
||||
|
||||
// Write metadata companion if provided.
|
||||
if metadata != nil && metadata.Len() > 0 {
|
||||
metaMap, err := starlarkDictToMap(metadata)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("files.put: metadata: %w", err)
|
||||
}
|
||||
metaJSON, err := json.Marshal(metaMap)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("files.put: marshal metadata: %w", err)
|
||||
}
|
||||
metaKey := cfg.metaKey(name)
|
||||
if err := cfg.Store.Put(ctx, metaKey, bytes.NewReader(metaJSON), int64(len(metaJSON)), "application/json"); err != nil {
|
||||
return nil, fmt.Errorf("files.put: write metadata: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return starlark.True, nil
|
||||
}
|
||||
}
|
||||
|
||||
func filesGet(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
|
||||
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
|
||||
var name string
|
||||
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := validateFileName(name); err != nil {
|
||||
return nil, fmt.Errorf("files.get: %w", err)
|
||||
}
|
||||
|
||||
key := cfg.scopedKey(name)
|
||||
rc, size, ct, err := cfg.Store.Get(ctx, key)
|
||||
if err == storage.ErrNotFound {
|
||||
return starlark.None, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("files.get: %w", err)
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
data, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("files.get: read: %w", err)
|
||||
}
|
||||
|
||||
// Load metadata companion.
|
||||
metaDict := starlark.NewDict(0)
|
||||
metaKey := cfg.metaKey(name)
|
||||
if metaRC, _, _, metaErr := cfg.Store.Get(ctx, metaKey); metaErr == nil {
|
||||
metaBytes, _ := io.ReadAll(metaRC)
|
||||
metaRC.Close()
|
||||
var m map[string]any
|
||||
if json.Unmarshal(metaBytes, &m) == nil {
|
||||
metaDict = mapToStarlarkDict(m)
|
||||
}
|
||||
}
|
||||
|
||||
result := starlark.NewDict(4)
|
||||
result.SetKey(starlark.String("content"), starlark.Bytes(data))
|
||||
result.SetKey(starlark.String("content_type"), starlark.String(ct))
|
||||
result.SetKey(starlark.String("size"), starlark.MakeInt64(size))
|
||||
result.SetKey(starlark.String("metadata"), metaDict)
|
||||
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
|
||||
func filesMeta(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
|
||||
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
|
||||
var name string
|
||||
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := validateFileName(name); err != nil {
|
||||
return nil, fmt.Errorf("files.meta: %w", err)
|
||||
}
|
||||
|
||||
// Check the main object exists.
|
||||
key := cfg.scopedKey(name)
|
||||
exists, err := cfg.Store.Exists(ctx, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("files.meta: %w", err)
|
||||
}
|
||||
if !exists {
|
||||
return starlark.None, nil
|
||||
}
|
||||
|
||||
// Load metadata companion.
|
||||
metaDict := starlark.NewDict(0)
|
||||
metaKey := cfg.metaKey(name)
|
||||
if metaRC, _, _, metaErr := cfg.Store.Get(ctx, metaKey); metaErr == nil {
|
||||
metaBytes, _ := io.ReadAll(metaRC)
|
||||
metaRC.Close()
|
||||
var m map[string]any
|
||||
if json.Unmarshal(metaBytes, &m) == nil {
|
||||
metaDict = mapToStarlarkDict(m)
|
||||
}
|
||||
}
|
||||
|
||||
return metaDict, nil
|
||||
}
|
||||
}
|
||||
|
||||
func filesList(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
|
||||
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
|
||||
var prefix string
|
||||
var limit int = filesDefaultLimit
|
||||
|
||||
if err := starlark.UnpackArgs(b.Name(), args, kwargs,
|
||||
"prefix?", &prefix,
|
||||
"limit?", &limit,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if limit <= 0 || limit > 1000 {
|
||||
limit = filesDefaultLimit
|
||||
}
|
||||
|
||||
// Build the full scoped prefix.
|
||||
scopedPrefix := cfg.scopedPrefix()
|
||||
if prefix != "" {
|
||||
// Validate the user-supplied prefix portion.
|
||||
if strings.Contains(prefix, "..") {
|
||||
return nil, fmt.Errorf("files.list: path traversal not allowed")
|
||||
}
|
||||
scopedPrefix += prefix
|
||||
}
|
||||
|
||||
// Request more than limit to account for _meta/ entries we'll filter out.
|
||||
entries, err := cfg.Store.List(ctx, scopedPrefix, limit*2)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("files.list: %w", err)
|
||||
}
|
||||
|
||||
// Filter out _meta/ companions and strip the package prefix.
|
||||
pkgPrefix := cfg.scopedPrefix()
|
||||
var results []starlark.Value
|
||||
for _, e := range entries {
|
||||
// Strip package prefix to get the extension-relative key.
|
||||
relKey := strings.TrimPrefix(e.Key, pkgPrefix)
|
||||
|
||||
// Skip metadata companions.
|
||||
if strings.HasPrefix(relKey, filesMetaPrefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
d := starlark.NewDict(3)
|
||||
d.SetKey(starlark.String("name"), starlark.String(relKey))
|
||||
d.SetKey(starlark.String("size"), starlark.MakeInt64(e.Size))
|
||||
d.SetKey(starlark.String("content_type"), starlark.String(e.ContentType))
|
||||
results = append(results, d)
|
||||
|
||||
if len(results) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return starlark.NewList(results), nil
|
||||
}
|
||||
}
|
||||
|
||||
func filesDelete(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
|
||||
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
|
||||
var name string
|
||||
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := validateFileName(name); err != nil {
|
||||
return nil, fmt.Errorf("files.delete: %w", err)
|
||||
}
|
||||
|
||||
// Delete main object.
|
||||
key := cfg.scopedKey(name)
|
||||
if err := cfg.Store.Delete(ctx, key); err != nil {
|
||||
return nil, fmt.Errorf("files.delete: %w", err)
|
||||
}
|
||||
|
||||
// Delete metadata companion (best-effort, idempotent).
|
||||
metaKey := cfg.metaKey(name)
|
||||
cfg.Store.Delete(ctx, metaKey)
|
||||
|
||||
return starlark.True, nil
|
||||
}
|
||||
}
|
||||
|
||||
func filesDeletePrefix(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
|
||||
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
|
||||
var prefix string
|
||||
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &prefix); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if strings.Contains(prefix, "..") {
|
||||
return nil, fmt.Errorf("files.delete_prefix: path traversal not allowed")
|
||||
}
|
||||
|
||||
scopedPrefix := cfg.scopedPrefix() + prefix
|
||||
if err := cfg.Store.DeletePrefix(ctx, scopedPrefix); err != nil {
|
||||
return nil, fmt.Errorf("files.delete_prefix: %w", err)
|
||||
}
|
||||
|
||||
// Also delete metadata companions under this prefix.
|
||||
metaPrefix := cfg.scopedPrefix() + filesMetaPrefix + prefix
|
||||
cfg.Store.DeletePrefix(ctx, metaPrefix)
|
||||
|
||||
return starlark.True, nil
|
||||
}
|
||||
}
|
||||
|
||||
func filesExists(ctx context.Context, cfg FilesModuleConfig) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
|
||||
return func(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
|
||||
var name string
|
||||
if err := starlark.UnpackPositionalArgs(b.Name(), args, kwargs, 1, &name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := validateFileName(name); err != nil {
|
||||
return nil, fmt.Errorf("files.exists: %w", err)
|
||||
}
|
||||
|
||||
key := cfg.scopedKey(name)
|
||||
exists, err := cfg.Store.Exists(ctx, key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("files.exists: %w", err)
|
||||
}
|
||||
|
||||
return starlark.Bool(exists), nil
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helpers ──────────────────────────────────
|
||||
|
||||
// scopedKey returns the full ObjectStore key for a file name.
|
||||
func (cfg FilesModuleConfig) scopedKey(name string) string {
|
||||
return fmt.Sprintf("ext/%s/%s", cfg.PackageID, name)
|
||||
}
|
||||
|
||||
// metaKey returns the ObjectStore key for a file's metadata companion.
|
||||
func (cfg FilesModuleConfig) metaKey(name string) string {
|
||||
return fmt.Sprintf("ext/%s/%s%s", cfg.PackageID, filesMetaPrefix, name)
|
||||
}
|
||||
|
||||
// scopedPrefix returns the ObjectStore key prefix for this package.
|
||||
func (cfg FilesModuleConfig) scopedPrefix() string {
|
||||
return fmt.Sprintf("ext/%s/", cfg.PackageID)
|
||||
}
|
||||
|
||||
// validateFileName checks that a file name is safe.
|
||||
func validateFileName(name string) error {
|
||||
if name == "" {
|
||||
return fmt.Errorf("file name is required")
|
||||
}
|
||||
if len(name) > filesMaxNameLen {
|
||||
return fmt.Errorf("file name exceeds %d characters", filesMaxNameLen)
|
||||
}
|
||||
if strings.HasPrefix(name, "/") || strings.HasPrefix(name, "\\") {
|
||||
return fmt.Errorf("absolute paths not allowed: %q", name)
|
||||
}
|
||||
if strings.Contains(name, "..") {
|
||||
return fmt.Errorf("path traversal not allowed: %q", name)
|
||||
}
|
||||
if strings.HasPrefix(name, filesMetaPrefix) {
|
||||
return fmt.Errorf("names starting with %q are reserved", filesMetaPrefix)
|
||||
}
|
||||
for _, r := range name {
|
||||
if unicode.IsControl(r) {
|
||||
return fmt.Errorf("control characters not allowed in file name")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// filesMaxSize returns the maximum file size from env or default.
|
||||
func filesMaxSize() int64 {
|
||||
if v := os.Getenv("EXT_FILES_MAX_SIZE"); v != "" {
|
||||
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return filesDefaultMaxSize
|
||||
}
|
||||
|
||||
// starlarkDictToMap converts a *starlark.Dict to map[string]any.
|
||||
func starlarkDictToMap(d *starlark.Dict) (map[string]any, error) {
|
||||
m := make(map[string]any, d.Len())
|
||||
for _, item := range d.Items() {
|
||||
k, ok := starlark.AsString(item[0])
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("metadata keys must be strings, got %s", item[0].Type())
|
||||
}
|
||||
m[k] = starlarkValueToGo(item[1])
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// starlarkValueToGo converts a starlark.Value to a Go value for JSON serialization.
|
||||
func starlarkValueToGo(v starlark.Value) any {
|
||||
switch x := v.(type) {
|
||||
case starlark.String:
|
||||
return string(x)
|
||||
case starlark.Int:
|
||||
if i, ok := x.Int64(); ok {
|
||||
return i
|
||||
}
|
||||
return x.String()
|
||||
case starlark.Float:
|
||||
return float64(x)
|
||||
case starlark.Bool:
|
||||
return bool(x)
|
||||
case *starlark.List:
|
||||
out := make([]any, x.Len())
|
||||
for i := 0; i < x.Len(); i++ {
|
||||
out[i] = starlarkValueToGo(x.Index(i))
|
||||
}
|
||||
return out
|
||||
case *starlark.Dict:
|
||||
m, _ := starlarkDictToMap(x)
|
||||
return m
|
||||
default:
|
||||
return v.String()
|
||||
}
|
||||
}
|
||||
|
||||
// mapToStarlarkDict converts a map[string]any to *starlark.Dict.
|
||||
func mapToStarlarkDict(m map[string]any) *starlark.Dict {
|
||||
d := starlark.NewDict(len(m))
|
||||
for k, v := range m {
|
||||
d.SetKey(starlark.String(k), goValueToStarlark(v))
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// goValueToStarlark converts a Go value (from JSON) to starlark.Value.
|
||||
func goValueToStarlark(v any) starlark.Value {
|
||||
switch x := v.(type) {
|
||||
case string:
|
||||
return starlark.String(x)
|
||||
case float64:
|
||||
if x == float64(int64(x)) {
|
||||
return starlark.MakeInt64(int64(x))
|
||||
}
|
||||
return starlark.Float(x)
|
||||
case bool:
|
||||
return starlark.Bool(x)
|
||||
case []any:
|
||||
elems := make([]starlark.Value, len(x))
|
||||
for i, e := range x {
|
||||
elems[i] = goValueToStarlark(e)
|
||||
}
|
||||
return starlark.NewList(elems)
|
||||
case map[string]any:
|
||||
return mapToStarlarkDict(x)
|
||||
default:
|
||||
return starlark.None
|
||||
}
|
||||
}
|
||||
410
server/sandbox/files_module_test.go
Normal file
410
server/sandbox/files_module_test.go
Normal file
@@ -0,0 +1,410 @@
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.starlark.net/starlark"
|
||||
|
||||
"armature/storage"
|
||||
)
|
||||
|
||||
// memStore is an in-memory ObjectStore for testing.
|
||||
type memStore struct {
|
||||
objects map[string]memObject
|
||||
}
|
||||
|
||||
type memObject struct {
|
||||
data []byte
|
||||
contentType string
|
||||
}
|
||||
|
||||
func newMemStore() *memStore {
|
||||
return &memStore{objects: make(map[string]memObject)}
|
||||
}
|
||||
|
||||
func (m *memStore) Put(_ context.Context, key string, r io.Reader, size int64, contentType string) error {
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.objects[key] = memObject{data: data, contentType: contentType}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memStore) Get(_ context.Context, key string) (io.ReadCloser, int64, string, error) {
|
||||
obj, ok := m.objects[key]
|
||||
if !ok {
|
||||
return nil, 0, "", storage.ErrNotFound
|
||||
}
|
||||
return io.NopCloser(&memReader{data: obj.data}), int64(len(obj.data)), obj.contentType, nil
|
||||
}
|
||||
|
||||
func (m *memStore) Delete(_ context.Context, key string) error {
|
||||
delete(m.objects, key)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memStore) DeletePrefix(_ context.Context, prefix string) error {
|
||||
for k := range m.objects {
|
||||
if strings.HasPrefix(k, prefix) {
|
||||
delete(m.objects, k)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memStore) Exists(_ context.Context, key string) (bool, error) {
|
||||
_, ok := m.objects[key]
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
func (m *memStore) Healthy(_ context.Context) error { return nil }
|
||||
|
||||
func (m *memStore) Stats(_ context.Context) (*storage.StorageStats, error) {
|
||||
return &storage.StorageStats{Backend: "mem", Configured: true, Healthy: true}, nil
|
||||
}
|
||||
|
||||
func (m *memStore) Backend() string { return "mem" }
|
||||
|
||||
func (m *memStore) List(_ context.Context, prefix string, limit int) ([]storage.ObjectEntry, error) {
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
var entries []storage.ObjectEntry
|
||||
for k, obj := range m.objects {
|
||||
if strings.HasPrefix(k, prefix) {
|
||||
entries = append(entries, storage.ObjectEntry{
|
||||
Key: k,
|
||||
Size: int64(len(obj.data)),
|
||||
ContentType: obj.contentType,
|
||||
})
|
||||
if len(entries) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// memReader wraps a byte slice as io.ReadCloser.
|
||||
type memReader struct {
|
||||
data []byte
|
||||
pos int
|
||||
}
|
||||
|
||||
func (r *memReader) Read(p []byte) (int, error) {
|
||||
if r.pos >= len(r.data) {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(p, r.data[r.pos:])
|
||||
r.pos += n
|
||||
if r.pos >= len(r.data) {
|
||||
return n, io.EOF
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (r *memReader) Close() error { return nil }
|
||||
|
||||
// ── Test helpers ──────────────────────────────
|
||||
|
||||
func execWithFiles(t *testing.T, canWrite bool, script string) (*Result, error) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
store := newMemStore()
|
||||
mod := BuildFilesModule(ctx, FilesModuleConfig{
|
||||
PackageID: "test-pkg",
|
||||
CanWrite: canWrite,
|
||||
Store: store,
|
||||
})
|
||||
sb := New(DefaultConfig())
|
||||
return sb.Exec(ctx, "test.star", script, map[string]starlark.Value{
|
||||
"files": mod,
|
||||
})
|
||||
}
|
||||
|
||||
func execWithFilesStore(t *testing.T, store *memStore, canWrite bool, script string) (*Result, error) {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
mod := BuildFilesModule(ctx, FilesModuleConfig{
|
||||
PackageID: "test-pkg",
|
||||
CanWrite: canWrite,
|
||||
Store: store,
|
||||
})
|
||||
sb := New(DefaultConfig())
|
||||
return sb.Exec(ctx, "test.star", script, map[string]starlark.Value{
|
||||
"files": mod,
|
||||
})
|
||||
}
|
||||
|
||||
// ── Put + Get round-trip ──────────────────────
|
||||
|
||||
func TestFilesPut_StringContent(t *testing.T) {
|
||||
_, err := execWithFiles(t, true, `
|
||||
files.put("hello.txt", "hello world", content_type="text/plain")
|
||||
result = files.get("hello.txt")
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("exec error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesPut_BytesContent(t *testing.T) {
|
||||
result, err := execWithFiles(t, true, `
|
||||
files.put("bin.dat", b"\x00\x01\x02\xff")
|
||||
got = files.get("bin.dat")
|
||||
size = got["size"]
|
||||
ct = got["content_type"]
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("exec error: %v", err)
|
||||
}
|
||||
|
||||
size, _ := starlark.AsInt32(result.Globals["size"])
|
||||
if size != 4 {
|
||||
t.Errorf("size = %d, want 4", size)
|
||||
}
|
||||
ct := string(result.Globals["ct"].(starlark.String))
|
||||
if ct != "application/octet-stream" {
|
||||
t.Errorf("content_type = %q, want application/octet-stream", ct)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesPut_WithMetadata(t *testing.T) {
|
||||
result, err := execWithFiles(t, true, `
|
||||
files.put("doc.pdf", "pdf content", content_type="application/pdf", metadata={"author": "test", "pages": 5})
|
||||
meta = files.meta("doc.pdf")
|
||||
author = meta["author"]
|
||||
pages = meta["pages"]
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("exec error: %v", err)
|
||||
}
|
||||
|
||||
author := string(result.Globals["author"].(starlark.String))
|
||||
if author != "test" {
|
||||
t.Errorf("author = %q, want %q", author, "test")
|
||||
}
|
||||
pages, _ := starlark.AsInt32(result.Globals["pages"])
|
||||
if pages != 5 {
|
||||
t.Errorf("pages = %d, want 5", pages)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesPut_SizeLimit(t *testing.T) {
|
||||
// Set a small limit for testing.
|
||||
t.Setenv("EXT_FILES_MAX_SIZE", "10")
|
||||
_, err := execWithFiles(t, true, `
|
||||
files.put("big.bin", "this content is longer than ten bytes")
|
||||
`)
|
||||
if err == nil {
|
||||
t.Fatal("expected size limit error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "exceeds limit") {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Get ───────────────────────────────────────
|
||||
|
||||
func TestFilesGet_NotFound(t *testing.T) {
|
||||
result, err := execWithFiles(t, false, `
|
||||
got = files.get("nonexistent.txt")
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("exec error: %v", err)
|
||||
}
|
||||
if result.Globals["got"] != starlark.None {
|
||||
t.Errorf("expected None, got %v", result.Globals["got"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesGet_ReturnsAllFields(t *testing.T) {
|
||||
result, err := execWithFiles(t, true, `
|
||||
files.put("test.txt", "data", content_type="text/plain", metadata={"k": "v"})
|
||||
got = files.get("test.txt")
|
||||
has_content = "content" in got
|
||||
has_ct = "content_type" in got
|
||||
has_size = "size" in got
|
||||
has_meta = "metadata" in got
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("exec error: %v", err)
|
||||
}
|
||||
for _, key := range []string{"has_content", "has_ct", "has_size", "has_meta"} {
|
||||
if result.Globals[key] != starlark.True {
|
||||
t.Errorf("%s = False, want True", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Meta ──────────────────────────────────────
|
||||
|
||||
func TestFilesMeta_NotFound(t *testing.T) {
|
||||
result, err := execWithFiles(t, false, `
|
||||
got = files.meta("nonexistent.txt")
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("exec error: %v", err)
|
||||
}
|
||||
if result.Globals["got"] != starlark.None {
|
||||
t.Errorf("expected None, got %v", result.Globals["got"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesMeta_NoCompanion(t *testing.T) {
|
||||
result, err := execWithFiles(t, true, `
|
||||
files.put("plain.txt", "data")
|
||||
meta = files.meta("plain.txt")
|
||||
count = len(meta)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("exec error: %v", err)
|
||||
}
|
||||
count, _ := starlark.AsInt32(result.Globals["count"])
|
||||
if count != 0 {
|
||||
t.Errorf("meta length = %d, want 0", count)
|
||||
}
|
||||
}
|
||||
|
||||
// ── List ──────────────────────────────────────
|
||||
|
||||
func TestFilesList_PrefixFilter(t *testing.T) {
|
||||
result, err := execWithFiles(t, true, `
|
||||
files.put("images/a.png", "a")
|
||||
files.put("images/b.png", "b")
|
||||
files.put("docs/c.pdf", "c")
|
||||
all_entries = files.list()
|
||||
img_entries = files.list(prefix="images/")
|
||||
all_count = len(all_entries)
|
||||
img_count = len(img_entries)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("exec error: %v", err)
|
||||
}
|
||||
|
||||
allCount, _ := starlark.AsInt32(result.Globals["all_count"])
|
||||
imgCount, _ := starlark.AsInt32(result.Globals["img_count"])
|
||||
|
||||
if allCount != 3 {
|
||||
t.Errorf("all_count = %d, want 3", allCount)
|
||||
}
|
||||
if imgCount != 2 {
|
||||
t.Errorf("img_count = %d, want 2", imgCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesList_ExcludesMeta(t *testing.T) {
|
||||
result, err := execWithFiles(t, true, `
|
||||
files.put("file.txt", "data", metadata={"key": "val"})
|
||||
entries = files.list()
|
||||
count = len(entries)
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("exec error: %v", err)
|
||||
}
|
||||
|
||||
count, _ := starlark.AsInt32(result.Globals["count"])
|
||||
if count != 1 {
|
||||
t.Errorf("count = %d, want 1 (should exclude _meta/ companion)", count)
|
||||
}
|
||||
}
|
||||
|
||||
// ── Delete ────────────────────────────────────
|
||||
|
||||
func TestFilesDelete_Idempotent(t *testing.T) {
|
||||
_, err := execWithFiles(t, true, `
|
||||
files.put("temp.txt", "temp")
|
||||
files.delete("temp.txt")
|
||||
files.delete("temp.txt")
|
||||
exists = files.exists("temp.txt")
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("exec error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesDeletePrefix(t *testing.T) {
|
||||
result, err := execWithFiles(t, true, `
|
||||
files.put("uploads/a.txt", "a")
|
||||
files.put("uploads/b.txt", "b")
|
||||
files.put("keep.txt", "keep")
|
||||
files.delete_prefix("uploads/")
|
||||
a_exists = files.exists("uploads/a.txt")
|
||||
b_exists = files.exists("uploads/b.txt")
|
||||
keep_exists = files.exists("keep.txt")
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("exec error: %v", err)
|
||||
}
|
||||
|
||||
if result.Globals["a_exists"] != starlark.False {
|
||||
t.Error("uploads/a.txt should be deleted")
|
||||
}
|
||||
if result.Globals["b_exists"] != starlark.False {
|
||||
t.Error("uploads/b.txt should be deleted")
|
||||
}
|
||||
if result.Globals["keep_exists"] != starlark.True {
|
||||
t.Error("keep.txt should be preserved")
|
||||
}
|
||||
}
|
||||
|
||||
// ── Exists ────────────────────────────────────
|
||||
|
||||
func TestFilesExists(t *testing.T) {
|
||||
result, err := execWithFiles(t, true, `
|
||||
files.put("exists.txt", "yes")
|
||||
yes = files.exists("exists.txt")
|
||||
no = files.exists("nope.txt")
|
||||
`)
|
||||
if err != nil {
|
||||
t.Fatalf("exec error: %v", err)
|
||||
}
|
||||
|
||||
if result.Globals["yes"] != starlark.True {
|
||||
t.Error("exists should be True")
|
||||
}
|
||||
if result.Globals["no"] != starlark.False {
|
||||
t.Error("nope should be False")
|
||||
}
|
||||
}
|
||||
|
||||
// ── Name Validation ──────────────────────────
|
||||
|
||||
func TestFilesNameValidation(t *testing.T) {
|
||||
badNames := []string{
|
||||
"../escape",
|
||||
"/absolute",
|
||||
"_meta/reserved",
|
||||
"",
|
||||
}
|
||||
|
||||
for _, name := range badNames {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
script := fmt.Sprintf(`files.exists(%q)`, name)
|
||||
_, err := execWithFiles(t, false, script)
|
||||
if err == nil {
|
||||
t.Errorf("expected error for name %q", name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── Write Permission Gate ────────────────────
|
||||
|
||||
func TestFilesWritePermission(t *testing.T) {
|
||||
// Read-only mode: put/delete/delete_prefix should not be available.
|
||||
_, err := execWithFiles(t, false, `
|
||||
files.put("test.txt", "data")
|
||||
`)
|
||||
if err == nil {
|
||||
t.Fatal("expected error: put should not be available in read-only mode")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "no .put field") && !strings.Contains(err.Error(), "has no .put") {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -33,6 +33,7 @@ import (
|
||||
"armature/events"
|
||||
"armature/metrics"
|
||||
"armature/models"
|
||||
"armature/storage"
|
||||
"armature/store"
|
||||
)
|
||||
|
||||
@@ -77,6 +78,7 @@ type Runner struct {
|
||||
dbPostgres bool // true = use $N placeholders; false = use ?
|
||||
allowPrivateIPs bool
|
||||
bus *events.Bus
|
||||
objectStore storage.ObjectStore // nil = files module unavailable
|
||||
}
|
||||
|
||||
// NewRunner creates a runner with the given sandbox and dependencies.
|
||||
@@ -116,6 +118,11 @@ func (r *Runner) SetBus(bus *events.Bus) {
|
||||
r.bus = bus
|
||||
}
|
||||
|
||||
// SetObjectStore attaches the blob storage backend for the files module.
|
||||
func (r *Runner) SetObjectStore(s storage.ObjectStore) {
|
||||
r.objectStore = s
|
||||
}
|
||||
|
||||
// SetAllowPrivateIPs disables the SSRF check that blocks connections to
|
||||
// private/loopback IPs. For self-hosted environments where extensions
|
||||
// reach internal services. Controlled by EXT_ALLOW_PRIVATE_IPS env var.
|
||||
@@ -336,8 +343,9 @@ func (r *Runner) buildModulesWithLibCtx(ctx context.Context, packageID string, m
|
||||
|
||||
modules := make(map[string]starlark.Value)
|
||||
|
||||
// Track db permission level: 0=none, 1=read, 2=write
|
||||
// Track tiered permission levels: 0=none, 1=read, 2=write
|
||||
dbLevel := 0
|
||||
filesLevel := 0
|
||||
hasBatchExec := false
|
||||
|
||||
for _, perm := range granted {
|
||||
@@ -363,6 +371,14 @@ func (r *Runner) buildModulesWithLibCtx(ctx context.Context, packageID string, m
|
||||
case models.ExtPermDBWrite:
|
||||
dbLevel = 2
|
||||
|
||||
case models.ExtPermFilesRead:
|
||||
if filesLevel < 1 {
|
||||
filesLevel = 1
|
||||
}
|
||||
|
||||
case models.ExtPermFilesWrite:
|
||||
filesLevel = 2
|
||||
|
||||
case models.ExtPermWorkflowAccess:
|
||||
modules["workflow"] = BuildWorkflowModule(ctx, r.stores)
|
||||
|
||||
@@ -391,6 +407,15 @@ func (r *Runner) buildModulesWithLibCtx(ctx context.Context, packageID string, m
|
||||
})
|
||||
}
|
||||
|
||||
// Wire files module at the highest granted level.
|
||||
if filesLevel > 0 && r.objectStore != nil {
|
||||
modules["files"] = BuildFilesModule(ctx, FilesModuleConfig{
|
||||
PackageID: packageID,
|
||||
CanWrite: filesLevel == 2,
|
||||
Store: r.objectStore,
|
||||
})
|
||||
}
|
||||
|
||||
// A package reads its own admin + team + user settings.
|
||||
userID := ""
|
||||
teamID := ""
|
||||
|
||||
Reference in New Issue
Block a user