switchboard/core
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Releases 9 Wiki Activity

Changeset 0.12.0 (#63)

Browse Source
This commit is contained in:
xcaliber
2026-02-25 21:38:49 +00:00
parent c9d8e9457e
commit 88216ec4cb
59 changed files with 13115 additions and 139 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
server/.env.example
View File

@@ -1,4 +1,4 @@
# Chat Switchboard v0.9 - Server Environment Variables
# Chat Switchboard v0.12 - Server Environment Variables
# Copy this file to .env and fill in the values
# ── Server ───────────────────────────────────
@@ -44,3 +44,22 @@ CORS_ALLOWED_ORIGINS=http://localhost:3000,http://localhost:8080
# ── Logging ──────────────────────────────────
LOG_LEVEL=info
# ── File Storage ─────────────────────────────
# Backend: "pvc" (local filesystem), "s3" (S3-compatible), or empty (auto-detect)
STORAGE_BACKEND=
STORAGE_PATH=/data/storage
# S3 settings (only when STORAGE_BACKEND=s3)
# Works with MinIO, Ceph RGW, AWS S3.
# S3_ENDPOINT=http://minio:9000
# S3_BUCKET=switchboard-storage
# S3_ACCESS_KEY=
# S3_SECRET_KEY=
# S3_REGION=us-east-1
# S3_PREFIX= # optional key prefix for shared buckets
# S3_FORCE_PATH_STYLE=true # required for MinIO, Ceph RGW
# ── Extraction ───────────────────────────────
# EXTRACTION_MODE=inline # "inline" or "sidecar"
# EXTRACTION_CONCURRENCY=3

44
server/config/config.go
View File

@@ -2,6 +2,7 @@ package config
import (
"os"
"strconv"
"github.com/joho/godotenv"
)
@@ -26,6 +27,28 @@ type Config struct {
// Used to derive AES-256 key for global/team provider API keys.
// Personal keys use per-user UEK (derived from password).
EncryptionKey string
// File storage (v0.12.0+)
// STORAGE_BACKEND: "pvc" or "s3". Empty = auto-detect (pvc if path writable).
// STORAGE_PATH: mount point for PVC backend (default /data/storage).
StorageBackend string
StoragePath string
// S3-compatible storage (v0.12.0+)
// Works with AWS S3, MinIO, Ceph RGW, or any S3-compatible API.
S3Endpoint string // custom endpoint URL (required for MinIO/Ceph, optional for AWS)
S3Bucket string // bucket name (required when STORAGE_BACKEND=s3)
S3Region string // AWS region (default "us-east-1")
S3AccessKey string // access key ID
S3SecretKey string // secret access key
S3Prefix string // optional key prefix within bucket (e.g. "switchboard/")
S3ForcePathStyle bool // use path-style URLs (required for MinIO, most self-hosted)
// Extraction pipeline (v0.12.0+)
// EXTRACTION_MODE: "inline" (in-process) or "sidecar" (shared PVC watcher).
// EXTRACTION_CONCURRENCY: max concurrent extraction jobs (default 3).
ExtractionMode string
ExtractionConcurrency int
}
// Load reads configuration from environment variables.
@@ -45,6 +68,18 @@ func Load() *Config {
AdminEmail: getEnv("SWITCHBOARD_ADMIN_EMAIL", ""),
SeedUsers: getEnv("SEED_USERS", ""),
EncryptionKey: getEnv("ENCRYPTION_KEY", ""),
StorageBackend: getEnv("STORAGE_BACKEND", ""),
StoragePath: getEnv("STORAGE_PATH", "/data/storage"),
S3Endpoint: getEnv("S3_ENDPOINT", ""),
S3Bucket: getEnv("S3_BUCKET", ""),
S3Region: getEnv("S3_REGION", "us-east-1"),
S3AccessKey: getEnv("S3_ACCESS_KEY", ""),
S3SecretKey: getEnv("S3_SECRET_KEY", ""),
S3Prefix: getEnv("S3_PREFIX", ""),
S3ForcePathStyle: getEnv("S3_FORCE_PATH_STYLE", "true") == "true",
ExtractionMode: getEnv("EXTRACTION_MODE", "inline"),
ExtractionConcurrency: getEnvInt("EXTRACTION_CONCURRENCY", 3),
}
}
@@ -71,3 +106,12 @@ func getEnv(key, fallback string) string {
}
return fallback
}
func getEnvInt(key string, fallback int) int {
if v := os.Getenv(key); v != "" {
if n, err := strconv.Atoi(v); err == nil {
return n
}
}
return fallback
}

115
server/crypto/rekey.go Normal file
View File

@@ -0,0 +1,115 @@
package crypto
import (
"database/sql"
"fmt"
"log"
)
// Rekey re-encrypts all global/team API keys from oldKey to newKey.
// Personal BYOK keys are unaffected (they're keyed to the user's UEK,
// not the environment-derived key).
//
// Runs in a single transaction — all-or-nothing. On failure, no keys
// are modified.
//
// Usage:
//
// switchboard vault rekey
// ENCRYPTION_KEY = current/old key
// NEW_ENCRYPTION_KEY = new key to re-encrypt with
func Rekey(db *sql.DB, oldEnvKey, newEnvKey string) error {
if oldEnvKey == "" {
return fmt.Errorf("ENCRYPTION_KEY (current key) is required")
}
if newEnvKey == "" {
return fmt.Errorf("NEW_ENCRYPTION_KEY is required")
}
if oldEnvKey == newEnvKey {
return fmt.Errorf("ENCRYPTION_KEY and NEW_ENCRYPTION_KEY are identical — nothing to do")
}
oldKey, err := DeriveKeyFromEnv(oldEnvKey)
if err != nil {
return fmt.Errorf("failed to derive old key: %w", err)
}
newKey, err := DeriveKeyFromEnv(newEnvKey)
if err != nil {
return fmt.Errorf("failed to derive new key: %w", err)
}
tx, err := db.Begin()
if err != nil {
return fmt.Errorf("failed to begin transaction: %w", err)
}
defer tx.Rollback() // no-op after commit
// Find all global/team keys that are encrypted
rows, err := tx.Query(`
SELECT id, api_key_enc, key_nonce
FROM provider_configs
WHERE key_scope IN ('global', 'team')
AND api_key_enc IS NOT NULL
`)
if err != nil {
return fmt.Errorf("failed to query encrypted keys: %w", err)
}
type rekeyRow struct {
id string
ciphertext []byte
nonce []byte
}
var toRekey []rekeyRow
for rows.Next() {
var r rekeyRow
if err := rows.Scan(&r.id, &r.ciphertext, &r.nonce); err != nil {
rows.Close()
return fmt.Errorf("failed to scan row: %w", err)
}
toRekey = append(toRekey, r)
}
rows.Close()
if err := rows.Err(); err != nil {
return fmt.Errorf("row iteration error: %w", err)
}
if len(toRekey) == 0 {
log.Println("No global/team encrypted keys found — nothing to rekey.")
return nil
}
log.Printf("Rekeying %d provider config(s)...", len(toRekey))
// Decrypt with old key, re-encrypt with new key, update in place
for _, r := range toRekey {
plaintext, err := Decrypt(r.ciphertext, r.nonce, oldKey)
if err != nil {
return fmt.Errorf("failed to decrypt config %s with old key: %w (is ENCRYPTION_KEY correct?)", r.id, err)
}
newCiphertext, newNonce, err := Encrypt(plaintext, newKey)
if err != nil {
return fmt.Errorf("failed to re-encrypt config %s: %w", r.id, err)
}
_, err = tx.Exec(`
UPDATE provider_configs
SET api_key_enc = $1, key_nonce = $2
WHERE id = $3
`, newCiphertext, newNonce, r.id)
if err != nil {
return fmt.Errorf("failed to update config %s: %w", r.id, err)
}
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("failed to commit rekey transaction: %w", err)
}
log.Printf("✅ Successfully rekeyed %d provider config(s).", len(toRekey))
log.Println("Update ENCRYPTION_KEY to the new value and restart the server.")
return nil
}

46
server/crypto/status.go Normal file
View File

@@ -0,0 +1,46 @@
package crypto
import (
"database/sql"
"fmt"
)
// VaultStatusInfo holds vault health information for admin display.
type VaultStatusInfo struct {
EncryptionKeySet bool `json:"encryption_key_set"`
EncryptedKeys int `json:"encrypted_keys"` // provider_configs with api_key_enc
VaultUsers int `json:"vault_users"` // users with vault_set = true
}
// VaultStatus gathers vault health metrics from the database.
// Used by both the CLI (`switchboard vault status`) and the admin API endpoint.
func VaultStatus(db *sql.DB, encryptionKey string) (*VaultStatusInfo, error) {
if db == nil {
return nil, fmt.Errorf("database not available")
}
info := &VaultStatusInfo{
EncryptionKeySet: encryptionKey != "",
}
// Count encrypted provider keys (global + team + personal)
err := db.QueryRow(`
SELECT COUNT(*) FROM provider_configs
WHERE api_key_enc IS NOT NULL
`).Scan(&info.EncryptedKeys)
if err != nil {
return nil, fmt.Errorf("count encrypted keys: %w", err)
}
// Count users with active vaults
err = db.QueryRow(`
SELECT COUNT(*) FROM users
WHERE vault_set = true
`).Scan(&info.VaultUsers)
if err != nil {
// vault_set column might not exist on very old installs
info.VaultUsers = 0
}
return info, nil
}

32
server/database/migrations/007_attachments.sql Normal file
View File

@@ -0,0 +1,32 @@
-- 007_attachments.sql
-- File attachments for chat messages (v0.12.0)
--
-- Blobs live in object storage (PVC / S3). This table holds metadata.
-- Access control: always join through channels — attachments inherit
-- channel membership as their access boundary.
CREATE TABLE IF NOT EXISTS attachments (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
channel_id UUID NOT NULL REFERENCES channels(id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users(id),
message_id UUID REFERENCES messages(id) ON DELETE SET NULL,
filename VARCHAR(255) NOT NULL,
content_type VARCHAR(127) NOT NULL,
size_bytes BIGINT NOT NULL,
storage_key TEXT NOT NULL,
extracted_text TEXT,
metadata JSONB DEFAULT '{}'::jsonb,
created_at TIMESTAMPTZ DEFAULT NOW()
);
-- Primary access pattern: find attachments for a channel (auth check joins here)
CREATE INDEX IF NOT EXISTS idx_attachments_channel ON attachments(channel_id);
-- Quota calculation: SUM(size_bytes) WHERE user_id = $1
CREATE INDEX IF NOT EXISTS idx_attachments_user_size ON attachments(user_id);
-- Find attachments for a specific message
CREATE INDEX IF NOT EXISTS idx_attachments_message ON attachments(message_id) WHERE message_id IS NOT NULL;
-- Orphan cleanup: find unlinked attachments older than threshold
CREATE INDEX IF NOT EXISTS idx_attachments_orphan ON attachments(created_at) WHERE message_id IS NULL;

294
server/extraction/queue.go Normal file
View File

@@ -0,0 +1,294 @@
package extraction
import (
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"sync"
"time"
)
// ── Status Constants ───────────────────────
const (
StatusPending = "pending"
StatusProcessing = "processing"
StatusComplete = "complete"
StatusFailed = "failed"
StatusSkipped = "skipped" // not extractable (images, etc.)
)
// ── Queue Item ─────────────────────────────
// QueueItem is the status.json written to the processing directory.
// The sidecar extractor watches this directory for pending items.
type QueueItem struct {
AttachmentID string `json:"attachment_id"`
StorageKey string `json:"storage_key"`
ContentType string `json:"content_type"`
Filename string `json:"filename"`
Status string `json:"status"`
Error string `json:"error,omitempty"`
OutputKey string `json:"output_key,omitempty"` // path to extracted text
QueuedAt string `json:"queued_at"`
StartedAt string `json:"started_at,omitempty"`
CompletedAt string `json:"completed_at,omitempty"`
}
// ── Queue Manager ──────────────────────────
// Queue manages the filesystem-based extraction queue.
// Processing state lives in {storagePath}/processing/{attachment_id}/status.json.
// The sidecar extractor watches for pending items and updates status.
type Queue struct {
storagePath string
concurrency int
sem chan struct{} // semaphore for concurrent extraction limit
mu sync.Mutex
}
// NewQueue creates a new extraction queue manager.
// storagePath is the PVC mount (e.g., /data/storage).
// concurrency limits parallel extractions (default 3).
func NewQueue(storagePath string, concurrency int) (*Queue, error) {
if concurrency <= 0 {
concurrency = 3
}
procDir := filepath.Join(storagePath, "processing")
if err := os.MkdirAll(procDir, 0750); err != nil {
return nil, fmt.Errorf("extraction: create processing dir: %w", err)
}
return &Queue{
storagePath: storagePath,
concurrency: concurrency,
sem: make(chan struct{}, concurrency),
}, nil
}
// Enqueue adds an attachment to the extraction queue.
// Creates {storagePath}/processing/{id}/status.json with status "pending".
func (q *Queue) Enqueue(attachmentID, storageKey, contentType, filename string) error {
q.mu.Lock()
defer q.mu.Unlock()
itemDir := filepath.Join(q.storagePath, "processing", attachmentID)
if err := os.MkdirAll(itemDir, 0750); err != nil {
return fmt.Errorf("extraction: create item dir: %w", err)
}
item := QueueItem{
AttachmentID: attachmentID,
StorageKey: storageKey,
ContentType: contentType,
Filename: filename,
Status: StatusPending,
QueuedAt: time.Now().UTC().Format(time.RFC3339),
}
return q.writeStatus(attachmentID, &item)
}
// GetStatus reads the current extraction status for an attachment.
func (q *Queue) GetStatus(attachmentID string) (*QueueItem, error) {
statusPath := filepath.Join(q.storagePath, "processing", attachmentID, "status.json")
data, err := os.ReadFile(statusPath)
if err != nil {
if os.IsNotExist(err) {
return nil, nil // not queued
}
return nil, fmt.Errorf("extraction: read status: %w", err)
}
var item QueueItem
if err := json.Unmarshal(data, &item); err != nil {
return nil, fmt.Errorf("extraction: parse status: %w", err)
}
return &item, nil
}
// MarkComplete updates status to complete and records the output key.
func (q *Queue) MarkComplete(attachmentID, outputKey string) error {
q.mu.Lock()
defer q.mu.Unlock()
item, err := q.GetStatus(attachmentID)
if err != nil || item == nil {
return fmt.Errorf("extraction: item not found: %s", attachmentID)
}
item.Status = StatusComplete
item.OutputKey = outputKey
item.CompletedAt = time.Now().UTC().Format(time.RFC3339)
return q.writeStatus(attachmentID, item)
}
// MarkFailed updates status to failed with an error message.
func (q *Queue) MarkFailed(attachmentID, errMsg string) error {
q.mu.Lock()
defer q.mu.Unlock()
item, err := q.GetStatus(attachmentID)
if err != nil || item == nil {
return fmt.Errorf("extraction: item not found: %s", attachmentID)
}
item.Status = StatusFailed
item.Error = errMsg
item.CompletedAt = time.Now().UTC().Format(time.RFC3339)
return q.writeStatus(attachmentID, item)
}
// Cleanup removes the processing directory for a completed/failed item.
func (q *Queue) Cleanup(attachmentID string) error {
itemDir := filepath.Join(q.storagePath, "processing", attachmentID)
return os.RemoveAll(itemDir)
}
// RecoverStale scans for items stuck in "processing" state (crash recovery).
// Items older than maxAge are reset to "pending" for re-processing.
func (q *Queue) RecoverStale(maxAge time.Duration) (int, error) {
procDir := filepath.Join(q.storagePath, "processing")
entries, err := os.ReadDir(procDir)
if err != nil {
if os.IsNotExist(err) {
return 0, nil
}
return 0, err
}
recovered := 0
cutoff := time.Now().Add(-maxAge)
for _, entry := range entries {
if !entry.IsDir() {
continue
}
item, err := q.GetStatus(entry.Name())
if err != nil || item == nil {
continue
}
if item.Status != StatusProcessing {
continue
}
// Check if started_at is older than maxAge
if item.StartedAt != "" {
startedAt, err := time.Parse(time.RFC3339, item.StartedAt)
if err == nil && startedAt.Before(cutoff) {
q.mu.Lock()
item.Status = StatusPending
item.StartedAt = ""
item.Error = "recovered from stale processing state"
q.writeStatus(entry.Name(), item)
q.mu.Unlock()
recovered++
log.Printf("extraction: recovered stale item %s", entry.Name())
}
}
}
return recovered, nil
}
// ListPending returns all items with status "pending".
func (q *Queue) ListPending() ([]QueueItem, error) {
return q.listByStatus(StatusPending)
}
// ListAll returns all items in the processing directory.
func (q *Queue) ListAll() ([]QueueItem, error) {
procDir := filepath.Join(q.storagePath, "processing")
entries, err := os.ReadDir(procDir)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
var items []QueueItem
for _, entry := range entries {
if !entry.IsDir() {
continue
}
item, err := q.GetStatus(entry.Name())
if err != nil || item == nil {
continue
}
items = append(items, *item)
}
return items, nil
}
// ── Internal Helpers ───────────────────────
func (q *Queue) writeStatus(attachmentID string, item *QueueItem) error {
statusPath := filepath.Join(q.storagePath, "processing", attachmentID, "status.json")
data, err := json.MarshalIndent(item, "", " ")
if err != nil {
return err
}
// Atomic write: temp + rename
tmpPath := statusPath + ".tmp"
if err := os.WriteFile(tmpPath, data, 0640); err != nil {
return err
}
return os.Rename(tmpPath, statusPath)
}
func (q *Queue) listByStatus(status string) ([]QueueItem, error) {
all, err := q.ListAll()
if err != nil {
return nil, err
}
var filtered []QueueItem
for _, item := range all {
if item.Status == status {
filtered = append(filtered, item)
}
}
return filtered, nil
}
// ── Extractable Check ──────────────────────
// IsExtractable returns true if the content type supports text extraction.
// Images and plain text do not need extraction (handled inline by upload handler).
var extractableTypes = map[string]bool{
"application/pdf": true,
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": true,
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true,
"application/vnd.openxmlformats-officedocument.presentationml.presentation": true,
"application/msword": true,
"application/vnd.ms-excel": true,
"application/vnd.oasis.opendocument.text": true,
"application/vnd.oasis.opendocument.spreadsheet": true,
"application/vnd.oasis.opendocument.presentation": true,
"application/rtf": true,
}
// IsExtractable returns true if the given content type requires sidecar extraction.
func IsExtractable(contentType string) bool {
return extractableTypes[contentType]
}
// ── Null Queue ─────────────────────────────
// NullQueue is a no-op implementation for when extraction is disabled.
type NullQueue struct{}
func (NullQueue) Enqueue(_, _, _, _ string) error { return nil }
func (NullQueue) GetStatus(_ string) (*QueueItem, error) { return nil, nil }
func (NullQueue) MarkComplete(_, _ string) error { return nil }
func (NullQueue) MarkFailed(_, _ string) error { return nil }
func (NullQueue) Cleanup(_ string) error { return nil }
func (NullQueue) RecoverStale(_ time.Duration) (int, error) { return 0, nil }
func (NullQueue) ListPending() ([]QueueItem, error) { return nil, nil }
func (NullQueue) ListAll() ([]QueueItem, error) { return nil, nil }

240
server/extraction/queue_test.go Normal file
View File

@@ -0,0 +1,240 @@
package extraction
import (
"os"
"path/filepath"
"testing"
"time"
)
func tempQueue(t *testing.T) *Queue {
t.Helper()
dir := t.TempDir()
q, err := NewQueue(dir, 3)
if err != nil {
t.Fatalf("NewQueue: %v", err)
}
return q
}
func TestEnqueue_CreatesStatusFile(t *testing.T) {
q := tempQueue(t)
err := q.Enqueue("att-123", "attachments/ch/att-123_doc.pdf", "application/pdf", "doc.pdf")
if err != nil {
t.Fatalf("Enqueue: %v", err)
}
item, err := q.GetStatus("att-123")
if err != nil {
t.Fatalf("GetStatus: %v", err)
}
if item == nil {
t.Fatal("expected non-nil item")
}
if item.Status != StatusPending {
t.Errorf("status = %q, want %q", item.Status, StatusPending)
}
if item.AttachmentID != "att-123" {
t.Errorf("attachment_id = %q, want att-123", item.AttachmentID)
}
if item.ContentType != "application/pdf" {
t.Errorf("content_type = %q, want application/pdf", item.ContentType)
}
}
func TestGetStatus_NotQueued(t *testing.T) {
q := tempQueue(t)
item, err := q.GetStatus("nonexistent")
if err != nil {
t.Fatalf("GetStatus: %v", err)
}
if item != nil {
t.Error("expected nil for non-queued item")
}
}
func TestMarkComplete(t *testing.T) {
q := tempQueue(t)
q.Enqueue("att-456", "key", "application/pdf", "test.pdf")
err := q.MarkComplete("att-456", "processing/att-456/extracted.txt")
if err != nil {
t.Fatalf("MarkComplete: %v", err)
}
item, _ := q.GetStatus("att-456")
if item.Status != StatusComplete {
t.Errorf("status = %q, want %q", item.Status, StatusComplete)
}
if item.OutputKey != "processing/att-456/extracted.txt" {
t.Errorf("output_key = %q", item.OutputKey)
}
if item.CompletedAt == "" {
t.Error("completed_at should be set")
}
}
func TestMarkFailed(t *testing.T) {
q := tempQueue(t)
q.Enqueue("att-789", "key", "application/pdf", "test.pdf")
err := q.MarkFailed("att-789", "libreoffice crashed")
if err != nil {
t.Fatalf("MarkFailed: %v", err)
}
item, _ := q.GetStatus("att-789")
if item.Status != StatusFailed {
t.Errorf("status = %q, want %q", item.Status, StatusFailed)
}
if item.Error != "libreoffice crashed" {
t.Errorf("error = %q", item.Error)
}
}
func TestCleanup_RemovesDirectory(t *testing.T) {
q := tempQueue(t)
q.Enqueue("att-cleanup", "key", "application/pdf", "test.pdf")
// Verify directory exists
dir := filepath.Join(q.storagePath, "processing", "att-cleanup")
if _, err := os.Stat(dir); err != nil {
t.Fatal("expected processing dir to exist before cleanup")
}
if err := q.Cleanup("att-cleanup"); err != nil {
t.Fatalf("Cleanup: %v", err)
}
if _, err := os.Stat(dir); !os.IsNotExist(err) {
t.Error("expected processing dir to be removed after cleanup")
}
}
func TestListPending(t *testing.T) {
q := tempQueue(t)
q.Enqueue("att-a", "key-a", "application/pdf", "a.pdf")
q.Enqueue("att-b", "key-b", "application/pdf", "b.pdf")
q.MarkComplete("att-b", "out")
pending, err := q.ListPending()
if err != nil {
t.Fatalf("ListPending: %v", err)
}
if len(pending) != 1 {
t.Fatalf("expected 1 pending, got %d", len(pending))
}
if pending[0].AttachmentID != "att-a" {
t.Errorf("pending[0].AttachmentID = %q, want att-a", pending[0].AttachmentID)
}
}
func TestListAll(t *testing.T) {
q := tempQueue(t)
q.Enqueue("att-1", "key-1", "application/pdf", "1.pdf")
q.Enqueue("att-2", "key-2", "application/pdf", "2.pdf")
q.Enqueue("att-3", "key-3", "application/pdf", "3.pdf")
all, err := q.ListAll()
if err != nil {
t.Fatalf("ListAll: %v", err)
}
if len(all) != 3 {
t.Errorf("expected 3 items, got %d", len(all))
}
}
func TestRecoverStale(t *testing.T) {
q := tempQueue(t)
// Create item and manually set it to "processing" with old timestamp
q.Enqueue("att-stale", "key", "application/pdf", "stale.pdf")
item, _ := q.GetStatus("att-stale")
item.Status = StatusProcessing
item.StartedAt = time.Now().Add(-2 * time.Hour).UTC().Format(time.RFC3339)
q.mu.Lock()
q.writeStatus("att-stale", item)
q.mu.Unlock()
// Recover items stale for > 1 hour
recovered, err := q.RecoverStale(1 * time.Hour)
if err != nil {
t.Fatalf("RecoverStale: %v", err)
}
if recovered != 1 {
t.Errorf("recovered = %d, want 1", recovered)
}
// Verify it's back to pending
item, _ = q.GetStatus("att-stale")
if item.Status != StatusPending {
t.Errorf("status = %q, want %q", item.Status, StatusPending)
}
}
func TestRecoverStale_SkipsRecent(t *testing.T) {
q := tempQueue(t)
// Create item "processing" but recent (not stale)
q.Enqueue("att-recent", "key", "application/pdf", "recent.pdf")
item, _ := q.GetStatus("att-recent")
item.Status = StatusProcessing
item.StartedAt = time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339)
q.mu.Lock()
q.writeStatus("att-recent", item)
q.mu.Unlock()
recovered, err := q.RecoverStale(1 * time.Hour)
if err != nil {
t.Fatalf("RecoverStale: %v", err)
}
if recovered != 0 {
t.Errorf("recovered = %d, want 0 (item is recent)", recovered)
}
}
func TestIsExtractable(t *testing.T) {
tests := []struct {
contentType string
want bool
}{
{"application/pdf", true},
{"application/vnd.openxmlformats-officedocument.wordprocessingml.document", true},
{"image/png", false},
{"text/plain", false},
{"image/jpeg", false},
{"application/rtf", true},
}
for _, tc := range tests {
got := IsExtractable(tc.contentType)
if got != tc.want {
t.Errorf("IsExtractable(%q) = %v, want %v", tc.contentType, got, tc.want)
}
}
}
func TestAtomicWrite(t *testing.T) {
q := tempQueue(t)
// Enqueue and verify the file is valid JSON
q.Enqueue("att-atomic", "key", "application/pdf", "test.pdf")
statusPath := filepath.Join(q.storagePath, "processing", "att-atomic", "status.json")
data, err := os.ReadFile(statusPath)
if err != nil {
t.Fatalf("ReadFile: %v", err)
}
// Verify no .tmp file left behind
tmpPath := statusPath + ".tmp"
if _, err := os.Stat(tmpPath); !os.IsNotExist(err) {
t.Error("temp file should not exist after atomic write")
}
if len(data) == 0 {
t.Error("status.json should not be empty")
}
}

1
server/go.mod
View File

@@ -8,6 +8,7 @@ require (
github.com/gorilla/websocket v1.5.3
github.com/joho/godotenv v1.5.1
github.com/lib/pq v1.10.9
github.com/minio/minio-go/v7 v7.0.82
golang.org/x/crypto v0.14.0
)

23
server/handlers/admin.go
View File

@@ -277,9 +277,10 @@ func (h *AdminHandler) PublicSettings(c *gin.Context) {
}
c.JSON(http.StatusOK, gin.H{
"banner": banner,
"branding": branding,
"has_admin_prompt": hasAdminPrompt,
"banner": banner,
"branding": branding,
"has_admin_prompt": hasAdminPrompt,
"storage_configured": storageConfigured,
"policies": gin.H{
"allow_registration": policies["allow_registration"],
"allow_user_byok": policies["allow_user_byok"],
@@ -288,6 +289,22 @@ func (h *AdminHandler) PublicSettings(c *gin.Context) {
})
}
// ── Vault Status ────────────────────────────
func (h *AdminHandler) VaultStatus(c *gin.Context) {
hasKey := h.vault != nil && h.vault.HasEnvKey()
keyStr := ""
if hasKey {
keyStr = "set" // non-empty signals to VaultStatus that key is configured
}
status, err := crypto.VaultStatus(database.DB, keyStr)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get vault status"})
return
}
c.JSON(http.StatusOK, status)
}
// ── Provider Configs (Global) ───────────────
func (h *AdminHandler) ListGlobalConfigs(c *gin.Context) {

476
server/handlers/attachments.go Normal file
View File

@@ -0,0 +1,476 @@
package handlers
import (
"context"
"fmt"
"io"
"log"
"net/http"
"path/filepath"
"strings"
"time"
"github.com/gin-gonic/gin"
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/extraction"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/storage"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// ── Default Limits ─────────────────────────
// Overridable via global_settings keys.
const (
defaultMaxFileSize = 10 * 1024 * 1024 // 10 MB
defaultMaxUploadSize = 50 * 1024 * 1024 // 50 MB total per request (future: multi-file)
defaultMaxAttachmentsPerMsg = 5 // future: multi-file per message
defaultOrphanMaxAge = 24 * time.Hour
)
// allowedMIMETypes is the default allowlist. Admin can override via global_settings.
var allowedMIMETypes = map[string]bool{
// Images
"image/jpeg": true, "image/png": true, "image/gif": true,
"image/webp": true, "image/svg+xml": true,
// Documents
"application/pdf": true,
"text/plain": true, "text/markdown": true, "text/csv": true,
// Microsoft Office
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": true,
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": true,
"application/vnd.openxmlformats-officedocument.presentationml.presentation": true,
"application/msword": true, "application/vnd.ms-excel": true,
// OpenDocument
"application/vnd.oasis.opendocument.text": true,
"application/vnd.oasis.opendocument.spreadsheet": true,
"application/vnd.oasis.opendocument.presentation": true,
// Other
"application/rtf": true,
}
// ── Handler ────────────────────────────────
type AttachmentHandler struct {
stores store.Stores
objStore storage.ObjectStore
extQueue *extraction.Queue // nil if extraction disabled
}
func NewAttachmentHandler(stores store.Stores, objStore storage.ObjectStore, extQueue *extraction.Queue) *AttachmentHandler {
return &AttachmentHandler{stores: stores, objStore: objStore, extQueue: extQueue}
}
// ── Upload ─────────────────────────────────
// POST /api/v1/channels/:id/attachments
// Multipart form: file field "file", returns attachment metadata.
func (h *AttachmentHandler) Upload(c *gin.Context) {
if h.objStore == nil {
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "file storage not configured"})
return
}
userID := getUserID(c)
channelID := c.Param("id")
// Verify channel ownership
if !h.verifyChannelAccess(c, channelID, userID) {
return
}
// Parse multipart
file, header, err := c.Request.FormFile("file")
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "missing file field"})
return
}
defer file.Close()
// Size check
if header.Size > defaultMaxFileSize {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("file too large (max %d MB)", defaultMaxFileSize/(1024*1024)),
})
return
}
// MIME detection: read first 512 bytes for sniffing, then reset
buf := make([]byte, 512)
n, _ := file.Read(buf)
detectedType := http.DetectContentType(buf[:n])
// Reset reader to start
if _, err := file.Seek(0, io.SeekStart); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to process file"})
return
}
// Normalize MIME type (strip params like charset)
contentType := detectedType
if idx := strings.Index(contentType, ";"); idx > 0 {
contentType = strings.TrimSpace(contentType[:idx])
}
// For types that DetectContentType can't distinguish (returns application/octet-stream),
// fall back to extension-based detection
if contentType == "application/octet-stream" {
ext := strings.ToLower(filepath.Ext(header.Filename))
if mapped, ok := extToMIME[ext]; ok {
contentType = mapped
}
}
// Allowlist check
if !allowedMIMETypes[contentType] {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("file type %q not allowed", contentType),
})
return
}
// Build storage key: attachments/{channel_id}/{attachment_id}_{filename}
// We generate the ID first via a temp UUID, then use it in the key.
att := &models.Attachment{
ChannelID: channelID,
UserID: userID,
Filename: sanitizeFilename(header.Filename),
ContentType: contentType,
SizeBytes: header.Size,
Metadata: models.JSONMap{
"extraction_status": "pending",
},
}
// Create PG row first to get the UUID
// storage_key is set after we have the ID
att.StorageKey = "placeholder"
if err := h.stores.Attachments.Create(c.Request.Context(), att); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create attachment record"})
return
}
// Now build the real storage key and update
att.StorageKey = fmt.Sprintf("attachments/%s/%s_%s", channelID, att.ID, att.Filename)
// Write to object store
if err := h.objStore.Put(c.Request.Context(), att.StorageKey, file, header.Size, contentType); err != nil {
// Rollback PG row on storage failure
h.stores.Attachments.Delete(c.Request.Context(), att.ID)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to store file"})
return
}
// Update storage_key in PG
database.DB.ExecContext(c.Request.Context(),
`UPDATE attachments SET storage_key = $1 WHERE id = $2`,
att.StorageKey, att.ID)
// For images, mark extraction as not needed (complete immediately)
if strings.HasPrefix(contentType, "image/") {
h.stores.Attachments.UpdateMetadata(c.Request.Context(), att.ID, map[string]interface{}{
"extraction_status": "complete",
})
att.Metadata["extraction_status"] = "complete"
}
// For text/plain, extract inline (trivial — just read the file)
if contentType == "text/plain" || contentType == "text/markdown" || contentType == "text/csv" {
if _, err := file.Seek(0, io.SeekStart); err == nil {
if textBytes, err := io.ReadAll(file); err == nil {
text := string(textBytes)
h.stores.Attachments.SetExtractedText(c.Request.Context(), att.ID, text)
h.stores.Attachments.UpdateMetadata(c.Request.Context(), att.ID, map[string]interface{}{
"extraction_status": "complete",
})
att.Metadata["extraction_status"] = "complete"
}
}
}
// For documents requiring extraction (PDF, DOCX, etc.), enqueue for sidecar
if extraction.IsExtractable(contentType) && h.extQueue != nil {
if err := h.extQueue.Enqueue(att.ID, att.StorageKey, contentType, att.Filename); err != nil {
log.Printf("extraction enqueue failed for %s: %v", att.ID, err)
// Non-fatal: file is uploaded, just won't have extracted text
}
}
c.JSON(http.StatusCreated, att)
}
// ── Download ───────────────────────────────
// GET /api/v1/attachments/:id/download
// Streams file content with auth check via channel membership.
func (h *AttachmentHandler) Download(c *gin.Context) {
if h.objStore == nil {
c.JSON(http.StatusServiceUnavailable, gin.H{"error": "file storage not configured"})
return
}
userID := getUserID(c)
attID := c.Param("id")
att, err := h.stores.Attachments.GetByID(c.Request.Context(), attID)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "attachment not found"})
return
}
// Channel-scoped access check
if !h.verifyChannelAccess(c, att.ChannelID, userID) {
return
}
reader, size, _, err := h.objStore.Get(c.Request.Context(), att.StorageKey)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to read file"})
return
}
defer reader.Close()
c.Header("Content-Type", att.ContentType)
c.Header("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, att.Filename))
c.Header("Content-Length", fmt.Sprintf("%d", size))
c.Status(http.StatusOK)
io.Copy(c.Writer, reader)
}
// ── Get Metadata ───────────────────────────
// GET /api/v1/attachments/:id
func (h *AttachmentHandler) GetMetadata(c *gin.Context) {
userID := getUserID(c)
attID := c.Param("id")
att, err := h.stores.Attachments.GetByID(c.Request.Context(), attID)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "attachment not found"})
return
}
if !h.verifyChannelAccess(c, att.ChannelID, userID) {
return
}
c.JSON(http.StatusOK, att)
}
// ── List Channel Attachments ───────────────
// GET /api/v1/channels/:id/attachments
func (h *AttachmentHandler) ListByChannel(c *gin.Context) {
userID := getUserID(c)
channelID := c.Param("id")
if !h.verifyChannelAccess(c, channelID, userID) {
return
}
attachments, err := h.stores.Attachments.GetByChannel(c.Request.Context(), channelID)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list attachments"})
return
}
if attachments == nil {
attachments = []models.Attachment{}
}
c.JSON(http.StatusOK, gin.H{"attachments": attachments})
}
// ── Delete ─────────────────────────────────
// DELETE /api/v1/attachments/:id
func (h *AttachmentHandler) DeleteAttachment(c *gin.Context) {
userID := getUserID(c)
attID := c.Param("id")
att, err := h.stores.Attachments.GetByID(c.Request.Context(), attID)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "attachment not found"})
return
}
if !h.verifyChannelAccess(c, att.ChannelID, userID) {
return
}
// Delete from PG (returns the row for storage cleanup)
deleted, err := h.stores.Attachments.Delete(c.Request.Context(), attID)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete attachment"})
return
}
// Clean up storage (async-safe: fire and forget with background context)
if h.objStore != nil && deleted != nil {
storageKey := deleted.StorageKey
go func() {
ctx := context.Background()
if err := h.objStore.Delete(ctx, storageKey); err != nil {
log.Printf("storage cleanup failed for %s: %v", storageKey, err)
}
// Also clean up thumbnail if it exists
h.objStore.Delete(ctx, storageKey+"_thumb.jpg")
}()
}
c.JSON(http.StatusOK, gin.H{"message": "attachment deleted"})
}
// ── Admin: Orphan Cleanup ──────────────────
// POST /admin/storage/cleanup
func (h *AttachmentHandler) CleanupOrphans(c *gin.Context) {
orphans, err := h.stores.Attachments.ListOrphans(c.Request.Context(), defaultOrphanMaxAge)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list orphans"})
return
}
deleted := 0
var freedBytes int64
for _, att := range orphans {
if _, err := h.stores.Attachments.Delete(c.Request.Context(), att.ID); err != nil {
log.Printf("orphan cleanup: failed to delete %s from PG: %v", att.ID, err)
continue
}
if h.objStore != nil {
h.objStore.Delete(c.Request.Context(), att.StorageKey)
h.objStore.Delete(c.Request.Context(), att.StorageKey+"_thumb.jpg")
}
deleted++
freedBytes += att.SizeBytes
}
c.JSON(http.StatusOK, gin.H{
"deleted": deleted,
"freed_bytes": freedBytes,
"scanned": len(orphans),
})
}
// ── Admin: Orphan Count ────────────────────
// GET /admin/storage/orphans (for the admin panel card)
func (h *AttachmentHandler) OrphanCount(c *gin.Context) {
orphans, err := h.stores.Attachments.ListOrphans(c.Request.Context(), defaultOrphanMaxAge)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to count orphans"})
return
}
var totalBytes int64
for _, att := range orphans {
totalBytes += att.SizeBytes
}
c.JSON(http.StatusOK, gin.H{
"count": len(orphans),
"reclaimable_bytes": totalBytes,
})
}
// ── Admin: Extraction Queue Status ─────────
// GET /admin/storage/extraction
func (h *AttachmentHandler) ExtractionStatus(c *gin.Context) {
if h.extQueue == nil {
c.JSON(http.StatusOK, gin.H{
"enabled": false,
"items": []interface{}{},
})
return
}
items, err := h.extQueue.ListAll()
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list extraction queue"})
return
}
if items == nil {
items = []extraction.QueueItem{}
}
// Count by status
counts := map[string]int{}
for _, item := range items {
counts[item.Status]++
}
c.JSON(http.StatusOK, gin.H{
"enabled": true,
"total": len(items),
"counts": counts,
"items": items,
})
}
// ── Channel Delete Hook ────────────────────
// Called by ChannelHandler.DeleteChannel to clean up storage.
func (h *AttachmentHandler) CleanupChannelStorage(channelID string) {
if h.objStore == nil {
return
}
// CASCADE already deleted PG rows. Clean up filesystem.
prefix := fmt.Sprintf("attachments/%s", channelID)
if err := h.objStore.DeletePrefix(context.Background(), prefix); err != nil {
log.Printf("storage cleanup for channel %s failed: %v", channelID, err)
}
}
// ── Helpers ────────────────────────────────
// verifyChannelAccess checks that the requesting user owns the channel.
// Future RBAC (v0.20.0): replace with rbac.Can(userID, channelID, permission).
func (h *AttachmentHandler) verifyChannelAccess(c *gin.Context, channelID, userID string) bool {
var ownerID string
err := database.DB.QueryRowContext(c.Request.Context(),
`SELECT user_id FROM channels WHERE id = $1`, channelID).Scan(&ownerID)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "channel not found"})
return false
}
if ownerID != userID {
// Check if user is admin (admins can access any channel)
role, _ := c.Get("role")
if role != "admin" {
c.JSON(http.StatusForbidden, gin.H{"error": "access denied"})
return false
}
}
return true
}
// sanitizeFilename cleans a filename for safe storage.
func sanitizeFilename(name string) string {
// Take only the base name (strip path separators)
name = filepath.Base(name)
// Replace problematic characters
replacer := strings.NewReplacer(
"/", "_", "\\", "_", "..", "_", "\x00", "",
)
name = replacer.Replace(name)
if name == "" || name == "." {
name = "unnamed"
}
// Truncate to 200 chars (leave room for UUID prefix in storage key)
if len(name) > 200 {
ext := filepath.Ext(name)
name = name[:200-len(ext)] + ext
}
return name
}
// extToMIME maps file extensions to MIME types for cases where
// http.DetectContentType returns application/octet-stream.
var extToMIME = map[string]string{
".pdf": "application/pdf",
".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
".doc": "application/msword",
".xls": "application/vnd.ms-excel",
".odt": "application/vnd.oasis.opendocument.text",
".ods": "application/vnd.oasis.opendocument.spreadsheet",
".odp": "application/vnd.oasis.opendocument.presentation",
".rtf": "application/rtf",
".md": "text/markdown",
".csv": "text/csv",
".svg": "image/svg+xml",
}

84
server/handlers/channels.go
View File

@@ -2,6 +2,7 @@ package handlers
import (
"database/sql"
"encoding/json"
"math"
"net/http"
"strconv"
@@ -27,33 +28,35 @@ type createChannelRequest struct {
}
type updateChannelRequest struct {
Title *string `json:"title,omitempty"`
Description *string `json:"description,omitempty"`
Model *string `json:"model,omitempty"`
SystemPrompt *string `json:"system_prompt,omitempty"`
APIConfigID *string `json:"provider_config_id,omitempty"`
IsArchived *bool `json:"is_archived,omitempty"`
IsPinned *bool `json:"is_pinned,omitempty"`
Folder *string `json:"folder,omitempty"`
Tags []string `json:"tags,omitempty"`
Title *string `json:"title,omitempty"`
Description *string `json:"description,omitempty"`
Model *string `json:"model,omitempty"`
SystemPrompt *string `json:"system_prompt,omitempty"`
APIConfigID *string `json:"provider_config_id,omitempty"`
IsArchived *bool `json:"is_archived,omitempty"`
IsPinned *bool `json:"is_pinned,omitempty"`
Folder *string `json:"folder,omitempty"`
Tags []string `json:"tags,omitempty"`
Settings *json.RawMessage `json:"settings,omitempty"` // JSONB merge into existing settings
}
type channelResponse struct {
ID string `json:"id"`
UserID string `json:"user_id"`
Title string `json:"title"`
Type string `json:"type"`
Description *string `json:"description"`
Model *string `json:"model"`
APIConfigID *string `json:"provider_config_id"`
SystemPrompt *string `json:"system_prompt"`
IsArchived bool `json:"is_archived"`
IsPinned bool `json:"is_pinned"`
Folder *string `json:"folder"`
Tags []string `json:"tags"`
MessageCount int `json:"message_count"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
ID string `json:"id"`
UserID string `json:"user_id"`
Title string `json:"title"`
Type string `json:"type"`
Description *string `json:"description"`
Model *string `json:"model"`
APIConfigID *string `json:"provider_config_id"`
SystemPrompt *string `json:"system_prompt"`
IsArchived bool `json:"is_archived"`
IsPinned bool `json:"is_pinned"`
Folder *string `json:"folder"`
Tags []string `json:"tags"`
Settings json.RawMessage `json:"settings,omitempty"`
MessageCount int `json:"message_count"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"`
}
type paginatedResponse struct {
@@ -72,6 +75,16 @@ func NewChannelHandler() *ChannelHandler {
return &ChannelHandler{}
}
// channelDeleteHook is called after a channel is successfully deleted.
// Set at startup by SetChannelDeleteHook to clean up storage files.
var channelDeleteHook func(channelID string)
// SetChannelDeleteHook registers a callback invoked after channel deletion.
// Used to clean up attachment files on the storage backend.
func SetChannelDeleteHook(fn func(channelID string)) {
channelDeleteHook = fn
}
// ── Helpers ─────────────────────────────────
// getUserID extracts the authenticated user's ID from context.
@@ -141,7 +154,7 @@ func (h *ChannelHandler) ListChannels(c *gin.Context) {
// Fetch channels with message count
query := `
SELECT c.id, c.user_id, c.title, c.type, c.description, c.model, c.provider_config_id,
c.system_prompt, c.is_archived, c.is_pinned, c.folder, c.tags,
c.system_prompt, c.is_archived, c.is_pinned, c.folder, c.tags, c.settings,
COALESCE(mc.cnt, 0) AS message_count,
c.created_at, c.updated_at
FROM channels c
@@ -186,7 +199,7 @@ func (h *ChannelHandler) ListChannels(c *gin.Context) {
err := rows.Scan(
&ch.ID, &ch.UserID, &ch.Title, &ch.Type, &ch.Description, &ch.Model, &ch.APIConfigID,
&ch.SystemPrompt, &ch.IsArchived, &ch.IsPinned, &ch.Folder,
pq.Array(&tags),
pq.Array(&tags), &ch.Settings,
&ch.MessageCount, &ch.CreatedAt, &ch.UpdatedAt,
)
if err != nil {
@@ -236,13 +249,13 @@ func (h *ChannelHandler) CreateChannel(c *gin.Context) {
INSERT INTO channels (user_id, title, type, description, model, system_prompt, provider_config_id, folder, tags)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
RETURNING id, user_id, title, type, description, model, provider_config_id, system_prompt,
is_archived, is_pinned, folder, tags, created_at, updated_at
is_archived, is_pinned, folder, tags, settings, created_at, updated_at
`, userID, req.Title, channelType, req.Description, req.Model, req.SystemPrompt, req.APIConfigID,
req.Folder, pq.Array(req.Tags),
).Scan(
&ch.ID, &ch.UserID, &ch.Title, &ch.Type, &ch.Description, &ch.Model, &ch.APIConfigID,
&ch.SystemPrompt, &ch.IsArchived, &ch.IsPinned, &ch.Folder,
pq.Array(&tags), &ch.CreatedAt, &ch.UpdatedAt,
pq.Array(&tags), &ch.Settings, &ch.CreatedAt, &ch.UpdatedAt,
)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create channel"})
@@ -284,7 +297,7 @@ func (h *ChannelHandler) GetChannel(c *gin.Context) {
var tags []string
err := database.DB.QueryRow(`
SELECT c.id, c.user_id, c.title, c.type, c.description, c.model, c.provider_config_id,
c.system_prompt, c.is_archived, c.is_pinned, c.folder, c.tags,
c.system_prompt, c.is_archived, c.is_pinned, c.folder, c.tags, c.settings,
COALESCE(mc.cnt, 0) AS message_count,
c.created_at, c.updated_at
FROM channels c
@@ -295,7 +308,7 @@ func (h *ChannelHandler) GetChannel(c *gin.Context) {
`, channelID, userID).Scan(
&ch.ID, &ch.UserID, &ch.Title, &ch.Type, &ch.Description, &ch.Model, &ch.APIConfigID,
&ch.SystemPrompt, &ch.IsArchived, &ch.IsPinned, &ch.Folder,
pq.Array(&tags),
pq.Array(&tags), &ch.Settings,
&ch.MessageCount, &ch.CreatedAt, &ch.UpdatedAt,
)
@@ -383,6 +396,12 @@ func (h *ChannelHandler) UpdateChannel(c *gin.Context) {
if req.Tags != nil {
addClause("tags", pq.Array(req.Tags))
}
if req.Settings != nil {
// JSONB merge: new settings keys overwrite existing, unmentioned keys preserved
setClauses = append(setClauses, "settings = COALESCE(settings, '{}'::jsonb) || $"+strconv.Itoa(argN)+"::jsonb")
args = append(args, []byte(*req.Settings))
argN++
}
if len(setClauses) == 0 {
c.JSON(http.StatusBadRequest, gin.H{"error": "no fields to update"})
@@ -430,5 +449,10 @@ func (h *ChannelHandler) DeleteChannel(c *gin.Context) {
return
}
// Clean up storage files (CASCADE already removed PG attachment rows)
if channelDeleteHook != nil {
go channelDeleteHook(channelID)
}
c.JSON(http.StatusOK, gin.H{"message": "channel deleted"})
}

204
server/handlers/completion.go
View File

@@ -3,49 +3,54 @@ package handlers
import (
"context"
"database/sql"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"strings"
"github.com/gin-gonic/gin"
capspkg "git.gobha.me/xcaliber/chat-switchboard/capabilities"
"git.gobha.me/xcaliber/chat-switchboard/crypto"
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/events"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/providers"
"git.gobha.me/xcaliber/chat-switchboard/crypto"
"git.gobha.me/xcaliber/chat-switchboard/storage"
"git.gobha.me/xcaliber/chat-switchboard/store"
capspkg "git.gobha.me/xcaliber/chat-switchboard/capabilities"
"git.gobha.me/xcaliber/chat-switchboard/tools"
)
// ── Request Types ───────────────────────────
type completionRequest struct {
ChannelID string `json:"channel_id"` // preferred; validated manually below
ChatID string `json:"chat_id"` // deprecated alias — maps to channel_id
Content string `json:"content" binding:"required"`
Model string `json:"model,omitempty"`
PresetID string `json:"preset_id,omitempty"` // if set, unwraps preset → base model + config
APIConfigID string `json:"provider_config_id,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
Stream *bool `json:"stream,omitempty"`
ChannelID string `json:"channel_id"` // preferred; validated manually below
ChatID string `json:"chat_id"` // deprecated alias — maps to channel_id
Content string `json:"content" binding:"required"`
Model string `json:"model,omitempty"`
PresetID string `json:"preset_id,omitempty"` // if set, unwraps preset → base model + config
APIConfigID string `json:"provider_config_id,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
Stream *bool `json:"stream,omitempty"`
AttachmentIDs []string `json:"attachment_ids,omitempty"` // staged attachment UUIDs to include in request
}
// CompletionHandler proxies LLM requests through the backend.
type CompletionHandler struct {
vault *crypto.KeyResolver
stores store.Stores
hub *events.Hub // WebSocket hub for browser tool bridge
vault *crypto.KeyResolver
stores store.Stores
hub *events.Hub // WebSocket hub for browser tool bridge
objStore storage.ObjectStore // file storage for attachment content (nil = disabled)
}
// NewCompletionHandler creates a new handler.
func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub) *CompletionHandler {
return &CompletionHandler{vault: vault, stores: stores, hub: hub}
func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub, objStore storage.ObjectStore) *CompletionHandler {
return &CompletionHandler{vault: vault, stores: stores, hub: hub, objStore: objStore}
}
// ── Chat Completion ─────────────────────────
@@ -138,26 +143,53 @@ func (h *CompletionHandler) Complete(c *gin.Context) {
return
}
// Add the new user message
messages = append(messages, providers.Message{
// Resolve capabilities early — needed for vision gating below
caps := h.getModelCapabilities(c, model, configID)
// Build user message — multimodal if attachments are present
userMsg := providers.Message{
Role: "user",
Content: req.Content,
})
}
// Persist user message
if _, err := h.persistMessage(channelID, userID, "user", req.Content, "", 0, 0, nil, nil); err != nil {
if len(req.AttachmentIDs) > 0 && h.objStore != nil {
parts, augContent, _, err := h.buildMultimodalParts(c, channelID, req.Content, req.AttachmentIDs, caps)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
if len(parts) > 0 {
// Multimodal (images present): use ContentParts
userMsg.ContentParts = parts
} else if augContent != req.Content {
// Doc-only: use augmented text content
userMsg.Content = augContent
}
}
messages = append(messages, userMsg)
// Persist user message (text-only for storage — multimodal parts are ephemeral)
msgID, err := h.persistMessage(channelID, userID, "user", req.Content, "", 0, 0, nil, nil)
if err != nil {
log.Printf("Failed to persist user message: %v", err)
}
// Link attachments to the persisted message
if msgID != "" && len(req.AttachmentIDs) > 0 {
for _, attID := range req.AttachmentIDs {
if err := h.stores.Attachments.SetMessageID(c.Request.Context(), attID, msgID); err != nil {
log.Printf("Failed to link attachment %s to message %s: %v", attID, msgID, err)
}
}
}
// Build provider request
provReq := providers.CompletionRequest{
Model: model,
Messages: messages,
}
// Resolve capabilities for this model — auto-set defaults
caps := h.getModelCapabilities(c, model, configID)
if req.MaxTokens > 0 {
provReq.MaxTokens = req.MaxTokens
} else {
@@ -405,6 +437,128 @@ func (h *CompletionHandler) getModelCapabilities(c *gin.Context, model, apiConfi
return ResolveModelCaps(c, model, apiConfigID)
}
// ── Multimodal Assembly ─────────────────────
// Builds content parts from attachments for the user message.
//
// Returns:
// - parts: ContentParts array (non-nil only when images are present)
// - augContent: enriched text content with document context (for doc-only case)
// - validIDs: attachment IDs that were successfully processed
// - error: if any attachment is invalid or vision is needed but missing
//
// Rules:
// - Images → base64 data URI (requires vision capability)
// - Documents with extracted_text → text injection
// - Documents without extraction → filename placeholder
// - Text is always the first part
func (h *CompletionHandler) buildMultimodalParts(
c *gin.Context,
channelID, textContent string,
attachmentIDs []string,
caps models.ModelCapabilities,
) ([]providers.ContentPart, string, []string, error) {
parts := []providers.ContentPart{
{Type: "text", Text: textContent},
}
var docTexts []string
var validIDs []string
hasImage := false
for _, attID := range attachmentIDs {
att, err := h.stores.Attachments.GetByID(c.Request.Context(), attID)
if err != nil {
return nil, "", nil, fmt.Errorf("attachment %s not found", attID)
}
// Security: verify attachment belongs to this channel
if att.ChannelID != channelID {
return nil, "", nil, fmt.Errorf("attachment %s does not belong to this channel", attID)
}
if isImageContentType(att.ContentType) {
// Vision gating: reject images if model lacks vision
if !caps.Vision {
return nil, "", nil, fmt.Errorf("model does not support image input; remove image attachments or choose a vision-capable model")
}
// Read image from storage, base64 encode, build data URI
reader, _, _, err := h.objStore.Get(c.Request.Context(), att.StorageKey)
if err != nil {
log.Printf("Failed to read attachment %s from storage: %v", attID, err)
parts = append(parts, providers.ContentPart{
Type: "text",
Text: fmt.Sprintf("[Image: %s — failed to read from storage]", att.Filename),
})
validIDs = append(validIDs, attID)
continue
}
data, err := io.ReadAll(reader)
reader.Close()
if err != nil {
log.Printf("Failed to read attachment %s bytes: %v", attID, err)
validIDs = append(validIDs, attID)
continue
}
b64 := base64.StdEncoding.EncodeToString(data)
dataURI := fmt.Sprintf("data:%s;base64,%s", att.ContentType, b64)
parts = append(parts, providers.ContentPart{
Type: "image_url",
ImageURL: &providers.ImageURL{
URL: dataURI,
Detail: "auto",
},
})
hasImage = true
} else if att.ExtractedText != nil && *att.ExtractedText != "" {
// Document with extracted text → inject as context
docText := fmt.Sprintf("[Document: %s]\n%s", att.Filename, *att.ExtractedText)
parts = append(parts, providers.ContentPart{
Type: "text",
Text: docText,
})
docTexts = append(docTexts, docText)
} else {
// Document without extraction (pending, failed, or not extractable)
status := "pending"
if s, ok := att.Metadata["extraction_status"].(string); ok {
status = s
}
placeholder := fmt.Sprintf("[Attached file: %s (extraction %s)]", att.Filename, status)
parts = append(parts, providers.ContentPart{
Type: "text",
Text: placeholder,
})
docTexts = append(docTexts, placeholder)
}
validIDs = append(validIDs, attID)
}
// If images present → use ContentParts (multimodal array)
if hasImage {
return parts, textContent, validIDs, nil
}
// Doc-only: merge document context into a single text string (more efficient)
augmented := textContent
for _, dt := range docTexts {
augmented += "\n\n" + dt
}
return nil, augmented, validIDs, nil
}
// isImageContentType returns true for MIME types that should be sent as
// base64 image content parts rather than text extraction.
func isImageContentType(ct string) bool {
return strings.HasPrefix(ct, "image/")
}
// ── Config Resolution ───────────────────────
// Priority: request.provider_config_id → chat.provider_config_id → user's first active config

14
server/handlers/integration_test.go
View File

@@ -17,7 +17,7 @@ import (
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/middleware"
"git.gobha.me/xcaliber/chat-switchboard/roles"
"git.gobha.me/xcaliber/chat-switchboard/store/postgres"
postgres "git.gobha.me/xcaliber/chat-switchboard/store/postgres"
)
// ── Test Harness ────────────────────────────
@@ -132,9 +132,19 @@ func setupHarness(t *testing.T) *testHarness {
channels := NewChannelHandler()
protected.GET("/channels", channels.ListChannels)
protected.POST("/channels", channels.CreateChannel)
protected.GET("/channels/:id", channels.GetChannel)
protected.DELETE("/channels/:id", channels.DeleteChannel)
// Attachments (nil storage = upload returns 503, but metadata works)
attachH := NewAttachmentHandler(stores, nil, nil)
protected.POST("/channels/:id/attachments", attachH.Upload)
protected.GET("/channels/:id/attachments", attachH.ListByChannel)
protected.GET("/attachments/:id", attachH.GetMetadata)
protected.GET("/attachments/:id/download", attachH.Download)
protected.DELETE("/attachments/:id", attachH.DeleteAttachment)
// Completions
completions := NewCompletionHandler(nil, stores, nil)
completions := NewCompletionHandler(nil, stores, nil, nil)
protected.POST("/chat/completions", completions.Complete)
// Admin routes

14
server/handlers/messages.go
View File

@@ -15,6 +15,7 @@ import (
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/events"
"git.gobha.me/xcaliber/chat-switchboard/providers"
"git.gobha.me/xcaliber/chat-switchboard/storage"
"git.gobha.me/xcaliber/chat-switchboard/store"
"git.gobha.me/xcaliber/chat-switchboard/tools"
)
@@ -58,14 +59,15 @@ type cursorRequest struct {
// MessageHandler holds dependencies for message endpoints.
type MessageHandler struct {
vault *crypto.KeyResolver
stores store.Stores
hub *events.Hub
vault *crypto.KeyResolver
stores store.Stores
hub *events.Hub
objStore storage.ObjectStore
}
// NewMessageHandler creates a new message handler.
func NewMessageHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub) *MessageHandler {
return &MessageHandler{vault: vault, stores: stores, hub: hub}
func NewMessageHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub, objStore storage.ObjectStore) *MessageHandler {
return &MessageHandler{vault: vault, stores: stores, hub: hub, objStore: objStore}
}
// ── List Messages (flat, all branches) ──────
@@ -360,7 +362,7 @@ func (h *MessageHandler) Regenerate(c *gin.Context) {
// ── Resolve model + provider ──
comp := NewCompletionHandler(h.vault, h.stores, h.hub)
comp := NewCompletionHandler(h.vault, h.stores, h.hub, h.objStore)
var presetSystemPrompt string
model := req.Model

62
server/handlers/storage.go Normal file
View File

@@ -0,0 +1,62 @@
package handlers
import (
"net/http"
"github.com/gin-gonic/gin"
"git.gobha.me/xcaliber/chat-switchboard/storage"
)
// storageConfigured is a package-level flag set during init.
// Read by PublicSettings to include in the boot payload.
var storageConfigured bool
// SetStorageConfigured sets the package-level flag indicating whether
// file storage is available. Called from main.go after storage init.
func SetStorageConfigured(configured bool) {
storageConfigured = configured
}
// StorageHandler handles file storage admin endpoints.
type StorageHandler struct {
store storage.ObjectStore
}
// NewStorageHandler creates a StorageHandler.
// store may be nil if storage is not configured.
func NewStorageHandler(store storage.ObjectStore) *StorageHandler {
return &StorageHandler{store: store}
}
// Configured returns true if a storage backend is available.
func (h *StorageHandler) Configured() bool {
return h.store != nil
}
// Status returns storage backend status and statistics.
//
// GET /admin/storage/status
func (h *StorageHandler) Status(c *gin.Context) {
if h.store == nil {
c.JSON(http.StatusOK, gin.H{
"backend": "none",
"configured": false,
"healthy": false,
})
return
}
stats, err := h.store.Stats(c.Request.Context())
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"error": "failed to collect storage stats",
"backend": h.store.Backend(),
"configured": true,
"healthy": false,
})
return
}
c.JSON(http.StatusOK, stats)
}

148
server/main.go
View File

@@ -1,7 +1,11 @@
package main
import (
"fmt"
"log"
"os"
"strings"
"time"
"github.com/gin-gonic/gin"
@@ -9,16 +13,30 @@ import (
"git.gobha.me/xcaliber/chat-switchboard/crypto"
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/events"
"git.gobha.me/xcaliber/chat-switchboard/extraction"
"git.gobha.me/xcaliber/chat-switchboard/handlers"
"git.gobha.me/xcaliber/chat-switchboard/middleware"
"git.gobha.me/xcaliber/chat-switchboard/providers"
"git.gobha.me/xcaliber/chat-switchboard/roles"
"git.gobha.me/xcaliber/chat-switchboard/storage"
"git.gobha.me/xcaliber/chat-switchboard/store"
postgres "git.gobha.me/xcaliber/chat-switchboard/store/postgres"
_ "git.gobha.me/xcaliber/chat-switchboard/tools" // registers built-in tools via init()
)
func main() {
// ── Subcommand dispatch ──────────────────
// Usage: switchboard vault rekey
if len(os.Args) > 2 && os.Args[1] == "vault" {
runVaultCommand(os.Args[2])
return
}
if len(os.Args) > 1 && os.Args[1] == "version" {
fmt.Println("switchboard", Version)
return
}
// ── Server startup ──────────────────────
cfg := config.Load()
// Register LLM providers
@@ -27,6 +45,7 @@ func main() {
var stores store.Stores
uekCache := crypto.NewUEKCache()
var keyResolver *crypto.KeyResolver
var objStore storage.ObjectStore
if err := database.Connect(cfg); err != nil {
log.Printf("⚠ Database unavailable: %v", err)
@@ -73,6 +92,51 @@ func main() {
}
defer database.Close()
// ── File Storage ─────────────────────────
// Auto-detects PVC if STORAGE_PATH is writable. Explicit STORAGE_BACKEND
// overrides auto-detection. nil objStore = storage features disabled.
var s3Cfg *storage.S3Config
if cfg.S3Bucket != "" {
s3Cfg = &storage.S3Config{
Endpoint: cfg.S3Endpoint,
Bucket: cfg.S3Bucket,
Region: cfg.S3Region,
AccessKey: cfg.S3AccessKey,
SecretKey: cfg.S3SecretKey,
Prefix: cfg.S3Prefix,
ForcePathStyle: cfg.S3ForcePathStyle,
}
}
if s, err := storage.Init(cfg.StorageBackend, cfg.StoragePath, s3Cfg); err != nil {
if cfg.StorageBackend != "" {
// Explicit backend requested but failed — fatal
log.Fatalf("❌ Storage init failed: %v", err)
}
log.Printf("⚠ Storage init failed: %v", err)
} else {
objStore = s
}
handlers.SetStorageConfigured(objStore != nil)
// ── Extraction Queue ────────────────────
// Filesystem-based queue for document text extraction (PDF, DOCX, etc.)
// Nil if storage is disabled.
var extQueue *extraction.Queue
if objStore != nil && cfg.StoragePath != "" {
q, err := extraction.NewQueue(cfg.StoragePath, cfg.ExtractionConcurrency)
if err != nil {
log.Printf("⚠ Extraction queue init failed: %v", err)
} else {
extQueue = q
// Recover items stuck in "processing" from previous crash
if recovered, err := extQueue.RecoverStale(30 * time.Minute); err != nil {
log.Printf("⚠ Extraction recovery failed: %v", err)
} else if recovered > 0 {
log.Printf(" 📋 Recovered %d stale extraction items", recovered)
}
}
}
// Role resolver for model role dispatch (needs stores + vault)
roleResolver := roles.NewResolver(stores, keyResolver)
@@ -148,7 +212,7 @@ func main() {
protected.DELETE("/channels/:id", channels.DeleteChannel)
// Messages
msgs := handlers.NewMessageHandler(keyResolver, stores, hub)
msgs := handlers.NewMessageHandler(keyResolver, stores, hub, objStore)
protected.GET("/channels/:id/messages", msgs.ListMessages)
protected.POST("/channels/:id/messages", msgs.CreateMessage)
@@ -160,7 +224,7 @@ func main() {
protected.GET("/channels/:id/messages/:msgId/siblings", msgs.ListSiblings)
// Chat Completions
comp := handlers.NewCompletionHandler(keyResolver, stores, hub)
comp := handlers.NewCompletionHandler(keyResolver, stores, hub, objStore)
protected.POST("/chat/completions", comp.Complete)
// Summarize & Continue
@@ -222,6 +286,17 @@ func main() {
protected.PUT("/notes/:id", notes.Update)
protected.DELETE("/notes/:id", notes.Delete)
// Attachments (file upload/download)
attachH := handlers.NewAttachmentHandler(stores, objStore, extQueue)
protected.POST("/channels/:id/attachments", attachH.Upload)
protected.GET("/channels/:id/attachments", attachH.ListByChannel)
protected.GET("/attachments/:id", attachH.GetMetadata)
protected.GET("/attachments/:id/download", attachH.Download)
protected.DELETE("/attachments/:id", attachH.DeleteAttachment)
// Hook: clean up storage files when channels are deleted
handlers.SetChannelDeleteHook(attachH.CleanupChannelStorage)
// Teams (user: my teams)
teams := handlers.NewTeamHandler(keyResolver)
protected.GET("/teams/mine", teams.MyTeams)
@@ -352,6 +427,19 @@ func main() {
admin.PUT("/pricing", usageH.UpsertPricing)
admin.DELETE("/pricing/:provider/:model", usageH.DeletePricing)
// Storage status
storageH := handlers.NewStorageHandler(objStore)
admin.GET("/storage/status", storageH.Status)
// Vault
admin.GET("/vault/status", adm.VaultStatus)
// Storage management (orphan cleanup)
attachAdm := handlers.NewAttachmentHandler(stores, objStore, extQueue)
admin.GET("/storage/orphans", attachAdm.OrphanCount)
admin.POST("/storage/cleanup", attachAdm.CleanupOrphans)
admin.GET("/storage/extraction", attachAdm.ExtractionStatus)
// Extensions (admin)
extAdm := handlers.NewExtensionHandler(stores)
admin.GET("/extensions", extAdm.AdminListExtensions)
@@ -369,8 +457,64 @@ func main() {
log.Printf(" Base path: %s", bp)
log.Printf(" Schema: %s", database.SchemaVersion())
log.Printf(" Providers: %v", providers.List())
if objStore != nil {
log.Printf(" Storage: %s", objStore.Backend())
if extQueue != nil {
log.Printf(" Extraction: enabled (concurrency=%d)", cfg.ExtractionConcurrency)
} else {
log.Printf(" Extraction: disabled")
}
} else {
log.Printf(" Storage: disabled")
}
log.Printf(" EventBus: ready, WebSocket on %s/ws", cfg.BasePath)
if err := r.Run(":" + cfg.Port); err != nil {
log.Fatalf("Failed to start server: %v", err)
}
}
// ── Vault CLI Commands ──────────────────────
func runVaultCommand(subcmd string) {
cfg := config.Load()
switch strings.ToLower(subcmd) {
case "rekey":
if cfg.DatabaseURL == "" {
log.Fatal("DATABASE_URL is required for vault operations")
}
if err := database.Connect(cfg); err != nil {
log.Fatalf("❌ Database connection failed: %v", err)
}
defer database.Close()
oldKey := os.Getenv("ENCRYPTION_KEY")
newKey := os.Getenv("NEW_ENCRYPTION_KEY")
if err := crypto.Rekey(database.DB, oldKey, newKey); err != nil {
log.Fatalf("❌ Vault rekey failed: %v", err)
}
case "status":
if cfg.DatabaseURL == "" {
log.Fatal("DATABASE_URL is required for vault operations")
}
if err := database.Connect(cfg); err != nil {
log.Fatalf("❌ Database connection failed: %v", err)
}
defer database.Close()
status, err := crypto.VaultStatus(database.DB, cfg.EncryptionKey)
if err != nil {
log.Fatalf("❌ Failed to get vault status: %v", err)
}
fmt.Printf("Encryption key set: %v\n", status.EncryptionKeySet)
fmt.Printf("Encrypted keys: %d\n", status.EncryptedKeys)
fmt.Printf("Vault users (active): %d\n", status.VaultUsers)
default:
fmt.Fprintf(os.Stderr, "Unknown vault command: %s\n", subcmd)
fmt.Fprintf(os.Stderr, "Usage: switchboard vault <rekey|status>\n")
os.Exit(1)
}
}

18
server/models/models.go
View File

@@ -389,6 +389,24 @@ type Note struct {
TeamID *string `json:"team_id,omitempty" db:"team_id"`
}
// =========================================
// ATTACHMENTS
// =========================================
type Attachment struct {
ID string `json:"id" db:"id"`
ChannelID string `json:"channel_id" db:"channel_id"`
UserID string `json:"user_id" db:"user_id"`
MessageID *string `json:"message_id,omitempty" db:"message_id"`
Filename string `json:"filename" db:"filename"`
ContentType string `json:"content_type" db:"content_type"`
SizeBytes int64 `json:"size_bytes" db:"size_bytes"`
StorageKey string `json:"-" db:"storage_key"` // never expose filesystem path
ExtractedText *string `json:"extracted_text,omitempty" db:"extracted_text"`
Metadata JSONMap `json:"metadata,omitempty" db:"metadata"`
CreatedAt time.Time `json:"created_at" db:"created_at"`
}
// =========================================
// AUDIT LOG
// =========================================

75
server/providers/anthropic.go
View File

@@ -263,6 +263,40 @@ func (p *AnthropicProvider) doRequest(ctx context.Context, cfg ProviderConfig, r
Content: m.Content,
},
}
} else if len(m.ContentParts) > 0 && m.Role == "user" {
// Multimodal user message: convert ContentParts to Anthropic blocks
for _, p := range m.ContentParts {
switch p.Type {
case "image_url":
if p.ImageURL != nil {
// Parse data URI: "data:image/jpeg;base64,{data}"
mediaType, b64Data := parseDataURI(p.ImageURL.URL)
if b64Data != "" {
antMsg.Content = append(antMsg.Content, anthropicContentBlock{
Type: "image",
Source: &anthropicImageSource{
Type: "base64",
MediaType: mediaType,
Data: b64Data,
},
})
}
}
default: // "text", "document"
if p.Text != "" {
antMsg.Content = append(antMsg.Content, anthropicContentBlock{
Type: "text",
Text: p.Text,
})
}
}
}
if len(antMsg.Content) == 0 {
// Fallback: shouldn't happen, but safety net
antMsg.Content = []anthropicContentBlock{
{Type: "text", Text: m.Content},
}
}
} else {
// Regular text message
antMsg.Content = []anthropicContentBlock{
@@ -354,6 +388,8 @@ type anthropicContentBlock struct {
Type string `json:"type"`
// text
Text string `json:"text,omitempty"`
// image (type="image")
Source *anthropicImageSource `json:"source,omitempty"`
// tool_use
ID string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
@@ -363,6 +399,15 @@ type anthropicContentBlock struct {
Content string `json:"content,omitempty"`
}
// anthropicImageSource is the Anthropic-native image format.
// Anthropic uses {"type":"base64","media_type":"image/jpeg","data":"..."}
// rather than OpenAI's data URI approach.
type anthropicImageSource struct {
Type string `json:"type"` // "base64"
MediaType string `json:"media_type"` // "image/jpeg", "image/png", etc.
Data string `json:"data"` // raw base64 (no data: prefix)
}
type anthropicMessage struct {
Role string `json:"role"`
Content []anthropicContentBlock `json:"content"`
@@ -425,3 +470,33 @@ type anthropicStreamEvent struct {
Name string `json:"name"`
} `json:"content_block,omitempty"`
}
// parseDataURI splits a data URI into media type and base64 data.
// Input: "data:image/jpeg;base64,/9j/4AAQ..."
// Output: "image/jpeg", "/9j/4AAQ..."
// Returns empty strings if the URI is malformed.
func parseDataURI(uri string) (mediaType, data string) {
// Strip "data:" prefix
if !strings.HasPrefix(uri, "data:") {
return "", ""
}
rest := uri[5:]
// Split on comma (separates header from data)
commaIdx := strings.Index(rest, ",")
if commaIdx < 0 {
return "", ""
}
header := rest[:commaIdx]
data = rest[commaIdx+1:]
// Extract media type from header (before ";base64")
if semiIdx := strings.Index(header, ";"); semiIdx >= 0 {
mediaType = header[:semiIdx]
} else {
mediaType = header
}
return mediaType, data
}

202
server/providers/multimodal_test.go Normal file
View File

@@ -0,0 +1,202 @@
package providers
import (
"encoding/json"
"testing"
)
// ── ContentPart Tests ──────────────────────
func TestContentPart_TextOnly(t *testing.T) {
msg := Message{
Role: "user",
Content: "hello",
ContentParts: []ContentPart{
{Type: "text", Text: "hello"},
},
}
if len(msg.ContentParts) != 1 {
t.Fatalf("expected 1 part, got %d", len(msg.ContentParts))
}
if msg.ContentParts[0].Type != "text" {
t.Errorf("type = %q, want text", msg.ContentParts[0].Type)
}
}
func TestContentPart_WithImage(t *testing.T) {
msg := Message{
Role: "user",
Content: "describe this",
ContentParts: []ContentPart{
{Type: "text", Text: "describe this"},
{Type: "image_url", ImageURL: &ImageURL{URL: "data:image/png;base64,abc123", Detail: "auto"}},
},
}
if len(msg.ContentParts) != 2 {
t.Fatalf("expected 2 parts, got %d", len(msg.ContentParts))
}
if msg.ContentParts[1].ImageURL == nil {
t.Fatal("expected non-nil ImageURL")
}
if msg.ContentParts[1].ImageURL.URL != "data:image/png;base64,abc123" {
t.Errorf("URL = %q", msg.ContentParts[1].ImageURL.URL)
}
}
// ── OpenAI Multimodal Serialization ────────
func TestOpenAI_TextOnlyContent_String(t *testing.T) {
// When Content is a string, it should serialize as a JSON string
msg := openaiMessage{
Role: "user",
Content: "hello world",
}
data, err := json.Marshal(msg)
if err != nil {
t.Fatalf("Marshal: %v", err)
}
var raw map[string]interface{}
json.Unmarshal(data, &raw)
content := raw["content"]
if s, ok := content.(string); !ok || s != "hello world" {
t.Errorf("content = %v (%T), want string 'hello world'", content, content)
}
}
func TestOpenAI_MultimodalContent_Array(t *testing.T) {
// When Content is an array, it should serialize as a JSON array
msg := openaiMessage{
Role: "user",
Content: []openaiContentPart{
{Type: "text", Text: "describe this"},
{Type: "image_url", ImageURL: &openaiImageURL{URL: "data:image/png;base64,abc", Detail: "auto"}},
},
}
data, err := json.Marshal(msg)
if err != nil {
t.Fatalf("Marshal: %v", err)
}
var raw map[string]interface{}
json.Unmarshal(data, &raw)
content := raw["content"]
arr, ok := content.([]interface{})
if !ok {
t.Fatalf("content is %T, want array", content)
}
if len(arr) != 2 {
t.Fatalf("content has %d elements, want 2", len(arr))
}
// Verify first part is text
part0 := arr[0].(map[string]interface{})
if part0["type"] != "text" {
t.Errorf("part[0].type = %v, want text", part0["type"])
}
// Verify second part is image_url
part1 := arr[1].(map[string]interface{})
if part1["type"] != "image_url" {
t.Errorf("part[1].type = %v, want image_url", part1["type"])
}
}
// ── Anthropic Image Source ─────────────────
func TestAnthropic_ImageSourceBlock(t *testing.T) {
block := anthropicContentBlock{
Type: "image",
Source: &anthropicImageSource{
Type: "base64",
MediaType: "image/jpeg",
Data: "abc123",
},
}
data, err := json.Marshal(block)
if err != nil {
t.Fatalf("Marshal: %v", err)
}
var raw map[string]interface{}
json.Unmarshal(data, &raw)
if raw["type"] != "image" {
t.Errorf("type = %v", raw["type"])
}
source := raw["source"].(map[string]interface{})
if source["type"] != "base64" {
t.Errorf("source.type = %v", source["type"])
}
if source["media_type"] != "image/jpeg" {
t.Errorf("source.media_type = %v", source["media_type"])
}
if source["data"] != "abc123" {
t.Errorf("source.data = %v", source["data"])
}
}
func TestAnthropic_TextBlockUnchanged(t *testing.T) {
block := anthropicContentBlock{
Type: "text",
Text: "hello",
}
data, err := json.Marshal(block)
if err != nil {
t.Fatalf("Marshal: %v", err)
}
var raw map[string]interface{}
json.Unmarshal(data, &raw)
if raw["type"] != "text" {
t.Errorf("type = %v", raw["type"])
}
if raw["text"] != "hello" {
t.Errorf("text = %v", raw["text"])
}
// Should NOT have source field
if _, ok := raw["source"]; ok {
t.Error("text block should not have source field")
}
}
// ── parseDataURI ───────────────────────────
func TestParseDataURI_Valid(t *testing.T) {
tests := []struct {
uri string
wantType string
wantData string
}{
{"data:image/jpeg;base64,/9j/4AAQ", "image/jpeg", "/9j/4AAQ"},
{"data:image/png;base64,abc123", "image/png", "abc123"},
{"data:image/webp;base64,RIFF", "image/webp", "RIFF"},
}
for _, tc := range tests {
mediaType, data := parseDataURI(tc.uri)
if mediaType != tc.wantType {
t.Errorf("parseDataURI(%q) mediaType = %q, want %q", tc.uri, mediaType, tc.wantType)
}
if data != tc.wantData {
t.Errorf("parseDataURI(%q) data = %q, want %q", tc.uri, data, tc.wantData)
}
}
}
func TestParseDataURI_Invalid(t *testing.T) {
tests := []string{
"",
"not-a-data-uri",
"data:nocomma",
"https://example.com/image.png",
}
for _, uri := range tests {
mediaType, data := parseDataURI(uri)
if mediaType != "" || data != "" {
t.Errorf("parseDataURI(%q) = (%q, %q), want empty", uri, mediaType, data)
}
}
}

68
server/providers/openai.go
View File

@@ -37,8 +37,14 @@ func (p *OpenAIProvider) ChatCompletion(ctx context.Context, cfg ProviderConfig,
return nil, fmt.Errorf("no choices in response")
}
// Extract content string from response (responses are always string, not array)
var contentStr string
if s, ok := resp.Choices[0].Message.Content.(string); ok {
contentStr = s
}
result := &CompletionResponse{
Content: resp.Choices[0].Message.Content,
Content: contentStr,
Model: resp.Model,
FinishReason: resp.Choices[0].FinishReason,
InputTokens: resp.Usage.PromptTokens,
@@ -360,11 +366,42 @@ func (p *OpenAIProvider) doRequest(ctx context.Context, cfg ProviderConfig, req
})
}
// Convert messages — handle all roles including tool
// Convert messages — handle all roles including tool and multimodal
for _, m := range req.Messages {
oaiMsg := openaiMessage{
Role: m.Role,
Content: m.Content,
Role: m.Role,
}
// Build content: multimodal parts or plain string
if len(m.ContentParts) > 0 && m.Role == "user" {
// Multimodal: convert to OpenAI content array format
parts := make([]openaiContentPart, 0, len(m.ContentParts))
for _, p := range m.ContentParts {
switch p.Type {
case "image_url":
if p.ImageURL != nil {
detail := p.ImageURL.Detail
if detail == "" {
detail = "auto"
}
parts = append(parts, openaiContentPart{
Type: "image_url",
ImageURL: &openaiImageURL{
URL: p.ImageURL.URL,
Detail: detail,
},
})
}
default: // "text", "document" → text
parts = append(parts, openaiContentPart{
Type: "text",
Text: p.Text,
})
}
}
oaiMsg.Content = parts
} else {
oaiMsg.Content = m.Content
}
// Assistant messages with tool calls
@@ -425,11 +462,24 @@ func (p *OpenAIProvider) doRequest(ctx context.Context, cfg ProviderConfig, req
// ── OpenAI Wire Types ───────────────────────
type openaiMessage struct {
Role string `json:"role"`
Content string `json:"content,omitempty"`
ToolCalls []openaiToolCall `json:"tool_calls,omitempty"` // assistant requesting tools
ToolCallID string `json:"tool_call_id,omitempty"` // role=tool result
Name string `json:"name,omitempty"` // tool name for role=tool
Role string `json:"role"`
Content interface{} `json:"content,omitempty"` // string or []openaiContentPart
ToolCalls []openaiToolCall `json:"tool_calls,omitempty"` // assistant requesting tools
ToolCallID string `json:"tool_call_id,omitempty"` // role=tool result
Name string `json:"name,omitempty"` // tool name for role=tool
}
// openaiContentPart represents a single element in a multimodal content array.
// OpenAI format: [{"type":"text","text":"..."}, {"type":"image_url","image_url":{"url":"..."}}]
type openaiContentPart struct {
Type string `json:"type"`
Text string `json:"text,omitempty"`
ImageURL *openaiImageURL `json:"image_url,omitempty"`
}
type openaiImageURL struct {
URL string `json:"url"`
Detail string `json:"detail,omitempty"`
}
type openaiToolCallFunction struct {

24
server/providers/provider.go
View File

@@ -52,12 +52,36 @@ type Message struct {
Role string `json:"role"` // user, assistant, system, tool
Content string `json:"content"`
// Multimodal content parts (v0.12.0+). When set, providers use these
// instead of Content for the user message. Content is still set as
// a fallback and for message persistence (extracted text summary).
ContentParts []ContentPart `json:"content_parts,omitempty"`
// Tool calling fields (assistant → tool_calls, tool → tool_call_id)
ToolCalls []ToolCall `json:"tool_calls,omitempty"` // set when assistant requests tools
ToolCallID string `json:"tool_call_id,omitempty"` // set for role="tool" result messages
Name string `json:"name,omitempty"` // tool name for role="tool"
}
// ContentPart is a single element in a multimodal message.
// Providers convert these to their native wire format.
type ContentPart struct {
Type string `json:"type"` // "text", "image_url", "document"
// Text content (type="text")
Text string `json:"text,omitempty"`
// Image content (type="image_url") — base64 data URI
ImageURL *ImageURL `json:"image_url,omitempty"`
}
// ImageURL holds image data for multimodal requests.
// URL is a data URI: "data:image/jpeg;base64,..."
type ImageURL struct {
URL string `json:"url"`
Detail string `json:"detail,omitempty"` // "auto", "low", "high"
}
// ToolCall represents a function call requested by the LLM.
type ToolCall struct {
ID string `json:"id"`

78
server/storage/init.go Normal file
View File

@@ -0,0 +1,78 @@
package storage
import (
"fmt"
"log"
"os"
)
// Init creates an ObjectStore based on the configuration.
//
// Auto-detection when backend is empty:
// - If storagePath is writable → PVC (implicit)
// - If storagePath is missing/unwritable → nil (storage disabled)
//
// Explicit backends:
// - "pvc" → PVC, fail if path is not writable
// - "s3" → S3-compatible (MinIO, Ceph RGW, AWS), fail if not reachable
func Init(backend, storagePath string, s3Cfg *S3Config) (ObjectStore, error) {
switch backend {
case "pvc":
// Explicit PVC — fail if not writable
s, err := NewPVC(storagePath)
if err != nil {
return nil, fmt.Errorf("storage: PVC backend at %q: %w", storagePath, err)
}
log.Printf(" 📁 Storage: PVC at %s", storagePath)
return s, nil
case "s3":
if s3Cfg == nil {
return nil, fmt.Errorf("storage: S3 backend requires S3_BUCKET and credentials")
}
s, err := NewS3(*s3Cfg)
if err != nil {
return nil, fmt.Errorf("storage: S3 backend: %w", err)
}
endpoint := s3Cfg.Endpoint
if endpoint == "" {
endpoint = "AWS"
}
log.Printf(" 📁 Storage: S3 at %s/%s (prefix=%q)", endpoint, s3Cfg.Bucket, s3Cfg.Prefix)
return s, nil
case "":
// Auto-detect: try PVC if path exists or is creatable
s, err := NewPVC(storagePath)
if err != nil {
// Path not writable — storage disabled, not an error
log.Printf(" 📁 Storage: disabled (path %s not writable)", storagePath)
return nil, nil
}
log.Printf(" 📁 Storage: PVC at %s (auto-detected)", storagePath)
return s, nil
default:
return nil, fmt.Errorf("storage: unknown backend %q (expected pvc or s3)", backend)
}
}
// IsPathWritable checks if a directory path exists and is writable.
// Used by auto-detection. Does not create the directory.
func IsPathWritable(path string) bool {
info, err := os.Stat(path)
if err != nil {
return false
}
if !info.IsDir() {
return false
}
// Try writing a probe file
probe := path + "/.writable-probe"
if err := os.WriteFile(probe, []byte("ok"), 0o644); err != nil {
return false
}
os.Remove(probe)
return true
}

278
server/storage/pvc.go Normal file
View File

@@ -0,0 +1,278 @@
package storage
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
)
// ── PVC Store ──────────────────────────────
// PVCStore implements ObjectStore using the local filesystem.
// Used with Kubernetes PersistentVolumeClaims (CephFS, NFS, local PV)
// or plain Docker volume mounts.
type PVCStore struct {
basePath string
// Content-type cache: key → content type.
// Populated on Put, used by Get. PVC doesn't store metadata
// alongside files, so we keep MIME types in memory. On restart,
// Get falls back to "application/octet-stream" — the DB has the
// authoritative content_type for serving to clients.
mu sync.RWMutex
mimeMap map[string]string
}
// NewPVC creates a PVCStore rooted at basePath.
// Returns an error if the path does not exist or is not writable.
func NewPVC(basePath string) (*PVCStore, error) {
// Ensure base path exists
if err := os.MkdirAll(basePath, 0o755); err != nil {
return nil, fmt.Errorf("storage/pvc: cannot create base path %q: %w", basePath, err)
}
// Create well-known subdirectories
for _, sub := range []string{"attachments", "processing"} {
if err := os.MkdirAll(filepath.Join(basePath, sub), 0o755); err != nil {
return nil, fmt.Errorf("storage/pvc: cannot create %s dir: %w", sub, err)
}
}
s := &PVCStore{
basePath: basePath,
mimeMap: make(map[string]string),
}
// Validate writability
if err := s.Healthy(context.Background()); err != nil {
return nil, err
}
return s, nil
}
func (s *PVCStore) Backend() string { return "pvc" }
// Put writes data to the filesystem at the resolved key path.
func (s *PVCStore) Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error {
if err := validateKey(key); err != nil {
return err
}
absPath := s.resolve(key)
// Ensure parent directory exists
if err := os.MkdirAll(filepath.Dir(absPath), 0o755); err != nil {
return fmt.Errorf("storage/pvc: mkdir for %q: %w", key, err)
}
// Write to temp file first, then rename (atomic on same filesystem)
tmp, err := os.CreateTemp(filepath.Dir(absPath), ".upload-*")
if err != nil {
return fmt.Errorf("storage/pvc: create temp for %q: %w", key, err)
}
tmpName := tmp.Name()
defer func() {
tmp.Close()
// Clean up temp file on failure
if err != nil {
os.Remove(tmpName)
}
}()
written, err := io.Copy(tmp, r)
if err != nil {
return fmt.Errorf("storage/pvc: write %q: %w", key, err)
}
if err = tmp.Close(); err != nil {
return fmt.Errorf("storage/pvc: close %q: %w", key, err)
}
// Verify size if provided (0 means unknown)
if size > 0 && written != size {
os.Remove(tmpName)
return fmt.Errorf("storage/pvc: size mismatch for %q: expected %d, got %d", key, size, written)
}
// Atomic rename
if err = os.Rename(tmpName, absPath); err != nil {
return fmt.Errorf("storage/pvc: rename %q: %w", key, err)
}
// Cache content type
if contentType != "" {
s.mu.Lock()
s.mimeMap[key] = contentType
s.mu.Unlock()
}
return nil
}
// Get returns a reader for the file at key.
func (s *PVCStore) Get(ctx context.Context, key string) (io.ReadCloser, int64, string, error) {
if err := validateKey(key); err != nil {
return nil, 0, "", err
}
absPath := s.resolve(key)
info, err := os.Stat(absPath)
if err != nil {
if os.IsNotExist(err) {
return nil, 0, "", ErrNotFound
}
return nil, 0, "", fmt.Errorf("storage/pvc: stat %q: %w", key, err)
}
f, err := os.Open(absPath)
if err != nil {
return nil, 0, "", fmt.Errorf("storage/pvc: open %q: %w", key, err)
}
// Look up cached content type
s.mu.RLock()
ct := s.mimeMap[key]
s.mu.RUnlock()
if ct == "" {
ct = "application/octet-stream"
}
return f, info.Size(), ct, nil
}
// Delete removes the file at key.
func (s *PVCStore) Delete(ctx context.Context, key string) error {
if err := validateKey(key); err != nil {
return err
}
absPath := s.resolve(key)
err := os.Remove(absPath)
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("storage/pvc: delete %q: %w", key, err)
}
// Clean up cache
s.mu.Lock()
delete(s.mimeMap, key)
s.mu.Unlock()
return nil
}
// DeletePrefix removes all files and directories under the given prefix.
func (s *PVCStore) DeletePrefix(ctx context.Context, prefix string) error {
if err := validateKey(prefix); err != nil {
return err
}
absPath := s.resolve(prefix)
// Check if the path exists
if _, err := os.Stat(absPath); os.IsNotExist(err) {
return nil // Nothing to delete
}
if err := os.RemoveAll(absPath); err != nil {
return fmt.Errorf("storage/pvc: delete prefix %q: %w", prefix, err)
}
// Clean up cache entries with this prefix
s.mu.Lock()
for k := range s.mimeMap {
if strings.HasPrefix(k, prefix) {
delete(s.mimeMap, k)
}
}
s.mu.Unlock()
return nil
}
// Exists checks if a file exists at key.
func (s *PVCStore) Exists(ctx context.Context, key string) (bool, error) {
if err := validateKey(key); err != nil {
return false, err
}
_, err := os.Stat(s.resolve(key))
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return false, fmt.Errorf("storage/pvc: exists %q: %w", key, err)
}
// Healthy verifies the storage path is writable.
func (s *PVCStore) Healthy(ctx context.Context) error {
probe := filepath.Join(s.basePath, ".health-probe")
if err := os.WriteFile(probe, []byte("ok"), 0o644); err != nil {
return fmt.Errorf("storage/pvc: not writable: %w", err)
}
os.Remove(probe)
return nil
}
// Stats returns file count and total size under the base path.
func (s *PVCStore) Stats(ctx context.Context) (*StorageStats, error) {
stats := &StorageStats{
Backend: "pvc",
Path: s.basePath,
Configured: true,
}
// Check health
if err := s.Healthy(ctx); err != nil {
stats.Healthy = false
return stats, nil
}
stats.Healthy = true
// Walk attachments directory for counts
attDir := filepath.Join(s.basePath, "attachments")
err := filepath.Walk(attDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return nil // skip errors (e.g. permission denied)
}
if !info.IsDir() {
stats.TotalFiles++
stats.TotalBytes += info.Size()
}
return nil
})
if err != nil && !os.IsNotExist(err) {
return stats, fmt.Errorf("storage/pvc: stats walk: %w", err)
}
return stats, nil
}
// ── Helpers ────────────────────────────────
// resolve converts a storage key to an absolute filesystem path.
func (s *PVCStore) resolve(key string) string {
return filepath.Join(s.basePath, filepath.FromSlash(key))
}
// validateKey ensures the key is safe (no path traversal, no absolute paths).
func validateKey(key string) error {
if key == "" {
return fmt.Errorf("storage: empty key")
}
if strings.HasPrefix(key, "/") || strings.HasPrefix(key, "\\") {
return fmt.Errorf("storage: absolute key not allowed: %q", key)
}
cleaned := filepath.Clean(key)
if strings.Contains(cleaned, "..") {
return fmt.Errorf("storage: path traversal not allowed: %q", key)
}
return nil
}

295
server/storage/pvc_test.go Normal file
View File

@@ -0,0 +1,295 @@
package storage
import (
"bytes"
"context"
"io"
"os"
"path/filepath"
"testing"
)
func tempStore(t *testing.T) *PVCStore {
t.Helper()
dir := t.TempDir()
s, err := NewPVC(dir)
if err != nil {
t.Fatalf("NewPVC(%q): %v", dir, err)
}
return s
}
func TestPVC_PutGetRoundTrip(t *testing.T) {
s := tempStore(t)
ctx := context.Background()
data := []byte("hello, storage world")
key := "attachments/channel-1/att-1_test.txt"
// Put
err := s.Put(ctx, key, bytes.NewReader(data), int64(len(data)), "text/plain")
if err != nil {
t.Fatalf("Put: %v", err)
}
// Get
rc, size, ct, err := s.Get(ctx, key)
if err != nil {
t.Fatalf("Get: %v", err)
}
defer rc.Close()
if size != int64(len(data)) {
t.Errorf("size = %d, want %d", size, len(data))
}
if ct != "text/plain" {
t.Errorf("content_type = %q, want %q", ct, "text/plain")
}
got, _ := io.ReadAll(rc)
if !bytes.Equal(got, data) {
t.Errorf("data = %q, want %q", got, data)
}
}
func TestPVC_PutAtomicWrite(t *testing.T) {
s := tempStore(t)
ctx := context.Background()
key := "attachments/ch/file.bin"
data := []byte("atomic test data")
err := s.Put(ctx, key, bytes.NewReader(data), int64(len(data)), "application/octet-stream")
if err != nil {
t.Fatalf("Put: %v", err)
}
// File should exist at final path, no temp files left
absPath := filepath.Join(s.basePath, "attachments", "ch")
entries, _ := os.ReadDir(absPath)
for _, e := range entries {
if e.Name() != "file.bin" {
t.Errorf("unexpected file in dir: %s", e.Name())
}
}
}
func TestPVC_PutSizeMismatch(t *testing.T) {
s := tempStore(t)
ctx := context.Background()
data := []byte("short")
err := s.Put(ctx, "attachments/ch/f.txt", bytes.NewReader(data), 999, "text/plain")
if err == nil {
t.Fatal("expected size mismatch error")
}
}
func TestPVC_GetNotFound(t *testing.T) {
s := tempStore(t)
ctx := context.Background()
_, _, _, err := s.Get(ctx, "attachments/nonexistent/file.txt")
if err != ErrNotFound {
t.Errorf("err = %v, want ErrNotFound", err)
}
}
func TestPVC_Delete(t *testing.T) {
s := tempStore(t)
ctx := context.Background()
key := "attachments/ch/delete-me.txt"
_ = s.Put(ctx, key, bytes.NewReader([]byte("bye")), 3, "text/plain")
err := s.Delete(ctx, key)
if err != nil {
t.Fatalf("Delete: %v", err)
}
exists, _ := s.Exists(ctx, key)
if exists {
t.Error("file still exists after Delete")
}
}
func TestPVC_DeleteIdempotent(t *testing.T) {
s := tempStore(t)
ctx := context.Background()
// Delete a file that doesn't exist — should not error
err := s.Delete(ctx, "attachments/no-such/file.txt")
if err != nil {
t.Errorf("Delete nonexistent: %v", err)
}
}
func TestPVC_DeletePrefix(t *testing.T) {
s := tempStore(t)
ctx := context.Background()
// Create several files under a channel prefix
prefix := "attachments/channel-abc/"
for _, name := range []string{"a.txt", "b.png", "c.pdf"} {
key := prefix + name
_ = s.Put(ctx, key, bytes.NewReader([]byte("x")), 1, "text/plain")
}
// Also create a file in a different channel
other := "attachments/channel-other/keep.txt"
_ = s.Put(ctx, other, bytes.NewReader([]byte("y")), 1, "text/plain")
// Delete the channel prefix
err := s.DeletePrefix(ctx, "attachments/channel-abc")
if err != nil {
t.Fatalf("DeletePrefix: %v", err)
}
// Verify channel files are gone
for _, name := range []string{"a.txt", "b.png", "c.pdf"} {
exists, _ := s.Exists(ctx, prefix+name)
if exists {
t.Errorf("%s still exists after DeletePrefix", name)
}
}
// Verify other channel is untouched
exists, _ := s.Exists(ctx, other)
if !exists {
t.Error("other channel file was incorrectly deleted")
}
}
func TestPVC_DeletePrefixNonexistent(t *testing.T) {
s := tempStore(t)
ctx := context.Background()
// Should not error
err := s.DeletePrefix(ctx, "attachments/does-not-exist/")
if err != nil {
t.Errorf("DeletePrefix nonexistent: %v", err)
}
}
func TestPVC_Exists(t *testing.T) {
s := tempStore(t)
ctx := context.Background()
key := "attachments/ch/exists.txt"
_ = s.Put(ctx, key, bytes.NewReader([]byte("hi")), 2, "text/plain")
exists, err := s.Exists(ctx, key)
if err != nil {
t.Fatalf("Exists: %v", err)
}
if !exists {
t.Error("Exists returned false for existing file")
}
exists, err = s.Exists(ctx, "attachments/ch/nope.txt")
if err != nil {
t.Fatalf("Exists (missing): %v", err)
}
if exists {
t.Error("Exists returned true for missing file")
}
}
func TestPVC_Healthy(t *testing.T) {
s := tempStore(t)
if err := s.Healthy(context.Background()); err != nil {
t.Errorf("Healthy: %v", err)
}
}
func TestPVC_Stats(t *testing.T) {
s := tempStore(t)
ctx := context.Background()
// Empty store
stats, err := s.Stats(ctx)
if err != nil {
t.Fatalf("Stats: %v", err)
}
if stats.TotalFiles != 0 || stats.TotalBytes != 0 {
t.Errorf("empty store: files=%d bytes=%d", stats.TotalFiles, stats.TotalBytes)
}
if !stats.Healthy || !stats.Configured {
t.Error("expected healthy + configured")
}
// Add some files
for i := 0; i < 3; i++ {
key := filepath.Join("attachments", "ch", string(rune('a'+i))+".txt")
data := bytes.Repeat([]byte("x"), 100*(i+1))
_ = s.Put(ctx, key, bytes.NewReader(data), int64(len(data)), "text/plain")
}
stats, err = s.Stats(ctx)
if err != nil {
t.Fatalf("Stats: %v", err)
}
if stats.TotalFiles != 3 {
t.Errorf("files = %d, want 3", stats.TotalFiles)
}
if stats.TotalBytes != 600 { // 100 + 200 + 300
t.Errorf("bytes = %d, want 600", stats.TotalBytes)
}
}
func TestPVC_PathTraversal(t *testing.T) {
s := tempStore(t)
ctx := context.Background()
bad := []string{
"../etc/passwd",
"attachments/../../etc/shadow",
"/absolute/path",
"",
}
for _, key := range bad {
err := s.Put(ctx, key, bytes.NewReader([]byte("x")), 1, "text/plain")
if err == nil {
t.Errorf("Put(%q) should have failed", key)
}
_, _, _, err = s.Get(ctx, key)
if err == nil {
t.Errorf("Get(%q) should have failed", key)
}
err = s.Delete(ctx, key)
if err == nil {
t.Errorf("Delete(%q) should have failed", key)
}
}
}
func TestPVC_SubdirCreation(t *testing.T) {
s := tempStore(t)
// Verify well-known subdirs were created
for _, sub := range []string{"attachments", "processing"} {
info, err := os.Stat(filepath.Join(s.basePath, sub))
if err != nil {
t.Errorf("subdir %q not created: %v", sub, err)
continue
}
if !info.IsDir() {
t.Errorf("%q is not a directory", sub)
}
}
}
func TestNewPVC_InvalidPath(t *testing.T) {
// Path that can't be created (nested under a file)
tmp := t.TempDir()
filePath := filepath.Join(tmp, "blocker")
os.WriteFile(filePath, []byte("x"), 0o644)
_, err := NewPVC(filepath.Join(filePath, "subdir"))
if err == nil {
t.Error("expected error for invalid path")
}
}

312
server/storage/s3.go Normal file
View File

@@ -0,0 +1,312 @@
package storage
import (
"bytes"
"context"
"fmt"
"io"
"log"
"strings"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
// ── S3 Store ──────────────────────────────
// S3Config holds all configuration for the S3 backend.
type S3Config struct {
Endpoint string // e.g. "minio.example.com:9000" or "s3.amazonaws.com"
Bucket string // bucket name
Region string // AWS region (default "us-east-1")
AccessKey string // access key ID
SecretKey string // secret access key
Prefix string // optional key prefix (e.g. "switchboard/")
ForcePathStyle bool // path-style URLs (required for MinIO, most self-hosted)
UseSSL bool // use HTTPS (derived from endpoint scheme)
}
// S3Store implements ObjectStore using any S3-compatible API.
// Tested with: MinIO, Ceph RGW, AWS S3.
type S3Store struct {
client *minio.Client
bucket string
prefix string // prepended to all keys (e.g. "switchboard/")
endpoint string // original endpoint for display in Stats
}
// NewS3 creates an S3Store connected to the given endpoint.
// Returns an error if the bucket does not exist or is not accessible.
func NewS3(cfg S3Config) (*S3Store, error) {
if cfg.Bucket == "" {
return nil, fmt.Errorf("storage/s3: bucket name is required")
}
if cfg.AccessKey == "" || cfg.SecretKey == "" {
return nil, fmt.Errorf("storage/s3: access key and secret key are required")
}
if cfg.Region == "" {
cfg.Region = "us-east-1"
}
// Parse endpoint: strip scheme if present, detect SSL
endpoint := cfg.Endpoint
useSSL := cfg.UseSSL
if strings.HasPrefix(endpoint, "https://") {
endpoint = strings.TrimPrefix(endpoint, "https://")
useSSL = true
} else if strings.HasPrefix(endpoint, "http://") {
endpoint = strings.TrimPrefix(endpoint, "http://")
useSSL = false
}
// Default to SSL for AWS endpoints
if endpoint == "" || strings.Contains(endpoint, "amazonaws.com") {
useSSL = true
}
opts := &minio.Options{
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
Secure: useSSL,
Region: cfg.Region,
}
// BucketLookup controls path-style vs virtual-hosted-style URLs.
// MinIO, Ceph, and most self-hosted S3 require path-style.
if cfg.ForcePathStyle {
opts.BucketLookup = minio.BucketLookupPath
} else {
opts.BucketLookup = minio.BucketLookupAuto
}
client, err := minio.New(endpoint, opts)
if err != nil {
return nil, fmt.Errorf("storage/s3: client init: %w", err)
}
// Normalize prefix: ensure trailing slash if non-empty, no leading slash
prefix := strings.TrimPrefix(cfg.Prefix, "/")
if prefix != "" && !strings.HasSuffix(prefix, "/") {
prefix += "/"
}
s := &S3Store{
client: client,
bucket: cfg.Bucket,
prefix: prefix,
endpoint: endpoint,
}
// Validate: check bucket exists and is accessible
ctx := context.Background()
exists, err := client.BucketExists(ctx, cfg.Bucket)
if err != nil {
return nil, fmt.Errorf("storage/s3: cannot reach endpoint: %w", err)
}
if !exists {
return nil, fmt.Errorf("storage/s3: bucket %q does not exist", cfg.Bucket)
}
return s, nil
}
func (s *S3Store) Backend() string { return "s3" }
// Put writes data to S3 at the given key.
func (s *S3Store) Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error {
if err := validateKey(key); err != nil {
return err
}
if contentType == "" {
contentType = "application/octet-stream"
}
opts := minio.PutObjectOptions{
ContentType: contentType,
}
// If size is unknown (0), we need to buffer or use -1.
// minio-go handles -1 with multipart upload.
uploadSize := size
if uploadSize == 0 {
uploadSize = -1
}
_, err := s.client.PutObject(ctx, s.bucket, s.fullKey(key), r, uploadSize, opts)
if err != nil {
return fmt.Errorf("storage/s3: put %q: %w", key, err)
}
return nil
}
// Get returns a reader for the object at key.
func (s *S3Store) Get(ctx context.Context, key string) (io.ReadCloser, int64, string, error) {
if err := validateKey(key); err != nil {
return nil, 0, "", err
}
obj, err := s.client.GetObject(ctx, s.bucket, s.fullKey(key), minio.GetObjectOptions{})
if err != nil {
return nil, 0, "", fmt.Errorf("storage/s3: get %q: %w", key, err)
}
// Stat the object to get size and content type.
// GetObject is lazy — the actual HTTP request happens on Read or Stat.
info, err := obj.Stat()
if err != nil {
obj.Close()
if isS3NotFound(err) {
return nil, 0, "", ErrNotFound
}
return nil, 0, "", fmt.Errorf("storage/s3: stat %q: %w", key, err)
}
ct := info.ContentType
if ct == "" {
ct = "application/octet-stream"
}
return obj, info.Size, ct, nil
}
// Delete removes the object at key.
func (s *S3Store) Delete(ctx context.Context, key string) error {
if err := validateKey(key); err != nil {
return err
}
err := s.client.RemoveObject(ctx, s.bucket, s.fullKey(key), minio.RemoveObjectOptions{})
if err != nil {
// S3 RemoveObject is already idempotent (no error for missing keys)
return fmt.Errorf("storage/s3: delete %q: %w", key, err)
}
return nil
}
// DeletePrefix removes all objects under the given prefix.
func (s *S3Store) DeletePrefix(ctx context.Context, prefix string) error {
if err := validateKey(prefix); err != nil {
return err
}
fullPrefix := s.fullKey(prefix)
// Ensure trailing slash for directory-like prefix
if !strings.HasSuffix(fullPrefix, "/") {
fullPrefix += "/"
}
// List all objects under prefix
objectsCh := s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{
Prefix: fullPrefix,
Recursive: true,
})
// Build removal channel
errCh := s.client.RemoveObjects(ctx, s.bucket, toRemoveChannel(objectsCh), minio.RemoveObjectsOptions{})
// Drain errors
var firstErr error
for e := range errCh {
if e.Err != nil && firstErr == nil {
firstErr = fmt.Errorf("storage/s3: delete prefix %q: %w", prefix, e.Err)
}
}
return firstErr
}
// Exists checks if an object exists at key.
func (s *S3Store) Exists(ctx context.Context, key string) (bool, error) {
if err := validateKey(key); err != nil {
return false, err
}
_, err := s.client.StatObject(ctx, s.bucket, s.fullKey(key), minio.StatObjectOptions{})
if err != nil {
if isS3NotFound(err) {
return false, nil
}
return false, fmt.Errorf("storage/s3: exists %q: %w", key, err)
}
return true, nil
}
// Healthy checks connectivity by performing a HeadBucket.
func (s *S3Store) Healthy(ctx context.Context) error {
exists, err := s.client.BucketExists(ctx, s.bucket)
if err != nil {
return fmt.Errorf("storage/s3: health check failed: %w", err)
}
if !exists {
return fmt.Errorf("storage/s3: bucket %q no longer exists", s.bucket)
}
// Write + read a probe object to verify full access
probeKey := s.fullKey(".health-probe")
_, err = s.client.PutObject(ctx, s.bucket, probeKey,
bytes.NewReader([]byte("ok")), 2,
minio.PutObjectOptions{ContentType: "text/plain"})
if err != nil {
return fmt.Errorf("storage/s3: not writable: %w", err)
}
s.client.RemoveObject(ctx, s.bucket, probeKey, minio.RemoveObjectOptions{})
return nil
}
// Stats returns object count and total size in the bucket (under prefix).
func (s *S3Store) Stats(ctx context.Context) (*StorageStats, error) {
stats := &StorageStats{
Backend: "s3",
Bucket: s.bucket,
Endpoint: s.endpoint,
Configured: true,
}
// Check health first
if err := s.Healthy(ctx); err != nil {
stats.Healthy = false
return stats, nil
}
stats.Healthy = true
// Count objects under the attachments prefix
attPrefix := s.fullKey("attachments/")
objectsCh := s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{
Prefix: attPrefix,
Recursive: true,
})
for obj := range objectsCh {
if obj.Err != nil {
log.Printf("storage/s3: stats list error: %v", obj.Err)
break
}
stats.TotalFiles++
stats.TotalBytes += obj.Size
}
return stats, nil
}
// ── Helpers ────────────────────────────────
// fullKey prepends the configured prefix to a storage key.
func (s *S3Store) fullKey(key string) string {
return s.prefix + key
}
// isS3NotFound checks if an error represents a "not found" condition.
func isS3NotFound(err error) bool {
if err == nil {
return false
}
resp := minio.ToErrorResponse(err)
return resp.Code == "NoSuchKey" || resp.StatusCode == 404
}
// toRemoveChannel converts a ListObjects channel to a RemoveObjects input channel.
func toRemoveChannel(objects <-chan minio.ObjectInfo) <-chan minio.ObjectInfo {
// RemoveObjects accepts the same channel type — pass through directly.
// The channel is already producing ObjectInfo values with Key set.
return objects
}

326
server/storage/s3_test.go Normal file
View File

@@ -0,0 +1,326 @@
package storage
import (
"bytes"
"context"
"io"
"os"
"testing"
)
// ── Unit Tests (always run) ──────────────
func TestS3_NewS3_Validation(t *testing.T) {
// Missing bucket
_, err := NewS3(S3Config{
AccessKey: "test",
SecretKey: "test",
})
if err == nil {
t.Error("expected error for missing bucket")
}
// Missing credentials
_, err = NewS3(S3Config{
Bucket: "test-bucket",
})
if err == nil {
t.Error("expected error for missing credentials")
}
// Missing access key only
_, err = NewS3(S3Config{
Bucket: "test-bucket",
SecretKey: "test",
})
if err == nil {
t.Error("expected error for missing access key")
}
}
func TestS3_FullKey(t *testing.T) {
tests := []struct {
prefix string
key string
want string
}{
{"", "attachments/ch/f.txt", "attachments/ch/f.txt"},
{"switchboard/", "attachments/ch/f.txt", "switchboard/attachments/ch/f.txt"},
{"prefix", "attachments/ch/f.txt", "prefix/attachments/ch/f.txt"}, // trailing slash added by NewS3
}
for _, tt := range tests {
s := &S3Store{prefix: tt.prefix}
// Normalize prefix same as NewS3 does
if s.prefix != "" && s.prefix[len(s.prefix)-1] != '/' {
s.prefix += "/"
}
got := s.fullKey(tt.key)
if got != tt.want {
t.Errorf("fullKey(%q, %q) = %q, want %q", tt.prefix, tt.key, got, tt.want)
}
}
}
func TestS3_KeyValidation(t *testing.T) {
// validateKey is shared between PVC and S3 — test with S3 context
bad := []string{
"",
"../etc/passwd",
"/absolute/path",
"attachments/../../etc/shadow",
}
for _, key := range bad {
if err := validateKey(key); err == nil {
t.Errorf("validateKey(%q) should have failed", key)
}
}
good := []string{
"attachments/ch/f.txt",
"attachments/channel-1/att-abc_test.pdf",
"processing/abc123/status.json",
}
for _, key := range good {
if err := validateKey(key); err != nil {
t.Errorf("validateKey(%q) failed: %v", key, err)
}
}
}
// ── Integration Tests (require live S3 endpoint) ──────────────
//
// Set these environment variables to run:
//
// S3_TEST_ENDPOINT=localhost:9000
// S3_TEST_BUCKET=switchboard-test
// S3_TEST_ACCESS_KEY=minioadmin
// S3_TEST_SECRET_KEY=minioadmin
//
// Example with MinIO:
//
// docker run -d -p 9000:9000 -p 9001:9001 \
// -e MINIO_ROOT_USER=minioadmin -e MINIO_ROOT_PASSWORD=minioadmin \
// minio/minio server /data --console-address :9001
// mc alias set local http://localhost:9000 minioadmin minioadmin
// mc mb local/switchboard-test
func testS3Store(t *testing.T) *S3Store {
t.Helper()
endpoint := os.Getenv("S3_TEST_ENDPOINT")
bucket := os.Getenv("S3_TEST_BUCKET")
accessKey := os.Getenv("S3_TEST_ACCESS_KEY")
secretKey := os.Getenv("S3_TEST_SECRET_KEY")
if endpoint == "" || bucket == "" {
t.Skip("S3_TEST_ENDPOINT and S3_TEST_BUCKET not set — skipping S3 integration tests")
}
s, err := NewS3(S3Config{
Endpoint: endpoint,
Bucket: bucket,
Region: "us-east-1",
AccessKey: accessKey,
SecretKey: secretKey,
Prefix: "test-" + t.Name() + "/",
ForcePathStyle: true,
})
if err != nil {
t.Fatalf("NewS3: %v", err)
}
// Clean up prefix after test
t.Cleanup(func() {
ctx := context.Background()
_ = s.DeletePrefix(ctx, "attachments")
_ = s.DeletePrefix(ctx, "processing")
})
return s
}
func TestS3_Integration_PutGetRoundTrip(t *testing.T) {
s := testS3Store(t)
ctx := context.Background()
data := []byte("hello, S3 storage world")
key := "attachments/channel-1/att-1_test.txt"
err := s.Put(ctx, key, bytes.NewReader(data), int64(len(data)), "text/plain")
if err != nil {
t.Fatalf("Put: %v", err)
}
rc, size, ct, err := s.Get(ctx, key)
if err != nil {
t.Fatalf("Get: %v", err)
}
defer rc.Close()
if size != int64(len(data)) {
t.Errorf("size = %d, want %d", size, len(data))
}
if ct != "text/plain" {
t.Errorf("content_type = %q, want %q", ct, "text/plain")
}
got, _ := io.ReadAll(rc)
if !bytes.Equal(got, data) {
t.Errorf("data = %q, want %q", got, data)
}
}
func TestS3_Integration_GetNotFound(t *testing.T) {
s := testS3Store(t)
ctx := context.Background()
_, _, _, err := s.Get(ctx, "attachments/nonexistent/file.txt")
if err != ErrNotFound {
t.Errorf("err = %v, want ErrNotFound", err)
}
}
func TestS3_Integration_Delete(t *testing.T) {
s := testS3Store(t)
ctx := context.Background()
key := "attachments/ch/delete-me.txt"
_ = s.Put(ctx, key, bytes.NewReader([]byte("bye")), 3, "text/plain")
err := s.Delete(ctx, key)
if err != nil {
t.Fatalf("Delete: %v", err)
}
exists, _ := s.Exists(ctx, key)
if exists {
t.Error("file still exists after Delete")
}
}
func TestS3_Integration_DeleteIdempotent(t *testing.T) {
s := testS3Store(t)
ctx := context.Background()
// S3 delete is inherently idempotent
err := s.Delete(ctx, "attachments/no-such/file.txt")
if err != nil {
t.Errorf("Delete nonexistent: %v", err)
}
}
func TestS3_Integration_DeletePrefix(t *testing.T) {
s := testS3Store(t)
ctx := context.Background()
prefix := "attachments/channel-abc/"
for _, name := range []string{"a.txt", "b.png", "c.pdf"} {
key := prefix + name
_ = s.Put(ctx, key, bytes.NewReader([]byte("x")), 1, "text/plain")
}
other := "attachments/channel-other/keep.txt"
_ = s.Put(ctx, other, bytes.NewReader([]byte("y")), 1, "text/plain")
err := s.DeletePrefix(ctx, "attachments/channel-abc")
if err != nil {
t.Fatalf("DeletePrefix: %v", err)
}
for _, name := range []string{"a.txt", "b.png", "c.pdf"} {
exists, _ := s.Exists(ctx, prefix+name)
if exists {
t.Errorf("%s still exists after DeletePrefix", name)
}
}
exists, _ := s.Exists(ctx, other)
if !exists {
t.Error("other channel file was incorrectly deleted")
}
}
func TestS3_Integration_Exists(t *testing.T) {
s := testS3Store(t)
ctx := context.Background()
key := "attachments/ch/exists.txt"
_ = s.Put(ctx, key, bytes.NewReader([]byte("hi")), 2, "text/plain")
exists, err := s.Exists(ctx, key)
if err != nil {
t.Fatalf("Exists: %v", err)
}
if !exists {
t.Error("Exists returned false for existing file")
}
exists, err = s.Exists(ctx, "attachments/ch/nope.txt")
if err != nil {
t.Fatalf("Exists (missing): %v", err)
}
if exists {
t.Error("Exists returned true for missing file")
}
}
func TestS3_Integration_Healthy(t *testing.T) {
s := testS3Store(t)
if err := s.Healthy(context.Background()); err != nil {
t.Errorf("Healthy: %v", err)
}
}
func TestS3_Integration_Stats(t *testing.T) {
s := testS3Store(t)
ctx := context.Background()
// Add some files
for i := 0; i < 3; i++ {
key := "attachments/ch/" + string(rune('a'+i)) + ".txt"
data := bytes.Repeat([]byte("x"), 100*(i+1))
_ = s.Put(ctx, key, bytes.NewReader(data), int64(len(data)), "text/plain")
}
stats, err := s.Stats(ctx)
if err != nil {
t.Fatalf("Stats: %v", err)
}
if stats.TotalFiles != 3 {
t.Errorf("files = %d, want 3", stats.TotalFiles)
}
if stats.TotalBytes != 600 {
t.Errorf("bytes = %d, want 600", stats.TotalBytes)
}
if !stats.Healthy || !stats.Configured {
t.Error("expected healthy + configured")
}
if stats.Backend != "s3" {
t.Errorf("backend = %q, want s3", stats.Backend)
}
}
func TestS3_Integration_PutUnknownSize(t *testing.T) {
s := testS3Store(t)
ctx := context.Background()
data := []byte("unknown size upload")
key := "attachments/ch/unknown-size.txt"
// size=0 means unknown — S3 should handle via multipart
err := s.Put(ctx, key, bytes.NewReader(data), 0, "text/plain")
if err != nil {
t.Fatalf("Put with size=0: %v", err)
}
rc, size, _, err := s.Get(ctx, key)
if err != nil {
t.Fatalf("Get: %v", err)
}
defer rc.Close()
if size != int64(len(data)) {
t.Errorf("size = %d, want %d", size, len(data))
}
}

73
server/storage/storage.go Normal file
View File

@@ -0,0 +1,73 @@
package storage
import (
"context"
"errors"
"io"
)
// ── Errors ─────────────────────────────────
var (
// ErrNotFound is returned when a requested object does not exist.
ErrNotFound = errors.New("storage: object not found")
// ErrNotConfigured is returned when storage operations are attempted
// but no backend has been configured or the backend is unhealthy.
ErrNotConfigured = errors.New("storage: backend not configured")
)
// ── ObjectStore Interface ──────────────────
// ObjectStore is the abstraction for blob storage.
// Implementations: PVC (filesystem), S3 (minio-go v7).
//
// Keys are slash-delimited paths relative to the storage root:
//
// attachments/{channel_id}/{attachment_id}_{filename}
//
// Implementations must create intermediate directories as needed.
type ObjectStore interface {
// Put writes data to the given key.
// Creates parent directories as needed. Overwrites if exists.
Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error
// Get returns a reader for the given key.
// Returns the reader, size in bytes, content type, and error.
// Caller must close the returned reader.
// Returns ErrNotFound if key does not exist.
Get(ctx context.Context, key string) (io.ReadCloser, int64, string, error)
// Delete removes the object at key.
// No error if the key does not exist (idempotent).
Delete(ctx context.Context, key string) error
// DeletePrefix removes all objects under the given prefix.
// Used for channel deletion (bulk cleanup).
// Example: DeletePrefix(ctx, "attachments/{channel_id}/")
DeletePrefix(ctx context.Context, prefix string) error
// Exists checks if an object exists at key without reading it.
Exists(ctx context.Context, key string) (bool, error)
// Healthy returns nil if the backend is operational (writable).
Healthy(ctx context.Context) error
// Stats returns aggregate storage statistics.
Stats(ctx context.Context) (*StorageStats, error)
// Backend returns the backend type identifier ("pvc", "s3").
Backend() string
}
// StorageStats holds aggregate storage metrics.
type StorageStats struct {
Backend string `json:"backend"`
Path string `json:"path,omitempty"` // PVC only
Endpoint string `json:"endpoint,omitempty"` // S3 only
Bucket string `json:"bucket,omitempty"` // S3 only
Healthy bool `json:"healthy"`
Configured bool `json:"configured"`
TotalFiles int64 `json:"total_files"`
TotalBytes int64 `json:"total_bytes"`
}

19
server/store/interfaces.go
View File

@@ -33,6 +33,7 @@ type Stores struct {
Usage UsageStore
Pricing PricingStore
Extensions ExtensionStore
Attachments AttachmentStore
}
// =========================================
@@ -340,6 +341,24 @@ type ExtensionStore interface {
DeleteUserSettings(ctx context.Context, extID, userID string) error
}
// =========================================
// ATTACHMENT STORE
// =========================================
type AttachmentStore interface {
Create(ctx context.Context, a *models.Attachment) error
GetByID(ctx context.Context, id string) (*models.Attachment, error)
GetByChannel(ctx context.Context, channelID string) ([]models.Attachment, error)
GetByMessage(ctx context.Context, messageID string) ([]models.Attachment, error)
SetMessageID(ctx context.Context, attachmentID, messageID string) error
UpdateMetadata(ctx context.Context, id string, metadata map[string]interface{}) error
SetExtractedText(ctx context.Context, id string, text string) error
Delete(ctx context.Context, id string) (*models.Attachment, error) // returns deleted row for storage cleanup
DeleteByChannel(ctx context.Context, channelID string) ([]string, error) // returns storage_keys
UserUsageBytes(ctx context.Context, userID string) (int64, error)
ListOrphans(ctx context.Context, olderThan time.Duration) ([]models.Attachment, error)
}
// =========================================
// SHARED TYPES
// =========================================

187
server/store/postgres/attachment.go Normal file
View File

@@ -0,0 +1,187 @@
package postgres
import (
"context"
"database/sql"
"encoding/json"
"time"
"git.gobha.me/xcaliber/chat-switchboard/models"
)
type AttachmentStore struct{}
func NewAttachmentStore() *AttachmentStore { return &AttachmentStore{} }
// ── columns shared across queries ──────────
const attachmentCols = `id, channel_id, user_id, message_id, filename, content_type,
size_bytes, storage_key, extracted_text, metadata, created_at`
// scanAttachment scans a row into an Attachment struct.
func scanAttachment(row interface{ Scan(dest ...interface{}) error }) (*models.Attachment, error) {
var a models.Attachment
var messageID sql.NullString
var extractedText sql.NullString
var metadataJSON []byte
err := row.Scan(
&a.ID, &a.ChannelID, &a.UserID, &messageID,
&a.Filename, &a.ContentType, &a.SizeBytes,
&a.StorageKey, &extractedText, &metadataJSON, &a.CreatedAt,
)
if err != nil {
return nil, err
}
a.MessageID = NullableStringPtr(messageID)
if extractedText.Valid {
a.ExtractedText = &extractedText.String
}
if len(metadataJSON) > 0 {
json.Unmarshal(metadataJSON, &a.Metadata)
}
return &a, nil
}
func (s *AttachmentStore) Create(ctx context.Context, a *models.Attachment) error {
return DB.QueryRowContext(ctx, `
INSERT INTO attachments (channel_id, user_id, message_id, filename, content_type,
size_bytes, storage_key, extracted_text, metadata)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
RETURNING id, created_at`,
a.ChannelID, a.UserID, models.NullString(a.MessageID),
a.Filename, a.ContentType, a.SizeBytes,
a.StorageKey, models.NullString(a.ExtractedText),
ToJSON(a.Metadata),
).Scan(&a.ID, &a.CreatedAt)
}
func (s *AttachmentStore) GetByID(ctx context.Context, id string) (*models.Attachment, error) {
row := DB.QueryRowContext(ctx, `SELECT `+attachmentCols+` FROM attachments WHERE id = $1`, id)
return scanAttachment(row)
}
func (s *AttachmentStore) GetByChannel(ctx context.Context, channelID string) ([]models.Attachment, error) {
rows, err := DB.QueryContext(ctx,
`SELECT `+attachmentCols+` FROM attachments WHERE channel_id = $1 ORDER BY created_at`, channelID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []models.Attachment
for rows.Next() {
a, err := scanAttachment(rows)
if err != nil {
return nil, err
}
out = append(out, *a)
}
return out, rows.Err()
}
func (s *AttachmentStore) GetByMessage(ctx context.Context, messageID string) ([]models.Attachment, error) {
rows, err := DB.QueryContext(ctx,
`SELECT `+attachmentCols+` FROM attachments WHERE message_id = $1 ORDER BY created_at`, messageID)
if err != nil {
return nil, err
}
defer rows.Close()
var out []models.Attachment
for rows.Next() {
a, err := scanAttachment(rows)
if err != nil {
return nil, err
}
out = append(out, *a)
}
return out, rows.Err()
}
func (s *AttachmentStore) SetMessageID(ctx context.Context, attachmentID, messageID string) error {
_, err := DB.ExecContext(ctx,
`UPDATE attachments SET message_id = $1 WHERE id = $2`,
messageID, attachmentID)
return err
}
func (s *AttachmentStore) UpdateMetadata(ctx context.Context, id string, metadata map[string]interface{}) error {
// Merge into existing metadata using jsonb || operator
metaJSON, err := json.Marshal(metadata)
if err != nil {
return err
}
_, err = DB.ExecContext(ctx,
`UPDATE attachments SET metadata = metadata || $1::jsonb WHERE id = $2`,
metaJSON, id)
return err
}
func (s *AttachmentStore) SetExtractedText(ctx context.Context, id string, text string) error {
_, err := DB.ExecContext(ctx,
`UPDATE attachments SET extracted_text = $1 WHERE id = $2`,
text, id)
return err
}
// Delete removes an attachment and returns the deleted row (for storage cleanup).
func (s *AttachmentStore) Delete(ctx context.Context, id string) (*models.Attachment, error) {
row := DB.QueryRowContext(ctx,
`DELETE FROM attachments WHERE id = $1
RETURNING `+attachmentCols, id)
return scanAttachment(row)
}
// DeleteByChannel removes all attachments for a channel and returns storage keys.
func (s *AttachmentStore) DeleteByChannel(ctx context.Context, channelID string) ([]string, error) {
rows, err := DB.QueryContext(ctx,
`DELETE FROM attachments WHERE channel_id = $1 RETURNING storage_key`, channelID)
if err != nil {
return nil, err
}
defer rows.Close()
var keys []string
for rows.Next() {
var key string
if err := rows.Scan(&key); err != nil {
return keys, err
}
keys = append(keys, key)
}
return keys, rows.Err()
}
func (s *AttachmentStore) UserUsageBytes(ctx context.Context, userID string) (int64, error) {
var total sql.NullInt64
err := DB.QueryRowContext(ctx,
`SELECT COALESCE(SUM(size_bytes), 0) FROM attachments WHERE user_id = $1`,
userID).Scan(&total)
if err != nil {
return 0, err
}
return total.Int64, nil
}
func (s *AttachmentStore) ListOrphans(ctx context.Context, olderThan time.Duration) ([]models.Attachment, error) {
cutoff := time.Now().Add(-olderThan)
rows, err := DB.QueryContext(ctx,
`SELECT `+attachmentCols+` FROM attachments
WHERE message_id IS NULL AND created_at < $1
ORDER BY created_at`, cutoff)
if err != nil {
return nil, err
}
defer rows.Close()
var out []models.Attachment
for rows.Next() {
a, err := scanAttachment(rows)
if err != nil {
return nil, err
}
out = append(out, *a)
}
return out, rows.Err()
}

1
server/store/postgres/stores.go
View File

@@ -26,5 +26,6 @@ func NewStores(db *sql.DB) store.Stores {
Usage: NewUsageStore(),
Pricing: NewPricingStore(),
Extensions: NewExtensionStore(),
Attachments: NewAttachmentStore(),
}
}