Changeset 0.12.0 (#63)

2026-02-25 21:38:49 +00:00
parent c9d8e9457e
commit 88216ec4cb
59 changed files with 13115 additions and 139 deletions
--- a/server/extraction/queue.go
+++ b/server/extraction/queue.go
@@ -0,0 +1,294 @@
+package extraction
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+)
+
+// ── Status Constants ───────────────────────
+
+const (
+	StatusPending    = "pending"
+	StatusProcessing = "processing"
+	StatusComplete   = "complete"
+	StatusFailed     = "failed"
+	StatusSkipped    = "skipped" // not extractable (images, etc.)
+)
+
+// ── Queue Item ─────────────────────────────
+
+// QueueItem is the status.json written to the processing directory.
+// The sidecar extractor watches this directory for pending items.
+type QueueItem struct {
+	AttachmentID string `json:"attachment_id"`
+	StorageKey   string `json:"storage_key"`
+	ContentType  string `json:"content_type"`
+	Filename     string `json:"filename"`
+	Status       string `json:"status"`
+	Error        string `json:"error,omitempty"`
+	OutputKey    string `json:"output_key,omitempty"` // path to extracted text
+	QueuedAt     string `json:"queued_at"`
+	StartedAt    string `json:"started_at,omitempty"`
+	CompletedAt  string `json:"completed_at,omitempty"`
+}
+
+// ── Queue Manager ──────────────────────────
+
+// Queue manages the filesystem-based extraction queue.
+// Processing state lives in {storagePath}/processing/{attachment_id}/status.json.
+// The sidecar extractor watches for pending items and updates status.
+type Queue struct {
+	storagePath string
+	concurrency int
+	sem         chan struct{} // semaphore for concurrent extraction limit
+	mu          sync.Mutex
+}
+
+// NewQueue creates a new extraction queue manager.
+// storagePath is the PVC mount (e.g., /data/storage).
+// concurrency limits parallel extractions (default 3).
+func NewQueue(storagePath string, concurrency int) (*Queue, error) {
+	if concurrency <= 0 {
+		concurrency = 3
+	}
+
+	procDir := filepath.Join(storagePath, "processing")
+	if err := os.MkdirAll(procDir, 0750); err != nil {
+		return nil, fmt.Errorf("extraction: create processing dir: %w", err)
+	}
+
+	return &Queue{
+		storagePath: storagePath,
+		concurrency: concurrency,
+		sem:         make(chan struct{}, concurrency),
+	}, nil
+}
+
+// Enqueue adds an attachment to the extraction queue.
+// Creates {storagePath}/processing/{id}/status.json with status "pending".
+func (q *Queue) Enqueue(attachmentID, storageKey, contentType, filename string) error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	itemDir := filepath.Join(q.storagePath, "processing", attachmentID)
+	if err := os.MkdirAll(itemDir, 0750); err != nil {
+		return fmt.Errorf("extraction: create item dir: %w", err)
+	}
+
+	item := QueueItem{
+		AttachmentID: attachmentID,
+		StorageKey:   storageKey,
+		ContentType:  contentType,
+		Filename:     filename,
+		Status:       StatusPending,
+		QueuedAt:     time.Now().UTC().Format(time.RFC3339),
+	}
+
+	return q.writeStatus(attachmentID, &item)
+}
+
+// GetStatus reads the current extraction status for an attachment.
+func (q *Queue) GetStatus(attachmentID string) (*QueueItem, error) {
+	statusPath := filepath.Join(q.storagePath, "processing", attachmentID, "status.json")
+	data, err := os.ReadFile(statusPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil // not queued
+		}
+		return nil, fmt.Errorf("extraction: read status: %w", err)
+	}
+
+	var item QueueItem
+	if err := json.Unmarshal(data, &item); err != nil {
+		return nil, fmt.Errorf("extraction: parse status: %w", err)
+	}
+	return &item, nil
+}
+
+// MarkComplete updates status to complete and records the output key.
+func (q *Queue) MarkComplete(attachmentID, outputKey string) error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	item, err := q.GetStatus(attachmentID)
+	if err != nil || item == nil {
+		return fmt.Errorf("extraction: item not found: %s", attachmentID)
+	}
+
+	item.Status = StatusComplete
+	item.OutputKey = outputKey
+	item.CompletedAt = time.Now().UTC().Format(time.RFC3339)
+	return q.writeStatus(attachmentID, item)
+}
+
+// MarkFailed updates status to failed with an error message.
+func (q *Queue) MarkFailed(attachmentID, errMsg string) error {
+	q.mu.Lock()
+	defer q.mu.Unlock()
+
+	item, err := q.GetStatus(attachmentID)
+	if err != nil || item == nil {
+		return fmt.Errorf("extraction: item not found: %s", attachmentID)
+	}
+
+	item.Status = StatusFailed
+	item.Error = errMsg
+	item.CompletedAt = time.Now().UTC().Format(time.RFC3339)
+	return q.writeStatus(attachmentID, item)
+}
+
+// Cleanup removes the processing directory for a completed/failed item.
+func (q *Queue) Cleanup(attachmentID string) error {
+	itemDir := filepath.Join(q.storagePath, "processing", attachmentID)
+	return os.RemoveAll(itemDir)
+}
+
+// RecoverStale scans for items stuck in "processing" state (crash recovery).
+// Items older than maxAge are reset to "pending" for re-processing.
+func (q *Queue) RecoverStale(maxAge time.Duration) (int, error) {
+	procDir := filepath.Join(q.storagePath, "processing")
+	entries, err := os.ReadDir(procDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return 0, nil
+		}
+		return 0, err
+	}
+
+	recovered := 0
+	cutoff := time.Now().Add(-maxAge)
+
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+		item, err := q.GetStatus(entry.Name())
+		if err != nil || item == nil {
+			continue
+		}
+
+		if item.Status != StatusProcessing {
+			continue
+		}
+
+		// Check if started_at is older than maxAge
+		if item.StartedAt != "" {
+			startedAt, err := time.Parse(time.RFC3339, item.StartedAt)
+			if err == nil && startedAt.Before(cutoff) {
+				q.mu.Lock()
+				item.Status = StatusPending
+				item.StartedAt = ""
+				item.Error = "recovered from stale processing state"
+				q.writeStatus(entry.Name(), item)
+				q.mu.Unlock()
+				recovered++
+				log.Printf("extraction: recovered stale item %s", entry.Name())
+			}
+		}
+	}
+
+	return recovered, nil
+}
+
+// ListPending returns all items with status "pending".
+func (q *Queue) ListPending() ([]QueueItem, error) {
+	return q.listByStatus(StatusPending)
+}
+
+// ListAll returns all items in the processing directory.
+func (q *Queue) ListAll() ([]QueueItem, error) {
+	procDir := filepath.Join(q.storagePath, "processing")
+	entries, err := os.ReadDir(procDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	var items []QueueItem
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+		item, err := q.GetStatus(entry.Name())
+		if err != nil || item == nil {
+			continue
+		}
+		items = append(items, *item)
+	}
+	return items, nil
+}
+
+// ── Internal Helpers ───────────────────────
+
+func (q *Queue) writeStatus(attachmentID string, item *QueueItem) error {
+	statusPath := filepath.Join(q.storagePath, "processing", attachmentID, "status.json")
+	data, err := json.MarshalIndent(item, "", "  ")
+	if err != nil {
+		return err
+	}
+
+	// Atomic write: temp + rename
+	tmpPath := statusPath + ".tmp"
+	if err := os.WriteFile(tmpPath, data, 0640); err != nil {
+		return err
+	}
+	return os.Rename(tmpPath, statusPath)
+}
+
+func (q *Queue) listByStatus(status string) ([]QueueItem, error) {
+	all, err := q.ListAll()
+	if err != nil {
+		return nil, err
+	}
+
+	var filtered []QueueItem
+	for _, item := range all {
+		if item.Status == status {
+			filtered = append(filtered, item)
+		}
+	}
+	return filtered, nil
+}
+
+// ── Extractable Check ──────────────────────
+
+// IsExtractable returns true if the content type supports text extraction.
+// Images and plain text do not need extraction (handled inline by upload handler).
+var extractableTypes = map[string]bool{
+	"application/pdf": true,
+	"application/vnd.openxmlformats-officedocument.wordprocessingml.document":   true,
+	"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet":         true,
+	"application/vnd.openxmlformats-officedocument.presentationml.presentation": true,
+	"application/msword":        true,
+	"application/vnd.ms-excel":  true,
+	"application/vnd.oasis.opendocument.text":         true,
+	"application/vnd.oasis.opendocument.spreadsheet":  true,
+	"application/vnd.oasis.opendocument.presentation": true,
+	"application/rtf": true,
+}
+
+// IsExtractable returns true if the given content type requires sidecar extraction.
+func IsExtractable(contentType string) bool {
+	return extractableTypes[contentType]
+}
+
+// ── Null Queue ─────────────────────────────
+
+// NullQueue is a no-op implementation for when extraction is disabled.
+type NullQueue struct{}
+
+func (NullQueue) Enqueue(_, _, _, _ string) error          { return nil }
+func (NullQueue) GetStatus(_ string) (*QueueItem, error)   { return nil, nil }
+func (NullQueue) MarkComplete(_, _ string) error           { return nil }
+func (NullQueue) MarkFailed(_, _ string) error             { return nil }
+func (NullQueue) Cleanup(_ string) error                   { return nil }
+func (NullQueue) RecoverStale(_ time.Duration) (int, error) { return 0, nil }
+func (NullQueue) ListPending() ([]QueueItem, error)        { return nil, nil }
+func (NullQueue) ListAll() ([]QueueItem, error)            { return nil, nil }
--- a/server/extraction/queue_test.go
+++ b/server/extraction/queue_test.go
@@ -0,0 +1,240 @@
+package extraction
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func tempQueue(t *testing.T) *Queue {
+	t.Helper()
+	dir := t.TempDir()
+	q, err := NewQueue(dir, 3)
+	if err != nil {
+		t.Fatalf("NewQueue: %v", err)
+	}
+	return q
+}
+
+func TestEnqueue_CreatesStatusFile(t *testing.T) {
+	q := tempQueue(t)
+
+	err := q.Enqueue("att-123", "attachments/ch/att-123_doc.pdf", "application/pdf", "doc.pdf")
+	if err != nil {
+		t.Fatalf("Enqueue: %v", err)
+	}
+
+	item, err := q.GetStatus("att-123")
+	if err != nil {
+		t.Fatalf("GetStatus: %v", err)
+	}
+	if item == nil {
+		t.Fatal("expected non-nil item")
+	}
+	if item.Status != StatusPending {
+		t.Errorf("status = %q, want %q", item.Status, StatusPending)
+	}
+	if item.AttachmentID != "att-123" {
+		t.Errorf("attachment_id = %q, want att-123", item.AttachmentID)
+	}
+	if item.ContentType != "application/pdf" {
+		t.Errorf("content_type = %q, want application/pdf", item.ContentType)
+	}
+}
+
+func TestGetStatus_NotQueued(t *testing.T) {
+	q := tempQueue(t)
+
+	item, err := q.GetStatus("nonexistent")
+	if err != nil {
+		t.Fatalf("GetStatus: %v", err)
+	}
+	if item != nil {
+		t.Error("expected nil for non-queued item")
+	}
+}
+
+func TestMarkComplete(t *testing.T) {
+	q := tempQueue(t)
+	q.Enqueue("att-456", "key", "application/pdf", "test.pdf")
+
+	err := q.MarkComplete("att-456", "processing/att-456/extracted.txt")
+	if err != nil {
+		t.Fatalf("MarkComplete: %v", err)
+	}
+
+	item, _ := q.GetStatus("att-456")
+	if item.Status != StatusComplete {
+		t.Errorf("status = %q, want %q", item.Status, StatusComplete)
+	}
+	if item.OutputKey != "processing/att-456/extracted.txt" {
+		t.Errorf("output_key = %q", item.OutputKey)
+	}
+	if item.CompletedAt == "" {
+		t.Error("completed_at should be set")
+	}
+}
+
+func TestMarkFailed(t *testing.T) {
+	q := tempQueue(t)
+	q.Enqueue("att-789", "key", "application/pdf", "test.pdf")
+
+	err := q.MarkFailed("att-789", "libreoffice crashed")
+	if err != nil {
+		t.Fatalf("MarkFailed: %v", err)
+	}
+
+	item, _ := q.GetStatus("att-789")
+	if item.Status != StatusFailed {
+		t.Errorf("status = %q, want %q", item.Status, StatusFailed)
+	}
+	if item.Error != "libreoffice crashed" {
+		t.Errorf("error = %q", item.Error)
+	}
+}
+
+func TestCleanup_RemovesDirectory(t *testing.T) {
+	q := tempQueue(t)
+	q.Enqueue("att-cleanup", "key", "application/pdf", "test.pdf")
+
+	// Verify directory exists
+	dir := filepath.Join(q.storagePath, "processing", "att-cleanup")
+	if _, err := os.Stat(dir); err != nil {
+		t.Fatal("expected processing dir to exist before cleanup")
+	}
+
+	if err := q.Cleanup("att-cleanup"); err != nil {
+		t.Fatalf("Cleanup: %v", err)
+	}
+
+	if _, err := os.Stat(dir); !os.IsNotExist(err) {
+		t.Error("expected processing dir to be removed after cleanup")
+	}
+}
+
+func TestListPending(t *testing.T) {
+	q := tempQueue(t)
+	q.Enqueue("att-a", "key-a", "application/pdf", "a.pdf")
+	q.Enqueue("att-b", "key-b", "application/pdf", "b.pdf")
+	q.MarkComplete("att-b", "out")
+
+	pending, err := q.ListPending()
+	if err != nil {
+		t.Fatalf("ListPending: %v", err)
+	}
+	if len(pending) != 1 {
+		t.Fatalf("expected 1 pending, got %d", len(pending))
+	}
+	if pending[0].AttachmentID != "att-a" {
+		t.Errorf("pending[0].AttachmentID = %q, want att-a", pending[0].AttachmentID)
+	}
+}
+
+func TestListAll(t *testing.T) {
+	q := tempQueue(t)
+	q.Enqueue("att-1", "key-1", "application/pdf", "1.pdf")
+	q.Enqueue("att-2", "key-2", "application/pdf", "2.pdf")
+	q.Enqueue("att-3", "key-3", "application/pdf", "3.pdf")
+
+	all, err := q.ListAll()
+	if err != nil {
+		t.Fatalf("ListAll: %v", err)
+	}
+	if len(all) != 3 {
+		t.Errorf("expected 3 items, got %d", len(all))
+	}
+}
+
+func TestRecoverStale(t *testing.T) {
+	q := tempQueue(t)
+
+	// Create item and manually set it to "processing" with old timestamp
+	q.Enqueue("att-stale", "key", "application/pdf", "stale.pdf")
+	item, _ := q.GetStatus("att-stale")
+	item.Status = StatusProcessing
+	item.StartedAt = time.Now().Add(-2 * time.Hour).UTC().Format(time.RFC3339)
+	q.mu.Lock()
+	q.writeStatus("att-stale", item)
+	q.mu.Unlock()
+
+	// Recover items stale for > 1 hour
+	recovered, err := q.RecoverStale(1 * time.Hour)
+	if err != nil {
+		t.Fatalf("RecoverStale: %v", err)
+	}
+	if recovered != 1 {
+		t.Errorf("recovered = %d, want 1", recovered)
+	}
+
+	// Verify it's back to pending
+	item, _ = q.GetStatus("att-stale")
+	if item.Status != StatusPending {
+		t.Errorf("status = %q, want %q", item.Status, StatusPending)
+	}
+}
+
+func TestRecoverStale_SkipsRecent(t *testing.T) {
+	q := tempQueue(t)
+
+	// Create item "processing" but recent (not stale)
+	q.Enqueue("att-recent", "key", "application/pdf", "recent.pdf")
+	item, _ := q.GetStatus("att-recent")
+	item.Status = StatusProcessing
+	item.StartedAt = time.Now().Add(-5 * time.Minute).UTC().Format(time.RFC3339)
+	q.mu.Lock()
+	q.writeStatus("att-recent", item)
+	q.mu.Unlock()
+
+	recovered, err := q.RecoverStale(1 * time.Hour)
+	if err != nil {
+		t.Fatalf("RecoverStale: %v", err)
+	}
+	if recovered != 0 {
+		t.Errorf("recovered = %d, want 0 (item is recent)", recovered)
+	}
+}
+
+func TestIsExtractable(t *testing.T) {
+	tests := []struct {
+		contentType string
+		want        bool
+	}{
+		{"application/pdf", true},
+		{"application/vnd.openxmlformats-officedocument.wordprocessingml.document", true},
+		{"image/png", false},
+		{"text/plain", false},
+		{"image/jpeg", false},
+		{"application/rtf", true},
+	}
+
+	for _, tc := range tests {
+		got := IsExtractable(tc.contentType)
+		if got != tc.want {
+			t.Errorf("IsExtractable(%q) = %v, want %v", tc.contentType, got, tc.want)
+		}
+	}
+}
+
+func TestAtomicWrite(t *testing.T) {
+	q := tempQueue(t)
+
+	// Enqueue and verify the file is valid JSON
+	q.Enqueue("att-atomic", "key", "application/pdf", "test.pdf")
+
+	statusPath := filepath.Join(q.storagePath, "processing", "att-atomic", "status.json")
+	data, err := os.ReadFile(statusPath)
+	if err != nil {
+		t.Fatalf("ReadFile: %v", err)
+	}
+
+	// Verify no .tmp file left behind
+	tmpPath := statusPath + ".tmp"
+	if _, err := os.Stat(tmpPath); !os.IsNotExist(err) {
+		t.Error("temp file should not exist after atomic write")
+	}
+
+	if len(data) == 0 {
+		t.Error("status.json should not be empty")
+	}
+}