Changeset 0.15.0 (#71)

2026-02-26 21:19:55 +00:00
parent e2149e249d
commit 2d79ff593b
35 changed files with 3218 additions and 504 deletions
--- a/server/compaction/compaction_test.go
+++ b/server/compaction/compaction_test.go
@@ -0,0 +1,401 @@
+package compaction
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+	"time"
+
+	"git.gobha.me/xcaliber/chat-switchboard/database"
+	"git.gobha.me/xcaliber/chat-switchboard/models"
+	postgres "git.gobha.me/xcaliber/chat-switchboard/store/postgres"
+)
+
+// ── Helpers ─────────────────────────────────
+
+func requireDB(t *testing.T) {
+	t.Helper()
+	database.RequireTestDB(t)
+	database.TruncateAll(t)
+}
+
+func seedChannel(t *testing.T, userID, title string, msgCount, contentSize int) (channelID string, msgIDs []string) {
+	t.Helper()
+	channelID = database.SeedTestChannel(t, userID, title)
+	msgIDs = database.SeedTestMessages(t, channelID, msgCount, contentSize)
+	if len(msgIDs) > 0 {
+		database.SeedTestCursor(t, channelID, userID, msgIDs[len(msgIDs)-1])
+	}
+	return
+}
+
+// setGlobalSetting writes a global_settings key for scanner config tests.
+func setGlobalSetting(t *testing.T, key string, value interface{}) {
+	t.Helper()
+	valMap := models.JSONMap{"value": value}
+	valJSON, _ := json.Marshal(valMap)
+	_, err := database.DB.Exec(`
+		INSERT INTO global_settings (key, value) VALUES ($1, $2)
+		ON CONFLICT (key) DO UPDATE SET value = $2
+	`, key, string(valJSON))
+	if err != nil {
+		t.Fatalf("setGlobalSetting(%s): %v", key, err)
+	}
+}
+
+// touchChannelTime backdates a channel's updated_at.
+func touchChannelTime(t *testing.T, channelID string, age time.Duration) {
+	t.Helper()
+	target := time.Now().Add(-age)
+	_, err := database.DB.Exec(`UPDATE channels SET updated_at = $1 WHERE id = $2`, target, channelID)
+	if err != nil {
+		t.Fatalf("touchChannelTime: %v", err)
+	}
+}
+
+// setChannelSettings sets the channel.settings JSONB.
+func setChannelSettings(t *testing.T, channelID string, settings models.JSONMap) {
+	t.Helper()
+	sJSON, _ := json.Marshal(settings)
+	_, err := database.DB.Exec(`UPDATE channels SET settings = $1 WHERE id = $2`, string(sJSON), channelID)
+	if err != nil {
+		t.Fatalf("setChannelSettings: %v", err)
+	}
+}
+
+// ═══════════════════════════════════════════
+// Estimator Tests (no DB — duplicated here for package-level access)
+// ═══════════════════════════════════════════
+// Note: estimator_test.go covers the unit tests more thoroughly.
+// These are here to verify the package compiles with DB tests.
+
+func TestEstimateTokens_Basic(t *testing.T) {
+	if got := EstimateTokens("hello world!"); got != 3 { // 12 chars → 3
+		t.Errorf("EstimateTokens = %d, want 3", got)
+	}
+}
+
+// ═══════════════════════════════════════════
+// Scanner: Candidate Query
+// ═══════════════════════════════════════════
+
+func TestScanner_FindCandidates_ReturnsQualifying(t *testing.T) {
+	requireDB(t)
+	stores := postgres.NewStores(database.TestDB)
+	svc := NewService(stores, nil) // no resolver needed for this test
+	sc := NewScanner(svc, stores, ScannerConfig{})
+
+	userID := database.SeedTestUser(t, "scanuser1", "scan1@test.com")
+
+	// Create a channel with enough messages to qualify
+	// 20 messages × 2000 chars = 40K chars (> candidateMinChars=20K)
+	channelID, _ := seedChannel(t, userID, "Big Chat", 20, 2000)
+
+	// Backdate so it passes the activity gap (>2min old) and recency (<7 days)
+	touchChannelTime(t, channelID, 5*time.Minute)
+
+	ctx := context.Background()
+	candidates := sc.findCandidates(ctx)
+
+	if len(candidates) == 0 {
+		t.Fatal("expected at least 1 candidate, got 0")
+	}
+
+	found := false
+	for _, c := range candidates {
+		if c.ID == channelID {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Fatalf("channel %s not in candidates (got %d candidates)", channelID, len(candidates))
+	}
+}
+
+func TestScanner_FindCandidates_ExcludesTooFewMessages(t *testing.T) {
+	requireDB(t)
+	stores := postgres.NewStores(database.TestDB)
+	svc := NewService(stores, nil)
+	sc := NewScanner(svc, stores, ScannerConfig{})
+
+	userID := database.SeedTestUser(t, "scanuser2", "scan2@test.com")
+
+	// Only 4 messages — below candidateMinMessages=10
+	channelID, _ := seedChannel(t, userID, "Small Chat", 4, 2000)
+	touchChannelTime(t, channelID, 5*time.Minute)
+
+	ctx := context.Background()
+	candidates := sc.findCandidates(ctx)
+
+	for _, c := range candidates {
+		if c.ID == channelID {
+			t.Fatal("channel with <10 messages should not be a candidate")
+		}
+	}
+}
+
+func TestScanner_FindCandidates_ExcludesTooRecent(t *testing.T) {
+	requireDB(t)
+	stores := postgres.NewStores(database.TestDB)
+	svc := NewService(stores, nil)
+	sc := NewScanner(svc, stores, ScannerConfig{})
+
+	userID := database.SeedTestUser(t, "scanuser3", "scan3@test.com")
+
+	// Enough messages but updated just now (< candidateActivityGap=2min)
+	seedChannel(t, userID, "Fresh Chat", 20, 2000)
+	// Don't backdate — should be excluded
+
+	ctx := context.Background()
+	candidates := sc.findCandidates(ctx)
+
+	for _, c := range candidates {
+		if c.UserID == userID {
+			t.Fatal("channel updated just now should not be a candidate (activity gap)")
+		}
+	}
+}
+
+func TestScanner_FindCandidates_ExcludesArchived(t *testing.T) {
+	requireDB(t)
+	stores := postgres.NewStores(database.TestDB)
+	svc := NewService(stores, nil)
+	sc := NewScanner(svc, stores, ScannerConfig{})
+
+	userID := database.SeedTestUser(t, "scanuser4", "scan4@test.com")
+	channelID, _ := seedChannel(t, userID, "Archived Chat", 20, 2000)
+	touchChannelTime(t, channelID, 5*time.Minute)
+
+	// Archive the channel
+	database.DB.Exec(`UPDATE channels SET is_archived = true WHERE id = $1`, channelID)
+
+	ctx := context.Background()
+	candidates := sc.findCandidates(ctx)
+
+	for _, c := range candidates {
+		if c.ID == channelID {
+			t.Fatal("archived channel should not be a candidate")
+		}
+	}
+}
+
+// ═══════════════════════════════════════════
+// Scanner: Cooldown + Dedup
+// ═══════════════════════════════════════════
+
+func TestScanner_Cooldown(t *testing.T) {
+	sc := &Scanner{
+		lastCompacted: make(map[string]time.Time),
+	}
+
+	channelID := "test-cooldown-channel"
+
+	// No cooldown initially
+	sc.mu.Lock()
+	_, inCooldown := sc.lastCompacted[channelID]
+	sc.mu.Unlock()
+	if inCooldown {
+		t.Fatal("should not be in cooldown initially")
+	}
+
+	// Record compaction
+	sc.mu.Lock()
+	sc.lastCompacted[channelID] = time.Now()
+	sc.mu.Unlock()
+
+	// Now it's in cooldown
+	sc.mu.Lock()
+	last, ok := sc.lastCompacted[channelID]
+	sc.mu.Unlock()
+	if !ok {
+		t.Fatal("should be in cooldown after recording")
+	}
+	if time.Since(last) > time.Second {
+		t.Fatal("cooldown timestamp should be recent")
+	}
+}
+
+func TestScanner_InFlightDedup(t *testing.T) {
+	sc := &Scanner{}
+	// sync.Map zero value is ready to use
+
+	channelID := "test-dedup-channel"
+
+	// First load — not in flight
+	_, loaded := sc.inFlight.LoadOrStore(channelID, struct{}{})
+	if loaded {
+		t.Fatal("should not be loaded on first attempt")
+	}
+
+	// Second load — already in flight
+	_, loaded = sc.inFlight.LoadOrStore(channelID, struct{}{})
+	if !loaded {
+		t.Fatal("should be loaded on second attempt (dedup)")
+	}
+
+	// Clean up
+	sc.inFlight.Delete(channelID)
+	_, loaded = sc.inFlight.LoadOrStore(channelID, struct{}{})
+	if loaded {
+		t.Fatal("should not be loaded after delete")
+	}
+}
+
+// ═══════════════════════════════════════════
+// Scanner: Channel Opt-Out
+// ═══════════════════════════════════════════
+
+func TestScanner_ShouldCompact_ChannelOptOut(t *testing.T) {
+	requireDB(t)
+	stores := postgres.NewStores(database.TestDB)
+	svc := NewService(stores, nil) // no resolver → IsConfigured returns false
+	sc := NewScanner(svc, stores, ScannerConfig{})
+
+	userID := database.SeedTestUser(t, "optout_user", "optout@test.com")
+	channelID, _ := seedChannel(t, userID, "Opt-Out Chat", 20, 2000)
+
+	// Opt out via channel settings
+	setChannelSettings(t, channelID, models.JSONMap{"auto_compaction": false})
+
+	ch := models.Channel{Settings: models.JSONMap{"auto_compaction": false}}
+	ch.ID = channelID
+	ch.UserID = userID
+
+	ctx := context.Background()
+	if sc.shouldCompact(ctx, &ch) {
+		t.Fatal("shouldCompact should return false for opted-out channel")
+	}
+}
+
+// ═══════════════════════════════════════════
+// Scanner: Settings Helpers
+// ═══════════════════════════════════════════
+
+func TestScanner_IsEnabled(t *testing.T) {
+	requireDB(t)
+	stores := postgres.NewStores(database.TestDB)
+	svc := NewService(stores, nil)
+	sc := NewScanner(svc, stores, ScannerConfig{})
+	ctx := context.Background()
+
+	// Default: disabled
+	if sc.isEnabled(ctx) {
+		t.Fatal("should be disabled by default")
+	}
+
+	// Enable
+	setGlobalSetting(t, "auto_compaction_enabled", true)
+	if !sc.isEnabled(ctx) {
+		t.Fatal("should be enabled after setting to true")
+	}
+
+	// Disable again
+	setGlobalSetting(t, "auto_compaction_enabled", false)
+	if sc.isEnabled(ctx) {
+		t.Fatal("should be disabled after setting to false")
+	}
+}
+
+func TestScanner_GetThreshold_Default(t *testing.T) {
+	requireDB(t)
+	stores := postgres.NewStores(database.TestDB)
+	svc := NewService(stores, nil)
+	sc := NewScanner(svc, stores, ScannerConfig{})
+
+	ch := &models.Channel{}
+	got := sc.getThreshold(ch)
+	if got != DefaultThreshold {
+		t.Errorf("default threshold = %f, want %f", got, DefaultThreshold)
+	}
+}
+
+func TestScanner_GetThreshold_GlobalOverride(t *testing.T) {
+	requireDB(t)
+	stores := postgres.NewStores(database.TestDB)
+	svc := NewService(stores, nil)
+	sc := NewScanner(svc, stores, ScannerConfig{})
+
+	setGlobalSetting(t, "auto_compaction_threshold", 0.85)
+
+	ch := &models.Channel{}
+	got := sc.getThreshold(ch)
+	if got != 0.85 {
+		t.Errorf("global threshold = %f, want 0.85", got)
+	}
+}
+
+func TestScanner_GetThreshold_ChannelOverride(t *testing.T) {
+	requireDB(t)
+	stores := postgres.NewStores(database.TestDB)
+	svc := NewService(stores, nil)
+	sc := NewScanner(svc, stores, ScannerConfig{})
+
+	setGlobalSetting(t, "auto_compaction_threshold", 0.85)
+
+	ch := &models.Channel{
+		Settings: models.JSONMap{"compaction_threshold": 0.60},
+	}
+	got := sc.getThreshold(ch)
+	if got != 0.60 {
+		t.Errorf("channel threshold = %f, want 0.60", got)
+	}
+}
+
+func TestScanner_GetCooldownDuration_Default(t *testing.T) {
+	requireDB(t)
+	stores := postgres.NewStores(database.TestDB)
+	svc := NewService(stores, nil)
+	sc := NewScanner(svc, stores, ScannerConfig{})
+	ctx := context.Background()
+
+	got := sc.getCooldownDuration(ctx)
+	if got != DefaultCooldown {
+		t.Errorf("default cooldown = %s, want %s", got, DefaultCooldown)
+	}
+}
+
+func TestScanner_GetCooldownDuration_Override(t *testing.T) {
+	requireDB(t)
+	stores := postgres.NewStores(database.TestDB)
+	svc := NewService(stores, nil)
+	sc := NewScanner(svc, stores, ScannerConfig{})
+	ctx := context.Background()
+
+	setGlobalSetting(t, "auto_compaction_cooldown_minutes", float64(15))
+
+	got := sc.getCooldownDuration(ctx)
+	want := 15 * time.Minute
+	if got != want {
+		t.Errorf("cooldown = %s, want %s", got, want)
+	}
+}
+
+// ═══════════════════════════════════════════
+// Context Budget Guard Rail
+// ═══════════════════════════════════════════
+
+func TestEstimateTokens_GuardRailMath(t *testing.T) {
+	// Simulate a large conversation that exceeds a 32K utility model
+	// 100K chars ≈ 25K tokens of conversation + system prompt overhead
+	largeContent := make([]byte, 100000)
+	contentTokens := EstimateTokens(string(largeContent))
+	systemTokens := EstimateTokens("You are a conversation summarizer...") + 8 // +overhead
+
+	totalPrompt := contentTokens + systemTokens
+	utilityBudget := 32000 // 32K model
+	inputCeiling := int(float64(utilityBudget) * 0.80)
+
+	if totalPrompt <= inputCeiling {
+		t.Errorf("100K chars (%d tokens) should exceed 32K model ceiling (%d tokens)",
+			totalPrompt, inputCeiling)
+	}
+
+	// Smaller conversation should fit
+	smallContent := make([]byte, 50000)
+	smallTokens := EstimateTokens(string(smallContent)) + systemTokens
+	if smallTokens > inputCeiling {
+		t.Errorf("50K chars (%d tokens) should fit in 32K model ceiling (%d tokens)",
+			smallTokens, inputCeiling)
+	}
+}