402 lines
12 KiB
Go
402 lines
12 KiB
Go
package compaction
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"testing"
|
||
"time"
|
||
|
||
"git.gobha.me/xcaliber/chat-switchboard/database"
|
||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||
postgres "git.gobha.me/xcaliber/chat-switchboard/store/postgres"
|
||
)
|
||
|
||
// ── Helpers ─────────────────────────────────
|
||
|
||
func requireDB(t *testing.T) {
|
||
t.Helper()
|
||
database.RequireTestDB(t)
|
||
database.TruncateAll(t)
|
||
}
|
||
|
||
func seedChannel(t *testing.T, userID, title string, msgCount, contentSize int) (channelID string, msgIDs []string) {
|
||
t.Helper()
|
||
channelID = database.SeedTestChannel(t, userID, title)
|
||
msgIDs = database.SeedTestMessages(t, channelID, msgCount, contentSize)
|
||
if len(msgIDs) > 0 {
|
||
database.SeedTestCursor(t, channelID, userID, msgIDs[len(msgIDs)-1])
|
||
}
|
||
return
|
||
}
|
||
|
||
// setGlobalSetting writes a global_settings key for scanner config tests.
|
||
func setGlobalSetting(t *testing.T, key string, value interface{}) {
|
||
t.Helper()
|
||
valMap := models.JSONMap{"value": value}
|
||
valJSON, _ := json.Marshal(valMap)
|
||
_, err := database.DB.Exec(`
|
||
INSERT INTO global_settings (key, value) VALUES ($1, $2)
|
||
ON CONFLICT (key) DO UPDATE SET value = $2
|
||
`, key, string(valJSON))
|
||
if err != nil {
|
||
t.Fatalf("setGlobalSetting(%s): %v", key, err)
|
||
}
|
||
}
|
||
|
||
// touchChannelTime backdates a channel's updated_at.
|
||
func touchChannelTime(t *testing.T, channelID string, age time.Duration) {
|
||
t.Helper()
|
||
target := time.Now().Add(-age)
|
||
_, err := database.DB.Exec(`UPDATE channels SET updated_at = $1 WHERE id = $2`, target, channelID)
|
||
if err != nil {
|
||
t.Fatalf("touchChannelTime: %v", err)
|
||
}
|
||
}
|
||
|
||
// setChannelSettings sets the channel.settings JSONB.
|
||
func setChannelSettings(t *testing.T, channelID string, settings models.JSONMap) {
|
||
t.Helper()
|
||
sJSON, _ := json.Marshal(settings)
|
||
_, err := database.DB.Exec(`UPDATE channels SET settings = $1 WHERE id = $2`, string(sJSON), channelID)
|
||
if err != nil {
|
||
t.Fatalf("setChannelSettings: %v", err)
|
||
}
|
||
}
|
||
|
||
// ═══════════════════════════════════════════
|
||
// Estimator Tests (no DB — duplicated here for package-level access)
|
||
// ═══════════════════════════════════════════
|
||
// Note: estimator_test.go covers the unit tests more thoroughly.
|
||
// These are here to verify the package compiles with DB tests.
|
||
|
||
func TestEstimateTokens_Basic(t *testing.T) {
|
||
if got := EstimateTokens("hello world!"); got != 3 { // 12 chars → 3
|
||
t.Errorf("EstimateTokens = %d, want 3", got)
|
||
}
|
||
}
|
||
|
||
// ═══════════════════════════════════════════
|
||
// Scanner: Candidate Query
|
||
// ═══════════════════════════════════════════
|
||
|
||
func TestScanner_FindCandidates_ReturnsQualifying(t *testing.T) {
|
||
requireDB(t)
|
||
stores := postgres.NewStores(database.TestDB)
|
||
svc := NewService(stores, nil) // no resolver needed for this test
|
||
sc := NewScanner(svc, stores, ScannerConfig{})
|
||
|
||
userID := database.SeedTestUser(t, "scanuser1", "scan1@test.com")
|
||
|
||
// Create a channel with enough messages to qualify
|
||
// 20 messages × 2000 chars = 40K chars (> candidateMinChars=20K)
|
||
channelID, _ := seedChannel(t, userID, "Big Chat", 20, 2000)
|
||
|
||
// Backdate so it passes the activity gap (>2min old) and recency (<7 days)
|
||
touchChannelTime(t, channelID, 5*time.Minute)
|
||
|
||
ctx := context.Background()
|
||
candidates := sc.findCandidates(ctx)
|
||
|
||
if len(candidates) == 0 {
|
||
t.Fatal("expected at least 1 candidate, got 0")
|
||
}
|
||
|
||
found := false
|
||
for _, c := range candidates {
|
||
if c.ID == channelID {
|
||
found = true
|
||
break
|
||
}
|
||
}
|
||
if !found {
|
||
t.Fatalf("channel %s not in candidates (got %d candidates)", channelID, len(candidates))
|
||
}
|
||
}
|
||
|
||
func TestScanner_FindCandidates_ExcludesTooFewMessages(t *testing.T) {
|
||
requireDB(t)
|
||
stores := postgres.NewStores(database.TestDB)
|
||
svc := NewService(stores, nil)
|
||
sc := NewScanner(svc, stores, ScannerConfig{})
|
||
|
||
userID := database.SeedTestUser(t, "scanuser2", "scan2@test.com")
|
||
|
||
// Only 4 messages — below candidateMinMessages=10
|
||
channelID, _ := seedChannel(t, userID, "Small Chat", 4, 2000)
|
||
touchChannelTime(t, channelID, 5*time.Minute)
|
||
|
||
ctx := context.Background()
|
||
candidates := sc.findCandidates(ctx)
|
||
|
||
for _, c := range candidates {
|
||
if c.ID == channelID {
|
||
t.Fatal("channel with <10 messages should not be a candidate")
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestScanner_FindCandidates_ExcludesTooRecent(t *testing.T) {
|
||
requireDB(t)
|
||
stores := postgres.NewStores(database.TestDB)
|
||
svc := NewService(stores, nil)
|
||
sc := NewScanner(svc, stores, ScannerConfig{})
|
||
|
||
userID := database.SeedTestUser(t, "scanuser3", "scan3@test.com")
|
||
|
||
// Enough messages but updated just now (< candidateActivityGap=2min)
|
||
seedChannel(t, userID, "Fresh Chat", 20, 2000)
|
||
// Don't backdate — should be excluded
|
||
|
||
ctx := context.Background()
|
||
candidates := sc.findCandidates(ctx)
|
||
|
||
for _, c := range candidates {
|
||
if c.UserID == userID {
|
||
t.Fatal("channel updated just now should not be a candidate (activity gap)")
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestScanner_FindCandidates_ExcludesArchived(t *testing.T) {
|
||
requireDB(t)
|
||
stores := postgres.NewStores(database.TestDB)
|
||
svc := NewService(stores, nil)
|
||
sc := NewScanner(svc, stores, ScannerConfig{})
|
||
|
||
userID := database.SeedTestUser(t, "scanuser4", "scan4@test.com")
|
||
channelID, _ := seedChannel(t, userID, "Archived Chat", 20, 2000)
|
||
touchChannelTime(t, channelID, 5*time.Minute)
|
||
|
||
// Archive the channel
|
||
database.DB.Exec(`UPDATE channels SET is_archived = true WHERE id = $1`, channelID)
|
||
|
||
ctx := context.Background()
|
||
candidates := sc.findCandidates(ctx)
|
||
|
||
for _, c := range candidates {
|
||
if c.ID == channelID {
|
||
t.Fatal("archived channel should not be a candidate")
|
||
}
|
||
}
|
||
}
|
||
|
||
// ═══════════════════════════════════════════
|
||
// Scanner: Cooldown + Dedup
|
||
// ═══════════════════════════════════════════
|
||
|
||
func TestScanner_Cooldown(t *testing.T) {
|
||
sc := &Scanner{
|
||
lastCompacted: make(map[string]time.Time),
|
||
}
|
||
|
||
channelID := "test-cooldown-channel"
|
||
|
||
// No cooldown initially
|
||
sc.mu.Lock()
|
||
_, inCooldown := sc.lastCompacted[channelID]
|
||
sc.mu.Unlock()
|
||
if inCooldown {
|
||
t.Fatal("should not be in cooldown initially")
|
||
}
|
||
|
||
// Record compaction
|
||
sc.mu.Lock()
|
||
sc.lastCompacted[channelID] = time.Now()
|
||
sc.mu.Unlock()
|
||
|
||
// Now it's in cooldown
|
||
sc.mu.Lock()
|
||
last, ok := sc.lastCompacted[channelID]
|
||
sc.mu.Unlock()
|
||
if !ok {
|
||
t.Fatal("should be in cooldown after recording")
|
||
}
|
||
if time.Since(last) > time.Second {
|
||
t.Fatal("cooldown timestamp should be recent")
|
||
}
|
||
}
|
||
|
||
func TestScanner_InFlightDedup(t *testing.T) {
|
||
sc := &Scanner{}
|
||
// sync.Map zero value is ready to use
|
||
|
||
channelID := "test-dedup-channel"
|
||
|
||
// First load — not in flight
|
||
_, loaded := sc.inFlight.LoadOrStore(channelID, struct{}{})
|
||
if loaded {
|
||
t.Fatal("should not be loaded on first attempt")
|
||
}
|
||
|
||
// Second load — already in flight
|
||
_, loaded = sc.inFlight.LoadOrStore(channelID, struct{}{})
|
||
if !loaded {
|
||
t.Fatal("should be loaded on second attempt (dedup)")
|
||
}
|
||
|
||
// Clean up
|
||
sc.inFlight.Delete(channelID)
|
||
_, loaded = sc.inFlight.LoadOrStore(channelID, struct{}{})
|
||
if loaded {
|
||
t.Fatal("should not be loaded after delete")
|
||
}
|
||
}
|
||
|
||
// ═══════════════════════════════════════════
|
||
// Scanner: Channel Opt-Out
|
||
// ═══════════════════════════════════════════
|
||
|
||
func TestScanner_ShouldCompact_ChannelOptOut(t *testing.T) {
|
||
requireDB(t)
|
||
stores := postgres.NewStores(database.TestDB)
|
||
svc := NewService(stores, nil) // no resolver → IsConfigured returns false
|
||
sc := NewScanner(svc, stores, ScannerConfig{})
|
||
|
||
userID := database.SeedTestUser(t, "optout_user", "optout@test.com")
|
||
channelID, _ := seedChannel(t, userID, "Opt-Out Chat", 20, 2000)
|
||
|
||
// Opt out via channel settings
|
||
setChannelSettings(t, channelID, models.JSONMap{"auto_compaction": false})
|
||
|
||
ch := models.Channel{Settings: models.JSONMap{"auto_compaction": false}}
|
||
ch.ID = channelID
|
||
ch.UserID = userID
|
||
|
||
ctx := context.Background()
|
||
if sc.shouldCompact(ctx, &ch) {
|
||
t.Fatal("shouldCompact should return false for opted-out channel")
|
||
}
|
||
}
|
||
|
||
// ═══════════════════════════════════════════
|
||
// Scanner: Settings Helpers
|
||
// ═══════════════════════════════════════════
|
||
|
||
func TestScanner_IsEnabled(t *testing.T) {
|
||
requireDB(t)
|
||
stores := postgres.NewStores(database.TestDB)
|
||
svc := NewService(stores, nil)
|
||
sc := NewScanner(svc, stores, ScannerConfig{})
|
||
ctx := context.Background()
|
||
|
||
// Default: disabled
|
||
if sc.isEnabled(ctx) {
|
||
t.Fatal("should be disabled by default")
|
||
}
|
||
|
||
// Enable
|
||
setGlobalSetting(t, "auto_compaction_enabled", true)
|
||
if !sc.isEnabled(ctx) {
|
||
t.Fatal("should be enabled after setting to true")
|
||
}
|
||
|
||
// Disable again
|
||
setGlobalSetting(t, "auto_compaction_enabled", false)
|
||
if sc.isEnabled(ctx) {
|
||
t.Fatal("should be disabled after setting to false")
|
||
}
|
||
}
|
||
|
||
func TestScanner_GetThreshold_Default(t *testing.T) {
|
||
requireDB(t)
|
||
stores := postgres.NewStores(database.TestDB)
|
||
svc := NewService(stores, nil)
|
||
sc := NewScanner(svc, stores, ScannerConfig{})
|
||
|
||
ch := &models.Channel{}
|
||
got := sc.getThreshold(ch)
|
||
if got != DefaultThreshold {
|
||
t.Errorf("default threshold = %f, want %f", got, DefaultThreshold)
|
||
}
|
||
}
|
||
|
||
func TestScanner_GetThreshold_GlobalOverride(t *testing.T) {
|
||
requireDB(t)
|
||
stores := postgres.NewStores(database.TestDB)
|
||
svc := NewService(stores, nil)
|
||
sc := NewScanner(svc, stores, ScannerConfig{})
|
||
|
||
setGlobalSetting(t, "auto_compaction_threshold", 0.85)
|
||
|
||
ch := &models.Channel{}
|
||
got := sc.getThreshold(ch)
|
||
if got != 0.85 {
|
||
t.Errorf("global threshold = %f, want 0.85", got)
|
||
}
|
||
}
|
||
|
||
func TestScanner_GetThreshold_ChannelOverride(t *testing.T) {
|
||
requireDB(t)
|
||
stores := postgres.NewStores(database.TestDB)
|
||
svc := NewService(stores, nil)
|
||
sc := NewScanner(svc, stores, ScannerConfig{})
|
||
|
||
setGlobalSetting(t, "auto_compaction_threshold", 0.85)
|
||
|
||
ch := &models.Channel{
|
||
Settings: models.JSONMap{"compaction_threshold": 0.60},
|
||
}
|
||
got := sc.getThreshold(ch)
|
||
if got != 0.60 {
|
||
t.Errorf("channel threshold = %f, want 0.60", got)
|
||
}
|
||
}
|
||
|
||
func TestScanner_GetCooldownDuration_Default(t *testing.T) {
|
||
requireDB(t)
|
||
stores := postgres.NewStores(database.TestDB)
|
||
svc := NewService(stores, nil)
|
||
sc := NewScanner(svc, stores, ScannerConfig{})
|
||
ctx := context.Background()
|
||
|
||
got := sc.getCooldownDuration(ctx)
|
||
if got != DefaultCooldown {
|
||
t.Errorf("default cooldown = %s, want %s", got, DefaultCooldown)
|
||
}
|
||
}
|
||
|
||
func TestScanner_GetCooldownDuration_Override(t *testing.T) {
|
||
requireDB(t)
|
||
stores := postgres.NewStores(database.TestDB)
|
||
svc := NewService(stores, nil)
|
||
sc := NewScanner(svc, stores, ScannerConfig{})
|
||
ctx := context.Background()
|
||
|
||
setGlobalSetting(t, "auto_compaction_cooldown_minutes", float64(15))
|
||
|
||
got := sc.getCooldownDuration(ctx)
|
||
want := 15 * time.Minute
|
||
if got != want {
|
||
t.Errorf("cooldown = %s, want %s", got, want)
|
||
}
|
||
}
|
||
|
||
// ═══════════════════════════════════════════
|
||
// Context Budget Guard Rail
|
||
// ═══════════════════════════════════════════
|
||
|
||
func TestEstimateTokens_GuardRailMath(t *testing.T) {
|
||
// Simulate a large conversation that exceeds a 32K utility model
|
||
// 100K chars ≈ 25K tokens of conversation + system prompt overhead
|
||
largeContent := make([]byte, 100000)
|
||
contentTokens := EstimateTokens(string(largeContent))
|
||
systemTokens := EstimateTokens("You are a conversation summarizer...") + 8 // +overhead
|
||
|
||
totalPrompt := contentTokens + systemTokens
|
||
utilityBudget := 32000 // 32K model
|
||
inputCeiling := int(float64(utilityBudget) * 0.80)
|
||
|
||
if totalPrompt <= inputCeiling {
|
||
t.Errorf("100K chars (%d tokens) should exceed 32K model ceiling (%d tokens)",
|
||
totalPrompt, inputCeiling)
|
||
}
|
||
|
||
// Smaller conversation should fit
|
||
smallContent := make([]byte, 50000)
|
||
smallTokens := EstimateTokens(string(smallContent)) + systemTokens
|
||
if smallTokens > inputCeiling {
|
||
t.Errorf("50K chars (%d tokens) should fit in 32K model ceiling (%d tokens)",
|
||
smallTokens, inputCeiling)
|
||
}
|
||
}
|