package memory import ( "context" "log" "sync" "time" "git.gobha.me/xcaliber/chat-switchboard/database" "git.gobha.me/xcaliber/chat-switchboard/store" ) // ScannerConfig holds tunable parameters for the background scanner. type ScannerConfig struct { Interval time.Duration // tick interval (default 10m) Concurrency int // max parallel extractions (default 2) } // Scanner periodically finds conversations needing extraction. type Scanner struct { extractor *Extractor stores store.Stores cfg ScannerConfig stopCh chan struct{} wg sync.WaitGroup inFlight sync.Map // channelID+userID → true } // NewScanner creates a background extraction scanner. func NewScanner(ext *Extractor, stores store.Stores, cfg ScannerConfig) *Scanner { if cfg.Interval <= 0 { cfg.Interval = 10 * time.Minute } if cfg.Concurrency <= 0 { cfg.Concurrency = 2 } return &Scanner{ extractor: ext, stores: stores, cfg: cfg, stopCh: make(chan struct{}), } } // Start begins the background scanning loop. func (s *Scanner) Start() { s.wg.Add(1) go func() { defer s.wg.Done() log.Printf("🧠 memory extraction scanner started (interval=%s, concurrency=%d)", s.cfg.Interval, s.cfg.Concurrency) ticker := time.NewTicker(s.cfg.Interval) defer ticker.Stop() for { select { case <-s.stopCh: log.Println("🧠 memory extraction scanner stopped") return case <-ticker.C: s.scan() } } }() } // Stop gracefully shuts down the scanner. func (s *Scanner) Stop() { close(s.stopCh) s.wg.Wait() } // candidate represents a conversation that may need extraction. type candidate struct { ChannelID string UserID string TeamID string PersonaID string // non-empty if a persona is active on the channel } func (s *Scanner) scan() { ctx := context.Background() // Check global kill switch. // Default: ENABLED. Only disable if the key exists and is explicitly false. // This inverts the previous behavior where a missing key blocked all extraction. if s.stores.GlobalConfig != nil { val, err := s.stores.GlobalConfig.Get(ctx, "memory_extraction_enabled") if err == nil && val != nil { if enabled, ok := val["value"].(bool); ok && !enabled { return // explicitly disabled } } // If key is missing (err != nil or val == nil), extraction is enabled. } // ── Extraction ── candidates, err := s.findCandidates(ctx) if err != nil { log.Printf("⚠ memory scanner: find candidates: %v", err) return } if len(candidates) > 0 { sem := make(chan struct{}, s.cfg.Concurrency) var wg sync.WaitGroup for _, c := range candidates { key := c.ChannelID + ":" + c.UserID if _, loaded := s.inFlight.LoadOrStore(key, true); loaded { continue } sem <- struct{}{} wg.Add(1) go func(cand candidate) { defer wg.Done() defer func() { <-sem }() defer s.inFlight.Delete(cand.ChannelID + ":" + cand.UserID) extractCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() if err := s.extractor.Extract(extractCtx, cand.ChannelID, cand.UserID, cand.TeamID, cand.PersonaID); err != nil { log.Printf("⚠ memory extraction failed: channel=%s user=%s: %v", cand.ChannelID, cand.UserID, err) } }(c) } wg.Wait() } // ── v0.30.1: Compaction (decay + prune) ── // Runs after extraction on every tick. Gated by memory_compaction_enabled. s.runCompaction(ctx) } // runCompaction applies confidence decay and pruning to stale memories. func (s *Scanner) runCompaction(ctx context.Context) { if s.stores.Memories == nil { return } // Check compaction kill switch (default: enabled) if s.stores.GlobalConfig != nil { val, err := s.stores.GlobalConfig.Get(ctx, "memory_compaction_enabled") if err == nil && val != nil { if enabled, ok := val["value"].(bool); ok && !enabled { return } } } // Decay: reduce confidence for memories not updated in 7 days olderThan := time.Now().Add(-7 * 24 * time.Hour).UTC().Format(time.RFC3339) decayed, err := s.stores.Memories.DecayConfidence(ctx, olderThan, 0.9) if err != nil { log.Printf("⚠ memory compaction: decay failed: %v", err) } else if decayed > 0 { log.Printf("🧠 memory compaction: decayed %d memories", decayed) } // Prune: archive memories with confidence below 0.15 pruned, err := s.stores.Memories.Prune(ctx, 0.15) if err != nil { log.Printf("⚠ memory compaction: prune failed: %v", err) } else if pruned > 0 { log.Printf("🧠 memory compaction: pruned %d memories", pruned) } } // findCandidates queries for channels with enough new activity since last extraction. // // Fixes from v0.28.0 audit: // - Uses channels.user_id (was: non-existent created_by) // - Uses channels.team_id directly (was: JOIN to non-existent channel_users table) // - JOINs channel_participants + personas to check memory_enabled (H6) // - Filters out channels whose active persona has memory_enabled=false func (s *Scanner) findCandidates(ctx context.Context) ([]candidate, error) { db := database.DB if db == nil { return nil, nil } var query string if database.IsSQLite() { query = ` SELECT c.id, c.user_id, COALESCE(c.team_id, ''), COALESCE(cp.participant_id, '') FROM channels c LEFT JOIN channel_participants cp ON cp.channel_id = c.id AND cp.participant_type = 'persona' LEFT JOIN personas p ON p.id = cp.participant_id LEFT JOIN memory_extraction_log mel ON mel.channel_id = c.id AND mel.user_id = c.user_id WHERE c.user_id IS NOT NULL AND (cp.id IS NULL OR p.memory_enabled = 1) AND ( SELECT COUNT(*) FROM messages m WHERE m.channel_id = c.id AND (mel.last_message_id IS NULL OR m.id > mel.last_message_id) ) >= 8 AND ( SELECT MAX(m2.created_at) FROM messages m2 WHERE m2.channel_id = c.id ) < datetime('now', '-5 minutes') LIMIT 20 ` } else { query = ` SELECT c.id, c.user_id, COALESCE(c.team_id::text, ''), COALESCE(cp.participant_id::text, '') FROM channels c LEFT JOIN channel_participants cp ON cp.channel_id = c.id AND cp.participant_type = 'persona' LEFT JOIN personas p ON p.id = cp.participant_id LEFT JOIN memory_extraction_log mel ON mel.channel_id = c.id AND mel.user_id = c.user_id WHERE c.user_id IS NOT NULL AND (cp.id IS NULL OR p.memory_enabled = true) AND ( SELECT COUNT(*) FROM messages m WHERE m.channel_id = c.id AND (mel.last_message_id IS NULL OR m.id::text > mel.last_message_id::text) ) >= 8 AND ( SELECT MAX(m2.created_at) FROM messages m2 WHERE m2.channel_id = c.id ) < now() - interval '5 minutes' LIMIT 20 ` } rows, err := db.QueryContext(ctx, query) if err != nil { return nil, err } defer rows.Close() var candidates []candidate for rows.Next() { var c candidate if err := rows.Scan(&c.ChannelID, &c.UserID, &c.TeamID, &c.PersonaID); err != nil { continue } candidates = append(candidates, c) } return candidates, rows.Err() }