254 lines
6.9 KiB
Go
254 lines
6.9 KiB
Go
package memory
|
|
|
|
import (
|
|
"context"
|
|
"log"
|
|
"sync"
|
|
"time"
|
|
|
|
"chat-switchboard/database"
|
|
"chat-switchboard/store"
|
|
)
|
|
|
|
// ScannerConfig holds tunable parameters for the background scanner.
|
|
type ScannerConfig struct {
|
|
Interval time.Duration // tick interval (default 10m)
|
|
Concurrency int // max parallel extractions (default 2)
|
|
}
|
|
|
|
// Scanner periodically finds conversations needing extraction.
|
|
type Scanner struct {
|
|
extractor *Extractor
|
|
stores store.Stores
|
|
cfg ScannerConfig
|
|
stopCh chan struct{}
|
|
wg sync.WaitGroup
|
|
inFlight sync.Map // channelID+userID → true
|
|
}
|
|
|
|
// NewScanner creates a background extraction scanner.
|
|
func NewScanner(ext *Extractor, stores store.Stores, cfg ScannerConfig) *Scanner {
|
|
if cfg.Interval <= 0 {
|
|
cfg.Interval = 10 * time.Minute
|
|
}
|
|
if cfg.Concurrency <= 0 {
|
|
cfg.Concurrency = 2
|
|
}
|
|
return &Scanner{
|
|
extractor: ext,
|
|
stores: stores,
|
|
cfg: cfg,
|
|
stopCh: make(chan struct{}),
|
|
}
|
|
}
|
|
|
|
// Start begins the background scanning loop.
|
|
func (s *Scanner) Start() {
|
|
s.wg.Add(1)
|
|
go func() {
|
|
defer s.wg.Done()
|
|
log.Printf("🧠 memory extraction scanner started (interval=%s, concurrency=%d)",
|
|
s.cfg.Interval, s.cfg.Concurrency)
|
|
|
|
ticker := time.NewTicker(s.cfg.Interval)
|
|
defer ticker.Stop()
|
|
|
|
for {
|
|
select {
|
|
case <-s.stopCh:
|
|
log.Println("🧠 memory extraction scanner stopped")
|
|
return
|
|
case <-ticker.C:
|
|
s.scan()
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Stop gracefully shuts down the scanner.
|
|
func (s *Scanner) Stop() {
|
|
close(s.stopCh)
|
|
s.wg.Wait()
|
|
}
|
|
|
|
// candidate represents a conversation that may need extraction.
|
|
type candidate struct {
|
|
ChannelID string
|
|
UserID string
|
|
TeamID string
|
|
PersonaID string // non-empty if a persona is active on the channel
|
|
}
|
|
|
|
func (s *Scanner) scan() {
|
|
ctx := context.Background()
|
|
|
|
// Check global kill switch.
|
|
// Default: ENABLED. Only disable if the key exists and is explicitly false.
|
|
// This inverts the previous behavior where a missing key blocked all extraction.
|
|
if s.stores.GlobalConfig != nil {
|
|
val, err := s.stores.GlobalConfig.Get(ctx, "memory_extraction_enabled")
|
|
if err == nil && val != nil {
|
|
if enabled, ok := val["value"].(bool); ok && !enabled {
|
|
return // explicitly disabled
|
|
}
|
|
}
|
|
// If key is missing (err != nil or val == nil), extraction is enabled.
|
|
}
|
|
|
|
// ── Extraction ──
|
|
candidates, err := s.findCandidates(ctx)
|
|
if err != nil {
|
|
log.Printf("⚠ memory scanner: find candidates: %v", err)
|
|
return
|
|
}
|
|
|
|
if len(candidates) > 0 {
|
|
sem := make(chan struct{}, s.cfg.Concurrency)
|
|
var wg sync.WaitGroup
|
|
|
|
for _, c := range candidates {
|
|
key := c.ChannelID + ":" + c.UserID
|
|
if _, loaded := s.inFlight.LoadOrStore(key, true); loaded {
|
|
continue
|
|
}
|
|
|
|
sem <- struct{}{}
|
|
wg.Add(1)
|
|
|
|
go func(cand candidate) {
|
|
defer wg.Done()
|
|
defer func() { <-sem }()
|
|
defer s.inFlight.Delete(cand.ChannelID + ":" + cand.UserID)
|
|
|
|
extractCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
|
defer cancel()
|
|
|
|
if err := s.extractor.Extract(extractCtx, cand.ChannelID, cand.UserID, cand.TeamID, cand.PersonaID); err != nil {
|
|
log.Printf("⚠ memory extraction failed: channel=%s user=%s: %v",
|
|
cand.ChannelID, cand.UserID, err)
|
|
}
|
|
}(c)
|
|
}
|
|
|
|
wg.Wait()
|
|
}
|
|
|
|
// ── v0.30.1: Compaction (decay + prune) ──
|
|
// Runs after extraction on every tick. Gated by memory_compaction_enabled.
|
|
s.runCompaction(ctx)
|
|
}
|
|
|
|
// runCompaction applies confidence decay and pruning to stale memories.
|
|
func (s *Scanner) runCompaction(ctx context.Context) {
|
|
if s.stores.Memories == nil {
|
|
return
|
|
}
|
|
|
|
// Check compaction kill switch (default: enabled)
|
|
if s.stores.GlobalConfig != nil {
|
|
val, err := s.stores.GlobalConfig.Get(ctx, "memory_compaction_enabled")
|
|
if err == nil && val != nil {
|
|
if enabled, ok := val["value"].(bool); ok && !enabled {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// Decay: reduce confidence for memories not updated in 7 days
|
|
olderThan := time.Now().Add(-7 * 24 * time.Hour).UTC().Format(time.RFC3339)
|
|
decayed, err := s.stores.Memories.DecayConfidence(ctx, olderThan, 0.9)
|
|
if err != nil {
|
|
log.Printf("⚠ memory compaction: decay failed: %v", err)
|
|
} else if decayed > 0 {
|
|
log.Printf("🧠 memory compaction: decayed %d memories", decayed)
|
|
}
|
|
|
|
// Prune: archive memories with confidence below 0.15
|
|
pruned, err := s.stores.Memories.Prune(ctx, 0.15)
|
|
if err != nil {
|
|
log.Printf("⚠ memory compaction: prune failed: %v", err)
|
|
} else if pruned > 0 {
|
|
log.Printf("🧠 memory compaction: pruned %d memories", pruned)
|
|
}
|
|
}
|
|
|
|
// findCandidates queries for channels with enough new activity since last extraction.
|
|
//
|
|
// Fixes from v0.28.0 audit:
|
|
// - Uses channels.user_id (was: non-existent created_by)
|
|
// - Uses channels.team_id directly (was: JOIN to non-existent channel_users table)
|
|
// - JOINs channel_participants + personas to check memory_enabled (H6)
|
|
// - Filters out channels whose active persona has memory_enabled=false
|
|
func (s *Scanner) findCandidates(ctx context.Context) ([]candidate, error) {
|
|
db := database.DB
|
|
if db == nil {
|
|
return nil, nil
|
|
}
|
|
|
|
var query string
|
|
if database.IsSQLite() {
|
|
query = `
|
|
SELECT c.id, c.user_id, COALESCE(c.team_id, ''),
|
|
COALESCE(cp.participant_id, '')
|
|
FROM channels c
|
|
LEFT JOIN channel_participants cp
|
|
ON cp.channel_id = c.id AND cp.participant_type = 'persona'
|
|
LEFT JOIN personas p
|
|
ON p.id = cp.participant_id
|
|
LEFT JOIN memory_extraction_log mel
|
|
ON mel.channel_id = c.id AND mel.user_id = c.user_id
|
|
WHERE c.user_id IS NOT NULL
|
|
AND (cp.id IS NULL OR p.memory_enabled = 1)
|
|
AND (
|
|
SELECT COUNT(*) FROM messages m
|
|
WHERE m.channel_id = c.id
|
|
AND (mel.last_message_id IS NULL OR m.id > mel.last_message_id)
|
|
) >= 8
|
|
AND (
|
|
SELECT MAX(m2.created_at) FROM messages m2 WHERE m2.channel_id = c.id
|
|
) < datetime('now', '-5 minutes')
|
|
LIMIT 20
|
|
`
|
|
} else {
|
|
query = `
|
|
SELECT c.id, c.user_id, COALESCE(c.team_id::text, ''),
|
|
COALESCE(cp.participant_id::text, '')
|
|
FROM channels c
|
|
LEFT JOIN channel_participants cp
|
|
ON cp.channel_id = c.id AND cp.participant_type = 'persona'
|
|
LEFT JOIN personas p
|
|
ON p.id = cp.participant_id
|
|
LEFT JOIN memory_extraction_log mel
|
|
ON mel.channel_id = c.id AND mel.user_id = c.user_id
|
|
WHERE c.user_id IS NOT NULL
|
|
AND (cp.id IS NULL OR p.memory_enabled = true)
|
|
AND (
|
|
SELECT COUNT(*) FROM messages m
|
|
WHERE m.channel_id = c.id
|
|
AND (mel.last_message_id IS NULL OR m.id::text > mel.last_message_id::text)
|
|
) >= 8
|
|
AND (
|
|
SELECT MAX(m2.created_at) FROM messages m2 WHERE m2.channel_id = c.id
|
|
) < now() - interval '5 minutes'
|
|
LIMIT 20
|
|
`
|
|
}
|
|
|
|
rows, err := db.QueryContext(ctx, query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var candidates []candidate
|
|
for rows.Next() {
|
|
var c candidate
|
|
if err := rows.Scan(&c.ChannelID, &c.UserID, &c.TeamID, &c.PersonaID); err != nil {
|
|
continue
|
|
}
|
|
candidates = append(candidates, c)
|
|
}
|
|
|
|
return candidates, rows.Err()
|
|
}
|