Changeset 0.18.0 (#79)
This commit is contained in:
261
server/memory/extractor.go
Normal file
261
server/memory/extractor.go
Normal file
@@ -0,0 +1,261 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/database"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/providers"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/roles"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
// defaultExtractionPrompt is used when the persona has no custom prompt.
|
||||
const defaultExtractionPrompt = `You are a memory extraction assistant. Analyze the following conversation and extract memorable facts about the user.
|
||||
|
||||
Focus on:
|
||||
- Personal preferences (language, tools, frameworks, communication style)
|
||||
- Technical details (deployment stack, database choices, architecture patterns)
|
||||
- Project context (project names, team size, deadlines, goals)
|
||||
- Biographical facts (role, company, location, experience level)
|
||||
|
||||
Respond ONLY with a JSON array of extracted facts. Each fact must have:
|
||||
- "key": short label (2-5 words, lowercase)
|
||||
- "value": the detail/fact (1-2 sentences max)
|
||||
- "confidence": 0.0-1.0 how certain you are
|
||||
|
||||
Example:
|
||||
[
|
||||
{"key": "preferred language", "value": "Go with Gin framework for backend services", "confidence": 0.95},
|
||||
{"key": "deployment target", "value": "Kubernetes on AWS with PostgreSQL databases", "confidence": 0.8}
|
||||
]
|
||||
|
||||
If no memorable facts are found, respond with an empty array: []`
|
||||
|
||||
// maxConversationChars limits the conversation text sent for extraction.
|
||||
const maxConversationChars = 30000
|
||||
|
||||
// minMessagesForExtraction is the minimum new messages before extraction triggers.
|
||||
const minMessagesForExtraction = 6
|
||||
|
||||
// extractedFact is the JSON structure returned by the utility model.
|
||||
type extractedFact struct {
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
|
||||
// Extractor analyzes conversations and extracts memorable facts.
|
||||
type Extractor struct {
|
||||
stores store.Stores
|
||||
roleResolver *roles.Resolver
|
||||
embedder *knowledge.Embedder
|
||||
}
|
||||
|
||||
// NewExtractor creates a new memory extraction service.
|
||||
func NewExtractor(stores store.Stores, rr *roles.Resolver, embedder *knowledge.Embedder) *Extractor {
|
||||
return &Extractor{stores: stores, roleResolver: rr, embedder: embedder}
|
||||
}
|
||||
|
||||
// Extract analyzes a conversation and saves extracted facts as pending_review memories.
|
||||
func (e *Extractor) Extract(ctx context.Context, channelID, userID, teamID, personaID string) error {
|
||||
if e.stores.Memories == nil || e.stores.Messages == nil {
|
||||
return fmt.Errorf("memory or message store not available")
|
||||
}
|
||||
|
||||
// Check extraction log for last processed message
|
||||
var lastMessageID string
|
||||
row := database.DB.QueryRowContext(ctx,
|
||||
database.Q(`SELECT last_message_id FROM memory_extraction_log WHERE channel_id = $1 AND user_id = $2`),
|
||||
channelID, userID)
|
||||
row.Scan(&lastMessageID) // ignore error — may not exist yet
|
||||
|
||||
// Load recent messages for this channel
|
||||
messages, err := e.stores.Messages.ListForChannel(ctx, channelID, store.ListOptions{Limit: 200})
|
||||
if err != nil {
|
||||
return fmt.Errorf("load messages: %w", err)
|
||||
}
|
||||
|
||||
// Filter to new messages only (messages come back newest-first)
|
||||
var newMessages []models.Message
|
||||
for i := len(messages) - 1; i >= 0; i-- {
|
||||
if lastMessageID == "" || messages[i].ID > lastMessageID {
|
||||
newMessages = append(newMessages, messages[i])
|
||||
}
|
||||
}
|
||||
|
||||
if len(newMessages) < minMessagesForExtraction {
|
||||
return nil // not enough new content
|
||||
}
|
||||
|
||||
// Build conversation text
|
||||
var sb strings.Builder
|
||||
for _, m := range newMessages {
|
||||
role := m.Role
|
||||
if role == "user" {
|
||||
role = "User"
|
||||
} else {
|
||||
role = "Assistant"
|
||||
}
|
||||
line := fmt.Sprintf("[%s]: %s\n", role, m.Content)
|
||||
if sb.Len()+len(line) > maxConversationChars {
|
||||
break
|
||||
}
|
||||
sb.WriteString(line)
|
||||
}
|
||||
|
||||
if sb.Len() < 200 {
|
||||
return nil // too little content
|
||||
}
|
||||
|
||||
// Get extraction prompt (persona-specific or default)
|
||||
prompt := defaultExtractionPrompt
|
||||
if personaID != "" && e.stores.Personas != nil {
|
||||
persona, err := e.stores.Personas.GetByID(ctx, personaID)
|
||||
if err == nil && persona.MemoryExtractionPrompt != nil && *persona.MemoryExtractionPrompt != "" {
|
||||
prompt = *persona.MemoryExtractionPrompt
|
||||
}
|
||||
}
|
||||
|
||||
// Call utility model via role resolver
|
||||
var tID *string
|
||||
if teamID != "" {
|
||||
tID = &teamID
|
||||
}
|
||||
|
||||
apiMessages := []providers.Message{
|
||||
{Role: "system", Content: prompt},
|
||||
{Role: "user", Content: "Analyze this conversation and extract memorable facts:\n\n" + sb.String()},
|
||||
}
|
||||
|
||||
result, err := e.roleResolver.Complete(ctx, "utility", userID, tID, apiMessages)
|
||||
if err != nil {
|
||||
return fmt.Errorf("extraction completion: %w", err)
|
||||
}
|
||||
|
||||
// Parse response
|
||||
facts, err := parseExtractionResponse(result.Content)
|
||||
if err != nil {
|
||||
log.Printf("⚠ memory extraction parse failed for channel %s: %v", channelID, err)
|
||||
return nil // don't fail on parse errors
|
||||
}
|
||||
|
||||
if len(facts) == 0 {
|
||||
log.Printf("🧠 memory extraction: channel %s → 0 facts (nothing memorable)", channelID)
|
||||
}
|
||||
|
||||
// Determine scope
|
||||
scope := models.MemoryScopeUser
|
||||
ownerID := userID
|
||||
var memUserID *string
|
||||
if personaID != "" {
|
||||
scope = models.MemoryScopePersonaUser
|
||||
ownerID = personaID
|
||||
memUserID = &userID
|
||||
}
|
||||
|
||||
// Save extracted facts
|
||||
saved := 0
|
||||
for _, f := range facts {
|
||||
if f.Key == "" || f.Value == "" || f.Confidence < 0.3 {
|
||||
continue
|
||||
}
|
||||
|
||||
mem := &models.Memory{
|
||||
ID: store.NewID(),
|
||||
Scope: scope,
|
||||
OwnerID: ownerID,
|
||||
UserID: memUserID,
|
||||
Key: f.Key,
|
||||
Value: f.Value,
|
||||
SourceChannelID: &channelID,
|
||||
Confidence: f.Confidence,
|
||||
Status: models.MemoryStatusPendingReview,
|
||||
}
|
||||
|
||||
if err := e.stores.Memories.Upsert(ctx, mem); err != nil {
|
||||
log.Printf("⚠ memory save failed: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Embed if available
|
||||
e.embedMemory(ctx, mem, userID, tID)
|
||||
saved++
|
||||
}
|
||||
|
||||
// Update extraction log
|
||||
latestID := newMessages[len(newMessages)-1].ID
|
||||
if database.IsSQLite() {
|
||||
_, err = database.DB.ExecContext(ctx, `
|
||||
INSERT INTO memory_extraction_log (id, channel_id, user_id, last_message_id, memory_count)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(channel_id, user_id) DO UPDATE SET
|
||||
last_message_id = excluded.last_message_id,
|
||||
extracted_at = datetime('now'),
|
||||
memory_count = memory_extraction_log.memory_count + excluded.memory_count
|
||||
`, store.NewID(), channelID, userID, latestID, saved)
|
||||
} else {
|
||||
_, err = database.DB.ExecContext(ctx, `
|
||||
INSERT INTO memory_extraction_log (channel_id, user_id, last_message_id, memory_count)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT(channel_id, user_id) DO UPDATE SET
|
||||
last_message_id = EXCLUDED.last_message_id,
|
||||
extracted_at = now(),
|
||||
memory_count = memory_extraction_log.memory_count + EXCLUDED.memory_count
|
||||
`, channelID, userID, latestID, saved)
|
||||
}
|
||||
|
||||
if saved > 0 {
|
||||
log.Printf("✅ memory extraction: channel %s → %d facts", channelID, saved)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// embedMemory generates and stores an embedding vector for a memory.
|
||||
func (e *Extractor) embedMemory(ctx context.Context, m *models.Memory, userID string, teamID *string) {
|
||||
if e.embedder == nil || !e.embedder.IsConfigured(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
text := m.Key + ": " + m.Value
|
||||
result, err := e.embedder.EmbedChunks(ctx, userID, teamID, []string{text})
|
||||
if err != nil || len(result.Vectors) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
vecJSON, _ := json.Marshal(result.Vectors[0])
|
||||
if database.IsSQLite() {
|
||||
database.DB.ExecContext(ctx,
|
||||
`UPDATE memories SET embedding = ? WHERE id = ?`,
|
||||
string(vecJSON), m.ID)
|
||||
} else {
|
||||
database.DB.ExecContext(ctx,
|
||||
`UPDATE memories SET embedding = $1::vector WHERE id = $2`,
|
||||
string(vecJSON), m.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// parseExtractionResponse parses the utility model's JSON response.
|
||||
func parseExtractionResponse(content string) ([]extractedFact, error) {
|
||||
content = strings.TrimSpace(content)
|
||||
|
||||
// Strip markdown code fences
|
||||
if strings.HasPrefix(content, "```") {
|
||||
lines := strings.Split(content, "\n")
|
||||
if len(lines) > 2 {
|
||||
lines = lines[1 : len(lines)-1]
|
||||
content = strings.Join(lines, "\n")
|
||||
}
|
||||
}
|
||||
|
||||
var facts []extractedFact
|
||||
if err := json.Unmarshal([]byte(content), &facts); err != nil {
|
||||
return nil, fmt.Errorf("parse facts JSON: %w", err)
|
||||
}
|
||||
return facts, nil
|
||||
}
|
||||
197
server/memory/scanner.go
Normal file
197
server/memory/scanner.go
Normal file
@@ -0,0 +1,197 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/database"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
// ScannerConfig holds tunable parameters for the background scanner.
|
||||
type ScannerConfig struct {
|
||||
Interval time.Duration // tick interval (default 10m)
|
||||
Concurrency int // max parallel extractions (default 2)
|
||||
}
|
||||
|
||||
// Scanner periodically finds conversations needing extraction.
|
||||
type Scanner struct {
|
||||
extractor *Extractor
|
||||
stores store.Stores
|
||||
cfg ScannerConfig
|
||||
stopCh chan struct{}
|
||||
wg sync.WaitGroup
|
||||
inFlight sync.Map // channelID+userID → true
|
||||
}
|
||||
|
||||
// NewScanner creates a background extraction scanner.
|
||||
func NewScanner(ext *Extractor, stores store.Stores, cfg ScannerConfig) *Scanner {
|
||||
if cfg.Interval <= 0 {
|
||||
cfg.Interval = 10 * time.Minute
|
||||
}
|
||||
if cfg.Concurrency <= 0 {
|
||||
cfg.Concurrency = 2
|
||||
}
|
||||
return &Scanner{
|
||||
extractor: ext,
|
||||
stores: stores,
|
||||
cfg: cfg,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins the background scanning loop.
|
||||
func (s *Scanner) Start() {
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
log.Printf("🧠 memory extraction scanner started (interval=%s, concurrency=%d)",
|
||||
s.cfg.Interval, s.cfg.Concurrency)
|
||||
|
||||
ticker := time.NewTicker(s.cfg.Interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-s.stopCh:
|
||||
log.Println("🧠 memory extraction scanner stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.scan()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the scanner.
|
||||
func (s *Scanner) Stop() {
|
||||
close(s.stopCh)
|
||||
s.wg.Wait()
|
||||
}
|
||||
|
||||
// candidate represents a conversation that may need extraction.
|
||||
type candidate struct {
|
||||
ChannelID string
|
||||
UserID string
|
||||
TeamID string
|
||||
}
|
||||
|
||||
func (s *Scanner) scan() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Check global kill switch
|
||||
if s.stores.GlobalConfig != nil {
|
||||
val, err := s.stores.GlobalConfig.Get(ctx, "memory_extraction_enabled")
|
||||
if err != nil || val == nil {
|
||||
return // disabled or not configured
|
||||
}
|
||||
// val is a map; check the "value" key
|
||||
if enabled, ok := val["value"].(bool); !ok || !enabled {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
candidates, err := s.findCandidates(ctx)
|
||||
if err != nil {
|
||||
log.Printf("⚠ memory scanner: find candidates: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(candidates) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Semaphore for concurrency control
|
||||
sem := make(chan struct{}, s.cfg.Concurrency)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, c := range candidates {
|
||||
key := c.ChannelID + ":" + c.UserID
|
||||
if _, loaded := s.inFlight.LoadOrStore(key, true); loaded {
|
||||
continue // already in progress
|
||||
}
|
||||
|
||||
sem <- struct{}{} // acquire
|
||||
wg.Add(1)
|
||||
|
||||
go func(cand candidate) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }() // release
|
||||
defer s.inFlight.Delete(cand.ChannelID + ":" + cand.UserID)
|
||||
|
||||
extractCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
if err := s.extractor.Extract(extractCtx, cand.ChannelID, cand.UserID, cand.TeamID, ""); err != nil {
|
||||
log.Printf("⚠ memory extraction failed: channel=%s user=%s: %v",
|
||||
cand.ChannelID, cand.UserID, err)
|
||||
}
|
||||
}(c)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// findCandidates queries for channels with enough new activity since last extraction.
|
||||
func (s *Scanner) findCandidates(ctx context.Context) ([]candidate, error) {
|
||||
db := database.DB
|
||||
if db == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var query string
|
||||
if database.IsSQLite() {
|
||||
query = `
|
||||
SELECT c.id, c.created_by, COALESCE(cu.team_id, '')
|
||||
FROM channels c
|
||||
LEFT JOIN channel_users cu ON cu.channel_id = c.id AND cu.user_id = c.created_by
|
||||
LEFT JOIN memory_extraction_log mel ON mel.channel_id = c.id AND mel.user_id = c.created_by
|
||||
WHERE c.created_by IS NOT NULL
|
||||
AND (
|
||||
SELECT COUNT(*) FROM messages m
|
||||
WHERE m.channel_id = c.id
|
||||
AND (mel.last_message_id IS NULL OR m.id > mel.last_message_id)
|
||||
) >= 8
|
||||
AND (
|
||||
SELECT MAX(m2.created_at) FROM messages m2 WHERE m2.channel_id = c.id
|
||||
) < datetime('now', '-5 minutes')
|
||||
LIMIT 20
|
||||
`
|
||||
} else {
|
||||
query = `
|
||||
SELECT c.id, c.created_by, COALESCE(cu.team_id, '')
|
||||
FROM channels c
|
||||
LEFT JOIN channel_users cu ON cu.channel_id = c.id AND cu.user_id = c.created_by
|
||||
LEFT JOIN memory_extraction_log mel ON mel.channel_id = c.id AND mel.user_id = c.created_by
|
||||
WHERE c.created_by IS NOT NULL
|
||||
AND (
|
||||
SELECT COUNT(*) FROM messages m
|
||||
WHERE m.channel_id = c.id
|
||||
AND (mel.last_message_id IS NULL OR m.id::text > mel.last_message_id::text)
|
||||
) >= 8
|
||||
AND (
|
||||
SELECT MAX(m2.created_at) FROM messages m2 WHERE m2.channel_id = c.id
|
||||
) < now() - interval '5 minutes'
|
||||
LIMIT 20
|
||||
`
|
||||
}
|
||||
|
||||
rows, err := db.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var candidates []candidate
|
||||
for rows.Next() {
|
||||
var c candidate
|
||||
if err := rows.Scan(&c.ChannelID, &c.UserID, &c.TeamID); err != nil {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, c)
|
||||
}
|
||||
|
||||
return candidates, rows.Err()
|
||||
}
|
||||
Reference in New Issue
Block a user