package memory import ( "context" "encoding/json" "fmt" "log" "strings" "git.gobha.me/xcaliber/chat-switchboard/database" "git.gobha.me/xcaliber/chat-switchboard/knowledge" "git.gobha.me/xcaliber/chat-switchboard/models" "git.gobha.me/xcaliber/chat-switchboard/providers" "git.gobha.me/xcaliber/chat-switchboard/roles" "git.gobha.me/xcaliber/chat-switchboard/store" ) // defaultExtractionPrompt is used when the persona has no custom prompt. const defaultExtractionPrompt = `You are a memory extraction assistant. Analyze the following conversation and extract memorable facts about the user. Focus on: - Personal preferences (language, tools, frameworks, communication style) - Technical details (deployment stack, database choices, architecture patterns) - Project context (project names, team size, deadlines, goals) - Biographical facts (role, company, location, experience level) Respond ONLY with a JSON array of extracted facts. Each fact must have: - "key": short label (2-5 words, lowercase) - "value": the detail/fact (1-2 sentences max) - "confidence": 0.0-1.0 how certain you are Example: [ {"key": "preferred language", "value": "Go with Gin framework for backend services", "confidence": 0.95}, {"key": "deployment target", "value": "Kubernetes on AWS with PostgreSQL databases", "confidence": 0.8} ] If no memorable facts are found, respond with an empty array: []` // maxConversationChars limits the conversation text sent for extraction. const maxConversationChars = 30000 // minMessagesForExtraction is the minimum new messages before extraction triggers. const minMessagesForExtraction = 6 // extractedFact is the JSON structure returned by the utility model. type extractedFact struct { Key string `json:"key"` Value string `json:"value"` Confidence float64 `json:"confidence"` } // Extractor analyzes conversations and extracts memorable facts. type Extractor struct { stores store.Stores roleResolver *roles.Resolver embedder *knowledge.Embedder } // NewExtractor creates a new memory extraction service. func NewExtractor(stores store.Stores, rr *roles.Resolver, embedder *knowledge.Embedder) *Extractor { return &Extractor{stores: stores, roleResolver: rr, embedder: embedder} } // Extract analyzes a conversation and saves extracted facts as pending_review memories. func (e *Extractor) Extract(ctx context.Context, channelID, userID, teamID, personaID string) error { if e.stores.Memories == nil || e.stores.Messages == nil { return fmt.Errorf("memory or message store not available") } // Check extraction log for last processed message var lastMessageID string row := database.DB.QueryRowContext(ctx, database.Q(`SELECT last_message_id FROM memory_extraction_log WHERE channel_id = $1 AND user_id = $2`), channelID, userID) row.Scan(&lastMessageID) // ignore error — may not exist yet // Load recent messages for this channel messages, err := e.stores.Messages.ListForChannel(ctx, channelID, store.ListOptions{Limit: 200}) if err != nil { return fmt.Errorf("load messages: %w", err) } // Filter to new messages only (messages come back newest-first) var newMessages []models.Message for i := len(messages) - 1; i >= 0; i-- { if lastMessageID == "" || messages[i].ID > lastMessageID { newMessages = append(newMessages, messages[i]) } } if len(newMessages) < minMessagesForExtraction { return nil // not enough new content } // Build conversation text var sb strings.Builder for _, m := range newMessages { role := m.Role if role == "user" { role = "User" } else { role = "Assistant" } line := fmt.Sprintf("[%s]: %s\n", role, m.Content) if sb.Len()+len(line) > maxConversationChars { break } sb.WriteString(line) } if sb.Len() < 200 { return nil // too little content } // Get extraction prompt (persona-specific or default) prompt := defaultExtractionPrompt if personaID != "" && e.stores.Personas != nil { persona, err := e.stores.Personas.GetByID(ctx, personaID) if err == nil { // Respect persona memory_enabled flag (H6 — audit v0.28.0) if !persona.MemoryEnabled { return nil } if persona.MemoryExtractionPrompt != nil && *persona.MemoryExtractionPrompt != "" { prompt = *persona.MemoryExtractionPrompt } } } // Call utility model via role resolver var tID *string if teamID != "" { tID = &teamID } apiMessages := []providers.Message{ {Role: "system", Content: prompt}, {Role: "user", Content: "Analyze this conversation and extract memorable facts:\n\n" + sb.String()}, } result, err := e.roleResolver.Complete(ctx, "utility", userID, tID, apiMessages) if err != nil { return fmt.Errorf("extraction completion: %w", err) } // Parse response facts, err := parseExtractionResponse(result.Content) if err != nil { log.Printf("⚠ memory extraction parse failed for channel %s: %v", channelID, err) return nil // don't fail on parse errors } if len(facts) == 0 { log.Printf("🧠 memory extraction: channel %s → 0 facts (nothing memorable)", channelID) } // Determine scope scope := models.MemoryScopeUser ownerID := userID var memUserID *string if personaID != "" { scope = models.MemoryScopePersonaUser ownerID = personaID memUserID = &userID } // Save extracted facts saved := 0 for _, f := range facts { if f.Key == "" || f.Value == "" || f.Confidence < 0.3 { continue } mem := &models.Memory{ ID: store.NewID(), Scope: scope, OwnerID: ownerID, UserID: memUserID, Key: f.Key, Value: f.Value, SourceChannelID: &channelID, Confidence: f.Confidence, Status: models.MemoryStatusPendingReview, } if err := e.stores.Memories.Upsert(ctx, mem); err != nil { log.Printf("⚠ memory save failed: %v", err) continue } // Embed if available e.embedMemory(ctx, mem, userID, tID) saved++ } // Update extraction log latestID := newMessages[len(newMessages)-1].ID if database.IsSQLite() { _, err = database.DB.ExecContext(ctx, ` INSERT INTO memory_extraction_log (id, channel_id, user_id, last_message_id, memory_count) VALUES (?, ?, ?, ?, ?) ON CONFLICT(channel_id, user_id) DO UPDATE SET last_message_id = excluded.last_message_id, extracted_at = datetime('now'), memory_count = memory_extraction_log.memory_count + excluded.memory_count `, store.NewID(), channelID, userID, latestID, saved) } else { _, err = database.DB.ExecContext(ctx, ` INSERT INTO memory_extraction_log (channel_id, user_id, last_message_id, memory_count) VALUES ($1, $2, $3, $4) ON CONFLICT(channel_id, user_id) DO UPDATE SET last_message_id = EXCLUDED.last_message_id, extracted_at = now(), memory_count = memory_extraction_log.memory_count + EXCLUDED.memory_count `, channelID, userID, latestID, saved) } if saved > 0 { log.Printf("✅ memory extraction: channel %s → %d facts", channelID, saved) } return err } // embedMemory generates and stores an embedding vector for a memory. func (e *Extractor) embedMemory(ctx context.Context, m *models.Memory, userID string, teamID *string) { if e.embedder == nil || !e.embedder.IsConfigured(ctx) { return } text := m.Key + ": " + m.Value result, err := e.embedder.EmbedChunks(ctx, userID, teamID, []string{text}) if err != nil || len(result.Vectors) == 0 { return } vecJSON, _ := json.Marshal(result.Vectors[0]) if database.IsSQLite() { database.DB.ExecContext(ctx, `UPDATE memories SET embedding = ? WHERE id = ?`, string(vecJSON), m.ID) } else { database.DB.ExecContext(ctx, `UPDATE memories SET embedding = $1::vector WHERE id = $2`, string(vecJSON), m.ID) } } // parseExtractionResponse parses the utility model's JSON response. func parseExtractionResponse(content string) ([]extractedFact, error) { content = strings.TrimSpace(content) // Strip markdown code fences if strings.HasPrefix(content, "```") { lines := strings.Split(content, "\n") if len(lines) > 2 { lines = lines[1 : len(lines)-1] content = strings.Join(lines, "\n") } } var facts []extractedFact if err := json.Unmarshal([]byte(content), &facts); err != nil { return nil, fmt.Errorf("parse facts JSON: %w", err) } return facts, nil }