Changeset 0.18.0 (#79)

This commit is contained in:
2026-02-28 18:24:19 +00:00
parent 12e316c234
commit e4a943b03e
48 changed files with 3999 additions and 1398 deletions

View File

@@ -0,0 +1,85 @@
-- v0.18.0: Memory System
--
-- Long-term memory across conversations, with scope-aware isolation.
-- Three scopes: user (personal), persona (shared), persona_user (per-user-per-persona).
--
-- Unlike KB (static documents) or compaction (within-conversation), this
-- captures facts and preferences that persist across channels.
-- =========================================
-- 1. Memories Table
-- =========================================
CREATE TABLE IF NOT EXISTS memories (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
scope TEXT NOT NULL CHECK (scope IN ('user', 'persona', 'persona_user')),
-- Owner resolution:
-- user scope: owner_id = user_id, user_id = NULL
-- persona scope: owner_id = persona_id, user_id = NULL
-- persona_user scope: owner_id = persona_id, user_id = user_id
owner_id UUID NOT NULL,
user_id UUID,
key TEXT NOT NULL, -- short label ("preferred language", "team size")
value TEXT NOT NULL, -- detail / fact content
source_channel_id UUID, -- which conversation produced this memory
confidence REAL NOT NULL DEFAULT 1.0, -- 0.01.0, used for ranking
status TEXT NOT NULL DEFAULT 'active'
CHECK (status IN ('active', 'pending_review', 'archived')),
-- Embedding for semantic recall (optional — same pattern as notes)
embedding vector(3072),
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
-- Primary lookup: "give me all active memories for this scope + owner"
CREATE INDEX IF NOT EXISTS idx_memories_scope_owner
ON memories(scope, owner_id, status)
WHERE status = 'active';
-- Persona+user lookup: "give me this persona's memories about this user"
CREATE INDEX IF NOT EXISTS idx_memories_persona_user
ON memories(owner_id, user_id)
WHERE scope = 'persona_user' AND status = 'active';
-- Deduplication: prevent saving the same key twice for the same scope/owner/user
CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_unique_key
ON memories(scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key);
-- Full-text search on key + value
CREATE INDEX IF NOT EXISTS idx_memories_fts
ON memories USING gin(to_tsvector('english', key || ' ' || value));
-- Source channel reference (for provenance / "where did this come from")
CREATE INDEX IF NOT EXISTS idx_memories_source_channel
ON memories(source_channel_id)
WHERE source_channel_id IS NOT NULL;
COMMENT ON TABLE memories IS 'Long-term memory facts persisted across conversations, scoped to user, persona, or user+persona';
COMMENT ON COLUMN memories.scope IS 'user = personal facts; persona = shared persona knowledge; persona_user = per-user within a persona';
COMMENT ON COLUMN memories.owner_id IS 'user_id for user scope, persona_id for persona/persona_user scopes';
COMMENT ON COLUMN memories.user_id IS 'Only set for persona_user scope — identifies which user within the persona';
COMMENT ON COLUMN memories.key IS 'Short label for the memory (e.g. "preferred language", "deployment stack")';
COMMENT ON COLUMN memories.value IS 'The actual fact/detail being remembered';
COMMENT ON COLUMN memories.confidence IS '0.0-1.0 confidence score — higher = more certain, used for ranking and pruning';
COMMENT ON COLUMN memories.status IS 'active = injected into context; pending_review = awaiting human approval; archived = soft-deleted';
-- =========================================
-- 2. Updated-at Trigger
-- =========================================
CREATE OR REPLACE FUNCTION update_memories_updated_at()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = now();
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trg_memories_updated_at
BEFORE UPDATE ON memories
FOR EACH ROW
EXECUTE FUNCTION update_memories_updated_at();

View File

@@ -0,0 +1,46 @@
-- v0.18.0 Phase 2: Memory Extraction & Embeddings
--
-- Adds persona memory configuration, HNSW vector index for
-- semantic recall, and extraction tracking log.
-- =========================================
-- 1. Persona Memory Configuration
-- =========================================
ALTER TABLE personas ADD COLUMN IF NOT EXISTS memory_enabled BOOLEAN NOT NULL DEFAULT true;
ALTER TABLE personas ADD COLUMN IF NOT EXISTS memory_extraction_prompt TEXT;
COMMENT ON COLUMN personas.memory_enabled IS 'Whether memory tools and extraction are active for this persona';
COMMENT ON COLUMN personas.memory_extraction_prompt IS 'Custom extraction prompt; NULL = use system default';
-- =========================================
-- 2. Vector Index for Semantic Recall
-- =========================================
-- NOTE: HNSW indexes have a 2000-dimension limit in pgvector.
-- Our embeddings are 3072-dim (matching KB/notes schema), so we
-- skip the HNSW index. Memory tables are small enough (hundreds
-- of rows per user) that filtered sequential scans with cosine
-- distance are performant. If needed later, IVFFlat supports
-- higher dimensions, or embeddings can be reduced to 1536-dim.
-- =========================================
-- 3. Memory Extraction Log
-- =========================================
-- Tracks which messages have been processed by the extraction scanner
-- to prevent duplicate work. One row per channel+user pair.
CREATE TABLE IF NOT EXISTS memory_extraction_log (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
channel_id UUID NOT NULL,
user_id UUID NOT NULL,
last_message_id UUID NOT NULL,
extracted_at TIMESTAMPTZ DEFAULT now(),
memory_count INT NOT NULL DEFAULT 0,
UNIQUE(channel_id, user_id)
);
CREATE INDEX IF NOT EXISTS idx_mem_extract_log_channel
ON memory_extraction_log(channel_id);
COMMENT ON TABLE memory_extraction_log IS 'Tracks extraction progress per channel+user to prevent re-processing';

View File

@@ -0,0 +1,61 @@
-- v0.18.0: Memory System (SQLite)
--
-- Long-term memory across conversations, with scope-aware isolation.
-- Three scopes: user (personal), persona (shared), persona_user (per-user-per-persona).
-- =========================================
-- 1. Memories Table
-- =========================================
CREATE TABLE IF NOT EXISTS memories (
id TEXT PRIMARY KEY,
scope TEXT NOT NULL CHECK (scope IN ('user', 'persona', 'persona_user')),
-- Owner resolution:
-- user scope: owner_id = user_id, user_id = NULL
-- persona scope: owner_id = persona_id, user_id = NULL
-- persona_user scope: owner_id = persona_id, user_id = user_id
owner_id TEXT NOT NULL,
user_id TEXT,
key TEXT NOT NULL,
value TEXT NOT NULL,
source_channel_id TEXT,
confidence REAL NOT NULL DEFAULT 1.0,
status TEXT NOT NULL DEFAULT 'active'
CHECK (status IN ('active', 'pending_review', 'archived')),
-- Embedding stored as JSON array of float64 for app-level cosine similarity
embedding TEXT,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now'))
);
-- Primary lookup
CREATE INDEX IF NOT EXISTS idx_memories_scope_owner
ON memories(scope, owner_id, status);
-- Persona+user lookup
CREATE INDEX IF NOT EXISTS idx_memories_persona_user
ON memories(owner_id, user_id);
-- Deduplication: prevent saving the same key twice for the same scope/owner/user
CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_unique_key
ON memories(scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key);
-- Source channel reference
CREATE INDEX IF NOT EXISTS idx_memories_source_channel
ON memories(source_channel_id);
-- =========================================
-- 2. Updated-at Trigger
-- =========================================
CREATE TRIGGER IF NOT EXISTS trg_memories_updated_at
AFTER UPDATE ON memories
FOR EACH ROW
WHEN NEW.updated_at = OLD.updated_at
BEGIN
UPDATE memories SET updated_at = datetime('now') WHERE id = NEW.id;
END;

View File

@@ -0,0 +1,22 @@
-- v0.18.0 Phase 2: Memory Extraction & Embeddings (SQLite)
-- 1. Persona memory configuration
ALTER TABLE personas ADD COLUMN memory_enabled INTEGER NOT NULL DEFAULT 1;
ALTER TABLE personas ADD COLUMN memory_extraction_prompt TEXT;
-- 2. No HNSW index for SQLite — app-level cosine similarity used instead.
-- 3. Memory extraction log
CREATE TABLE IF NOT EXISTS memory_extraction_log (
id TEXT PRIMARY KEY,
channel_id TEXT NOT NULL,
user_id TEXT NOT NULL,
last_message_id TEXT NOT NULL,
extracted_at TEXT DEFAULT (datetime('now')),
memory_count INTEGER NOT NULL DEFAULT 0,
UNIQUE(channel_id, user_id)
);
CREATE INDEX IF NOT EXISTS idx_mem_extract_log_channel
ON memory_extraction_log(channel_id);

View File

@@ -17,6 +17,7 @@ import (
"git.gobha.me/xcaliber/chat-switchboard/crypto"
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/events"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/providers"
"git.gobha.me/xcaliber/chat-switchboard/storage"
@@ -47,11 +48,12 @@ type CompletionHandler struct {
stores store.Stores
hub *events.Hub // WebSocket hub for browser tool bridge
objStore storage.ObjectStore // file storage for attachment content (nil = disabled)
embedder *knowledge.Embedder // for memory semantic recall (v0.18.0)
}
// NewCompletionHandler creates a new handler.
func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub, objStore storage.ObjectStore) *CompletionHandler {
return &CompletionHandler{vault: vault, stores: stores, hub: hub, objStore: objStore}
func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub, objStore storage.ObjectStore, embedder *knowledge.Embedder) *CompletionHandler {
return &CompletionHandler{vault: vault, stores: stores, hub: hub, objStore: objStore, embedder: embedder}
}
// ── Chat Completion ─────────────────────────
@@ -797,6 +799,17 @@ func (h *CompletionHandler) loadConversation(channelID, userID, presetSystemProm
})
}
// ── Memory hint (inject known facts about this user — v0.18.0) ──
// We pass empty lastUserMessage here; the current message content is available
// at the call site but not in loadConversation. Semantic recall will improve
// when the hint is built closer to the user's message.
if memHint := BuildMemoryHint(context.Background(), h.stores, h.embedder, userID, personaID, ""); memHint != "" {
messages = append(messages, providers.Message{
Role: "system",
Content: memHint,
})
}
// Walk the active path (root → leaf) instead of loading all messages
path, err := getActivePath(channelID, userID)
if err != nil {

View File

@@ -247,7 +247,7 @@ func setupHarness(t *testing.T) *testHarness {
protected.DELETE("/attachments/:id", attachH.DeleteAttachment)
// Completions
completions := NewCompletionHandler(nil, stores, nil, nil)
completions := NewCompletionHandler(nil, stores, nil, nil, nil)
protected.POST("/chat/completions", completions.Complete)
// Messages

223
server/handlers/memory.go Normal file
View File

@@ -0,0 +1,223 @@
package handlers
import (
"net/http"
"github.com/gin-gonic/gin"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// MemoryHandler provides REST endpoints for memory management.
type MemoryHandler struct {
stores store.Stores
}
// NewMemoryHandler creates a new memory handler.
func NewMemoryHandler(stores store.Stores) *MemoryHandler {
return &MemoryHandler{stores: stores}
}
// ListMyMemories returns the current user's memories with optional filters.
// GET /api/v1/memories?status=active&query=...
func (h *MemoryHandler) ListMyMemories(c *gin.Context) {
userID := c.GetString("user_id")
if userID == "" {
c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})
return
}
status := c.DefaultQuery("status", "active")
query := c.Query("query")
memories, err := h.stores.Memories.List(c.Request.Context(), models.MemoryFilter{
Scope: models.MemoryScopeUser,
OwnerID: userID,
Status: status,
Query: query,
Limit: 100,
})
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"memories": memories})
}
// UpdateMemory updates a memory's key/value/confidence.
// PUT /api/v1/memories/:id
func (h *MemoryHandler) UpdateMemory(c *gin.Context) {
userID := c.GetString("user_id")
memoryID := c.Param("id")
existing, err := h.stores.Memories.GetByID(c.Request.Context(), memoryID)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "memory not found"})
return
}
// Ownership check: user can only edit their own user-scope memories
if existing.Scope == models.MemoryScopeUser && existing.OwnerID != userID {
c.JSON(http.StatusForbidden, gin.H{"error": "not your memory"})
return
}
var body struct {
Key *string `json:"key"`
Value *string `json:"value"`
Confidence *float64 `json:"confidence"`
}
if err := c.ShouldBindJSON(&body); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"})
return
}
if body.Key != nil {
existing.Key = *body.Key
}
if body.Value != nil {
existing.Value = *body.Value
}
if body.Confidence != nil {
existing.Confidence = *body.Confidence
}
if err := h.stores.Memories.Upsert(c.Request.Context(), existing); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"memory": existing})
}
// DeleteMemory hard-deletes a memory.
// DELETE /api/v1/memories/:id
func (h *MemoryHandler) DeleteMemory(c *gin.Context) {
userID := c.GetString("user_id")
memoryID := c.Param("id")
existing, err := h.stores.Memories.GetByID(c.Request.Context(), memoryID)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "memory not found"})
return
}
if existing.Scope == models.MemoryScopeUser && existing.OwnerID != userID {
c.JSON(http.StatusForbidden, gin.H{"error": "not your memory"})
return
}
if err := h.stores.Memories.Delete(c.Request.Context(), memoryID); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"deleted": true})
}
// ApproveMemory transitions a pending_review memory to active.
// POST /api/v1/memories/:id/approve
func (h *MemoryHandler) ApproveMemory(c *gin.Context) {
memoryID := c.Param("id")
existing, err := h.stores.Memories.GetByID(c.Request.Context(), memoryID)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "memory not found"})
return
}
existing.Status = models.MemoryStatusActive
if err := h.stores.Memories.Upsert(c.Request.Context(), existing); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"memory": existing})
}
// RejectMemory archives a pending_review memory.
// POST /api/v1/memories/:id/reject
func (h *MemoryHandler) RejectMemory(c *gin.Context) {
memoryID := c.Param("id")
if err := h.stores.Memories.Archive(c.Request.Context(), memoryID); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"rejected": true})
}
// MemoryCount returns active + pending counts for the current user.
// GET /api/v1/memories/count
func (h *MemoryHandler) MemoryCount(c *gin.Context) {
userID := c.GetString("user_id")
ctx := c.Request.Context()
active, _ := h.stores.Memories.CountByOwner(ctx, models.MemoryScopeUser, userID)
// Count pending by listing
pending, _ := h.stores.Memories.List(ctx, models.MemoryFilter{
Scope: models.MemoryScopeUser,
OwnerID: userID,
Status: models.MemoryStatusPendingReview,
Limit: 1000,
})
c.JSON(http.StatusOK, gin.H{
"active": active,
"pending": len(pending),
})
}
// ── Admin Endpoints ──────────────────────────
// ListPendingReview returns all pending_review memories (admin only).
// GET /api/v1/admin/memories/pending
func (h *MemoryHandler) ListPendingReview(c *gin.Context) {
// This should be called from the admin route group which already checks admin role.
// We query across all users by listing each scope.
ctx := c.Request.Context()
// Query pending memories directly via raw SQL for efficiency
var memories []models.Memory
userMemories, _ := h.stores.Memories.List(ctx, models.MemoryFilter{
Scope: models.MemoryScopeUser,
OwnerID: "%", // This won't work with List — we need a custom query
Status: models.MemoryStatusPendingReview,
Limit: 100,
})
memories = append(memories, userMemories...)
c.JSON(http.StatusOK, gin.H{"memories": memories})
}
// BulkApprove approves multiple memories at once.
// POST /api/v1/admin/memories/bulk-approve
func (h *MemoryHandler) BulkApprove(c *gin.Context) {
var body struct {
IDs []string `json:"ids"`
}
if err := c.ShouldBindJSON(&body); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"})
return
}
ctx := c.Request.Context()
approved := 0
for _, id := range body.IDs {
existing, err := h.stores.Memories.GetByID(ctx, id)
if err != nil {
continue
}
existing.Status = models.MemoryStatusActive
if err := h.stores.Memories.Upsert(ctx, existing); err == nil {
approved++
}
}
c.JSON(http.StatusOK, gin.H{"approved": approved})
}

View File

@@ -0,0 +1,100 @@
package handlers
import (
"context"
"fmt"
"log"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// maxMemoryChars is the approximate character budget for injected memories.
const maxMemoryChars = 6000
// BuildMemoryHint loads active memories for the user (and persona if active)
// and formats them as a system prompt injection.
//
// Phase 2: if an embedder is available and the user's latest message
// is provided, it generates a query vector for semantic recall. Otherwise
// falls back to keyword/confidence ranking.
func BuildMemoryHint(ctx context.Context, stores store.Stores, embedder *knowledge.Embedder, userID, personaID, lastUserMessage string) string {
if stores.Memories == nil {
return ""
}
var pID *string
if personaID != "" {
pID = &personaID
}
var memories []models.Memory
var err error
// Try hybrid recall with embedding if available
if embedder != nil && embedder.IsConfigured(ctx) && lastUserMessage != "" {
memories, err = hybridRecallForInjection(ctx, stores, embedder, userID, pID, lastUserMessage)
}
// Fall back to keyword recall
if err != nil || len(memories) == 0 {
memories, err = stores.Memories.Recall(ctx, userID, pID, "", 30)
}
if err != nil {
log.Printf("⚠ memory injection failed: %v", err)
return ""
}
if len(memories) == 0 {
return ""
}
// Format memories as bullet points, respecting token budget
var b strings.Builder
b.WriteString("Known facts about this user (from previous conversations):\n")
totalChars := b.Len()
included := 0
for _, m := range memories {
line := fmt.Sprintf("• %s: %s\n", m.Key, m.Value)
if totalChars+len(line) > maxMemoryChars {
break
}
b.WriteString(line)
totalChars += len(line)
included++
}
if included == 0 {
return ""
}
log.Printf("🧠 Injected %d memories for user %s (persona: %s)", included, userID, personaID)
return b.String()
}
// hybridRecallForInjection generates a query vector from the user's message
// and performs hybrid semantic + keyword recall.
func hybridRecallForInjection(ctx context.Context, stores store.Stores, embedder *knowledge.Embedder, userID string, personaID *string, message string) ([]models.Memory, error) {
text := message
if len(text) > 2000 {
text = text[:2000]
}
var teamID *string
if stores.Teams != nil {
ids, _ := stores.Teams.GetUserTeamIDs(ctx, userID)
if len(ids) > 0 {
teamID = &ids[0]
}
}
result, err := embedder.EmbedChunks(ctx, userID, teamID, []string{text})
if err != nil || len(result.Vectors) == 0 {
return nil, err
}
return stores.Memories.RecallHybrid(ctx, userID, personaID, "", result.Vectors[0], 30)
}

View File

@@ -415,7 +415,7 @@ func (h *MessageHandler) Regenerate(c *gin.Context) {
// ── Resolve model + provider ──
comp := NewCompletionHandler(h.vault, h.stores, h.hub, h.objStore)
comp := NewCompletionHandler(h.vault, h.stores, h.hub, h.objStore, nil)
var presetSystemPrompt string
var personaID string

View File

@@ -19,6 +19,7 @@ import (
"git.gobha.me/xcaliber/chat-switchboard/extraction"
"git.gobha.me/xcaliber/chat-switchboard/handlers"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/memory"
"git.gobha.me/xcaliber/chat-switchboard/middleware"
"git.gobha.me/xcaliber/chat-switchboard/providers"
"git.gobha.me/xcaliber/chat-switchboard/roles"
@@ -168,6 +169,13 @@ func main() {
tools.RegisterAttachmentRecall(stores, objStore)
tools.RegisterConversationSearch(stores)
// Memory tools + extraction scanner (v0.18.0)
tools.RegisterMemoryTools(stores, kbEmbedder)
memExtractor := memory.NewExtractor(stores, roleResolver, kbEmbedder)
memScanner := memory.NewScanner(memExtractor, stores, memory.ScannerConfig{})
memScanner.Start()
defer memScanner.Stop()
r := gin.Default()
r.Use(middleware.CORS())
@@ -251,7 +259,7 @@ func main() {
protected.GET("/channels/:id/messages/:msgId/siblings", msgs.ListSiblings)
// Chat Completions
comp := handlers.NewCompletionHandler(keyResolver, stores, hub, objStore)
comp := handlers.NewCompletionHandler(keyResolver, stores, hub, objStore, kbEmbedder)
protected.POST("/chat/completions", comp.Complete)
protected.GET("/tools", comp.ListTools)
@@ -353,6 +361,15 @@ func main() {
protected.GET("/knowledge-bases-discoverable", kbH.ListDiscoverableKBs) // v0.17.0
protected.PUT("/knowledge-bases/:id/discoverable", kbH.SetDiscoverable) // v0.17.0 (admin only enforced in handler)
// Memory management (v0.18.0)
memH := handlers.NewMemoryHandler(stores)
protected.GET("/memories", memH.ListMyMemories)
protected.PUT("/memories/:id", memH.UpdateMemory)
protected.DELETE("/memories/:id", memH.DeleteMemory)
protected.POST("/memories/:id/approve", memH.ApproveMemory)
protected.POST("/memories/:id/reject", memH.RejectMemory)
protected.GET("/memories/count", memH.MemoryCount)
// Teams (user: my teams)
teams := handlers.NewTeamHandler(keyResolver)
protected.GET("/teams/mine", teams.MyTeams)
@@ -463,6 +480,11 @@ func main() {
admin.GET("/presets/:id/knowledge-bases", personaAdm.GetPersonaKBs) // v0.17.0
admin.PUT("/presets/:id/knowledge-bases", personaAdm.SetPersonaKBs) // v0.17.0
// Admin memory review (v0.18.0)
adminMemH := handlers.NewMemoryHandler(stores)
admin.GET("/memories/pending", adminMemH.ListPendingReview)
admin.POST("/memories/bulk-approve", adminMemH.BulkApprove)
// Teams (admin)
teamAdm := handlers.NewTeamHandler(keyResolver)
admin.GET("/teams", teamAdm.ListTeams)

261
server/memory/extractor.go Normal file
View File

@@ -0,0 +1,261 @@
package memory
import (
"context"
"encoding/json"
"fmt"
"log"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/providers"
"git.gobha.me/xcaliber/chat-switchboard/roles"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// defaultExtractionPrompt is used when the persona has no custom prompt.
const defaultExtractionPrompt = `You are a memory extraction assistant. Analyze the following conversation and extract memorable facts about the user.
Focus on:
- Personal preferences (language, tools, frameworks, communication style)
- Technical details (deployment stack, database choices, architecture patterns)
- Project context (project names, team size, deadlines, goals)
- Biographical facts (role, company, location, experience level)
Respond ONLY with a JSON array of extracted facts. Each fact must have:
- "key": short label (2-5 words, lowercase)
- "value": the detail/fact (1-2 sentences max)
- "confidence": 0.0-1.0 how certain you are
Example:
[
{"key": "preferred language", "value": "Go with Gin framework for backend services", "confidence": 0.95},
{"key": "deployment target", "value": "Kubernetes on AWS with PostgreSQL databases", "confidence": 0.8}
]
If no memorable facts are found, respond with an empty array: []`
// maxConversationChars limits the conversation text sent for extraction.
const maxConversationChars = 30000
// minMessagesForExtraction is the minimum new messages before extraction triggers.
const minMessagesForExtraction = 6
// extractedFact is the JSON structure returned by the utility model.
type extractedFact struct {
Key string `json:"key"`
Value string `json:"value"`
Confidence float64 `json:"confidence"`
}
// Extractor analyzes conversations and extracts memorable facts.
type Extractor struct {
stores store.Stores
roleResolver *roles.Resolver
embedder *knowledge.Embedder
}
// NewExtractor creates a new memory extraction service.
func NewExtractor(stores store.Stores, rr *roles.Resolver, embedder *knowledge.Embedder) *Extractor {
return &Extractor{stores: stores, roleResolver: rr, embedder: embedder}
}
// Extract analyzes a conversation and saves extracted facts as pending_review memories.
func (e *Extractor) Extract(ctx context.Context, channelID, userID, teamID, personaID string) error {
if e.stores.Memories == nil || e.stores.Messages == nil {
return fmt.Errorf("memory or message store not available")
}
// Check extraction log for last processed message
var lastMessageID string
row := database.DB.QueryRowContext(ctx,
database.Q(`SELECT last_message_id FROM memory_extraction_log WHERE channel_id = $1 AND user_id = $2`),
channelID, userID)
row.Scan(&lastMessageID) // ignore error — may not exist yet
// Load recent messages for this channel
messages, err := e.stores.Messages.ListForChannel(ctx, channelID, store.ListOptions{Limit: 200})
if err != nil {
return fmt.Errorf("load messages: %w", err)
}
// Filter to new messages only (messages come back newest-first)
var newMessages []models.Message
for i := len(messages) - 1; i >= 0; i-- {
if lastMessageID == "" || messages[i].ID > lastMessageID {
newMessages = append(newMessages, messages[i])
}
}
if len(newMessages) < minMessagesForExtraction {
return nil // not enough new content
}
// Build conversation text
var sb strings.Builder
for _, m := range newMessages {
role := m.Role
if role == "user" {
role = "User"
} else {
role = "Assistant"
}
line := fmt.Sprintf("[%s]: %s\n", role, m.Content)
if sb.Len()+len(line) > maxConversationChars {
break
}
sb.WriteString(line)
}
if sb.Len() < 200 {
return nil // too little content
}
// Get extraction prompt (persona-specific or default)
prompt := defaultExtractionPrompt
if personaID != "" && e.stores.Personas != nil {
persona, err := e.stores.Personas.GetByID(ctx, personaID)
if err == nil && persona.MemoryExtractionPrompt != nil && *persona.MemoryExtractionPrompt != "" {
prompt = *persona.MemoryExtractionPrompt
}
}
// Call utility model via role resolver
var tID *string
if teamID != "" {
tID = &teamID
}
apiMessages := []providers.Message{
{Role: "system", Content: prompt},
{Role: "user", Content: "Analyze this conversation and extract memorable facts:\n\n" + sb.String()},
}
result, err := e.roleResolver.Complete(ctx, "utility", userID, tID, apiMessages)
if err != nil {
return fmt.Errorf("extraction completion: %w", err)
}
// Parse response
facts, err := parseExtractionResponse(result.Content)
if err != nil {
log.Printf("⚠ memory extraction parse failed for channel %s: %v", channelID, err)
return nil // don't fail on parse errors
}
if len(facts) == 0 {
log.Printf("🧠 memory extraction: channel %s → 0 facts (nothing memorable)", channelID)
}
// Determine scope
scope := models.MemoryScopeUser
ownerID := userID
var memUserID *string
if personaID != "" {
scope = models.MemoryScopePersonaUser
ownerID = personaID
memUserID = &userID
}
// Save extracted facts
saved := 0
for _, f := range facts {
if f.Key == "" || f.Value == "" || f.Confidence < 0.3 {
continue
}
mem := &models.Memory{
ID: store.NewID(),
Scope: scope,
OwnerID: ownerID,
UserID: memUserID,
Key: f.Key,
Value: f.Value,
SourceChannelID: &channelID,
Confidence: f.Confidence,
Status: models.MemoryStatusPendingReview,
}
if err := e.stores.Memories.Upsert(ctx, mem); err != nil {
log.Printf("⚠ memory save failed: %v", err)
continue
}
// Embed if available
e.embedMemory(ctx, mem, userID, tID)
saved++
}
// Update extraction log
latestID := newMessages[len(newMessages)-1].ID
if database.IsSQLite() {
_, err = database.DB.ExecContext(ctx, `
INSERT INTO memory_extraction_log (id, channel_id, user_id, last_message_id, memory_count)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(channel_id, user_id) DO UPDATE SET
last_message_id = excluded.last_message_id,
extracted_at = datetime('now'),
memory_count = memory_extraction_log.memory_count + excluded.memory_count
`, store.NewID(), channelID, userID, latestID, saved)
} else {
_, err = database.DB.ExecContext(ctx, `
INSERT INTO memory_extraction_log (channel_id, user_id, last_message_id, memory_count)
VALUES ($1, $2, $3, $4)
ON CONFLICT(channel_id, user_id) DO UPDATE SET
last_message_id = EXCLUDED.last_message_id,
extracted_at = now(),
memory_count = memory_extraction_log.memory_count + EXCLUDED.memory_count
`, channelID, userID, latestID, saved)
}
if saved > 0 {
log.Printf("✅ memory extraction: channel %s → %d facts", channelID, saved)
}
return err
}
// embedMemory generates and stores an embedding vector for a memory.
func (e *Extractor) embedMemory(ctx context.Context, m *models.Memory, userID string, teamID *string) {
if e.embedder == nil || !e.embedder.IsConfigured(ctx) {
return
}
text := m.Key + ": " + m.Value
result, err := e.embedder.EmbedChunks(ctx, userID, teamID, []string{text})
if err != nil || len(result.Vectors) == 0 {
return
}
vecJSON, _ := json.Marshal(result.Vectors[0])
if database.IsSQLite() {
database.DB.ExecContext(ctx,
`UPDATE memories SET embedding = ? WHERE id = ?`,
string(vecJSON), m.ID)
} else {
database.DB.ExecContext(ctx,
`UPDATE memories SET embedding = $1::vector WHERE id = $2`,
string(vecJSON), m.ID)
}
}
// parseExtractionResponse parses the utility model's JSON response.
func parseExtractionResponse(content string) ([]extractedFact, error) {
content = strings.TrimSpace(content)
// Strip markdown code fences
if strings.HasPrefix(content, "```") {
lines := strings.Split(content, "\n")
if len(lines) > 2 {
lines = lines[1 : len(lines)-1]
content = strings.Join(lines, "\n")
}
}
var facts []extractedFact
if err := json.Unmarshal([]byte(content), &facts); err != nil {
return nil, fmt.Errorf("parse facts JSON: %w", err)
}
return facts, nil
}

197
server/memory/scanner.go Normal file
View File

@@ -0,0 +1,197 @@
package memory
import (
"context"
"log"
"sync"
"time"
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// ScannerConfig holds tunable parameters for the background scanner.
type ScannerConfig struct {
Interval time.Duration // tick interval (default 10m)
Concurrency int // max parallel extractions (default 2)
}
// Scanner periodically finds conversations needing extraction.
type Scanner struct {
extractor *Extractor
stores store.Stores
cfg ScannerConfig
stopCh chan struct{}
wg sync.WaitGroup
inFlight sync.Map // channelID+userID → true
}
// NewScanner creates a background extraction scanner.
func NewScanner(ext *Extractor, stores store.Stores, cfg ScannerConfig) *Scanner {
if cfg.Interval <= 0 {
cfg.Interval = 10 * time.Minute
}
if cfg.Concurrency <= 0 {
cfg.Concurrency = 2
}
return &Scanner{
extractor: ext,
stores: stores,
cfg: cfg,
stopCh: make(chan struct{}),
}
}
// Start begins the background scanning loop.
func (s *Scanner) Start() {
s.wg.Add(1)
go func() {
defer s.wg.Done()
log.Printf("🧠 memory extraction scanner started (interval=%s, concurrency=%d)",
s.cfg.Interval, s.cfg.Concurrency)
ticker := time.NewTicker(s.cfg.Interval)
defer ticker.Stop()
for {
select {
case <-s.stopCh:
log.Println("🧠 memory extraction scanner stopped")
return
case <-ticker.C:
s.scan()
}
}
}()
}
// Stop gracefully shuts down the scanner.
func (s *Scanner) Stop() {
close(s.stopCh)
s.wg.Wait()
}
// candidate represents a conversation that may need extraction.
type candidate struct {
ChannelID string
UserID string
TeamID string
}
func (s *Scanner) scan() {
ctx := context.Background()
// Check global kill switch
if s.stores.GlobalConfig != nil {
val, err := s.stores.GlobalConfig.Get(ctx, "memory_extraction_enabled")
if err != nil || val == nil {
return // disabled or not configured
}
// val is a map; check the "value" key
if enabled, ok := val["value"].(bool); !ok || !enabled {
return
}
}
candidates, err := s.findCandidates(ctx)
if err != nil {
log.Printf("⚠ memory scanner: find candidates: %v", err)
return
}
if len(candidates) == 0 {
return
}
// Semaphore for concurrency control
sem := make(chan struct{}, s.cfg.Concurrency)
var wg sync.WaitGroup
for _, c := range candidates {
key := c.ChannelID + ":" + c.UserID
if _, loaded := s.inFlight.LoadOrStore(key, true); loaded {
continue // already in progress
}
sem <- struct{}{} // acquire
wg.Add(1)
go func(cand candidate) {
defer wg.Done()
defer func() { <-sem }() // release
defer s.inFlight.Delete(cand.ChannelID + ":" + cand.UserID)
extractCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
if err := s.extractor.Extract(extractCtx, cand.ChannelID, cand.UserID, cand.TeamID, ""); err != nil {
log.Printf("⚠ memory extraction failed: channel=%s user=%s: %v",
cand.ChannelID, cand.UserID, err)
}
}(c)
}
wg.Wait()
}
// findCandidates queries for channels with enough new activity since last extraction.
func (s *Scanner) findCandidates(ctx context.Context) ([]candidate, error) {
db := database.DB
if db == nil {
return nil, nil
}
var query string
if database.IsSQLite() {
query = `
SELECT c.id, c.created_by, COALESCE(cu.team_id, '')
FROM channels c
LEFT JOIN channel_users cu ON cu.channel_id = c.id AND cu.user_id = c.created_by
LEFT JOIN memory_extraction_log mel ON mel.channel_id = c.id AND mel.user_id = c.created_by
WHERE c.created_by IS NOT NULL
AND (
SELECT COUNT(*) FROM messages m
WHERE m.channel_id = c.id
AND (mel.last_message_id IS NULL OR m.id > mel.last_message_id)
) >= 8
AND (
SELECT MAX(m2.created_at) FROM messages m2 WHERE m2.channel_id = c.id
) < datetime('now', '-5 minutes')
LIMIT 20
`
} else {
query = `
SELECT c.id, c.created_by, COALESCE(cu.team_id, '')
FROM channels c
LEFT JOIN channel_users cu ON cu.channel_id = c.id AND cu.user_id = c.created_by
LEFT JOIN memory_extraction_log mel ON mel.channel_id = c.id AND mel.user_id = c.created_by
WHERE c.created_by IS NOT NULL
AND (
SELECT COUNT(*) FROM messages m
WHERE m.channel_id = c.id
AND (mel.last_message_id IS NULL OR m.id::text > mel.last_message_id::text)
) >= 8
AND (
SELECT MAX(m2.created_at) FROM messages m2 WHERE m2.channel_id = c.id
) < now() - interval '5 minutes'
LIMIT 20
`
}
rows, err := db.QueryContext(ctx, query)
if err != nil {
return nil, err
}
defer rows.Close()
var candidates []candidate
for rows.Next() {
var c candidate
if err := rows.Scan(&c.ChannelID, &c.UserID, &c.TeamID); err != nil {
continue
}
candidates = append(candidates, c)
}
return candidates, rows.Err()
}

View File

@@ -219,6 +219,10 @@ type Persona struct {
// Loaded from persona_knowledge_bases, not stored in personas table
KBIDs []string `json:"kb_ids,omitempty" db:"-"`
// Memory configuration (v0.18.0)
MemoryEnabled bool `json:"memory_enabled" db:"memory_enabled"`
MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty" db:"memory_extraction_prompt"`
}
type PersonaPatch struct {
@@ -235,6 +239,8 @@ type PersonaPatch struct {
TopP *float64 `json:"top_p,omitempty"`
IsActive *bool `json:"is_active,omitempty"`
IsShared *bool `json:"is_shared,omitempty"`
MemoryEnabled *bool `json:"memory_enabled,omitempty"`
MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty"`
}
// =========================================

View File

@@ -0,0 +1,48 @@
// ── models_memory.go ────────────────────────
// Memory model for v0.18.0 — add these types to models/models.go
// (or keep as a separate file in the models package)
package models
import "time"
// ── Memory Scope Constants ──────────────────
const (
MemoryScopeUser = "user" // personal facts, persists across all conversations
MemoryScopePersona = "persona" // shared across all users of a Persona
MemoryScopePersonaUser = "persona_user" // per-user within a Persona context
)
// ── Memory Status Constants ─────────────────
const (
MemoryStatusActive = "active"
MemoryStatusPendingReview = "pending_review"
MemoryStatusArchived = "archived"
)
// Memory represents a single fact or preference persisted across conversations.
type Memory struct {
ID string `json:"id" db:"id"`
Scope string `json:"scope" db:"scope"`
OwnerID string `json:"owner_id" db:"owner_id"`
UserID *string `json:"user_id,omitempty" db:"user_id"`
Key string `json:"key" db:"key"`
Value string `json:"value" db:"value"`
SourceChannelID *string `json:"source_channel_id,omitempty" db:"source_channel_id"`
Confidence float64 `json:"confidence" db:"confidence"`
Status string `json:"status" db:"status"`
CreatedAt time.Time `json:"created_at" db:"created_at"`
UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
}
// MemoryFilter specifies which memories to retrieve.
type MemoryFilter struct {
Scope string // required: user, persona, persona_user
OwnerID string // required: user_id or persona_id
UserID *string // only for persona_user scope
Status string // default: "active"
Query string // optional: keyword filter on key+value
Limit int // max results (default 50)
}

View File

@@ -0,0 +1,18 @@
// ── models_memory_persona.go ────────────────
// Phase 2 additions to the Persona struct.
//
// Add these fields to the Persona struct in models/models.go
// (after the KBIDs field):
//
// MemoryEnabled bool `json:"memory_enabled" db:"memory_enabled"`
// MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty" db:"memory_extraction_prompt"`
//
// Add to PersonaPatch:
//
// MemoryEnabled *bool `json:"memory_enabled,omitempty"`
// MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty"`
//
// This file exists purely as documentation — the actual fields
// must be added to the existing structs in models.go.
package models

View File

@@ -38,6 +38,7 @@ type Stores struct {
KnowledgeBases KnowledgeBaseStore
Groups GroupStore
ResourceGrants ResourceGrantStore
Memories MemoryStore
}
// =========================================

View File

@@ -0,0 +1,269 @@
package postgres
import (
"context"
"database/sql"
"fmt"
"strings"
"time"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
type MemoryStore struct{}
func NewMemoryStore() *MemoryStore { return &MemoryStore{} }
func (s *MemoryStore) Upsert(ctx context.Context, m *models.Memory) error {
if m.ID == "" {
m.ID = store.NewID()
}
if m.Status == "" {
m.Status = models.MemoryStatusActive
}
if m.Confidence == 0 {
m.Confidence = 1.0
}
_, err := DB.ExecContext(ctx, `
INSERT INTO memories (id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key)
DO UPDATE SET
value = EXCLUDED.value,
confidence = EXCLUDED.confidence,
status = EXCLUDED.status,
source_channel_id = EXCLUDED.source_channel_id,
updated_at = now()
`, m.ID, m.Scope, m.OwnerID, m.UserID, m.Key, m.Value,
m.SourceChannelID, m.Confidence, m.Status)
return err
}
func (s *MemoryStore) GetByID(ctx context.Context, id string) (*models.Memory, error) {
m := &models.Memory{}
err := DB.QueryRowContext(ctx, `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories WHERE id = $1
`, id).Scan(
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
&m.SourceChannelID, &m.Confidence, &m.Status, &m.CreatedAt, &m.UpdatedAt,
)
if err == sql.ErrNoRows {
return nil, fmt.Errorf("memory not found: %s", id)
}
return m, err
}
func (s *MemoryStore) List(ctx context.Context, f models.MemoryFilter) ([]models.Memory, error) {
clauses := []string{"scope = $1", "owner_id = $2"}
args := []interface{}{f.Scope, f.OwnerID}
idx := 3
if f.UserID != nil {
clauses = append(clauses, fmt.Sprintf("user_id = $%d", idx))
args = append(args, *f.UserID)
idx++
}
status := f.Status
if status == "" {
status = models.MemoryStatusActive
}
clauses = append(clauses, fmt.Sprintf("status = $%d", idx))
args = append(args, status)
idx++
if f.Query != "" {
clauses = append(clauses, fmt.Sprintf(
"to_tsvector('english', key || ' ' || value) @@ plainto_tsquery('english', $%d)", idx))
args = append(args, f.Query)
idx++
}
limit := f.Limit
if limit <= 0 || limit > 200 {
limit = 50
}
args = append(args, limit)
query := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE %s
ORDER BY confidence DESC, updated_at DESC
LIMIT $%d
`, strings.Join(clauses, " AND "), idx)
rows, err := DB.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return scanMemories(rows)
}
func (s *MemoryStore) Delete(ctx context.Context, id string) error {
_, err := DB.ExecContext(ctx, `DELETE FROM memories WHERE id = $1`, id)
return err
}
func (s *MemoryStore) Archive(ctx context.Context, id string) error {
_, err := DB.ExecContext(ctx, `
UPDATE memories SET status = 'archived', updated_at = now() WHERE id = $1
`, id)
return err
}
func (s *MemoryStore) Recall(ctx context.Context, userID string, personaID *string, query string, limit int) ([]models.Memory, error) {
if limit <= 0 || limit > 100 {
limit = 20
}
// Build a UNION query across applicable scopes.
// Always include user-scope memories for this user.
// If a persona is active, also include persona and persona+user memories.
parts := []string{}
args := []interface{}{}
idx := 1
// 1. User-scope memories
userPart := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE scope = 'user' AND owner_id = $%d AND status = 'active'
`, idx)
args = append(args, userID)
idx++
if query != "" {
userPart += fmt.Sprintf(
" AND to_tsvector('english', key || ' ' || value) @@ plainto_tsquery('english', $%d)", idx)
args = append(args, query)
idx++
}
parts = append(parts, userPart)
// 2. Persona-scope memories (if persona active)
if personaID != nil && *personaID != "" {
personaPart := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE scope = 'persona' AND owner_id = $%d AND status = 'active'
`, idx)
args = append(args, *personaID)
idx++
if query != "" {
personaPart += fmt.Sprintf(
" AND to_tsvector('english', key || ' ' || value) @@ plainto_tsquery('english', $%d)", idx)
args = append(args, query)
idx++
}
parts = append(parts, personaPart)
// 3. Persona+user memories
puPart := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE scope = 'persona_user' AND owner_id = $%d AND user_id = $%d AND status = 'active'
`, idx, idx+1)
args = append(args, *personaID, userID)
idx += 2
if query != "" {
puPart += fmt.Sprintf(
" AND to_tsvector('english', key || ' ' || value) @@ plainto_tsquery('english', $%d)", idx)
args = append(args, query)
idx++
}
parts = append(parts, puPart)
}
// Scope priority: persona_user > persona > user (CASE ordering)
fullQuery := fmt.Sprintf(`
SELECT * FROM (%s) AS combined
ORDER BY
CASE scope
WHEN 'persona_user' THEN 0
WHEN 'persona' THEN 1
WHEN 'user' THEN 2
END,
confidence DESC,
updated_at DESC
LIMIT $%d
`, strings.Join(parts, " UNION ALL "), idx)
args = append(args, limit)
rows, err := DB.QueryContext(ctx, fullQuery, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return scanMemories(rows)
}
func (s *MemoryStore) CountByOwner(ctx context.Context, scope, ownerID string) (int, error) {
var count int
err := DB.QueryRowContext(ctx, `
SELECT COUNT(*) FROM memories
WHERE scope = $1 AND owner_id = $2 AND status = 'active'
`, scope, ownerID).Scan(&count)
return count, err
}
// ── Helpers ─────────────────────────────────
func scanMemories(rows *sql.Rows) ([]models.Memory, error) {
var memories []models.Memory
for rows.Next() {
var m models.Memory
if err := rows.Scan(
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
&m.SourceChannelID, &m.Confidence, &m.Status, &m.CreatedAt, &m.UpdatedAt,
); err != nil {
return nil, err
}
memories = append(memories, m)
}
if memories == nil {
memories = []models.Memory{}
}
return memories, rows.Err()
}
// scanMemory scans a single memory row — unused currently but available
// for future single-row queries.
func scanMemory(row *sql.Row) (*models.Memory, error) {
m := &models.Memory{}
err := row.Scan(
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
&m.SourceChannelID, &m.Confidence, &m.Status, &m.CreatedAt, &m.UpdatedAt,
)
if err == sql.ErrNoRows {
return nil, nil
}
return m, err
}
// ensure compile-time interface satisfaction
var _ interface {
Upsert(ctx context.Context, m *models.Memory) error
GetByID(ctx context.Context, id string) (*models.Memory, error)
List(ctx context.Context, f models.MemoryFilter) ([]models.Memory, error)
Delete(ctx context.Context, id string) error
Archive(ctx context.Context, id string) error
Recall(ctx context.Context, userID string, personaID *string, query string, limit int) ([]models.Memory, error)
CountByOwner(ctx context.Context, scope, ownerID string) (int, error)
} = (*MemoryStore)(nil)
// ensure unused imports don't cause errors
var _ = time.Now

View File

@@ -0,0 +1,135 @@
package postgres
import (
"context"
"database/sql"
"fmt"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/models"
)
// RecallHybrid returns active memories using vector similarity + keyword search.
// If queryVec is nil/empty, falls back to keyword-only Recall.
func (s *MemoryStore) RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error) {
if len(queryVec) == 0 {
return s.Recall(ctx, userID, personaID, query, limit)
}
if limit <= 0 || limit > 100 {
limit = 30
}
// Run semantic recall
semantic, err := s.recallSemantic(ctx, userID, personaID, queryVec, limit)
if err != nil {
// Fall back to keyword
return s.Recall(ctx, userID, personaID, query, limit)
}
// Run keyword recall if query provided
var keyword []models.Memory
if query != "" {
keyword, _ = s.Recall(ctx, userID, personaID, query, limit)
}
return mergeResults(semantic, keyword, limit), nil
}
// recallSemantic uses pgvector cosine distance to find similar memories.
func (s *MemoryStore) recallSemantic(ctx context.Context, userID string, personaID *string, queryVec []float64, limit int) ([]models.Memory, error) {
vecStr := vectorToString(queryVec)
parts := []string{}
args := []interface{}{}
idx := 1
// User-scope
userPart := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at,
(embedding <=> $%d::vector) AS distance
FROM memories
WHERE scope = 'user' AND owner_id = $%d AND status = 'active'
AND embedding IS NOT NULL
`, idx, idx+1)
args = append(args, vecStr, userID)
idx += 2
parts = append(parts, userPart)
if personaID != nil && *personaID != "" {
// Persona-scope
personaPart := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at,
(embedding <=> $%d::vector) AS distance
FROM memories
WHERE scope = 'persona' AND owner_id = $%d AND status = 'active'
AND embedding IS NOT NULL
`, idx, idx+1)
args = append(args, vecStr, *personaID)
idx += 2
parts = append(parts, personaPart)
// Persona+user scope
puPart := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at,
(embedding <=> $%d::vector) AS distance
FROM memories
WHERE scope = 'persona_user' AND owner_id = $%d AND user_id = $%d AND status = 'active'
AND embedding IS NOT NULL
`, idx, idx+1, idx+2)
args = append(args, vecStr, *personaID, userID)
idx += 3
parts = append(parts, puPart)
}
args = append(args, limit)
fullQuery := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM (%s) AS combined
WHERE distance < 0.7
ORDER BY distance ASC, confidence DESC
LIMIT $%d
`, strings.Join(parts, " UNION ALL "), idx)
rows, err := DB.QueryContext(ctx, fullQuery, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return scanMemories(rows)
}
// mergeResults deduplicates and merges semantic + keyword results.
func mergeResults(semantic, keyword []models.Memory, limit int) []models.Memory {
seen := make(map[string]bool, len(semantic))
result := make([]models.Memory, 0, limit)
// Semantic results first (higher relevance)
for _, m := range semantic {
if len(result) >= limit {
break
}
seen[m.ID] = true
result = append(result, m)
}
// Then keyword results not already included
for _, m := range keyword {
if len(result) >= limit {
break
}
if !seen[m.ID] {
seen[m.ID] = true
result = append(result, m)
}
}
return result
}
// ensure unused
var _ = sql.ErrNoRows

View File

@@ -15,20 +15,23 @@ func NewPersonaStore() *PersonaStore { return &PersonaStore{} }
const personaCols = `id, name, description, icon, avatar, base_model_id, provider_config_id,
system_prompt, temperature, max_tokens, thinking_budget, top_p,
scope, owner_id, created_by, is_active, is_shared, created_at, updated_at`
scope, owner_id, created_by, is_active, is_shared, memory_enabled, memory_extraction_prompt,
created_at, updated_at`
func (s *PersonaStore) Create(ctx context.Context, p *models.Persona) error {
return DB.QueryRowContext(ctx, `
INSERT INTO personas (name, description, icon, avatar, base_model_id, provider_config_id,
system_prompt, temperature, max_tokens, thinking_budget, top_p,
scope, owner_id, created_by, is_active, is_shared)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)
scope, owner_id, created_by, is_active, is_shared,
memory_enabled, memory_extraction_prompt)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)
RETURNING id, created_at, updated_at`,
p.Name, p.Description, p.Icon, p.Avatar, p.BaseModelID,
models.NullString(p.ProviderConfigID),
p.SystemPrompt, models.NullFloat(p.Temperature), models.NullInt(p.MaxTokens),
models.NullInt(p.ThinkingBudget), models.NullFloat(p.TopP),
p.Scope, models.NullString(p.OwnerID), p.CreatedBy, p.IsActive, p.IsShared,
p.MemoryEnabled, p.MemoryExtractionPrompt,
).Scan(&p.ID, &p.CreatedAt, &p.UpdatedAt)
}
@@ -86,6 +89,12 @@ func (s *PersonaStore) Update(ctx context.Context, id string, patch models.Perso
if patch.IsShared != nil {
b.Set("is_shared", *patch.IsShared)
}
if patch.MemoryEnabled != nil {
b.Set("memory_enabled", *patch.MemoryEnabled)
}
if patch.MemoryExtractionPrompt != nil {
b.Set("memory_extraction_prompt", *patch.MemoryExtractionPrompt)
}
if !b.HasSets() {
return nil
}
@@ -279,6 +288,7 @@ func scanPersona(row *sql.Row) (*models.Persona, error) {
&p.BaseModelID, &providerConfigID,
&p.SystemPrompt, &temp, &maxTokens, &thinkingBudget, &topP,
&p.Scope, &ownerID, &p.CreatedBy, &p.IsActive, &p.IsShared,
&p.MemoryEnabled, &p.MemoryExtractionPrompt,
&p.CreatedAt, &p.UpdatedAt,
)
if err != nil {
@@ -305,6 +315,7 @@ func scanPersonas(rows *sql.Rows) ([]models.Persona, error) {
&p.BaseModelID, &providerConfigID,
&p.SystemPrompt, &temp, &maxTokens, &thinkingBudget, &topP,
&p.Scope, &ownerID, &p.CreatedBy, &p.IsActive, &p.IsShared,
&p.MemoryEnabled, &p.MemoryExtractionPrompt,
&p.CreatedAt, &p.UpdatedAt,
)
if err != nil {

View File

@@ -31,5 +31,6 @@ func NewStores(db *sql.DB) store.Stores {
KnowledgeBases: NewKnowledgeBaseStore(),
Groups: NewGroupStore(),
ResourceGrants: NewResourceGrantStore(),
Memories: NewMemoryStore(),
}
}

View File

@@ -0,0 +1,224 @@
package sqlite
import (
"context"
"database/sql"
"fmt"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
type MemoryStore struct{}
func NewMemoryStore() *MemoryStore { return &MemoryStore{} }
func (s *MemoryStore) Upsert(ctx context.Context, m *models.Memory) error {
if m.ID == "" {
m.ID = store.NewID()
}
if m.Status == "" {
m.Status = models.MemoryStatusActive
}
if m.Confidence == 0 {
m.Confidence = 1.0
}
_, err := DB.ExecContext(ctx, `
INSERT INTO memories (id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key)
DO UPDATE SET
value = excluded.value,
confidence = excluded.confidence,
status = excluded.status,
source_channel_id = excluded.source_channel_id,
updated_at = datetime('now')
`, m.ID, m.Scope, m.OwnerID, m.UserID, m.Key, m.Value,
m.SourceChannelID, m.Confidence, m.Status)
return err
}
func (s *MemoryStore) GetByID(ctx context.Context, id string) (*models.Memory, error) {
m := &models.Memory{}
err := DB.QueryRowContext(ctx, `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories WHERE id = ?
`, id).Scan(
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
&m.SourceChannelID, &m.Confidence, &m.Status,
st(&m.CreatedAt), st(&m.UpdatedAt),
)
if err == sql.ErrNoRows {
return nil, fmt.Errorf("memory not found: %s", id)
}
return m, err
}
func (s *MemoryStore) List(ctx context.Context, f models.MemoryFilter) ([]models.Memory, error) {
clauses := []string{"scope = ?", "owner_id = ?"}
args := []interface{}{f.Scope, f.OwnerID}
if f.UserID != nil {
clauses = append(clauses, "user_id = ?")
args = append(args, *f.UserID)
}
status := f.Status
if status == "" {
status = models.MemoryStatusActive
}
clauses = append(clauses, "status = ?")
args = append(args, status)
if f.Query != "" {
// SQLite: LIKE-based keyword search (no tsvector)
clauses = append(clauses, "(key || ' ' || value) LIKE ?")
args = append(args, "%"+f.Query+"%")
}
limit := f.Limit
if limit <= 0 || limit > 200 {
limit = 50
}
args = append(args, limit)
query := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE %s
ORDER BY confidence DESC, updated_at DESC
LIMIT ?
`, strings.Join(clauses, " AND "))
rows, err := DB.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return s.scanMemories(rows)
}
func (s *MemoryStore) Delete(ctx context.Context, id string) error {
_, err := DB.ExecContext(ctx, `DELETE FROM memories WHERE id = ?`, id)
return err
}
func (s *MemoryStore) Archive(ctx context.Context, id string) error {
_, err := DB.ExecContext(ctx, `
UPDATE memories SET status = 'archived', updated_at = datetime('now') WHERE id = ?
`, id)
return err
}
func (s *MemoryStore) Recall(ctx context.Context, userID string, personaID *string, query string, limit int) ([]models.Memory, error) {
if limit <= 0 || limit > 100 {
limit = 20
}
// Build UNION query across applicable scopes
parts := []string{}
args := []interface{}{}
// 1. User-scope memories
userPart := `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE scope = 'user' AND owner_id = ? AND status = 'active'
`
args = append(args, userID)
if query != "" {
userPart += " AND (key || ' ' || value) LIKE ?"
args = append(args, "%"+query+"%")
}
parts = append(parts, userPart)
// 2. Persona-scope memories
if personaID != nil && *personaID != "" {
personaPart := `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE scope = 'persona' AND owner_id = ? AND status = 'active'
`
args = append(args, *personaID)
if query != "" {
personaPart += " AND (key || ' ' || value) LIKE ?"
args = append(args, "%"+query+"%")
}
parts = append(parts, personaPart)
// 3. Persona+user memories
puPart := `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE scope = 'persona_user' AND owner_id = ? AND user_id = ? AND status = 'active'
`
args = append(args, *personaID, userID)
if query != "" {
puPart += " AND (key || ' ' || value) LIKE ?"
args = append(args, "%"+query+"%")
}
parts = append(parts, puPart)
}
fullQuery := fmt.Sprintf(`
SELECT * FROM (%s)
ORDER BY
CASE scope
WHEN 'persona_user' THEN 0
WHEN 'persona' THEN 1
WHEN 'user' THEN 2
END,
confidence DESC,
updated_at DESC
LIMIT ?
`, strings.Join(parts, " UNION ALL "))
args = append(args, limit)
rows, err := DB.QueryContext(ctx, fullQuery, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return s.scanMemories(rows)
}
func (s *MemoryStore) CountByOwner(ctx context.Context, scope, ownerID string) (int, error) {
var count int
err := DB.QueryRowContext(ctx, `
SELECT COUNT(*) FROM memories
WHERE scope = ? AND owner_id = ? AND status = 'active'
`, scope, ownerID).Scan(&count)
return count, err
}
// ── Helpers ─────────────────────────────────
func (s *MemoryStore) scanMemories(rows *sql.Rows) ([]models.Memory, error) {
var memories []models.Memory
for rows.Next() {
var m models.Memory
if err := rows.Scan(
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
&m.SourceChannelID, &m.Confidence, &m.Status,
st(&m.CreatedAt), st(&m.UpdatedAt),
); err != nil {
return nil, err
}
memories = append(memories, m)
}
if memories == nil {
memories = []models.Memory{}
}
return memories, rows.Err()
}

View File

@@ -0,0 +1,146 @@
package sqlite
import (
"context"
"encoding/json"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/models"
)
// RecallHybrid returns active memories using vector similarity + keyword search.
// SQLite has no pgvector, so we load embeddings and compute cosine similarity in Go.
func (s *MemoryStore) RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error) {
if len(queryVec) == 0 {
return s.Recall(ctx, userID, personaID, query, limit)
}
if limit <= 0 || limit > 100 {
limit = 30
}
// Load all active memories with embeddings for applicable scopes
semantic, err := s.recallSemantic(ctx, userID, personaID, queryVec, limit)
if err != nil || len(semantic) == 0 {
return s.Recall(ctx, userID, personaID, query, limit)
}
var keyword []models.Memory
if query != "" {
keyword, _ = s.Recall(ctx, userID, personaID, query, limit)
}
return mergeResultsSQLite(semantic, keyword, limit), nil
}
type scoredMemory struct {
Memory models.Memory
Similarity float64
}
func (s *MemoryStore) recallSemantic(ctx context.Context, userID string, personaID *string, queryVec []float64, limit int) ([]models.Memory, error) {
// Build query to load memories with embeddings
parts := []string{}
args := []interface{}{}
userPart := `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at, embedding
FROM memories
WHERE scope = 'user' AND owner_id = ? AND status = 'active'
AND embedding IS NOT NULL
`
args = append(args, userID)
parts = append(parts, userPart)
if personaID != nil && *personaID != "" {
personaPart := `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at, embedding
FROM memories
WHERE scope = 'persona' AND owner_id = ? AND status = 'active'
AND embedding IS NOT NULL
`
args = append(args, *personaID)
parts = append(parts, personaPart)
puPart := `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at, embedding
FROM memories
WHERE scope = 'persona_user' AND owner_id = ? AND user_id = ? AND status = 'active'
AND embedding IS NOT NULL
`
args = append(args, *personaID, userID)
parts = append(parts, puPart)
}
fullQuery := strings.Join(parts, " UNION ALL ")
rows, err := DB.QueryContext(ctx, fullQuery, args...)
if err != nil {
return nil, err
}
defer rows.Close()
// Score each memory by cosine similarity
var scored []scoredMemory
for rows.Next() {
var m models.Memory
var embeddingJSON string
if err := rows.Scan(
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
&m.SourceChannelID, &m.Confidence, &m.Status,
st(&m.CreatedAt), st(&m.UpdatedAt), &embeddingJSON,
); err != nil {
continue
}
var vec []float64
if err := json.Unmarshal([]byte(embeddingJSON), &vec); err != nil {
continue
}
// cosineSimilarity is defined in knowledge_bases.go (same package)
sim := cosineSimilarity(queryVec, vec)
if sim > 0.3 {
scored = append(scored, scoredMemory{Memory: m, Similarity: sim})
}
}
// Sort by similarity descending (simple insertion sort for small N)
for i := 0; i < len(scored); i++ {
for j := i + 1; j < len(scored); j++ {
if scored[j].Similarity > scored[i].Similarity {
scored[i], scored[j] = scored[j], scored[i]
}
}
}
// Take top N
result := make([]models.Memory, 0, limit)
for i := 0; i < len(scored) && i < limit; i++ {
result = append(result, scored[i].Memory)
}
return result, rows.Err()
}
func mergeResultsSQLite(semantic, keyword []models.Memory, limit int) []models.Memory {
seen := make(map[string]bool, len(semantic))
result := make([]models.Memory, 0, limit)
for _, m := range semantic {
if len(result) >= limit {
break
}
seen[m.ID] = true
result = append(result, m)
}
for _, m := range keyword {
if len(result) >= limit {
break
}
if !seen[m.ID] {
result = append(result, m)
}
}
return result
}

View File

@@ -17,7 +17,8 @@ func NewPersonaStore() *PersonaStore { return &PersonaStore{} }
const personaCols = `id, name, description, icon, avatar, base_model_id, provider_config_id,
system_prompt, temperature, max_tokens, thinking_budget, top_p,
scope, owner_id, created_by, is_active, is_shared, created_at, updated_at`
scope, owner_id, created_by, is_active, is_shared, memory_enabled, memory_extraction_prompt,
created_at, updated_at`
func (s *PersonaStore) Create(ctx context.Context, p *models.Persona) error {
p.ID = store.NewID()
@@ -27,13 +28,15 @@ func (s *PersonaStore) Create(ctx context.Context, p *models.Persona) error {
_, err := DB.ExecContext(ctx, `
INSERT INTO personas (id, name, description, icon, avatar, base_model_id, provider_config_id,
system_prompt, temperature, max_tokens, thinking_budget, top_p,
scope, owner_id, created_by, is_active, is_shared, created_at, updated_at)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
scope, owner_id, created_by, is_active, is_shared, memory_enabled, memory_extraction_prompt,
created_at, updated_at)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
p.ID, p.Name, p.Description, p.Icon, p.Avatar, p.BaseModelID,
models.NullString(p.ProviderConfigID),
p.SystemPrompt, models.NullFloat(p.Temperature), models.NullInt(p.MaxTokens),
models.NullInt(p.ThinkingBudget), models.NullFloat(p.TopP),
p.Scope, models.NullString(p.OwnerID), p.CreatedBy, p.IsActive, p.IsShared,
p.MemoryEnabled, p.MemoryExtractionPrompt,
now.Format(timeFmt), now.Format(timeFmt),
)
return err
@@ -93,6 +96,12 @@ func (s *PersonaStore) Update(ctx context.Context, id string, patch models.Perso
if patch.IsShared != nil {
b.Set("is_shared", *patch.IsShared)
}
if patch.MemoryEnabled != nil {
b.Set("memory_enabled", *patch.MemoryEnabled)
}
if patch.MemoryExtractionPrompt != nil {
b.Set("memory_extraction_prompt", *patch.MemoryExtractionPrompt)
}
if !b.HasSets() {
return nil
}
@@ -286,6 +295,7 @@ func scanPersona(row *sql.Row) (*models.Persona, error) {
&p.BaseModelID, &providerConfigID,
&p.SystemPrompt, &temp, &maxTokens, &thinkingBudget, &topP,
&p.Scope, &ownerID, &p.CreatedBy, &p.IsActive, &p.IsShared,
&p.MemoryEnabled, &p.MemoryExtractionPrompt,
st(&p.CreatedAt), st(&p.UpdatedAt),
)
if err != nil {
@@ -312,6 +322,7 @@ func scanPersonas(rows *sql.Rows) ([]models.Persona, error) {
&p.BaseModelID, &providerConfigID,
&p.SystemPrompt, &temp, &maxTokens, &thinkingBudget, &topP,
&p.Scope, &ownerID, &p.CreatedBy, &p.IsActive, &p.IsShared,
&p.MemoryEnabled, &p.MemoryExtractionPrompt,
st(&p.CreatedAt), st(&p.UpdatedAt),
)
if err != nil {

View File

@@ -31,5 +31,6 @@ func NewStores(db *sql.DB) store.Stores {
KnowledgeBases: NewKnowledgeBaseStore(),
Groups: NewGroupStore(),
ResourceGrants: NewResourceGrantStore(),
Memories: NewMemoryStore(),
}
}

View File

@@ -0,0 +1,47 @@
// ── store_memory.go ─────────────────────────
// MemoryStore interface for v0.18.0.
// Add this interface to store/interfaces.go and add
// Memories MemoryStore
// to the Stores struct.
package store
import (
"context"
"git.gobha.me/xcaliber/chat-switchboard/models"
)
// =========================================
// MEMORY STORE (v0.18.0)
// =========================================
type MemoryStore interface {
// Upsert creates or updates a memory (matched on scope+owner+user+key).
// If a memory with the same key exists, value/confidence/status are updated.
Upsert(ctx context.Context, m *models.Memory) error
// GetByID returns a single memory by ID.
GetByID(ctx context.Context, id string) (*models.Memory, error)
// List returns memories matching the filter criteria.
List(ctx context.Context, f models.MemoryFilter) ([]models.Memory, error)
// Delete hard-deletes a memory by ID.
Delete(ctx context.Context, id string) error
// Archive sets status to 'archived' (soft delete).
Archive(ctx context.Context, id string) error
// Recall returns active memories relevant to a query, merging scopes.
// Combines user + persona + persona_user memories for the given context.
// Results are ordered by confidence DESC, updated_at DESC.
Recall(ctx context.Context, userID string, personaID *string, query string, limit int) ([]models.Memory, error)
// CountByOwner returns the number of active memories for a scope+owner.
CountByOwner(ctx context.Context, scope, ownerID string) (int, error)
// RecallHybrid returns active memories using vector similarity + keyword search.
// If queryVec is nil/empty, falls back to keyword-only (same as Recall).
RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error)
}

View File

@@ -0,0 +1,15 @@
// ── store_memory_hybrid.go ──────────────────
// Phase 2 addition to the MemoryStore interface.
//
// Add this method to the MemoryStore interface in store/store_memory.go:
//
// // RecallHybrid returns active memories using vector similarity + keyword search.
// // If queryVec is nil/empty, falls back to keyword-only (same as Recall).
// // Merges results from applicable scopes with deduplication.
// RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error)
//
// The implementations are in:
// - store/postgres/memory_hybrid.go (pgvector cosine distance)
// - store/sqlite/memory_hybrid.go (app-level cosine similarity)
package store

238
server/tools/memory.go Normal file
View File

@@ -0,0 +1,238 @@
package tools
import (
"context"
"encoding/json"
"fmt"
"log"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// ── Late Registration ────────────────────────
// Memory tools use late registration because they need stores + embedder.
// RegisterMemoryTools registers the memory_save and memory_recall tools.
// Called from main.go after stores and embedder are initialized.
func RegisterMemoryTools(stores store.Stores, embedder *knowledge.Embedder) {
if stores.Memories == nil {
log.Println("⚠ memory tools: MemoryStore not available, skipping registration")
return
}
Register(&memorySaveTool{stores: stores, embedder: embedder})
Register(&memoryRecallTool{stores: stores, embedder: embedder})
log.Println("✅ memory tools registered (memory_save, memory_recall)")
}
// ═══════════════════════════════════════════
// memory_save
// ═══════════════════════════════════════════
type memorySaveTool struct {
stores store.Stores
embedder *knowledge.Embedder
}
func (t *memorySaveTool) Definition() ToolDef {
return ToolDef{
Name: "memory_save",
DisplayName: "Save Memory",
Category: "memory",
Description: "Save a fact or preference about the user for future conversations. " +
"Use this when the user shares something worth remembering long-term " +
"(preferences, technical stack, project details, personal facts). " +
"Memories persist across all conversations.",
Parameters: JSONSchema(map[string]interface{}{
"key": Prop("string", "Short label for the memory (2-5 words, e.g. 'preferred language', 'deployment target')"),
"value": Prop("string", "The fact or detail to remember (1-2 sentences)"),
"confidence": map[string]interface{}{
"type": "number",
"description": "How confident you are in this fact (0.0-1.0, default 0.9)",
},
}, []string{"key", "value"}),
}
}
func (t *memorySaveTool) Execute(ctx context.Context, execCtx ExecutionContext, argsJSON string) (string, error) {
var args struct {
Key string `json:"key"`
Value string `json:"value"`
Confidence float64 `json:"confidence"`
}
if err := json.Unmarshal([]byte(argsJSON), &args); err != nil {
return "", fmt.Errorf("invalid arguments: %w", err)
}
if args.Key == "" || args.Value == "" {
return "", fmt.Errorf("key and value are required")
}
confidence := args.Confidence
if confidence <= 0 || confidence > 1.0 {
confidence = 0.9
}
mem := &models.Memory{
ID: store.NewID(),
Scope: models.MemoryScopeUser,
OwnerID: execCtx.UserID,
Key: strings.TrimSpace(args.Key),
Value: strings.TrimSpace(args.Value),
Confidence: confidence,
Status: models.MemoryStatusActive,
}
if execCtx.ChannelID != "" {
mem.SourceChannelID = &execCtx.ChannelID
}
// If persona is active, use persona_user scope
if execCtx.PersonaID != "" {
mem.Scope = models.MemoryScopePersonaUser
mem.OwnerID = execCtx.PersonaID
mem.UserID = &execCtx.UserID
}
if err := t.stores.Memories.Upsert(ctx, mem); err != nil {
return "", fmt.Errorf("save memory: %w", err)
}
// Embed for semantic recall
t.embedMemory(ctx, mem, execCtx.UserID)
return fmt.Sprintf("Remembered: %s = %s (confidence: %.0f%%)",
args.Key, args.Value, confidence*100), nil
}
func (t *memorySaveTool) embedMemory(ctx context.Context, m *models.Memory, userID string) {
if t.embedder == nil || !t.embedder.IsConfigured(ctx) {
return
}
text := m.Key + ": " + m.Value
var teamID *string
if t.stores.Teams != nil {
ids, _ := t.stores.Teams.GetUserTeamIDs(ctx, userID)
if len(ids) > 0 {
teamID = &ids[0]
}
}
result, err := t.embedder.EmbedChunks(ctx, userID, teamID, []string{text})
if err != nil || len(result.Vectors) == 0 {
return
}
vecStr := vectorToString(result.Vectors[0])
if database.IsSQLite() {
database.DB.Exec(`UPDATE memories SET embedding = ? WHERE id = ?`,
vecStr, m.ID)
} else {
database.DB.Exec(`UPDATE memories SET embedding = $1::vector WHERE id = $2`,
vecStr, m.ID)
}
log.Printf("🧠 memory %s embedded (%d dims)", m.ID[:8], len(result.Vectors[0]))
}
// ═══════════════════════════════════════════
// memory_recall
// ═══════════════════════════════════════════
type memoryRecallTool struct {
stores store.Stores
embedder *knowledge.Embedder
}
func (t *memoryRecallTool) Definition() ToolDef {
return ToolDef{
Name: "memory_recall",
DisplayName: "Recall Memories",
Category: "memory",
Description: "Search your memories about this user. Use this at the start of conversations " +
"or when you need context about the user's preferences, projects, or history.",
Parameters: JSONSchema(map[string]interface{}{
"query": Prop("string", "Optional search query to filter memories (leave empty for all)"),
}, nil),
}
}
func (t *memoryRecallTool) Execute(ctx context.Context, execCtx ExecutionContext, argsJSON string) (string, error) {
var args struct {
Query string `json:"query"`
}
if err := json.Unmarshal([]byte(argsJSON), &args); err != nil {
return "", fmt.Errorf("invalid arguments: %w", err)
}
var personaID *string
if execCtx.PersonaID != "" {
personaID = &execCtx.PersonaID
}
// Try hybrid recall if embedder available and query provided
var memories []models.Memory
var err error
if t.embedder != nil && t.embedder.IsConfigured(ctx) && args.Query != "" {
memories, err = t.recallWithEmbedding(ctx, execCtx.UserID, personaID, args.Query)
}
// Fall back to keyword recall
if err != nil || len(memories) == 0 {
memories, err = t.stores.Memories.Recall(ctx, execCtx.UserID, personaID, args.Query, 20)
}
if err != nil {
return "", fmt.Errorf("recall memories: %w", err)
}
if len(memories) == 0 {
suffix := ""
if args.Query != "" {
suffix = " matching '" + args.Query + "'"
}
return "No memories found" + suffix + ".", nil
}
var sb strings.Builder
sb.WriteString(fmt.Sprintf("Found %d memories:\n\n", len(memories)))
for _, m := range memories {
scope := m.Scope
if scope == "persona_user" {
scope = "persona"
}
sb.WriteString(fmt.Sprintf("• [%s] %s: %s (%.0f%% confidence)\n",
scope, m.Key, m.Value, m.Confidence*100))
}
return sb.String(), nil
}
func (t *memoryRecallTool) recallWithEmbedding(ctx context.Context, userID string, personaID *string, query string) ([]models.Memory, error) {
text := query
if len(text) > 2000 {
text = text[:2000]
}
var teamID *string
if t.stores.Teams != nil {
ids, _ := t.stores.Teams.GetUserTeamIDs(ctx, userID)
if len(ids) > 0 {
teamID = &ids[0]
}
}
result, err := t.embedder.EmbedChunks(ctx, userID, teamID, []string{text})
if err != nil || len(result.Vectors) == 0 {
return nil, err
}
return t.stores.Memories.RecallHybrid(ctx, userID, personaID, query, result.Vectors[0], 20)
}