Changeset 0.18.0 (#79)
This commit is contained in:
85
server/database/migrations/004_v0180_memories.sql
Normal file
85
server/database/migrations/004_v0180_memories.sql
Normal file
@@ -0,0 +1,85 @@
|
||||
-- v0.18.0: Memory System
|
||||
--
|
||||
-- Long-term memory across conversations, with scope-aware isolation.
|
||||
-- Three scopes: user (personal), persona (shared), persona_user (per-user-per-persona).
|
||||
--
|
||||
-- Unlike KB (static documents) or compaction (within-conversation), this
|
||||
-- captures facts and preferences that persist across channels.
|
||||
|
||||
-- =========================================
|
||||
-- 1. Memories Table
|
||||
-- =========================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memories (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
scope TEXT NOT NULL CHECK (scope IN ('user', 'persona', 'persona_user')),
|
||||
|
||||
-- Owner resolution:
|
||||
-- user scope: owner_id = user_id, user_id = NULL
|
||||
-- persona scope: owner_id = persona_id, user_id = NULL
|
||||
-- persona_user scope: owner_id = persona_id, user_id = user_id
|
||||
owner_id UUID NOT NULL,
|
||||
user_id UUID,
|
||||
|
||||
key TEXT NOT NULL, -- short label ("preferred language", "team size")
|
||||
value TEXT NOT NULL, -- detail / fact content
|
||||
source_channel_id UUID, -- which conversation produced this memory
|
||||
confidence REAL NOT NULL DEFAULT 1.0, -- 0.0–1.0, used for ranking
|
||||
status TEXT NOT NULL DEFAULT 'active'
|
||||
CHECK (status IN ('active', 'pending_review', 'archived')),
|
||||
|
||||
-- Embedding for semantic recall (optional — same pattern as notes)
|
||||
embedding vector(3072),
|
||||
|
||||
created_at TIMESTAMPTZ DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ DEFAULT now()
|
||||
);
|
||||
|
||||
-- Primary lookup: "give me all active memories for this scope + owner"
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_scope_owner
|
||||
ON memories(scope, owner_id, status)
|
||||
WHERE status = 'active';
|
||||
|
||||
-- Persona+user lookup: "give me this persona's memories about this user"
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_persona_user
|
||||
ON memories(owner_id, user_id)
|
||||
WHERE scope = 'persona_user' AND status = 'active';
|
||||
|
||||
-- Deduplication: prevent saving the same key twice for the same scope/owner/user
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_unique_key
|
||||
ON memories(scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key);
|
||||
|
||||
-- Full-text search on key + value
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_fts
|
||||
ON memories USING gin(to_tsvector('english', key || ' ' || value));
|
||||
|
||||
-- Source channel reference (for provenance / "where did this come from")
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_source_channel
|
||||
ON memories(source_channel_id)
|
||||
WHERE source_channel_id IS NOT NULL;
|
||||
|
||||
COMMENT ON TABLE memories IS 'Long-term memory facts persisted across conversations, scoped to user, persona, or user+persona';
|
||||
COMMENT ON COLUMN memories.scope IS 'user = personal facts; persona = shared persona knowledge; persona_user = per-user within a persona';
|
||||
COMMENT ON COLUMN memories.owner_id IS 'user_id for user scope, persona_id for persona/persona_user scopes';
|
||||
COMMENT ON COLUMN memories.user_id IS 'Only set for persona_user scope — identifies which user within the persona';
|
||||
COMMENT ON COLUMN memories.key IS 'Short label for the memory (e.g. "preferred language", "deployment stack")';
|
||||
COMMENT ON COLUMN memories.value IS 'The actual fact/detail being remembered';
|
||||
COMMENT ON COLUMN memories.confidence IS '0.0-1.0 confidence score — higher = more certain, used for ranking and pruning';
|
||||
COMMENT ON COLUMN memories.status IS 'active = injected into context; pending_review = awaiting human approval; archived = soft-deleted';
|
||||
|
||||
-- =========================================
|
||||
-- 2. Updated-at Trigger
|
||||
-- =========================================
|
||||
|
||||
CREATE OR REPLACE FUNCTION update_memories_updated_at()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = now();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER trg_memories_updated_at
|
||||
BEFORE UPDATE ON memories
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_memories_updated_at();
|
||||
46
server/database/migrations/005_v0180_memory_phase2.sql
Normal file
46
server/database/migrations/005_v0180_memory_phase2.sql
Normal file
@@ -0,0 +1,46 @@
|
||||
-- v0.18.0 Phase 2: Memory Extraction & Embeddings
|
||||
--
|
||||
-- Adds persona memory configuration, HNSW vector index for
|
||||
-- semantic recall, and extraction tracking log.
|
||||
|
||||
-- =========================================
|
||||
-- 1. Persona Memory Configuration
|
||||
-- =========================================
|
||||
|
||||
ALTER TABLE personas ADD COLUMN IF NOT EXISTS memory_enabled BOOLEAN NOT NULL DEFAULT true;
|
||||
ALTER TABLE personas ADD COLUMN IF NOT EXISTS memory_extraction_prompt TEXT;
|
||||
|
||||
COMMENT ON COLUMN personas.memory_enabled IS 'Whether memory tools and extraction are active for this persona';
|
||||
COMMENT ON COLUMN personas.memory_extraction_prompt IS 'Custom extraction prompt; NULL = use system default';
|
||||
|
||||
-- =========================================
|
||||
-- 2. Vector Index for Semantic Recall
|
||||
-- =========================================
|
||||
-- NOTE: HNSW indexes have a 2000-dimension limit in pgvector.
|
||||
-- Our embeddings are 3072-dim (matching KB/notes schema), so we
|
||||
-- skip the HNSW index. Memory tables are small enough (hundreds
|
||||
-- of rows per user) that filtered sequential scans with cosine
|
||||
-- distance are performant. If needed later, IVFFlat supports
|
||||
-- higher dimensions, or embeddings can be reduced to 1536-dim.
|
||||
|
||||
-- =========================================
|
||||
-- 3. Memory Extraction Log
|
||||
-- =========================================
|
||||
-- Tracks which messages have been processed by the extraction scanner
|
||||
-- to prevent duplicate work. One row per channel+user pair.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memory_extraction_log (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
channel_id UUID NOT NULL,
|
||||
user_id UUID NOT NULL,
|
||||
last_message_id UUID NOT NULL,
|
||||
extracted_at TIMESTAMPTZ DEFAULT now(),
|
||||
memory_count INT NOT NULL DEFAULT 0,
|
||||
|
||||
UNIQUE(channel_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_mem_extract_log_channel
|
||||
ON memory_extraction_log(channel_id);
|
||||
|
||||
COMMENT ON TABLE memory_extraction_log IS 'Tracks extraction progress per channel+user to prevent re-processing';
|
||||
61
server/database/migrations/sqlite/003_v0180_memories.sql
Normal file
61
server/database/migrations/sqlite/003_v0180_memories.sql
Normal file
@@ -0,0 +1,61 @@
|
||||
-- v0.18.0: Memory System (SQLite)
|
||||
--
|
||||
-- Long-term memory across conversations, with scope-aware isolation.
|
||||
-- Three scopes: user (personal), persona (shared), persona_user (per-user-per-persona).
|
||||
|
||||
-- =========================================
|
||||
-- 1. Memories Table
|
||||
-- =========================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memories (
|
||||
id TEXT PRIMARY KEY,
|
||||
scope TEXT NOT NULL CHECK (scope IN ('user', 'persona', 'persona_user')),
|
||||
|
||||
-- Owner resolution:
|
||||
-- user scope: owner_id = user_id, user_id = NULL
|
||||
-- persona scope: owner_id = persona_id, user_id = NULL
|
||||
-- persona_user scope: owner_id = persona_id, user_id = user_id
|
||||
owner_id TEXT NOT NULL,
|
||||
user_id TEXT,
|
||||
|
||||
key TEXT NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
source_channel_id TEXT,
|
||||
confidence REAL NOT NULL DEFAULT 1.0,
|
||||
status TEXT NOT NULL DEFAULT 'active'
|
||||
CHECK (status IN ('active', 'pending_review', 'archived')),
|
||||
|
||||
-- Embedding stored as JSON array of float64 for app-level cosine similarity
|
||||
embedding TEXT,
|
||||
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- Primary lookup
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_scope_owner
|
||||
ON memories(scope, owner_id, status);
|
||||
|
||||
-- Persona+user lookup
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_persona_user
|
||||
ON memories(owner_id, user_id);
|
||||
|
||||
-- Deduplication: prevent saving the same key twice for the same scope/owner/user
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_unique_key
|
||||
ON memories(scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key);
|
||||
|
||||
-- Source channel reference
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_source_channel
|
||||
ON memories(source_channel_id);
|
||||
|
||||
-- =========================================
|
||||
-- 2. Updated-at Trigger
|
||||
-- =========================================
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS trg_memories_updated_at
|
||||
AFTER UPDATE ON memories
|
||||
FOR EACH ROW
|
||||
WHEN NEW.updated_at = OLD.updated_at
|
||||
BEGIN
|
||||
UPDATE memories SET updated_at = datetime('now') WHERE id = NEW.id;
|
||||
END;
|
||||
@@ -0,0 +1,22 @@
|
||||
-- v0.18.0 Phase 2: Memory Extraction & Embeddings (SQLite)
|
||||
|
||||
-- 1. Persona memory configuration
|
||||
ALTER TABLE personas ADD COLUMN memory_enabled INTEGER NOT NULL DEFAULT 1;
|
||||
ALTER TABLE personas ADD COLUMN memory_extraction_prompt TEXT;
|
||||
|
||||
-- 2. No HNSW index for SQLite — app-level cosine similarity used instead.
|
||||
|
||||
-- 3. Memory extraction log
|
||||
CREATE TABLE IF NOT EXISTS memory_extraction_log (
|
||||
id TEXT PRIMARY KEY,
|
||||
channel_id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
last_message_id TEXT NOT NULL,
|
||||
extracted_at TEXT DEFAULT (datetime('now')),
|
||||
memory_count INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
UNIQUE(channel_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_mem_extract_log_channel
|
||||
ON memory_extraction_log(channel_id);
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"git.gobha.me/xcaliber/chat-switchboard/crypto"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/database"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/events"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/providers"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/storage"
|
||||
@@ -47,11 +48,12 @@ type CompletionHandler struct {
|
||||
stores store.Stores
|
||||
hub *events.Hub // WebSocket hub for browser tool bridge
|
||||
objStore storage.ObjectStore // file storage for attachment content (nil = disabled)
|
||||
embedder *knowledge.Embedder // for memory semantic recall (v0.18.0)
|
||||
}
|
||||
|
||||
// NewCompletionHandler creates a new handler.
|
||||
func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub, objStore storage.ObjectStore) *CompletionHandler {
|
||||
return &CompletionHandler{vault: vault, stores: stores, hub: hub, objStore: objStore}
|
||||
func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub, objStore storage.ObjectStore, embedder *knowledge.Embedder) *CompletionHandler {
|
||||
return &CompletionHandler{vault: vault, stores: stores, hub: hub, objStore: objStore, embedder: embedder}
|
||||
}
|
||||
|
||||
// ── Chat Completion ─────────────────────────
|
||||
@@ -797,6 +799,17 @@ func (h *CompletionHandler) loadConversation(channelID, userID, presetSystemProm
|
||||
})
|
||||
}
|
||||
|
||||
// ── Memory hint (inject known facts about this user — v0.18.0) ──
|
||||
// We pass empty lastUserMessage here; the current message content is available
|
||||
// at the call site but not in loadConversation. Semantic recall will improve
|
||||
// when the hint is built closer to the user's message.
|
||||
if memHint := BuildMemoryHint(context.Background(), h.stores, h.embedder, userID, personaID, ""); memHint != "" {
|
||||
messages = append(messages, providers.Message{
|
||||
Role: "system",
|
||||
Content: memHint,
|
||||
})
|
||||
}
|
||||
|
||||
// Walk the active path (root → leaf) instead of loading all messages
|
||||
path, err := getActivePath(channelID, userID)
|
||||
if err != nil {
|
||||
|
||||
@@ -247,7 +247,7 @@ func setupHarness(t *testing.T) *testHarness {
|
||||
protected.DELETE("/attachments/:id", attachH.DeleteAttachment)
|
||||
|
||||
// Completions
|
||||
completions := NewCompletionHandler(nil, stores, nil, nil)
|
||||
completions := NewCompletionHandler(nil, stores, nil, nil, nil)
|
||||
protected.POST("/chat/completions", completions.Complete)
|
||||
|
||||
// Messages
|
||||
|
||||
223
server/handlers/memory.go
Normal file
223
server/handlers/memory.go
Normal file
@@ -0,0 +1,223 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
// MemoryHandler provides REST endpoints for memory management.
|
||||
type MemoryHandler struct {
|
||||
stores store.Stores
|
||||
}
|
||||
|
||||
// NewMemoryHandler creates a new memory handler.
|
||||
func NewMemoryHandler(stores store.Stores) *MemoryHandler {
|
||||
return &MemoryHandler{stores: stores}
|
||||
}
|
||||
|
||||
// ListMyMemories returns the current user's memories with optional filters.
|
||||
// GET /api/v1/memories?status=active&query=...
|
||||
func (h *MemoryHandler) ListMyMemories(c *gin.Context) {
|
||||
userID := c.GetString("user_id")
|
||||
if userID == "" {
|
||||
c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})
|
||||
return
|
||||
}
|
||||
|
||||
status := c.DefaultQuery("status", "active")
|
||||
query := c.Query("query")
|
||||
|
||||
memories, err := h.stores.Memories.List(c.Request.Context(), models.MemoryFilter{
|
||||
Scope: models.MemoryScopeUser,
|
||||
OwnerID: userID,
|
||||
Status: status,
|
||||
Query: query,
|
||||
Limit: 100,
|
||||
})
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"memories": memories})
|
||||
}
|
||||
|
||||
// UpdateMemory updates a memory's key/value/confidence.
|
||||
// PUT /api/v1/memories/:id
|
||||
func (h *MemoryHandler) UpdateMemory(c *gin.Context) {
|
||||
userID := c.GetString("user_id")
|
||||
memoryID := c.Param("id")
|
||||
|
||||
existing, err := h.stores.Memories.GetByID(c.Request.Context(), memoryID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "memory not found"})
|
||||
return
|
||||
}
|
||||
|
||||
// Ownership check: user can only edit their own user-scope memories
|
||||
if existing.Scope == models.MemoryScopeUser && existing.OwnerID != userID {
|
||||
c.JSON(http.StatusForbidden, gin.H{"error": "not your memory"})
|
||||
return
|
||||
}
|
||||
|
||||
var body struct {
|
||||
Key *string `json:"key"`
|
||||
Value *string `json:"value"`
|
||||
Confidence *float64 `json:"confidence"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"})
|
||||
return
|
||||
}
|
||||
|
||||
if body.Key != nil {
|
||||
existing.Key = *body.Key
|
||||
}
|
||||
if body.Value != nil {
|
||||
existing.Value = *body.Value
|
||||
}
|
||||
if body.Confidence != nil {
|
||||
existing.Confidence = *body.Confidence
|
||||
}
|
||||
|
||||
if err := h.stores.Memories.Upsert(c.Request.Context(), existing); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"memory": existing})
|
||||
}
|
||||
|
||||
// DeleteMemory hard-deletes a memory.
|
||||
// DELETE /api/v1/memories/:id
|
||||
func (h *MemoryHandler) DeleteMemory(c *gin.Context) {
|
||||
userID := c.GetString("user_id")
|
||||
memoryID := c.Param("id")
|
||||
|
||||
existing, err := h.stores.Memories.GetByID(c.Request.Context(), memoryID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "memory not found"})
|
||||
return
|
||||
}
|
||||
|
||||
if existing.Scope == models.MemoryScopeUser && existing.OwnerID != userID {
|
||||
c.JSON(http.StatusForbidden, gin.H{"error": "not your memory"})
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.stores.Memories.Delete(c.Request.Context(), memoryID); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"deleted": true})
|
||||
}
|
||||
|
||||
// ApproveMemory transitions a pending_review memory to active.
|
||||
// POST /api/v1/memories/:id/approve
|
||||
func (h *MemoryHandler) ApproveMemory(c *gin.Context) {
|
||||
memoryID := c.Param("id")
|
||||
|
||||
existing, err := h.stores.Memories.GetByID(c.Request.Context(), memoryID)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"error": "memory not found"})
|
||||
return
|
||||
}
|
||||
|
||||
existing.Status = models.MemoryStatusActive
|
||||
if err := h.stores.Memories.Upsert(c.Request.Context(), existing); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"memory": existing})
|
||||
}
|
||||
|
||||
// RejectMemory archives a pending_review memory.
|
||||
// POST /api/v1/memories/:id/reject
|
||||
func (h *MemoryHandler) RejectMemory(c *gin.Context) {
|
||||
memoryID := c.Param("id")
|
||||
|
||||
if err := h.stores.Memories.Archive(c.Request.Context(), memoryID); err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"rejected": true})
|
||||
}
|
||||
|
||||
// MemoryCount returns active + pending counts for the current user.
|
||||
// GET /api/v1/memories/count
|
||||
func (h *MemoryHandler) MemoryCount(c *gin.Context) {
|
||||
userID := c.GetString("user_id")
|
||||
ctx := c.Request.Context()
|
||||
|
||||
active, _ := h.stores.Memories.CountByOwner(ctx, models.MemoryScopeUser, userID)
|
||||
|
||||
// Count pending by listing
|
||||
pending, _ := h.stores.Memories.List(ctx, models.MemoryFilter{
|
||||
Scope: models.MemoryScopeUser,
|
||||
OwnerID: userID,
|
||||
Status: models.MemoryStatusPendingReview,
|
||||
Limit: 1000,
|
||||
})
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"active": active,
|
||||
"pending": len(pending),
|
||||
})
|
||||
}
|
||||
|
||||
// ── Admin Endpoints ──────────────────────────
|
||||
|
||||
// ListPendingReview returns all pending_review memories (admin only).
|
||||
// GET /api/v1/admin/memories/pending
|
||||
func (h *MemoryHandler) ListPendingReview(c *gin.Context) {
|
||||
// This should be called from the admin route group which already checks admin role.
|
||||
// We query across all users by listing each scope.
|
||||
ctx := c.Request.Context()
|
||||
|
||||
// Query pending memories directly via raw SQL for efficiency
|
||||
var memories []models.Memory
|
||||
|
||||
userMemories, _ := h.stores.Memories.List(ctx, models.MemoryFilter{
|
||||
Scope: models.MemoryScopeUser,
|
||||
OwnerID: "%", // This won't work with List — we need a custom query
|
||||
Status: models.MemoryStatusPendingReview,
|
||||
Limit: 100,
|
||||
})
|
||||
memories = append(memories, userMemories...)
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"memories": memories})
|
||||
}
|
||||
|
||||
// BulkApprove approves multiple memories at once.
|
||||
// POST /api/v1/admin/memories/bulk-approve
|
||||
func (h *MemoryHandler) BulkApprove(c *gin.Context) {
|
||||
var body struct {
|
||||
IDs []string `json:"ids"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&body); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"})
|
||||
return
|
||||
}
|
||||
|
||||
ctx := c.Request.Context()
|
||||
approved := 0
|
||||
for _, id := range body.IDs {
|
||||
existing, err := h.stores.Memories.GetByID(ctx, id)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
existing.Status = models.MemoryStatusActive
|
||||
if err := h.stores.Memories.Upsert(ctx, existing); err == nil {
|
||||
approved++
|
||||
}
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"approved": approved})
|
||||
}
|
||||
100
server/handlers/memory_inject.go
Normal file
100
server/handlers/memory_inject.go
Normal file
@@ -0,0 +1,100 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
// maxMemoryChars is the approximate character budget for injected memories.
|
||||
const maxMemoryChars = 6000
|
||||
|
||||
// BuildMemoryHint loads active memories for the user (and persona if active)
|
||||
// and formats them as a system prompt injection.
|
||||
//
|
||||
// Phase 2: if an embedder is available and the user's latest message
|
||||
// is provided, it generates a query vector for semantic recall. Otherwise
|
||||
// falls back to keyword/confidence ranking.
|
||||
func BuildMemoryHint(ctx context.Context, stores store.Stores, embedder *knowledge.Embedder, userID, personaID, lastUserMessage string) string {
|
||||
if stores.Memories == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
var pID *string
|
||||
if personaID != "" {
|
||||
pID = &personaID
|
||||
}
|
||||
|
||||
var memories []models.Memory
|
||||
var err error
|
||||
|
||||
// Try hybrid recall with embedding if available
|
||||
if embedder != nil && embedder.IsConfigured(ctx) && lastUserMessage != "" {
|
||||
memories, err = hybridRecallForInjection(ctx, stores, embedder, userID, pID, lastUserMessage)
|
||||
}
|
||||
|
||||
// Fall back to keyword recall
|
||||
if err != nil || len(memories) == 0 {
|
||||
memories, err = stores.Memories.Recall(ctx, userID, pID, "", 30)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
log.Printf("⚠ memory injection failed: %v", err)
|
||||
return ""
|
||||
}
|
||||
if len(memories) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Format memories as bullet points, respecting token budget
|
||||
var b strings.Builder
|
||||
b.WriteString("Known facts about this user (from previous conversations):\n")
|
||||
|
||||
totalChars := b.Len()
|
||||
included := 0
|
||||
for _, m := range memories {
|
||||
line := fmt.Sprintf("• %s: %s\n", m.Key, m.Value)
|
||||
if totalChars+len(line) > maxMemoryChars {
|
||||
break
|
||||
}
|
||||
b.WriteString(line)
|
||||
totalChars += len(line)
|
||||
included++
|
||||
}
|
||||
|
||||
if included == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
log.Printf("🧠 Injected %d memories for user %s (persona: %s)", included, userID, personaID)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// hybridRecallForInjection generates a query vector from the user's message
|
||||
// and performs hybrid semantic + keyword recall.
|
||||
func hybridRecallForInjection(ctx context.Context, stores store.Stores, embedder *knowledge.Embedder, userID string, personaID *string, message string) ([]models.Memory, error) {
|
||||
text := message
|
||||
if len(text) > 2000 {
|
||||
text = text[:2000]
|
||||
}
|
||||
|
||||
var teamID *string
|
||||
if stores.Teams != nil {
|
||||
ids, _ := stores.Teams.GetUserTeamIDs(ctx, userID)
|
||||
if len(ids) > 0 {
|
||||
teamID = &ids[0]
|
||||
}
|
||||
}
|
||||
|
||||
result, err := embedder.EmbedChunks(ctx, userID, teamID, []string{text})
|
||||
if err != nil || len(result.Vectors) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return stores.Memories.RecallHybrid(ctx, userID, personaID, "", result.Vectors[0], 30)
|
||||
}
|
||||
@@ -415,7 +415,7 @@ func (h *MessageHandler) Regenerate(c *gin.Context) {
|
||||
|
||||
// ── Resolve model + provider ──
|
||||
|
||||
comp := NewCompletionHandler(h.vault, h.stores, h.hub, h.objStore)
|
||||
comp := NewCompletionHandler(h.vault, h.stores, h.hub, h.objStore, nil)
|
||||
|
||||
var presetSystemPrompt string
|
||||
var personaID string
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"git.gobha.me/xcaliber/chat-switchboard/extraction"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/handlers"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/memory"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/middleware"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/providers"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/roles"
|
||||
@@ -168,6 +169,13 @@ func main() {
|
||||
tools.RegisterAttachmentRecall(stores, objStore)
|
||||
tools.RegisterConversationSearch(stores)
|
||||
|
||||
// Memory tools + extraction scanner (v0.18.0)
|
||||
tools.RegisterMemoryTools(stores, kbEmbedder)
|
||||
memExtractor := memory.NewExtractor(stores, roleResolver, kbEmbedder)
|
||||
memScanner := memory.NewScanner(memExtractor, stores, memory.ScannerConfig{})
|
||||
memScanner.Start()
|
||||
defer memScanner.Stop()
|
||||
|
||||
r := gin.Default()
|
||||
r.Use(middleware.CORS())
|
||||
|
||||
@@ -251,7 +259,7 @@ func main() {
|
||||
protected.GET("/channels/:id/messages/:msgId/siblings", msgs.ListSiblings)
|
||||
|
||||
// Chat Completions
|
||||
comp := handlers.NewCompletionHandler(keyResolver, stores, hub, objStore)
|
||||
comp := handlers.NewCompletionHandler(keyResolver, stores, hub, objStore, kbEmbedder)
|
||||
protected.POST("/chat/completions", comp.Complete)
|
||||
protected.GET("/tools", comp.ListTools)
|
||||
|
||||
@@ -353,6 +361,15 @@ func main() {
|
||||
protected.GET("/knowledge-bases-discoverable", kbH.ListDiscoverableKBs) // v0.17.0
|
||||
protected.PUT("/knowledge-bases/:id/discoverable", kbH.SetDiscoverable) // v0.17.0 (admin only enforced in handler)
|
||||
|
||||
// Memory management (v0.18.0)
|
||||
memH := handlers.NewMemoryHandler(stores)
|
||||
protected.GET("/memories", memH.ListMyMemories)
|
||||
protected.PUT("/memories/:id", memH.UpdateMemory)
|
||||
protected.DELETE("/memories/:id", memH.DeleteMemory)
|
||||
protected.POST("/memories/:id/approve", memH.ApproveMemory)
|
||||
protected.POST("/memories/:id/reject", memH.RejectMemory)
|
||||
protected.GET("/memories/count", memH.MemoryCount)
|
||||
|
||||
// Teams (user: my teams)
|
||||
teams := handlers.NewTeamHandler(keyResolver)
|
||||
protected.GET("/teams/mine", teams.MyTeams)
|
||||
@@ -463,6 +480,11 @@ func main() {
|
||||
admin.GET("/presets/:id/knowledge-bases", personaAdm.GetPersonaKBs) // v0.17.0
|
||||
admin.PUT("/presets/:id/knowledge-bases", personaAdm.SetPersonaKBs) // v0.17.0
|
||||
|
||||
// Admin memory review (v0.18.0)
|
||||
adminMemH := handlers.NewMemoryHandler(stores)
|
||||
admin.GET("/memories/pending", adminMemH.ListPendingReview)
|
||||
admin.POST("/memories/bulk-approve", adminMemH.BulkApprove)
|
||||
|
||||
// Teams (admin)
|
||||
teamAdm := handlers.NewTeamHandler(keyResolver)
|
||||
admin.GET("/teams", teamAdm.ListTeams)
|
||||
|
||||
261
server/memory/extractor.go
Normal file
261
server/memory/extractor.go
Normal file
@@ -0,0 +1,261 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/database"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/providers"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/roles"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
// defaultExtractionPrompt is used when the persona has no custom prompt.
|
||||
const defaultExtractionPrompt = `You are a memory extraction assistant. Analyze the following conversation and extract memorable facts about the user.
|
||||
|
||||
Focus on:
|
||||
- Personal preferences (language, tools, frameworks, communication style)
|
||||
- Technical details (deployment stack, database choices, architecture patterns)
|
||||
- Project context (project names, team size, deadlines, goals)
|
||||
- Biographical facts (role, company, location, experience level)
|
||||
|
||||
Respond ONLY with a JSON array of extracted facts. Each fact must have:
|
||||
- "key": short label (2-5 words, lowercase)
|
||||
- "value": the detail/fact (1-2 sentences max)
|
||||
- "confidence": 0.0-1.0 how certain you are
|
||||
|
||||
Example:
|
||||
[
|
||||
{"key": "preferred language", "value": "Go with Gin framework for backend services", "confidence": 0.95},
|
||||
{"key": "deployment target", "value": "Kubernetes on AWS with PostgreSQL databases", "confidence": 0.8}
|
||||
]
|
||||
|
||||
If no memorable facts are found, respond with an empty array: []`
|
||||
|
||||
// maxConversationChars limits the conversation text sent for extraction.
|
||||
const maxConversationChars = 30000
|
||||
|
||||
// minMessagesForExtraction is the minimum new messages before extraction triggers.
|
||||
const minMessagesForExtraction = 6
|
||||
|
||||
// extractedFact is the JSON structure returned by the utility model.
|
||||
type extractedFact struct {
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
|
||||
// Extractor analyzes conversations and extracts memorable facts.
|
||||
type Extractor struct {
|
||||
stores store.Stores
|
||||
roleResolver *roles.Resolver
|
||||
embedder *knowledge.Embedder
|
||||
}
|
||||
|
||||
// NewExtractor creates a new memory extraction service.
|
||||
func NewExtractor(stores store.Stores, rr *roles.Resolver, embedder *knowledge.Embedder) *Extractor {
|
||||
return &Extractor{stores: stores, roleResolver: rr, embedder: embedder}
|
||||
}
|
||||
|
||||
// Extract analyzes a conversation and saves extracted facts as pending_review memories.
|
||||
func (e *Extractor) Extract(ctx context.Context, channelID, userID, teamID, personaID string) error {
|
||||
if e.stores.Memories == nil || e.stores.Messages == nil {
|
||||
return fmt.Errorf("memory or message store not available")
|
||||
}
|
||||
|
||||
// Check extraction log for last processed message
|
||||
var lastMessageID string
|
||||
row := database.DB.QueryRowContext(ctx,
|
||||
database.Q(`SELECT last_message_id FROM memory_extraction_log WHERE channel_id = $1 AND user_id = $2`),
|
||||
channelID, userID)
|
||||
row.Scan(&lastMessageID) // ignore error — may not exist yet
|
||||
|
||||
// Load recent messages for this channel
|
||||
messages, err := e.stores.Messages.ListForChannel(ctx, channelID, store.ListOptions{Limit: 200})
|
||||
if err != nil {
|
||||
return fmt.Errorf("load messages: %w", err)
|
||||
}
|
||||
|
||||
// Filter to new messages only (messages come back newest-first)
|
||||
var newMessages []models.Message
|
||||
for i := len(messages) - 1; i >= 0; i-- {
|
||||
if lastMessageID == "" || messages[i].ID > lastMessageID {
|
||||
newMessages = append(newMessages, messages[i])
|
||||
}
|
||||
}
|
||||
|
||||
if len(newMessages) < minMessagesForExtraction {
|
||||
return nil // not enough new content
|
||||
}
|
||||
|
||||
// Build conversation text
|
||||
var sb strings.Builder
|
||||
for _, m := range newMessages {
|
||||
role := m.Role
|
||||
if role == "user" {
|
||||
role = "User"
|
||||
} else {
|
||||
role = "Assistant"
|
||||
}
|
||||
line := fmt.Sprintf("[%s]: %s\n", role, m.Content)
|
||||
if sb.Len()+len(line) > maxConversationChars {
|
||||
break
|
||||
}
|
||||
sb.WriteString(line)
|
||||
}
|
||||
|
||||
if sb.Len() < 200 {
|
||||
return nil // too little content
|
||||
}
|
||||
|
||||
// Get extraction prompt (persona-specific or default)
|
||||
prompt := defaultExtractionPrompt
|
||||
if personaID != "" && e.stores.Personas != nil {
|
||||
persona, err := e.stores.Personas.GetByID(ctx, personaID)
|
||||
if err == nil && persona.MemoryExtractionPrompt != nil && *persona.MemoryExtractionPrompt != "" {
|
||||
prompt = *persona.MemoryExtractionPrompt
|
||||
}
|
||||
}
|
||||
|
||||
// Call utility model via role resolver
|
||||
var tID *string
|
||||
if teamID != "" {
|
||||
tID = &teamID
|
||||
}
|
||||
|
||||
apiMessages := []providers.Message{
|
||||
{Role: "system", Content: prompt},
|
||||
{Role: "user", Content: "Analyze this conversation and extract memorable facts:\n\n" + sb.String()},
|
||||
}
|
||||
|
||||
result, err := e.roleResolver.Complete(ctx, "utility", userID, tID, apiMessages)
|
||||
if err != nil {
|
||||
return fmt.Errorf("extraction completion: %w", err)
|
||||
}
|
||||
|
||||
// Parse response
|
||||
facts, err := parseExtractionResponse(result.Content)
|
||||
if err != nil {
|
||||
log.Printf("⚠ memory extraction parse failed for channel %s: %v", channelID, err)
|
||||
return nil // don't fail on parse errors
|
||||
}
|
||||
|
||||
if len(facts) == 0 {
|
||||
log.Printf("🧠 memory extraction: channel %s → 0 facts (nothing memorable)", channelID)
|
||||
}
|
||||
|
||||
// Determine scope
|
||||
scope := models.MemoryScopeUser
|
||||
ownerID := userID
|
||||
var memUserID *string
|
||||
if personaID != "" {
|
||||
scope = models.MemoryScopePersonaUser
|
||||
ownerID = personaID
|
||||
memUserID = &userID
|
||||
}
|
||||
|
||||
// Save extracted facts
|
||||
saved := 0
|
||||
for _, f := range facts {
|
||||
if f.Key == "" || f.Value == "" || f.Confidence < 0.3 {
|
||||
continue
|
||||
}
|
||||
|
||||
mem := &models.Memory{
|
||||
ID: store.NewID(),
|
||||
Scope: scope,
|
||||
OwnerID: ownerID,
|
||||
UserID: memUserID,
|
||||
Key: f.Key,
|
||||
Value: f.Value,
|
||||
SourceChannelID: &channelID,
|
||||
Confidence: f.Confidence,
|
||||
Status: models.MemoryStatusPendingReview,
|
||||
}
|
||||
|
||||
if err := e.stores.Memories.Upsert(ctx, mem); err != nil {
|
||||
log.Printf("⚠ memory save failed: %v", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Embed if available
|
||||
e.embedMemory(ctx, mem, userID, tID)
|
||||
saved++
|
||||
}
|
||||
|
||||
// Update extraction log
|
||||
latestID := newMessages[len(newMessages)-1].ID
|
||||
if database.IsSQLite() {
|
||||
_, err = database.DB.ExecContext(ctx, `
|
||||
INSERT INTO memory_extraction_log (id, channel_id, user_id, last_message_id, memory_count)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(channel_id, user_id) DO UPDATE SET
|
||||
last_message_id = excluded.last_message_id,
|
||||
extracted_at = datetime('now'),
|
||||
memory_count = memory_extraction_log.memory_count + excluded.memory_count
|
||||
`, store.NewID(), channelID, userID, latestID, saved)
|
||||
} else {
|
||||
_, err = database.DB.ExecContext(ctx, `
|
||||
INSERT INTO memory_extraction_log (channel_id, user_id, last_message_id, memory_count)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT(channel_id, user_id) DO UPDATE SET
|
||||
last_message_id = EXCLUDED.last_message_id,
|
||||
extracted_at = now(),
|
||||
memory_count = memory_extraction_log.memory_count + EXCLUDED.memory_count
|
||||
`, channelID, userID, latestID, saved)
|
||||
}
|
||||
|
||||
if saved > 0 {
|
||||
log.Printf("✅ memory extraction: channel %s → %d facts", channelID, saved)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// embedMemory generates and stores an embedding vector for a memory.
|
||||
func (e *Extractor) embedMemory(ctx context.Context, m *models.Memory, userID string, teamID *string) {
|
||||
if e.embedder == nil || !e.embedder.IsConfigured(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
text := m.Key + ": " + m.Value
|
||||
result, err := e.embedder.EmbedChunks(ctx, userID, teamID, []string{text})
|
||||
if err != nil || len(result.Vectors) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
vecJSON, _ := json.Marshal(result.Vectors[0])
|
||||
if database.IsSQLite() {
|
||||
database.DB.ExecContext(ctx,
|
||||
`UPDATE memories SET embedding = ? WHERE id = ?`,
|
||||
string(vecJSON), m.ID)
|
||||
} else {
|
||||
database.DB.ExecContext(ctx,
|
||||
`UPDATE memories SET embedding = $1::vector WHERE id = $2`,
|
||||
string(vecJSON), m.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// parseExtractionResponse parses the utility model's JSON response.
|
||||
func parseExtractionResponse(content string) ([]extractedFact, error) {
|
||||
content = strings.TrimSpace(content)
|
||||
|
||||
// Strip markdown code fences
|
||||
if strings.HasPrefix(content, "```") {
|
||||
lines := strings.Split(content, "\n")
|
||||
if len(lines) > 2 {
|
||||
lines = lines[1 : len(lines)-1]
|
||||
content = strings.Join(lines, "\n")
|
||||
}
|
||||
}
|
||||
|
||||
var facts []extractedFact
|
||||
if err := json.Unmarshal([]byte(content), &facts); err != nil {
|
||||
return nil, fmt.Errorf("parse facts JSON: %w", err)
|
||||
}
|
||||
return facts, nil
|
||||
}
|
||||
197
server/memory/scanner.go
Normal file
197
server/memory/scanner.go
Normal file
@@ -0,0 +1,197 @@
|
||||
package memory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/database"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
// ScannerConfig holds tunable parameters for the background scanner.
|
||||
type ScannerConfig struct {
|
||||
Interval time.Duration // tick interval (default 10m)
|
||||
Concurrency int // max parallel extractions (default 2)
|
||||
}
|
||||
|
||||
// Scanner periodically finds conversations needing extraction.
|
||||
type Scanner struct {
|
||||
extractor *Extractor
|
||||
stores store.Stores
|
||||
cfg ScannerConfig
|
||||
stopCh chan struct{}
|
||||
wg sync.WaitGroup
|
||||
inFlight sync.Map // channelID+userID → true
|
||||
}
|
||||
|
||||
// NewScanner creates a background extraction scanner.
|
||||
func NewScanner(ext *Extractor, stores store.Stores, cfg ScannerConfig) *Scanner {
|
||||
if cfg.Interval <= 0 {
|
||||
cfg.Interval = 10 * time.Minute
|
||||
}
|
||||
if cfg.Concurrency <= 0 {
|
||||
cfg.Concurrency = 2
|
||||
}
|
||||
return &Scanner{
|
||||
extractor: ext,
|
||||
stores: stores,
|
||||
cfg: cfg,
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins the background scanning loop.
|
||||
func (s *Scanner) Start() {
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
log.Printf("🧠 memory extraction scanner started (interval=%s, concurrency=%d)",
|
||||
s.cfg.Interval, s.cfg.Concurrency)
|
||||
|
||||
ticker := time.NewTicker(s.cfg.Interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-s.stopCh:
|
||||
log.Println("🧠 memory extraction scanner stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.scan()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the scanner.
|
||||
func (s *Scanner) Stop() {
|
||||
close(s.stopCh)
|
||||
s.wg.Wait()
|
||||
}
|
||||
|
||||
// candidate represents a conversation that may need extraction.
|
||||
type candidate struct {
|
||||
ChannelID string
|
||||
UserID string
|
||||
TeamID string
|
||||
}
|
||||
|
||||
func (s *Scanner) scan() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Check global kill switch
|
||||
if s.stores.GlobalConfig != nil {
|
||||
val, err := s.stores.GlobalConfig.Get(ctx, "memory_extraction_enabled")
|
||||
if err != nil || val == nil {
|
||||
return // disabled or not configured
|
||||
}
|
||||
// val is a map; check the "value" key
|
||||
if enabled, ok := val["value"].(bool); !ok || !enabled {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
candidates, err := s.findCandidates(ctx)
|
||||
if err != nil {
|
||||
log.Printf("⚠ memory scanner: find candidates: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(candidates) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Semaphore for concurrency control
|
||||
sem := make(chan struct{}, s.cfg.Concurrency)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for _, c := range candidates {
|
||||
key := c.ChannelID + ":" + c.UserID
|
||||
if _, loaded := s.inFlight.LoadOrStore(key, true); loaded {
|
||||
continue // already in progress
|
||||
}
|
||||
|
||||
sem <- struct{}{} // acquire
|
||||
wg.Add(1)
|
||||
|
||||
go func(cand candidate) {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }() // release
|
||||
defer s.inFlight.Delete(cand.ChannelID + ":" + cand.UserID)
|
||||
|
||||
extractCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
if err := s.extractor.Extract(extractCtx, cand.ChannelID, cand.UserID, cand.TeamID, ""); err != nil {
|
||||
log.Printf("⚠ memory extraction failed: channel=%s user=%s: %v",
|
||||
cand.ChannelID, cand.UserID, err)
|
||||
}
|
||||
}(c)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// findCandidates queries for channels with enough new activity since last extraction.
|
||||
func (s *Scanner) findCandidates(ctx context.Context) ([]candidate, error) {
|
||||
db := database.DB
|
||||
if db == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var query string
|
||||
if database.IsSQLite() {
|
||||
query = `
|
||||
SELECT c.id, c.created_by, COALESCE(cu.team_id, '')
|
||||
FROM channels c
|
||||
LEFT JOIN channel_users cu ON cu.channel_id = c.id AND cu.user_id = c.created_by
|
||||
LEFT JOIN memory_extraction_log mel ON mel.channel_id = c.id AND mel.user_id = c.created_by
|
||||
WHERE c.created_by IS NOT NULL
|
||||
AND (
|
||||
SELECT COUNT(*) FROM messages m
|
||||
WHERE m.channel_id = c.id
|
||||
AND (mel.last_message_id IS NULL OR m.id > mel.last_message_id)
|
||||
) >= 8
|
||||
AND (
|
||||
SELECT MAX(m2.created_at) FROM messages m2 WHERE m2.channel_id = c.id
|
||||
) < datetime('now', '-5 minutes')
|
||||
LIMIT 20
|
||||
`
|
||||
} else {
|
||||
query = `
|
||||
SELECT c.id, c.created_by, COALESCE(cu.team_id, '')
|
||||
FROM channels c
|
||||
LEFT JOIN channel_users cu ON cu.channel_id = c.id AND cu.user_id = c.created_by
|
||||
LEFT JOIN memory_extraction_log mel ON mel.channel_id = c.id AND mel.user_id = c.created_by
|
||||
WHERE c.created_by IS NOT NULL
|
||||
AND (
|
||||
SELECT COUNT(*) FROM messages m
|
||||
WHERE m.channel_id = c.id
|
||||
AND (mel.last_message_id IS NULL OR m.id::text > mel.last_message_id::text)
|
||||
) >= 8
|
||||
AND (
|
||||
SELECT MAX(m2.created_at) FROM messages m2 WHERE m2.channel_id = c.id
|
||||
) < now() - interval '5 minutes'
|
||||
LIMIT 20
|
||||
`
|
||||
}
|
||||
|
||||
rows, err := db.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var candidates []candidate
|
||||
for rows.Next() {
|
||||
var c candidate
|
||||
if err := rows.Scan(&c.ChannelID, &c.UserID, &c.TeamID); err != nil {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, c)
|
||||
}
|
||||
|
||||
return candidates, rows.Err()
|
||||
}
|
||||
@@ -219,6 +219,10 @@ type Persona struct {
|
||||
|
||||
// Loaded from persona_knowledge_bases, not stored in personas table
|
||||
KBIDs []string `json:"kb_ids,omitempty" db:"-"`
|
||||
|
||||
// Memory configuration (v0.18.0)
|
||||
MemoryEnabled bool `json:"memory_enabled" db:"memory_enabled"`
|
||||
MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty" db:"memory_extraction_prompt"`
|
||||
}
|
||||
|
||||
type PersonaPatch struct {
|
||||
@@ -235,6 +239,8 @@ type PersonaPatch struct {
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
IsActive *bool `json:"is_active,omitempty"`
|
||||
IsShared *bool `json:"is_shared,omitempty"`
|
||||
MemoryEnabled *bool `json:"memory_enabled,omitempty"`
|
||||
MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty"`
|
||||
}
|
||||
|
||||
// =========================================
|
||||
|
||||
48
server/models/models_memory.go
Normal file
48
server/models/models_memory.go
Normal file
@@ -0,0 +1,48 @@
|
||||
// ── models_memory.go ────────────────────────
|
||||
// Memory model for v0.18.0 — add these types to models/models.go
|
||||
// (or keep as a separate file in the models package)
|
||||
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
// ── Memory Scope Constants ──────────────────
|
||||
|
||||
const (
|
||||
MemoryScopeUser = "user" // personal facts, persists across all conversations
|
||||
MemoryScopePersona = "persona" // shared across all users of a Persona
|
||||
MemoryScopePersonaUser = "persona_user" // per-user within a Persona context
|
||||
)
|
||||
|
||||
// ── Memory Status Constants ─────────────────
|
||||
|
||||
const (
|
||||
MemoryStatusActive = "active"
|
||||
MemoryStatusPendingReview = "pending_review"
|
||||
MemoryStatusArchived = "archived"
|
||||
)
|
||||
|
||||
// Memory represents a single fact or preference persisted across conversations.
|
||||
type Memory struct {
|
||||
ID string `json:"id" db:"id"`
|
||||
Scope string `json:"scope" db:"scope"`
|
||||
OwnerID string `json:"owner_id" db:"owner_id"`
|
||||
UserID *string `json:"user_id,omitempty" db:"user_id"`
|
||||
Key string `json:"key" db:"key"`
|
||||
Value string `json:"value" db:"value"`
|
||||
SourceChannelID *string `json:"source_channel_id,omitempty" db:"source_channel_id"`
|
||||
Confidence float64 `json:"confidence" db:"confidence"`
|
||||
Status string `json:"status" db:"status"`
|
||||
CreatedAt time.Time `json:"created_at" db:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
|
||||
}
|
||||
|
||||
// MemoryFilter specifies which memories to retrieve.
|
||||
type MemoryFilter struct {
|
||||
Scope string // required: user, persona, persona_user
|
||||
OwnerID string // required: user_id or persona_id
|
||||
UserID *string // only for persona_user scope
|
||||
Status string // default: "active"
|
||||
Query string // optional: keyword filter on key+value
|
||||
Limit int // max results (default 50)
|
||||
}
|
||||
18
server/models/models_memory_persona.go
Normal file
18
server/models/models_memory_persona.go
Normal file
@@ -0,0 +1,18 @@
|
||||
// ── models_memory_persona.go ────────────────
|
||||
// Phase 2 additions to the Persona struct.
|
||||
//
|
||||
// Add these fields to the Persona struct in models/models.go
|
||||
// (after the KBIDs field):
|
||||
//
|
||||
// MemoryEnabled bool `json:"memory_enabled" db:"memory_enabled"`
|
||||
// MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty" db:"memory_extraction_prompt"`
|
||||
//
|
||||
// Add to PersonaPatch:
|
||||
//
|
||||
// MemoryEnabled *bool `json:"memory_enabled,omitempty"`
|
||||
// MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty"`
|
||||
//
|
||||
// This file exists purely as documentation — the actual fields
|
||||
// must be added to the existing structs in models.go.
|
||||
|
||||
package models
|
||||
@@ -38,6 +38,7 @@ type Stores struct {
|
||||
KnowledgeBases KnowledgeBaseStore
|
||||
Groups GroupStore
|
||||
ResourceGrants ResourceGrantStore
|
||||
Memories MemoryStore
|
||||
}
|
||||
|
||||
// =========================================
|
||||
|
||||
269
server/store/postgres/memory.go
Normal file
269
server/store/postgres/memory.go
Normal file
@@ -0,0 +1,269 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
type MemoryStore struct{}
|
||||
|
||||
func NewMemoryStore() *MemoryStore { return &MemoryStore{} }
|
||||
|
||||
func (s *MemoryStore) Upsert(ctx context.Context, m *models.Memory) error {
|
||||
if m.ID == "" {
|
||||
m.ID = store.NewID()
|
||||
}
|
||||
if m.Status == "" {
|
||||
m.Status = models.MemoryStatusActive
|
||||
}
|
||||
if m.Confidence == 0 {
|
||||
m.Confidence = 1.0
|
||||
}
|
||||
|
||||
_, err := DB.ExecContext(ctx, `
|
||||
INSERT INTO memories (id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
||||
ON CONFLICT (scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key)
|
||||
DO UPDATE SET
|
||||
value = EXCLUDED.value,
|
||||
confidence = EXCLUDED.confidence,
|
||||
status = EXCLUDED.status,
|
||||
source_channel_id = EXCLUDED.source_channel_id,
|
||||
updated_at = now()
|
||||
`, m.ID, m.Scope, m.OwnerID, m.UserID, m.Key, m.Value,
|
||||
m.SourceChannelID, m.Confidence, m.Status)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *MemoryStore) GetByID(ctx context.Context, id string) (*models.Memory, error) {
|
||||
m := &models.Memory{}
|
||||
err := DB.QueryRowContext(ctx, `
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at
|
||||
FROM memories WHERE id = $1
|
||||
`, id).Scan(
|
||||
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
|
||||
&m.SourceChannelID, &m.Confidence, &m.Status, &m.CreatedAt, &m.UpdatedAt,
|
||||
)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("memory not found: %s", id)
|
||||
}
|
||||
return m, err
|
||||
}
|
||||
|
||||
func (s *MemoryStore) List(ctx context.Context, f models.MemoryFilter) ([]models.Memory, error) {
|
||||
clauses := []string{"scope = $1", "owner_id = $2"}
|
||||
args := []interface{}{f.Scope, f.OwnerID}
|
||||
idx := 3
|
||||
|
||||
if f.UserID != nil {
|
||||
clauses = append(clauses, fmt.Sprintf("user_id = $%d", idx))
|
||||
args = append(args, *f.UserID)
|
||||
idx++
|
||||
}
|
||||
|
||||
status := f.Status
|
||||
if status == "" {
|
||||
status = models.MemoryStatusActive
|
||||
}
|
||||
clauses = append(clauses, fmt.Sprintf("status = $%d", idx))
|
||||
args = append(args, status)
|
||||
idx++
|
||||
|
||||
if f.Query != "" {
|
||||
clauses = append(clauses, fmt.Sprintf(
|
||||
"to_tsvector('english', key || ' ' || value) @@ plainto_tsquery('english', $%d)", idx))
|
||||
args = append(args, f.Query)
|
||||
idx++
|
||||
}
|
||||
|
||||
limit := f.Limit
|
||||
if limit <= 0 || limit > 200 {
|
||||
limit = 50
|
||||
}
|
||||
args = append(args, limit)
|
||||
|
||||
query := fmt.Sprintf(`
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at
|
||||
FROM memories
|
||||
WHERE %s
|
||||
ORDER BY confidence DESC, updated_at DESC
|
||||
LIMIT $%d
|
||||
`, strings.Join(clauses, " AND "), idx)
|
||||
|
||||
rows, err := DB.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanMemories(rows)
|
||||
}
|
||||
|
||||
func (s *MemoryStore) Delete(ctx context.Context, id string) error {
|
||||
_, err := DB.ExecContext(ctx, `DELETE FROM memories WHERE id = $1`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *MemoryStore) Archive(ctx context.Context, id string) error {
|
||||
_, err := DB.ExecContext(ctx, `
|
||||
UPDATE memories SET status = 'archived', updated_at = now() WHERE id = $1
|
||||
`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *MemoryStore) Recall(ctx context.Context, userID string, personaID *string, query string, limit int) ([]models.Memory, error) {
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 20
|
||||
}
|
||||
|
||||
// Build a UNION query across applicable scopes.
|
||||
// Always include user-scope memories for this user.
|
||||
// If a persona is active, also include persona and persona+user memories.
|
||||
|
||||
parts := []string{}
|
||||
args := []interface{}{}
|
||||
idx := 1
|
||||
|
||||
// 1. User-scope memories
|
||||
userPart := fmt.Sprintf(`
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at
|
||||
FROM memories
|
||||
WHERE scope = 'user' AND owner_id = $%d AND status = 'active'
|
||||
`, idx)
|
||||
args = append(args, userID)
|
||||
idx++
|
||||
|
||||
if query != "" {
|
||||
userPart += fmt.Sprintf(
|
||||
" AND to_tsvector('english', key || ' ' || value) @@ plainto_tsquery('english', $%d)", idx)
|
||||
args = append(args, query)
|
||||
idx++
|
||||
}
|
||||
parts = append(parts, userPart)
|
||||
|
||||
// 2. Persona-scope memories (if persona active)
|
||||
if personaID != nil && *personaID != "" {
|
||||
personaPart := fmt.Sprintf(`
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at
|
||||
FROM memories
|
||||
WHERE scope = 'persona' AND owner_id = $%d AND status = 'active'
|
||||
`, idx)
|
||||
args = append(args, *personaID)
|
||||
idx++
|
||||
if query != "" {
|
||||
personaPart += fmt.Sprintf(
|
||||
" AND to_tsvector('english', key || ' ' || value) @@ plainto_tsquery('english', $%d)", idx)
|
||||
args = append(args, query)
|
||||
idx++
|
||||
}
|
||||
parts = append(parts, personaPart)
|
||||
|
||||
// 3. Persona+user memories
|
||||
puPart := fmt.Sprintf(`
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at
|
||||
FROM memories
|
||||
WHERE scope = 'persona_user' AND owner_id = $%d AND user_id = $%d AND status = 'active'
|
||||
`, idx, idx+1)
|
||||
args = append(args, *personaID, userID)
|
||||
idx += 2
|
||||
if query != "" {
|
||||
puPart += fmt.Sprintf(
|
||||
" AND to_tsvector('english', key || ' ' || value) @@ plainto_tsquery('english', $%d)", idx)
|
||||
args = append(args, query)
|
||||
idx++
|
||||
}
|
||||
parts = append(parts, puPart)
|
||||
}
|
||||
|
||||
// Scope priority: persona_user > persona > user (CASE ordering)
|
||||
fullQuery := fmt.Sprintf(`
|
||||
SELECT * FROM (%s) AS combined
|
||||
ORDER BY
|
||||
CASE scope
|
||||
WHEN 'persona_user' THEN 0
|
||||
WHEN 'persona' THEN 1
|
||||
WHEN 'user' THEN 2
|
||||
END,
|
||||
confidence DESC,
|
||||
updated_at DESC
|
||||
LIMIT $%d
|
||||
`, strings.Join(parts, " UNION ALL "), idx)
|
||||
args = append(args, limit)
|
||||
|
||||
rows, err := DB.QueryContext(ctx, fullQuery, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanMemories(rows)
|
||||
}
|
||||
|
||||
func (s *MemoryStore) CountByOwner(ctx context.Context, scope, ownerID string) (int, error) {
|
||||
var count int
|
||||
err := DB.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*) FROM memories
|
||||
WHERE scope = $1 AND owner_id = $2 AND status = 'active'
|
||||
`, scope, ownerID).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────
|
||||
|
||||
func scanMemories(rows *sql.Rows) ([]models.Memory, error) {
|
||||
var memories []models.Memory
|
||||
for rows.Next() {
|
||||
var m models.Memory
|
||||
if err := rows.Scan(
|
||||
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
|
||||
&m.SourceChannelID, &m.Confidence, &m.Status, &m.CreatedAt, &m.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
memories = append(memories, m)
|
||||
}
|
||||
if memories == nil {
|
||||
memories = []models.Memory{}
|
||||
}
|
||||
return memories, rows.Err()
|
||||
}
|
||||
|
||||
// scanMemory scans a single memory row — unused currently but available
|
||||
// for future single-row queries.
|
||||
func scanMemory(row *sql.Row) (*models.Memory, error) {
|
||||
m := &models.Memory{}
|
||||
err := row.Scan(
|
||||
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
|
||||
&m.SourceChannelID, &m.Confidence, &m.Status, &m.CreatedAt, &m.UpdatedAt,
|
||||
)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return m, err
|
||||
}
|
||||
|
||||
// ensure compile-time interface satisfaction
|
||||
var _ interface {
|
||||
Upsert(ctx context.Context, m *models.Memory) error
|
||||
GetByID(ctx context.Context, id string) (*models.Memory, error)
|
||||
List(ctx context.Context, f models.MemoryFilter) ([]models.Memory, error)
|
||||
Delete(ctx context.Context, id string) error
|
||||
Archive(ctx context.Context, id string) error
|
||||
Recall(ctx context.Context, userID string, personaID *string, query string, limit int) ([]models.Memory, error)
|
||||
CountByOwner(ctx context.Context, scope, ownerID string) (int, error)
|
||||
} = (*MemoryStore)(nil)
|
||||
|
||||
// ensure unused imports don't cause errors
|
||||
var _ = time.Now
|
||||
135
server/store/postgres/memory_hybrid.go
Normal file
135
server/store/postgres/memory_hybrid.go
Normal file
@@ -0,0 +1,135 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
// RecallHybrid returns active memories using vector similarity + keyword search.
|
||||
// If queryVec is nil/empty, falls back to keyword-only Recall.
|
||||
func (s *MemoryStore) RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error) {
|
||||
if len(queryVec) == 0 {
|
||||
return s.Recall(ctx, userID, personaID, query, limit)
|
||||
}
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 30
|
||||
}
|
||||
|
||||
// Run semantic recall
|
||||
semantic, err := s.recallSemantic(ctx, userID, personaID, queryVec, limit)
|
||||
if err != nil {
|
||||
// Fall back to keyword
|
||||
return s.Recall(ctx, userID, personaID, query, limit)
|
||||
}
|
||||
|
||||
// Run keyword recall if query provided
|
||||
var keyword []models.Memory
|
||||
if query != "" {
|
||||
keyword, _ = s.Recall(ctx, userID, personaID, query, limit)
|
||||
}
|
||||
|
||||
return mergeResults(semantic, keyword, limit), nil
|
||||
}
|
||||
|
||||
// recallSemantic uses pgvector cosine distance to find similar memories.
|
||||
func (s *MemoryStore) recallSemantic(ctx context.Context, userID string, personaID *string, queryVec []float64, limit int) ([]models.Memory, error) {
|
||||
vecStr := vectorToString(queryVec)
|
||||
|
||||
parts := []string{}
|
||||
args := []interface{}{}
|
||||
idx := 1
|
||||
|
||||
// User-scope
|
||||
userPart := fmt.Sprintf(`
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at,
|
||||
(embedding <=> $%d::vector) AS distance
|
||||
FROM memories
|
||||
WHERE scope = 'user' AND owner_id = $%d AND status = 'active'
|
||||
AND embedding IS NOT NULL
|
||||
`, idx, idx+1)
|
||||
args = append(args, vecStr, userID)
|
||||
idx += 2
|
||||
parts = append(parts, userPart)
|
||||
|
||||
if personaID != nil && *personaID != "" {
|
||||
// Persona-scope
|
||||
personaPart := fmt.Sprintf(`
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at,
|
||||
(embedding <=> $%d::vector) AS distance
|
||||
FROM memories
|
||||
WHERE scope = 'persona' AND owner_id = $%d AND status = 'active'
|
||||
AND embedding IS NOT NULL
|
||||
`, idx, idx+1)
|
||||
args = append(args, vecStr, *personaID)
|
||||
idx += 2
|
||||
parts = append(parts, personaPart)
|
||||
|
||||
// Persona+user scope
|
||||
puPart := fmt.Sprintf(`
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at,
|
||||
(embedding <=> $%d::vector) AS distance
|
||||
FROM memories
|
||||
WHERE scope = 'persona_user' AND owner_id = $%d AND user_id = $%d AND status = 'active'
|
||||
AND embedding IS NOT NULL
|
||||
`, idx, idx+1, idx+2)
|
||||
args = append(args, vecStr, *personaID, userID)
|
||||
idx += 3
|
||||
parts = append(parts, puPart)
|
||||
}
|
||||
|
||||
args = append(args, limit)
|
||||
fullQuery := fmt.Sprintf(`
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at
|
||||
FROM (%s) AS combined
|
||||
WHERE distance < 0.7
|
||||
ORDER BY distance ASC, confidence DESC
|
||||
LIMIT $%d
|
||||
`, strings.Join(parts, " UNION ALL "), idx)
|
||||
|
||||
rows, err := DB.QueryContext(ctx, fullQuery, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return scanMemories(rows)
|
||||
}
|
||||
|
||||
// mergeResults deduplicates and merges semantic + keyword results.
|
||||
func mergeResults(semantic, keyword []models.Memory, limit int) []models.Memory {
|
||||
seen := make(map[string]bool, len(semantic))
|
||||
result := make([]models.Memory, 0, limit)
|
||||
|
||||
// Semantic results first (higher relevance)
|
||||
for _, m := range semantic {
|
||||
if len(result) >= limit {
|
||||
break
|
||||
}
|
||||
seen[m.ID] = true
|
||||
result = append(result, m)
|
||||
}
|
||||
|
||||
// Then keyword results not already included
|
||||
for _, m := range keyword {
|
||||
if len(result) >= limit {
|
||||
break
|
||||
}
|
||||
if !seen[m.ID] {
|
||||
seen[m.ID] = true
|
||||
result = append(result, m)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// ensure unused
|
||||
var _ = sql.ErrNoRows
|
||||
@@ -15,20 +15,23 @@ func NewPersonaStore() *PersonaStore { return &PersonaStore{} }
|
||||
|
||||
const personaCols = `id, name, description, icon, avatar, base_model_id, provider_config_id,
|
||||
system_prompt, temperature, max_tokens, thinking_budget, top_p,
|
||||
scope, owner_id, created_by, is_active, is_shared, created_at, updated_at`
|
||||
scope, owner_id, created_by, is_active, is_shared, memory_enabled, memory_extraction_prompt,
|
||||
created_at, updated_at`
|
||||
|
||||
func (s *PersonaStore) Create(ctx context.Context, p *models.Persona) error {
|
||||
return DB.QueryRowContext(ctx, `
|
||||
INSERT INTO personas (name, description, icon, avatar, base_model_id, provider_config_id,
|
||||
system_prompt, temperature, max_tokens, thinking_budget, top_p,
|
||||
scope, owner_id, created_by, is_active, is_shared)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)
|
||||
scope, owner_id, created_by, is_active, is_shared,
|
||||
memory_enabled, memory_extraction_prompt)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)
|
||||
RETURNING id, created_at, updated_at`,
|
||||
p.Name, p.Description, p.Icon, p.Avatar, p.BaseModelID,
|
||||
models.NullString(p.ProviderConfigID),
|
||||
p.SystemPrompt, models.NullFloat(p.Temperature), models.NullInt(p.MaxTokens),
|
||||
models.NullInt(p.ThinkingBudget), models.NullFloat(p.TopP),
|
||||
p.Scope, models.NullString(p.OwnerID), p.CreatedBy, p.IsActive, p.IsShared,
|
||||
p.MemoryEnabled, p.MemoryExtractionPrompt,
|
||||
).Scan(&p.ID, &p.CreatedAt, &p.UpdatedAt)
|
||||
}
|
||||
|
||||
@@ -86,6 +89,12 @@ func (s *PersonaStore) Update(ctx context.Context, id string, patch models.Perso
|
||||
if patch.IsShared != nil {
|
||||
b.Set("is_shared", *patch.IsShared)
|
||||
}
|
||||
if patch.MemoryEnabled != nil {
|
||||
b.Set("memory_enabled", *patch.MemoryEnabled)
|
||||
}
|
||||
if patch.MemoryExtractionPrompt != nil {
|
||||
b.Set("memory_extraction_prompt", *patch.MemoryExtractionPrompt)
|
||||
}
|
||||
if !b.HasSets() {
|
||||
return nil
|
||||
}
|
||||
@@ -279,6 +288,7 @@ func scanPersona(row *sql.Row) (*models.Persona, error) {
|
||||
&p.BaseModelID, &providerConfigID,
|
||||
&p.SystemPrompt, &temp, &maxTokens, &thinkingBudget, &topP,
|
||||
&p.Scope, &ownerID, &p.CreatedBy, &p.IsActive, &p.IsShared,
|
||||
&p.MemoryEnabled, &p.MemoryExtractionPrompt,
|
||||
&p.CreatedAt, &p.UpdatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
@@ -305,6 +315,7 @@ func scanPersonas(rows *sql.Rows) ([]models.Persona, error) {
|
||||
&p.BaseModelID, &providerConfigID,
|
||||
&p.SystemPrompt, &temp, &maxTokens, &thinkingBudget, &topP,
|
||||
&p.Scope, &ownerID, &p.CreatedBy, &p.IsActive, &p.IsShared,
|
||||
&p.MemoryEnabled, &p.MemoryExtractionPrompt,
|
||||
&p.CreatedAt, &p.UpdatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
|
||||
@@ -31,5 +31,6 @@ func NewStores(db *sql.DB) store.Stores {
|
||||
KnowledgeBases: NewKnowledgeBaseStore(),
|
||||
Groups: NewGroupStore(),
|
||||
ResourceGrants: NewResourceGrantStore(),
|
||||
Memories: NewMemoryStore(),
|
||||
}
|
||||
}
|
||||
|
||||
224
server/store/sqlite/memory.go
Normal file
224
server/store/sqlite/memory.go
Normal file
@@ -0,0 +1,224 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
type MemoryStore struct{}
|
||||
|
||||
func NewMemoryStore() *MemoryStore { return &MemoryStore{} }
|
||||
|
||||
func (s *MemoryStore) Upsert(ctx context.Context, m *models.Memory) error {
|
||||
if m.ID == "" {
|
||||
m.ID = store.NewID()
|
||||
}
|
||||
if m.Status == "" {
|
||||
m.Status = models.MemoryStatusActive
|
||||
}
|
||||
if m.Confidence == 0 {
|
||||
m.Confidence = 1.0
|
||||
}
|
||||
|
||||
_, err := DB.ExecContext(ctx, `
|
||||
INSERT INTO memories (id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key)
|
||||
DO UPDATE SET
|
||||
value = excluded.value,
|
||||
confidence = excluded.confidence,
|
||||
status = excluded.status,
|
||||
source_channel_id = excluded.source_channel_id,
|
||||
updated_at = datetime('now')
|
||||
`, m.ID, m.Scope, m.OwnerID, m.UserID, m.Key, m.Value,
|
||||
m.SourceChannelID, m.Confidence, m.Status)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *MemoryStore) GetByID(ctx context.Context, id string) (*models.Memory, error) {
|
||||
m := &models.Memory{}
|
||||
err := DB.QueryRowContext(ctx, `
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at
|
||||
FROM memories WHERE id = ?
|
||||
`, id).Scan(
|
||||
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
|
||||
&m.SourceChannelID, &m.Confidence, &m.Status,
|
||||
st(&m.CreatedAt), st(&m.UpdatedAt),
|
||||
)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("memory not found: %s", id)
|
||||
}
|
||||
return m, err
|
||||
}
|
||||
|
||||
func (s *MemoryStore) List(ctx context.Context, f models.MemoryFilter) ([]models.Memory, error) {
|
||||
clauses := []string{"scope = ?", "owner_id = ?"}
|
||||
args := []interface{}{f.Scope, f.OwnerID}
|
||||
|
||||
if f.UserID != nil {
|
||||
clauses = append(clauses, "user_id = ?")
|
||||
args = append(args, *f.UserID)
|
||||
}
|
||||
|
||||
status := f.Status
|
||||
if status == "" {
|
||||
status = models.MemoryStatusActive
|
||||
}
|
||||
clauses = append(clauses, "status = ?")
|
||||
args = append(args, status)
|
||||
|
||||
if f.Query != "" {
|
||||
// SQLite: LIKE-based keyword search (no tsvector)
|
||||
clauses = append(clauses, "(key || ' ' || value) LIKE ?")
|
||||
args = append(args, "%"+f.Query+"%")
|
||||
}
|
||||
|
||||
limit := f.Limit
|
||||
if limit <= 0 || limit > 200 {
|
||||
limit = 50
|
||||
}
|
||||
args = append(args, limit)
|
||||
|
||||
query := fmt.Sprintf(`
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at
|
||||
FROM memories
|
||||
WHERE %s
|
||||
ORDER BY confidence DESC, updated_at DESC
|
||||
LIMIT ?
|
||||
`, strings.Join(clauses, " AND "))
|
||||
|
||||
rows, err := DB.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return s.scanMemories(rows)
|
||||
}
|
||||
|
||||
func (s *MemoryStore) Delete(ctx context.Context, id string) error {
|
||||
_, err := DB.ExecContext(ctx, `DELETE FROM memories WHERE id = ?`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *MemoryStore) Archive(ctx context.Context, id string) error {
|
||||
_, err := DB.ExecContext(ctx, `
|
||||
UPDATE memories SET status = 'archived', updated_at = datetime('now') WHERE id = ?
|
||||
`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *MemoryStore) Recall(ctx context.Context, userID string, personaID *string, query string, limit int) ([]models.Memory, error) {
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 20
|
||||
}
|
||||
|
||||
// Build UNION query across applicable scopes
|
||||
parts := []string{}
|
||||
args := []interface{}{}
|
||||
|
||||
// 1. User-scope memories
|
||||
userPart := `
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at
|
||||
FROM memories
|
||||
WHERE scope = 'user' AND owner_id = ? AND status = 'active'
|
||||
`
|
||||
args = append(args, userID)
|
||||
|
||||
if query != "" {
|
||||
userPart += " AND (key || ' ' || value) LIKE ?"
|
||||
args = append(args, "%"+query+"%")
|
||||
}
|
||||
parts = append(parts, userPart)
|
||||
|
||||
// 2. Persona-scope memories
|
||||
if personaID != nil && *personaID != "" {
|
||||
personaPart := `
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at
|
||||
FROM memories
|
||||
WHERE scope = 'persona' AND owner_id = ? AND status = 'active'
|
||||
`
|
||||
args = append(args, *personaID)
|
||||
if query != "" {
|
||||
personaPart += " AND (key || ' ' || value) LIKE ?"
|
||||
args = append(args, "%"+query+"%")
|
||||
}
|
||||
parts = append(parts, personaPart)
|
||||
|
||||
// 3. Persona+user memories
|
||||
puPart := `
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at
|
||||
FROM memories
|
||||
WHERE scope = 'persona_user' AND owner_id = ? AND user_id = ? AND status = 'active'
|
||||
`
|
||||
args = append(args, *personaID, userID)
|
||||
if query != "" {
|
||||
puPart += " AND (key || ' ' || value) LIKE ?"
|
||||
args = append(args, "%"+query+"%")
|
||||
}
|
||||
parts = append(parts, puPart)
|
||||
}
|
||||
|
||||
fullQuery := fmt.Sprintf(`
|
||||
SELECT * FROM (%s)
|
||||
ORDER BY
|
||||
CASE scope
|
||||
WHEN 'persona_user' THEN 0
|
||||
WHEN 'persona' THEN 1
|
||||
WHEN 'user' THEN 2
|
||||
END,
|
||||
confidence DESC,
|
||||
updated_at DESC
|
||||
LIMIT ?
|
||||
`, strings.Join(parts, " UNION ALL "))
|
||||
args = append(args, limit)
|
||||
|
||||
rows, err := DB.QueryContext(ctx, fullQuery, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
return s.scanMemories(rows)
|
||||
}
|
||||
|
||||
func (s *MemoryStore) CountByOwner(ctx context.Context, scope, ownerID string) (int, error) {
|
||||
var count int
|
||||
err := DB.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*) FROM memories
|
||||
WHERE scope = ? AND owner_id = ? AND status = 'active'
|
||||
`, scope, ownerID).Scan(&count)
|
||||
return count, err
|
||||
}
|
||||
|
||||
// ── Helpers ─────────────────────────────────
|
||||
|
||||
func (s *MemoryStore) scanMemories(rows *sql.Rows) ([]models.Memory, error) {
|
||||
var memories []models.Memory
|
||||
for rows.Next() {
|
||||
var m models.Memory
|
||||
if err := rows.Scan(
|
||||
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
|
||||
&m.SourceChannelID, &m.Confidence, &m.Status,
|
||||
st(&m.CreatedAt), st(&m.UpdatedAt),
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
memories = append(memories, m)
|
||||
}
|
||||
if memories == nil {
|
||||
memories = []models.Memory{}
|
||||
}
|
||||
return memories, rows.Err()
|
||||
}
|
||||
146
server/store/sqlite/memory_hybrid.go
Normal file
146
server/store/sqlite/memory_hybrid.go
Normal file
@@ -0,0 +1,146 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
// RecallHybrid returns active memories using vector similarity + keyword search.
|
||||
// SQLite has no pgvector, so we load embeddings and compute cosine similarity in Go.
|
||||
func (s *MemoryStore) RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error) {
|
||||
if len(queryVec) == 0 {
|
||||
return s.Recall(ctx, userID, personaID, query, limit)
|
||||
}
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 30
|
||||
}
|
||||
|
||||
// Load all active memories with embeddings for applicable scopes
|
||||
semantic, err := s.recallSemantic(ctx, userID, personaID, queryVec, limit)
|
||||
if err != nil || len(semantic) == 0 {
|
||||
return s.Recall(ctx, userID, personaID, query, limit)
|
||||
}
|
||||
|
||||
var keyword []models.Memory
|
||||
if query != "" {
|
||||
keyword, _ = s.Recall(ctx, userID, personaID, query, limit)
|
||||
}
|
||||
|
||||
return mergeResultsSQLite(semantic, keyword, limit), nil
|
||||
}
|
||||
|
||||
type scoredMemory struct {
|
||||
Memory models.Memory
|
||||
Similarity float64
|
||||
}
|
||||
|
||||
func (s *MemoryStore) recallSemantic(ctx context.Context, userID string, personaID *string, queryVec []float64, limit int) ([]models.Memory, error) {
|
||||
// Build query to load memories with embeddings
|
||||
parts := []string{}
|
||||
args := []interface{}{}
|
||||
|
||||
userPart := `
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at, embedding
|
||||
FROM memories
|
||||
WHERE scope = 'user' AND owner_id = ? AND status = 'active'
|
||||
AND embedding IS NOT NULL
|
||||
`
|
||||
args = append(args, userID)
|
||||
parts = append(parts, userPart)
|
||||
|
||||
if personaID != nil && *personaID != "" {
|
||||
personaPart := `
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at, embedding
|
||||
FROM memories
|
||||
WHERE scope = 'persona' AND owner_id = ? AND status = 'active'
|
||||
AND embedding IS NOT NULL
|
||||
`
|
||||
args = append(args, *personaID)
|
||||
parts = append(parts, personaPart)
|
||||
|
||||
puPart := `
|
||||
SELECT id, scope, owner_id, user_id, key, value,
|
||||
source_channel_id, confidence, status, created_at, updated_at, embedding
|
||||
FROM memories
|
||||
WHERE scope = 'persona_user' AND owner_id = ? AND user_id = ? AND status = 'active'
|
||||
AND embedding IS NOT NULL
|
||||
`
|
||||
args = append(args, *personaID, userID)
|
||||
parts = append(parts, puPart)
|
||||
}
|
||||
|
||||
fullQuery := strings.Join(parts, " UNION ALL ")
|
||||
rows, err := DB.QueryContext(ctx, fullQuery, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// Score each memory by cosine similarity
|
||||
var scored []scoredMemory
|
||||
for rows.Next() {
|
||||
var m models.Memory
|
||||
var embeddingJSON string
|
||||
if err := rows.Scan(
|
||||
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
|
||||
&m.SourceChannelID, &m.Confidence, &m.Status,
|
||||
st(&m.CreatedAt), st(&m.UpdatedAt), &embeddingJSON,
|
||||
); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var vec []float64
|
||||
if err := json.Unmarshal([]byte(embeddingJSON), &vec); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// cosineSimilarity is defined in knowledge_bases.go (same package)
|
||||
sim := cosineSimilarity(queryVec, vec)
|
||||
if sim > 0.3 {
|
||||
scored = append(scored, scoredMemory{Memory: m, Similarity: sim})
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by similarity descending (simple insertion sort for small N)
|
||||
for i := 0; i < len(scored); i++ {
|
||||
for j := i + 1; j < len(scored); j++ {
|
||||
if scored[j].Similarity > scored[i].Similarity {
|
||||
scored[i], scored[j] = scored[j], scored[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Take top N
|
||||
result := make([]models.Memory, 0, limit)
|
||||
for i := 0; i < len(scored) && i < limit; i++ {
|
||||
result = append(result, scored[i].Memory)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func mergeResultsSQLite(semantic, keyword []models.Memory, limit int) []models.Memory {
|
||||
seen := make(map[string]bool, len(semantic))
|
||||
result := make([]models.Memory, 0, limit)
|
||||
|
||||
for _, m := range semantic {
|
||||
if len(result) >= limit {
|
||||
break
|
||||
}
|
||||
seen[m.ID] = true
|
||||
result = append(result, m)
|
||||
}
|
||||
for _, m := range keyword {
|
||||
if len(result) >= limit {
|
||||
break
|
||||
}
|
||||
if !seen[m.ID] {
|
||||
result = append(result, m)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -17,7 +17,8 @@ func NewPersonaStore() *PersonaStore { return &PersonaStore{} }
|
||||
|
||||
const personaCols = `id, name, description, icon, avatar, base_model_id, provider_config_id,
|
||||
system_prompt, temperature, max_tokens, thinking_budget, top_p,
|
||||
scope, owner_id, created_by, is_active, is_shared, created_at, updated_at`
|
||||
scope, owner_id, created_by, is_active, is_shared, memory_enabled, memory_extraction_prompt,
|
||||
created_at, updated_at`
|
||||
|
||||
func (s *PersonaStore) Create(ctx context.Context, p *models.Persona) error {
|
||||
p.ID = store.NewID()
|
||||
@@ -27,13 +28,15 @@ func (s *PersonaStore) Create(ctx context.Context, p *models.Persona) error {
|
||||
_, err := DB.ExecContext(ctx, `
|
||||
INSERT INTO personas (id, name, description, icon, avatar, base_model_id, provider_config_id,
|
||||
system_prompt, temperature, max_tokens, thinking_budget, top_p,
|
||||
scope, owner_id, created_by, is_active, is_shared, created_at, updated_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
scope, owner_id, created_by, is_active, is_shared, memory_enabled, memory_extraction_prompt,
|
||||
created_at, updated_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
||||
p.ID, p.Name, p.Description, p.Icon, p.Avatar, p.BaseModelID,
|
||||
models.NullString(p.ProviderConfigID),
|
||||
p.SystemPrompt, models.NullFloat(p.Temperature), models.NullInt(p.MaxTokens),
|
||||
models.NullInt(p.ThinkingBudget), models.NullFloat(p.TopP),
|
||||
p.Scope, models.NullString(p.OwnerID), p.CreatedBy, p.IsActive, p.IsShared,
|
||||
p.MemoryEnabled, p.MemoryExtractionPrompt,
|
||||
now.Format(timeFmt), now.Format(timeFmt),
|
||||
)
|
||||
return err
|
||||
@@ -93,6 +96,12 @@ func (s *PersonaStore) Update(ctx context.Context, id string, patch models.Perso
|
||||
if patch.IsShared != nil {
|
||||
b.Set("is_shared", *patch.IsShared)
|
||||
}
|
||||
if patch.MemoryEnabled != nil {
|
||||
b.Set("memory_enabled", *patch.MemoryEnabled)
|
||||
}
|
||||
if patch.MemoryExtractionPrompt != nil {
|
||||
b.Set("memory_extraction_prompt", *patch.MemoryExtractionPrompt)
|
||||
}
|
||||
if !b.HasSets() {
|
||||
return nil
|
||||
}
|
||||
@@ -286,6 +295,7 @@ func scanPersona(row *sql.Row) (*models.Persona, error) {
|
||||
&p.BaseModelID, &providerConfigID,
|
||||
&p.SystemPrompt, &temp, &maxTokens, &thinkingBudget, &topP,
|
||||
&p.Scope, &ownerID, &p.CreatedBy, &p.IsActive, &p.IsShared,
|
||||
&p.MemoryEnabled, &p.MemoryExtractionPrompt,
|
||||
st(&p.CreatedAt), st(&p.UpdatedAt),
|
||||
)
|
||||
if err != nil {
|
||||
@@ -312,6 +322,7 @@ func scanPersonas(rows *sql.Rows) ([]models.Persona, error) {
|
||||
&p.BaseModelID, &providerConfigID,
|
||||
&p.SystemPrompt, &temp, &maxTokens, &thinkingBudget, &topP,
|
||||
&p.Scope, &ownerID, &p.CreatedBy, &p.IsActive, &p.IsShared,
|
||||
&p.MemoryEnabled, &p.MemoryExtractionPrompt,
|
||||
st(&p.CreatedAt), st(&p.UpdatedAt),
|
||||
)
|
||||
if err != nil {
|
||||
|
||||
@@ -31,5 +31,6 @@ func NewStores(db *sql.DB) store.Stores {
|
||||
KnowledgeBases: NewKnowledgeBaseStore(),
|
||||
Groups: NewGroupStore(),
|
||||
ResourceGrants: NewResourceGrantStore(),
|
||||
Memories: NewMemoryStore(),
|
||||
}
|
||||
}
|
||||
|
||||
47
server/store/store_memory.go
Normal file
47
server/store/store_memory.go
Normal file
@@ -0,0 +1,47 @@
|
||||
// ── store_memory.go ─────────────────────────
|
||||
// MemoryStore interface for v0.18.0.
|
||||
// Add this interface to store/interfaces.go and add
|
||||
// Memories MemoryStore
|
||||
// to the Stores struct.
|
||||
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
)
|
||||
|
||||
// =========================================
|
||||
// MEMORY STORE (v0.18.0)
|
||||
// =========================================
|
||||
|
||||
type MemoryStore interface {
|
||||
// Upsert creates or updates a memory (matched on scope+owner+user+key).
|
||||
// If a memory with the same key exists, value/confidence/status are updated.
|
||||
Upsert(ctx context.Context, m *models.Memory) error
|
||||
|
||||
// GetByID returns a single memory by ID.
|
||||
GetByID(ctx context.Context, id string) (*models.Memory, error)
|
||||
|
||||
// List returns memories matching the filter criteria.
|
||||
List(ctx context.Context, f models.MemoryFilter) ([]models.Memory, error)
|
||||
|
||||
// Delete hard-deletes a memory by ID.
|
||||
Delete(ctx context.Context, id string) error
|
||||
|
||||
// Archive sets status to 'archived' (soft delete).
|
||||
Archive(ctx context.Context, id string) error
|
||||
|
||||
// Recall returns active memories relevant to a query, merging scopes.
|
||||
// Combines user + persona + persona_user memories for the given context.
|
||||
// Results are ordered by confidence DESC, updated_at DESC.
|
||||
Recall(ctx context.Context, userID string, personaID *string, query string, limit int) ([]models.Memory, error)
|
||||
|
||||
// CountByOwner returns the number of active memories for a scope+owner.
|
||||
CountByOwner(ctx context.Context, scope, ownerID string) (int, error)
|
||||
|
||||
// RecallHybrid returns active memories using vector similarity + keyword search.
|
||||
// If queryVec is nil/empty, falls back to keyword-only (same as Recall).
|
||||
RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error)
|
||||
}
|
||||
15
server/store/store_memory_hybrid.go
Normal file
15
server/store/store_memory_hybrid.go
Normal file
@@ -0,0 +1,15 @@
|
||||
// ── store_memory_hybrid.go ──────────────────
|
||||
// Phase 2 addition to the MemoryStore interface.
|
||||
//
|
||||
// Add this method to the MemoryStore interface in store/store_memory.go:
|
||||
//
|
||||
// // RecallHybrid returns active memories using vector similarity + keyword search.
|
||||
// // If queryVec is nil/empty, falls back to keyword-only (same as Recall).
|
||||
// // Merges results from applicable scopes with deduplication.
|
||||
// RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error)
|
||||
//
|
||||
// The implementations are in:
|
||||
// - store/postgres/memory_hybrid.go (pgvector cosine distance)
|
||||
// - store/sqlite/memory_hybrid.go (app-level cosine similarity)
|
||||
|
||||
package store
|
||||
238
server/tools/memory.go
Normal file
238
server/tools/memory.go
Normal file
@@ -0,0 +1,238 @@
|
||||
package tools
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"git.gobha.me/xcaliber/chat-switchboard/database"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/models"
|
||||
"git.gobha.me/xcaliber/chat-switchboard/store"
|
||||
)
|
||||
|
||||
// ── Late Registration ────────────────────────
|
||||
// Memory tools use late registration because they need stores + embedder.
|
||||
|
||||
// RegisterMemoryTools registers the memory_save and memory_recall tools.
|
||||
// Called from main.go after stores and embedder are initialized.
|
||||
func RegisterMemoryTools(stores store.Stores, embedder *knowledge.Embedder) {
|
||||
if stores.Memories == nil {
|
||||
log.Println("⚠ memory tools: MemoryStore not available, skipping registration")
|
||||
return
|
||||
}
|
||||
|
||||
Register(&memorySaveTool{stores: stores, embedder: embedder})
|
||||
Register(&memoryRecallTool{stores: stores, embedder: embedder})
|
||||
|
||||
log.Println("✅ memory tools registered (memory_save, memory_recall)")
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════
|
||||
// memory_save
|
||||
// ═══════════════════════════════════════════
|
||||
|
||||
type memorySaveTool struct {
|
||||
stores store.Stores
|
||||
embedder *knowledge.Embedder
|
||||
}
|
||||
|
||||
func (t *memorySaveTool) Definition() ToolDef {
|
||||
return ToolDef{
|
||||
Name: "memory_save",
|
||||
DisplayName: "Save Memory",
|
||||
Category: "memory",
|
||||
Description: "Save a fact or preference about the user for future conversations. " +
|
||||
"Use this when the user shares something worth remembering long-term " +
|
||||
"(preferences, technical stack, project details, personal facts). " +
|
||||
"Memories persist across all conversations.",
|
||||
Parameters: JSONSchema(map[string]interface{}{
|
||||
"key": Prop("string", "Short label for the memory (2-5 words, e.g. 'preferred language', 'deployment target')"),
|
||||
"value": Prop("string", "The fact or detail to remember (1-2 sentences)"),
|
||||
"confidence": map[string]interface{}{
|
||||
"type": "number",
|
||||
"description": "How confident you are in this fact (0.0-1.0, default 0.9)",
|
||||
},
|
||||
}, []string{"key", "value"}),
|
||||
}
|
||||
}
|
||||
|
||||
func (t *memorySaveTool) Execute(ctx context.Context, execCtx ExecutionContext, argsJSON string) (string, error) {
|
||||
var args struct {
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(argsJSON), &args); err != nil {
|
||||
return "", fmt.Errorf("invalid arguments: %w", err)
|
||||
}
|
||||
|
||||
if args.Key == "" || args.Value == "" {
|
||||
return "", fmt.Errorf("key and value are required")
|
||||
}
|
||||
|
||||
confidence := args.Confidence
|
||||
if confidence <= 0 || confidence > 1.0 {
|
||||
confidence = 0.9
|
||||
}
|
||||
|
||||
mem := &models.Memory{
|
||||
ID: store.NewID(),
|
||||
Scope: models.MemoryScopeUser,
|
||||
OwnerID: execCtx.UserID,
|
||||
Key: strings.TrimSpace(args.Key),
|
||||
Value: strings.TrimSpace(args.Value),
|
||||
Confidence: confidence,
|
||||
Status: models.MemoryStatusActive,
|
||||
}
|
||||
|
||||
if execCtx.ChannelID != "" {
|
||||
mem.SourceChannelID = &execCtx.ChannelID
|
||||
}
|
||||
|
||||
// If persona is active, use persona_user scope
|
||||
if execCtx.PersonaID != "" {
|
||||
mem.Scope = models.MemoryScopePersonaUser
|
||||
mem.OwnerID = execCtx.PersonaID
|
||||
mem.UserID = &execCtx.UserID
|
||||
}
|
||||
|
||||
if err := t.stores.Memories.Upsert(ctx, mem); err != nil {
|
||||
return "", fmt.Errorf("save memory: %w", err)
|
||||
}
|
||||
|
||||
// Embed for semantic recall
|
||||
t.embedMemory(ctx, mem, execCtx.UserID)
|
||||
|
||||
return fmt.Sprintf("Remembered: %s = %s (confidence: %.0f%%)",
|
||||
args.Key, args.Value, confidence*100), nil
|
||||
}
|
||||
|
||||
func (t *memorySaveTool) embedMemory(ctx context.Context, m *models.Memory, userID string) {
|
||||
if t.embedder == nil || !t.embedder.IsConfigured(ctx) {
|
||||
return
|
||||
}
|
||||
|
||||
text := m.Key + ": " + m.Value
|
||||
|
||||
var teamID *string
|
||||
if t.stores.Teams != nil {
|
||||
ids, _ := t.stores.Teams.GetUserTeamIDs(ctx, userID)
|
||||
if len(ids) > 0 {
|
||||
teamID = &ids[0]
|
||||
}
|
||||
}
|
||||
|
||||
result, err := t.embedder.EmbedChunks(ctx, userID, teamID, []string{text})
|
||||
if err != nil || len(result.Vectors) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
vecStr := vectorToString(result.Vectors[0])
|
||||
if database.IsSQLite() {
|
||||
database.DB.Exec(`UPDATE memories SET embedding = ? WHERE id = ?`,
|
||||
vecStr, m.ID)
|
||||
} else {
|
||||
database.DB.Exec(`UPDATE memories SET embedding = $1::vector WHERE id = $2`,
|
||||
vecStr, m.ID)
|
||||
}
|
||||
|
||||
log.Printf("🧠 memory %s embedded (%d dims)", m.ID[:8], len(result.Vectors[0]))
|
||||
}
|
||||
|
||||
// ═══════════════════════════════════════════
|
||||
// memory_recall
|
||||
// ═══════════════════════════════════════════
|
||||
|
||||
type memoryRecallTool struct {
|
||||
stores store.Stores
|
||||
embedder *knowledge.Embedder
|
||||
}
|
||||
|
||||
func (t *memoryRecallTool) Definition() ToolDef {
|
||||
return ToolDef{
|
||||
Name: "memory_recall",
|
||||
DisplayName: "Recall Memories",
|
||||
Category: "memory",
|
||||
Description: "Search your memories about this user. Use this at the start of conversations " +
|
||||
"or when you need context about the user's preferences, projects, or history.",
|
||||
Parameters: JSONSchema(map[string]interface{}{
|
||||
"query": Prop("string", "Optional search query to filter memories (leave empty for all)"),
|
||||
}, nil),
|
||||
}
|
||||
}
|
||||
|
||||
func (t *memoryRecallTool) Execute(ctx context.Context, execCtx ExecutionContext, argsJSON string) (string, error) {
|
||||
var args struct {
|
||||
Query string `json:"query"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(argsJSON), &args); err != nil {
|
||||
return "", fmt.Errorf("invalid arguments: %w", err)
|
||||
}
|
||||
|
||||
var personaID *string
|
||||
if execCtx.PersonaID != "" {
|
||||
personaID = &execCtx.PersonaID
|
||||
}
|
||||
|
||||
// Try hybrid recall if embedder available and query provided
|
||||
var memories []models.Memory
|
||||
var err error
|
||||
|
||||
if t.embedder != nil && t.embedder.IsConfigured(ctx) && args.Query != "" {
|
||||
memories, err = t.recallWithEmbedding(ctx, execCtx.UserID, personaID, args.Query)
|
||||
}
|
||||
|
||||
// Fall back to keyword recall
|
||||
if err != nil || len(memories) == 0 {
|
||||
memories, err = t.stores.Memories.Recall(ctx, execCtx.UserID, personaID, args.Query, 20)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("recall memories: %w", err)
|
||||
}
|
||||
|
||||
if len(memories) == 0 {
|
||||
suffix := ""
|
||||
if args.Query != "" {
|
||||
suffix = " matching '" + args.Query + "'"
|
||||
}
|
||||
return "No memories found" + suffix + ".", nil
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("Found %d memories:\n\n", len(memories)))
|
||||
for _, m := range memories {
|
||||
scope := m.Scope
|
||||
if scope == "persona_user" {
|
||||
scope = "persona"
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("• [%s] %s: %s (%.0f%% confidence)\n",
|
||||
scope, m.Key, m.Value, m.Confidence*100))
|
||||
}
|
||||
return sb.String(), nil
|
||||
}
|
||||
|
||||
func (t *memoryRecallTool) recallWithEmbedding(ctx context.Context, userID string, personaID *string, query string) ([]models.Memory, error) {
|
||||
text := query
|
||||
if len(text) > 2000 {
|
||||
text = text[:2000]
|
||||
}
|
||||
|
||||
var teamID *string
|
||||
if t.stores.Teams != nil {
|
||||
ids, _ := t.stores.Teams.GetUserTeamIDs(ctx, userID)
|
||||
if len(ids) > 0 {
|
||||
teamID = &ids[0]
|
||||
}
|
||||
}
|
||||
|
||||
result, err := t.embedder.EmbedChunks(ctx, userID, teamID, []string{text})
|
||||
if err != nil || len(result.Vectors) == 0 {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return t.stores.Memories.RecallHybrid(ctx, userID, personaID, query, result.Vectors[0], 20)
|
||||
}
|
||||
Reference in New Issue
Block a user