Changeset 0.18.0 (#79)
This commit is contained in:
85
server/database/migrations/004_v0180_memories.sql
Normal file
85
server/database/migrations/004_v0180_memories.sql
Normal file
@@ -0,0 +1,85 @@
|
||||
-- v0.18.0: Memory System
|
||||
--
|
||||
-- Long-term memory across conversations, with scope-aware isolation.
|
||||
-- Three scopes: user (personal), persona (shared), persona_user (per-user-per-persona).
|
||||
--
|
||||
-- Unlike KB (static documents) or compaction (within-conversation), this
|
||||
-- captures facts and preferences that persist across channels.
|
||||
|
||||
-- =========================================
|
||||
-- 1. Memories Table
|
||||
-- =========================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memories (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
scope TEXT NOT NULL CHECK (scope IN ('user', 'persona', 'persona_user')),
|
||||
|
||||
-- Owner resolution:
|
||||
-- user scope: owner_id = user_id, user_id = NULL
|
||||
-- persona scope: owner_id = persona_id, user_id = NULL
|
||||
-- persona_user scope: owner_id = persona_id, user_id = user_id
|
||||
owner_id UUID NOT NULL,
|
||||
user_id UUID,
|
||||
|
||||
key TEXT NOT NULL, -- short label ("preferred language", "team size")
|
||||
value TEXT NOT NULL, -- detail / fact content
|
||||
source_channel_id UUID, -- which conversation produced this memory
|
||||
confidence REAL NOT NULL DEFAULT 1.0, -- 0.0–1.0, used for ranking
|
||||
status TEXT NOT NULL DEFAULT 'active'
|
||||
CHECK (status IN ('active', 'pending_review', 'archived')),
|
||||
|
||||
-- Embedding for semantic recall (optional — same pattern as notes)
|
||||
embedding vector(3072),
|
||||
|
||||
created_at TIMESTAMPTZ DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ DEFAULT now()
|
||||
);
|
||||
|
||||
-- Primary lookup: "give me all active memories for this scope + owner"
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_scope_owner
|
||||
ON memories(scope, owner_id, status)
|
||||
WHERE status = 'active';
|
||||
|
||||
-- Persona+user lookup: "give me this persona's memories about this user"
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_persona_user
|
||||
ON memories(owner_id, user_id)
|
||||
WHERE scope = 'persona_user' AND status = 'active';
|
||||
|
||||
-- Deduplication: prevent saving the same key twice for the same scope/owner/user
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_unique_key
|
||||
ON memories(scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key);
|
||||
|
||||
-- Full-text search on key + value
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_fts
|
||||
ON memories USING gin(to_tsvector('english', key || ' ' || value));
|
||||
|
||||
-- Source channel reference (for provenance / "where did this come from")
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_source_channel
|
||||
ON memories(source_channel_id)
|
||||
WHERE source_channel_id IS NOT NULL;
|
||||
|
||||
COMMENT ON TABLE memories IS 'Long-term memory facts persisted across conversations, scoped to user, persona, or user+persona';
|
||||
COMMENT ON COLUMN memories.scope IS 'user = personal facts; persona = shared persona knowledge; persona_user = per-user within a persona';
|
||||
COMMENT ON COLUMN memories.owner_id IS 'user_id for user scope, persona_id for persona/persona_user scopes';
|
||||
COMMENT ON COLUMN memories.user_id IS 'Only set for persona_user scope — identifies which user within the persona';
|
||||
COMMENT ON COLUMN memories.key IS 'Short label for the memory (e.g. "preferred language", "deployment stack")';
|
||||
COMMENT ON COLUMN memories.value IS 'The actual fact/detail being remembered';
|
||||
COMMENT ON COLUMN memories.confidence IS '0.0-1.0 confidence score — higher = more certain, used for ranking and pruning';
|
||||
COMMENT ON COLUMN memories.status IS 'active = injected into context; pending_review = awaiting human approval; archived = soft-deleted';
|
||||
|
||||
-- =========================================
|
||||
-- 2. Updated-at Trigger
|
||||
-- =========================================
|
||||
|
||||
CREATE OR REPLACE FUNCTION update_memories_updated_at()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = now();
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
CREATE TRIGGER trg_memories_updated_at
|
||||
BEFORE UPDATE ON memories
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_memories_updated_at();
|
||||
46
server/database/migrations/005_v0180_memory_phase2.sql
Normal file
46
server/database/migrations/005_v0180_memory_phase2.sql
Normal file
@@ -0,0 +1,46 @@
|
||||
-- v0.18.0 Phase 2: Memory Extraction & Embeddings
|
||||
--
|
||||
-- Adds persona memory configuration, HNSW vector index for
|
||||
-- semantic recall, and extraction tracking log.
|
||||
|
||||
-- =========================================
|
||||
-- 1. Persona Memory Configuration
|
||||
-- =========================================
|
||||
|
||||
ALTER TABLE personas ADD COLUMN IF NOT EXISTS memory_enabled BOOLEAN NOT NULL DEFAULT true;
|
||||
ALTER TABLE personas ADD COLUMN IF NOT EXISTS memory_extraction_prompt TEXT;
|
||||
|
||||
COMMENT ON COLUMN personas.memory_enabled IS 'Whether memory tools and extraction are active for this persona';
|
||||
COMMENT ON COLUMN personas.memory_extraction_prompt IS 'Custom extraction prompt; NULL = use system default';
|
||||
|
||||
-- =========================================
|
||||
-- 2. Vector Index for Semantic Recall
|
||||
-- =========================================
|
||||
-- NOTE: HNSW indexes have a 2000-dimension limit in pgvector.
|
||||
-- Our embeddings are 3072-dim (matching KB/notes schema), so we
|
||||
-- skip the HNSW index. Memory tables are small enough (hundreds
|
||||
-- of rows per user) that filtered sequential scans with cosine
|
||||
-- distance are performant. If needed later, IVFFlat supports
|
||||
-- higher dimensions, or embeddings can be reduced to 1536-dim.
|
||||
|
||||
-- =========================================
|
||||
-- 3. Memory Extraction Log
|
||||
-- =========================================
|
||||
-- Tracks which messages have been processed by the extraction scanner
|
||||
-- to prevent duplicate work. One row per channel+user pair.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memory_extraction_log (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
channel_id UUID NOT NULL,
|
||||
user_id UUID NOT NULL,
|
||||
last_message_id UUID NOT NULL,
|
||||
extracted_at TIMESTAMPTZ DEFAULT now(),
|
||||
memory_count INT NOT NULL DEFAULT 0,
|
||||
|
||||
UNIQUE(channel_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_mem_extract_log_channel
|
||||
ON memory_extraction_log(channel_id);
|
||||
|
||||
COMMENT ON TABLE memory_extraction_log IS 'Tracks extraction progress per channel+user to prevent re-processing';
|
||||
61
server/database/migrations/sqlite/003_v0180_memories.sql
Normal file
61
server/database/migrations/sqlite/003_v0180_memories.sql
Normal file
@@ -0,0 +1,61 @@
|
||||
-- v0.18.0: Memory System (SQLite)
|
||||
--
|
||||
-- Long-term memory across conversations, with scope-aware isolation.
|
||||
-- Three scopes: user (personal), persona (shared), persona_user (per-user-per-persona).
|
||||
|
||||
-- =========================================
|
||||
-- 1. Memories Table
|
||||
-- =========================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memories (
|
||||
id TEXT PRIMARY KEY,
|
||||
scope TEXT NOT NULL CHECK (scope IN ('user', 'persona', 'persona_user')),
|
||||
|
||||
-- Owner resolution:
|
||||
-- user scope: owner_id = user_id, user_id = NULL
|
||||
-- persona scope: owner_id = persona_id, user_id = NULL
|
||||
-- persona_user scope: owner_id = persona_id, user_id = user_id
|
||||
owner_id TEXT NOT NULL,
|
||||
user_id TEXT,
|
||||
|
||||
key TEXT NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
source_channel_id TEXT,
|
||||
confidence REAL NOT NULL DEFAULT 1.0,
|
||||
status TEXT NOT NULL DEFAULT 'active'
|
||||
CHECK (status IN ('active', 'pending_review', 'archived')),
|
||||
|
||||
-- Embedding stored as JSON array of float64 for app-level cosine similarity
|
||||
embedding TEXT,
|
||||
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- Primary lookup
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_scope_owner
|
||||
ON memories(scope, owner_id, status);
|
||||
|
||||
-- Persona+user lookup
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_persona_user
|
||||
ON memories(owner_id, user_id);
|
||||
|
||||
-- Deduplication: prevent saving the same key twice for the same scope/owner/user
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_unique_key
|
||||
ON memories(scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key);
|
||||
|
||||
-- Source channel reference
|
||||
CREATE INDEX IF NOT EXISTS idx_memories_source_channel
|
||||
ON memories(source_channel_id);
|
||||
|
||||
-- =========================================
|
||||
-- 2. Updated-at Trigger
|
||||
-- =========================================
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS trg_memories_updated_at
|
||||
AFTER UPDATE ON memories
|
||||
FOR EACH ROW
|
||||
WHEN NEW.updated_at = OLD.updated_at
|
||||
BEGIN
|
||||
UPDATE memories SET updated_at = datetime('now') WHERE id = NEW.id;
|
||||
END;
|
||||
@@ -0,0 +1,22 @@
|
||||
-- v0.18.0 Phase 2: Memory Extraction & Embeddings (SQLite)
|
||||
|
||||
-- 1. Persona memory configuration
|
||||
ALTER TABLE personas ADD COLUMN memory_enabled INTEGER NOT NULL DEFAULT 1;
|
||||
ALTER TABLE personas ADD COLUMN memory_extraction_prompt TEXT;
|
||||
|
||||
-- 2. No HNSW index for SQLite — app-level cosine similarity used instead.
|
||||
|
||||
-- 3. Memory extraction log
|
||||
CREATE TABLE IF NOT EXISTS memory_extraction_log (
|
||||
id TEXT PRIMARY KEY,
|
||||
channel_id TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
last_message_id TEXT NOT NULL,
|
||||
extracted_at TEXT DEFAULT (datetime('now')),
|
||||
memory_count INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
UNIQUE(channel_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_mem_extract_log_channel
|
||||
ON memory_extraction_log(channel_id);
|
||||
Reference in New Issue
Block a user