Changeset 0.14.0 (#67)

server/database/migrations/008_knowledge_bases.sql

@@ -0,0 +1,99 @@
+-- 008_knowledge_bases.sql
+-- Knowledge Bases: KB metadata, documents, chunks, channel links.
+-- Requires: pgvector extension already installed (via db-bootstrap.sh).
+
+-- ── Knowledge Bases ──────────────────────────
+
+CREATE TABLE IF NOT EXISTS knowledge_bases (
+    id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    name            TEXT NOT NULL,
+    description     TEXT NOT NULL DEFAULT '',
+    scope           TEXT NOT NULL DEFAULT 'global',  -- global, team, personal
+    owner_id        UUID REFERENCES users(id) ON DELETE CASCADE,
+    team_id         UUID REFERENCES teams(id) ON DELETE CASCADE,
+
+    -- Snapshot of embedding config at creation time.
+    -- Changing the model requires a full re-embed (rebuild endpoint).
+    -- { "provider_config_id": "...", "model_id": "...", "dimensions": 1536 }
+    embedding_config JSONB NOT NULL DEFAULT '{}',
+
+    -- Denormalized stats, updated by ingest pipeline.
+    document_count  INT NOT NULL DEFAULT 0,
+    chunk_count     INT NOT NULL DEFAULT 0,
+    total_bytes     BIGINT NOT NULL DEFAULT 0,
+
+    status          TEXT NOT NULL DEFAULT 'active',  -- active, processing, error
+    created_at      TIMESTAMPTZ NOT NULL DEFAULT now(),
+    updated_at      TIMESTAMPTZ NOT NULL DEFAULT now(),
+
+    CONSTRAINT kb_scope_check CHECK (
+        (scope = 'global'   AND owner_id IS NULL) OR
+        (scope = 'team'     AND team_id IS NOT NULL) OR
+        (scope = 'personal' AND owner_id IS NOT NULL)
+    )
+);
+
+CREATE INDEX IF NOT EXISTS idx_kb_scope ON knowledge_bases(scope);
+CREATE INDEX IF NOT EXISTS idx_kb_owner ON knowledge_bases(owner_id) WHERE owner_id IS NOT NULL;
+CREATE INDEX IF NOT EXISTS idx_kb_team  ON knowledge_bases(team_id)  WHERE team_id IS NOT NULL;
+
+-- ── KB Documents ─────────────────────────────
+-- One row per uploaded file. Blobs live in ObjectStore under:
+--   kb/{kb_id}/{doc_id}_{filename}
+
+CREATE TABLE IF NOT EXISTS kb_documents (
+    id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    kb_id           UUID NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
+    filename        TEXT NOT NULL,
+    content_type    TEXT NOT NULL,
+    size_bytes      BIGINT NOT NULL,
+    storage_key     TEXT NOT NULL,
+    extracted_text  TEXT,                     -- full text for re-chunking
+    chunk_count     INT NOT NULL DEFAULT 0,
+    status          TEXT NOT NULL DEFAULT 'pending',  -- pending, chunking, embedding, ready, error
+    error           TEXT,
+    uploaded_by     UUID NOT NULL REFERENCES users(id),
+    created_at      TIMESTAMPTZ NOT NULL DEFAULT now(),
+    updated_at      TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS idx_kbdoc_kb ON kb_documents(kb_id);
+
+-- ── KB Chunks ────────────────────────────────
+-- Chunked text + embedding vector for similarity search.
+-- vector(3072) accommodates all current embedding models:
+--   OpenAI ada-002 / text-embedding-3-small = 1536
+--   OpenAI text-embedding-3-large = 3072
+--   Open-source models = typically 768-1024
+-- Smaller vectors are zero-padded at insert time.
+
+CREATE TABLE IF NOT EXISTS kb_chunks (
+    id              UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    kb_id           UUID NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
+    document_id     UUID NOT NULL REFERENCES kb_documents(id) ON DELETE CASCADE,
+    chunk_index     INT NOT NULL,
+    content         TEXT NOT NULL,
+    token_count     INT NOT NULL DEFAULT 0,
+    embedding       VECTOR(3072),
+    metadata        JSONB NOT NULL DEFAULT '{}',
+    created_at      TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS idx_kbchunk_kb  ON kb_chunks(kb_id);
+CREATE INDEX IF NOT EXISTS idx_kbchunk_doc ON kb_chunks(document_id);
+
+-- NOTE: IVFFlat index for similarity search is created after first data
+-- load (needs rows to train). The ingest handler creates it:
+--   CREATE INDEX idx_kbchunk_embedding ON kb_chunks
+--       USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
+
+-- ── Channel-KB Links ─────────────────────────
+-- Which KBs are active for a given channel.
+
+CREATE TABLE IF NOT EXISTS channel_knowledge_bases (
+    channel_id      UUID NOT NULL REFERENCES channels(id) ON DELETE CASCADE,
+    kb_id           UUID NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
+    enabled         BOOLEAN NOT NULL DEFAULT true,
+    added_at        TIMESTAMPTZ NOT NULL DEFAULT now(),
+    PRIMARY KEY (channel_id, kb_id)
+);

server/database/migrations/009_notes_embedding.sql

@@ -0,0 +1,12 @@
+-- 009_notes_embedding.sql
+-- Adds vector embedding column to notes for semantic search.
+-- Uses the same vector(3072) dimension as kb_chunks for uniformity.
+
+ALTER TABLE notes ADD COLUMN IF NOT EXISTS embedding vector(3072);
+
+-- NOTE: pgvector HNSW indexes are limited to 2000 dimensions.
+-- For 3072-dim vectors, IVFFlat is the option but requires training
+-- data (rows). The notes table is typically small enough that a
+-- sequential scan is fast. If needed, create after data exists:
+--   CREATE INDEX idx_notes_embedding ON notes
+--       USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);