Changeset 0.14.0 (#67)
This commit is contained in:
99
server/database/migrations/008_knowledge_bases.sql
Normal file
99
server/database/migrations/008_knowledge_bases.sql
Normal file
@@ -0,0 +1,99 @@
|
||||
-- 008_knowledge_bases.sql
|
||||
-- Knowledge Bases: KB metadata, documents, chunks, channel links.
|
||||
-- Requires: pgvector extension already installed (via db-bootstrap.sh).
|
||||
|
||||
-- ── Knowledge Bases ──────────────────────────
|
||||
|
||||
CREATE TABLE IF NOT EXISTS knowledge_bases (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
name TEXT NOT NULL,
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
scope TEXT NOT NULL DEFAULT 'global', -- global, team, personal
|
||||
owner_id UUID REFERENCES users(id) ON DELETE CASCADE,
|
||||
team_id UUID REFERENCES teams(id) ON DELETE CASCADE,
|
||||
|
||||
-- Snapshot of embedding config at creation time.
|
||||
-- Changing the model requires a full re-embed (rebuild endpoint).
|
||||
-- { "provider_config_id": "...", "model_id": "...", "dimensions": 1536 }
|
||||
embedding_config JSONB NOT NULL DEFAULT '{}',
|
||||
|
||||
-- Denormalized stats, updated by ingest pipeline.
|
||||
document_count INT NOT NULL DEFAULT 0,
|
||||
chunk_count INT NOT NULL DEFAULT 0,
|
||||
total_bytes BIGINT NOT NULL DEFAULT 0,
|
||||
|
||||
status TEXT NOT NULL DEFAULT 'active', -- active, processing, error
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT kb_scope_check CHECK (
|
||||
(scope = 'global' AND owner_id IS NULL) OR
|
||||
(scope = 'team' AND team_id IS NOT NULL) OR
|
||||
(scope = 'personal' AND owner_id IS NOT NULL)
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kb_scope ON knowledge_bases(scope);
|
||||
CREATE INDEX IF NOT EXISTS idx_kb_owner ON knowledge_bases(owner_id) WHERE owner_id IS NOT NULL;
|
||||
CREATE INDEX IF NOT EXISTS idx_kb_team ON knowledge_bases(team_id) WHERE team_id IS NOT NULL;
|
||||
|
||||
-- ── KB Documents ─────────────────────────────
|
||||
-- One row per uploaded file. Blobs live in ObjectStore under:
|
||||
-- kb/{kb_id}/{doc_id}_{filename}
|
||||
|
||||
CREATE TABLE IF NOT EXISTS kb_documents (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
kb_id UUID NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
|
||||
filename TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
storage_key TEXT NOT NULL,
|
||||
extracted_text TEXT, -- full text for re-chunking
|
||||
chunk_count INT NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL DEFAULT 'pending', -- pending, chunking, embedding, ready, error
|
||||
error TEXT,
|
||||
uploaded_by UUID NOT NULL REFERENCES users(id),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kbdoc_kb ON kb_documents(kb_id);
|
||||
|
||||
-- ── KB Chunks ────────────────────────────────
|
||||
-- Chunked text + embedding vector for similarity search.
|
||||
-- vector(3072) accommodates all current embedding models:
|
||||
-- OpenAI ada-002 / text-embedding-3-small = 1536
|
||||
-- OpenAI text-embedding-3-large = 3072
|
||||
-- Open-source models = typically 768-1024
|
||||
-- Smaller vectors are zero-padded at insert time.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS kb_chunks (
|
||||
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
kb_id UUID NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
|
||||
document_id UUID NOT NULL REFERENCES kb_documents(id) ON DELETE CASCADE,
|
||||
chunk_index INT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
token_count INT NOT NULL DEFAULT 0,
|
||||
embedding VECTOR(3072),
|
||||
metadata JSONB NOT NULL DEFAULT '{}',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kbchunk_kb ON kb_chunks(kb_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_kbchunk_doc ON kb_chunks(document_id);
|
||||
|
||||
-- NOTE: IVFFlat index for similarity search is created after first data
|
||||
-- load (needs rows to train). The ingest handler creates it:
|
||||
-- CREATE INDEX idx_kbchunk_embedding ON kb_chunks
|
||||
-- USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
|
||||
|
||||
-- ── Channel-KB Links ─────────────────────────
|
||||
-- Which KBs are active for a given channel.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS channel_knowledge_bases (
|
||||
channel_id UUID NOT NULL REFERENCES channels(id) ON DELETE CASCADE,
|
||||
kb_id UUID NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
|
||||
enabled BOOLEAN NOT NULL DEFAULT true,
|
||||
added_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (channel_id, kb_id)
|
||||
);
|
||||
12
server/database/migrations/009_notes_embedding.sql
Normal file
12
server/database/migrations/009_notes_embedding.sql
Normal file
@@ -0,0 +1,12 @@
|
||||
-- 009_notes_embedding.sql
|
||||
-- Adds vector embedding column to notes for semantic search.
|
||||
-- Uses the same vector(3072) dimension as kb_chunks for uniformity.
|
||||
|
||||
ALTER TABLE notes ADD COLUMN IF NOT EXISTS embedding vector(3072);
|
||||
|
||||
-- NOTE: pgvector HNSW indexes are limited to 2000 dimensions.
|
||||
-- For 3072-dim vectors, IVFFlat is the option but requires training
|
||||
-- data (rows). The notes table is typically small enough that a
|
||||
-- sequential scan is fast. If needed, create after data exists:
|
||||
-- CREATE INDEX idx_notes_embedding ON notes
|
||||
-- USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
|
||||
@@ -62,16 +62,23 @@ func SetupTestDB() func() {
|
||||
return func() {}
|
||||
}
|
||||
|
||||
// Attempt to create the test DB. If the user lacks CREATE DATABASE
|
||||
// permissions (e.g. CI already created it via admin bootstrap step),
|
||||
// that's fine — we'll just connect to the existing one.
|
||||
// Check if test DB already exists (e.g. created by CI bootstrap with
|
||||
// admin privileges and extensions like pgvector already installed).
|
||||
// Only create if it doesn't exist — never drop, because recreating
|
||||
// as the app user would lose extensions that require superuser.
|
||||
createdByUs := false
|
||||
adminDB.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", testDBName))
|
||||
if _, err := adminDB.Exec(fmt.Sprintf("CREATE DATABASE %s", testDBName)); err != nil {
|
||||
log.Printf("⚠ Cannot CREATE DATABASE %s (will try to connect to existing): %v", testDBName, err)
|
||||
var dbExists bool
|
||||
err = adminDB.QueryRow("SELECT 1 FROM pg_database WHERE datname = $1", testDBName).Scan(&dbExists)
|
||||
if err != nil {
|
||||
// DB doesn't exist — create it
|
||||
if _, err := adminDB.Exec(fmt.Sprintf("CREATE DATABASE %s", testDBName)); err != nil {
|
||||
log.Printf("⚠ Cannot CREATE DATABASE %s (will try to connect to existing): %v", testDBName, err)
|
||||
} else {
|
||||
createdByUs = true
|
||||
log.Printf("✓ Created test database: %s", testDBName)
|
||||
}
|
||||
} else {
|
||||
createdByUs = true
|
||||
log.Printf("✓ Created test database: %s", testDBName)
|
||||
log.Printf("✓ Test database %s already exists (keeping extensions)", testDBName)
|
||||
}
|
||||
|
||||
// Build DSN for the test DB
|
||||
@@ -100,7 +107,7 @@ func SetupTestDB() func() {
|
||||
TestDB = DB
|
||||
|
||||
// Ensure required extensions (may already exist from CI bootstrap)
|
||||
for _, ext := range []string{"uuid-ossp", "pgcrypto"} {
|
||||
for _, ext := range []string{"uuid-ossp", "pgcrypto", "vector"} {
|
||||
DB.Exec(fmt.Sprintf(`CREATE EXTENSION IF NOT EXISTS "%s"`, ext))
|
||||
}
|
||||
|
||||
@@ -166,6 +173,10 @@ func TruncateAll(t *testing.T) {
|
||||
"messages",
|
||||
"channel_models",
|
||||
"channel_members",
|
||||
"channel_knowledge_bases",
|
||||
"kb_chunks",
|
||||
"kb_documents",
|
||||
"knowledge_bases",
|
||||
"channels",
|
||||
"user_model_settings",
|
||||
"model_catalog",
|
||||
|
||||
Reference in New Issue
Block a user