Changeset 0.14.0 (#67)

2026-02-26 15:59:26 +00:00
parent 1a71658b24
commit e2149e249d
38 changed files with 5171 additions and 141 deletions
--- a/server/knowledge/chunker.go
+++ b/server/knowledge/chunker.go
@@ -0,0 +1,244 @@
+package knowledge
+
+import (
+	"strings"
+	"unicode/utf8"
+)
+
+// ── Configuration ────────────────────────────
+
+// ChunkConfig controls how text is split into chunks.
+type ChunkConfig struct {
+	ChunkSize    int      // target chars per chunk (default 1000)
+	ChunkOverlap int      // overlap chars between adjacent chunks (default 200)
+	Separators   []string // split hierarchy, tried in order
+}
+
+// DefaultChunkConfig returns sensible defaults for general documents.
+// ~250 tokens per chunk at ~4 chars/token.
+func DefaultChunkConfig() ChunkConfig {
+	return ChunkConfig{
+		ChunkSize:    1000,
+		ChunkOverlap: 200,
+		Separators:   []string{"\n\n", "\n", ". ", " "},
+	}
+}
+
+// ── Output ───────────────────────────────────
+
+// Chunk is a single text segment with position metadata.
+type Chunk struct {
+	Content    string         // chunk text
+	Index      int            // ordinal within document (0-based)
+	TokenCount int            // estimated token count (~chars/4)
+	Metadata   map[string]any // extensible: page, heading, etc.
+}
+
+// ── Splitter ─────────────────────────────────
+
+// SplitText splits text into overlapping chunks using recursive
+// character splitting. It tries separators in order, splitting on
+// the first one that produces segments smaller than ChunkSize.
+// Segments that are still too large are recursively split with the
+// next separator.
+func SplitText(text string, cfg ChunkConfig) []Chunk {
+	if cfg.ChunkSize <= 0 {
+		cfg.ChunkSize = 1000
+	}
+	if cfg.ChunkOverlap < 0 {
+		cfg.ChunkOverlap = 0
+	}
+	if cfg.ChunkOverlap >= cfg.ChunkSize {
+		cfg.ChunkOverlap = cfg.ChunkSize / 5
+	}
+	if len(cfg.Separators) == 0 {
+		cfg.Separators = DefaultChunkConfig().Separators
+	}
+
+	text = strings.TrimSpace(text)
+	if text == "" {
+		return nil
+	}
+
+	// If text fits in one chunk, return it directly.
+	if utf8.RuneCountInString(text) <= cfg.ChunkSize {
+		return []Chunk{{
+			Content:    text,
+			Index:      0,
+			TokenCount: estimateTokens(text),
+			Metadata:   map[string]any{},
+		}}
+	}
+
+	// Recursively split, then merge into overlapping windows.
+	segments := recursiveSplit(text, cfg.Separators, cfg.ChunkSize)
+	return mergeSegments(segments, cfg.ChunkSize, cfg.ChunkOverlap)
+}
+
+// recursiveSplit tries each separator in order. For the first separator
+// that actually splits the text, it checks if each piece fits within
+// chunkSize. Pieces that are still too large recurse with the remaining
+// separators. Pieces that fit are kept as-is.
+func recursiveSplit(text string, separators []string, chunkSize int) []string {
+	if utf8.RuneCountInString(text) <= chunkSize {
+		return []string{text}
+	}
+
+	if len(separators) == 0 {
+		// No separators left — hard split by character count.
+		return hardSplit(text, chunkSize)
+	}
+
+	sep := separators[0]
+	rest := separators[1:]
+
+	parts := splitKeepSep(text, sep)
+	if len(parts) <= 1 {
+		// This separator didn't split anything — try next.
+		return recursiveSplit(text, rest, chunkSize)
+	}
+
+	var result []string
+	for _, part := range parts {
+		part = strings.TrimSpace(part)
+		if part == "" {
+			continue
+		}
+		if utf8.RuneCountInString(part) <= chunkSize {
+			result = append(result, part)
+		} else {
+			// Still too large — recurse with remaining separators.
+			result = append(result, recursiveSplit(part, rest, chunkSize)...)
+		}
+	}
+	return result
+}
+
+// splitKeepSep splits text on sep. Unlike strings.Split, the separator
+// is kept at the end of each segment (except the last) to preserve
+// context like sentence-ending periods.
+func splitKeepSep(text, sep string) []string {
+	parts := strings.Split(text, sep)
+	if len(parts) <= 1 {
+		return parts
+	}
+
+	result := make([]string, 0, len(parts))
+	for i, part := range parts {
+		if i < len(parts)-1 {
+			result = append(result, part+sep)
+		} else if part != "" {
+			result = append(result, part)
+		}
+	}
+	return result
+}
+
+// hardSplit cuts text into pieces of at most chunkSize runes.
+// Last resort when no separator works.
+func hardSplit(text string, chunkSize int) []string {
+	runes := []rune(text)
+	var result []string
+	for i := 0; i < len(runes); i += chunkSize {
+		end := i + chunkSize
+		if end > len(runes) {
+			end = len(runes)
+		}
+		result = append(result, string(runes[i:end]))
+	}
+	return result
+}
+
+// mergeSegments combines small segments into chunks up to chunkSize,
+// then applies overlap between adjacent chunks.
+func mergeSegments(segments []string, chunkSize, overlap int) []Chunk {
+	if len(segments) == 0 {
+		return nil
+	}
+
+	// First pass: merge small adjacent segments into windows up to chunkSize.
+	var merged []string
+	var current strings.Builder
+
+	for _, seg := range segments {
+		segLen := utf8.RuneCountInString(seg)
+		curLen := utf8.RuneCountInString(current.String())
+
+		if curLen > 0 && curLen+segLen > chunkSize {
+			// Flush current buffer.
+			merged = append(merged, strings.TrimSpace(current.String()))
+			current.Reset()
+		}
+		if current.Len() > 0 {
+			current.WriteString(" ")
+		}
+		current.WriteString(seg)
+	}
+	if current.Len() > 0 {
+		merged = append(merged, strings.TrimSpace(current.String()))
+	}
+
+	if overlap <= 0 || len(merged) <= 1 {
+		// No overlap needed — convert directly to Chunks.
+		chunks := make([]Chunk, len(merged))
+		for i, text := range merged {
+			chunks[i] = Chunk{
+				Content:    text,
+				Index:      i,
+				TokenCount: estimateTokens(text),
+				Metadata:   map[string]any{},
+			}
+		}
+		return chunks
+	}
+
+	// Second pass: create overlapping chunks.
+	// Each chunk includes up to `overlap` chars from the end of the
+	// previous chunk as prefix context.
+	chunks := make([]Chunk, 0, len(merged))
+	for i, text := range merged {
+		if i > 0 {
+			prev := merged[i-1]
+			suffix := overlapSuffix(prev, overlap)
+			if suffix != "" {
+				text = suffix + " " + text
+			}
+		}
+		chunks = append(chunks, Chunk{
+			Content:    text,
+			Index:      i,
+			TokenCount: estimateTokens(text),
+			Metadata:   map[string]any{},
+		})
+	}
+	return chunks
+}
+
+// overlapSuffix returns the last `n` characters of s, breaking at a
+// word boundary to avoid splitting mid-word.
+func overlapSuffix(s string, n int) string {
+	runes := []rune(s)
+	if len(runes) <= n {
+		return s
+	}
+
+	start := len(runes) - n
+	suffix := string(runes[start:])
+
+	// Try to break at a space to avoid mid-word splits.
+	if idx := strings.Index(suffix, " "); idx >= 0 && idx < len(suffix)/2 {
+		suffix = suffix[idx+1:]
+	}
+	return suffix
+}
+
+// estimateTokens gives a rough token count. Most tokenizers average
+// ~4 characters per token for English text.
+func estimateTokens(s string) int {
+	n := utf8.RuneCountInString(s)
+	tokens := n / 4
+	if tokens == 0 && n > 0 {
+		tokens = 1
+	}
+	return tokens
+}
--- a/server/knowledge/chunker_test.go
+++ b/server/knowledge/chunker_test.go
@@ -0,0 +1,227 @@
+package knowledge
+
+import (
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+func TestSplitText_EmptyInput(t *testing.T) {
+	chunks := SplitText("", DefaultChunkConfig())
+	if chunks != nil {
+		t.Errorf("expected nil for empty input, got %d chunks", len(chunks))
+	}
+}
+
+func TestSplitText_WhitespaceOnly(t *testing.T) {
+	chunks := SplitText("   \n\n  ", DefaultChunkConfig())
+	if chunks != nil {
+		t.Errorf("expected nil for whitespace input, got %d chunks", len(chunks))
+	}
+}
+
+func TestSplitText_SmallText(t *testing.T) {
+	text := "Hello, world."
+	chunks := SplitText(text, DefaultChunkConfig())
+	if len(chunks) != 1 {
+		t.Fatalf("expected 1 chunk, got %d", len(chunks))
+	}
+	if chunks[0].Content != text {
+		t.Errorf("content mismatch: %q", chunks[0].Content)
+	}
+	if chunks[0].Index != 0 {
+		t.Errorf("expected index 0, got %d", chunks[0].Index)
+	}
+	if chunks[0].TokenCount <= 0 {
+		t.Error("expected positive token count")
+	}
+}
+
+func TestSplitText_ParagraphSplit(t *testing.T) {
+	// Two paragraphs, each under chunk size, but together over.
+	para := strings.Repeat("word ", 60) // ~300 chars each
+	text := para + "\n\n" + para
+
+	cfg := ChunkConfig{ChunkSize: 400, ChunkOverlap: 0, Separators: []string{"\n\n", "\n", ". ", " "}}
+	chunks := SplitText(text, cfg)
+
+	if len(chunks) < 2 {
+		t.Fatalf("expected at least 2 chunks from paragraph split, got %d", len(chunks))
+	}
+
+	for i, c := range chunks {
+		if utf8.RuneCountInString(c.Content) == 0 {
+			t.Errorf("chunk %d is empty", i)
+		}
+	}
+}
+
+func TestSplitText_OverlapPresent(t *testing.T) {
+	// Build text that will split into multiple chunks.
+	sentences := make([]string, 20)
+	for i := range sentences {
+		sentences[i] = "This is sentence number " + strings.Repeat("x", 40) + ". "
+	}
+	text := strings.Join(sentences, "")
+
+	cfg := ChunkConfig{ChunkSize: 200, ChunkOverlap: 50, Separators: []string{". ", " "}}
+	chunks := SplitText(text, cfg)
+
+	if len(chunks) < 3 {
+		t.Fatalf("expected at least 3 chunks, got %d", len(chunks))
+	}
+
+	// Verify chunks have sequential indexes.
+	for i, c := range chunks {
+		if c.Index != i {
+			t.Errorf("chunk %d has index %d", i, c.Index)
+		}
+	}
+
+	// With overlap > 0 and multiple chunks, later chunks should share
+	// some content with the previous chunk's tail.
+	// Just verify the overlap logic ran (chunks 1+ are longer or contain
+	// content from the previous chunk's ending).
+	if len(chunks) >= 2 {
+		// Chunk 1 should contain some overlap from chunk 0's tail.
+		// We can't check exact content easily, but the chunk should be
+		// non-trivially sized.
+		if len(chunks[1].Content) < 50 {
+			t.Errorf("chunk 1 seems too short for overlap: %d chars", len(chunks[1].Content))
+		}
+	}
+}
+
+func TestSplitText_ZeroOverlap(t *testing.T) {
+	text := strings.Repeat("A", 500) + "\n\n" + strings.Repeat("B", 500)
+	cfg := ChunkConfig{ChunkSize: 400, ChunkOverlap: 0, Separators: []string{"\n\n"}}
+	chunks := SplitText(text, cfg)
+
+	if len(chunks) < 2 {
+		t.Fatalf("expected at least 2 chunks, got %d", len(chunks))
+	}
+
+	// Without overlap, chunks should not share content.
+	for i, c := range chunks {
+		if c.Index != i {
+			t.Errorf("chunk %d index = %d", i, c.Index)
+		}
+	}
+}
+
+func TestSplitText_HardSplitFallback(t *testing.T) {
+	// No separators at all in the text — forces hard character split.
+	text := strings.Repeat("x", 500)
+	cfg := ChunkConfig{ChunkSize: 200, ChunkOverlap: 0, Separators: []string{"\n\n", "\n", ". ", " "}}
+	chunks := SplitText(text, cfg)
+
+	if len(chunks) < 2 {
+		t.Fatalf("expected multiple chunks from hard split, got %d", len(chunks))
+	}
+
+	// All chunks should be at most chunkSize.
+	for i, c := range chunks {
+		if utf8.RuneCountInString(c.Content) > cfg.ChunkSize {
+			t.Errorf("chunk %d exceeds chunk size: %d > %d",
+				i, utf8.RuneCountInString(c.Content), cfg.ChunkSize)
+		}
+	}
+}
+
+func TestSplitText_TokenEstimate(t *testing.T) {
+	text := strings.Repeat("word ", 100) // 500 chars
+	chunks := SplitText(text, ChunkConfig{ChunkSize: 2000, ChunkOverlap: 0})
+
+	if len(chunks) != 1 {
+		t.Fatalf("expected 1 chunk, got %d", len(chunks))
+	}
+
+	// ~500 chars / 4 = ~125 tokens
+	if chunks[0].TokenCount < 100 || chunks[0].TokenCount > 150 {
+		t.Errorf("expected ~125 tokens, got %d", chunks[0].TokenCount)
+	}
+}
+
+func TestSplitText_InvalidConfig(t *testing.T) {
+	text := "Hello world. This is a test."
+
+	// ChunkSize 0 should use default.
+	chunks := SplitText(text, ChunkConfig{ChunkSize: 0})
+	if len(chunks) != 1 {
+		t.Errorf("expected 1 chunk with zero chunk size (uses default), got %d", len(chunks))
+	}
+
+	// Overlap >= ChunkSize should be clamped.
+	chunks = SplitText(strings.Repeat("word ", 200), ChunkConfig{
+		ChunkSize:    100,
+		ChunkOverlap: 100,
+	})
+	if len(chunks) == 0 {
+		t.Error("expected chunks with overlap == chunk_size (should clamp)")
+	}
+}
+
+func TestSplitText_SentenceSplit(t *testing.T) {
+	text := "First sentence. Second sentence. Third sentence. Fourth sentence. Fifth sentence."
+	cfg := ChunkConfig{ChunkSize: 40, ChunkOverlap: 0, Separators: []string{"\n\n", "\n", ". ", " "}}
+	chunks := SplitText(text, cfg)
+
+	if len(chunks) < 2 {
+		t.Fatalf("expected multiple sentence-split chunks, got %d", len(chunks))
+	}
+
+	// Each chunk should be within bounds.
+	for i, c := range chunks {
+		if utf8.RuneCountInString(c.Content) == 0 {
+			t.Errorf("chunk %d is empty", i)
+		}
+	}
+}
+
+func TestSplitText_MetadataInitialized(t *testing.T) {
+	chunks := SplitText("Some text here.", DefaultChunkConfig())
+	if len(chunks) != 1 {
+		t.Fatal("expected 1 chunk")
+	}
+	if chunks[0].Metadata == nil {
+		t.Error("metadata should be initialized, got nil")
+	}
+}
+
+func TestEstimateTokens(t *testing.T) {
+	cases := []struct {
+		input    string
+		minTok   int
+		maxTok   int
+	}{
+		{"", 0, 0},
+		{"hi", 1, 1},
+		{"hello world", 2, 4},
+		{strings.Repeat("a", 100), 20, 30},
+	}
+	for _, tc := range cases {
+		got := estimateTokens(tc.input)
+		if got < tc.minTok || got > tc.maxTok {
+			t.Errorf("estimateTokens(%q): got %d, want [%d, %d]", tc.input, got, tc.minTok, tc.maxTok)
+		}
+	}
+}
+
+func TestOverlapSuffix(t *testing.T) {
+	s := "the quick brown fox jumps over the lazy dog"
+
+	// Should return last ~20 chars, breaking at word boundary.
+	suffix := overlapSuffix(s, 20)
+	if len(suffix) == 0 {
+		t.Error("expected non-empty suffix")
+	}
+	if len(suffix) > 25 { // some slack for word boundary adjustment
+		t.Errorf("suffix too long: %d chars: %q", len(suffix), suffix)
+	}
+
+	// Short string: return whole thing.
+	short := "hello"
+	if got := overlapSuffix(short, 100); got != short {
+		t.Errorf("expected %q, got %q", short, got)
+	}
+}
--- a/server/knowledge/embedder.go
+++ b/server/knowledge/embedder.go
@@ -0,0 +1,184 @@
+package knowledge
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"git.gobha.me/xcaliber/chat-switchboard/models"
+	"git.gobha.me/xcaliber/chat-switchboard/roles"
+	"git.gobha.me/xcaliber/chat-switchboard/store"
+)
+
+// ── Configuration ────────────────────────────
+
+const (
+	// DefaultBatchSize is the max chunks per embedding API call.
+	// Most providers handle 100 inputs per request comfortably.
+	DefaultBatchSize = 100
+
+	// MaxVectorDim is the column width in pgvector (vector(3072)).
+	// Vectors shorter than this are zero-padded on storage.
+	MaxVectorDim = 3072
+)
+
+// ── Embedder ─────────────────────────────────
+
+// Embedder generates vector embeddings for text chunks using the
+// embedding role resolver. It handles batching and dimension
+// normalization (zero-padding to MaxVectorDim).
+type Embedder struct {
+	resolver  *roles.Resolver
+	stores    store.Stores // optional — for usage tracking
+	batchSize int
+}
+
+// NewEmbedder creates an embedder that dispatches through the role resolver.
+func NewEmbedder(resolver *roles.Resolver) *Embedder {
+	return &Embedder{
+		resolver:  resolver,
+		batchSize: DefaultBatchSize,
+	}
+}
+
+// WithStores attaches stores for usage tracking. Returns self for chaining.
+func (e *Embedder) WithStores(s store.Stores) *Embedder {
+	e.stores = s
+	return e
+}
+
+// EmbedResult holds the output of a batch embedding operation.
+type EmbedResult struct {
+	Vectors       [][]float64 // one per input chunk, zero-padded to MaxVectorDim
+	Model         string      // model that produced the embeddings
+	Dimensions    int         // native dimension before padding
+	InputTokens   int         // total tokens consumed across all batches
+	ConfigID      string      // provider config that was used
+	ProviderScope string      // "personal", "team", or "global"
+}
+
+// EmbedChunks generates embeddings for a slice of text chunks.
+// Chunks are batched in groups of batchSize to avoid provider limits.
+// Uses the embedding role resolution chain: personal → team → global.
+//
+// Returns vectors in the same order as input chunks.
+func (e *Embedder) EmbedChunks(ctx context.Context, userID string, teamID *string, texts []string) (*EmbedResult, error) {
+	if len(texts) == 0 {
+		return &EmbedResult{Vectors: [][]float64{}}, nil
+	}
+
+	// Verify embedding role is configured before starting
+	cfg, err := e.resolver.GetConfig(ctx, roles.RoleEmbedding, userID, teamID)
+	if err != nil {
+		return nil, fmt.Errorf("embedding role not configured: %w", err)
+	}
+	if cfg.Primary == nil {
+		return nil, fmt.Errorf("embedding role has no primary binding")
+	}
+
+	allVectors := make([][]float64, 0, len(texts))
+	var model string
+	var nativeDim int
+	var configID, provScope string
+	totalTokens := 0
+
+	for i := 0; i < len(texts); i += e.batchSize {
+		end := i + e.batchSize
+		if end > len(texts) {
+			end = len(texts)
+		}
+		batch := texts[i:end]
+
+		log.Printf("  embed batch %d/%d (%d chunks)",
+			(i/e.batchSize)+1, (len(texts)+e.batchSize-1)/e.batchSize, len(batch))
+
+		result, err := e.resolver.Embed(ctx, roles.RoleEmbedding, userID, teamID, batch)
+		if err != nil {
+			return nil, fmt.Errorf("embed batch starting at chunk %d: %w", i, err)
+		}
+
+		if len(result.Embeddings) != len(batch) {
+			return nil, fmt.Errorf("embed batch %d: expected %d vectors, got %d",
+				i/e.batchSize, len(batch), len(result.Embeddings))
+		}
+
+		// Capture model info from first batch
+		if model == "" {
+			model = result.Model
+			configID = result.ConfigID
+			provScope = result.ProviderScope
+			if len(result.Embeddings) > 0 {
+				nativeDim = len(result.Embeddings[0])
+			}
+		}
+
+		totalTokens += result.InputTokens
+		allVectors = append(allVectors, result.Embeddings...)
+	}
+
+	// Zero-pad to MaxVectorDim for uniform storage
+	for i, vec := range allVectors {
+		allVectors[i] = padVector(vec, MaxVectorDim)
+	}
+
+	return &EmbedResult{
+		Vectors:       allVectors,
+		Model:         model,
+		Dimensions:    nativeDim,
+		InputTokens:   totalTokens,
+		ConfigID:      configID,
+		ProviderScope: provScope,
+	}, nil
+}
+
+// LogUsage records an embedding usage entry. Silently no-ops if stores
+// were not attached via WithStores or if there are no tokens to log.
+func (e *Embedder) LogUsage(ctx context.Context, userID string, channelID *string, result *EmbedResult) {
+	if e.stores.Usage == nil || result == nil || result.InputTokens == 0 {
+		return
+	}
+
+	role := "embedding"
+	entry := &models.UsageEntry{
+		ChannelID:        channelID,
+		UserID:           userID,
+		ProviderConfigID: strPtr(result.ConfigID),
+		ProviderScope:    result.ProviderScope,
+		ModelID:          result.Model,
+		Role:             &role,
+		PromptTokens:     result.InputTokens,
+		CompletionTokens: 0,
+	}
+
+	if err := e.stores.Usage.Log(ctx, entry); err != nil {
+		log.Printf("⚠ Failed to log embedding usage: %v", err)
+	}
+}
+
+// IsConfigured returns true if the embedding role has at least a primary binding.
+func (e *Embedder) IsConfigured(ctx context.Context) bool {
+	return e.resolver.IsConfigured(ctx, roles.RoleEmbedding)
+}
+
+// ── Helpers ──────────────────────────────────
+
+// padVector zero-pads a vector to the target dimension.
+// If the vector is already the target size or larger, it's truncated.
+func padVector(vec []float64, targetDim int) []float64 {
+	if len(vec) == targetDim {
+		return vec
+	}
+	if len(vec) > targetDim {
+		return vec[:targetDim]
+	}
+	padded := make([]float64, targetDim)
+	copy(padded, vec)
+	return padded
+}
+
+func strPtr(s string) *string {
+	if s == "" {
+		return nil
+	}
+	return &s
+}
--- a/server/knowledge/ingest.go
+++ b/server/knowledge/ingest.go
@@ -0,0 +1,258 @@
+package knowledge
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"strings"
+	"sync"
+	"time"
+
+	"git.gobha.me/xcaliber/chat-switchboard/models"
+	"git.gobha.me/xcaliber/chat-switchboard/storage"
+	"git.gobha.me/xcaliber/chat-switchboard/store"
+)
+
+// ── Configuration ────────────────────────────
+
+const (
+	// DefaultConcurrency limits parallel ingestion goroutines.
+	DefaultConcurrency = 3
+
+	// MaxTextLength is the upper bound for extracted text (10 MB).
+	// Documents larger than this are truncated with a warning.
+	MaxTextLength = 10 * 1024 * 1024
+
+	// contextTimeout for the entire ingestion of one document.
+	ingestTimeout = 10 * time.Minute
+)
+
+// textMIMETypes are content types we can extract text from directly.
+var textMIMETypes = map[string]bool{
+	"text/plain":    true,
+	"text/markdown": true,
+	"text/csv":      true,
+	"text/html":     true,
+}
+
+// ── Ingester ─────────────────────────────────
+
+// Ingester manages the document ingestion pipeline.
+// It runs chunk + embed as background goroutines with a concurrency
+// semaphore to avoid overwhelming the embedding provider.
+type Ingester struct {
+	stores   store.Stores
+	embedder *Embedder
+	objStore storage.ObjectStore
+	sem      chan struct{}
+	wg       sync.WaitGroup
+}
+
+// NewIngester creates an ingestion pipeline.
+func NewIngester(stores store.Stores, embedder *Embedder, objStore storage.ObjectStore, concurrency int) *Ingester {
+	if concurrency <= 0 {
+		concurrency = DefaultConcurrency
+	}
+	return &Ingester{
+		stores:   stores,
+		embedder: embedder,
+		objStore: objStore,
+		sem:      make(chan struct{}, concurrency),
+	}
+}
+
+// IngestDocument starts async ingestion for a document.
+// The document must already exist in kb_documents with status "pending"
+// and the file must be stored at doc.StorageKey in the object store.
+//
+// The goroutine progresses through: pending → extracting → chunking →
+// embedding → ready (or → error on failure).
+func (ing *Ingester) IngestDocument(kb *models.KnowledgeBase, doc *models.KBDocument, userID string, teamID *string) {
+	ing.wg.Add(1)
+	go func() {
+		defer ing.wg.Done()
+
+		// Acquire semaphore slot
+		ing.sem <- struct{}{}
+		defer func() { <-ing.sem }()
+
+		ctx, cancel := context.WithTimeout(context.Background(), ingestTimeout)
+		defer cancel()
+
+		if err := ing.ingest(ctx, kb, doc, userID, teamID); err != nil {
+			errMsg := err.Error()
+			log.Printf("❌ ingest doc %s (%s): %v", doc.ID, doc.Filename, err)
+			ing.stores.KnowledgeBases.UpdateDocumentStatus(ctx, doc.ID, "error", &errMsg)
+			ing.stores.KnowledgeBases.Update(ctx, kb.ID, map[string]interface{}{"status": "active"})
+		}
+	}()
+}
+
+// Wait blocks until all in-flight ingestion goroutines complete.
+// Call during graceful shutdown.
+func (ing *Ingester) Wait() {
+	ing.wg.Wait()
+}
+
+// ── Internal Pipeline ────────────────────────
+
+func (ing *Ingester) ingest(ctx context.Context, kb *models.KnowledgeBase, doc *models.KBDocument, userID string, teamID *string) error {
+	start := time.Now()
+	log.Printf("▶ ingest %s (%s, %d bytes)", doc.Filename, doc.ContentType, doc.SizeBytes)
+
+	// Mark KB as processing
+	ing.stores.KnowledgeBases.Update(ctx, kb.ID, map[string]interface{}{"status": "processing"})
+
+	// ── Step 1: Extract text ────────────────
+	ing.stores.KnowledgeBases.UpdateDocumentStatus(ctx, doc.ID, "extracting", nil)
+
+	text, err := ing.extractText(ctx, doc)
+	if err != nil {
+		return fmt.Errorf("extract: %w", err)
+	}
+
+	if strings.TrimSpace(text) == "" {
+		return fmt.Errorf("extract: no text content found in %s", doc.Filename)
+	}
+
+	// Truncate very large documents
+	if len(text) > MaxTextLength {
+		log.Printf("  ⚠ truncating %s from %d to %d chars", doc.Filename, len(text), MaxTextLength)
+		text = text[:MaxTextLength]
+	}
+
+	// Store extracted text for potential re-chunking later
+	if err := ing.stores.KnowledgeBases.UpdateDocumentText(ctx, doc.ID, text, 0); err != nil {
+		log.Printf("  ⚠ failed to store extracted text for %s: %v", doc.ID, err)
+	}
+
+	// ── Step 2: Chunk ───────────────────────
+	ing.stores.KnowledgeBases.UpdateDocumentStatus(ctx, doc.ID, "chunking", nil)
+
+	cfg := DefaultChunkConfig()
+	chunks := SplitText(text, cfg)
+	if len(chunks) == 0 {
+		return fmt.Errorf("chunk: produced zero chunks from %s", doc.Filename)
+	}
+
+	log.Printf("  ✓ chunked %s → %d chunks", doc.Filename, len(chunks))
+
+	// ── Step 3: Embed ───────────────────────
+	ing.stores.KnowledgeBases.UpdateDocumentStatus(ctx, doc.ID, "embedding", nil)
+
+	texts := make([]string, len(chunks))
+	for i, c := range chunks {
+		texts[i] = c.Content
+	}
+
+	embedResult, err := ing.embedder.EmbedChunks(ctx, userID, teamID, texts)
+	if err != nil {
+		return fmt.Errorf("embed: %w", err)
+	}
+
+	log.Printf("  ✓ embedded %d chunks (model=%s, dim=%d, tokens=%d)",
+		len(chunks), embedResult.Model, embedResult.Dimensions, embedResult.InputTokens)
+
+	// Track embedding usage (role = "embedding")
+	ing.embedder.LogUsage(ctx, userID, nil, embedResult)
+
+	// ── Step 4: Store chunks with vectors ───
+	dbChunks := make([]models.KBChunk, len(chunks))
+	for i, c := range chunks {
+		dbChunks[i] = models.KBChunk{
+			KBID:       kb.ID,
+			DocumentID: doc.ID,
+			ChunkIndex: c.Index,
+			Content:    c.Content,
+			TokenCount: c.TokenCount,
+			Embedding:  embedResult.Vectors[i],
+			Metadata: models.JSONMap{
+				"char_start": i * (cfg.ChunkSize - cfg.ChunkOverlap), // approximate
+			},
+		}
+	}
+
+	if err := ing.stores.KnowledgeBases.InsertChunks(ctx, dbChunks); err != nil {
+		return fmt.Errorf("store chunks: %w", err)
+	}
+
+	// ── Step 5: Update stats ────────────────
+	// Update document status + chunk count
+	if err := ing.stores.KnowledgeBases.UpdateDocumentText(ctx, doc.ID, text, len(chunks)); err != nil {
+		log.Printf("  ⚠ failed to update chunk count for %s: %v", doc.ID, err)
+	}
+	ing.stores.KnowledgeBases.UpdateDocumentStatus(ctx, doc.ID, "ready", nil)
+
+	// Update KB-level stats
+	if err := ing.stores.KnowledgeBases.UpdateStats(ctx, kb.ID); err != nil {
+		log.Printf("  ⚠ failed to update KB stats: %v", err)
+	}
+
+	// Store embedding model info in KB config if not set
+	if kb.EmbeddingConfig == nil || kb.EmbeddingConfig["model"] == nil {
+		ing.stores.KnowledgeBases.Update(ctx, kb.ID, map[string]interface{}{
+			"embedding_config": models.JSONMap{
+				"model":      embedResult.Model,
+				"dimensions": embedResult.Dimensions,
+			},
+		})
+	}
+
+	// Mark KB as active
+	ing.stores.KnowledgeBases.Update(ctx, kb.ID, map[string]interface{}{"status": "active"})
+
+	log.Printf("  ✓ ingest complete: %s (%d chunks, %s)",
+		doc.Filename, len(chunks), time.Since(start).Round(time.Millisecond))
+
+	return nil
+}
+
+// ── Text Extraction ──────────────────────────
+
+// extractText reads the document file and returns its text content.
+// For text-based files, reads directly from object store.
+// For complex formats (PDF, docx), returns an error — the extraction
+// sidecar integration is planned for a future phase.
+func (ing *Ingester) extractText(ctx context.Context, doc *models.KBDocument) (string, error) {
+	// If extracted text was already stored (e.g. from a rebuild), use it
+	if doc.ExtractedText != nil && *doc.ExtractedText != "" {
+		return *doc.ExtractedText, nil
+	}
+
+	// Text-based files: read directly
+	if textMIMETypes[doc.ContentType] {
+		return ing.readTextFile(ctx, doc.StorageKey)
+	}
+
+	// Complex document types are not yet supported for inline extraction.
+	// The extraction sidecar (v0.12.0) handles attachments but isn't yet
+	// wired into the KB pipeline. Planned for a future phase.
+	return "", fmt.Errorf("unsupported content type %q — text-based files only for now (txt, md, csv, html)", doc.ContentType)
+}
+
+// readTextFile reads a text file from the object store and returns its contents.
+func (ing *Ingester) readTextFile(ctx context.Context, storageKey string) (string, error) {
+	rc, size, _, err := ing.objStore.Get(ctx, storageKey)
+	if err != nil {
+		return "", fmt.Errorf("read %s: %w", storageKey, err)
+	}
+	defer rc.Close()
+
+	// Guard against absurdly large files
+	if size > MaxTextLength+1024 {
+		// Read up to limit
+		limited := io.LimitReader(rc, MaxTextLength)
+		data, err := io.ReadAll(limited)
+		if err != nil {
+			return "", fmt.Errorf("read %s: %w", storageKey, err)
+		}
+		return string(data), nil
+	}
+
+	data, err := io.ReadAll(rc)
+	if err != nil {
+		return "", fmt.Errorf("read %s: %w", storageKey, err)
+	}
+	return string(data), nil
+}