From c5159538ce7a0f163b811e49b81bd842ef490fa7 Mon Sep 17 00:00:00 2001 From: xcaliber Date: Sun, 1 Mar 2026 17:45:50 +0000 Subject: [PATCH] Changeset 0.21.2 (#88) --- CHANGELOG.md | 35 ++ VERSION | 2 +- docs/ROADMAP.md | 40 +- server/config/config.go | 19 + .../migrations/011_v0212_workspace_chunks.sql | 58 +++ .../sqlite/010_v0212_workspace_chunks.sql | 31 ++ server/handlers/workspaces.go | 40 ++ server/main.go | 20 + server/models/models_workspace.go | 45 ++- server/store/interfaces.go | 6 + server/store/postgres/workspace.go | 101 ++++- server/store/sqlite/workspace.go | 157 +++++++- server/tools/workspace.go | 8 + server/tools/workspace_search.go | 150 ++++++++ server/workspace/archive.go | 20 +- server/workspace/fs.go | 36 +- server/workspace/fs_test.go | 14 + server/workspace/indexer.go | 355 ++++++++++++++++++ 18 files changed, 1105 insertions(+), 32 deletions(-) create mode 100644 server/database/migrations/011_v0212_workspace_chunks.sql create mode 100644 server/database/migrations/sqlite/010_v0212_workspace_chunks.sql create mode 100644 server/tools/workspace_search.go create mode 100644 server/workspace/indexer.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bfc79f..9682786 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,41 @@ All notable changes to Chat Switchboard. +## [0.21.2] — 2026-03-01 + +### Added +- **Workspace Indexing Pipeline.** Background text chunking and embedding for workspace files, reusing the existing `knowledge.SplitText` chunker and `knowledge.Embedder` for vector generation. Indexable file types include 40+ source code extensions (Go, Rust, Python, TypeScript, Perl, etc.) plus all `text/*` MIME types. Code-aware chunking uses 1500-character chunks with function/class/type boundary separators (`\n\nfunc `, `\n\nfn `, `\n\nclass `, `\n\ndef `, `\n\ntype `, `\n\nsub `, `\n\npub `, `\n\nimpl `), falling back to paragraph and line breaks. +- **Content-addressed skip.** SHA256 computed during `WriteFile` is compared against the prior hash before triggering re-indexing. Unchanged files skip the chunk + embed pipeline entirely, making archive extracts and bulk writes efficient. +- **`workspace_chunks` table.** New table (Postgres + SQLite migrations) storing text chunks with vector embeddings per workspace file. Postgres uses sequential scan with pgvector `<=>` cosine operator (no IVFFlat/HNSW index — 3072-dim embeddings exceed the 2000-dim index limit; sequential scan is adequate at workspace scale). SQLite uses app-level cosine in Go. Columns: workspace_id, file_id, chunk_index, content, token_count, embedding, metadata (JSONB with line_start, language). +- **`workspace_files` additions.** `index_status` column (`pending`, `indexing`, `ready`, `error`, `skipped`) and `chunk_count` for tracking indexing progress per file. `workspaces` table gains `indexing_enabled` boolean (default true). +- **`WorkspaceStore` chunk methods.** Four new store interface methods implemented for both Postgres and SQLite: `InsertChunks` (batch insert with pgvector cast or JSON text), `DeleteChunksByFile`, `SimilaritySearch` (pgvector `<=>` operator or app-level cosine), `UpdateFileIndexStatus`. +- **`workspace.Indexer`.** Background indexing orchestrator with configurable concurrency semaphore. Single-file indexing (`IndexFile`) triggered on `WriteFile` when content changes. Batch indexing (`IndexBatch`) triggered after `ExtractArchive`. Async goroutines with 5-minute timeout per file. Embedding usage tracked via `Embedder.LogUsage`. +- **`workspace_search` tool.** Semantic search across workspace files via natural language query. Embeds the query, runs cosine similarity against workspace chunks, returns file paths, content snippets, relevance scores, and approximate line numbers. Optional `file_pattern` glob filtering (e.g. `*.go`, `src/*.ts`) applied post-search. Top-K capped at 20. +- **Indexer integration.** `workspace.FS.SetIndexer()` hooks the indexer into the write path. `WriteFile` triggers single-file indexing on content change. `ExtractArchive` triggers batch indexing for all extracted files. Both use the workspace owner's identity for embedding provider resolution. +- **Configuration.** `WORKSPACE_INDEXING_ENABLED` env var (global kill switch, default true). `WORKSPACE_INDEX_CONCURRENCY` env var (default 2). Per-workspace `indexing_enabled` toggle via `WorkspacePatch`. Indexing gracefully degrades when no embedding role is configured. +- **Index status endpoint.** `GET /api/v1/workspaces/:id/index-status` returns aggregated indexing progress: per-status file counts (pending, indexing, ready, error, skipped), total chunk count, and workspace indexing_enabled flag. + +### Changed +- `workspace_files` scanner updated in both Postgres and SQLite stores to include `index_status` and `chunk_count` columns. +- `workspaces` scanner updated to include `indexing_enabled` column. +- `WorkspacePatch` model extended with `IndexingEnabled` field. +- `WorkspaceToolNames()` now includes `workspace_search` in the disabled set when no workspace is bound. +- `RegisterWorkspaceSearchTool()` added for late registration of the search tool (requires embedder). + +## [0.21.1] — 2026-03-01 + +### Added +- **Workspace Tools.** Six AI-callable tools in the `workspace` category: `workspace_ls` (list files with content type/size), `workspace_read` (text content with 50KB soft limit, binary detection), `workspace_write` (create/overwrite with auto-mkdir), `workspace_rm` (delete with recursive guard), `workspace_mv` (move/rename with auto-mkdir), `workspace_patch` (find/replace operations, each match must be unique, 1MB file limit). Late registration via `RegisterWorkspaceTools()` after stores + FS init. `WorkspaceToolNames()` returns the full list for filtering. +- **Channel Workspace Binding.** `channels.workspace_id` nullable FK column (Postgres + SQLite migration 010/009). Channels with a bound workspace get workspace tools auto-injected into the completion tool set. +- **Project Workspace Binding.** `projects.workspace_id` nullable FK column. Projects can bind a workspace shared by all channels in the project. +- **Workspace Resolution Chain.** `ChannelStore.ResolveWorkspaceID()` resolves the effective workspace for a channel: channel `workspace_id` takes priority, falling back to the project's `workspace_id` via `project_channels` join. Both Postgres and SQLite implementations. Returns empty string when no workspace is bound. +- **Tool Injection in Completion Handler.** `buildToolDefs()` accepts `workspaceID` parameter; when empty, all workspace tool names are added to the disabled set. `ExecutionContext.WorkspaceID` field carries the resolved workspace ID through tool execution and streaming. +- **Workspace resolution wired into streaming.** Both `CompletionHandler` and `streamWithToolLoop` resolve the workspace ID from the channel and pass it through to tool execution contexts. + +### Deferred +- Frontend workspace UI (channel settings toggle, project Files tab, chat bar workspace indicator) deferred to a future release. +- Archive-to-workspace upload flow (detect archive MIME + workspace binding → extract to workspace) deferred. + ## [0.21.0] — 2026-03-01 ### Added diff --git a/VERSION b/VERSION index 8854156..2d62101 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.21.0 +0.21.2 \ No newline at end of file diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 8f460fe..73c7cdc 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -82,10 +82,10 @@ v0.21.0 Workspace Storage Primitive ✅ │ ┌───────┴──────────────┐ │ │ -v0.21.1 Workspace v0.21.3 Surface Infra +v0.21.1 Workspace ✅ v0.21.3 Surface Infra Tools + Bindings + REPL (parallel) │ │ -v0.21.2 Workspace v0.21.5 Editor Surface +v0.21.2 Workspace ✅ v0.21.5 Editor Surface Indexing + Search (Development Mode) │ │ v0.21.4 Git v0.21.6 Article Surface @@ -694,27 +694,35 @@ filesystem (source of truth) + DB metadata index (queryable cache). - [x] Owner-based authorization: user=self, channel=owner, project=member, team=member - [x] Unit tests: path cleaning, traversal detection, MIME detection, unsafe path filtering, write/read round-trip, delete, mkdir -### v0.21.1 — Workspace Tools + Channel/Project Binding +### v0.21.1 — Workspace Tools + Channel/Project Binding ✅ Make workspaces useful in chat mode via tool calls. -- [ ] Workspace tools (category: workspace): `workspace_ls`, `workspace_read` (50KB soft limit), `workspace_write` (auto-mkdir), `workspace_rm` (recursive guard), `workspace_mv`, `workspace_patch` (find/replace) -- [ ] `channels.workspace_id` FK, `projects.workspace_id` FK -- [ ] Workspace resolution: channel workspace → project workspace (override pattern) -- [ ] Archive upload flow: detect archive MIME + workspace binding → extract to workspace -- [ ] Frontend: channel settings workspace toggle, project Files tab with tree view, chat bar workspace indicator +- [x] Workspace tools (category: workspace): `workspace_ls`, `workspace_read` (50KB soft limit), `workspace_write` (auto-mkdir), `workspace_rm` (recursive guard), `workspace_mv`, `workspace_patch` (find/replace) +- [x] `channels.workspace_id` FK, `projects.workspace_id` FK +- [x] Workspace resolution: channel workspace → project workspace (override pattern) +- [x] Tool injection in completion handler when workspace bound +- [x] Frontend: deferred (channel settings workspace toggle, project Files tab, chat bar workspace indicator) -### v0.21.2 — Workspace Indexing + Semantic Search +### v0.21.2 — Workspace Indexing + Semantic Search ✅ Embed text files for semantic search. Reuses `knowledge.SplitText` + `knowledge.Embedder`. -- [ ] `workspace_chunks` table: workspace_id, file_id, chunk_index, content, embedding, metadata -- [ ] `workspace_files` additions: index_status, chunk_count -- [ ] Indexable types: text MIME types + 30+ code extensions -- [ ] Code-aware chunking: larger chunks (1500 chars), function/class boundary separators -- [ ] Content-addressed skip: sha256 check before re-chunking/re-embedding -- [ ] Background indexing: async goroutine with shared semaphore -- [ ] `workspace_search` tool: semantic search with optional glob filtering +- [x] `workspace_chunks` table (Postgres + SQLite migrations) +- [x] `workspace_files` additions: `index_status`, `chunk_count` columns +- [x] `workspaces.indexing_enabled` boolean (default true) +- [x] Store methods: `InsertChunks`, `DeleteChunksByFile`, `SimilaritySearch`, `UpdateFileIndexStatus` +- [x] Indexable types: text MIME types + 40+ code extensions + special filenames (Makefile, Dockerfile, etc.) +- [x] Code-aware chunking: 1500-char chunks with function/class/type/sub boundary separators +- [x] Content-addressed skip: sha256 comparison before re-chunking/re-embedding +- [x] `workspace.Indexer`: background goroutine with configurable concurrency semaphore +- [x] Single-file indexing via `FS.WriteFile` hook (`SetIndexer`) +- [x] Batch indexing via `ExtractArchive` (sequential within one semaphore slot) +- [x] `workspace_search` tool: semantic search with optional glob filtering, top-K capped at 20 +- [x] `GET /api/v1/workspaces/:id/index-status` endpoint (per-status file counts, total chunks) +- [x] Configuration: `WORKSPACE_INDEXING_ENABLED`, `WORKSPACE_INDEX_CONCURRENCY` env vars +- [x] Graceful degradation when no embedding role configured +- [x] Mock store updated for FS unit tests ### v0.21.3 — Surface Infrastructure + REPL diff --git a/server/config/config.go b/server/config/config.go index 28339dd..c6623fd 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -51,6 +51,12 @@ type Config struct { // EXTRACTION_CONCURRENCY: max concurrent extraction jobs (default 3). ExtractionMode string ExtractionConcurrency int + + // Workspace indexing (v0.21.2) + // WORKSPACE_INDEXING_ENABLED: global kill switch for workspace file indexing (default true). + // WORKSPACE_INDEX_CONCURRENCY: max concurrent indexing goroutines (default 2). + WorkspaceIndexingEnabled bool + WorkspaceIndexConcurrency int } // Load reads configuration from environment variables. @@ -83,6 +89,9 @@ func Load() *Config { S3ForcePathStyle: getEnv("S3_FORCE_PATH_STYLE", "true") == "true", ExtractionMode: getEnv("EXTRACTION_MODE", "inline"), ExtractionConcurrency: getEnvInt("EXTRACTION_CONCURRENCY", 3), + + WorkspaceIndexingEnabled: getEnvBool("WORKSPACE_INDEXING_ENABLED", true), + WorkspaceIndexConcurrency: getEnvInt("WORKSPACE_INDEX_CONCURRENCY", 2), } } @@ -143,3 +152,13 @@ func getEnvInt(key string, fallback int) int { } return fallback } + +func getEnvBool(key string, fallback bool) bool { + if v := os.Getenv(key); v != "" { + b, err := strconv.ParseBool(v) + if err == nil { + return b + } + } + return fallback +} diff --git a/server/database/migrations/011_v0212_workspace_chunks.sql b/server/database/migrations/011_v0212_workspace_chunks.sql new file mode 100644 index 0000000..a8361c0 --- /dev/null +++ b/server/database/migrations/011_v0212_workspace_chunks.sql @@ -0,0 +1,58 @@ +-- v0.21.2: Workspace indexing + semantic search +-- +-- New: workspace_chunks table for text embeddings per workspace file +-- Alter: workspace_files gains index_status and chunk_count columns +-- Alter: workspaces gains indexing_enabled column + +-- ========================================= +-- 1. workspace_files additions +-- ========================================= + +ALTER TABLE workspace_files + ADD COLUMN IF NOT EXISTS index_status VARCHAR(20) DEFAULT 'pending' + CHECK (index_status IN ('pending', 'indexing', 'ready', 'error', 'skipped')); + +ALTER TABLE workspace_files + ADD COLUMN IF NOT EXISTS chunk_count INT NOT NULL DEFAULT 0; + +CREATE INDEX IF NOT EXISTS idx_workspace_files_index_status + ON workspace_files(workspace_id, index_status) + WHERE index_status IN ('pending', 'indexing'); + +-- ========================================= +-- 2. workspaces addition +-- ========================================= + +ALTER TABLE workspaces + ADD COLUMN IF NOT EXISTS indexing_enabled BOOLEAN NOT NULL DEFAULT true; + +-- ========================================= +-- 3. workspace_chunks table +-- ========================================= + +CREATE TABLE IF NOT EXISTS workspace_chunks ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + file_id UUID NOT NULL REFERENCES workspace_files(id) ON DELETE CASCADE, + chunk_index INT NOT NULL, + content TEXT NOT NULL, + token_count INT NOT NULL DEFAULT 0, + embedding vector(3072), + metadata JSONB DEFAULT '{}', + created_at TIMESTAMPTZ DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_workspace_chunks_file + ON workspace_chunks(file_id); + +CREATE INDEX IF NOT EXISTS idx_workspace_chunks_workspace + ON workspace_chunks(workspace_id); + +-- Note: No vector index created here. pgvector IVFFlat/HNSW indexes are +-- limited to 2000 dimensions, but our embeddings are 3072-dim (zero-padded). +-- Sequential scan with <=> is adequate at workspace scale (thousands of +-- chunks, not millions). Add a partial/quantized index later if needed. + +COMMENT ON TABLE workspace_chunks IS 'Text chunks with vector embeddings for workspace file semantic search'; +COMMENT ON COLUMN workspace_chunks.embedding IS 'Vector embedding (3072-dim, zero-padded) for cosine similarity'; +COMMENT ON COLUMN workspace_chunks.metadata IS 'Extensible: line_start, heading, language, etc.'; diff --git a/server/database/migrations/sqlite/010_v0212_workspace_chunks.sql b/server/database/migrations/sqlite/010_v0212_workspace_chunks.sql new file mode 100644 index 0000000..ba83d5b --- /dev/null +++ b/server/database/migrations/sqlite/010_v0212_workspace_chunks.sql @@ -0,0 +1,31 @@ +-- v0.21.2: Workspace indexing + semantic search (SQLite) + +-- workspace_files additions +ALTER TABLE workspace_files ADD COLUMN index_status TEXT DEFAULT 'pending'; +ALTER TABLE workspace_files ADD COLUMN chunk_count INTEGER NOT NULL DEFAULT 0; + +CREATE INDEX IF NOT EXISTS idx_workspace_files_index_status + ON workspace_files(workspace_id, index_status); + +-- workspaces addition +ALTER TABLE workspaces ADD COLUMN indexing_enabled INTEGER NOT NULL DEFAULT 1; + +-- workspace_chunks table +-- Embeddings stored as JSON text for app-level cosine similarity. +CREATE TABLE IF NOT EXISTS workspace_chunks ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + file_id TEXT NOT NULL REFERENCES workspace_files(id) ON DELETE CASCADE, + chunk_index INTEGER NOT NULL, + content TEXT NOT NULL, + token_count INTEGER NOT NULL DEFAULT 0, + embedding TEXT, -- JSON array of floats + metadata TEXT DEFAULT '{}', -- JSON object + created_at TEXT DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_workspace_chunks_file + ON workspace_chunks(file_id); + +CREATE INDEX IF NOT EXISTS idx_workspace_chunks_workspace + ON workspace_chunks(workspace_id); diff --git a/server/handlers/workspaces.go b/server/handlers/workspaces.go index f0d7a5a..7ad36ac 100644 --- a/server/handlers/workspaces.go +++ b/server/handlers/workspaces.go @@ -394,6 +394,46 @@ func (h *WorkspaceHandler) Stats(c *gin.Context) { c.JSON(http.StatusOK, stats) } +// IndexStatus returns indexing progress for all files in the workspace (v0.21.2). +func (h *WorkspaceHandler) IndexStatus(c *gin.Context) { + w, ok := h.loadAndAuthorize(c) + if !ok { + return + } + + ctx := c.Request.Context() + files, err := h.stores.Workspaces.ListFiles(ctx, w.ID, "", true) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list files"}) + return + } + + // Aggregate counts by status + counts := map[string]int{ + "pending": 0, "indexing": 0, "ready": 0, + "error": 0, "skipped": 0, + } + totalChunks := 0 + for _, f := range files { + if f.IsDirectory { + continue + } + status := f.IndexStatus + if status == "" { + status = "pending" + } + counts[status]++ + totalChunks += f.ChunkCount + } + + c.JSON(http.StatusOK, gin.H{ + "workspace_id": w.ID, + "indexing_enabled": w.IndexingEnabled, + "status_counts": counts, + "total_chunks": totalChunks, + }) +} + // ── Authorization Helpers ─────────────────── // loadAndAuthorize loads a workspace by :id param and checks user access. diff --git a/server/main.go b/server/main.go index 9d1db31..d10813e 100644 --- a/server/main.go +++ b/server/main.go @@ -186,6 +186,25 @@ func main() { // Register note tools (late registration — needs stores + embedder for semantic search) tools.RegisterNoteTools(stores, kbEmbedder) + // ── Workspace Indexer (v0.21.2) ─────────── + // Background indexing pipeline for workspace files. + // Shares the embedder with KB ingestion; uses its own concurrency semaphore. + var wsIndexer *workspace.Indexer + if wfs != nil { + idxSem := make(chan struct{}, cfg.WorkspaceIndexConcurrency) + wsIndexer = workspace.NewIndexer(stores, kbEmbedder, wfs, idxSem, cfg.WorkspaceIndexingEnabled) + wfs.SetIndexer(wsIndexer) // hook into write path + defer wsIndexer.Wait() + if cfg.WorkspaceIndexingEnabled { + log.Printf(" 📑 Workspace indexer enabled (concurrency=%d)", cfg.WorkspaceIndexConcurrency) + } else { + log.Printf(" 📑 Workspace indexer disabled (WORKSPACE_INDEXING_ENABLED=false)") + } + } + + // Register workspace_search tool (needs embedder for query embedding) + tools.RegisterWorkspaceSearchTool(stores, kbEmbedder) + // Register context recall tools (v0.15.1) tools.RegisterAttachmentRecall(stores, objStore) tools.RegisterConversationSearch(stores) @@ -438,6 +457,7 @@ func main() { protected.GET("/workspaces/:id/archive/download", wsH.DownloadArchive) protected.POST("/workspaces/:id/reconcile", wsH.Reconcile) protected.GET("/workspaces/:id/stats", wsH.Stats) + protected.GET("/workspaces/:id/index-status", wsH.IndexStatus) } // Attachments (file upload/download) diff --git a/server/models/models_workspace.go b/server/models/models_workspace.go index 6937ac4..bf5cb43 100644 --- a/server/models/models_workspace.go +++ b/server/models/models_workspace.go @@ -31,6 +31,8 @@ type Workspace struct { MaxBytes *int64 `json:"max_bytes,omitempty" db:"max_bytes"` Status string `json:"status" db:"status"` + IndexingEnabled bool `json:"indexing_enabled" db:"indexing_enabled"` + // Computed fields (not DB columns) FileCount int `json:"file_count,omitempty"` TotalBytes int64 `json:"total_bytes,omitempty"` @@ -38,9 +40,10 @@ type Workspace struct { // WorkspacePatch holds optional fields for updating a workspace. type WorkspacePatch struct { - Name *string `json:"name,omitempty"` - MaxBytes *int64 `json:"max_bytes,omitempty"` - Status *string `json:"status,omitempty"` + Name *string `json:"name,omitempty"` + MaxBytes *int64 `json:"max_bytes,omitempty"` + Status *string `json:"status,omitempty"` + IndexingEnabled *bool `json:"indexing_enabled,omitempty"` } // WorkspaceFile represents metadata for a single file or directory in a workspace. @@ -52,6 +55,8 @@ type WorkspaceFile struct { ContentType string `json:"content_type,omitempty" db:"content_type"` SizeBytes int64 `json:"size_bytes" db:"size_bytes"` SHA256 string `json:"sha256,omitempty" db:"sha256"` + IndexStatus string `json:"index_status,omitempty" db:"index_status"` + ChunkCount int `json:"chunk_count,omitempty" db:"chunk_count"` CreatedAt time.Time `json:"created_at" db:"created_at"` UpdatedAt time.Time `json:"updated_at" db:"updated_at"` } @@ -64,3 +69,37 @@ type WorkspaceStats struct { TotalBytes int64 `json:"total_bytes"` MaxBytes *int64 `json:"max_bytes,omitempty"` } + +// ========================================= +// WORKSPACE CHUNKS (v0.21.2) +// ========================================= + +// Workspace file index status constants. +const ( + IndexStatusPending = "pending" + IndexStatusIndexing = "indexing" + IndexStatusReady = "ready" + IndexStatusError = "error" + IndexStatusSkipped = "skipped" +) + +// WorkspaceChunk is a text segment from a workspace file with its embedding vector. +type WorkspaceChunk struct { + ID string `json:"id" db:"id"` + WorkspaceID string `json:"workspace_id" db:"workspace_id"` + FileID string `json:"file_id" db:"file_id"` + ChunkIndex int `json:"chunk_index" db:"chunk_index"` + Content string `json:"content" db:"content"` + TokenCount int `json:"token_count" db:"token_count"` + Embedding []float64 `json:"-" db:"embedding"` // never serialized to API + Metadata JSONMap `json:"metadata" db:"metadata"` +} + +// WorkspaceChunkResult is a single result from workspace similarity search. +type WorkspaceChunkResult struct { + FilePath string `json:"file_path"` + Content string `json:"content"` + Score float64 `json:"score"` + LineHint int `json:"line_hint,omitempty"` + Metadata JSONMap `json:"metadata,omitempty"` +} diff --git a/server/store/interfaces.go b/server/store/interfaces.go index 968c4af..aea865a 100644 --- a/server/store/interfaces.go +++ b/server/store/interfaces.go @@ -539,6 +539,12 @@ type WorkspaceStore interface { // Stats GetStats(ctx context.Context, workspaceID string) (*models.WorkspaceStats, error) + + // Chunks (v0.21.2 — workspace indexing) + InsertChunks(ctx context.Context, chunks []models.WorkspaceChunk) error + DeleteChunksByFile(ctx context.Context, fileID string) error + SimilaritySearch(ctx context.Context, workspaceID string, queryVec []float64, threshold float64, limit int) ([]models.WorkspaceChunkResult, error) + UpdateFileIndexStatus(ctx context.Context, fileID, status string, chunkCount int) error } // ========================================= diff --git a/server/store/postgres/workspace.go b/server/store/postgres/workspace.go index 54cd671..3acff0e 100644 --- a/server/store/postgres/workspace.go +++ b/server/store/postgres/workspace.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "strings" "time" "git.gobha.me/xcaliber/chat-switchboard/models" @@ -15,8 +16,8 @@ func NewWorkspaceStore() *WorkspaceStore { return &WorkspaceStore{} } // ── columns ──────────────────────────────── -const workspaceCols = `id, owner_type, owner_id, name, root_path, max_bytes, status, created_at, updated_at` -const workspaceFileCols = `id, workspace_id, path, is_directory, content_type, size_bytes, sha256, created_at, updated_at` +const workspaceCols = `id, owner_type, owner_id, name, root_path, max_bytes, status, indexing_enabled, created_at, updated_at` +const workspaceFileCols = `id, workspace_id, path, is_directory, content_type, size_bytes, sha256, index_status, chunk_count, created_at, updated_at` // ── scanners ─────────────────────────────── @@ -25,7 +26,7 @@ func scanWorkspace(row interface{ Scan(dest ...interface{}) error }) (*models.Wo var maxBytes sql.NullInt64 err := row.Scan( &w.ID, &w.OwnerType, &w.OwnerID, &w.Name, - &w.RootPath, &maxBytes, &w.Status, + &w.RootPath, &maxBytes, &w.Status, &w.IndexingEnabled, &w.CreatedAt, &w.UpdatedAt, ) if err != nil { @@ -41,9 +42,11 @@ func scanWorkspaceFile(row interface{ Scan(dest ...interface{}) error }) (*model var f models.WorkspaceFile var contentType sql.NullString var sha256 sql.NullString + var indexStatus sql.NullString err := row.Scan( &f.ID, &f.WorkspaceID, &f.Path, &f.IsDirectory, &contentType, &f.SizeBytes, &sha256, + &indexStatus, &f.ChunkCount, &f.CreatedAt, &f.UpdatedAt, ) if err != nil { @@ -55,6 +58,9 @@ func scanWorkspaceFile(row interface{ Scan(dest ...interface{}) error }) (*model if sha256.Valid { f.SHA256 = sha256.String } + if indexStatus.Valid { + f.IndexStatus = indexStatus.String + } return &f, nil } @@ -97,6 +103,9 @@ func (s *WorkspaceStore) Update(ctx context.Context, id string, patch models.Wor if patch.Status != nil { b.Set("status", *patch.Status) } + if patch.IndexingEnabled != nil { + b.Set("indexing_enabled", *patch.IndexingEnabled) + } if !b.HasSets() { return nil } @@ -270,3 +279,89 @@ func (s *WorkspaceStore) GetStats(ctx context.Context, workspaceID string) (*mod return stats, nil } + +// ── Chunks (v0.21.2) ───────────────────────── + +func (s *WorkspaceStore) InsertChunks(ctx context.Context, chunks []models.WorkspaceChunk) error { + if len(chunks) == 0 { + return nil + } + + const cols = 8 // workspace_id, file_id, chunk_index, content, token_count, embedding, metadata, created_at + valueParts := make([]string, 0, len(chunks)) + args := make([]interface{}, 0, len(chunks)*cols) + + for i, c := range chunks { + base := i * cols + valueParts = append(valueParts, fmt.Sprintf( + "($%d, $%d, $%d, $%d, $%d, $%d::vector, $%d, NOW())", + base+1, base+2, base+3, base+4, base+5, base+6, base+7, + )) + args = append(args, c.WorkspaceID, c.FileID, c.ChunkIndex, + c.Content, c.TokenCount, vectorToString(c.Embedding), ToJSON(c.Metadata)) + } + + q := fmt.Sprintf(`INSERT INTO workspace_chunks + (workspace_id, file_id, chunk_index, content, token_count, embedding, metadata, created_at) + VALUES %s`, strings.Join(valueParts, ", ")) + + _, err := DB.ExecContext(ctx, q, args...) + return err +} + +func (s *WorkspaceStore) DeleteChunksByFile(ctx context.Context, fileID string) error { + _, err := DB.ExecContext(ctx, `DELETE FROM workspace_chunks WHERE file_id = $1`, fileID) + return err +} + +func (s *WorkspaceStore) SimilaritySearch(ctx context.Context, workspaceID string, queryVec []float64, threshold float64, limit int) ([]models.WorkspaceChunkResult, error) { + if len(queryVec) == 0 { + return nil, nil + } + if limit <= 0 { + limit = 10 + } + + rows, err := DB.QueryContext(ctx, ` + SELECT f.path, c.content, + 1 - (c.embedding <=> $1::vector) AS similarity, + c.metadata + FROM workspace_chunks c + JOIN workspace_files f ON c.file_id = f.id + WHERE c.workspace_id = $2 + AND 1 - (c.embedding <=> $1::vector) > $3 + ORDER BY c.embedding <=> $1::vector + LIMIT $4`, + vectorToString(queryVec), workspaceID, threshold, limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []models.WorkspaceChunkResult + for rows.Next() { + var r models.WorkspaceChunkResult + var metadataJSON []byte + if err := rows.Scan(&r.FilePath, &r.Content, &r.Score, &metadataJSON); err != nil { + return nil, err + } + ScanJSON(metadataJSON, &r.Metadata) + if r.Metadata != nil { + if lh, ok := r.Metadata["line_start"]; ok { + if n, ok := lh.(float64); ok { + r.LineHint = int(n) + } + } + } + results = append(results, r) + } + return results, rows.Err() +} + +func (s *WorkspaceStore) UpdateFileIndexStatus(ctx context.Context, fileID, status string, chunkCount int) error { + _, err := DB.ExecContext(ctx, + `UPDATE workspace_files SET index_status = $1, chunk_count = $2, updated_at = NOW() WHERE id = $3`, + status, chunkCount, fileID) + return err +} diff --git a/server/store/sqlite/workspace.go b/server/store/sqlite/workspace.go index e69a6fe..abad1e0 100644 --- a/server/store/sqlite/workspace.go +++ b/server/store/sqlite/workspace.go @@ -3,7 +3,11 @@ package sqlite import ( "context" "database/sql" + "encoding/json" "fmt" + "math" + "sort" + "strings" "time" "git.gobha.me/xcaliber/chat-switchboard/models" @@ -16,8 +20,8 @@ func NewWorkspaceStore() *WorkspaceStore { return &WorkspaceStore{} } // ── columns ──────────────────────────────── -const workspaceCols = `id, owner_type, owner_id, name, root_path, max_bytes, status, created_at, updated_at` -const workspaceFileCols = `id, workspace_id, path, is_directory, content_type, size_bytes, sha256, created_at, updated_at` +const workspaceCols = `id, owner_type, owner_id, name, root_path, max_bytes, status, indexing_enabled, created_at, updated_at` +const workspaceFileCols = `id, workspace_id, path, is_directory, content_type, size_bytes, sha256, index_status, chunk_count, created_at, updated_at` // ── scanners ─────────────────────────────── @@ -26,7 +30,7 @@ func scanWorkspace(row interface{ Scan(dest ...interface{}) error }) (*models.Wo var maxBytes sql.NullInt64 err := row.Scan( &w.ID, &w.OwnerType, &w.OwnerID, &w.Name, - &w.RootPath, &maxBytes, &w.Status, + &w.RootPath, &maxBytes, &w.Status, &w.IndexingEnabled, st(&w.CreatedAt), st(&w.UpdatedAt), ) if err != nil { @@ -42,9 +46,11 @@ func scanWorkspaceFile(row interface{ Scan(dest ...interface{}) error }) (*model var f models.WorkspaceFile var contentType sql.NullString var sha256 sql.NullString + var indexStatus sql.NullString err := row.Scan( &f.ID, &f.WorkspaceID, &f.Path, &f.IsDirectory, &contentType, &f.SizeBytes, &sha256, + &indexStatus, &f.ChunkCount, st(&f.CreatedAt), st(&f.UpdatedAt), ) if err != nil { @@ -56,6 +62,9 @@ func scanWorkspaceFile(row interface{ Scan(dest ...interface{}) error }) (*model if sha256.Valid { f.SHA256 = sha256.String } + if indexStatus.Valid { + f.IndexStatus = indexStatus.String + } return &f, nil } @@ -95,6 +104,9 @@ func (s *WorkspaceStore) Update(ctx context.Context, id string, patch models.Wor if patch.Status != nil { b.Set("status", *patch.Status) } + if patch.IndexingEnabled != nil { + b.Set("indexing_enabled", *patch.IndexingEnabled) + } if !b.HasSets() { return nil } @@ -272,3 +284,142 @@ func (s *WorkspaceStore) GetStats(ctx context.Context, workspaceID string) (*mod return stats, nil } + +// ── Chunks (v0.21.2) ───────────────────────── + +func (s *WorkspaceStore) InsertChunks(ctx context.Context, chunks []models.WorkspaceChunk) error { + if len(chunks) == 0 { + return nil + } + + valueParts := make([]string, 0, len(chunks)) + args := make([]interface{}, 0, len(chunks)*8) + + for _, c := range chunks { + c.ID = store.NewID() + valueParts = append(valueParts, "(?, ?, ?, ?, ?, ?, ?, ?)") + var embJSON interface{} + if len(c.Embedding) > 0 { + embJSON = ToJSON(c.Embedding) + } + args = append(args, c.ID, c.WorkspaceID, c.FileID, c.ChunkIndex, + c.Content, c.TokenCount, embJSON, ToJSON(c.Metadata)) + } + + q := fmt.Sprintf(`INSERT INTO workspace_chunks + (id, workspace_id, file_id, chunk_index, content, token_count, embedding, metadata) + VALUES %s`, strings.Join(valueParts, ", ")) + + _, err := DB.ExecContext(ctx, q, args...) + return err +} + +func (s *WorkspaceStore) DeleteChunksByFile(ctx context.Context, fileID string) error { + _, err := DB.ExecContext(ctx, `DELETE FROM workspace_chunks WHERE file_id = ?`, fileID) + return err +} + +// SimilaritySearch performs app-level cosine similarity for SQLite. +// Loads all chunks for the workspace, computes cosine distance in Go, +// and returns the top results above threshold. +func (s *WorkspaceStore) SimilaritySearch(ctx context.Context, workspaceID string, queryVec []float64, threshold float64, limit int) ([]models.WorkspaceChunkResult, error) { + if len(queryVec) == 0 { + return nil, nil + } + if limit <= 0 { + limit = 10 + } + + rows, err := DB.QueryContext(ctx, ` + SELECT f.path, c.content, c.embedding, c.metadata + FROM workspace_chunks c + JOIN workspace_files f ON c.file_id = f.id + WHERE c.workspace_id = ? + AND c.embedding IS NOT NULL`, + workspaceID) + if err != nil { + return nil, err + } + defer rows.Close() + + type candidate struct { + result models.WorkspaceChunkResult + similarity float64 + } + var candidates []candidate + + for rows.Next() { + var filePath, content, embJSON string + var metaJSON sql.NullString + if err := rows.Scan(&filePath, &content, &embJSON, &metaJSON); err != nil { + return nil, err + } + + var embedding []float64 + if err := json.Unmarshal([]byte(embJSON), &embedding); err != nil { + continue // skip malformed + } + + sim := wsCosineSimilarity(queryVec, embedding) + if sim > threshold { + r := models.WorkspaceChunkResult{ + FilePath: filePath, + Content: content, + Score: sim, + } + if metaJSON.Valid { + ScanJSON(metaJSON.String, &r.Metadata) + } + if r.Metadata != nil { + if lh, ok := r.Metadata["line_start"]; ok { + if n, ok := lh.(float64); ok { + r.LineHint = int(n) + } + } + } + candidates = append(candidates, candidate{result: r, similarity: sim}) + } + } + if err := rows.Err(); err != nil { + return nil, err + } + + sort.Slice(candidates, func(i, j int) bool { + return candidates[i].similarity > candidates[j].similarity + }) + + if len(candidates) > limit { + candidates = candidates[:limit] + } + + results := make([]models.WorkspaceChunkResult, len(candidates)) + for i, c := range candidates { + results[i] = c.result + } + return results, nil +} + +func (s *WorkspaceStore) UpdateFileIndexStatus(ctx context.Context, fileID, status string, chunkCount int) error { + _, err := DB.ExecContext(ctx, + `UPDATE workspace_files SET index_status = ?, chunk_count = ?, updated_at = datetime('now') WHERE id = ?`, + status, chunkCount, fileID) + return err +} + +// wsCosineSimilarity computes cosine similarity between two vectors. +// Named to avoid collision with the KB package-level function. +func wsCosineSimilarity(a, b []float64) float64 { + if len(a) != len(b) || len(a) == 0 { + return 0 + } + var dot, normA, normB float64 + for i := range a { + dot += a[i] * b[i] + normA += a[i] * a[i] + normB += b[i] * b[i] + } + if normA == 0 || normB == 0 { + return 0 + } + return dot / (math.Sqrt(normA) * math.Sqrt(normB)) +} \ No newline at end of file diff --git a/server/tools/workspace.go b/server/tools/workspace.go index 3fd1a12..e872e48 100644 --- a/server/tools/workspace.go +++ b/server/tools/workspace.go @@ -8,6 +8,7 @@ import ( "log" "strings" + "git.gobha.me/xcaliber/chat-switchboard/knowledge" "git.gobha.me/xcaliber/chat-switchboard/models" "git.gobha.me/xcaliber/chat-switchboard/store" "git.gobha.me/xcaliber/chat-switchboard/workspace" @@ -27,12 +28,19 @@ func RegisterWorkspaceTools(stores store.Stores, wfs *workspace.FS) { Register(&workspacePatchTool{stores: stores, wfs: wfs}) } +// RegisterWorkspaceSearchTool registers the semantic search tool separately +// because it requires the embedder (which may not be configured). +func RegisterWorkspaceSearchTool(stores store.Stores, embedder *knowledge.Embedder) { + Register(&workspaceSearchTool{stores: stores, embedder: embedder}) +} + // WorkspaceToolNames returns the names of all workspace tools. // Used by the completion handler to filter them out when no workspace is bound. func WorkspaceToolNames() []string { return []string{ "workspace_ls", "workspace_read", "workspace_write", "workspace_rm", "workspace_mv", "workspace_patch", + "workspace_search", } } diff --git a/server/tools/workspace_search.go b/server/tools/workspace_search.go new file mode 100644 index 0000000..0967070 --- /dev/null +++ b/server/tools/workspace_search.go @@ -0,0 +1,150 @@ +package tools + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "strings" + + "git.gobha.me/xcaliber/chat-switchboard/knowledge" + "git.gobha.me/xcaliber/chat-switchboard/models" + "git.gobha.me/xcaliber/chat-switchboard/store" +) + +// ── workspace_search ──────────────────────── + +type workspaceSearchTool struct { + stores store.Stores + embedder *knowledge.Embedder +} + +func (t *workspaceSearchTool) Definition() ToolDef { + return ToolDef{ + Name: "workspace_search", + DisplayName: "Workspace Search", + Description: "Semantic search across workspace files. Finds code, documentation, and text content relevant to a natural language query. Returns matching file paths, content snippets, relevance scores, and approximate line numbers.", + Category: "workspace", + Parameters: JSONSchema(map[string]interface{}{ + "query": Prop("string", "Natural language search query (e.g. 'database connection pooling', 'error handling middleware')"), + "top_k": map[string]interface{}{ + "type": "integer", + "description": "Maximum number of results to return (default 10, max 20)", + "default": 10, + }, + "file_pattern": Prop("string", "Optional glob pattern to filter results by file path (e.g. '*.go', 'src/*.ts'). Applied post-search."), + }, []string{"query"}), + } +} + +func (t *workspaceSearchTool) Execute(ctx context.Context, execCtx ExecutionContext, argsJSON string) (string, error) { + var args struct { + Query string `json:"query"` + TopK int `json:"top_k"` + FilePattern string `json:"file_pattern"` + } + if err := json.Unmarshal([]byte(argsJSON), &args); err != nil { + return "", fmt.Errorf("invalid arguments: %w", err) + } + + if args.Query == "" { + return "", fmt.Errorf("query is required") + } + if args.TopK <= 0 { + args.TopK = 10 + } + if args.TopK > 20 { + args.TopK = 20 + } + + // Resolve workspace + w, err := loadWorkspace(ctx, t.stores, execCtx) + if err != nil { + return "", err + } + + if !w.IndexingEnabled { + return "", fmt.Errorf("workspace indexing is disabled for this workspace") + } + + // Embed the query + embedResult, err := t.embedder.EmbedChunks(ctx, execCtx.UserID, nil, []string{args.Query}) + if err != nil { + return "", fmt.Errorf("failed to embed query: %w", err) + } + if len(embedResult.Vectors) == 0 { + return "", fmt.Errorf("embedding returned no vectors") + } + + queryVec := embedResult.Vectors[0] + + // Fetch more if we'll post-filter by glob + searchLimit := args.TopK + if args.FilePattern != "" { + searchLimit = args.TopK * 3 + if searchLimit > 50 { + searchLimit = 50 + } + } + + results, err := t.stores.Workspaces.SimilaritySearch(ctx, w.ID, queryVec, 0.3, searchLimit) + if err != nil { + return "", fmt.Errorf("search failed: %w", err) + } + + // Apply glob filter if specified + if args.FilePattern != "" { + results = filterByGlob(results, args.FilePattern) + } + + // Cap at top_k + if len(results) > args.TopK { + results = results[:args.TopK] + } + + // Format response + formatted := make([]map[string]interface{}, len(results)) + for i, r := range results { + m := map[string]interface{}{ + "file_path": r.FilePath, + "content": r.Content, + "score": fmt.Sprintf("%.3f", r.Score), + } + if r.LineHint > 0 { + m["line_hint"] = r.LineHint + } + formatted[i] = m + } + + resp := map[string]interface{}{ + "query": args.Query, + "count": len(formatted), + "results": formatted, + } + if args.FilePattern != "" { + resp["file_pattern"] = args.FilePattern + } + if len(formatted) == 0 { + resp["message"] = "No matching results found. Try a different query or check that workspace files have been indexed." + } + + b, _ := json.Marshal(resp) + return string(b), nil +} + +// filterByGlob filters search results by a glob pattern. +func filterByGlob(results []models.WorkspaceChunkResult, pattern string) []models.WorkspaceChunkResult { + var filtered []models.WorkspaceChunkResult + for _, r := range results { + // Try matching against basename + matched, _ := filepath.Match(pattern, filepath.Base(r.FilePath)) + if !matched && strings.Contains(pattern, "/") { + // Try full path match if pattern contains directory separator + matched, _ = filepath.Match(pattern, r.FilePath) + } + if matched { + filtered = append(filtered, r) + } + } + return filtered +} diff --git a/server/workspace/archive.go b/server/workspace/archive.go index 8ac46f2..8d5a7f0 100644 --- a/server/workspace/archive.go +++ b/server/workspace/archive.go @@ -33,14 +33,30 @@ const ( // Returns the number of files extracted. Respects workspace quota. // format must be "zip" or "tar.gz". func (fs *FS) ExtractArchive(ctx context.Context, w *models.Workspace, archivePath, format string) (int, error) { + var count int + var err error switch format { case "zip": - return fs.extractZip(ctx, w, archivePath) + count, err = fs.extractZip(ctx, w, archivePath) case "tar.gz", "tgz": - return fs.extractTarGz(ctx, w, archivePath) + count, err = fs.extractTarGz(ctx, w, archivePath) default: return 0, fmt.Errorf("workspace: unsupported archive format: %s", format) } + + // Trigger batch indexing for all extracted files (v0.21.2) + if err == nil && fs.indexer != nil && count > 0 { + files, listErr := fs.store.ListFiles(ctx, w.ID, "", true) + if listErr == nil { + var teamID *string + if w.OwnerType == models.WorkspaceOwnerTeam { + teamID = &w.OwnerID + } + fs.indexer.IndexBatch(w, files, w.OwnerID, teamID) + } + } + + return count, err } func (fs *FS) extractZip(ctx context.Context, w *models.Workspace, archivePath string) (int, error) { diff --git a/server/workspace/fs.go b/server/workspace/fs.go index a3071c4..4d729c8 100644 --- a/server/workspace/fs.go +++ b/server/workspace/fs.go @@ -46,6 +46,7 @@ const ( type FS struct { basePath string // e.g. /data/storage/workspaces store store.WorkspaceStore + indexer *Indexer // optional — set via SetIndexer for v0.21.2 indexing } // NewFS creates a workspace filesystem manager. @@ -53,6 +54,12 @@ func NewFS(basePath string, ws store.WorkspaceStore) *FS { return &FS{basePath: basePath, store: ws} } +// SetIndexer attaches the workspace indexer for background file indexing. +// Must be called after NewIndexer is created (circular dependency break). +func (fs *FS) SetIndexer(idx *Indexer) { + fs.indexer = idx +} + // Init ensures the base workspaces directory exists. func (fs *FS) Init() error { return os.MkdirAll(fs.basePath, 0750) @@ -183,6 +190,12 @@ func (fs *FS) WriteFile(ctx context.Context, w *models.Workspace, relPath string return fmt.Errorf("workspace: refusing to overwrite symlink: %s", relPath) } + // Check existing sha256 for content-addressed skip (v0.21.2) + var oldSHA256 string + if existing, err := fs.store.GetFile(ctx, w.ID, relPath); err == nil { + oldSHA256 = existing.SHA256 + } + // Ensure parent directory exists dir := filepath.Dir(abs) if err := os.MkdirAll(dir, 0750); err != nil { @@ -218,16 +231,31 @@ func (fs *FS) WriteFile(ctx context.Context, w *models.Workspace, relPath string // Update DB index contentType := detectContentType(relPath, abs) - hash := hex.EncodeToString(h.Sum(nil)) + newHash := hex.EncodeToString(h.Sum(nil)) - return fs.store.UpsertFile(ctx, &models.WorkspaceFile{ + f := &models.WorkspaceFile{ WorkspaceID: w.ID, Path: relPath, IsDirectory: false, ContentType: contentType, SizeBytes: n, - SHA256: hash, - }) + SHA256: newHash, + } + if err := fs.store.UpsertFile(ctx, f); err != nil { + return err + } + + // Trigger async indexing if content changed (v0.21.2) + if fs.indexer != nil && newHash != oldSHA256 { + // Use workspace owner for embedding provider resolution + var teamID *string + if w.OwnerType == models.WorkspaceOwnerTeam { + teamID = &w.OwnerID + } + fs.indexer.IndexFile(w, f, w.OwnerID, teamID) + } + + return nil } // DeleteFile removes a file or empty directory at relPath. diff --git a/server/workspace/fs_test.go b/server/workspace/fs_test.go index a20468e..144d470 100644 --- a/server/workspace/fs_test.go +++ b/server/workspace/fs_test.go @@ -262,6 +262,20 @@ func (m *mockWorkspaceStore) DeleteAllFiles(ctx context.Context, workspaceID str return nil } +// Chunk methods (v0.21.2) — no-ops for FS tests +func (m *mockWorkspaceStore) InsertChunks(ctx context.Context, chunks []models.WorkspaceChunk) error { + return nil +} +func (m *mockWorkspaceStore) DeleteChunksByFile(ctx context.Context, fileID string) error { + return nil +} +func (m *mockWorkspaceStore) SimilaritySearch(ctx context.Context, workspaceID string, queryVec []float64, threshold float64, limit int) ([]models.WorkspaceChunkResult, error) { + return nil, nil +} +func (m *mockWorkspaceStore) UpdateFileIndexStatus(ctx context.Context, fileID, status string, chunkCount int) error { + return nil +} + func TestWriteReadRoundTrip(t *testing.T) { tmpDir := t.TempDir() mock := newMockStore() diff --git a/server/workspace/indexer.go b/server/workspace/indexer.go new file mode 100644 index 0000000..14d4142 --- /dev/null +++ b/server/workspace/indexer.go @@ -0,0 +1,355 @@ +package workspace + +import ( + "context" + "fmt" + "io" + "log" + "path/filepath" + "strings" + "sync" + "time" + + "git.gobha.me/xcaliber/chat-switchboard/knowledge" + "git.gobha.me/xcaliber/chat-switchboard/models" + "git.gobha.me/xcaliber/chat-switchboard/notifications" + "git.gobha.me/xcaliber/chat-switchboard/store" +) + +// ── Configuration ──────────────────────────── + +const ( + // indexTimeout is the max time for indexing a single file. + indexTimeout = 5 * time.Minute + + // maxIndexTextLength caps text extracted from a single file (5 MB). + maxIndexTextLength = 5 * 1024 * 1024 +) + +// indexableExtensions defines source code and text file extensions +// eligible for chunking and embedding. MIME detection covers text/*, +// but code files often have application/* types — this map catches them. +var indexableExtensions = map[string]bool{ + ".go": true, ".py": true, ".js": true, ".ts": true, ".jsx": true, + ".tsx": true, ".rs": true, ".c": true, ".cpp": true, ".h": true, + ".hpp": true, ".java": true, ".rb": true, ".php": true, ".swift": true, + ".kt": true, ".scala": true, ".sh": true, ".bash": true, ".zsh": true, + ".sql": true, ".yaml": true, ".yml": true, ".toml": true, ".json": true, + ".xml": true, ".css": true, ".scss": true, ".less": true, + ".md": true, ".txt": true, ".csv": true, ".html": true, ".htm": true, + ".dockerfile": true, ".tf": true, ".hcl": true, ".proto": true, + ".graphql": true, ".vue": true, ".svelte": true, + ".pl": true, ".pm": true, // Perl + ".lua": true, ".zig": true, ".nim": true, ".ex": true, ".exs": true, + // Config / data + ".ini": true, ".cfg": true, ".conf": true, ".env": true, + ".mk": true, ".cmake": true, ".makefile": true, + ".r": true, ".jl": true, // R, Julia +} + +// CodeChunkConfig returns chunking settings optimized for source code. +// Larger chunks (1500 chars) because functions can be long, with +// separators tuned to split on function/class/type boundaries. +func CodeChunkConfig() knowledge.ChunkConfig { + return knowledge.ChunkConfig{ + ChunkSize: 1500, + ChunkOverlap: 200, + Separators: []string{ + "\n\nfunc ", "\n\nfn ", // Go, Rust + "\n\nclass ", "\n\ndef ", // Python, Ruby + "\n\ntype ", "\n\nsub ", // Go types, Perl + "\n\npub ", "\n\nimpl ", // Rust + "\n\n", "\n", " ", // fallback hierarchy + }, + } +} + +// isIndexable returns true if the file is eligible for text indexing. +func isIndexable(path string, contentType string) bool { + // Check extension first (catches source code with non-text MIME types) + ext := strings.ToLower(filepath.Ext(path)) + if indexableExtensions[ext] { + return true + } + // Also index any text/* content type + if strings.HasPrefix(contentType, "text/") { + return true + } + // Specific filenames without extension + base := strings.ToLower(filepath.Base(path)) + switch base { + case "makefile", "dockerfile", "jenkinsfile", "gemfile", + "rakefile", "procfile", "vagrantfile", ".gitignore", + ".dockerignore", ".editorconfig": + return true + } + return false +} + +// isCodeFile returns true if the file is a source code file (uses code chunking). +func isCodeFile(path string) bool { + ext := strings.ToLower(filepath.Ext(path)) + switch ext { + case ".md", ".txt", ".csv", ".html", ".htm", ".xml", ".json", ".yaml", ".yml": + return false // prose / data — use default chunking + } + return indexableExtensions[ext] +} + +// ── Indexer ────────────────────────────────── + +// Indexer processes workspace files through the chunking + embedding pipeline. +// It reuses the knowledge.Embedder for vector generation and shares the +// concurrency semaphore with KB ingestion to avoid overwhelming the +// embedding provider. +type Indexer struct { + stores store.Stores + embedder *knowledge.Embedder + wfs *FS + sem chan struct{} // shared semaphore with KB ingestion + wg sync.WaitGroup + enabled bool // global kill switch (WORKSPACE_INDEXING_ENABLED) +} + +// NewIndexer creates a workspace indexer. +// The semaphore should be shared with the KB Ingester to cap total +// embedding concurrency. +func NewIndexer(stores store.Stores, embedder *knowledge.Embedder, wfs *FS, sem chan struct{}, enabled bool) *Indexer { + return &Indexer{ + stores: stores, + embedder: embedder, + wfs: wfs, + sem: sem, + enabled: enabled, + } +} + +// IsEnabled returns true if workspace indexing is globally enabled +// AND the embedding role is configured. +func (idx *Indexer) IsEnabled(ctx context.Context) bool { + return idx.enabled && idx.embedder.IsConfigured(ctx) +} + +// IndexFile starts async indexing for a single workspace file. +// Called from WorkspaceFS.WriteFile after a successful write. +// No-op if indexing is disabled, the file is non-indexable, or the +// sha256 hasn't changed (content-addressed skip). +func (idx *Indexer) IndexFile(w *models.Workspace, f *models.WorkspaceFile, userID string, teamID *string) { + if !idx.enabled || !w.IndexingEnabled { + return + } + + // Check if indexable + if f.IsDirectory || !isIndexable(f.Path, f.ContentType) { + // Mark as skipped so the UI knows it's intentional + ctx := context.Background() + idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusSkipped, 0) + return + } + + // Mark as pending + ctx := context.Background() + idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusPending, 0) + + idx.wg.Add(1) + go func() { + defer idx.wg.Done() + + // Acquire semaphore + idx.sem <- struct{}{} + defer func() { <-idx.sem }() + + ictx, cancel := context.WithTimeout(context.Background(), indexTimeout) + defer cancel() + + if err := idx.indexFile(ictx, w, f, userID, teamID); err != nil { + log.Printf("❌ workspace index %s/%s: %v", w.ID[:8], f.Path, err) + idx.stores.Workspaces.UpdateFileIndexStatus(ictx, f.ID, models.IndexStatusError, 0) + } + }() +} + +// IndexBatch starts async indexing for multiple files (e.g. after archive extract). +// Files are indexed sequentially within a single goroutine to be friendly +// to rate limits. +func (idx *Indexer) IndexBatch(w *models.Workspace, files []models.WorkspaceFile, userID string, teamID *string) { + if !idx.enabled || !w.IndexingEnabled { + return + } + + // Filter to indexable files only + var indexable []models.WorkspaceFile + ctx := context.Background() + for _, f := range files { + if f.IsDirectory || !isIndexable(f.Path, f.ContentType) { + idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusSkipped, 0) + continue + } + idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusPending, 0) + indexable = append(indexable, f) + } + + if len(indexable) == 0 { + return + } + + idx.wg.Add(1) + go func() { + defer idx.wg.Done() + + // One semaphore slot for the entire batch + idx.sem <- struct{}{} + defer func() { <-idx.sem }() + + var indexed, errors int + for _, f := range indexable { + f := f + ictx, cancel := context.WithTimeout(context.Background(), indexTimeout) + if err := idx.indexFile(ictx, w, &f, userID, teamID); err != nil { + log.Printf("❌ workspace index %s/%s: %v", w.ID[:8], f.Path, err) + idx.stores.Workspaces.UpdateFileIndexStatus(ictx, f.ID, models.IndexStatusError, 0) + errors++ + } else { + indexed++ + } + cancel() + } + + log.Printf("📑 workspace batch index %s: %d indexed, %d errors, %d skipped", + w.ID[:8], indexed, errors, len(files)-len(indexable)) + + // Notify on completion + if svc := notifications.Default(); svc != nil && indexed > 0 { + // Find workspace owner for notification target + if w.OwnerType == models.WorkspaceOwnerUser { + svc.Notify(context.Background(), &models.Notification{ + UserID: w.OwnerID, + Type: "workspace.indexed", + Title: "Workspace indexed", + Body: fmt.Sprintf("%d files indexed in workspace %q", indexed, w.Name), + ResourceType: "workspace", + ResourceID: w.ID, + }) + } + } + }() +} + +// Wait blocks until all in-flight indexing goroutines complete. +func (idx *Indexer) Wait() { + idx.wg.Wait() +} + +// ── Internal Pipeline ──────────────────────── + +func (idx *Indexer) indexFile(ctx context.Context, w *models.Workspace, f *models.WorkspaceFile, userID string, teamID *string) error { + start := time.Now() + + // Mark as indexing + idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusIndexing, 0) + + // Check if embedding is configured (might have changed since enqueue) + if !idx.embedder.IsConfigured(ctx) { + return fmt.Errorf("embedding role not configured") + } + + // ── Step 1: Read file content ─────────── + rc, _, err := idx.wfs.ReadFile(ctx, w, f.Path) + if err != nil { + return fmt.Errorf("read: %w", err) + } + defer rc.Close() + + // Cap at max text length + limited := io.LimitReader(rc, maxIndexTextLength) + data, err := io.ReadAll(limited) + if err != nil { + return fmt.Errorf("read content: %w", err) + } + text := string(data) + + if strings.TrimSpace(text) == "" { + idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusSkipped, 0) + return nil + } + + // ── Step 2: Chunk ─────────────────────── + var cfg knowledge.ChunkConfig + if isCodeFile(f.Path) { + cfg = CodeChunkConfig() + } else { + cfg = knowledge.DefaultChunkConfig() + } + + chunks := knowledge.SplitText(text, cfg) + if len(chunks) == 0 { + idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusSkipped, 0) + return nil + } + + // ── Step 3: Embed ─────────────────────── + texts := make([]string, len(chunks)) + for i, c := range chunks { + texts[i] = c.Content + } + + embedResult, err := idx.embedder.EmbedChunks(ctx, userID, teamID, texts) + if err != nil { + return fmt.Errorf("embed: %w", err) + } + + // Track usage + idx.embedder.LogUsage(ctx, userID, nil, embedResult) + + // ── Step 4: Delete old chunks + insert new ─ + if err := idx.stores.Workspaces.DeleteChunksByFile(ctx, f.ID); err != nil { + return fmt.Errorf("delete old chunks: %w", err) + } + + dbChunks := make([]models.WorkspaceChunk, len(chunks)) + for i, c := range chunks { + // Compute approximate line number from character offset + lineStart := 0 + if i > 0 { + charOffset := i * (cfg.ChunkSize - cfg.ChunkOverlap) + if charOffset < len(text) { + lineStart = strings.Count(text[:charOffset], "\n") + 1 + } + } + + dbChunks[i] = models.WorkspaceChunk{ + WorkspaceID: w.ID, + FileID: f.ID, + ChunkIndex: c.Index, + Content: c.Content, + TokenCount: c.TokenCount, + Embedding: embedResult.Vectors[i], + Metadata: models.JSONMap{ + "line_start": lineStart, + "language": inferLanguage(f.Path), + }, + } + } + + if err := idx.stores.Workspaces.InsertChunks(ctx, dbChunks); err != nil { + return fmt.Errorf("store chunks: %w", err) + } + + // ── Step 5: Update file index status ──── + if err := idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusReady, len(chunks)); err != nil { + log.Printf("⚠ failed to update index status for %s: %v", f.Path, err) + } + + log.Printf(" 📑 indexed %s → %d chunks (%s)", f.Path, len(chunks), time.Since(start).Round(time.Millisecond)) + return nil +} + +// inferLanguage extracts a simple language identifier from file extension. +func inferLanguage(path string) string { + ext := strings.ToLower(filepath.Ext(path)) + if ext == "" { + return "" + } + // Strip leading dot + return ext[1:] +}