switchboard/core
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Releases 9 Wiki Activity

Changeset 0.21.2 (#88)

Browse Source
This commit is contained in:
xcaliber
2026-03-01 17:45:50 +00:00
parent 09c7281552
commit c5159538ce
18 changed files with 1105 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
server/config/config.go
View File

@@ -51,6 +51,12 @@ type Config struct {
// EXTRACTION_CONCURRENCY: max concurrent extraction jobs (default 3).
ExtractionMode string
ExtractionConcurrency int
// Workspace indexing (v0.21.2)
// WORKSPACE_INDEXING_ENABLED: global kill switch for workspace file indexing (default true).
// WORKSPACE_INDEX_CONCURRENCY: max concurrent indexing goroutines (default 2).
WorkspaceIndexingEnabled bool
WorkspaceIndexConcurrency int
}
// Load reads configuration from environment variables.
@@ -83,6 +89,9 @@ func Load() *Config {
S3ForcePathStyle: getEnv("S3_FORCE_PATH_STYLE", "true") == "true",
ExtractionMode: getEnv("EXTRACTION_MODE", "inline"),
ExtractionConcurrency: getEnvInt("EXTRACTION_CONCURRENCY", 3),
WorkspaceIndexingEnabled: getEnvBool("WORKSPACE_INDEXING_ENABLED", true),
WorkspaceIndexConcurrency: getEnvInt("WORKSPACE_INDEX_CONCURRENCY", 2),
}
}
@@ -143,3 +152,13 @@ func getEnvInt(key string, fallback int) int {
}
return fallback
}
func getEnvBool(key string, fallback bool) bool {
if v := os.Getenv(key); v != "" {
b, err := strconv.ParseBool(v)
if err == nil {
return b
}
}
return fallback
}

58
server/database/migrations/011_v0212_workspace_chunks.sql Normal file
View File

@@ -0,0 +1,58 @@
-- v0.21.2: Workspace indexing + semantic search
--
-- New: workspace_chunks table for text embeddings per workspace file
-- Alter: workspace_files gains index_status and chunk_count columns
-- Alter: workspaces gains indexing_enabled column
-- =========================================
-- 1. workspace_files additions
-- =========================================
ALTER TABLE workspace_files
ADD COLUMN IF NOT EXISTS index_status VARCHAR(20) DEFAULT 'pending'
CHECK (index_status IN ('pending', 'indexing', 'ready', 'error', 'skipped'));
ALTER TABLE workspace_files
ADD COLUMN IF NOT EXISTS chunk_count INT NOT NULL DEFAULT 0;
CREATE INDEX IF NOT EXISTS idx_workspace_files_index_status
ON workspace_files(workspace_id, index_status)
WHERE index_status IN ('pending', 'indexing');
-- =========================================
-- 2. workspaces addition
-- =========================================
ALTER TABLE workspaces
ADD COLUMN IF NOT EXISTS indexing_enabled BOOLEAN NOT NULL DEFAULT true;
-- =========================================
-- 3. workspace_chunks table
-- =========================================
CREATE TABLE IF NOT EXISTS workspace_chunks (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
file_id UUID NOT NULL REFERENCES workspace_files(id) ON DELETE CASCADE,
chunk_index INT NOT NULL,
content TEXT NOT NULL,
token_count INT NOT NULL DEFAULT 0,
embedding vector(3072),
metadata JSONB DEFAULT '{}',
created_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_workspace_chunks_file
ON workspace_chunks(file_id);
CREATE INDEX IF NOT EXISTS idx_workspace_chunks_workspace
ON workspace_chunks(workspace_id);
-- Note: No vector index created here. pgvector IVFFlat/HNSW indexes are
-- limited to 2000 dimensions, but our embeddings are 3072-dim (zero-padded).
-- Sequential scan with <=> is adequate at workspace scale (thousands of
-- chunks, not millions). Add a partial/quantized index later if needed.
COMMENT ON TABLE workspace_chunks IS 'Text chunks with vector embeddings for workspace file semantic search';
COMMENT ON COLUMN workspace_chunks.embedding IS 'Vector embedding (3072-dim, zero-padded) for cosine similarity';
COMMENT ON COLUMN workspace_chunks.metadata IS 'Extensible: line_start, heading, language, etc.';

31
server/database/migrations/sqlite/010_v0212_workspace_chunks.sql Normal file
View File

@@ -0,0 +1,31 @@
-- v0.21.2: Workspace indexing + semantic search (SQLite)
-- workspace_files additions
ALTER TABLE workspace_files ADD COLUMN index_status TEXT DEFAULT 'pending';
ALTER TABLE workspace_files ADD COLUMN chunk_count INTEGER NOT NULL DEFAULT 0;
CREATE INDEX IF NOT EXISTS idx_workspace_files_index_status
ON workspace_files(workspace_id, index_status);
-- workspaces addition
ALTER TABLE workspaces ADD COLUMN indexing_enabled INTEGER NOT NULL DEFAULT 1;
-- workspace_chunks table
-- Embeddings stored as JSON text for app-level cosine similarity.
CREATE TABLE IF NOT EXISTS workspace_chunks (
id TEXT PRIMARY KEY,
workspace_id TEXT NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
file_id TEXT NOT NULL REFERENCES workspace_files(id) ON DELETE CASCADE,
chunk_index INTEGER NOT NULL,
content TEXT NOT NULL,
token_count INTEGER NOT NULL DEFAULT 0,
embedding TEXT, -- JSON array of floats
metadata TEXT DEFAULT '{}', -- JSON object
created_at TEXT DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_workspace_chunks_file
ON workspace_chunks(file_id);
CREATE INDEX IF NOT EXISTS idx_workspace_chunks_workspace
ON workspace_chunks(workspace_id);

40
server/handlers/workspaces.go
View File

@@ -394,6 +394,46 @@ func (h *WorkspaceHandler) Stats(c *gin.Context) {
c.JSON(http.StatusOK, stats)
}
// IndexStatus returns indexing progress for all files in the workspace (v0.21.2).
func (h *WorkspaceHandler) IndexStatus(c *gin.Context) {
w, ok := h.loadAndAuthorize(c)
if !ok {
return
}
ctx := c.Request.Context()
files, err := h.stores.Workspaces.ListFiles(ctx, w.ID, "", true)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to list files"})
return
}
// Aggregate counts by status
counts := map[string]int{
"pending": 0, "indexing": 0, "ready": 0,
"error": 0, "skipped": 0,
}
totalChunks := 0
for _, f := range files {
if f.IsDirectory {
continue
}
status := f.IndexStatus
if status == "" {
status = "pending"
}
counts[status]++
totalChunks += f.ChunkCount
}
c.JSON(http.StatusOK, gin.H{
"workspace_id": w.ID,
"indexing_enabled": w.IndexingEnabled,
"status_counts": counts,
"total_chunks": totalChunks,
})
}
// ── Authorization Helpers ───────────────────
// loadAndAuthorize loads a workspace by :id param and checks user access.

20
server/main.go
View File

@@ -186,6 +186,25 @@ func main() {
// Register note tools (late registration — needs stores + embedder for semantic search)
tools.RegisterNoteTools(stores, kbEmbedder)
// ── Workspace Indexer (v0.21.2) ───────────
// Background indexing pipeline for workspace files.
// Shares the embedder with KB ingestion; uses its own concurrency semaphore.
var wsIndexer *workspace.Indexer
if wfs != nil {
idxSem := make(chan struct{}, cfg.WorkspaceIndexConcurrency)
wsIndexer = workspace.NewIndexer(stores, kbEmbedder, wfs, idxSem, cfg.WorkspaceIndexingEnabled)
wfs.SetIndexer(wsIndexer) // hook into write path
defer wsIndexer.Wait()
if cfg.WorkspaceIndexingEnabled {
log.Printf(" 📑 Workspace indexer enabled (concurrency=%d)", cfg.WorkspaceIndexConcurrency)
} else {
log.Printf(" 📑 Workspace indexer disabled (WORKSPACE_INDEXING_ENABLED=false)")
}
}
// Register workspace_search tool (needs embedder for query embedding)
tools.RegisterWorkspaceSearchTool(stores, kbEmbedder)
// Register context recall tools (v0.15.1)
tools.RegisterAttachmentRecall(stores, objStore)
tools.RegisterConversationSearch(stores)
@@ -438,6 +457,7 @@ func main() {
protected.GET("/workspaces/:id/archive/download", wsH.DownloadArchive)
protected.POST("/workspaces/:id/reconcile", wsH.Reconcile)
protected.GET("/workspaces/:id/stats", wsH.Stats)
protected.GET("/workspaces/:id/index-status", wsH.IndexStatus)
}
// Attachments (file upload/download)

45
server/models/models_workspace.go
View File

@@ -31,6 +31,8 @@ type Workspace struct {
MaxBytes *int64 `json:"max_bytes,omitempty" db:"max_bytes"`
Status string `json:"status" db:"status"`
IndexingEnabled bool `json:"indexing_enabled" db:"indexing_enabled"`
// Computed fields (not DB columns)
FileCount int `json:"file_count,omitempty"`
TotalBytes int64 `json:"total_bytes,omitempty"`
@@ -38,9 +40,10 @@ type Workspace struct {
// WorkspacePatch holds optional fields for updating a workspace.
type WorkspacePatch struct {
Name *string `json:"name,omitempty"`
MaxBytes *int64 `json:"max_bytes,omitempty"`
Status *string `json:"status,omitempty"`
Name *string `json:"name,omitempty"`
MaxBytes *int64 `json:"max_bytes,omitempty"`
Status *string `json:"status,omitempty"`
IndexingEnabled *bool `json:"indexing_enabled,omitempty"`
}
// WorkspaceFile represents metadata for a single file or directory in a workspace.
@@ -52,6 +55,8 @@ type WorkspaceFile struct {
ContentType string `json:"content_type,omitempty" db:"content_type"`
SizeBytes int64 `json:"size_bytes" db:"size_bytes"`
SHA256 string `json:"sha256,omitempty" db:"sha256"`
IndexStatus string `json:"index_status,omitempty" db:"index_status"`
ChunkCount int `json:"chunk_count,omitempty" db:"chunk_count"`
CreatedAt time.Time `json:"created_at" db:"created_at"`
UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
}
@@ -64,3 +69,37 @@ type WorkspaceStats struct {
TotalBytes int64 `json:"total_bytes"`
MaxBytes *int64 `json:"max_bytes,omitempty"`
}
// =========================================
// WORKSPACE CHUNKS (v0.21.2)
// =========================================
// Workspace file index status constants.
const (
IndexStatusPending = "pending"
IndexStatusIndexing = "indexing"
IndexStatusReady = "ready"
IndexStatusError = "error"
IndexStatusSkipped = "skipped"
)
// WorkspaceChunk is a text segment from a workspace file with its embedding vector.
type WorkspaceChunk struct {
ID string `json:"id" db:"id"`
WorkspaceID string `json:"workspace_id" db:"workspace_id"`
FileID string `json:"file_id" db:"file_id"`
ChunkIndex int `json:"chunk_index" db:"chunk_index"`
Content string `json:"content" db:"content"`
TokenCount int `json:"token_count" db:"token_count"`
Embedding []float64 `json:"-" db:"embedding"` // never serialized to API
Metadata JSONMap `json:"metadata" db:"metadata"`
}
// WorkspaceChunkResult is a single result from workspace similarity search.
type WorkspaceChunkResult struct {
FilePath string `json:"file_path"`
Content string `json:"content"`
Score float64 `json:"score"`
LineHint int `json:"line_hint,omitempty"`
Metadata JSONMap `json:"metadata,omitempty"`
}

6
server/store/interfaces.go
View File

@@ -539,6 +539,12 @@ type WorkspaceStore interface {
// Stats
GetStats(ctx context.Context, workspaceID string) (*models.WorkspaceStats, error)
// Chunks (v0.21.2 — workspace indexing)
InsertChunks(ctx context.Context, chunks []models.WorkspaceChunk) error
DeleteChunksByFile(ctx context.Context, fileID string) error
SimilaritySearch(ctx context.Context, workspaceID string, queryVec []float64, threshold float64, limit int) ([]models.WorkspaceChunkResult, error)
UpdateFileIndexStatus(ctx context.Context, fileID, status string, chunkCount int) error
}
// =========================================

101
server/store/postgres/workspace.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"fmt"
"strings"
"time"
"git.gobha.me/xcaliber/chat-switchboard/models"
@@ -15,8 +16,8 @@ func NewWorkspaceStore() *WorkspaceStore { return &WorkspaceStore{} }
// ── columns ────────────────────────────────
const workspaceCols = `id, owner_type, owner_id, name, root_path, max_bytes, status, created_at, updated_at`
const workspaceFileCols = `id, workspace_id, path, is_directory, content_type, size_bytes, sha256, created_at, updated_at`
const workspaceCols = `id, owner_type, owner_id, name, root_path, max_bytes, status, indexing_enabled, created_at, updated_at`
const workspaceFileCols = `id, workspace_id, path, is_directory, content_type, size_bytes, sha256, index_status, chunk_count, created_at, updated_at`
// ── scanners ───────────────────────────────
@@ -25,7 +26,7 @@ func scanWorkspace(row interface{ Scan(dest ...interface{}) error }) (*models.Wo
var maxBytes sql.NullInt64
err := row.Scan(
&w.ID, &w.OwnerType, &w.OwnerID, &w.Name,
&w.RootPath, &maxBytes, &w.Status,
&w.RootPath, &maxBytes, &w.Status, &w.IndexingEnabled,
&w.CreatedAt, &w.UpdatedAt,
)
if err != nil {
@@ -41,9 +42,11 @@ func scanWorkspaceFile(row interface{ Scan(dest ...interface{}) error }) (*model
var f models.WorkspaceFile
var contentType sql.NullString
var sha256 sql.NullString
var indexStatus sql.NullString
err := row.Scan(
&f.ID, &f.WorkspaceID, &f.Path, &f.IsDirectory,
&contentType, &f.SizeBytes, &sha256,
&indexStatus, &f.ChunkCount,
&f.CreatedAt, &f.UpdatedAt,
)
if err != nil {
@@ -55,6 +58,9 @@ func scanWorkspaceFile(row interface{ Scan(dest ...interface{}) error }) (*model
if sha256.Valid {
f.SHA256 = sha256.String
}
if indexStatus.Valid {
f.IndexStatus = indexStatus.String
}
return &f, nil
}
@@ -97,6 +103,9 @@ func (s *WorkspaceStore) Update(ctx context.Context, id string, patch models.Wor
if patch.Status != nil {
b.Set("status", *patch.Status)
}
if patch.IndexingEnabled != nil {
b.Set("indexing_enabled", *patch.IndexingEnabled)
}
if !b.HasSets() {
return nil
}
@@ -270,3 +279,89 @@ func (s *WorkspaceStore) GetStats(ctx context.Context, workspaceID string) (*mod
return stats, nil
}
// ── Chunks (v0.21.2) ─────────────────────────
func (s *WorkspaceStore) InsertChunks(ctx context.Context, chunks []models.WorkspaceChunk) error {
if len(chunks) == 0 {
return nil
}
const cols = 8 // workspace_id, file_id, chunk_index, content, token_count, embedding, metadata, created_at
valueParts := make([]string, 0, len(chunks))
args := make([]interface{}, 0, len(chunks)*cols)
for i, c := range chunks {
base := i * cols
valueParts = append(valueParts, fmt.Sprintf(
"($%d, $%d, $%d, $%d, $%d, $%d::vector, $%d, NOW())",
base+1, base+2, base+3, base+4, base+5, base+6, base+7,
))
args = append(args, c.WorkspaceID, c.FileID, c.ChunkIndex,
c.Content, c.TokenCount, vectorToString(c.Embedding), ToJSON(c.Metadata))
}
q := fmt.Sprintf(`INSERT INTO workspace_chunks
(workspace_id, file_id, chunk_index, content, token_count, embedding, metadata, created_at)
VALUES %s`, strings.Join(valueParts, ", "))
_, err := DB.ExecContext(ctx, q, args...)
return err
}
func (s *WorkspaceStore) DeleteChunksByFile(ctx context.Context, fileID string) error {
_, err := DB.ExecContext(ctx, `DELETE FROM workspace_chunks WHERE file_id = $1`, fileID)
return err
}
func (s *WorkspaceStore) SimilaritySearch(ctx context.Context, workspaceID string, queryVec []float64, threshold float64, limit int) ([]models.WorkspaceChunkResult, error) {
if len(queryVec) == 0 {
return nil, nil
}
if limit <= 0 {
limit = 10
}
rows, err := DB.QueryContext(ctx, `
SELECT f.path, c.content,
1 - (c.embedding <=> $1::vector) AS similarity,
c.metadata
FROM workspace_chunks c
JOIN workspace_files f ON c.file_id = f.id
WHERE c.workspace_id = $2
AND 1 - (c.embedding <=> $1::vector) > $3
ORDER BY c.embedding <=> $1::vector
LIMIT $4`,
vectorToString(queryVec), workspaceID, threshold, limit,
)
if err != nil {
return nil, err
}
defer rows.Close()
var results []models.WorkspaceChunkResult
for rows.Next() {
var r models.WorkspaceChunkResult
var metadataJSON []byte
if err := rows.Scan(&r.FilePath, &r.Content, &r.Score, &metadataJSON); err != nil {
return nil, err
}
ScanJSON(metadataJSON, &r.Metadata)
if r.Metadata != nil {
if lh, ok := r.Metadata["line_start"]; ok {
if n, ok := lh.(float64); ok {
r.LineHint = int(n)
}
}
}
results = append(results, r)
}
return results, rows.Err()
}
func (s *WorkspaceStore) UpdateFileIndexStatus(ctx context.Context, fileID, status string, chunkCount int) error {
_, err := DB.ExecContext(ctx,
`UPDATE workspace_files SET index_status = $1, chunk_count = $2, updated_at = NOW() WHERE id = $3`,
status, chunkCount, fileID)
return err
}

157
server/store/sqlite/workspace.go
View File

@@ -3,7 +3,11 @@ package sqlite
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"math"
"sort"
"strings"
"time"
"git.gobha.me/xcaliber/chat-switchboard/models"
@@ -16,8 +20,8 @@ func NewWorkspaceStore() *WorkspaceStore { return &WorkspaceStore{} }
// ── columns ────────────────────────────────
const workspaceCols = `id, owner_type, owner_id, name, root_path, max_bytes, status, created_at, updated_at`
const workspaceFileCols = `id, workspace_id, path, is_directory, content_type, size_bytes, sha256, created_at, updated_at`
const workspaceCols = `id, owner_type, owner_id, name, root_path, max_bytes, status, indexing_enabled, created_at, updated_at`
const workspaceFileCols = `id, workspace_id, path, is_directory, content_type, size_bytes, sha256, index_status, chunk_count, created_at, updated_at`
// ── scanners ───────────────────────────────
@@ -26,7 +30,7 @@ func scanWorkspace(row interface{ Scan(dest ...interface{}) error }) (*models.Wo
var maxBytes sql.NullInt64
err := row.Scan(
&w.ID, &w.OwnerType, &w.OwnerID, &w.Name,
&w.RootPath, &maxBytes, &w.Status,
&w.RootPath, &maxBytes, &w.Status, &w.IndexingEnabled,
st(&w.CreatedAt), st(&w.UpdatedAt),
)
if err != nil {
@@ -42,9 +46,11 @@ func scanWorkspaceFile(row interface{ Scan(dest ...interface{}) error }) (*model
var f models.WorkspaceFile
var contentType sql.NullString
var sha256 sql.NullString
var indexStatus sql.NullString
err := row.Scan(
&f.ID, &f.WorkspaceID, &f.Path, &f.IsDirectory,
&contentType, &f.SizeBytes, &sha256,
&indexStatus, &f.ChunkCount,
st(&f.CreatedAt), st(&f.UpdatedAt),
)
if err != nil {
@@ -56,6 +62,9 @@ func scanWorkspaceFile(row interface{ Scan(dest ...interface{}) error }) (*model
if sha256.Valid {
f.SHA256 = sha256.String
}
if indexStatus.Valid {
f.IndexStatus = indexStatus.String
}
return &f, nil
}
@@ -95,6 +104,9 @@ func (s *WorkspaceStore) Update(ctx context.Context, id string, patch models.Wor
if patch.Status != nil {
b.Set("status", *patch.Status)
}
if patch.IndexingEnabled != nil {
b.Set("indexing_enabled", *patch.IndexingEnabled)
}
if !b.HasSets() {
return nil
}
@@ -272,3 +284,142 @@ func (s *WorkspaceStore) GetStats(ctx context.Context, workspaceID string) (*mod
return stats, nil
}
// ── Chunks (v0.21.2) ─────────────────────────
func (s *WorkspaceStore) InsertChunks(ctx context.Context, chunks []models.WorkspaceChunk) error {
if len(chunks) == 0 {
return nil
}
valueParts := make([]string, 0, len(chunks))
args := make([]interface{}, 0, len(chunks)*8)
for _, c := range chunks {
c.ID = store.NewID()
valueParts = append(valueParts, "(?, ?, ?, ?, ?, ?, ?, ?)")
var embJSON interface{}
if len(c.Embedding) > 0 {
embJSON = ToJSON(c.Embedding)
}
args = append(args, c.ID, c.WorkspaceID, c.FileID, c.ChunkIndex,
c.Content, c.TokenCount, embJSON, ToJSON(c.Metadata))
}
q := fmt.Sprintf(`INSERT INTO workspace_chunks
(id, workspace_id, file_id, chunk_index, content, token_count, embedding, metadata)
VALUES %s`, strings.Join(valueParts, ", "))
_, err := DB.ExecContext(ctx, q, args...)
return err
}
func (s *WorkspaceStore) DeleteChunksByFile(ctx context.Context, fileID string) error {
_, err := DB.ExecContext(ctx, `DELETE FROM workspace_chunks WHERE file_id = ?`, fileID)
return err
}
// SimilaritySearch performs app-level cosine similarity for SQLite.
// Loads all chunks for the workspace, computes cosine distance in Go,
// and returns the top results above threshold.
func (s *WorkspaceStore) SimilaritySearch(ctx context.Context, workspaceID string, queryVec []float64, threshold float64, limit int) ([]models.WorkspaceChunkResult, error) {
if len(queryVec) == 0 {
return nil, nil
}
if limit <= 0 {
limit = 10
}
rows, err := DB.QueryContext(ctx, `
SELECT f.path, c.content, c.embedding, c.metadata
FROM workspace_chunks c
JOIN workspace_files f ON c.file_id = f.id
WHERE c.workspace_id = ?
AND c.embedding IS NOT NULL`,
workspaceID)
if err != nil {
return nil, err
}
defer rows.Close()
type candidate struct {
result models.WorkspaceChunkResult
similarity float64
}
var candidates []candidate
for rows.Next() {
var filePath, content, embJSON string
var metaJSON sql.NullString
if err := rows.Scan(&filePath, &content, &embJSON, &metaJSON); err != nil {
return nil, err
}
var embedding []float64
if err := json.Unmarshal([]byte(embJSON), &embedding); err != nil {
continue // skip malformed
}
sim := wsCosineSimilarity(queryVec, embedding)
if sim > threshold {
r := models.WorkspaceChunkResult{
FilePath: filePath,
Content: content,
Score: sim,
}
if metaJSON.Valid {
ScanJSON(metaJSON.String, &r.Metadata)
}
if r.Metadata != nil {
if lh, ok := r.Metadata["line_start"]; ok {
if n, ok := lh.(float64); ok {
r.LineHint = int(n)
}
}
}
candidates = append(candidates, candidate{result: r, similarity: sim})
}
}
if err := rows.Err(); err != nil {
return nil, err
}
sort.Slice(candidates, func(i, j int) bool {
return candidates[i].similarity > candidates[j].similarity
})
if len(candidates) > limit {
candidates = candidates[:limit]
}
results := make([]models.WorkspaceChunkResult, len(candidates))
for i, c := range candidates {
results[i] = c.result
}
return results, nil
}
func (s *WorkspaceStore) UpdateFileIndexStatus(ctx context.Context, fileID, status string, chunkCount int) error {
_, err := DB.ExecContext(ctx,
`UPDATE workspace_files SET index_status = ?, chunk_count = ?, updated_at = datetime('now') WHERE id = ?`,
status, chunkCount, fileID)
return err
}
// wsCosineSimilarity computes cosine similarity between two vectors.
// Named to avoid collision with the KB package-level function.
func wsCosineSimilarity(a, b []float64) float64 {
if len(a) != len(b) || len(a) == 0 {
return 0
}
var dot, normA, normB float64
for i := range a {
dot += a[i] * b[i]
normA += a[i] * a[i]
normB += b[i] * b[i]
}
if normA == 0 || normB == 0 {
return 0
}
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
}

8
server/tools/workspace.go
View File

@@ -8,6 +8,7 @@ import (
"log"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
"git.gobha.me/xcaliber/chat-switchboard/workspace"
@@ -27,12 +28,19 @@ func RegisterWorkspaceTools(stores store.Stores, wfs *workspace.FS) {
Register(&workspacePatchTool{stores: stores, wfs: wfs})
}
// RegisterWorkspaceSearchTool registers the semantic search tool separately
// because it requires the embedder (which may not be configured).
func RegisterWorkspaceSearchTool(stores store.Stores, embedder *knowledge.Embedder) {
Register(&workspaceSearchTool{stores: stores, embedder: embedder})
}
// WorkspaceToolNames returns the names of all workspace tools.
// Used by the completion handler to filter them out when no workspace is bound.
func WorkspaceToolNames() []string {
return []string{
"workspace_ls", "workspace_read", "workspace_write",
"workspace_rm", "workspace_mv", "workspace_patch",
"workspace_search",
}
}

150
server/tools/workspace_search.go Normal file
View File

@@ -0,0 +1,150 @@
package tools
import (
"context"
"encoding/json"
"fmt"
"path/filepath"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// ── workspace_search ────────────────────────
type workspaceSearchTool struct {
stores store.Stores
embedder *knowledge.Embedder
}
func (t *workspaceSearchTool) Definition() ToolDef {
return ToolDef{
Name: "workspace_search",
DisplayName: "Workspace Search",
Description: "Semantic search across workspace files. Finds code, documentation, and text content relevant to a natural language query. Returns matching file paths, content snippets, relevance scores, and approximate line numbers.",
Category: "workspace",
Parameters: JSONSchema(map[string]interface{}{
"query": Prop("string", "Natural language search query (e.g. 'database connection pooling', 'error handling middleware')"),
"top_k": map[string]interface{}{
"type": "integer",
"description": "Maximum number of results to return (default 10, max 20)",
"default": 10,
},
"file_pattern": Prop("string", "Optional glob pattern to filter results by file path (e.g. '*.go', 'src/*.ts'). Applied post-search."),
}, []string{"query"}),
}
}
func (t *workspaceSearchTool) Execute(ctx context.Context, execCtx ExecutionContext, argsJSON string) (string, error) {
var args struct {
Query string `json:"query"`
TopK int `json:"top_k"`
FilePattern string `json:"file_pattern"`
}
if err := json.Unmarshal([]byte(argsJSON), &args); err != nil {
return "", fmt.Errorf("invalid arguments: %w", err)
}
if args.Query == "" {
return "", fmt.Errorf("query is required")
}
if args.TopK <= 0 {
args.TopK = 10
}
if args.TopK > 20 {
args.TopK = 20
}
// Resolve workspace
w, err := loadWorkspace(ctx, t.stores, execCtx)
if err != nil {
return "", err
}
if !w.IndexingEnabled {
return "", fmt.Errorf("workspace indexing is disabled for this workspace")
}
// Embed the query
embedResult, err := t.embedder.EmbedChunks(ctx, execCtx.UserID, nil, []string{args.Query})
if err != nil {
return "", fmt.Errorf("failed to embed query: %w", err)
}
if len(embedResult.Vectors) == 0 {
return "", fmt.Errorf("embedding returned no vectors")
}
queryVec := embedResult.Vectors[0]
// Fetch more if we'll post-filter by glob
searchLimit := args.TopK
if args.FilePattern != "" {
searchLimit = args.TopK * 3
if searchLimit > 50 {
searchLimit = 50
}
}
results, err := t.stores.Workspaces.SimilaritySearch(ctx, w.ID, queryVec, 0.3, searchLimit)
if err != nil {
return "", fmt.Errorf("search failed: %w", err)
}
// Apply glob filter if specified
if args.FilePattern != "" {
results = filterByGlob(results, args.FilePattern)
}
// Cap at top_k
if len(results) > args.TopK {
results = results[:args.TopK]
}
// Format response
formatted := make([]map[string]interface{}, len(results))
for i, r := range results {
m := map[string]interface{}{
"file_path": r.FilePath,
"content": r.Content,
"score": fmt.Sprintf("%.3f", r.Score),
}
if r.LineHint > 0 {
m["line_hint"] = r.LineHint
}
formatted[i] = m
}
resp := map[string]interface{}{
"query": args.Query,
"count": len(formatted),
"results": formatted,
}
if args.FilePattern != "" {
resp["file_pattern"] = args.FilePattern
}
if len(formatted) == 0 {
resp["message"] = "No matching results found. Try a different query or check that workspace files have been indexed."
}
b, _ := json.Marshal(resp)
return string(b), nil
}
// filterByGlob filters search results by a glob pattern.
func filterByGlob(results []models.WorkspaceChunkResult, pattern string) []models.WorkspaceChunkResult {
var filtered []models.WorkspaceChunkResult
for _, r := range results {
// Try matching against basename
matched, _ := filepath.Match(pattern, filepath.Base(r.FilePath))
if !matched && strings.Contains(pattern, "/") {
// Try full path match if pattern contains directory separator
matched, _ = filepath.Match(pattern, r.FilePath)
}
if matched {
filtered = append(filtered, r)
}
}
return filtered
}

20
server/workspace/archive.go
View File

@@ -33,14 +33,30 @@ const (
// Returns the number of files extracted. Respects workspace quota.
// format must be "zip" or "tar.gz".
func (fs *FS) ExtractArchive(ctx context.Context, w *models.Workspace, archivePath, format string) (int, error) {
var count int
var err error
switch format {
case "zip":
return fs.extractZip(ctx, w, archivePath)
count, err = fs.extractZip(ctx, w, archivePath)
case "tar.gz", "tgz":
return fs.extractTarGz(ctx, w, archivePath)
count, err = fs.extractTarGz(ctx, w, archivePath)
default:
return 0, fmt.Errorf("workspace: unsupported archive format: %s", format)
}
// Trigger batch indexing for all extracted files (v0.21.2)
if err == nil && fs.indexer != nil && count > 0 {
files, listErr := fs.store.ListFiles(ctx, w.ID, "", true)
if listErr == nil {
var teamID *string
if w.OwnerType == models.WorkspaceOwnerTeam {
teamID = &w.OwnerID
}
fs.indexer.IndexBatch(w, files, w.OwnerID, teamID)
}
}
return count, err
}
func (fs *FS) extractZip(ctx context.Context, w *models.Workspace, archivePath string) (int, error) {

36
server/workspace/fs.go
View File

@@ -46,6 +46,7 @@ const (
type FS struct {
basePath string // e.g. /data/storage/workspaces
store store.WorkspaceStore
indexer *Indexer // optional — set via SetIndexer for v0.21.2 indexing
}
// NewFS creates a workspace filesystem manager.
@@ -53,6 +54,12 @@ func NewFS(basePath string, ws store.WorkspaceStore) *FS {
return &FS{basePath: basePath, store: ws}
}
// SetIndexer attaches the workspace indexer for background file indexing.
// Must be called after NewIndexer is created (circular dependency break).
func (fs *FS) SetIndexer(idx *Indexer) {
fs.indexer = idx
}
// Init ensures the base workspaces directory exists.
func (fs *FS) Init() error {
return os.MkdirAll(fs.basePath, 0750)
@@ -183,6 +190,12 @@ func (fs *FS) WriteFile(ctx context.Context, w *models.Workspace, relPath string
return fmt.Errorf("workspace: refusing to overwrite symlink: %s", relPath)
}
// Check existing sha256 for content-addressed skip (v0.21.2)
var oldSHA256 string
if existing, err := fs.store.GetFile(ctx, w.ID, relPath); err == nil {
oldSHA256 = existing.SHA256
}
// Ensure parent directory exists
dir := filepath.Dir(abs)
if err := os.MkdirAll(dir, 0750); err != nil {
@@ -218,16 +231,31 @@ func (fs *FS) WriteFile(ctx context.Context, w *models.Workspace, relPath string
// Update DB index
contentType := detectContentType(relPath, abs)
hash := hex.EncodeToString(h.Sum(nil))
newHash := hex.EncodeToString(h.Sum(nil))
return fs.store.UpsertFile(ctx, &models.WorkspaceFile{
f := &models.WorkspaceFile{
WorkspaceID: w.ID,
Path: relPath,
IsDirectory: false,
ContentType: contentType,
SizeBytes: n,
SHA256: hash,
})
SHA256: newHash,
}
if err := fs.store.UpsertFile(ctx, f); err != nil {
return err
}
// Trigger async indexing if content changed (v0.21.2)
if fs.indexer != nil && newHash != oldSHA256 {
// Use workspace owner for embedding provider resolution
var teamID *string
if w.OwnerType == models.WorkspaceOwnerTeam {
teamID = &w.OwnerID
}
fs.indexer.IndexFile(w, f, w.OwnerID, teamID)
}
return nil
}
// DeleteFile removes a file or empty directory at relPath.

14
server/workspace/fs_test.go
View File

@@ -262,6 +262,20 @@ func (m *mockWorkspaceStore) DeleteAllFiles(ctx context.Context, workspaceID str
return nil
}
// Chunk methods (v0.21.2) — no-ops for FS tests
func (m *mockWorkspaceStore) InsertChunks(ctx context.Context, chunks []models.WorkspaceChunk) error {
return nil
}
func (m *mockWorkspaceStore) DeleteChunksByFile(ctx context.Context, fileID string) error {
return nil
}
func (m *mockWorkspaceStore) SimilaritySearch(ctx context.Context, workspaceID string, queryVec []float64, threshold float64, limit int) ([]models.WorkspaceChunkResult, error) {
return nil, nil
}
func (m *mockWorkspaceStore) UpdateFileIndexStatus(ctx context.Context, fileID, status string, chunkCount int) error {
return nil
}
func TestWriteReadRoundTrip(t *testing.T) {
tmpDir := t.TempDir()
mock := newMockStore()

355
server/workspace/indexer.go Normal file
View File

@@ -0,0 +1,355 @@
package workspace
import (
"context"
"fmt"
"io"
"log"
"path/filepath"
"strings"
"sync"
"time"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/notifications"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// ── Configuration ────────────────────────────
const (
// indexTimeout is the max time for indexing a single file.
indexTimeout = 5 * time.Minute
// maxIndexTextLength caps text extracted from a single file (5 MB).
maxIndexTextLength = 5 * 1024 * 1024
)
// indexableExtensions defines source code and text file extensions
// eligible for chunking and embedding. MIME detection covers text/*,
// but code files often have application/* types — this map catches them.
var indexableExtensions = map[string]bool{
".go": true, ".py": true, ".js": true, ".ts": true, ".jsx": true,
".tsx": true, ".rs": true, ".c": true, ".cpp": true, ".h": true,
".hpp": true, ".java": true, ".rb": true, ".php": true, ".swift": true,
".kt": true, ".scala": true, ".sh": true, ".bash": true, ".zsh": true,
".sql": true, ".yaml": true, ".yml": true, ".toml": true, ".json": true,
".xml": true, ".css": true, ".scss": true, ".less": true,
".md": true, ".txt": true, ".csv": true, ".html": true, ".htm": true,
".dockerfile": true, ".tf": true, ".hcl": true, ".proto": true,
".graphql": true, ".vue": true, ".svelte": true,
".pl": true, ".pm": true, // Perl
".lua": true, ".zig": true, ".nim": true, ".ex": true, ".exs": true,
// Config / data
".ini": true, ".cfg": true, ".conf": true, ".env": true,
".mk": true, ".cmake": true, ".makefile": true,
".r": true, ".jl": true, // R, Julia
}
// CodeChunkConfig returns chunking settings optimized for source code.
// Larger chunks (1500 chars) because functions can be long, with
// separators tuned to split on function/class/type boundaries.
func CodeChunkConfig() knowledge.ChunkConfig {
return knowledge.ChunkConfig{
ChunkSize: 1500,
ChunkOverlap: 200,
Separators: []string{
"\n\nfunc ", "\n\nfn ", // Go, Rust
"\n\nclass ", "\n\ndef ", // Python, Ruby
"\n\ntype ", "\n\nsub ", // Go types, Perl
"\n\npub ", "\n\nimpl ", // Rust
"\n\n", "\n", " ", // fallback hierarchy
},
}
}
// isIndexable returns true if the file is eligible for text indexing.
func isIndexable(path string, contentType string) bool {
// Check extension first (catches source code with non-text MIME types)
ext := strings.ToLower(filepath.Ext(path))
if indexableExtensions[ext] {
return true
}
// Also index any text/* content type
if strings.HasPrefix(contentType, "text/") {
return true
}
// Specific filenames without extension
base := strings.ToLower(filepath.Base(path))
switch base {
case "makefile", "dockerfile", "jenkinsfile", "gemfile",
"rakefile", "procfile", "vagrantfile", ".gitignore",
".dockerignore", ".editorconfig":
return true
}
return false
}
// isCodeFile returns true if the file is a source code file (uses code chunking).
func isCodeFile(path string) bool {
ext := strings.ToLower(filepath.Ext(path))
switch ext {
case ".md", ".txt", ".csv", ".html", ".htm", ".xml", ".json", ".yaml", ".yml":
return false // prose / data — use default chunking
}
return indexableExtensions[ext]
}
// ── Indexer ──────────────────────────────────
// Indexer processes workspace files through the chunking + embedding pipeline.
// It reuses the knowledge.Embedder for vector generation and shares the
// concurrency semaphore with KB ingestion to avoid overwhelming the
// embedding provider.
type Indexer struct {
stores store.Stores
embedder *knowledge.Embedder
wfs *FS
sem chan struct{} // shared semaphore with KB ingestion
wg sync.WaitGroup
enabled bool // global kill switch (WORKSPACE_INDEXING_ENABLED)
}
// NewIndexer creates a workspace indexer.
// The semaphore should be shared with the KB Ingester to cap total
// embedding concurrency.
func NewIndexer(stores store.Stores, embedder *knowledge.Embedder, wfs *FS, sem chan struct{}, enabled bool) *Indexer {
return &Indexer{
stores: stores,
embedder: embedder,
wfs: wfs,
sem: sem,
enabled: enabled,
}
}
// IsEnabled returns true if workspace indexing is globally enabled
// AND the embedding role is configured.
func (idx *Indexer) IsEnabled(ctx context.Context) bool {
return idx.enabled && idx.embedder.IsConfigured(ctx)
}
// IndexFile starts async indexing for a single workspace file.
// Called from WorkspaceFS.WriteFile after a successful write.
// No-op if indexing is disabled, the file is non-indexable, or the
// sha256 hasn't changed (content-addressed skip).
func (idx *Indexer) IndexFile(w *models.Workspace, f *models.WorkspaceFile, userID string, teamID *string) {
if !idx.enabled || !w.IndexingEnabled {
return
}
// Check if indexable
if f.IsDirectory || !isIndexable(f.Path, f.ContentType) {
// Mark as skipped so the UI knows it's intentional
ctx := context.Background()
idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusSkipped, 0)
return
}
// Mark as pending
ctx := context.Background()
idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusPending, 0)
idx.wg.Add(1)
go func() {
defer idx.wg.Done()
// Acquire semaphore
idx.sem <- struct{}{}
defer func() { <-idx.sem }()
ictx, cancel := context.WithTimeout(context.Background(), indexTimeout)
defer cancel()
if err := idx.indexFile(ictx, w, f, userID, teamID); err != nil {
log.Printf("❌ workspace index %s/%s: %v", w.ID[:8], f.Path, err)
idx.stores.Workspaces.UpdateFileIndexStatus(ictx, f.ID, models.IndexStatusError, 0)
}
}()
}
// IndexBatch starts async indexing for multiple files (e.g. after archive extract).
// Files are indexed sequentially within a single goroutine to be friendly
// to rate limits.
func (idx *Indexer) IndexBatch(w *models.Workspace, files []models.WorkspaceFile, userID string, teamID *string) {
if !idx.enabled || !w.IndexingEnabled {
return
}
// Filter to indexable files only
var indexable []models.WorkspaceFile
ctx := context.Background()
for _, f := range files {
if f.IsDirectory || !isIndexable(f.Path, f.ContentType) {
idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusSkipped, 0)
continue
}
idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusPending, 0)
indexable = append(indexable, f)
}
if len(indexable) == 0 {
return
}
idx.wg.Add(1)
go func() {
defer idx.wg.Done()
// One semaphore slot for the entire batch
idx.sem <- struct{}{}
defer func() { <-idx.sem }()
var indexed, errors int
for _, f := range indexable {
f := f
ictx, cancel := context.WithTimeout(context.Background(), indexTimeout)
if err := idx.indexFile(ictx, w, &f, userID, teamID); err != nil {
log.Printf("❌ workspace index %s/%s: %v", w.ID[:8], f.Path, err)
idx.stores.Workspaces.UpdateFileIndexStatus(ictx, f.ID, models.IndexStatusError, 0)
errors++
} else {
indexed++
}
cancel()
}
log.Printf("📑 workspace batch index %s: %d indexed, %d errors, %d skipped",
w.ID[:8], indexed, errors, len(files)-len(indexable))
// Notify on completion
if svc := notifications.Default(); svc != nil && indexed > 0 {
// Find workspace owner for notification target
if w.OwnerType == models.WorkspaceOwnerUser {
svc.Notify(context.Background(), &models.Notification{
UserID: w.OwnerID,
Type: "workspace.indexed",
Title: "Workspace indexed",
Body: fmt.Sprintf("%d files indexed in workspace %q", indexed, w.Name),
ResourceType: "workspace",
ResourceID: w.ID,
})
}
}
}()
}
// Wait blocks until all in-flight indexing goroutines complete.
func (idx *Indexer) Wait() {
idx.wg.Wait()
}
// ── Internal Pipeline ────────────────────────
func (idx *Indexer) indexFile(ctx context.Context, w *models.Workspace, f *models.WorkspaceFile, userID string, teamID *string) error {
start := time.Now()
// Mark as indexing
idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusIndexing, 0)
// Check if embedding is configured (might have changed since enqueue)
if !idx.embedder.IsConfigured(ctx) {
return fmt.Errorf("embedding role not configured")
}
// ── Step 1: Read file content ───────────
rc, _, err := idx.wfs.ReadFile(ctx, w, f.Path)
if err != nil {
return fmt.Errorf("read: %w", err)
}
defer rc.Close()
// Cap at max text length
limited := io.LimitReader(rc, maxIndexTextLength)
data, err := io.ReadAll(limited)
if err != nil {
return fmt.Errorf("read content: %w", err)
}
text := string(data)
if strings.TrimSpace(text) == "" {
idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusSkipped, 0)
return nil
}
// ── Step 2: Chunk ───────────────────────
var cfg knowledge.ChunkConfig
if isCodeFile(f.Path) {
cfg = CodeChunkConfig()
} else {
cfg = knowledge.DefaultChunkConfig()
}
chunks := knowledge.SplitText(text, cfg)
if len(chunks) == 0 {
idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusSkipped, 0)
return nil
}
// ── Step 3: Embed ───────────────────────
texts := make([]string, len(chunks))
for i, c := range chunks {
texts[i] = c.Content
}
embedResult, err := idx.embedder.EmbedChunks(ctx, userID, teamID, texts)
if err != nil {
return fmt.Errorf("embed: %w", err)
}
// Track usage
idx.embedder.LogUsage(ctx, userID, nil, embedResult)
// ── Step 4: Delete old chunks + insert new ─
if err := idx.stores.Workspaces.DeleteChunksByFile(ctx, f.ID); err != nil {
return fmt.Errorf("delete old chunks: %w", err)
}
dbChunks := make([]models.WorkspaceChunk, len(chunks))
for i, c := range chunks {
// Compute approximate line number from character offset
lineStart := 0
if i > 0 {
charOffset := i * (cfg.ChunkSize - cfg.ChunkOverlap)
if charOffset < len(text) {
lineStart = strings.Count(text[:charOffset], "\n") + 1
}
}
dbChunks[i] = models.WorkspaceChunk{
WorkspaceID: w.ID,
FileID: f.ID,
ChunkIndex: c.Index,
Content: c.Content,
TokenCount: c.TokenCount,
Embedding: embedResult.Vectors[i],
Metadata: models.JSONMap{
"line_start": lineStart,
"language": inferLanguage(f.Path),
},
}
}
if err := idx.stores.Workspaces.InsertChunks(ctx, dbChunks); err != nil {
return fmt.Errorf("store chunks: %w", err)
}
// ── Step 5: Update file index status ────
if err := idx.stores.Workspaces.UpdateFileIndexStatus(ctx, f.ID, models.IndexStatusReady, len(chunks)); err != nil {
log.Printf("⚠ failed to update index status for %s: %v", f.Path, err)
}
log.Printf(" 📑 indexed %s → %d chunks (%s)", f.Path, len(chunks), time.Since(start).Round(time.Millisecond))
return nil
}
// inferLanguage extracts a simple language identifier from file extension.
func inferLanguage(path string) string {
ext := strings.ToLower(filepath.Ext(path))
if ext == "" {
return ""
}
// Strip leading dot
return ext[1:]
}