switchboard/core
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Releases 9 Wiki Activity

Changeset 0.21.2 (#88)

Browse Source
This commit is contained in:
xcaliber
2026-03-01 17:45:50 +00:00
parent 09c7281552
commit c5159538ce
18 changed files with 1105 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
server/store/interfaces.go
View File

@@ -539,6 +539,12 @@ type WorkspaceStore interface {
// Stats
GetStats(ctx context.Context, workspaceID string) (*models.WorkspaceStats, error)
// Chunks (v0.21.2 — workspace indexing)
InsertChunks(ctx context.Context, chunks []models.WorkspaceChunk) error
DeleteChunksByFile(ctx context.Context, fileID string) error
SimilaritySearch(ctx context.Context, workspaceID string, queryVec []float64, threshold float64, limit int) ([]models.WorkspaceChunkResult, error)
UpdateFileIndexStatus(ctx context.Context, fileID, status string, chunkCount int) error
}
// =========================================

101
server/store/postgres/workspace.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"fmt"
"strings"
"time"
"git.gobha.me/xcaliber/chat-switchboard/models"
@@ -15,8 +16,8 @@ func NewWorkspaceStore() *WorkspaceStore { return &WorkspaceStore{} }
// ── columns ────────────────────────────────
const workspaceCols = `id, owner_type, owner_id, name, root_path, max_bytes, status, created_at, updated_at`
const workspaceFileCols = `id, workspace_id, path, is_directory, content_type, size_bytes, sha256, created_at, updated_at`
const workspaceCols = `id, owner_type, owner_id, name, root_path, max_bytes, status, indexing_enabled, created_at, updated_at`
const workspaceFileCols = `id, workspace_id, path, is_directory, content_type, size_bytes, sha256, index_status, chunk_count, created_at, updated_at`
// ── scanners ───────────────────────────────
@@ -25,7 +26,7 @@ func scanWorkspace(row interface{ Scan(dest ...interface{}) error }) (*models.Wo
var maxBytes sql.NullInt64
err := row.Scan(
&w.ID, &w.OwnerType, &w.OwnerID, &w.Name,
&w.RootPath, &maxBytes, &w.Status,
&w.RootPath, &maxBytes, &w.Status, &w.IndexingEnabled,
&w.CreatedAt, &w.UpdatedAt,
)
if err != nil {
@@ -41,9 +42,11 @@ func scanWorkspaceFile(row interface{ Scan(dest ...interface{}) error }) (*model
var f models.WorkspaceFile
var contentType sql.NullString
var sha256 sql.NullString
var indexStatus sql.NullString
err := row.Scan(
&f.ID, &f.WorkspaceID, &f.Path, &f.IsDirectory,
&contentType, &f.SizeBytes, &sha256,
&indexStatus, &f.ChunkCount,
&f.CreatedAt, &f.UpdatedAt,
)
if err != nil {
@@ -55,6 +58,9 @@ func scanWorkspaceFile(row interface{ Scan(dest ...interface{}) error }) (*model
if sha256.Valid {
f.SHA256 = sha256.String
}
if indexStatus.Valid {
f.IndexStatus = indexStatus.String
}
return &f, nil
}
@@ -97,6 +103,9 @@ func (s *WorkspaceStore) Update(ctx context.Context, id string, patch models.Wor
if patch.Status != nil {
b.Set("status", *patch.Status)
}
if patch.IndexingEnabled != nil {
b.Set("indexing_enabled", *patch.IndexingEnabled)
}
if !b.HasSets() {
return nil
}
@@ -270,3 +279,89 @@ func (s *WorkspaceStore) GetStats(ctx context.Context, workspaceID string) (*mod
return stats, nil
}
// ── Chunks (v0.21.2) ─────────────────────────
func (s *WorkspaceStore) InsertChunks(ctx context.Context, chunks []models.WorkspaceChunk) error {
if len(chunks) == 0 {
return nil
}
const cols = 8 // workspace_id, file_id, chunk_index, content, token_count, embedding, metadata, created_at
valueParts := make([]string, 0, len(chunks))
args := make([]interface{}, 0, len(chunks)*cols)
for i, c := range chunks {
base := i * cols
valueParts = append(valueParts, fmt.Sprintf(
"($%d, $%d, $%d, $%d, $%d, $%d::vector, $%d, NOW())",
base+1, base+2, base+3, base+4, base+5, base+6, base+7,
))
args = append(args, c.WorkspaceID, c.FileID, c.ChunkIndex,
c.Content, c.TokenCount, vectorToString(c.Embedding), ToJSON(c.Metadata))
}
q := fmt.Sprintf(`INSERT INTO workspace_chunks
(workspace_id, file_id, chunk_index, content, token_count, embedding, metadata, created_at)
VALUES %s`, strings.Join(valueParts, ", "))
_, err := DB.ExecContext(ctx, q, args...)
return err
}
func (s *WorkspaceStore) DeleteChunksByFile(ctx context.Context, fileID string) error {
_, err := DB.ExecContext(ctx, `DELETE FROM workspace_chunks WHERE file_id = $1`, fileID)
return err
}
func (s *WorkspaceStore) SimilaritySearch(ctx context.Context, workspaceID string, queryVec []float64, threshold float64, limit int) ([]models.WorkspaceChunkResult, error) {
if len(queryVec) == 0 {
return nil, nil
}
if limit <= 0 {
limit = 10
}
rows, err := DB.QueryContext(ctx, `
SELECT f.path, c.content,
1 - (c.embedding <=> $1::vector) AS similarity,
c.metadata
FROM workspace_chunks c
JOIN workspace_files f ON c.file_id = f.id
WHERE c.workspace_id = $2
AND 1 - (c.embedding <=> $1::vector) > $3
ORDER BY c.embedding <=> $1::vector
LIMIT $4`,
vectorToString(queryVec), workspaceID, threshold, limit,
)
if err != nil {
return nil, err
}
defer rows.Close()
var results []models.WorkspaceChunkResult
for rows.Next() {
var r models.WorkspaceChunkResult
var metadataJSON []byte
if err := rows.Scan(&r.FilePath, &r.Content, &r.Score, &metadataJSON); err != nil {
return nil, err
}
ScanJSON(metadataJSON, &r.Metadata)
if r.Metadata != nil {
if lh, ok := r.Metadata["line_start"]; ok {
if n, ok := lh.(float64); ok {
r.LineHint = int(n)
}
}
}
results = append(results, r)
}
return results, rows.Err()
}
func (s *WorkspaceStore) UpdateFileIndexStatus(ctx context.Context, fileID, status string, chunkCount int) error {
_, err := DB.ExecContext(ctx,
`UPDATE workspace_files SET index_status = $1, chunk_count = $2, updated_at = NOW() WHERE id = $3`,
status, chunkCount, fileID)
return err
}

157
server/store/sqlite/workspace.go
View File

@@ -3,7 +3,11 @@ package sqlite
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"math"
"sort"
"strings"
"time"
"git.gobha.me/xcaliber/chat-switchboard/models"
@@ -16,8 +20,8 @@ func NewWorkspaceStore() *WorkspaceStore { return &WorkspaceStore{} }
// ── columns ────────────────────────────────
const workspaceCols = `id, owner_type, owner_id, name, root_path, max_bytes, status, created_at, updated_at`
const workspaceFileCols = `id, workspace_id, path, is_directory, content_type, size_bytes, sha256, created_at, updated_at`
const workspaceCols = `id, owner_type, owner_id, name, root_path, max_bytes, status, indexing_enabled, created_at, updated_at`
const workspaceFileCols = `id, workspace_id, path, is_directory, content_type, size_bytes, sha256, index_status, chunk_count, created_at, updated_at`
// ── scanners ───────────────────────────────
@@ -26,7 +30,7 @@ func scanWorkspace(row interface{ Scan(dest ...interface{}) error }) (*models.Wo
var maxBytes sql.NullInt64
err := row.Scan(
&w.ID, &w.OwnerType, &w.OwnerID, &w.Name,
&w.RootPath, &maxBytes, &w.Status,
&w.RootPath, &maxBytes, &w.Status, &w.IndexingEnabled,
st(&w.CreatedAt), st(&w.UpdatedAt),
)
if err != nil {
@@ -42,9 +46,11 @@ func scanWorkspaceFile(row interface{ Scan(dest ...interface{}) error }) (*model
var f models.WorkspaceFile
var contentType sql.NullString
var sha256 sql.NullString
var indexStatus sql.NullString
err := row.Scan(
&f.ID, &f.WorkspaceID, &f.Path, &f.IsDirectory,
&contentType, &f.SizeBytes, &sha256,
&indexStatus, &f.ChunkCount,
st(&f.CreatedAt), st(&f.UpdatedAt),
)
if err != nil {
@@ -56,6 +62,9 @@ func scanWorkspaceFile(row interface{ Scan(dest ...interface{}) error }) (*model
if sha256.Valid {
f.SHA256 = sha256.String
}
if indexStatus.Valid {
f.IndexStatus = indexStatus.String
}
return &f, nil
}
@@ -95,6 +104,9 @@ func (s *WorkspaceStore) Update(ctx context.Context, id string, patch models.Wor
if patch.Status != nil {
b.Set("status", *patch.Status)
}
if patch.IndexingEnabled != nil {
b.Set("indexing_enabled", *patch.IndexingEnabled)
}
if !b.HasSets() {
return nil
}
@@ -272,3 +284,142 @@ func (s *WorkspaceStore) GetStats(ctx context.Context, workspaceID string) (*mod
return stats, nil
}
// ── Chunks (v0.21.2) ─────────────────────────
func (s *WorkspaceStore) InsertChunks(ctx context.Context, chunks []models.WorkspaceChunk) error {
if len(chunks) == 0 {
return nil
}
valueParts := make([]string, 0, len(chunks))
args := make([]interface{}, 0, len(chunks)*8)
for _, c := range chunks {
c.ID = store.NewID()
valueParts = append(valueParts, "(?, ?, ?, ?, ?, ?, ?, ?)")
var embJSON interface{}
if len(c.Embedding) > 0 {
embJSON = ToJSON(c.Embedding)
}
args = append(args, c.ID, c.WorkspaceID, c.FileID, c.ChunkIndex,
c.Content, c.TokenCount, embJSON, ToJSON(c.Metadata))
}
q := fmt.Sprintf(`INSERT INTO workspace_chunks
(id, workspace_id, file_id, chunk_index, content, token_count, embedding, metadata)
VALUES %s`, strings.Join(valueParts, ", "))
_, err := DB.ExecContext(ctx, q, args...)
return err
}
func (s *WorkspaceStore) DeleteChunksByFile(ctx context.Context, fileID string) error {
_, err := DB.ExecContext(ctx, `DELETE FROM workspace_chunks WHERE file_id = ?`, fileID)
return err
}
// SimilaritySearch performs app-level cosine similarity for SQLite.
// Loads all chunks for the workspace, computes cosine distance in Go,
// and returns the top results above threshold.
func (s *WorkspaceStore) SimilaritySearch(ctx context.Context, workspaceID string, queryVec []float64, threshold float64, limit int) ([]models.WorkspaceChunkResult, error) {
if len(queryVec) == 0 {
return nil, nil
}
if limit <= 0 {
limit = 10
}
rows, err := DB.QueryContext(ctx, `
SELECT f.path, c.content, c.embedding, c.metadata
FROM workspace_chunks c
JOIN workspace_files f ON c.file_id = f.id
WHERE c.workspace_id = ?
AND c.embedding IS NOT NULL`,
workspaceID)
if err != nil {
return nil, err
}
defer rows.Close()
type candidate struct {
result models.WorkspaceChunkResult
similarity float64
}
var candidates []candidate
for rows.Next() {
var filePath, content, embJSON string
var metaJSON sql.NullString
if err := rows.Scan(&filePath, &content, &embJSON, &metaJSON); err != nil {
return nil, err
}
var embedding []float64
if err := json.Unmarshal([]byte(embJSON), &embedding); err != nil {
continue // skip malformed
}
sim := wsCosineSimilarity(queryVec, embedding)
if sim > threshold {
r := models.WorkspaceChunkResult{
FilePath: filePath,
Content: content,
Score: sim,
}
if metaJSON.Valid {
ScanJSON(metaJSON.String, &r.Metadata)
}
if r.Metadata != nil {
if lh, ok := r.Metadata["line_start"]; ok {
if n, ok := lh.(float64); ok {
r.LineHint = int(n)
}
}
}
candidates = append(candidates, candidate{result: r, similarity: sim})
}
}
if err := rows.Err(); err != nil {
return nil, err
}
sort.Slice(candidates, func(i, j int) bool {
return candidates[i].similarity > candidates[j].similarity
})
if len(candidates) > limit {
candidates = candidates[:limit]
}
results := make([]models.WorkspaceChunkResult, len(candidates))
for i, c := range candidates {
results[i] = c.result
}
return results, nil
}
func (s *WorkspaceStore) UpdateFileIndexStatus(ctx context.Context, fileID, status string, chunkCount int) error {
_, err := DB.ExecContext(ctx,
`UPDATE workspace_files SET index_status = ?, chunk_count = ?, updated_at = datetime('now') WHERE id = ?`,
status, chunkCount, fileID)
return err
}
// wsCosineSimilarity computes cosine similarity between two vectors.
// Named to avoid collision with the KB package-level function.
func wsCosineSimilarity(a, b []float64) float64 {
if len(a) != len(b) || len(a) == 0 {
return 0
}
var dot, normA, normB float64
for i := range a {
dot += a[i] * b[i]
normA += a[i] * a[i]
normB += b[i] * b[i]
}
if normA == 0 || normB == 0 {
return 0
}
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
}