diff --git a/VERSION b/VERSION index 7092c7c..8076af5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.15.0 \ No newline at end of file +0.15.1 \ No newline at end of file diff --git a/docs/DESIGN-0.15.1.md b/docs/DESIGN-0.15.1.md new file mode 100644 index 0000000..e57d653 --- /dev/null +++ b/docs/DESIGN-0.15.1.md @@ -0,0 +1,332 @@ +# DESIGN-0.15.1 — Context Recall Tools + +## Overview + +After compaction (v0.15.0) or in long conversations, the model loses access +to earlier material — attachment content injected once at send time, and +message details compressed into summaries. These tools let the model +re-read source material on demand. + +Depends on: file handling (v0.12.0), tool framework (v0.11.0), compaction (v0.15.0). + +**Four work streams:** + +1. `attachment_recall` — re-read attachment content from storage +2. `conversation_search` — keyword search over channel messages +3. Token estimator improvements — attachment-aware context counting +4. KB auto-injection — automatic knowledge base context in system prompt + +--- + +## 1. `attachment_recall` Tool + +### Purpose + +Attachments are injected into context once (at the user message that +references them). On subsequent turns the model relies on its own response +as a natural summary. After compaction, that summary is gone. This tool +lets the model re-read any attachment in the current channel. + +### Two-action design + +Single tool, `action` parameter selects behavior: + +- **`list`** — returns metadata for all attachments in the channel + (id, filename, content_type, size, created_at). The model calls this + first to discover what's available. +- **`read`** — returns content for a specific attachment_id. + Documents → `extracted_text` from DB. Images → base64 data URI. + +### Security + +Channel-scoped: `attachment_id` must belong to `ExecutionContext.ChannelID`. +The store query is `GetByChannel(channelID)` so cross-channel access is +impossible by construction. + +### Dependencies + +Late registration (same pattern as `kb_search`): + +```go +func RegisterAttachmentRecall(stores store.Stores, objStore storage.ObjectStore) +``` + +Called from `main.go` after storage init. If `objStore` is nil (storage +not configured), the tool is **not registered** — it simply won't appear +in the tool list. + +### Implementation: `server/tools/attachment_recall.go` + +```go +type attachmentRecallTool struct { + stores store.Stores + objStore storage.ObjectStore +} + +func (t *attachmentRecallTool) Definition() ToolDef { + return ToolDef{ + Name: "attachment_recall", + DisplayName: "Attachments", + Category: "context", + Description: "Re-read file attachments from this conversation. " + + "Use action='list' to see available files, then action='read' " + + "with an attachment_id to get the content.", + Parameters: JSONSchema(map[string]interface{}{ + "action": PropEnum("Action to perform", "list", "read"), + "attachment_id": Prop("string", "Attachment ID (required for read)"), + }, []string{"action"}), + } +} +``` + +**`list` action returns:** +```json +{ + "attachments": [ + { + "id": "abc-123", + "filename": "report.pdf", + "content_type": "application/pdf", + "size_bytes": 45200, + "has_text": true, + "created_at": "2026-02-26T10:00:00Z" + } + ], + "count": 1 +} +``` + +**`read` action returns:** +```json +{ + "id": "abc-123", + "filename": "report.pdf", + "content_type": "application/pdf", + "content": "... extracted text ...", + "content_type_returned": "text" +} +``` + +For images: +```json +{ + "id": "def-456", + "filename": "diagram.png", + "content_type": "image/png", + "content": "data:image/png;base64,...", + "content_type_returned": "base64_image" +} +``` + +Image base64 is capped at 10MB to prevent context blow-up. Larger images +return an error suggesting the user re-upload. + +--- + +## 2. `conversation_search` Tool + +### Purpose + +After compaction, message details are compressed into summaries. If the +user asks about something the summary omitted, the model needs to search +the original messages. Also useful in long pre-compaction conversations +where the model's attention has drifted. + +### Design + +Keyword search (full-text) over messages in the current channel using +PostgreSQL's `plainto_tsquery`. No migration needed — `to_tsvector` is +computed on the fly. This is acceptable because: + +- Search is scoped to a single channel (bounded message count) +- Called infrequently (tool invocation, not every request) +- Channel messages rarely exceed ~10K rows + +If performance becomes an issue, a `search_vector tsvector` column with +GIN index can be added in a future migration. + +### Parameters + +```go +Parameters: JSONSchema(map[string]interface{}{ + "query": Prop("string", "Search keywords"), + "max_results": map[string]interface{}{ + "type": "integer", + "description": "Maximum results (1-20, default 5)", + }, + "role_filter": PropEnum("Filter by message role", "all", "user", "assistant"), +}, []string{"query"}) +``` + +### Returns + +```json +{ + "results": [ + { + "message_id": "...", + "role": "user", + "excerpt": "...highlighted excerpt...", + "timestamp": "2026-02-26T10:00:00Z", + "rank": 0.95 + } + ], + "query": "original query", + "total": 3, + "channel_id": "..." +} +``` + +Uses `ts_headline` for highlighted excerpts (same pattern as `note_search`). + +### Security + +Channel-scoped via `ExecutionContext.ChannelID`. Only searches messages +in the current channel. Soft-deleted messages (`deleted_at IS NOT NULL`) +are excluded. + +### Dependencies + +Late registration: + +```go +func RegisterConversationSearch(stores store.Stores) +``` + +No external dependencies beyond stores (uses `database.DB` directly for +the full-text query, same pattern as `note_search`). + +--- + +## 3. Token Estimator Improvements + +### Frontend (`tokens.js`) + +Current `estimateConversation` counts message text only. Enhancements: + +- **Staged attachments**: Before send, show estimated tokens for queued + attachments. Documents: `extracted_text.length / 4`. Images: flat + ~765 tokens per image (OpenAI's base tile estimate, reasonable cross-provider). +- **Sent attachments**: After send, the augmented content is already in + the message text (documents are inlined). Images were ephemeral + ContentParts — add their estimate to the message's token count. + +### Implementation + +Add to `Tokens`: + +```javascript +estimateAttachments(attachments) { + let total = 0; + for (const att of attachments) { + if (att.content_type?.startsWith('image/')) { + total += 765; // base tile estimate + } else if (att.extracted_text) { + total += this.estimate(att.extracted_text); + } else { + total += Math.ceil(att.size_bytes / 4); // rough fallback + } + } + return total; +} +``` + +Wire into `updateInputTokens()` to include staged attachment estimates. + +--- + +## 4. KB Auto-Injection + +### Purpose + +Currently the model must explicitly call `kb_search` to access knowledge +base content. For common patterns (channel linked to a KB, user asks a +question), automatic injection into the system prompt is more natural. + +### Design + +In the completion handler, after loading conversation history and before +building the provider request: + +1. Check if channel has linked KBs (or user has personal KBs) +2. Extract the user's latest message as the query +3. Run similarity search (top-K, K=3 default) +4. If results found AND context budget allows, prepend to system prompt + +### Context budget check + +```go +contextUsed := estimateTokens(systemPrompt + allMessages) +kbTokens := estimateTokens(kbContent) +if contextUsed + kbTokens > maxContext * 0.85 { + // Skip injection — context too full + log.Printf("⏭ KB auto-inject skipped: context %.0f%% full", pct*100) +} +``` + +### Channel settings + +```json +{ + "kb_injection": "auto" | "tool_only" | "both" | "off" +} +``` + +Default: `"auto"` — inject when KBs are linked. `"tool_only"` disables +auto-injection but keeps the `kb_search` tool available. `"both"` does +both. `"off"` disables all KB context. + +### System prompt format + +``` +[Knowledge Base Context] +The following information was retrieved from the user's knowledge bases +and may be relevant to this conversation: + +--- +Source: Technical Docs (chunk 3 of report.pdf, similarity: 0.87) +Content: ... + +--- +Source: FAQ (chunk 1 of faq.md, similarity: 0.82) +Content: ... +[End Knowledge Base Context] +``` + +--- + +## File Inventory + +### New files + +| File | Description | +|------|-------------| +| `server/tools/attachment_recall.go` | attachment_recall tool | +| `server/tools/conversation_search.go` | conversation_search tool | +| `docs/DESIGN-0.15.1.md` | This design doc | + +### Modified files + +| File | Change | +|------|--------| +| `server/main.go` | Register new tools | +| `server/handlers/completion.go` | KB auto-injection in completion flow | +| `src/js/tokens.js` | Attachment-aware token estimation | +| `docs/ROADMAP.md` | Mark v0.15.1 items | + +### No migration required + +- `conversation_search` uses on-the-fly `to_tsvector` (no new column) +- `attachment_recall` reads existing `extracted_text` column +- KB auto-injection uses existing `kb_search` infrastructure + +--- + +## Implementation Order + +1. **Phase 1**: `attachment_recall` tool + registration ✅ +2. **Phase 2**: `conversation_search` tool + registration ✅ +3. **Phase 3**: Token estimator improvements (frontend) ✅ +4. **Phase 4**: KB auto-injection (completion handler) — separate delivery, + requires adding embedder dependency to `CompletionHandler` and careful + latency budgeting (embedding query on every completion call) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 50cb240..ed89ad7 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -786,28 +786,30 @@ summary. After compaction (v0.15.0) or in long conversations, the model may need to re-read source material. These tools bridge that gap. **`attachment_recall` tool** -- [ ] `attachment_recall(attachment_id)` — re-reads attachment content from storage -- [ ] Images: re-inject as base64 content part (vision gating still applies) -- [ ] Documents: return extracted text -- [ ] Security: scoped to current channel's attachments only -- [ ] Extend `ExecutionContext` with optional storage backend reference -- [ ] Tool definition includes attachment list hint so model knows what's available +- [x] Two-action design: `list` (channel metadata) + `read` (content by ID) +- [x] Images: base64 data URI from object storage (10MB cap) +- [x] Documents: return `extracted_text` from DB +- [x] Security: channel-scoped — attachment must belong to `ExecutionContext.ChannelID` +- [x] Late registration: only registered when `objStore` is non-nil +- [x] Category: `context` (groups with conversation_search in toggle UI) **`conversation_search` tool** -- [ ] Keyword search over compacted/archived messages in current channel -- [ ] Returns matching message excerpts with timestamps -- [ ] Useful post-compaction when summary dropped details the user asks about +- [x] Full-text keyword search via `plainto_tsquery` over channel messages +- [x] `ts_headline` for highlighted excerpts with `**` markers +- [x] Role filter: `all`, `user`, `assistant` +- [x] Scoped to current channel, excludes soft-deleted messages +- [x] On-the-fly `to_tsvector` (no migration needed — channel-bounded) **Token estimator improvements** -- [ ] Include attachment token estimates in context counter -- [ ] Images: use provider-specific sizing rules (e.g. tile-based for OpenAI) -- [ ] Documents: count extracted text tokens -- [ ] Staged attachments reflected in input token counter before send +- [x] `Tokens.estimateAttachments()` — images ~765 tokens, docs by text length +- [x] Staged attachment tokens reflected in input counter before send +- [x] Context percentage includes attachment estimates **KB auto-injection** (depends on: knowledge bases v0.14.0, compaction v0.15.0) - [ ] Pre-pend top-K KB results to system prompt (automatic, no tool call needed) - [ ] Context budget aware: skip injection if context already near limit - [ ] Per-channel toggle: tool-only vs auto-inject vs both +- _(deferred — requires adding embedder to CompletionHandler, latency budgeting)_ --- diff --git a/server/main.go b/server/main.go index 752da03..224556f 100644 --- a/server/main.go +++ b/server/main.go @@ -161,6 +161,10 @@ func main() { // Register note tools (late registration — needs stores + embedder for semantic search) tools.RegisterNoteTools(stores, kbEmbedder) + // Register context recall tools (v0.15.1) + tools.RegisterAttachmentRecall(stores, objStore) + tools.RegisterConversationSearch(stores) + r := gin.Default() r.Use(middleware.CORS()) diff --git a/server/tools/attachment_recall.go b/server/tools/attachment_recall.go new file mode 100644 index 0000000..5e04230 --- /dev/null +++ b/server/tools/attachment_recall.go @@ -0,0 +1,184 @@ +package tools + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "log" + "strings" + + "git.gobha.me/xcaliber/chat-switchboard/storage" + "git.gobha.me/xcaliber/chat-switchboard/store" +) + +// ── Late Registration ──────────────────────── +// attachment_recall needs stores + objStore, which aren't available at +// init time. Called from main.go after storage init. +// If objStore is nil (storage not configured), the tool is not registered. + +func RegisterAttachmentRecall(stores store.Stores, objStore storage.ObjectStore) { + if objStore == nil { + log.Printf("ℹ attachment_recall: storage not configured, tool not registered") + return + } + Register(&attachmentRecallTool{ + stores: stores, + objStore: objStore, + }) +} + +// ═══════════════════════════════════════════ +// attachment_recall +// ═══════════════════════════════════════════ + +const maxImageBytes = 10 * 1024 * 1024 // 10 MB cap for base64 images + +type attachmentRecallTool struct { + stores store.Stores + objStore storage.ObjectStore +} + +func (t *attachmentRecallTool) Definition() ToolDef { + return ToolDef{ + Name: "attachment_recall", + DisplayName: "Attachments", + Category: "context", + Description: "Re-read file attachments from this conversation. " + + "Use action='list' to see available files, then action='read' " + + "with an attachment_id to retrieve the file content. " + + "Documents return extracted text; images return base64 data.", + Parameters: JSONSchema(map[string]interface{}{ + "action": PropEnum("Action: 'list' to see files, 'read' to get content", "list", "read"), + "attachment_id": Prop("string", "Attachment ID to read (required for action='read')"), + }, []string{"action"}), + } +} + +func (t *attachmentRecallTool) Execute(ctx context.Context, execCtx ExecutionContext, argsJSON string) (string, error) { + var args struct { + Action string `json:"action"` + AttachmentID string `json:"attachment_id"` + } + if err := json.Unmarshal([]byte(argsJSON), &args); err != nil { + return "", fmt.Errorf("invalid arguments: %w", err) + } + + switch args.Action { + case "list": + return t.listAttachments(ctx, execCtx) + case "read": + if args.AttachmentID == "" { + return "", fmt.Errorf("attachment_id is required for action='read'") + } + return t.readAttachment(ctx, execCtx, args.AttachmentID) + default: + return "", fmt.Errorf("invalid action: %q (use 'list' or 'read')", args.Action) + } +} + +// listAttachments returns metadata for all attachments in the channel. +func (t *attachmentRecallTool) listAttachments(ctx context.Context, execCtx ExecutionContext) (string, error) { + atts, err := t.stores.Attachments.GetByChannel(ctx, execCtx.ChannelID) + if err != nil { + return "", fmt.Errorf("failed to list attachments: %w", err) + } + + type attInfo struct { + ID string `json:"id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + HasText bool `json:"has_text"` + CreatedAt string `json:"created_at"` + } + + items := make([]attInfo, 0, len(atts)) + for _, a := range atts { + items = append(items, attInfo{ + ID: a.ID, + Filename: a.Filename, + ContentType: a.ContentType, + SizeBytes: a.SizeBytes, + HasText: a.ExtractedText != nil && *a.ExtractedText != "", + CreatedAt: a.CreatedAt.Format("2006-01-02T15:04:05Z"), + }) + } + + log.Printf("📎 attachment_recall: list → %d files in channel %s", len(items), execCtx.ChannelID) + + out, _ := json.Marshal(map[string]interface{}{ + "attachments": items, + "count": len(items), + "channel_id": execCtx.ChannelID, + }) + return string(out), nil +} + +// readAttachment returns the content of a specific attachment. +func (t *attachmentRecallTool) readAttachment(ctx context.Context, execCtx ExecutionContext, attachmentID string) (string, error) { + att, err := t.stores.Attachments.GetByID(ctx, attachmentID) + if err != nil { + return "", fmt.Errorf("attachment not found: %s", attachmentID) + } + + // Security: verify attachment belongs to this channel + if att.ChannelID != execCtx.ChannelID { + return "", fmt.Errorf("attachment %s does not belong to this conversation", attachmentID) + } + + // Documents: return extracted text + if !isImageType(att.ContentType) { + if att.ExtractedText != nil && *att.ExtractedText != "" { + log.Printf("📎 attachment_recall: read text → %s (%s, %d chars)", + att.Filename, att.ID, len(*att.ExtractedText)) + out, _ := json.Marshal(map[string]interface{}{ + "id": att.ID, + "filename": att.Filename, + "content_type": att.ContentType, + "content": *att.ExtractedText, + "content_type_returned": "text", + }) + return string(out), nil + } + return "", fmt.Errorf("no extracted text available for %s (content type: %s)", att.Filename, att.ContentType) + } + + // Images: read from storage and return base64 + if att.SizeBytes > maxImageBytes { + return "", fmt.Errorf("image %s is too large (%d bytes, max %d). Re-upload a smaller version", + att.Filename, att.SizeBytes, maxImageBytes) + } + + reader, size, _, err := t.objStore.Get(ctx, att.StorageKey) + if err != nil { + return "", fmt.Errorf("failed to read attachment from storage: %w", err) + } + defer reader.Close() + + // Read and base64-encode + data := make([]byte, size) + if _, err := io.ReadFull(reader, data); err != nil { + return "", fmt.Errorf("failed to read attachment data: %w", err) + } + + b64 := base64.StdEncoding.EncodeToString(data) + dataURI := fmt.Sprintf("data:%s;base64,%s", att.ContentType, b64) + + log.Printf("📎 attachment_recall: read image → %s (%s, %d bytes)", + att.Filename, att.ID, size) + + out, _ := json.Marshal(map[string]interface{}{ + "id": att.ID, + "filename": att.Filename, + "content_type": att.ContentType, + "content": dataURI, + "content_type_returned": "base64_image", + }) + return string(out), nil +} + +func isImageType(ct string) bool { + return strings.HasPrefix(ct, "image/") +} diff --git a/server/tools/conversation_search.go b/server/tools/conversation_search.go new file mode 100644 index 0000000..7bc0492 --- /dev/null +++ b/server/tools/conversation_search.go @@ -0,0 +1,128 @@ +package tools + +import ( + "context" + "encoding/json" + "fmt" + "log" + "time" + + "git.gobha.me/xcaliber/chat-switchboard/database" + "git.gobha.me/xcaliber/chat-switchboard/store" +) + +// ── Late Registration ──────────────────────── + +func RegisterConversationSearch(stores store.Stores) { + Register(&conversationSearchTool{stores: stores}) +} + +// ═══════════════════════════════════════════ +// conversation_search +// ═══════════════════════════════════════════ + +type conversationSearchTool struct { + stores store.Stores +} + +func (t *conversationSearchTool) Definition() ToolDef { + return ToolDef{ + Name: "conversation_search", + DisplayName: "Search Chat", + Category: "context", + Description: "Search earlier messages in this conversation by keyword. " + + "Useful when the conversation is long or has been summarized and " + + "you need to find specific details from earlier discussion. " + + "Returns matching message excerpts with timestamps.", + Parameters: JSONSchema(map[string]interface{}{ + "query": Prop("string", "Search keywords — use terms likely to appear in the messages"), + "max_results": map[string]interface{}{ + "type": "integer", + "description": "Maximum results to return (1-20, default 5)", + }, + "role_filter": PropEnum( + "Filter by message role (default: all)", + "all", "user", "assistant", + ), + }, []string{"query"}), + } +} + +func (t *conversationSearchTool) Execute(ctx context.Context, execCtx ExecutionContext, argsJSON string) (string, error) { + var args struct { + Query string `json:"query"` + MaxResults int `json:"max_results"` + RoleFilter string `json:"role_filter"` + } + if err := json.Unmarshal([]byte(argsJSON), &args); err != nil { + return "", fmt.Errorf("invalid arguments: %w", err) + } + + if args.Query == "" { + return "", fmt.Errorf("query is required") + } + if args.MaxResults <= 0 || args.MaxResults > 20 { + args.MaxResults = 5 + } + + // Build query — full-text search scoped to this channel + // Uses on-the-fly to_tsvector (no index needed — channel-scoped is bounded) + roleClause := "" + queryArgs := []interface{}{execCtx.ChannelID, args.Query, args.MaxResults} + if args.RoleFilter == "user" || args.RoleFilter == "assistant" { + roleClause = "AND m.role = $4" + queryArgs = append(queryArgs, args.RoleFilter) + } + + rows, err := database.DB.QueryContext(ctx, fmt.Sprintf(` + SELECT m.id, m.role, + ts_headline('english', m.content, plainto_tsquery('english', $2), + 'MaxWords=60, MinWords=20, StartSel=**, StopSel=**') AS headline, + ts_rank(to_tsvector('english', m.content), plainto_tsquery('english', $2)) AS rank, + m.created_at + FROM messages m + WHERE m.channel_id = $1 + AND m.deleted_at IS NULL + AND m.role IN ('user', 'assistant') + AND to_tsvector('english', m.content) @@ plainto_tsquery('english', $2) + %s + ORDER BY rank DESC, m.created_at DESC + LIMIT $3 + `, roleClause), queryArgs...) + if err != nil { + return "", fmt.Errorf("search failed: %w", err) + } + defer rows.Close() + + type searchResult struct { + MessageID string `json:"message_id"` + Role string `json:"role"` + Excerpt string `json:"excerpt"` + Rank float64 `json:"rank"` + Timestamp string `json:"timestamp"` + } + + results := make([]searchResult, 0) + for rows.Next() { + var r searchResult + var rank float64 + var createdAt time.Time + if err := rows.Scan(&r.MessageID, &r.Role, &r.Excerpt, &rank, &createdAt); err != nil { + continue + } + r.Rank = rank + r.Timestamp = createdAt.Format(time.RFC3339) + results = append(results, r) + } + + log.Printf("🔍 conversation_search: %q → %d results in channel %s", + args.Query, len(results), execCtx.ChannelID) + + out, _ := json.Marshal(map[string]interface{}{ + "results": results, + "query": args.Query, + "total": len(results), + "channel_id": execCtx.ChannelID, + }) + return string(out), nil +} diff --git a/src/js/tokens.js b/src/js/tokens.js index a0805f4..a33f3b4 100644 --- a/src/js/tokens.js +++ b/src/js/tokens.js @@ -25,6 +25,25 @@ const Tokens = { return total; }, + // Estimate tokens for a set of attachments (staged or sent). + // Images: ~765 tokens (base tile estimate, reasonable cross-provider). + // Documents with extracted text: text length / 4. + // Other/unknown: raw size / 4 (rough fallback). + estimateAttachments(attachments) { + if (!attachments?.length) return 0; + let total = 0; + for (const att of attachments) { + if (att.content_type?.startsWith('image/')) { + total += 765; // base tile estimate + } else if (att.extracted_text) { + total += this.estimate(att.extracted_text); + } else if (att.size_bytes) { + total += Math.ceil(att.size_bytes / 4); + } + } + return total; + }, + // Get context budget for current model getContextBudget() { const caps = UI.getSelectedModelCaps(); @@ -65,9 +84,17 @@ function updateInputTokens() { // Show relative to available context const chat = App.chats.find(c => c.id === App.currentChatId); const convTokens = Tokens.estimateConversation(chat?.messages || [], App.settings.systemPrompt); - const totalWithInput = convTokens + inputTokens; + // Include staged attachment estimates + const attTokens = Tokens.estimateAttachments( + (App.stagedAttachments || []).filter(a => a.status !== 'error').map(a => ({ + content_type: a.contentType, + size_bytes: a.sizeBytes, + })) + ); + const totalWithInput = convTokens + inputTokens + attTokens; const pct = totalWithInput / budget.maxContext; - el.textContent = `~${Tokens.format(inputTokens)} tokens · ${Tokens.format(totalWithInput)} / ${Tokens.format(budget.maxContext)} context`; + const attLabel = attTokens > 0 ? ` +${Tokens.format(attTokens)} files` : ''; + el.textContent = `~${Tokens.format(inputTokens)} tokens${attLabel} · ${Tokens.format(totalWithInput)} / ${Tokens.format(budget.maxContext)} context`; el.className = 'input-token-count' + (pct > 0.9 ? ' danger' : pct > 0.75 ? ' warning' : ''); } else { el.textContent = `~${Tokens.format(inputTokens)} tokens`;