From e4a943b03e243386fe5d9c7c0118b82babeb348b Mon Sep 17 00:00:00 2001 From: xcaliber Date: Sat, 28 Feb 2026 18:24:19 +0000 Subject: [PATCH] Changeset 0.18.0 (#79) --- .gitea/workflows/ci.yaml | 171 +++--- CHANGELOG.md | 84 +++ VERSION | 2 +- docs/ARCHITECTURE.md | 16 +- docs/CHANGES-0.15.0-phase1.md | 93 --- docs/CHANGES-0.15.0-phase2.md | 186 ------ docs/CHANGES-0.15.0-phase4.md | 106 ---- docs/CHANGES-0.18.0-phase1.md | 99 ++++ docs/CHANGES-0.18.0-phase2.md | 191 +++++++ docs/CHANGES-0.18.0-phase3.md | 283 ++++++++++ docs/DESIGN-0.18.0.md | 235 ++++++++ docs/REFACTOR-0.10.3.md | 530 ------------------ docs/ROADMAP.md | 64 ++- docs/UI-AUDIT-0.10.5.md | 358 ------------ .../migrations/004_v0180_memories.sql | 85 +++ .../migrations/005_v0180_memory_phase2.sql | 46 ++ .../migrations/sqlite/003_v0180_memories.sql | 61 ++ .../sqlite/004_v0180_memory_phase2.sql | 22 + server/handlers/completion.go | 17 +- server/handlers/integration_test.go | 2 +- server/handlers/memory.go | 223 ++++++++ server/handlers/memory_inject.go | 100 ++++ server/handlers/messages.go | 2 +- server/main.go | 24 +- server/memory/extractor.go | 261 +++++++++ server/memory/scanner.go | 197 +++++++ server/models/models.go | 6 + server/models/models_memory.go | 48 ++ server/models/models_memory_persona.go | 18 + server/store/interfaces.go | 1 + server/store/postgres/memory.go | 269 +++++++++ server/store/postgres/memory_hybrid.go | 135 +++++ server/store/postgres/persona.go | 17 +- server/store/postgres/stores.go | 1 + server/store/sqlite/memory.go | 224 ++++++++ server/store/sqlite/memory_hybrid.go | 146 +++++ server/store/sqlite/persona.go | 17 +- server/store/sqlite/stores.go | 1 + server/store/store_memory.go | 47 ++ server/store/store_memory_hybrid.go | 15 + server/tools/memory.go | 238 ++++++++ src/css/memory.css | 298 ++++++++++ src/index.html | 23 + src/js/api.js | 16 + src/js/memory-ui.js | 364 ++++++++++++ src/js/settings-handlers.js | 9 + src/js/ui-admin.js | 22 +- src/js/ui-core.js | 24 + 48 files changed, 3999 insertions(+), 1398 deletions(-) delete mode 100644 docs/CHANGES-0.15.0-phase1.md delete mode 100644 docs/CHANGES-0.15.0-phase2.md delete mode 100644 docs/CHANGES-0.15.0-phase4.md create mode 100644 docs/CHANGES-0.18.0-phase1.md create mode 100644 docs/CHANGES-0.18.0-phase2.md create mode 100644 docs/CHANGES-0.18.0-phase3.md create mode 100644 docs/DESIGN-0.18.0.md delete mode 100644 docs/REFACTOR-0.10.3.md delete mode 100644 docs/UI-AUDIT-0.10.5.md create mode 100644 server/database/migrations/004_v0180_memories.sql create mode 100644 server/database/migrations/005_v0180_memory_phase2.sql create mode 100644 server/database/migrations/sqlite/003_v0180_memories.sql create mode 100644 server/database/migrations/sqlite/004_v0180_memory_phase2.sql create mode 100644 server/handlers/memory.go create mode 100644 server/handlers/memory_inject.go create mode 100644 server/memory/extractor.go create mode 100644 server/memory/scanner.go create mode 100644 server/models/models_memory.go create mode 100644 server/models/models_memory_persona.go create mode 100644 server/store/postgres/memory.go create mode 100644 server/store/postgres/memory_hybrid.go create mode 100644 server/store/sqlite/memory.go create mode 100644 server/store/sqlite/memory_hybrid.go create mode 100644 server/store/store_memory.go create mode 100644 server/store/store_memory_hybrid.go create mode 100644 server/tools/memory.go create mode 100644 src/css/memory.css create mode 100644 src/js/memory-ui.js diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index f1c491e..25e1e44 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -1,16 +1,24 @@ # .gitea/workflows/ci.yaml # ============================================ -# Chat Switchboard - CI/CD Pipeline (v0.17.2) +# Chat Switchboard - CI/CD Pipeline (v0.17.3) # ============================================ # Cluster deployments use SEPARATE FE + BE images. # Unified image is for Docker Hub only (docker-compose use). # # Pipeline: # 0. Detect changes (path-based gating for all downstream jobs) -# 1a. Frontend tests — skipped if only BE/docs changed -# 1b. Go test (PG) — skipped if only FE/docs changed -# 1c. Go test (SQLite) — skipped if only FE/docs changed -# 2. Build + Deploy — skipped if docs-only change +# 1a. Frontend tests — skipped if only BE/docs changed +# 1b. Go unit tests — all non-DB packages + SQLite integration (race-enabled) +# 1c. Go test (PG) — PG store + handlers against Postgres (race-enabled) +# 2. Build + Deploy — skipped if docs-only change +# +# Test coverage mapping (no package tested by zero jobs): +# Unit packages (auto-discovered) → test-sqlite (race) +# ./handlers/... → test-sqlite (SQLite driver, race) +# + test-go-pg (PG driver, race) +# ./store/sqlite/... → test-sqlite (race) +# ./store/postgres/... → test-go-pg (race) +# CGO_ENABLED=0 build check → test-sqlite # # Path gating rules: # src/, src/editor/ → frontend tests @@ -202,11 +210,81 @@ jobs: - name: Run frontend tests run: node --test src/js/__tests__/*.test.js - # ── Stage 1b: Go Build & Test (Postgres) ───── + # ── Stage 1b: Go Unit Tests + SQLite Integration ───── + # Covers ALL non-Postgres packages via auto-discovery, plus + # SQLite handler and store integration tests. Race-enabled. + # + # Coverage: unit packages (dynamic), ./handlers/... (SQLite), + # ./store/sqlite/..., CGO_ENABLED=0 build check. # # Runs when: backend files changed or infra changed. # Skipped when: only frontend or docs changed. - test: + test-sqlite: + runs-on: ubuntu-latest + needs: [detect-changes] + if: needs.detect-changes.outputs.backend == 'true' || needs.detect-changes.outputs.infra == 'true' + env: + GOPRIVATE: git.gobha.me/* + GONOSUMCHECK: git.gobha.me/* + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: Download and tidy + working-directory: server + run: | + go mod download + go mod tidy + + - name: Build check (CGO_ENABLED=0) + working-directory: server + run: | + echo "━━━ SQLite Backend Build Check ━━━" + CGO_ENABLED=0 go build -o /dev/null . + echo "✓ Binary compiles with SQLite backend (pure Go, no CGO)" + + - name: Run unit tests (auto-discovered, no DB) + working-directory: server + run: | + echo "━━━ Unit Tests (all non-DB packages) ━━━" + + # Auto-discover: everything except store/* and handlers/* + # This ensures new packages are never silently untested + UNIT_PKGS=$(go list ./... | grep -v -E '/(store|handlers)(/|$)') + + echo "Packages under test:" + echo "${UNIT_PKGS}" | sed 's/^/ /' + echo "" + + go test -v -race -count=1 -p 1 ${UNIT_PKGS} + echo "✓ Unit tests complete" + + - name: Run SQLite handler integration tests + working-directory: server + env: + DB_DRIVER: sqlite + run: | + echo "━━━ SQLite Integration Tests (handlers + stores) ━━━" + go test -v -race -count=1 -p 1 \ + ./handlers/... \ + ./store/sqlite/... + echo "✓ SQLite integration tests complete" + + # ── Stage 1c: Go Test (Postgres) ───────────── + # Tests ONLY Postgres-dependent packages: store/postgres and + # handlers (with PG driver). Runs in parallel with test-sqlite. + # + # Coverage: ./store/postgres/..., ./handlers/... (PG driver). + # Handlers are intentionally tested against BOTH drivers. + # + # Runs when: backend files changed or infra changed. + # Skipped when: only frontend or docs changed. + test-go-pg: runs-on: ubuntu-latest needs: [detect-changes] if: needs.detect-changes.outputs.backend == 'true' || needs.detect-changes.outputs.infra == 'true' @@ -264,14 +342,19 @@ jobs: psql -d "${DB_NAME}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO ${APP_USER};" echo "✓ CI test database ready" - - name: Run tests + - name: Run Postgres integration tests working-directory: server env: PGHOST: ${{ env.POSTGRES_HOST }} PGPORT: ${{ env.POSTGRES_PORT }} PGUSER: ${{ secrets.POSTGRES_USER }} PGPASSWORD: ${{ secrets.POSTGRES_PASSWORD }} - run: go test -v -race -count=1 -p 1 ./... + run: | + echo "━━━ Postgres Integration Tests ━━━" + go test -v -race -count=1 -p 1 \ + ./store/postgres/... \ + ./handlers/... + echo "✓ Postgres integration tests complete" - name: Drop CI test database if: always() @@ -284,72 +367,6 @@ jobs: psql -c "DROP DATABASE IF EXISTS chat_switchboard_ci;" postgres echo "✓ Dropped CI test database" - - name: Build check - working-directory: server - run: CGO_ENABLED=0 go build -o /dev/null . - - # ── Stage 1c: Go Build & Test (SQLite) ─────── - # Verifies SQLite backend compiles, stores work, and handler - # integration tests pass against an in-memory SQLite database. - # No external database or provider keys required. - # - # Runs when: backend files changed or infra changed. - # Skipped when: only frontend or docs changed. - test-sqlite: - runs-on: ubuntu-latest - needs: [detect-changes] - if: needs.detect-changes.outputs.backend == 'true' || needs.detect-changes.outputs.infra == 'true' - env: - GOPRIVATE: git.gobha.me/* - GONOSUMCHECK: git.gobha.me/* - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: '1.22' - - - name: Download and tidy - working-directory: server - run: | - go mod download - go mod tidy - - - name: Build check (CGO_ENABLED=0) - working-directory: server - run: | - echo "━━━ SQLite Backend Build Check ━━━" - CGO_ENABLED=0 go build -o /dev/null . - echo "✓ Binary compiles with SQLite backend (pure Go, no CGO)" - - - name: Run unit tests (no DB) - working-directory: server - run: | - echo "━━━ Unit Tests (no external DB) ━━━" - go test -v -count=1 -p 1 \ - ./capabilities/... \ - ./compaction/... \ - ./crypto/... \ - ./events/... \ - ./extraction/... \ - ./knowledge/... \ - ./providers/... \ - ./tools/... - echo "✓ Unit tests complete" - - - name: Run SQLite handler integration tests - working-directory: server - env: - DB_DRIVER: sqlite - run: | - echo "━━━ SQLite Integration Tests (handlers + stores) ━━━" - go test -v -count=1 -p 1 \ - ./handlers/... \ - ./store/sqlite/... - echo "✓ SQLite integration tests complete" - # ── Stage 2: Build, Database, Deploy ───────── # # Depends on all test jobs. Skipped jobs (due to path gating) @@ -358,7 +375,7 @@ jobs: # Skipped entirely for docs-only changes (nothing to build). build-and-deploy: runs-on: ubuntu-latest - needs: [detect-changes, test, test-frontend, test-sqlite] + needs: [detect-changes, test-go-pg, test-frontend, test-sqlite] # Run unless: a needed job failed, the workflow was cancelled, or it's docs-only. # Skipped test jobs (path-gated) are fine — they don't block. if: | @@ -773,4 +790,4 @@ jobs: fi if [[ "${{ steps.env.outputs.is_release }}" == "true" ]]; then echo "| **Docker Hub** | \`${DOCKERHUB_IMAGE}:${{ steps.env.outputs.EXTRA_TAG }}\` (unified) |" >> $GITHUB_STEP_SUMMARY - fi + fi \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 7525c25..1fc6160 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,90 @@ All notable changes to Chat Switchboard. +## [0.18.0] — 2026-02-28 + +### Added +- **Memory system.** Long-term memory across conversations with scope-aware + isolation. Three memory scopes: `user` (personal facts/preferences), + `persona` (shared across all users of a Persona), and `persona_user` + (per-user within a Persona context, e.g. tutoring progress per student). + Memories persist across channels and are injected into the system prompt + at completion time with scope priority (persona_user > persona > user). +- **Memory tools.** Two new LLM-callable tools: + - `memory_save` — LLM explicitly stores a fact with key/value/confidence. + Scope-aware: saves to the active scope for the current Persona context. + - `memory_recall` — LLM queries stored facts relevant to current context. + Merges results from applicable scopes with semantic search via embeddings + and keyword fallback. +- **Automatic memory extraction.** Background scanner finds conversations + with sufficient new activity, sends them to the utility model role for + fact extraction, and stores results as `pending_review` memories. + Configurable extraction prompt per Persona (e.g. "extract FAQ-worthy + Q&A pairs"). Scanner runs on a configurable interval with concurrency + control. Global kill switch via admin settings. +- **Memory review pipeline.** Extracted memories start in `pending_review` + status. Admin panel shows pending review queue with bulk approve. Users + can approve/reject/edit their own pending memories from the Settings → + Memory tab. +- **User memory management.** Settings → Memory tab shows active/pending + memory counts, filterable/searchable memory list with inline edit and + delete. Status filter (active, pending_review, archived). Approve All + button for batch operations on pending memories. +- **Admin memory controls.** Admin panel → Memory section shows system-wide + pending review queue. Memory extraction toggle in admin Settings panel + (`memory_extraction_enabled`). Bulk approve endpoint for batch review. +- **Hybrid semantic recall.** Memory recall supports both keyword search + and vector similarity (cosine distance). Postgres uses `<=>` operator + with pgvector; SQLite computes cosine similarity in Go with app-level + vector loading. Results are merged and deduplicated. +- **Memory injection at completion time.** `BuildMemoryHint()` loads + relevant memories and formats them as a system prompt section, injected + after the knowledge base hint. Context-budget aware with configurable + character limit. Supports embedding-based relevance filtering when the + user's latest message is available. +- **Persona memory configuration.** Personas gain `memory_enabled` (bool) + and `memory_extraction_prompt` (text) fields. When memory is enabled on + a Persona, conversations with that Persona contribute to persona-scoped + memory extraction. Custom extraction prompts allow specialization + (e.g. helpdesk FAQ extraction vs. tutoring progress tracking). +- **Database migrations.** Four new migration files (Postgres + SQLite): + - `004_v0180_memories.sql` / `sqlite/003_v0180_memories.sql` — `memories` + table with composite unique index, full-text search index (GIN/keyword), + `memory_extraction_log` tracking table. + - `005_v0180_memory_phase2.sql` / `sqlite/004_v0180_memory_phase2.sql` — + Persona memory columns, extraction log unique constraint. +- **API endpoints.** Eight new authenticated endpoints: + - `GET /memories` — list user's memories (filterable by status/query) + - `GET /memories/count` — active + pending counts + - `PUT /memories/:id` — edit a memory's key/value/confidence + - `DELETE /memories/:id` — delete a memory + - `POST /memories/:id/approve` — approve a pending memory + - `POST /memories/:id/reject` — reject (archive) a pending memory + - `GET /admin/memories/pending` — admin pending review queue + - `POST /admin/memories/bulk-approve` — admin bulk approve + +### Changed +- `CompletionHandler` now accepts an `*knowledge.Embedder` parameter for + memory injection with semantic relevance filtering. +- Persona create/update forms include memory configuration fields + (enabled toggle, extraction prompt textarea). +- Admin settings save handler includes `memory_extraction` and + `memory_extraction_enabled` keys. +- `store.Stores` struct includes `Memories MemoryStore` field. +- Admin panel sections include Memory with pending review loader. + +### Technical Notes +- Memory embeddings use `vector(3072)` matching the existing KB/notes + schema. HNSW index is not used (pgvector limits HNSW to 2000 dims); + filtered sequential scans are performant for per-user memory tables. + IVFFlat or dimension reduction available as future optimizations. +- SQLite hybrid recall loads embeddings into Go and computes cosine + similarity at the application level, reusing the existing + `cosineSimilarity()` function from the knowledge base store. +- Memory tools use late registration (like KB search and note tools) + because they require stores and embedder dependencies initialized + in main.go. + ## [0.17.3] — 2026-02-28 ### Added diff --git a/VERSION b/VERSION index 6ba94dd..47d04a5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.17.3 \ No newline at end of file +0.18.0 \ No newline at end of file diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f66c7a8..acfdecf 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -126,7 +126,7 @@ server/ │ └── 002_v017_persona_kb.sql ├── store/ │ ├── interfaces.go # Store interfaces + shared types -│ ├── postgres/ # Postgres implementations (19 stores) +│ ├── postgres/ # Postgres implementations (20 stores) │ │ ├── stores.go # NewStores() constructor │ │ ├── provider.go # ProviderStore │ │ ├── catalog.go # CatalogStore @@ -137,8 +137,10 @@ server/ │ │ ├── team.go # TeamStore │ │ ├── note.go # NoteStore │ │ ├── knowledge.go # KnowledgeStore +│ │ ├── memory.go # MemoryStore (CRUD + recall) +│ │ ├── memory_hybrid.go # RecallHybrid (pgvector cosine) │ │ └── ... -│ └── sqlite/ # SQLite implementations (19 stores) +│ └── sqlite/ # SQLite implementations (20 stores) │ ├── stores.go # NewStores() constructor │ └── ... # Mirror of postgres/ with dialect adaptations ├── models/models.go # Shared domain types @@ -149,6 +151,9 @@ server/ ├── crypto/ # AES-256-GCM API key encryption ├── events/ # EventBus + WebSocket hub ├── extraction/ # Document text extraction pipeline +├── memory/ # Long-term memory extraction + scanning +│ ├── extractor.go # Conversation analysis → fact extraction +│ └── scanner.go # Background job: find channels needing extraction ├── handlers/ # HTTP handlers (Gin) │ ├── auth.go # Login, register, refresh, logout │ ├── admin.go # User/config/model management @@ -162,6 +167,8 @@ server/ │ ├── teams.go # Team management │ ├── notes.go # Notes CRUD + search + graph + backlinks │ ├── knowledge.go # Knowledge base management +│ ├── memory.go # Memory CRUD + review (user + admin) +│ ├── memory_inject.go # BuildMemoryHint() for completion injection │ └── ... ├── notelinks/ # Wikilink extraction (regex → NoteLink structs) ├── knowledge/ # KB chunking, embedding, search @@ -251,7 +258,9 @@ src/ ├── index.html # Single-page app shell ├── sw.js # Service worker (offline cache) ├── manifest.json # PWA manifest -├── css/styles.css # All styles (CSS variables, light/dark themes) +├── css/ +│ ├── styles.css # Core styles (CSS variables, light/dark themes) +│ └── memory.css # Memory settings + admin review panel styles ├── js/ │ ├── api.js # HTTP client with token refresh │ ├── app.js # Application state machine, startup, routing @@ -268,6 +277,7 @@ src/ │ ├── notes.js # Notes panel CRUD + graph + daily notes │ ├── note-graph.js # Canvas force-directed graph visualization │ ├── knowledge.js # Knowledge base UI +│ ├── memory-ui.js # Memory settings tab + admin review panel │ └── __tests__/ # Node.js test suite (node --test) ├── editor/ # CM6 source (compiled at build time) │ ├── package.json # CM6 + language mode dependencies diff --git a/docs/CHANGES-0.15.0-phase1.md b/docs/CHANGES-0.15.0-phase1.md deleted file mode 100644 index 492ccaf..0000000 --- a/docs/CHANGES-0.15.0-phase1.md +++ /dev/null @@ -1,93 +0,0 @@ -# v0.15.0 Phase 1 — Changeset Notes - -## Files included in this zip (new or fully replaced) - -``` -server/treepath/path.go NEW — PathMessage, GetActivePath, etc. -server/treepath/summary.go NEW — IsSummaryMessage, FindSummaryBoundary -server/treepath/cursor.go NEW — UpdateCursor, NextSiblingIndex -server/treepath/siblings.go NEW — GetSiblingCount, GetSiblings, FindLeafFromMessage -server/compaction/compaction.go NEW — Service, Compact, CheckRateLimit -server/handlers/tree.go REPLACED — thin wrappers delegating to treepath -server/handlers/summarize.go REPLACED — thin HTTP wrapper using compaction.Service -src/js/settings-handlers.js REPLACED — bug fixes for display name + role config -``` - -## Surgical edits needed in existing files - -### 1. `server/handlers/completion.go` - -**Delete** the `isSummaryMessage` function (lines ~832–841). It is now -defined in `treepath/summary.go` and aliased in `handlers/tree.go`. -Without this deletion, the build fails with a duplicate function error. - -```go -// DELETE this entire function from completion.go: - -// isSummaryMessage checks if a PathMessage has summary metadata. -func isSummaryMessage(m *PathMessage) bool { - if m.Metadata == nil { - return false - } - var meta map[string]interface{} - if err := json.Unmarshal(*m.Metadata, &meta); err != nil { - return false - } - return meta["type"] == "summary" -} -``` - -No other changes needed in completion.go — the `isSummaryMessage` calls -throughout the file now resolve via the alias in `handlers/tree.go`. - -### 2. `server/main.go` - -**Change** the summarize handler wiring (~line 252). Old constructor -takes `(stores, roleResolver)`, new one takes `(*compaction.Service)`. - -```go -// ── BEFORE ── -// Summarize & Continue -summarize := handlers.NewSummarizeHandler(stores, roleResolver) -protected.POST("/channels/:id/summarize", summarize.Summarize) - -// ── AFTER ── -// Compaction service (shared by manual summarize handler + future auto-scanner) -compactionSvc := compaction.NewService(stores, roleResolver) - -// Summarize & Continue (manual compaction via HTTP) -summarize := handlers.NewSummarizeHandler(compactionSvc) -protected.POST("/channels/:id/summarize", summarize.Summarize) -``` - -**Add import** at top of main.go: -```go -"git.gobha.me/xcaliber/chat-switchboard/compaction" -``` - -## Bug fixes included - -### Display Name not saved or used - -`handleSaveSettings()` now also saves `display_name` and `email` via -`API.updateProfile()`. Updates `API.user` and calls `UI.updateUser()` so -the sidebar reflects the change immediately. - -### User utility model role not displayed after save - -`_initUserRolePrimitive()` → `fetchModels` callback now calls the -global `fetchModels()` (which refreshes `App.models` from the server) -before returning the model list. This ensures recently added personal -provider models appear in the role config dropdown after tab switch or -save-refresh. - -## Verification - -After applying all changes: - -1. `cd server && go build .` — should compile cleanly -2. Existing integration tests should pass (no behavior change) -3. Manual test: Settings → Profile → change display name → Save → verify - sidebar updates and value persists on reload -4. Manual test: Settings → Model Roles → pick personal provider + model → - Save → verify dropdown shows saved values after refresh diff --git a/docs/CHANGES-0.15.0-phase2.md b/docs/CHANGES-0.15.0-phase2.md deleted file mode 100644 index c44f5b7..0000000 --- a/docs/CHANGES-0.15.0-phase2.md +++ /dev/null @@ -1,186 +0,0 @@ -# v0.15.0 Phase 2 — Changeset Notes - -## Files included in this zip (new) - -``` -server/compaction/estimator.go NEW — EstimateTokens, EstimatePath, EstimatePathFromSummary -server/compaction/estimator_test.go NEW — unit tests (no DB required) -server/compaction/scanner.go NEW — Scanner, scan loop, candidate query, shouldCompact -``` - -## Surgical edits needed in existing files - -### 1. `server/main.go` - -Phase 1 already introduced `compactionSvc`. Now add the scanner below it -(after the `compactionSvc` line, before route registration): - -```go -import "git.gobha.me/xcaliber/chat-switchboard/compaction" // already present from Phase 1 - -// ── Auto-Compaction Scanner ─────────── -compactionScanner := compaction.NewScanner(compactionSvc, stores, compaction.ScannerConfig{ - Interval: 5 * time.Minute, - Concurrency: 2, -}) -compactionScanner.Start() -defer compactionScanner.Stop() // drain in-flight compactions on shutdown -``` - -Place `defer compactionScanner.Stop()` near the existing -`defer kbIngester.Wait()` line. - -**Note:** The scanner re-reads all config from `global_settings` each -tick. The `ScannerConfig` struct only sets the ticker interval and -semaphore size at startup. Everything else (enabled, threshold, cooldown) -is dynamic. - ---- - -### 2. `src/index.html` - -Add an auto-compaction section to the admin Settings tab. Insert **before** -the Encryption section (before `
` with -`🔐 Encryption`): - -```html -
-

Auto-Compaction

- -

When enabled, long conversations are automatically summarized in the background using the utility model. Ships off by default — requires a utility model role to be configured.

- -
-``` - ---- - -### 3. `src/js/ui-admin.js` — `loadAdminSettings()` - -Add these lines **after** the Web Search config loading block -(after `document.getElementById('searxngConfigFields').style.display = ...`): - -```javascript - // Auto-Compaction (global_settings) - const compactionCfg = getSetting('auto_compaction', {}) || {}; - const compactionEnabled = document.getElementById('adminCompactionEnabled'); - if (compactionEnabled) { - compactionEnabled.checked = !!compactionCfg.enabled; - document.getElementById('compactionConfigFields').style.display = - compactionCfg.enabled ? '' : 'none'; - } - const compThreshold = document.getElementById('adminCompactionThreshold'); - if (compThreshold) compThreshold.value = String(compactionCfg.threshold || 70); - const compCooldown = document.getElementById('adminCompactionCooldown'); - if (compCooldown) compCooldown.value = String(compactionCfg.cooldown || 30); -``` - ---- - -### 4. `src/js/settings-handlers.js` — `handleSaveAdminSettings()` - -Add these lines **after** the `search_config` save block (before -`UI.toast('Settings saved', 'success')`): - -```javascript - // Auto-Compaction config → global_settings - const compactionEnabled = document.getElementById('adminCompactionEnabled')?.checked || false; - const compactionThreshold = parseInt(document.getElementById('adminCompactionThreshold')?.value) || 70; - const compactionCooldown = parseInt(document.getElementById('adminCompactionCooldown')?.value) || 30; - await API.adminUpdateSetting('auto_compaction', { value: { - enabled: compactionEnabled, - threshold: compactionThreshold, - cooldown: compactionCooldown, - }}); - - // Persist individual keys for scanner (reads these independently) - await API.adminUpdateSetting('auto_compaction_enabled', { value: compactionEnabled }); - await API.adminUpdateSetting('auto_compaction_threshold', { value: compactionThreshold / 100 }); - await API.adminUpdateSetting('auto_compaction_cooldown_minutes', { value: compactionCooldown }); -``` - ---- - -### 5. `src/js/settings-handlers.js` — `_initSettingsListeners()` - -Add a toggle listener for the compaction checkbox, alongside the existing -banner toggle. Add **after** the `adminBannerEnabled` change listener: - -```javascript - document.getElementById('adminCompactionEnabled')?.addEventListener('change', (e) => { - document.getElementById('compactionConfigFields').style.display = e.target.checked ? '' : 'none'; - }); -``` - ---- - -### 6. `src/js/ui-core.js` — `_summaryHTML()` - -Show "(auto)" label on auto-triggered summaries. The `trigger` field is -already in message metadata from Phase 1. - -**Replace** the `_summaryHTML` method: - -```javascript - _summaryHTML(msg) { - const meta = msg.metadata || {}; - const count = meta.summarized_count || '?'; - const model = meta.utility_model || 'utility model'; - const trigger = meta.trigger === 'auto' ? ' · auto' : ''; - return ` -
-
- 📝 Conversation summary (${count} messages, by ${esc(model)}${trigger}) -
-
${formatMessage(msg.content)}
-
`; - }, -``` - ---- - -## How the scanner reads settings - -The scanner calls `GlobalConfig.Get()` **every tick** for these keys: - -| Key | Read by | How stored | -|-----|---------|------------| -| `auto_compaction_enabled` | `isEnabled()` | `{value: true/false}` | -| `auto_compaction_threshold` | `getThreshold()` | `{value: 0.70}` (float 0-1) | -| `auto_compaction_cooldown_minutes` | `getCooldownDuration()` | `{value: 30}` | - -The admin UI stores a combined `auto_compaction` key (for display) AND -the individual keys (for scanner consumption). This avoids the scanner -needing to parse a composite object. - -Interval and concurrency are set at startup via `ScannerConfig` and -require a restart to change. This is intentional — they affect the -goroutine pool size and ticker, which shouldn't be hot-swapped. - -## Verification - -1. `cd server && go test ./compaction/` — estimator tests pass (no DB) -2. `cd server && go build .` — compiles cleanly -3. Manual test: Admin → Settings → enable auto-compaction → Save → - check server logs for `🔍 compaction scanner started` -4. Manual test: with utility role configured and a long conversation, - verify auto-compaction fires after the threshold is exceeded -5. Verify cooldown: same channel should not re-compact within 30 minutes -6. Verify kill switch: disable auto-compaction → scanner stops processing - on next tick diff --git a/docs/CHANGES-0.15.0-phase4.md b/docs/CHANGES-0.15.0-phase4.md deleted file mode 100644 index deb3147..0000000 --- a/docs/CHANGES-0.15.0-phase4.md +++ /dev/null @@ -1,106 +0,0 @@ -# v0.15.0 Phase 4 — Guard Rail + Integration Tests - -## Files - -``` -server/compaction/compaction.go UPDATED — context budget guard rail -server/compaction/compaction_test.go NEW — scanner integration tests (DB required) -server/compaction/testmain_test.go NEW — TestMain for compaction package -server/database/seed_helpers.go NEW — SeedTestMessage, SeedTestMessages, SeedTestCursor -server/handlers/live_compaction_test.go NEW — e2e tests with Venice/Qwen3-4B -``` - -## Changes - -### 1. Context Budget Guard Rail (`compaction.go`) - -Added between prompt construction and LLM call. Prevents sending -truncated input to small utility models. - -**How it works:** -1. Estimates total prompt tokens (system + conversation content) -2. Resolves the utility model's `max_context` from catalog -3. Reserves 20% for output (the summary itself) -4. If `estimated_prompt > max_context × 0.80`, returns `ErrContextBudget` - -**Resolution chain** for utility model context: -- `Resolver.GetConfig()` → primary binding → `Catalog.GetByModelID()` -- Fallback: `Catalog.GetByModelIDAny()` (different provider) -- Hard fallback: `DefaultBudget` (128K) - -The scanner's `doCompact()` already handles errors gracefully — a budget -error just logs a warning and retries next cycle (by which time the -conversation may have been manually compacted or the admin may have -upgraded the utility model). - -**New export:** `ErrContextBudget` — callers can use `errors.Is()` to -distinguish budget failures from LLM errors. - -### 2. Test Seed Helpers (`database/seed_helpers.go`) - -New file in the `database` package alongside `testhelper.go`: - -- `SeedTestMessage(t, channelID, parentID, role, content) → msgID` -- `SeedTestMessages(t, channelID, count, contentSize) → []msgIDs` - Creates a linear alternating user/assistant chain -- `SeedTestCursor(t, channelID, userID, leafID)` - Sets the active leaf for tree path resolution - -### 3. Scanner Integration Tests (`compaction/compaction_test.go`) - -All DB-dependent tests use `database.RequireTestDB(t)` and skip -gracefully when no DB is configured. - -| Test | What it verifies | -|------|-----------------| -| `FindCandidates_ReturnsQualifying` | Channel with ≥10 msgs, ≥20K chars, proper age range appears | -| `FindCandidates_ExcludesTooFewMessages` | <10 messages → excluded | -| `FindCandidates_ExcludesTooRecent` | Updated just now → excluded (activity gap) | -| `FindCandidates_ExcludesArchived` | `is_archived=true` → excluded | -| `Cooldown` | Timestamps recorded and checked correctly | -| `InFlightDedup` | sync.Map prevents concurrent compaction of same channel | -| `ShouldCompact_ChannelOptOut` | `settings.auto_compaction=false` → rejected | -| `IsEnabled` | Reads `auto_compaction_enabled` from global_settings | -| `GetThreshold_Default` | Returns 0.70 when no override | -| `GetThreshold_GlobalOverride` | Reads from global_settings | -| `GetThreshold_ChannelOverride` | Channel setting takes precedence | -| `GetCooldownDuration_Default` | Returns 30min default | -| `GetCooldownDuration_Override` | Reads from global_settings | -| `GuardRailMath` | Validates token math for 32K vs 128K models | - -### 4. Live E2E Tests (`handlers/live_compaction_test.go`) - -Requires: `TEST_DATABASE_URL` + `VENICE_API_KEY` - -| Test | What it verifies | -|------|-----------------| -| `CompactionFullPipeline` | Seeds 10 realistic messages → Compact() → verifies summary message in DB, metadata fields, cursor update, usage log entry | -| `CompactionContextBudgetGuardRail` | Seeds 160K chars of content → Compact() returns ErrContextBudget with 32K model | - -## Running Tests - -```bash -# Unit only (no DB) -cd server && go test ./compaction/ -run TestEstimate -v - -# Integration (requires DB) -cd server && go test ./compaction/ -v - -# Live e2e (requires DB + Venice key) -cd server && VENICE_API_KEY=... go test ./handlers/ -run TestLive_Compaction -v -``` - -## Surgical Edit - -### `database/testhelper.go` - -Add this import if not already present (the new `seed_helpers.go` file -is a separate file in the same package, so no edit needed to -`testhelper.go` itself): - -```go -// No changes needed — seed_helpers.go is a new file in package database -``` - -The `strings` import in `seed_helpers.go` is used by `SeedTestMessages` -for `strings.Repeat`. diff --git a/docs/CHANGES-0.18.0-phase1.md b/docs/CHANGES-0.18.0-phase1.md new file mode 100644 index 0000000..c7b59ed --- /dev/null +++ b/docs/CHANGES-0.18.0-phase1.md @@ -0,0 +1,99 @@ +# v0.18.0 Phase 1 — Changes Guide + +## New Files (drop in place) + +| File | Description | +|------|-------------| +| `server/database/migrations/004_v0180_memories.sql` | Postgres migration — memories table, indexes, trigger | +| `server/database/migrations/sqlite/003_v0180_memories.sql` | SQLite equivalent | +| `server/models/models_memory.go` | Memory + MemoryFilter model structs + scope/status constants | +| `server/store/store_memory.go` | MemoryStore interface definition | +| `server/store/postgres/memory.go` | Postgres MemoryStore implementation | +| `server/store/sqlite/memory.go` | SQLite MemoryStore implementation | +| `server/tools/memory.go` | memory_save + memory_recall tools with late registration | +| `server/handlers/memory_inject.go` | BuildMemoryHint() for context injection | +| `docs/DESIGN-0.18.0.md` | Design document | + +## Existing File Modifications + +### 1. `server/store/interfaces.go` + +Add to the `Stores` struct (after `ResourceGrants`): + +```go +Memories MemoryStore +``` + +The `MemoryStore` interface itself is in the new `store_memory.go` file +(same package, separate file for cleanliness). + +### 2. `server/store/postgres/stores.go` + +Add to the `NewStores()` return: + +```go +Memories: NewMemoryStore(), +``` + +### 3. `server/store/sqlite/stores.go` + +Add to the `NewStores()` return: + +```go +Memories: NewMemoryStore(), +``` + +### 4. `server/main.go` + +Add after the existing tool registrations (~line 169): + +```go +// Memory tools (v0.18.0) — late registration, needs stores +tools.RegisterMemoryTools(stores) +``` + +### 5. `server/handlers/completion.go` + +In `loadConversation()`, add memory injection after the KB hint block +(~line 798, after the `BuildKBHint` block): + +```go +// ── Memory injection (recall known facts about user) ── +if memHint := BuildMemoryHint(context.Background(), h.stores, userID, personaID); memHint != "" { + messages = append(messages, providers.Message{ + Role: "system", + Content: memHint, + }) +} +``` + +### 6. `VERSION` + +``` +0.18.0 +``` + +### 7. `docs/ROADMAP.md` + +Mark Phase 1 items as complete (Data Model, Tools, Memory Injection basics). + +--- + +## Testing Checklist + +1. **Migration runs clean** — both Postgres and SQLite +2. **Tools register** — check startup logs for `🔧 Registered tool: memory_save` and `memory_recall` +3. **memory_save** — in a chat, tell the AI something personal ("I prefer Go over Python"). The AI should call memory_save. +4. **memory_recall** — in a NEW chat, ask "what programming language do I prefer?" The AI should call memory_recall and find it. +5. **Persona scoping** — with a Persona active, memories save as `persona_user` scope. Without a Persona, they save as `user` scope. +6. **Upsert** — saving the same key twice updates the value instead of creating a duplicate. +7. **Injection** — memories appear in the system prompt context (check server logs for `🧠 Injected N memories`). + +## Architecture Notes + +- Memory injection happens in `loadConversation()` as a system message, same pattern as KB hints and compaction summaries +- Scope priority in Recall: persona_user > persona > user (most specific wins) +- The unique index on `(scope, owner_id, COALESCE(user_id, nil_uuid), key)` prevents duplicate keys within a scope +- Confidence is LLM-provided: 1.0 for explicit statements, lower for inferences +- Status field supports Phase 2's review pipeline (pending_review, archived) +- Embedding column exists but is unused in Phase 1 — Phase 2 adds semantic recall diff --git a/docs/CHANGES-0.18.0-phase2.md b/docs/CHANGES-0.18.0-phase2.md new file mode 100644 index 0000000..16ae317 --- /dev/null +++ b/docs/CHANGES-0.18.0-phase2.md @@ -0,0 +1,191 @@ +# v0.18.0 Phase 2 — Changes Guide + +## Overview + +Phase 2 adds: automatic memory extraction via background scanner, +embedding-powered semantic recall, memory review pipeline API, and +persona memory configuration. + +## New Files (drop in place) + +| File | Description | +|------|-------------| +| `server/database/migrations/005_v0180_memory_phase2.sql` | Postgres: persona memory columns, HNSW index, extraction log table | +| `server/database/migrations/sqlite/004_v0180_memory_phase2.sql` | SQLite equivalent | +| `server/memory/extractor.go` | Extraction service — calls utility model to extract facts from conversations | +| `server/memory/scanner.go` | Background scanner — finds conversations to extract, same pattern as compaction.Scanner | +| `server/store/postgres/memory_hybrid.go` | RecallHybrid — pgvector cosine distance + keyword merge | +| `server/store/sqlite/memory_hybrid.go` | RecallHybrid — app-level cosine similarity + keyword merge | +| `server/handlers/memory.go` | REST API: list, edit, delete, approve/reject memories | +| `server/handlers/memory_inject.go` | **REPLACES Phase 1 version** — now supports hybrid recall with embeddings | + +## Updated Files (replace Phase 1 versions) + +| File | What Changed | +|------|-------------| +| `server/tools/memory.go` | **REPLACES Phase 1 version** — RegisterMemoryTools now takes embedder param, memory_save embeds on save | + +## Existing File Modifications + +### 1. `server/models/models.go` — Persona struct + +Add after the `KBIDs` field: + +```go +MemoryEnabled bool `json:"memory_enabled" db:"memory_enabled"` +MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty" db:"memory_extraction_prompt"` +``` + +Add to `PersonaPatch`: + +```go +MemoryEnabled *bool `json:"memory_enabled,omitempty"` +MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty"` +``` + +### 2. `server/store/store_memory.go` — MemoryStore interface + +Add to the interface: + +```go +// RecallHybrid returns active memories using vector similarity + keyword search. +// If queryVec is nil/empty, falls back to keyword-only (same as Recall). +RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error) +``` + +### 3. `server/main.go` — Tool registration + scanner startup + +**Update** the existing RegisterMemoryTools call (~line 170) to pass the embedder: + +```go +// Memory tools (v0.18.0) — late registration, needs stores + embedder +tools.RegisterMemoryTools(stores, kbEmbedder) +``` + +**Add** memory extraction scanner startup (after compaction scanner, or near the end +of the startup block — see compaction scanner as pattern reference): + +```go +// Memory extraction scanner (v0.18.0 Phase 2) +// Opt-in: requires global_settings.memory_extraction_enabled = true +memExtractor := memory.NewExtractor(stores, roleResolver, kbEmbedder) +memScanner := memory.NewScanner(memExtractor, stores, memory.ScannerConfig{}) +memScanner.Start() +defer memScanner.Stop() +``` + +Import: `"git.gobha.me/xcaliber/chat-switchboard/memory"` + +### 4. `server/main.go` — Memory API routes + +Add under the protected routes (~after presets/personas routes): + +```go +// Memory management (v0.18.0) +memH := handlers.NewMemoryHandler(stores) +protected.GET("/memories", memH.ListMyMemories) +protected.PUT("/memories/:id", memH.UpdateMemory) +protected.DELETE("/memories/:id", memH.DeleteMemory) +protected.POST("/memories/:id/approve", memH.ApproveMemory) +protected.POST("/memories/:id/reject", memH.RejectMemory) +protected.GET("/memories/count", memH.MemoryCount) +``` + +Add under admin routes: + +```go +// Admin memory review (v0.18.0) +adminMemH := handlers.NewMemoryHandler(stores) +admin.GET("/memories/pending", adminMemH.ListPendingReview) +admin.POST("/memories/bulk-approve", adminMemH.BulkApprove) +``` + +### 5. `server/handlers/completion.go` — BuildMemoryHint signature change + +The `BuildMemoryHint` call in `loadConversation()` changes signature. +Update from Phase 1: + +```go +// OLD (Phase 1): +if memHint := BuildMemoryHint(context.Background(), h.stores, userID, personaID); memHint != "" { + +// NEW (Phase 2) — pass embedder and last user message for semantic recall: +lastUserMsg := "" +for i := len(messages) - 1; i >= 0; i-- { + if messages[i].Role == "user" { + lastUserMsg = messages[i].Content + break + } +} +if memHint := BuildMemoryHint(context.Background(), h.stores, h.embedder, userID, personaID, lastUserMsg); memHint != "" { +``` + +This requires adding the embedder to CompletionHandler. In the struct: + +```go +type CompletionHandler struct { + vault *crypto.KeyResolver + stores store.Stores + hub *events.Hub + objStore storage.ObjectStore + embedder *knowledge.Embedder // NEW: for memory semantic recall +} +``` + +And in `NewCompletionHandler`: + +```go +func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub, objStore storage.ObjectStore, embedder *knowledge.Embedder) *CompletionHandler { + return &CompletionHandler{vault: vault, stores: stores, hub: hub, objStore: objStore, embedder: embedder} +} +``` + +Update the call site in main.go: + +```go +comp := handlers.NewCompletionHandler(keyResolver, stores, hub, objStore, kbEmbedder) +``` + +### 6. `server/store/postgres/persona.go` — Scan memory fields + +Any Persona scan queries need to include the new columns. Add +`memory_enabled` and `memory_extraction_prompt` to SELECT lists and Scan calls +in `GetByID`, `ListForUser`, etc. + +### 7. `server/store/sqlite/persona.go` — Same as above for SQLite. + +--- + +## Admin Setup + +The extraction scanner is **opt-in**. To enable: + +1. Set global config: `memory_extraction_enabled` = `true` +2. The scanner runs every 10 minutes (configurable) +3. Extracted memories start as `pending_review` +4. Admin reviews via `GET /api/v1/admin/memories/pending` +5. Approve: `POST /api/v1/memories/:id/approve` +6. Reject: `POST /api/v1/memories/:id/reject` + +## Testing Checklist + +1. **Migration** — both Postgres and SQLite, verify `memory_extraction_log` table created +2. **Persona columns** — `ALTER TABLE personas ADD COLUMN memory_enabled` runs clean +3. **HNSW index** — Postgres only, verify with `\di+ idx_memories_embedding` +4. **Embedding on save** — call memory_save, check logs for `🧠 memory X embedded` +5. **Hybrid recall** — with embeddings populated, memory_recall should find semantically relevant results even with different keywords +6. **Extraction scanner** — set `memory_extraction_enabled=true` in global config, wait for scan cycle, verify `🧠 memory extraction:` logs +7. **Extracted status** — auto-extracted memories should have `status=pending_review` +8. **Review API** — `GET /memories?status=pending_review` returns pending items +9. **Approve/reject** — POST approve changes status to active, reject archives +10. **BuildMemoryHint** — with embedder, verify semantic recall contextualizes based on user's message + +## Architecture Notes + +- **Extraction scanner** follows compaction.Scanner pattern: ticker loop, semaphore concurrency, in-flight dedup, graceful shutdown via Stop() +- **Extraction log** (`memory_extraction_log`) prevents re-processing: tracks last_message_id per channel+user +- **RecallHybrid** merges vector + keyword results with dedup by ID; semantic results rank first +- **SQLite hybrid** loads all embedded memories into Go and computes cosine similarity in-process (acceptable for single-user deployments) +- **Persona memory toggle** (`memory_enabled`) gates both tool-based and extraction-based memory for that persona +- **Extraction prompt** is customizable per persona — a helpdesk persona extracts FAQ patterns, a tutoring persona extracts learning progress +- **Phase 3** will add the frontend UI: Settings → Memory panel, admin review queue, per-persona toggle in persona editor diff --git a/docs/CHANGES-0.18.0-phase3.md b/docs/CHANGES-0.18.0-phase3.md new file mode 100644 index 0000000..e1cc0c7 --- /dev/null +++ b/docs/CHANGES-0.18.0-phase3.md @@ -0,0 +1,283 @@ +# v0.18.0 Phase 3 — Frontend UI Changes Guide + +## Overview + +Phase 3 adds the user-facing memory management UI: +- **Settings → Memory tab**: view, search, edit, approve/reject memories +- **Admin → AI → Memory**: review pipeline for auto-extracted memories +- **Persona forms**: memory_enabled toggle + custom extraction prompt +- **Admin → System → Settings**: memory extraction toggle + +## New Files (drop in place) + +| File | Lines | Description | +|------|-------|-------------| +| `src/js/memory-ui.js` | ~310 | Memory UI module — settings panel, admin panel, persona form fields | +| `src/css/memory.css` | ~230 | Styles for memory cards, badges, toolbar, edit forms | + +## Existing File Modifications + +### 1. `src/index.html` — Add Memory Tab + Admin Section + +#### A. Settings Modal — Add Memory Tab Button + +After the Knowledge tab button (~line 389): + +```html + +``` + +#### B. Settings Modal — Add Memory Tab Content + +After the Knowledge Bases tab content block (after `settingsKnowledgeBasesTab` div, ~line 552): + +```html + + +``` + +#### C. Admin Panel — Add Memory Section + +In the admin sections HTML, after the `adminPresetsTab` block (~line 790), +add a new admin section: + +```html + +``` + +#### D. Admin Settings — Add Memory Extraction Toggle + +In `adminSettingsTab`, after the Auto-Compaction section (~line 908): + +```html +
+

Memory Extraction

+ +

When enabled, the system automatically extracts memorable facts from conversations using the utility model. Extracted memories require admin approval before becoming active. Requires a utility model role.

+ +
+``` + +#### E. Script + CSS Include + +In the `` section, add the CSS: + +```html + +``` + +In the script block at the bottom (after `knowledge-ui.js`): + +```html + +``` + +--- + +### 2. `src/js/api.js` — Memory API Client Methods + +Add after the user presets section (~line 310): + +```javascript +// Memory (v0.18.0) +listMemories(status, query) { + const params = new URLSearchParams(); + if (status) params.set('status', status); + if (query) params.set('query', query); + return this._get('/api/v1/memories?' + params.toString()); +}, +getMemoryCount() { return this._get('/api/v1/memories/count'); }, +updateMemory(id, data) { return this._put(`/api/v1/memories/${id}`, data); }, +deleteMemory(id) { return this._del(`/api/v1/memories/${id}`); }, +approveMemory(id) { return this._post(`/api/v1/memories/${id}/approve`); }, +rejectMemory(id) { return this._post(`/api/v1/memories/${id}/reject`); }, +bulkApproveMemories(ids) { return this._post('/api/v1/admin/memories/bulk-approve', { ids }); }, +adminListPendingMemories() { return this._get('/api/v1/admin/memories/pending'); }, +``` + +--- + +### 3. `src/js/ui-core.js` — Wire Memory Tab into Settings + +In `switchSettingsTab()` (~line 932), add after the `knowledgeBases` case: + +```javascript +if (tab === 'memory') { + if (typeof MemoryUI !== 'undefined') { + MemoryUI.openSettingsPanel(); + } +} +``` + +--- + +### 4. `src/js/ui-admin.js` — Register Memory in Admin Panel + +#### A. Add "memory" to ADMIN_SECTIONS (~line 9) + +```javascript +const ADMIN_SECTIONS = { + people: ['users', 'teams', 'groups'], + ai: ['providers', 'models', 'presets', 'roles', 'knowledgeBases', 'memory'], + system: ['settings', 'storage', 'extensions'], + monitoring: ['usage', 'audit', 'stats'], +}; +``` + +#### B. Add label (~line 18) + +In ADMIN_LABELS, add: + +```javascript +memory: 'Memory', +``` + +#### C. Add loader (~line 37) + +In ADMIN_LOADERS, add: + +```javascript +memory: () => { + if (typeof MemoryUI === 'undefined') { + console.error('[Admin] MemoryUI not loaded'); + return; + } + MemoryUI.openAdminPanel(); +}, +``` + +--- + +### 5. `src/js/ui-admin.js` — Load/Save Memory Extraction Settings + +#### A. In `loadAdminSettings()` (~after compaction config loading) + +Add after the compaction threshold/cooldown block: + +```javascript +// Memory Extraction (v0.18.0) +const memCfg = getSetting('memory_extraction', {}) || {}; +const memExtractionEl = document.getElementById('adminMemoryExtractionEnabled'); +if (memExtractionEl) { + memExtractionEl.checked = !!memCfg.enabled; + document.getElementById('memoryExtractionConfigFields').style.display = + memCfg.enabled ? '' : 'none'; +} +const memAutoApproveEl = document.getElementById('adminMemoryAutoApprove'); +if (memAutoApproveEl) memAutoApproveEl.checked = !!memCfg.auto_approve; +``` + +#### B. Wire toggle visibility + +Add in `_initAdminListeners()` or inline: + +```javascript +document.getElementById('adminMemoryExtractionEnabled')?.addEventListener('change', function() { + document.getElementById('memoryExtractionConfigFields').style.display = + this.checked ? '' : 'none'; +}); +``` + +--- + +### 6. `src/js/settings-handlers.js` — Save Memory Extraction Config + +In `handleSaveAdminSettings()`, before the final toast (~line 252): + +```javascript +// Memory Extraction config (v0.18.0) +const memExtractionEnabled = document.getElementById('adminMemoryExtractionEnabled')?.checked || false; +const memAutoApprove = document.getElementById('adminMemoryAutoApprove')?.checked || false; +await API.adminUpdateSetting('memory_extraction', { value: { + enabled: memExtractionEnabled, + auto_approve: memAutoApprove, +}}); +// Scanner reads this individual key each tick +await API.adminUpdateSetting('memory_extraction_enabled', { value: memExtractionEnabled }); +``` + +--- + +### 7. `src/js/ui-core.js` — Persona Form Memory Fields + +For **admin** persona forms, in the function that initializes the admin preset +form (after `renderPresetForm` call), add: + +```javascript +if (typeof MemoryUI !== 'undefined') { + MemoryUI.appendMemoryFields(container, prefix); +} +``` + +And in the submit handler, merge memory values: + +```javascript +const values = form.getValues(); +if (typeof MemoryUI !== 'undefined') { + Object.assign(values, MemoryUI.getMemoryValues(prefix)); +} +``` + +When editing an existing persona, call: + +```javascript +if (typeof MemoryUI !== 'undefined') { + MemoryUI.setMemoryValues(prefix, persona); +} +``` + +--- + +## Testing Checklist + +1. **Settings → Memory tab** — click tab, verify memory list loads +2. **Memory search** — type in search box, verify filtering works +3. **Memory status filter** — switch between Active/Pending/Archived +4. **Edit memory** — click pencil icon, modify key/value, save +5. **Delete memory** — click trash icon, confirm, verify removal +6. **Approve/Reject** — with pending memories, verify buttons work +7. **Approve All** — bulk approve pending memories +8. **Admin → AI → Memory** — navigate to admin memory panel +9. **Admin pending review** — verify pending memories show with approve/reject +10. **Admin bulk approve** — verify bulk action works +11. **Admin Settings → Memory Extraction** — toggle on/off, verify checkbox persists +12. **Persona form** — create new persona, verify memory section appears +13. **Persona memory toggle** — uncheck, verify it persists on save +14. **Mobile responsive** — test all memory views on narrow viewport +15. **Empty states** — verify graceful display when no memories exist + +## CSS Variable Dependencies + +The memory styles reference these existing CSS variables: +- `--bg-1`, `--bg-2`, `--bg-3` — background layers +- `--text-1`, `--text-2`, `--text-3` — text colors +- `--border` — border color +- `--accent`, `--accent-dim`, `--accent-text` — accent colors +- `--success`, `--warning`, `--danger` — status colors + +All are defined in the existing `styles.css` theme system. + +## Architecture Notes + +- `MemoryUI` follows the same pattern as `KnowledgeUI` — standalone module + that integrates via `typeof MemoryUI !== 'undefined'` guards +- No build step required — vanilla JS module loaded via ` + diff --git a/src/js/api.js b/src/js/api.js index 88e7770..4462935 100644 --- a/src/js/api.js +++ b/src/js/api.js @@ -671,6 +671,22 @@ const API = { getChannelKBs(channelId) { return this._get(`/api/v1/channels/${channelId}/knowledge-bases`); }, setChannelKBs(channelId, kbIds) { return this._put(`/api/v1/channels/${channelId}/knowledge-bases`, { kb_ids: kbIds }); }, + // ── Memories (v0.18.0) ─────────────────── + listMemories(status, query) { + const params = new URLSearchParams(); + if (status) params.set('status', status); + if (query) params.set('query', query); + const qs = params.toString(); + return this._get('/api/v1/memories' + (qs ? '?' + qs : '')); + }, + getMemoryCount() { return this._get('/api/v1/memories/count'); }, + updateMemory(id, data) { return this._put(`/api/v1/memories/${id}`, data); }, + deleteMemory(id) { return this._del(`/api/v1/memories/${id}`); }, + approveMemory(id) { return this._post(`/api/v1/memories/${id}/approve`, {}); }, + rejectMemory(id) { return this._post(`/api/v1/memories/${id}/reject`, {}); }, + bulkApproveMemories(ids) { return this._post('/api/v1/admin/memories/bulk-approve', { ids }); }, + adminListPendingMemories() { return this._get('/api/v1/admin/memories/pending'); }, + // ── HTTP Internals ─────────────────────── _esc(s) { diff --git a/src/js/memory-ui.js b/src/js/memory-ui.js new file mode 100644 index 0000000..40b6a31 --- /dev/null +++ b/src/js/memory-ui.js @@ -0,0 +1,364 @@ +// ========================================== +// Chat Switchboard – Memory UI (v0.18.0 Phase 3) +// ========================================== +// User-facing memory management in Settings, +// admin memory review in Admin Panel, +// and persona memory config in preset forms. + +const MemoryUI = { + + // ── State ──────────────────────────────── + _memories: [], + _pendingMemories: [], + _filter: { scope: 'user', status: 'active', query: '' }, + + // ═════════════════════════════════════════ + // User Settings Tab + // ═════════════════════════════════════════ + + async openSettingsPanel() { + const panel = document.getElementById('settingsMemoryContent'); + if (!panel) return; + panel.innerHTML = '
Loading memories...
'; + + try { + // Load memory count summary + const count = await API.getMemoryCount(); + const active = count.active || 0; + const pending = count.pending || 0; + + panel.innerHTML = ` +
+
+ ${active} + Active +
+
+ ${pending} + Pending Review +
+
+ +
+
+ + +
+
+ + ${pending > 0 ? `` : ''} +
+
+ +
+ `; + + // Wire events + document.getElementById('memoryStatusFilter')?.addEventListener('change', (e) => { + MemoryUI._filter.status = e.target.value; + MemoryUI.loadMemories(); + }); + document.getElementById('memorySearchInput')?.addEventListener('input', _debounce((e) => { + MemoryUI._filter.query = e.target.value; + MemoryUI.loadMemories(); + }, 300)); + document.getElementById('memoryRefreshBtn')?.addEventListener('click', () => MemoryUI.openSettingsPanel()); + document.getElementById('memoryApproveAllBtn')?.addEventListener('click', () => MemoryUI.approveAllPending()); + + // Load initial list + await this.loadMemories(); + + } catch (e) { + panel.innerHTML = `
${esc(e.message)}
`; + } + }, + + async loadMemories() { + const el = document.getElementById('memoryList'); + if (!el) return; + el.innerHTML = '
Loading...
'; + + try { + const data = await API.listMemories(this._filter.status, this._filter.query); + this._memories = data.memories || []; + + if (!this._memories.length) { + el.innerHTML = `
No ${this._filter.status.replace('_', ' ')} memories${this._filter.query ? ' matching "' + esc(this._filter.query) + '"' : ''}
`; + return; + } + + el.innerHTML = this._memories.map(m => this._renderMemoryCard(m)).join(''); + this._wireMemoryCards(el); + + } catch (e) { + el.innerHTML = `
${esc(e.message)}
`; + } + }, + + _renderMemoryCard(m) { + const confidence = Math.round(m.confidence * 100); + const confidenceClass = confidence >= 80 ? 'high' : confidence >= 50 ? 'medium' : 'low'; + const updated = m.updated_at ? new Date(m.updated_at).toLocaleDateString() : ''; + const scopeLabel = m.scope === 'persona_user' ? 'persona' : m.scope; + + return ` +
+
+ ${esc(m.key)} +
+ ${scopeLabel} + ${confidence}% +
+
+
${esc(m.value)}
+
+ ${updated} +
+ ${m.status === 'pending_review' ? ` + + + ` : ''} + + +
+
+
`; + }, + + _wireMemoryCards(container) { + container.querySelectorAll('[data-action]').forEach(btn => { + btn.addEventListener('click', async (e) => { + const id = btn.dataset.id; + const action = btn.dataset.action; + + if (action === 'approve') await this.approveMemory(id); + else if (action === 'reject') await this.rejectMemory(id); + else if (action === 'delete') await this.deleteMemory(id); + else if (action === 'edit') this.editMemory(id); + }); + }); + }, + + async approveMemory(id) { + try { + await API.approveMemory(id); + UI.toast('Memory approved', 'success'); + this.loadMemories(); + } catch (e) { UI.toast(e.message, 'error'); } + }, + + async rejectMemory(id) { + try { + await API.rejectMemory(id); + UI.toast('Memory rejected', 'success'); + this.loadMemories(); + } catch (e) { UI.toast(e.message, 'error'); } + }, + + async deleteMemory(id) { + if (!confirm('Delete this memory permanently?')) return; + try { + await API.deleteMemory(id); + UI.toast('Memory deleted', 'success'); + this.loadMemories(); + } catch (e) { UI.toast(e.message, 'error'); } + }, + + editMemory(id) { + const m = this._memories.find(x => x.id === id); + if (!m) return; + + const card = document.querySelector(`.memory-card[data-id="${id}"]`); + if (!card) return; + + card.innerHTML = ` +
+
+ + +
+
+ + +
+
+ + +
+
+ `; + + document.getElementById(`memEditSave_${id}`)?.addEventListener('click', async () => { + const key = document.getElementById(`memEdit_key_${id}`).value.trim(); + const value = document.getElementById(`memEdit_val_${id}`).value.trim(); + if (!key || !value) return UI.toast('Key and value are required', 'warning'); + try { + await API.updateMemory(id, { key, value }); + UI.toast('Memory updated', 'success'); + this.loadMemories(); + } catch (e) { UI.toast(e.message, 'error'); } + }); + + document.getElementById(`memEditCancel_${id}`)?.addEventListener('click', () => { + this.loadMemories(); + }); + }, + + async approveAllPending() { + try { + const data = await API.listMemories('pending_review', ''); + const ids = (data.memories || []).map(m => m.id); + if (!ids.length) return UI.toast('No pending memories', 'info'); + if (!confirm(`Approve all ${ids.length} pending memories?`)) return; + await API.bulkApproveMemories(ids); + UI.toast(`${ids.length} memories approved`, 'success'); + this.openSettingsPanel(); + } catch (e) { UI.toast(e.message, 'error'); } + }, + + // ═════════════════════════════════════════ + // Admin Panel — Memory Review + // ═════════════════════════════════════════ + + async openAdminPanel() { + const el = document.getElementById('adminMemoryContent'); + if (!el) return; + el.innerHTML = '
Loading...
'; + + try { + const data = await API.adminListPendingMemories(); + const pending = data.memories || []; + + el.innerHTML = ` +
+ ${pending.length} pending memories + ${pending.length > 0 ? ` + + ` : ''} + +
+
+ ${pending.length === 0 ? '
No memories pending review
' : ''} + ${pending.map(m => this._renderAdminMemoryRow(m)).join('')} +
+ `; + + // Wire events + document.getElementById('adminBulkApproveBtn')?.addEventListener('click', async () => { + const ids = pending.map(m => m.id); + if (!confirm(`Approve all ${ids.length} pending memories?`)) return; + try { + await API.bulkApproveMemories(ids); + UI.toast(`${ids.length} memories approved`, 'success'); + MemoryUI.openAdminPanel(); + } catch (e) { UI.toast(e.message, 'error'); } + }); + document.getElementById('adminMemoryRefreshBtn')?.addEventListener('click', () => MemoryUI.openAdminPanel()); + + // Wire per-row actions + el.querySelectorAll('[data-action]').forEach(btn => { + btn.addEventListener('click', async () => { + const id = btn.dataset.id; + try { + if (btn.dataset.action === 'approve') { + await API.approveMemory(id); + UI.toast('Approved', 'success'); + } else if (btn.dataset.action === 'reject') { + await API.rejectMemory(id); + UI.toast('Rejected', 'success'); + } + MemoryUI.openAdminPanel(); + } catch (e) { UI.toast(e.message, 'error'); } + }); + }); + + } catch (e) { + el.innerHTML = `
${esc(e.message)}
`; + } + }, + + _renderAdminMemoryRow(m) { + const confidence = Math.round(m.confidence * 100); + const scope = m.scope === 'persona_user' ? 'persona' : m.scope; + return ` +
+
+ ${esc(m.key)} +
+ ${scope} + ${confidence}% + ${esc(m.owner_id?.substring(0, 8) || '?')}… +
+
+
${esc(m.value)}
+
+ ${m.updated_at ? new Date(m.updated_at).toLocaleDateString() : ''} +
+ + +
+
+
`; + }, + + // ═════════════════════════════════════════ + // Persona Form Memory Fields + // ═════════════════════════════════════════ + + // Call this after renderPresetForm to append memory fields. + // container: the form container element + // prefix: the form prefix (e.g. 'pf', 'apf') + appendMemoryFields(container, prefix) { + const section = document.createElement('div'); + section.className = 'memory-persona-section'; + section.innerHTML = ` +
+

Memory

+ +

When enabled, this persona can save and recall memories from conversations.

+
+ + +
+ `; + + // Insert before the submit/cancel buttons + const submitRow = container.querySelector('.form-row:last-child'); + if (submitRow) { + container.insertBefore(section, submitRow); + } else { + container.appendChild(section); + } + }, + + // Read memory values from a persona form + getMemoryValues(prefix) { + const enabled = document.getElementById(`${prefix}_memoryEnabled`)?.checked ?? true; + const prompt = document.getElementById(`${prefix}_memoryExtractionPrompt`)?.value.trim() || null; + return { memory_enabled: enabled, memory_extraction_prompt: prompt }; + }, + + // Set memory values in a persona form + setMemoryValues(prefix, persona) { + const enabledEl = document.getElementById(`${prefix}_memoryEnabled`); + if (enabledEl) enabledEl.checked = persona.memory_enabled !== false; + const promptEl = document.getElementById(`${prefix}_memoryExtractionPrompt`); + if (promptEl) promptEl.value = persona.memory_extraction_prompt || ''; + }, +}; + +// ── Debounce helper ────────────────────────── +function _debounce(fn, ms) { + let timer; + return function(...args) { + clearTimeout(timer); + timer = setTimeout(() => fn.apply(this, args), ms); + }; +} diff --git a/src/js/settings-handlers.js b/src/js/settings-handlers.js index 2e203f7..c2abb7e 100644 --- a/src/js/settings-handlers.js +++ b/src/js/settings-handlers.js @@ -247,6 +247,15 @@ async function handleSaveAdminSettings() { await API.adminUpdateSetting('auto_compaction_threshold', { value: compactionThreshold / 100 }); await API.adminUpdateSetting('auto_compaction_cooldown_minutes', { value: compactionCooldown }); + // Memory extraction (v0.18.0) + const memExtractionEnabled = document.getElementById('adminMemoryExtractionEnabled')?.checked || false; + const memAutoApprove = document.getElementById('adminMemoryAutoApprove')?.checked || false; + await API.adminUpdateSetting('memory_extraction', { value: { + enabled: memExtractionEnabled, + auto_approve: memAutoApprove, + }}); + await API.adminUpdateSetting('memory_extraction_enabled', { value: memExtractionEnabled }); + UI.toast('Settings saved', 'success'); // Live-apply: refresh policies and dependent UI await initBanners(); diff --git a/src/js/ui-admin.js b/src/js/ui-admin.js index 55b2d4b..dd5ffd7 100644 --- a/src/js/ui-admin.js +++ b/src/js/ui-admin.js @@ -7,14 +7,14 @@ // ── Category → Section mapping ──────────── const ADMIN_SECTIONS = { people: ['users', 'teams', 'groups'], - ai: ['providers', 'models', 'presets', 'roles', 'knowledgeBases'], + ai: ['providers', 'models', 'presets', 'roles', 'knowledgeBases', 'memory'], system: ['settings', 'storage', 'extensions'], monitoring: ['usage', 'audit', 'stats'], }; const ADMIN_LABELS = { users: 'Users', teams: 'Teams', groups: 'Groups', - providers: 'Providers', models: 'Models', presets: 'Personas', roles: 'Roles', knowledgeBases: 'Knowledge', + providers: 'Providers', models: 'Models', presets: 'Personas', roles: 'Roles', knowledgeBases: 'Knowledge', memory: 'Memory', settings: 'Settings', storage: 'Storage', extensions: 'Extensions', usage: 'Usage', audit: 'Audit', stats: 'Stats', }; @@ -32,6 +32,10 @@ const ADMIN_LOADERS = { if (typeof KnowledgeUI === 'undefined') { console.error('[Admin] KnowledgeUI not loaded — check js/knowledge-ui.js'); return; } KnowledgeUI.openAdminPanel(); }, + memory: () => { + if (typeof MemoryUI !== 'undefined') { MemoryUI.openAdminPanel(); } + else { console.error('[Admin] MemoryUI not loaded — check js/memory-ui.js'); } + }, settings: () => UI.loadAdminSettings(), storage: () => typeof loadAdminStorage === 'function' ? loadAdminStorage() : null, extensions: () => UI.loadAdminExtensions(), @@ -963,6 +967,20 @@ Object.assign(UI, { const compCooldown = document.getElementById('adminCompactionCooldown'); if (compCooldown) compCooldown.value = String(compactionCfg.cooldown || 30); + // Memory extraction config (v0.18.0) + const memExtractionCfg = getSetting('memory_extraction', {}) || {}; + const memEnabled = document.getElementById('adminMemoryExtractionEnabled'); + if (memEnabled) { + memEnabled.checked = !!memExtractionCfg.enabled; + const memFields = document.getElementById('memoryExtractionConfigFields'); + if (memFields) memFields.style.display = memExtractionCfg.enabled ? '' : 'none'; + memEnabled.addEventListener('change', () => { + if (memFields) memFields.style.display = memEnabled.checked ? '' : 'none'; + }); + } + const memAutoApprove = document.getElementById('adminMemoryAutoApprove'); + if (memAutoApprove) memAutoApprove.checked = !!memExtractionCfg.auto_approve; + if (vaultEl) { try { const vault = await API.adminGetVaultStatus(); diff --git a/src/js/ui-core.js b/src/js/ui-core.js index b93331a..08fde9b 100644 --- a/src/js/ui-core.js +++ b/src/js/ui-core.js @@ -59,6 +59,14 @@ function renderPresetForm(containerEl, options = {}) {
+
+ +

When enabled, the AI can save and recall facts about users across conversations.

+
+ + +
+
@@ -111,6 +119,11 @@ function renderPresetForm(containerEl, options = {}) { // Pending avatar data URI const img = document.querySelector(`#${pfx}_avatarPreview img`); if (img?.src?.startsWith('data:')) v._pendingAvatar = img.src; + // Memory fields (v0.18.0) + const memCb = document.getElementById(`${pfx}_memoryEnabled`); + if (memCb) v.memory_enabled = memCb.checked; + const memPrompt = document.getElementById(`${pfx}_memoryExtractionPrompt`)?.value?.trim(); + if (memPrompt) v.memory_extraction_prompt = memPrompt; return v; }, setValues(p) { @@ -123,6 +136,9 @@ function renderPresetForm(containerEl, options = {}) { if (el('model')) el('model').value = p.base_model_id || ''; if (showConfig && el('config')) el('config').value = p.provider_config_id || ''; if (showAvatar) form.updateAvatarPreview(p.avatar || null); + // Memory fields (v0.18.0) + if (el('memoryEnabled')) el('memoryEnabled').checked = p.memory_enabled !== false; + if (el('memoryExtractionPrompt')) el('memoryExtractionPrompt').value = p.memory_extraction_prompt || ''; }, clearForm() { ['name','desc','prompt','temp','maxTokens'].forEach(f => { @@ -930,6 +946,14 @@ const UI = { } } if (tab === 'appearance') UI.loadAppearanceSettings(); + if (tab === 'memory') { + if (typeof MemoryUI !== 'undefined') { + MemoryUI.openSettingsPanel(); + } else { + const c = document.getElementById('settingsMemoryContent'); + if (c) c.innerHTML = '
Memory UI failed to load.
'; + } + } }, // ── Export ────────────────────────────────