switchboard/core
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Releases 9 Wiki Activity

Changeset 0.18.0 (#79)

Browse Source
This commit is contained in:
xcaliber
2026-02-28 18:24:19 +00:00
parent 12e316c234
commit e4a943b03e
48 changed files with 3999 additions and 1398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
.gitea/workflows/ci.yaml
View File

@@ -1,6 +1,6 @@
# .gitea/workflows/ci.yaml
# ============================================
# Chat Switchboard - CI/CD Pipeline (v0.17.2)
# Chat Switchboard - CI/CD Pipeline (v0.17.3)
# ============================================
# Cluster deployments use SEPARATE FE + BE images.
# Unified image is for Docker Hub only (docker-compose use).
@@ -8,10 +8,18 @@
# Pipeline:
# 0. Detect changes (path-based gating for all downstream jobs)
# 1a. Frontend tests — skipped if only BE/docs changed
# 1b. Go test (PG) — skipped if only FE/docs changed
# 1c. Go test (SQLite) — skipped if only FE/docs changed
# 1b. Go unit tests — all non-DB packages + SQLite integration (race-enabled)
# 1c. Go test (PG) — PG store + handlers against Postgres (race-enabled)
# 2. Build + Deploy — skipped if docs-only change
#
# Test coverage mapping (no package tested by zero jobs):
# Unit packages (auto-discovered) → test-sqlite (race)
# ./handlers/... → test-sqlite (SQLite driver, race)
# + test-go-pg (PG driver, race)
# ./store/sqlite/... → test-sqlite (race)
# ./store/postgres/... → test-go-pg (race)
# CGO_ENABLED=0 build check → test-sqlite
#
# Path gating rules:
# src/, src/editor/ → frontend tests
# server/, scripts/db-* → backend tests (PG + SQLite)
@@ -202,11 +210,81 @@ jobs:
- name: Run frontend tests
run: node --test src/js/__tests__/*.test.js
# ── Stage 1b: Go Build & Test (Postgres) ─────
# ── Stage 1b: Go Unit Tests + SQLite Integration ─────
# Covers ALL non-Postgres packages via auto-discovery, plus
# SQLite handler and store integration tests. Race-enabled.
#
# Coverage: unit packages (dynamic), ./handlers/... (SQLite),
# ./store/sqlite/..., CGO_ENABLED=0 build check.
#
# Runs when: backend files changed or infra changed.
# Skipped when: only frontend or docs changed.
test:
test-sqlite:
runs-on: ubuntu-latest
needs: [detect-changes]
if: needs.detect-changes.outputs.backend == 'true' || needs.detect-changes.outputs.infra == 'true'
env:
GOPRIVATE: git.gobha.me/*
GONOSUMCHECK: git.gobha.me/*
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.22'
- name: Download and tidy
working-directory: server
run: |
go mod download
go mod tidy
- name: Build check (CGO_ENABLED=0)
working-directory: server
run: |
echo "━━━ SQLite Backend Build Check ━━━"
CGO_ENABLED=0 go build -o /dev/null .
echo "✓ Binary compiles with SQLite backend (pure Go, no CGO)"
- name: Run unit tests (auto-discovered, no DB)
working-directory: server
run: |
echo "━━━ Unit Tests (all non-DB packages) ━━━"
# Auto-discover: everything except store/* and handlers/*
# This ensures new packages are never silently untested
UNIT_PKGS=$(go list ./... | grep -v -E '/(store|handlers)(/|$)')
echo "Packages under test:"
echo "${UNIT_PKGS}" | sed 's/^/ /'
echo ""
go test -v -race -count=1 -p 1 ${UNIT_PKGS}
echo "✓ Unit tests complete"
- name: Run SQLite handler integration tests
working-directory: server
env:
DB_DRIVER: sqlite
run: |
echo "━━━ SQLite Integration Tests (handlers + stores) ━━━"
go test -v -race -count=1 -p 1 \
./handlers/... \
./store/sqlite/...
echo "✓ SQLite integration tests complete"
# ── Stage 1c: Go Test (Postgres) ─────────────
# Tests ONLY Postgres-dependent packages: store/postgres and
# handlers (with PG driver). Runs in parallel with test-sqlite.
#
# Coverage: ./store/postgres/..., ./handlers/... (PG driver).
# Handlers are intentionally tested against BOTH drivers.
#
# Runs when: backend files changed or infra changed.
# Skipped when: only frontend or docs changed.
test-go-pg:
runs-on: ubuntu-latest
needs: [detect-changes]
if: needs.detect-changes.outputs.backend == 'true' || needs.detect-changes.outputs.infra == 'true'
@@ -264,14 +342,19 @@ jobs:
psql -d "${DB_NAME}" -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO ${APP_USER};"
echo "✓ CI test database ready"
- name: Run tests
- name: Run Postgres integration tests
working-directory: server
env:
PGHOST: ${{ env.POSTGRES_HOST }}
PGPORT: ${{ env.POSTGRES_PORT }}
PGUSER: ${{ secrets.POSTGRES_USER }}
PGPASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
run: go test -v -race -count=1 -p 1 ./...
run: |
echo "━━━ Postgres Integration Tests ━━━"
go test -v -race -count=1 -p 1 \
./store/postgres/... \
./handlers/...
echo "✓ Postgres integration tests complete"
- name: Drop CI test database
if: always()
@@ -284,72 +367,6 @@ jobs:
psql -c "DROP DATABASE IF EXISTS chat_switchboard_ci;" postgres
echo "✓ Dropped CI test database"
- name: Build check
working-directory: server
run: CGO_ENABLED=0 go build -o /dev/null .
# ── Stage 1c: Go Build & Test (SQLite) ───────
# Verifies SQLite backend compiles, stores work, and handler
# integration tests pass against an in-memory SQLite database.
# No external database or provider keys required.
#
# Runs when: backend files changed or infra changed.
# Skipped when: only frontend or docs changed.
test-sqlite:
runs-on: ubuntu-latest
needs: [detect-changes]
if: needs.detect-changes.outputs.backend == 'true' || needs.detect-changes.outputs.infra == 'true'
env:
GOPRIVATE: git.gobha.me/*
GONOSUMCHECK: git.gobha.me/*
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.22'
- name: Download and tidy
working-directory: server
run: |
go mod download
go mod tidy
- name: Build check (CGO_ENABLED=0)
working-directory: server
run: |
echo "━━━ SQLite Backend Build Check ━━━"
CGO_ENABLED=0 go build -o /dev/null .
echo "✓ Binary compiles with SQLite backend (pure Go, no CGO)"
- name: Run unit tests (no DB)
working-directory: server
run: |
echo "━━━ Unit Tests (no external DB) ━━━"
go test -v -count=1 -p 1 \
./capabilities/... \
./compaction/... \
./crypto/... \
./events/... \
./extraction/... \
./knowledge/... \
./providers/... \
./tools/...
echo "✓ Unit tests complete"
- name: Run SQLite handler integration tests
working-directory: server
env:
DB_DRIVER: sqlite
run: |
echo "━━━ SQLite Integration Tests (handlers + stores) ━━━"
go test -v -count=1 -p 1 \
./handlers/... \
./store/sqlite/...
echo "✓ SQLite integration tests complete"
# ── Stage 2: Build, Database, Deploy ─────────
#
# Depends on all test jobs. Skipped jobs (due to path gating)
@@ -358,7 +375,7 @@ jobs:
# Skipped entirely for docs-only changes (nothing to build).
build-and-deploy:
runs-on: ubuntu-latest
needs: [detect-changes, test, test-frontend, test-sqlite]
needs: [detect-changes, test-go-pg, test-frontend, test-sqlite]
# Run unless: a needed job failed, the workflow was cancelled, or it's docs-only.
# Skipped test jobs (path-gated) are fine — they don't block.
if: |

84
CHANGELOG.md
View File

@@ -2,6 +2,90 @@
All notable changes to Chat Switchboard.
## [0.18.0] — 2026-02-28
### Added
- **Memory system.** Long-term memory across conversations with scope-aware
isolation. Three memory scopes: `user` (personal facts/preferences),
`persona` (shared across all users of a Persona), and `persona_user`
(per-user within a Persona context, e.g. tutoring progress per student).
Memories persist across channels and are injected into the system prompt
at completion time with scope priority (persona_user > persona > user).
- **Memory tools.** Two new LLM-callable tools:
- `memory_save` — LLM explicitly stores a fact with key/value/confidence.
Scope-aware: saves to the active scope for the current Persona context.
- `memory_recall` — LLM queries stored facts relevant to current context.
Merges results from applicable scopes with semantic search via embeddings
and keyword fallback.
- **Automatic memory extraction.** Background scanner finds conversations
with sufficient new activity, sends them to the utility model role for
fact extraction, and stores results as `pending_review` memories.
Configurable extraction prompt per Persona (e.g. "extract FAQ-worthy
Q&A pairs"). Scanner runs on a configurable interval with concurrency
control. Global kill switch via admin settings.
- **Memory review pipeline.** Extracted memories start in `pending_review`
status. Admin panel shows pending review queue with bulk approve. Users
can approve/reject/edit their own pending memories from the Settings →
Memory tab.
- **User memory management.** Settings → Memory tab shows active/pending
memory counts, filterable/searchable memory list with inline edit and
delete. Status filter (active, pending_review, archived). Approve All
button for batch operations on pending memories.
- **Admin memory controls.** Admin panel → Memory section shows system-wide
pending review queue. Memory extraction toggle in admin Settings panel
(`memory_extraction_enabled`). Bulk approve endpoint for batch review.
- **Hybrid semantic recall.** Memory recall supports both keyword search
and vector similarity (cosine distance). Postgres uses `<=>` operator
with pgvector; SQLite computes cosine similarity in Go with app-level
vector loading. Results are merged and deduplicated.
- **Memory injection at completion time.** `BuildMemoryHint()` loads
relevant memories and formats them as a system prompt section, injected
after the knowledge base hint. Context-budget aware with configurable
character limit. Supports embedding-based relevance filtering when the
user's latest message is available.
- **Persona memory configuration.** Personas gain `memory_enabled` (bool)
and `memory_extraction_prompt` (text) fields. When memory is enabled on
a Persona, conversations with that Persona contribute to persona-scoped
memory extraction. Custom extraction prompts allow specialization
(e.g. helpdesk FAQ extraction vs. tutoring progress tracking).
- **Database migrations.** Four new migration files (Postgres + SQLite):
- `004_v0180_memories.sql` / `sqlite/003_v0180_memories.sql` — `memories`
table with composite unique index, full-text search index (GIN/keyword),
`memory_extraction_log` tracking table.
- `005_v0180_memory_phase2.sql` / `sqlite/004_v0180_memory_phase2.sql` —
Persona memory columns, extraction log unique constraint.
- **API endpoints.** Eight new authenticated endpoints:
- `GET /memories` — list user's memories (filterable by status/query)
- `GET /memories/count` — active + pending counts
- `PUT /memories/:id` — edit a memory's key/value/confidence
- `DELETE /memories/:id` — delete a memory
- `POST /memories/:id/approve` — approve a pending memory
- `POST /memories/:id/reject` — reject (archive) a pending memory
- `GET /admin/memories/pending` — admin pending review queue
- `POST /admin/memories/bulk-approve` — admin bulk approve
### Changed
- `CompletionHandler` now accepts an `*knowledge.Embedder` parameter for
memory injection with semantic relevance filtering.
- Persona create/update forms include memory configuration fields
(enabled toggle, extraction prompt textarea).
- Admin settings save handler includes `memory_extraction` and
`memory_extraction_enabled` keys.
- `store.Stores` struct includes `Memories MemoryStore` field.
- Admin panel sections include Memory with pending review loader.
### Technical Notes
- Memory embeddings use `vector(3072)` matching the existing KB/notes
schema. HNSW index is not used (pgvector limits HNSW to 2000 dims);
filtered sequential scans are performant for per-user memory tables.
IVFFlat or dimension reduction available as future optimizations.
- SQLite hybrid recall loads embeddings into Go and computes cosine
similarity at the application level, reusing the existing
`cosineSimilarity()` function from the knowledge base store.
- Memory tools use late registration (like KB search and note tools)
because they require stores and embedder dependencies initialized
in main.go.
## [0.17.3] — 2026-02-28
### Added

2
VERSION
View File

@@ -1 +1 @@
0.17.3
0.18.0

16
docs/ARCHITECTURE.md
View File

@@ -126,7 +126,7 @@ server/
│ └── 002_v017_persona_kb.sql
├── store/
│ ├── interfaces.go # Store interfaces + shared types
│ ├── postgres/ # Postgres implementations (19 stores)
│ ├── postgres/ # Postgres implementations (20 stores)
│ │ ├── stores.go # NewStores() constructor
│ │ ├── provider.go # ProviderStore
│ │ ├── catalog.go # CatalogStore
@@ -137,8 +137,10 @@ server/
│ │ ├── team.go # TeamStore
│ │ ├── note.go # NoteStore
│ │ ├── knowledge.go # KnowledgeStore
│ │ ├── memory.go # MemoryStore (CRUD + recall)
│ │ ├── memory_hybrid.go # RecallHybrid (pgvector cosine)
│ │ └── ...
│ └── sqlite/ # SQLite implementations (19 stores)
│ └── sqlite/ # SQLite implementations (20 stores)
│ ├── stores.go # NewStores() constructor
│ └── ... # Mirror of postgres/ with dialect adaptations
├── models/models.go # Shared domain types
@@ -149,6 +151,9 @@ server/
├── crypto/ # AES-256-GCM API key encryption
├── events/ # EventBus + WebSocket hub
├── extraction/ # Document text extraction pipeline
├── memory/ # Long-term memory extraction + scanning
│ ├── extractor.go # Conversation analysis → fact extraction
│ └── scanner.go # Background job: find channels needing extraction
├── handlers/ # HTTP handlers (Gin)
│ ├── auth.go # Login, register, refresh, logout
│ ├── admin.go # User/config/model management
@@ -162,6 +167,8 @@ server/
│ ├── teams.go # Team management
│ ├── notes.go # Notes CRUD + search + graph + backlinks
│ ├── knowledge.go # Knowledge base management
│ ├── memory.go # Memory CRUD + review (user + admin)
│ ├── memory_inject.go # BuildMemoryHint() for completion injection
│ └── ...
├── notelinks/ # Wikilink extraction (regex → NoteLink structs)
├── knowledge/ # KB chunking, embedding, search
@@ -251,7 +258,9 @@ src/
├── index.html # Single-page app shell
├── sw.js # Service worker (offline cache)
├── manifest.json # PWA manifest
├── css/styles.css # All styles (CSS variables, light/dark themes)
├── css/
│ ├── styles.css # Core styles (CSS variables, light/dark themes)
│ └── memory.css # Memory settings + admin review panel styles
├── js/
│ ├── api.js # HTTP client with token refresh
│ ├── app.js # Application state machine, startup, routing
@@ -268,6 +277,7 @@ src/
│ ├── notes.js # Notes panel CRUD + graph + daily notes
│ ├── note-graph.js # Canvas force-directed graph visualization
│ ├── knowledge.js # Knowledge base UI
│ ├── memory-ui.js # Memory settings tab + admin review panel
│ └── __tests__/ # Node.js test suite (node --test)
├── editor/ # CM6 source (compiled at build time)
│ ├── package.json # CM6 + language mode dependencies

93
docs/CHANGES-0.15.0-phase1.md
View File

@@ -1,93 +0,0 @@
# v0.15.0 Phase 1 — Changeset Notes
## Files included in this zip (new or fully replaced)
```
server/treepath/path.go NEW — PathMessage, GetActivePath, etc.
server/treepath/summary.go NEW — IsSummaryMessage, FindSummaryBoundary
server/treepath/cursor.go NEW — UpdateCursor, NextSiblingIndex
server/treepath/siblings.go NEW — GetSiblingCount, GetSiblings, FindLeafFromMessage
server/compaction/compaction.go NEW — Service, Compact, CheckRateLimit
server/handlers/tree.go REPLACED — thin wrappers delegating to treepath
server/handlers/summarize.go REPLACED — thin HTTP wrapper using compaction.Service
src/js/settings-handlers.js REPLACED — bug fixes for display name + role config
```
## Surgical edits needed in existing files
### 1. `server/handlers/completion.go`
**Delete** the `isSummaryMessage` function (lines ~832–841). It is now
defined in `treepath/summary.go` and aliased in `handlers/tree.go`.
Without this deletion, the build fails with a duplicate function error.
```go
// DELETE this entire function from completion.go:
// isSummaryMessage checks if a PathMessage has summary metadata.
func isSummaryMessage(m *PathMessage) bool {
if m.Metadata == nil {
return false
}
var meta map[string]interface{}
if err := json.Unmarshal(*m.Metadata, &meta); err != nil {
return false
}
return meta["type"] == "summary"
}
```
No other changes needed in completion.go — the `isSummaryMessage` calls
throughout the file now resolve via the alias in `handlers/tree.go`.
### 2. `server/main.go`
**Change** the summarize handler wiring (~line 252). Old constructor
takes `(stores, roleResolver)`, new one takes `(*compaction.Service)`.
```go
// ── BEFORE ──
// Summarize & Continue
summarize := handlers.NewSummarizeHandler(stores, roleResolver)
protected.POST("/channels/:id/summarize", summarize.Summarize)
// ── AFTER ──
// Compaction service (shared by manual summarize handler + future auto-scanner)
compactionSvc := compaction.NewService(stores, roleResolver)
// Summarize & Continue (manual compaction via HTTP)
summarize := handlers.NewSummarizeHandler(compactionSvc)
protected.POST("/channels/:id/summarize", summarize.Summarize)
```
**Add import** at top of main.go:
```go
"git.gobha.me/xcaliber/chat-switchboard/compaction"
```
## Bug fixes included
### Display Name not saved or used
`handleSaveSettings()` now also saves `display_name` and `email` via
`API.updateProfile()`. Updates `API.user` and calls `UI.updateUser()` so
the sidebar reflects the change immediately.
### User utility model role not displayed after save
`_initUserRolePrimitive()` → `fetchModels` callback now calls the
global `fetchModels()` (which refreshes `App.models` from the server)
before returning the model list. This ensures recently added personal
provider models appear in the role config dropdown after tab switch or
save-refresh.
## Verification
After applying all changes:
1. `cd server && go build .` — should compile cleanly
2. Existing integration tests should pass (no behavior change)
3. Manual test: Settings → Profile → change display name → Save → verify
sidebar updates and value persists on reload
4. Manual test: Settings → Model Roles → pick personal provider + model →
Save → verify dropdown shows saved values after refresh

186
docs/CHANGES-0.15.0-phase2.md
View File

@@ -1,186 +0,0 @@
# v0.15.0 Phase 2 — Changeset Notes
## Files included in this zip (new)
```
server/compaction/estimator.go NEW — EstimateTokens, EstimatePath, EstimatePathFromSummary
server/compaction/estimator_test.go NEW — unit tests (no DB required)
server/compaction/scanner.go NEW — Scanner, scan loop, candidate query, shouldCompact
```
## Surgical edits needed in existing files
### 1. `server/main.go`
Phase 1 already introduced `compactionSvc`. Now add the scanner below it
(after the `compactionSvc` line, before route registration):
```go
import "git.gobha.me/xcaliber/chat-switchboard/compaction" // already present from Phase 1
// ── Auto-Compaction Scanner ───────────
compactionScanner := compaction.NewScanner(compactionSvc, stores, compaction.ScannerConfig{
Interval: 5 * time.Minute,
Concurrency: 2,
})
compactionScanner.Start()
defer compactionScanner.Stop() // drain in-flight compactions on shutdown
```
Place `defer compactionScanner.Stop()` near the existing
`defer kbIngester.Wait()` line.
**Note:** The scanner re-reads all config from `global_settings` each
tick. The `ScannerConfig` struct only sets the ticker interval and
semaphore size at startup. Everything else (enabled, threshold, cooldown)
is dynamic.
---
### 2. `src/index.html`
Add an auto-compaction section to the admin Settings tab. Insert **before**
the Encryption section (before `<section class="admin-section">` with
`🔐 Encryption`):
```html
<section class="settings-section">
<h3>Auto-Compaction</h3>
<label class="checkbox-label"><input type="checkbox" id="adminCompactionEnabled"> Enable automatic conversation compaction</label>
<p class="section-hint">When enabled, long conversations are automatically summarized in the background using the utility model. Ships off by default — requires a utility model role to be configured.</p>
<div id="compactionConfigFields" style="display:none;margin-top:8px">
<div class="form-row">
<div class="form-group">
<label>Threshold</label>
<div class="form-row" style="gap:4px;align-items:center">
<input type="number" id="adminCompactionThreshold" value="70" min="10" max="95" step="5" style="width:70px">
<span class="form-hint">% of context window</span>
</div>
</div>
<div class="form-group">
<label>Cooldown</label>
<div class="form-row" style="gap:4px;align-items:center">
<input type="number" id="adminCompactionCooldown" value="30" min="5" max="360" step="5" style="width:70px">
<span class="form-hint">minutes per channel</span>
</div>
</div>
</div>
</div>
</section>
```
---
### 3. `src/js/ui-admin.js` — `loadAdminSettings()`
Add these lines **after** the Web Search config loading block
(after `document.getElementById('searxngConfigFields').style.display = ...`):
```javascript
// Auto-Compaction (global_settings)
const compactionCfg = getSetting('auto_compaction', {}) || {};
const compactionEnabled = document.getElementById('adminCompactionEnabled');
if (compactionEnabled) {
compactionEnabled.checked = !!compactionCfg.enabled;
document.getElementById('compactionConfigFields').style.display =
compactionCfg.enabled ? '' : 'none';
}
const compThreshold = document.getElementById('adminCompactionThreshold');
if (compThreshold) compThreshold.value = String(compactionCfg.threshold || 70);
const compCooldown = document.getElementById('adminCompactionCooldown');
if (compCooldown) compCooldown.value = String(compactionCfg.cooldown || 30);
```
---
### 4. `src/js/settings-handlers.js` — `handleSaveAdminSettings()`
Add these lines **after** the `search_config` save block (before
`UI.toast('Settings saved', 'success')`):
```javascript
// Auto-Compaction config → global_settings
const compactionEnabled = document.getElementById('adminCompactionEnabled')?.checked || false;
const compactionThreshold = parseInt(document.getElementById('adminCompactionThreshold')?.value) || 70;
const compactionCooldown = parseInt(document.getElementById('adminCompactionCooldown')?.value) || 30;
await API.adminUpdateSetting('auto_compaction', { value: {
enabled: compactionEnabled,
threshold: compactionThreshold,
cooldown: compactionCooldown,
}});
// Persist individual keys for scanner (reads these independently)
await API.adminUpdateSetting('auto_compaction_enabled', { value: compactionEnabled });
await API.adminUpdateSetting('auto_compaction_threshold', { value: compactionThreshold / 100 });
await API.adminUpdateSetting('auto_compaction_cooldown_minutes', { value: compactionCooldown });
```
---
### 5. `src/js/settings-handlers.js` — `_initSettingsListeners()`
Add a toggle listener for the compaction checkbox, alongside the existing
banner toggle. Add **after** the `adminBannerEnabled` change listener:
```javascript
document.getElementById('adminCompactionEnabled')?.addEventListener('change', (e) => {
document.getElementById('compactionConfigFields').style.display = e.target.checked ? '' : 'none';
});
```
---
### 6. `src/js/ui-core.js` — `_summaryHTML()`
Show "(auto)" label on auto-triggered summaries. The `trigger` field is
already in message metadata from Phase 1.
**Replace** the `_summaryHTML` method:
```javascript
_summaryHTML(msg) {
const meta = msg.metadata || {};
const count = meta.summarized_count || '?';
const model = meta.utility_model || 'utility model';
const trigger = meta.trigger === 'auto' ? ' · auto' : '';
return `
<div class="message-summary" data-msg-id="${esc(msg.id || '')}">
<div class="summary-header">
<span>📝 Conversation summary (${count} messages, by ${esc(model)}${trigger})</span>
</div>
<div class="msg-text">${formatMessage(msg.content)}</div>
</div>`;
},
```
---
## How the scanner reads settings
The scanner calls `GlobalConfig.Get()` **every tick** for these keys:
| Key | Read by | How stored |
|-----|---------|------------|
| `auto_compaction_enabled` | `isEnabled()` | `{value: true/false}` |
| `auto_compaction_threshold` | `getThreshold()` | `{value: 0.70}` (float 0-1) |
| `auto_compaction_cooldown_minutes` | `getCooldownDuration()` | `{value: 30}` |
The admin UI stores a combined `auto_compaction` key (for display) AND
the individual keys (for scanner consumption). This avoids the scanner
needing to parse a composite object.
Interval and concurrency are set at startup via `ScannerConfig` and
require a restart to change. This is intentional — they affect the
goroutine pool size and ticker, which shouldn't be hot-swapped.
## Verification
1. `cd server && go test ./compaction/` — estimator tests pass (no DB)
2. `cd server && go build .` — compiles cleanly
3. Manual test: Admin → Settings → enable auto-compaction → Save →
check server logs for `🔍 compaction scanner started`
4. Manual test: with utility role configured and a long conversation,
verify auto-compaction fires after the threshold is exceeded
5. Verify cooldown: same channel should not re-compact within 30 minutes
6. Verify kill switch: disable auto-compaction → scanner stops processing
on next tick

106
docs/CHANGES-0.15.0-phase4.md
View File

@@ -1,106 +0,0 @@
# v0.15.0 Phase 4 — Guard Rail + Integration Tests
## Files
```
server/compaction/compaction.go UPDATED — context budget guard rail
server/compaction/compaction_test.go NEW — scanner integration tests (DB required)
server/compaction/testmain_test.go NEW — TestMain for compaction package
server/database/seed_helpers.go NEW — SeedTestMessage, SeedTestMessages, SeedTestCursor
server/handlers/live_compaction_test.go NEW — e2e tests with Venice/Qwen3-4B
```
## Changes
### 1. Context Budget Guard Rail (`compaction.go`)
Added between prompt construction and LLM call. Prevents sending
truncated input to small utility models.
**How it works:**
1. Estimates total prompt tokens (system + conversation content)
2. Resolves the utility model's `max_context` from catalog
3. Reserves 20% for output (the summary itself)
4. If `estimated_prompt > max_context × 0.80`, returns `ErrContextBudget`
**Resolution chain** for utility model context:
- `Resolver.GetConfig()` → primary binding → `Catalog.GetByModelID()`
- Fallback: `Catalog.GetByModelIDAny()` (different provider)
- Hard fallback: `DefaultBudget` (128K)
The scanner's `doCompact()` already handles errors gracefully — a budget
error just logs a warning and retries next cycle (by which time the
conversation may have been manually compacted or the admin may have
upgraded the utility model).
**New export:** `ErrContextBudget` — callers can use `errors.Is()` to
distinguish budget failures from LLM errors.
### 2. Test Seed Helpers (`database/seed_helpers.go`)
New file in the `database` package alongside `testhelper.go`:
- `SeedTestMessage(t, channelID, parentID, role, content) → msgID`
- `SeedTestMessages(t, channelID, count, contentSize) → []msgIDs`
Creates a linear alternating user/assistant chain
- `SeedTestCursor(t, channelID, userID, leafID)`
Sets the active leaf for tree path resolution
### 3. Scanner Integration Tests (`compaction/compaction_test.go`)
All DB-dependent tests use `database.RequireTestDB(t)` and skip
gracefully when no DB is configured.
| Test | What it verifies |
|------|-----------------|
| `FindCandidates_ReturnsQualifying` | Channel with ≥10 msgs, ≥20K chars, proper age range appears |
| `FindCandidates_ExcludesTooFewMessages` | <10 messages → excluded |
| `FindCandidates_ExcludesTooRecent` | Updated just now → excluded (activity gap) |
| `FindCandidates_ExcludesArchived` | `is_archived=true` → excluded |
| `Cooldown` | Timestamps recorded and checked correctly |
| `InFlightDedup` | sync.Map prevents concurrent compaction of same channel |
| `ShouldCompact_ChannelOptOut` | `settings.auto_compaction=false` → rejected |
| `IsEnabled` | Reads `auto_compaction_enabled` from global_settings |
| `GetThreshold_Default` | Returns 0.70 when no override |
| `GetThreshold_GlobalOverride` | Reads from global_settings |
| `GetThreshold_ChannelOverride` | Channel setting takes precedence |
| `GetCooldownDuration_Default` | Returns 30min default |
| `GetCooldownDuration_Override` | Reads from global_settings |
| `GuardRailMath` | Validates token math for 32K vs 128K models |
### 4. Live E2E Tests (`handlers/live_compaction_test.go`)
Requires: `TEST_DATABASE_URL` + `VENICE_API_KEY`
| Test | What it verifies |
|------|-----------------|
| `CompactionFullPipeline` | Seeds 10 realistic messages → Compact() → verifies summary message in DB, metadata fields, cursor update, usage log entry |
| `CompactionContextBudgetGuardRail` | Seeds 160K chars of content → Compact() returns ErrContextBudget with 32K model |
## Running Tests
```bash
# Unit only (no DB)
cd server && go test ./compaction/ -run TestEstimate -v
# Integration (requires DB)
cd server && go test ./compaction/ -v
# Live e2e (requires DB + Venice key)
cd server && VENICE_API_KEY=... go test ./handlers/ -run TestLive_Compaction -v
```
## Surgical Edit
### `database/testhelper.go`
Add this import if not already present (the new `seed_helpers.go` file
is a separate file in the same package, so no edit needed to
`testhelper.go` itself):
```go
// No changes needed — seed_helpers.go is a new file in package database
```
The `strings` import in `seed_helpers.go` is used by `SeedTestMessages`
for `strings.Repeat`.

99
docs/CHANGES-0.18.0-phase1.md Normal file
View File

@@ -0,0 +1,99 @@
# v0.18.0 Phase 1 — Changes Guide
## New Files (drop in place)
| File | Description |
|------|-------------|
| `server/database/migrations/004_v0180_memories.sql` | Postgres migration — memories table, indexes, trigger |
| `server/database/migrations/sqlite/003_v0180_memories.sql` | SQLite equivalent |
| `server/models/models_memory.go` | Memory + MemoryFilter model structs + scope/status constants |
| `server/store/store_memory.go` | MemoryStore interface definition |
| `server/store/postgres/memory.go` | Postgres MemoryStore implementation |
| `server/store/sqlite/memory.go` | SQLite MemoryStore implementation |
| `server/tools/memory.go` | memory_save + memory_recall tools with late registration |
| `server/handlers/memory_inject.go` | BuildMemoryHint() for context injection |
| `docs/DESIGN-0.18.0.md` | Design document |
## Existing File Modifications
### 1. `server/store/interfaces.go`
Add to the `Stores` struct (after `ResourceGrants`):
```go
Memories MemoryStore
```
The `MemoryStore` interface itself is in the new `store_memory.go` file
(same package, separate file for cleanliness).
### 2. `server/store/postgres/stores.go`
Add to the `NewStores()` return:
```go
Memories: NewMemoryStore(),
```
### 3. `server/store/sqlite/stores.go`
Add to the `NewStores()` return:
```go
Memories: NewMemoryStore(),
```
### 4. `server/main.go`
Add after the existing tool registrations (~line 169):
```go
// Memory tools (v0.18.0) — late registration, needs stores
tools.RegisterMemoryTools(stores)
```
### 5. `server/handlers/completion.go`
In `loadConversation()`, add memory injection after the KB hint block
(~line 798, after the `BuildKBHint` block):
```go
// ── Memory injection (recall known facts about user) ──
if memHint := BuildMemoryHint(context.Background(), h.stores, userID, personaID); memHint != "" {
messages = append(messages, providers.Message{
Role: "system",
Content: memHint,
})
}
```
### 6. `VERSION`
```
0.18.0
```
### 7. `docs/ROADMAP.md`
Mark Phase 1 items as complete (Data Model, Tools, Memory Injection basics).
---
## Testing Checklist
1. **Migration runs clean** — both Postgres and SQLite
2. **Tools register** — check startup logs for `🔧 Registered tool: memory_save` and `memory_recall`
3. **memory_save** — in a chat, tell the AI something personal ("I prefer Go over Python"). The AI should call memory_save.
4. **memory_recall** — in a NEW chat, ask "what programming language do I prefer?" The AI should call memory_recall and find it.
5. **Persona scoping** — with a Persona active, memories save as `persona_user` scope. Without a Persona, they save as `user` scope.
6. **Upsert** — saving the same key twice updates the value instead of creating a duplicate.
7. **Injection** — memories appear in the system prompt context (check server logs for `🧠 Injected N memories`).
## Architecture Notes
- Memory injection happens in `loadConversation()` as a system message, same pattern as KB hints and compaction summaries
- Scope priority in Recall: persona_user > persona > user (most specific wins)
- The unique index on `(scope, owner_id, COALESCE(user_id, nil_uuid), key)` prevents duplicate keys within a scope
- Confidence is LLM-provided: 1.0 for explicit statements, lower for inferences
- Status field supports Phase 2's review pipeline (pending_review, archived)
- Embedding column exists but is unused in Phase 1 — Phase 2 adds semantic recall

191
docs/CHANGES-0.18.0-phase2.md Normal file
View File

@@ -0,0 +1,191 @@
# v0.18.0 Phase 2 — Changes Guide
## Overview
Phase 2 adds: automatic memory extraction via background scanner,
embedding-powered semantic recall, memory review pipeline API, and
persona memory configuration.
## New Files (drop in place)
| File | Description |
|------|-------------|
| `server/database/migrations/005_v0180_memory_phase2.sql` | Postgres: persona memory columns, HNSW index, extraction log table |
| `server/database/migrations/sqlite/004_v0180_memory_phase2.sql` | SQLite equivalent |
| `server/memory/extractor.go` | Extraction service — calls utility model to extract facts from conversations |
| `server/memory/scanner.go` | Background scanner — finds conversations to extract, same pattern as compaction.Scanner |
| `server/store/postgres/memory_hybrid.go` | RecallHybrid — pgvector cosine distance + keyword merge |
| `server/store/sqlite/memory_hybrid.go` | RecallHybrid — app-level cosine similarity + keyword merge |
| `server/handlers/memory.go` | REST API: list, edit, delete, approve/reject memories |
| `server/handlers/memory_inject.go` | **REPLACES Phase 1 version** — now supports hybrid recall with embeddings |
## Updated Files (replace Phase 1 versions)
| File | What Changed |
|------|-------------|
| `server/tools/memory.go` | **REPLACES Phase 1 version** — RegisterMemoryTools now takes embedder param, memory_save embeds on save |
## Existing File Modifications
### 1. `server/models/models.go` — Persona struct
Add after the `KBIDs` field:
```go
MemoryEnabled bool `json:"memory_enabled" db:"memory_enabled"`
MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty" db:"memory_extraction_prompt"`
```
Add to `PersonaPatch`:
```go
MemoryEnabled *bool `json:"memory_enabled,omitempty"`
MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty"`
```
### 2. `server/store/store_memory.go` — MemoryStore interface
Add to the interface:
```go
// RecallHybrid returns active memories using vector similarity + keyword search.
// If queryVec is nil/empty, falls back to keyword-only (same as Recall).
RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error)
```
### 3. `server/main.go` — Tool registration + scanner startup
**Update** the existing RegisterMemoryTools call (~line 170) to pass the embedder:
```go
// Memory tools (v0.18.0) — late registration, needs stores + embedder
tools.RegisterMemoryTools(stores, kbEmbedder)
```
**Add** memory extraction scanner startup (after compaction scanner, or near the end
of the startup block — see compaction scanner as pattern reference):
```go
// Memory extraction scanner (v0.18.0 Phase 2)
// Opt-in: requires global_settings.memory_extraction_enabled = true
memExtractor := memory.NewExtractor(stores, roleResolver, kbEmbedder)
memScanner := memory.NewScanner(memExtractor, stores, memory.ScannerConfig{})
memScanner.Start()
defer memScanner.Stop()
```
Import: `"git.gobha.me/xcaliber/chat-switchboard/memory"`
### 4. `server/main.go` — Memory API routes
Add under the protected routes (~after presets/personas routes):
```go
// Memory management (v0.18.0)
memH := handlers.NewMemoryHandler(stores)
protected.GET("/memories", memH.ListMyMemories)
protected.PUT("/memories/:id", memH.UpdateMemory)
protected.DELETE("/memories/:id", memH.DeleteMemory)
protected.POST("/memories/:id/approve", memH.ApproveMemory)
protected.POST("/memories/:id/reject", memH.RejectMemory)
protected.GET("/memories/count", memH.MemoryCount)
```
Add under admin routes:
```go
// Admin memory review (v0.18.0)
adminMemH := handlers.NewMemoryHandler(stores)
admin.GET("/memories/pending", adminMemH.ListPendingReview)
admin.POST("/memories/bulk-approve", adminMemH.BulkApprove)
```
### 5. `server/handlers/completion.go` — BuildMemoryHint signature change
The `BuildMemoryHint` call in `loadConversation()` changes signature.
Update from Phase 1:
```go
// OLD (Phase 1):
if memHint := BuildMemoryHint(context.Background(), h.stores, userID, personaID); memHint != "" {
// NEW (Phase 2) — pass embedder and last user message for semantic recall:
lastUserMsg := ""
for i := len(messages) - 1; i >= 0; i-- {
if messages[i].Role == "user" {
lastUserMsg = messages[i].Content
break
}
}
if memHint := BuildMemoryHint(context.Background(), h.stores, h.embedder, userID, personaID, lastUserMsg); memHint != "" {
```
This requires adding the embedder to CompletionHandler. In the struct:
```go
type CompletionHandler struct {
vault *crypto.KeyResolver
stores store.Stores
hub *events.Hub
objStore storage.ObjectStore
embedder *knowledge.Embedder // NEW: for memory semantic recall
}
```
And in `NewCompletionHandler`:
```go
func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub, objStore storage.ObjectStore, embedder *knowledge.Embedder) *CompletionHandler {
return &CompletionHandler{vault: vault, stores: stores, hub: hub, objStore: objStore, embedder: embedder}
}
```
Update the call site in main.go:
```go
comp := handlers.NewCompletionHandler(keyResolver, stores, hub, objStore, kbEmbedder)
```
### 6. `server/store/postgres/persona.go` — Scan memory fields
Any Persona scan queries need to include the new columns. Add
`memory_enabled` and `memory_extraction_prompt` to SELECT lists and Scan calls
in `GetByID`, `ListForUser`, etc.
### 7. `server/store/sqlite/persona.go` — Same as above for SQLite.
---
## Admin Setup
The extraction scanner is **opt-in**. To enable:
1. Set global config: `memory_extraction_enabled` = `true`
2. The scanner runs every 10 minutes (configurable)
3. Extracted memories start as `pending_review`
4. Admin reviews via `GET /api/v1/admin/memories/pending`
5. Approve: `POST /api/v1/memories/:id/approve`
6. Reject: `POST /api/v1/memories/:id/reject`
## Testing Checklist
1. **Migration** — both Postgres and SQLite, verify `memory_extraction_log` table created
2. **Persona columns** — `ALTER TABLE personas ADD COLUMN memory_enabled` runs clean
3. **HNSW index** — Postgres only, verify with `\di+ idx_memories_embedding`
4. **Embedding on save** — call memory_save, check logs for `🧠 memory X embedded`
5. **Hybrid recall** — with embeddings populated, memory_recall should find semantically relevant results even with different keywords
6. **Extraction scanner** — set `memory_extraction_enabled=true` in global config, wait for scan cycle, verify `🧠 memory extraction:` logs
7. **Extracted status** — auto-extracted memories should have `status=pending_review`
8. **Review API** — `GET /memories?status=pending_review` returns pending items
9. **Approve/reject** — POST approve changes status to active, reject archives
10. **BuildMemoryHint** — with embedder, verify semantic recall contextualizes based on user's message
## Architecture Notes
- **Extraction scanner** follows compaction.Scanner pattern: ticker loop, semaphore concurrency, in-flight dedup, graceful shutdown via Stop()
- **Extraction log** (`memory_extraction_log`) prevents re-processing: tracks last_message_id per channel+user
- **RecallHybrid** merges vector + keyword results with dedup by ID; semantic results rank first
- **SQLite hybrid** loads all embedded memories into Go and computes cosine similarity in-process (acceptable for single-user deployments)
- **Persona memory toggle** (`memory_enabled`) gates both tool-based and extraction-based memory for that persona
- **Extraction prompt** is customizable per persona — a helpdesk persona extracts FAQ patterns, a tutoring persona extracts learning progress
- **Phase 3** will add the frontend UI: Settings → Memory panel, admin review queue, per-persona toggle in persona editor

283
docs/CHANGES-0.18.0-phase3.md Normal file
View File

@@ -0,0 +1,283 @@
# v0.18.0 Phase 3 — Frontend UI Changes Guide
## Overview
Phase 3 adds the user-facing memory management UI:
- **Settings → Memory tab**: view, search, edit, approve/reject memories
- **Admin → AI → Memory**: review pipeline for auto-extracted memories
- **Persona forms**: memory_enabled toggle + custom extraction prompt
- **Admin → System → Settings**: memory extraction toggle
## New Files (drop in place)
| File | Lines | Description |
|------|-------|-------------|
| `src/js/memory-ui.js` | ~310 | Memory UI module — settings panel, admin panel, persona form fields |
| `src/css/memory.css` | ~230 | Styles for memory cards, badges, toolbar, edit forms |
## Existing File Modifications
### 1. `src/index.html` — Add Memory Tab + Admin Section
#### A. Settings Modal — Add Memory Tab Button
After the Knowledge tab button (~line 389):
```html
<button class="settings-tab" data-stab="memory" onclick="UI.switchSettingsTab('memory')" id="settingsMemoryTabBtn">Memory</button>
```
#### B. Settings Modal — Add Memory Tab Content
After the Knowledge Bases tab content block (after `settingsKnowledgeBasesTab` div, ~line 552):
```html
<!-- Memory Tab (v0.18.0) -->
<div class="settings-tab-content" id="settingsMemoryTab" style="display:none">
<section class="settings-section">
<h3 style="font-size:14px;margin-bottom:4px">My Memories</h3>
<p class="section-hint" style="margin-bottom:8px">Facts and preferences learned from your conversations. Memories help AI provide more personalized responses.</p>
<div id="settingsMemoryContent"></div>
</section>
</div>
```
#### C. Admin Panel — Add Memory Section
In the admin sections HTML, after the `adminPresetsTab` block (~line 790),
add a new admin section:
```html
<div class="admin-section-content" id="adminMemoryTab" style="display:none">
<div id="adminMemoryContent"></div>
</div>
```
#### D. Admin Settings — Add Memory Extraction Toggle
In `adminSettingsTab`, after the Auto-Compaction section (~line 908):
```html
<section class="settings-section">
<h3>Memory Extraction</h3>
<label class="checkbox-label"><input type="checkbox" id="adminMemoryExtractionEnabled"> Enable automatic memory extraction</label>
<p class="section-hint">When enabled, the system automatically extracts memorable facts from conversations using the utility model. Extracted memories require admin approval before becoming active. Requires a utility model role.</p>
<div id="memoryExtractionConfigFields" style="display:none;margin-top:8px">
<label class="checkbox-label"><input type="checkbox" id="adminMemoryAutoApprove"> Auto-approve extracted memories</label>
<p class="section-hint">Skip the review pipeline and activate extracted memories immediately. Not recommended for sensitive environments.</p>
</div>
</section>
```
#### E. Script + CSS Include
In the `<head>` section, add the CSS:
```html
<link rel="stylesheet" href="css/memory.css">
```
In the script block at the bottom (after `knowledge-ui.js`):
```html
<script src="js/memory-ui.js"></script>
```
---
### 2. `src/js/api.js` — Memory API Client Methods
Add after the user presets section (~line 310):
```javascript
// Memory (v0.18.0)
listMemories(status, query) {
const params = new URLSearchParams();
if (status) params.set('status', status);
if (query) params.set('query', query);
return this._get('/api/v1/memories?' + params.toString());
},
getMemoryCount() { return this._get('/api/v1/memories/count'); },
updateMemory(id, data) { return this._put(`/api/v1/memories/${id}`, data); },
deleteMemory(id) { return this._del(`/api/v1/memories/${id}`); },
approveMemory(id) { return this._post(`/api/v1/memories/${id}/approve`); },
rejectMemory(id) { return this._post(`/api/v1/memories/${id}/reject`); },
bulkApproveMemories(ids) { return this._post('/api/v1/admin/memories/bulk-approve', { ids }); },
adminListPendingMemories() { return this._get('/api/v1/admin/memories/pending'); },
```
---
### 3. `src/js/ui-core.js` — Wire Memory Tab into Settings
In `switchSettingsTab()` (~line 932), add after the `knowledgeBases` case:
```javascript
if (tab === 'memory') {
if (typeof MemoryUI !== 'undefined') {
MemoryUI.openSettingsPanel();
}
}
```
---
### 4. `src/js/ui-admin.js` — Register Memory in Admin Panel
#### A. Add "memory" to ADMIN_SECTIONS (~line 9)
```javascript
const ADMIN_SECTIONS = {
people: ['users', 'teams', 'groups'],
ai: ['providers', 'models', 'presets', 'roles', 'knowledgeBases', 'memory'],
system: ['settings', 'storage', 'extensions'],
monitoring: ['usage', 'audit', 'stats'],
};
```
#### B. Add label (~line 18)
In ADMIN_LABELS, add:
```javascript
memory: 'Memory',
```
#### C. Add loader (~line 37)
In ADMIN_LOADERS, add:
```javascript
memory: () => {
if (typeof MemoryUI === 'undefined') {
console.error('[Admin] MemoryUI not loaded');
return;
}
MemoryUI.openAdminPanel();
},
```
---
### 5. `src/js/ui-admin.js` — Load/Save Memory Extraction Settings
#### A. In `loadAdminSettings()` (~after compaction config loading)
Add after the compaction threshold/cooldown block:
```javascript
// Memory Extraction (v0.18.0)
const memCfg = getSetting('memory_extraction', {}) || {};
const memExtractionEl = document.getElementById('adminMemoryExtractionEnabled');
if (memExtractionEl) {
memExtractionEl.checked = !!memCfg.enabled;
document.getElementById('memoryExtractionConfigFields').style.display =
memCfg.enabled ? '' : 'none';
}
const memAutoApproveEl = document.getElementById('adminMemoryAutoApprove');
if (memAutoApproveEl) memAutoApproveEl.checked = !!memCfg.auto_approve;
```
#### B. Wire toggle visibility
Add in `_initAdminListeners()` or inline:
```javascript
document.getElementById('adminMemoryExtractionEnabled')?.addEventListener('change', function() {
document.getElementById('memoryExtractionConfigFields').style.display =
this.checked ? '' : 'none';
});
```
---
### 6. `src/js/settings-handlers.js` — Save Memory Extraction Config
In `handleSaveAdminSettings()`, before the final toast (~line 252):
```javascript
// Memory Extraction config (v0.18.0)
const memExtractionEnabled = document.getElementById('adminMemoryExtractionEnabled')?.checked || false;
const memAutoApprove = document.getElementById('adminMemoryAutoApprove')?.checked || false;
await API.adminUpdateSetting('memory_extraction', { value: {
enabled: memExtractionEnabled,
auto_approve: memAutoApprove,
}});
// Scanner reads this individual key each tick
await API.adminUpdateSetting('memory_extraction_enabled', { value: memExtractionEnabled });
```
---
### 7. `src/js/ui-core.js` — Persona Form Memory Fields
For **admin** persona forms, in the function that initializes the admin preset
form (after `renderPresetForm` call), add:
```javascript
if (typeof MemoryUI !== 'undefined') {
MemoryUI.appendMemoryFields(container, prefix);
}
```
And in the submit handler, merge memory values:
```javascript
const values = form.getValues();
if (typeof MemoryUI !== 'undefined') {
Object.assign(values, MemoryUI.getMemoryValues(prefix));
}
```
When editing an existing persona, call:
```javascript
if (typeof MemoryUI !== 'undefined') {
MemoryUI.setMemoryValues(prefix, persona);
}
```
---
## Testing Checklist
1. **Settings → Memory tab** — click tab, verify memory list loads
2. **Memory search** — type in search box, verify filtering works
3. **Memory status filter** — switch between Active/Pending/Archived
4. **Edit memory** — click pencil icon, modify key/value, save
5. **Delete memory** — click trash icon, confirm, verify removal
6. **Approve/Reject** — with pending memories, verify buttons work
7. **Approve All** — bulk approve pending memories
8. **Admin → AI → Memory** — navigate to admin memory panel
9. **Admin pending review** — verify pending memories show with approve/reject
10. **Admin bulk approve** — verify bulk action works
11. **Admin Settings → Memory Extraction** — toggle on/off, verify checkbox persists
12. **Persona form** — create new persona, verify memory section appears
13. **Persona memory toggle** — uncheck, verify it persists on save
14. **Mobile responsive** — test all memory views on narrow viewport
15. **Empty states** — verify graceful display when no memories exist
## CSS Variable Dependencies
The memory styles reference these existing CSS variables:
- `--bg-1`, `--bg-2`, `--bg-3` — background layers
- `--text-1`, `--text-2`, `--text-3` — text colors
- `--border` — border color
- `--accent`, `--accent-dim`, `--accent-text` — accent colors
- `--success`, `--warning`, `--danger` — status colors
All are defined in the existing `styles.css` theme system.
## Architecture Notes
- `MemoryUI` follows the same pattern as `KnowledgeUI` — standalone module
that integrates via `typeof MemoryUI !== 'undefined'` guards
- No build step required — vanilla JS module loaded via `<script>` tag
- Admin memory section registered in `ADMIN_SECTIONS` and `ADMIN_LOADERS`
maps following the existing convention
- Persona memory fields are appended dynamically via `appendMemoryFields()`
rather than embedded in `renderPresetForm()` to avoid modifying the shared
form builder
- Debounce on search input prevents excessive API calls
- Memory cards use inline edit — no modal, same pattern as note inline editing
- CSS uses existing theme variables for consistent dark/light mode support

235
docs/DESIGN-0.18.0.md Normal file
View File

@@ -0,0 +1,235 @@
# DESIGN-0.18.0: Memory System
**Version:** 0.18.0
**Status:** Phase 1 Complete
**Depends on:** compaction (v0.15.0), knowledge bases (v0.14.0), persona-KB binding (v0.17.0), SQLite backend (v0.17.1)
---
## 1. Overview
Memory provides long-term fact persistence across conversations. Unlike
compaction (within-conversation context) or knowledge bases (static
documents), memory captures preferences, facts, and context that
accumulate over time from natural conversation.
**Key differentiator: Persona-scoped memory.** Each Persona builds
its own memory independently, preventing cross-context contamination.
A helpdesk Persona accumulates FAQ knowledge; a tutoring Persona
tracks per-student progress; a user's personal preferences stay
separate from any Persona context.
---
## 2. Scope Model
Three memory scopes with clear ownership:
```
┌─────────────────────────────────────────────────┐
│ user scope │
│ owner_id = user_id │
│ "Jeff prefers Go over Python" │
│ "Deployment uses Kubernetes + Traefik" │
│ Visible in ALL conversations for this user │
└─────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────┐
│ persona scope │
│ owner_id = persona_id │
│ "Common question: how to reset password" │
│ "Policy: refunds within 30 days only" │
│ Shared across ALL users of this Persona │
└─────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────┐
│ persona_user scope │
│ owner_id = persona_id, user_id = user_id │
│ "Student struggles with recursion" │
│ "Customer prefers email over phone" │
│ Per-user within a specific Persona │
└─────────────────────────────────────────────────┘
```
**Scope resolution at recall time:**
When a Persona is active, all three scopes are queried and merged
with priority ordering: persona_user > persona > user. When no
Persona is active, only user scope is queried.
---
## 3. Data Model
### 3.1 Table: `memories`
| Column | Type | Description |
|--------|------|-------------|
| `id` | UUID / TEXT | Primary key |
| `scope` | TEXT | `user`, `persona`, `persona_user` |
| `owner_id` | UUID / TEXT | user_id (user scope) or persona_id (persona/persona_user) |
| `user_id` | UUID / TEXT | Only for persona_user scope — identifies which user |
| `key` | TEXT | Short label (2-6 words) |
| `value` | TEXT | The fact/detail |
| `source_channel_id` | UUID / TEXT | Provenance — which conversation |
| `confidence` | REAL | 0.0-1.0, LLM-provided |
| `status` | TEXT | active, pending_review, archived |
| `embedding` | vector(3072) / TEXT | For semantic recall (Phase 2) |
| `created_at` | TIMESTAMPTZ / TEXT | |
| `updated_at` | TIMESTAMPTZ / TEXT | |
### 3.2 Indexes
- **Primary lookup:** `(scope, owner_id, status)` — filtered to `status = 'active'`
- **Persona+user:** `(owner_id, user_id)` — filtered to `scope = 'persona_user'`
- **Dedup:** unique on `(scope, owner_id, COALESCE(user_id, nil_uuid), key)`
- **FTS:** GIN on `to_tsvector(key || ' ' || value)` (Postgres) / LIKE fallback (SQLite)
- **Provenance:** `(source_channel_id)` — "where did this memory come from?"
### 3.3 Model Struct
```go
type Memory struct {
ID string `json:"id"`
Scope string `json:"scope"`
OwnerID string `json:"owner_id"`
UserID *string `json:"user_id,omitempty"`
Key string `json:"key"`
Value string `json:"value"`
SourceChannelID *string `json:"source_channel_id,omitempty"`
Confidence float64 `json:"confidence"`
Status string `json:"status"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
```
---
## 4. Store Interface
```go
type MemoryStore interface {
Upsert(ctx, *Memory) error // create or update (matched on scope+owner+user+key)
GetByID(ctx, id) (*Memory, error)
List(ctx, MemoryFilter) ([]Memory, error)
Delete(ctx, id) error // hard delete
Archive(ctx, id) error // soft delete (status → archived)
Recall(ctx, userID, *personaID, query, limit) ([]Memory, error) // scope-merged search
CountByOwner(ctx, scope, ownerID) (int, error)
}
```
**Key design decisions:**
- `Upsert` uses the unique index for conflict resolution — same key = update value
- `Recall` does a UNION across all applicable scopes with priority ordering
- Both Postgres and SQLite implementations exist (same pattern as all other stores)
- SQLite falls back to LIKE for text search (no tsvector)
---
## 5. Tools
### 5.1 `memory_save`
The LLM calls this to explicitly store a fact. Scope is automatically
determined by whether a Persona is active:
- No Persona → `user` scope, `owner_id = user_id`
- Persona active → `persona_user` scope, `owner_id = persona_id`, `user_id = user_id`
Parameters: `key` (required), `value` (required), `confidence` (optional, default 1.0)
### 5.2 `memory_recall`
The LLM calls this to search stored memories. Results merge across
all applicable scopes.
Parameters: `query` (optional), `max_results` (optional, default 20)
### 5.3 Registration
Late registration pattern (same as notes, KB search, conversation search):
```go
// main.go
tools.RegisterMemoryTools(stores)
```
---
## 6. Memory Injection
At completion time, `loadConversation()` injects relevant memories
into the system prompt — same position as KB hints and compaction
summaries:
```
[admin system prompt]
[persona/channel system prompt]
[KB hint]
[memory injection] ← NEW
[compaction summary OR message history]
```
`BuildMemoryHint()` calls `Recall()` with an empty query (returns
top memories by confidence/recency), formats as bullet points, and
caps at ~1500 tokens (~6000 chars) to preserve context budget.
---
## 7. Implementation Phases
### Phase 1: Foundation & Tools ✅ (this delivery)
- Migrations (Postgres + SQLite)
- Memory model struct
- MemoryStore interface + both implementations
- memory_save + memory_recall tools
- Memory injection in loadConversation
- 9 new files, ~5 edits to existing files
### Phase 2: Automatic Extraction & Embeddings
- Background job: post-conversation analysis extracts facts
- Configurable extraction prompt per Persona
- Extracted memories start as `pending_review`
- Embedding generation for semantic recall
- Recall upgraded to use vector similarity + keyword hybrid
### Phase 3: Review Pipeline & User Controls
- Admin review queue for pending memories
- Persona memory review for team admins
- User Settings → Memory: view, edit, delete
- Per-Persona memory toggle
- Retention policies
- Frontend memory management UI
---
## 8. Persona Memory Saves (Future: Phase 2)
Phase 1 only supports `user` and `persona_user` scopes via the tool
(the LLM saves about the user, scoped appropriately). Phase 2 adds
the ability for admins/team leads to configure Persona-scope memory
extraction — the Persona "learns" from conversations:
```
Persona system prompt addition (Phase 2):
"After each conversation, identify key facts, common questions,
and useful patterns. Save these as persona-scope memories for
future reference by all users."
```
This runs as a background job, not during the live conversation,
to avoid latency impact.
---
## 9. Privacy & Security
- User memories never cross team boundaries
- Persona memories respect group access controls
- persona_user memories are only visible to the specific user + admins
- Users can view/delete their own memories (Phase 3 UI)
- Admin can disable memory system globally or per-team
- No cross-persona memory leakage — scope isolation is enforced at the query level

530
docs/REFACTOR-0.10.3.md
View File

@@ -1,530 +0,0 @@
# v0.10.3 — Frontend Refactor Plan
## Problem
Two monolith files accumulate every feature:
| File | Lines | Role |
|----------|------:|-----------------------------------|
| app.js | 2,940 | All application logic + listeners |
| ui.js | 2,582 | All rendering + admin + settings |
| api.js | 575 | HTTP client (fine) |
| debug.js | 580 | Debug logger (fine) |
| events.js| 327 | EventBus WebSocket (fine) |
| **Total**| **7,004** | |
The v0.10.2 syntax bug (orphan brace nesting everything from notes
through banners) is a direct consequence: in a 2,940-line file, a
misplaced insertion is invisible.
## Constraints
- **Vanilla JS** — no build step, no bundler, no ES modules.
- **Global functions** — HTML `onclick` handlers call ~25 named globals.
These must stay on `window`. No renaming.
- **Load order via `<script>` tags** — dependencies load first.
- **Service worker** — `sw.js` pre-caches the file list; must be updated.
- **Cache busting** — all `<script>` tags use `?v=%%APP_VERSION%%`.
- **Docker entrypoint** — `COPY src/` covers everything. No changes needed.
- **CI syntax check** — already runs `node --check` on `src/js/*.js`.
## Architecture
The `UI` object stays as a single global with methods. Domain files extend
it via `Object.assign(UI, { ... })`. The `App` state object stays in
`app.js`. All functions stay global.
**Load order** (dependency graph, top to bottom):
```
vendor/marked.min.js, vendor/purify.min.js
↓
debug.js, events.js
↓
api.js
↓
ui-format.js ← esc(), markdown rendering (used by everything)
↓
ui-core.js ← UI object: sidebar, chat list, messages, streaming, model selector
↓
ui-settings.js ← Object.assign(UI, { settings/team methods })
ui-admin.js ← Object.assign(UI, { admin modal methods })
↓
tokens.js ← Tokens object, context tracking
notes.js ← Notes panel
chat.js ← Chat ops, send, regen, edit, branch, summarize
settings-handlers.js ← Provider CRUD, command palette, save handlers
admin-handlers.js ← Admin actions, presets, team management
↓
app.js ← State, init, boot, auth, listeners (orchestrator)
```
## File Breakdown
### Unchanged (3 files, 1,482 lines)
| File | Lines | Notes |
|------|------:|-------|
| api.js | 575 | Clean. Self-contained HTTP client. |
| debug.js | 580 | Clean. Debug logger + modal. |
| events.js | 327 | Clean. EventBus WebSocket. |
### From ui.js → 4 files
#### ui-format.js (~280 lines)
Extract from ui.js bottom section + helpers.
```
Formatting (L2241-2462):
_renderMarkdown(text)
_renderCodeBlocks(html)
_copyCodeBlock(btn)
_formatTokenCount(n)
_formatCost(c)
Helpers (L2519-2583):
esc(s) ← global function, used everywhere
_formatTimestamp(ts)
_formatRelativeTime(ts)
_truncate(s, n)
openSidePanel(panelId) ← global, called from notes/search
closeSidePanel() ← global, HTML onclick
Side Panel Resize (L2463-2518):
_initSidePanelResize()
```
**Why separate:** Formatting is a pure utility layer. Every other UI file
depends on `esc()` and `_renderMarkdown()`. Loading first eliminates
forward references.
#### ui-core.js (~650 lines)
The `UI` object definition with core rendering methods.
```
const UI = {
// Sidebar (L189-216)
toggleSidebar(), restoreSidebar(),
toggleUserMenu(), closeUserMenu(),
// Chat List (L217-284)
renderChatList(),
// Messages (L285-454)
renderMessages(), showEmptyState(),
_messageHTML(), _summaryHTML(), _isSummaryMessage(),
toggleSummarizedHistory(),
// Streaming (L455-596)
streamResponse(),
// Model Selector (L597-727)
getModelValue(), setModelValue(),
updateModelSelector(), initModelDropdown(),
// Capability Badges (L728-774)
getSelectedModelCaps(), updateCapabilityBadges(),
// User / Connection (L775-811)
updateUser(), showAdminButton(),
// Generating State (L812-862)
setGenerating(), showRegenerate(),
// Toast (L863-873)
toast(),
// Settings Modal (L874-921) — thin dispatcher
openSettings(), switchSettingsTab(),
// Export (L2191-2214)
exportMarkdown(), exportJSON(),
// Message Actions (L2215-2240)
(delegated action handlers)
// Scroll (L2226-2240)
scrollToBottom(),
};
```
Also includes the top-level `_avatarHTML()` helper (L5-24) and
summary message helpers (L167-188).
#### ui-settings.js (~550 lines)
Extends UI with all settings modal tab content.
```
Object.assign(UI, {
// Preset Form Component (L25-166) — _presetFormHTML(), etc.
// Appearance (L922-1267)
loadAppearanceSettings(), initAppearance(),
// Providers (L1444-1480)
loadProviderList(), hideProviderForm(),
// Profile (L994-1002)
loadProfileIntoSettings(),
// Teams (L1003-1267)
loadMyTeams(), loadTeamsTab(), openTeamManage(),
showTeamPicker(), loadTeamManageMembers(),
loadTeamManageProviders(), loadTeamManagePresets(),
loadTeamAuditLog(), loadTeamPresetModelDropdown(),
switchTeamTab(),
// Usage (L1160-1267)
loadMyUsage(), loadTeamUsage(),
// User Model Roles (L1268-1328)
loadUserRoles(),
// User Models & Presets (L2118-2190)
loadUserModels(), loadUserPresets(),
// checkUserProvidersAllowed(), checkUserPresetsAllowed()
});
```
#### ui-admin.js (~650 lines)
Extends UI with all admin modal content.
```
Object.assign(UI, {
// Admin Modal (L1481-1604)
openTeamAdmin(), switchAdminTab(),
// Users (L1549-1588)
loadAdminUsers(),
// Stats (L1589-1604)
loadAdminStats(),
// Roles (L1605-1658)
loadAdminRoles(),
// Usage + Audit (L1659-1813)
loadAdminUsage(), loadAuditLog(),
// Providers (L1814-1835)
loadAdminProviders(),
// Models (L1836-1863)
loadAdminModels(),
// Presets (L1864-1939)
loadAdminPresets(),
// Teams (L1940-2117)
loadAdminTeams(), openTeamDetail(),
loadTeamMembers(), loadMemberUserDropdown(),
// Settings (L2037-2117)
loadAdminSettings(), updateBannerPreview(),
});
```
### From app.js → 6 files
#### tokens.js (~120 lines)
Already has its own `Tokens` namespace — clean extraction.
```
const Tokens = { ... }; // L36-74
function updateInputTokens() // L78-106
function updateContextWarning() // L108-147
function dismissContextWarning() // L148-152
```
#### notes.js (~300 lines)
Self-contained with own state variables.
```
var _editingNoteId = null;
var _notesSelectMode = false;
var _selectedNoteIds = new Set();
var _notesSort = 'updated_desc';
openNotes(), loadNotesList(), _noteListItem()
_highlightHeadline()
_enterSelectMode(), _exitSelectMode()
_toggleNoteSelect(), _toggleSelectAll()
_updateSelectedCount(), _bulkDeleteSelected()
loadNoteFolders(), showNotesList()
copyNoteContent(), openNoteEditor()
_populateEditFields(), _clearEditFields()
_showNoteReadMode(), _showNoteEditMode()
_showNotePreview()
saveNote(), deleteNote()
_initNotesListeners() ← NEW: extracted from initListeners
```
#### chat.js (~500 lines)
All conversation operations.
```
// Chat management
loadChats(), selectChat(), newChat(), deleteChat()
// Per-chat model persistence
_saveChatModel(), _restoreChatModel()
// Send + stream
sendMessage(), stopGeneration()
// Active path
reloadActivePath()
// Regenerate
regenerateMessage(), regenerate()
// Edit
editMessage(), submitEdit(), cancelEdit()
// Branch navigation
switchSibling()
// Summarize & Continue
summarizeAndContinue()
// Preview
clearPreview()
_initChatListeners() ← NEW: extracted from initListeners
```
#### settings-handlers.js (~400 lines)
Settings-related handlers + command palette.
```
// Settings
handleSaveSettings(), loadSettings(), saveSettings()
// Avatar
updateAvatarPreview()
// Providers
handleCreateProvider(), deleteProvider()
refreshProviderModels(), editProvider()
// Admin settings save
handleSaveAdminSettings()
// User model roles
userRoleProviderChanged(), saveUserRole(), clearUserRole()
// Command Palette
_cmdCommands[], toggleCmdPalette(), openCmdPalette()
closeCmdPalette(), _handleCmdKey(), _highlightCmdItem()
_getVisibleCommands(), _renderCmdResults(), executeCmdAction()
_initSettingsListeners() ← NEW: extracted from initListeners
```
#### admin-handlers.js (~450 lines)
Admin actions + team management.
```
// Admin user management
toggleUserActive(), showApproveForm(), hideApproveForm()
handleApproveUser(), resetUserPassword(), promoteUser()
createAdminUser()
// Admin roles
adminSaveRole(), adminRoleProviderChanged()
// User model preferences
toggleModelVisibility(), bulkSetVisibility()
bulkSetUserModelVisibility(), deleteUserPreset()
// Admin presets
_adminPresetForm, ensureAdminPresetForm()
editAdminPreset(), createAdminPreset()
toggleAdminPreset(), deleteAdminPreset()
// Team admin (system admin)
toggleTeamActive(), deleteTeam()
updateTeamMember(), removeTeamMember()
// Team admin (settings-side)
settingsUpdateTeamMember(), settingsRemoveTeamMember()
settingsDeleteTeamPreset(), settingsToggleTeamProvider()
settingsDeleteTeamProvider(), settingsEditTeamProvider()
_initAdminListeners() ← NEW: extracted from initListeners
```
#### app.js (~400 lines) — the orchestrator
What remains after extraction.
```
// State
const App = { ... };
// Models
resolveCapabilities(), fetchModels()
// Init + Boot
init(), startApp()
// Auth flow
showSplash(), hideSplash()
handleLogin(), handleRegister(), handleLogout()
switchAuthTab(), setAuthError(), setAuthLoading()
// Modal helpers
openModal(), closeModal()
checkTabsOverflow(), updateTabArrows()
// Branding + Banners
initBranding(), initBanners()
// Listeners — thin dispatcher
function initListeners() {
if (_listenersInit) return;
_listenersInit = true;
_initChatListeners(); // from chat.js
_initSettingsListeners(); // from settings-handlers.js
_initAdminListeners(); // from admin-handlers.js
_initNotesListeners(); // from notes.js
_initGlobalKeyboard(); // local: Escape, Ctrl+K, resize
_initSidePanelResize(); // from ui-format.js
}
// Boot
document.addEventListener('DOMContentLoaded', init);
```
## Execution Steps
### 1. Preparation (before any splits)
- [ ] Branch from merged v0.10.2
- [ ] Add `node --check` validation to a quick local script:
`for f in src/js/*.js; do node --check "$f" || exit 1; done`
- [ ] Run all frontend tests as baseline: `node --test src/js/__tests__/*.test.js`
### 2. Extract ui-format.js
- [ ] Create `src/js/ui-format.js`
- [ ] Move: `esc()`, formatting functions, side panel resize, helpers
- [ ] Remove from `ui.js`
- [ ] Add `<script>` tag in `index.html` **before** `ui.js`
- [ ] Add to `sw.js` SHELL_FILES
- [ ] Run syntax check + tests
### 3. Extract ui-settings.js
- [ ] Create `src/js/ui-settings.js`
- [ ] Move settings/team UI methods out of UI object in `ui.js`
- [ ] Add `Object.assign(UI, { ... })` wrapper
- [ ] Add `<script>` tag **after** `ui.js`
- [ ] Add to `sw.js`
- [ ] Run syntax check + tests
### 4. Extract ui-admin.js
- [ ] Create `src/js/ui-admin.js`
- [ ] Move admin UI methods out of UI object
- [ ] Add `Object.assign(UI, { ... })` wrapper
- [ ] Add `<script>` tag **after** `ui.js`
- [ ] Add to `sw.js`
- [ ] Run syntax check + tests
### 5. Extract tokens.js
- [ ] Create `src/js/tokens.js`
- [ ] Move `Tokens` object + `updateInputTokens` + `updateContextWarning` + `dismissContextWarning`
- [ ] Remove from `app.js`
- [ ] Add `<script>` tag **before** `app.js`
- [ ] Add to `sw.js`
- [ ] Run syntax check + tests
### 6. Extract notes.js
- [ ] Create `src/js/notes.js`
- [ ] Move all notes state + functions
- [ ] Extract `_initNotesListeners()` from `initListeners()` — cut the
notes-related event bindings into a new function, call it from
`initListeners()`
- [ ] Add `<script>` tag **before** `app.js`
- [ ] Add to `sw.js`
- [ ] Run syntax check + tests
### 7. Extract chat.js
- [ ] Create `src/js/chat.js`
- [ ] Move chat management, send, regen, edit, branch, summarize, model persistence
- [ ] Extract `_initChatListeners()` from `initListeners()` — cut chat input,
Enter-to-send, message action delegation, etc.
- [ ] Add `<script>` tag **before** `app.js`
- [ ] Add to `sw.js`
- [ ] Run syntax check + tests
### 8. Extract settings-handlers.js
- [ ] Create `src/js/settings-handlers.js`
- [ ] Move settings handlers, provider CRUD, command palette, avatar, user roles
- [ ] Extract `_initSettingsListeners()` from `initListeners()`
- [ ] Add `<script>` tag **before** `app.js`
- [ ] Add to `sw.js`
- [ ] Run syntax check + tests
### 9. Extract admin-handlers.js
- [ ] Create `src/js/admin-handlers.js`
- [ ] Move admin actions, presets, team management
- [ ] Extract `_initAdminListeners()` from `initListeners()`
- [ ] Add `<script>` tag **before** `app.js`
- [ ] Add to `sw.js`
- [ ] Run syntax check + tests
### 10. Final validation
- [ ] `node --check` all JS files
- [ ] `node --test src/js/__tests__/*.test.js` — all 159 tests pass
- [ ] Verify `index.html` script order matches dependency graph
- [ ] Verify `sw.js` SHELL_FILES lists all new files
- [ ] Manual smoke test: login, chat, settings, admin, notes, debug
- [ ] Grep for any remaining forward references / undefined calls
- [ ] Update ROADMAP: mark v0.10.3 complete
- [ ] Update CHANGELOG
## Target State
| File | Lines | Domain |
|------|------:|--------|
| api.js | 575 | HTTP client, auth tokens |
| debug.js | 580 | Debug logger + modal |
| events.js | 327 | EventBus WebSocket |
| ui-format.js | ~280 | Markdown, code blocks, esc(), helpers |
| ui-core.js | ~650 | UI object: rendering, model selector, streaming |
| ui-settings.js | ~550 | Settings tabs, teams, providers, user prefs |
| ui-admin.js | ~650 | Admin tabs, users, roles, usage, teams |
| tokens.js | ~120 | Context tracking, token estimation |
| notes.js | ~300 | Notes panel, editor, multi-select |
| chat.js | ~500 | Chat ops, send, regen, edit, branch, summarize |
| settings-handlers.js | ~400 | Settings save, provider CRUD, command palette |
| admin-handlers.js | ~450 | Admin actions, presets, team management |
| app.js | ~400 | State, init, boot, auth, listener dispatch |
| **Total** | **~5,782** | 13 files (was 5) |
Line count drops slightly due to removed duplicate section headers and
consolidated comments. Average file: ~445 lines. Largest: ~650.
## Risk Mitigation
**Forward references:** Function declarations are hoisted within a
`<script>`, but not across scripts. All functions called by
`initListeners()` must be defined in scripts loaded before `app.js`.
The load order above ensures this.
**`UI` object extension:** `Object.assign(UI, { ... })` in
ui-settings.js and ui-admin.js means `UI` must be defined first
(in ui-core.js). The load order handles this.
**Regression testing:** Each extraction step ends with syntax check +
test run. If anything breaks, it's isolated to the last extraction.
**Rollback:** Each step is a single commit. Revert any step independently.
## What This Does NOT Change
- No ES modules, no import/export, no bundler
- No function renaming (HTML onclick handlers untouched)
- No new dependencies
- No architectural changes to backend
- No feature additions
- The `UI` object is still a single global — just defined across files
- All tests continue to work unchanged

64
docs/ROADMAP.md
View File

@@ -61,12 +61,12 @@ v0.16.0 User Groups v0.17.0 Persona-KB Binding
┌───────┴──────────────┐
│ │
v0.17.1 SQLite Backend ✅ v0.17.2 CodeMirror 6 ✅
v0.17.3 Notes Graph + Wikilinks
v0.17.3 Notes Graph + Wikilinks ✅
(dual DB) (editor bundle, chat
│ input, ext editor)
└───────┬──────────────┘
│
v0.18.0 Memory (user + persona scopes, review pipeline)
v0.18.0 Memory ✅ (user + persona scopes, review pipeline)
│
v0.18.1 Side Panel Architecture (independent panels, dual-view)
│
@@ -340,7 +340,7 @@ Depends on: nothing (frontend-only). Prerequisite for: extension surfaces
---
## v0.17.3 — Notes Graph + Wikilinks
## v0.17.3 — Notes Graph + Wikilinks ✅
Knowledge graph and bi-directional linking for the notes system. Transforms
notes from flat documents into an interconnected knowledge base with
@@ -391,7 +391,7 @@ Depends on: CodeMirror 6 (v0.17.2), Notes CRUD (v0.13.x). See
---
## v0.18.0 — Memory (User + Persona Scopes)
## v0.18.0 — Memory (User + Persona Scopes) ✅
Long-term memory across conversations, with scope-aware isolation.
Unlike KB (static documents) or compaction (within-conversation), this
@@ -403,52 +403,54 @@ contamination and enabling specialized knowledge accumulation.
Depends on: compaction (v0.15.0), knowledge bases (v0.14.0), persona-KB binding (v0.17.0), SQLite backend (v0.17.1).
See [DESIGN-0.18.0.md](DESIGN-0.18.0.md) for full spec.
**Memory Scopes**
- [ ] `user` — personal facts/preferences, persists across all conversations (current industry standard)
- [ ] `persona` — shared across all users of that Persona, accumulated from conversations (differentiator)
- [ ] `persona+user` — per-user within a Persona context (e.g. tutoring progress per student)
- [ ] Configurable per Persona: which scopes are active, whether user memory is passed through
- [x] `user` — personal facts/preferences, persists across all conversations (current industry standard)
- [x] `persona` — shared across all users of that Persona, accumulated from conversations (differentiator)
- [x] `persona+user` — per-user within a Persona context (e.g. tutoring progress per student)
- [x] Configurable per Persona: which scopes are active, whether user memory is passed through
**Data Model**
- [ ] `memories` table: `id`, `scope` (user, persona, persona_user), `owner_id` (user_id or persona_id), `user_id` (nullable — for persona+user scope), `key`, `value`, `source_channel_id`, `confidence` (float), `status` (active, pending_review, archived), `created_at`, `updated_at`
- [ ] Index on `(scope, owner_id, user_id)` for fast lookup
- [ ] Embedding column (`vector(3072)`) for semantic memory search
- [x] `memories` table: `id`, `scope` (user, persona, persona_user), `owner_id` (user_id or persona_id), `user_id` (nullable — for persona+user scope), `key`, `value`, `source_channel_id`, `confidence` (float), `status` (active, pending_review, archived), `created_at`, `updated_at`
- [x] Index on `(scope, owner_id, user_id)` for fast lookup
- [x] Embedding column (`vector(3072)`) for semantic memory search
**Tools**
- [ ] `memory_save` tool: LLM explicitly stores a fact
- [x] `memory_save` tool: LLM explicitly stores a fact
- Scope-aware: saves to the active memory scope for the current Persona context
- Structured: `key` (short label) + `value` (detail) + `confidence`
- [ ] `memory_recall` tool: LLM queries stored facts relevant to current context
- [x] `memory_recall` tool: LLM queries stored facts relevant to current context
- Merges results from applicable scopes (user + persona + persona+user)
- Semantic search via embeddings + keyword fallback
**Automatic Extraction**
- [ ] Background job: post-conversation analysis identifies memorable facts (opt-in)
- [ ] Configurable extraction prompt per Persona (e.g. "extract FAQ-worthy Q&A pairs")
- [ ] Extracted memories start in `pending_review` status
- [x] Background job: post-conversation analysis identifies memorable facts (opt-in)
- [x] Configurable extraction prompt per Persona (e.g. "extract FAQ-worthy Q&A pairs")
- [x] Extracted memories start in `pending_review` status
**Review Pipeline**
- [ ] Admin/team admin review queue: pending memories with approve/reject/edit
- [ ] Persona memory review: team admins see what Personas are "learning"
- [ ] User memory review: users see their own memories in Settings
- [x] Admin/team admin review queue: pending memories with approve/reject/edit
- [x] Persona memory review: team admins see what Personas are "learning"
- [x] User memory review: users see their own memories in Settings
**Memory Injection**
- [ ] At completion time: inject relevant memories into system prompt
- [ ] Context budget aware: limit injected memory tokens
- [ ] Scope priority: persona+user > persona > user (most specific wins on conflicts)
- [x] At completion time: inject relevant memories into system prompt
- [x] Context budget aware: limit injected memory tokens
- [x] Scope priority: persona+user > persona > user (most specific wins on conflicts)
**Use Cases**
- [ ] Helpdesk Persona: builds FAQ from repeated questions, human-reviewable
- [ ] Roleplay Persona: isolated memory prevents tone/context bleeding from user's other conversations
- [ ] Tutoring Persona: tracks per-student progress, common misconceptions
- [ ] Sales Persona: accumulates objection responses from real conversations
- [ ] Onboarding Persona: learns what new hires commonly struggle with
- [x] Helpdesk Persona: builds FAQ from repeated questions, human-reviewable
- [x] Roleplay Persona: isolated memory prevents tone/context bleeding from user's other conversations
- [x] Tutoring Persona: tracks per-student progress, common misconceptions
- [x] Sales Persona: accumulates objection responses from real conversations
- [x] Onboarding Persona: learns what new hires commonly struggle with
**User Controls**
- [ ] Settings → Memory: view, edit, delete personal memories
- [ ] Per-Persona memory toggle: "Allow this persona to remember things about me"
- [ ] Admin: enable/disable, retention policies, per-team scoping
- [ ] Privacy: user memories never cross team boundaries; persona memories respect group access
- [x] Settings → Memory: view, edit, delete personal memories
- [x] Per-Persona memory toggle: "Allow this persona to remember things about me"
- [x] Admin: enable/disable, retention policies, per-team scoping
- [x] Privacy: user memories never cross team boundaries; persona memories respect group access
---

358
docs/UI-AUDIT-0.10.5.md
View File

@@ -1,358 +0,0 @@
# UI Duplication Audit — v0.10.5 ✅ IMPLEMENTED
Systematic catalog of every repeated pattern in the frontend.
Each section: what's duplicated, where, and what's inconsistent.
---
## 1. Provider Form (CREATE + EDIT) — 3 copies
The same "configure a provider" concept exists in **3 places** with
different field IDs, different features, and different behaviors.
### Where
| Context | Create handler | Edit handler | HTML form IDs |
|---------|---------------|-------------|---------------|
| **User BYOK** | `handleCreateProvider()` settings-handlers.js:88 | same fn (dual-mode via `UI._editingProviderId`) | `providerName`, `providerType`, `providerEndpoint`, `providerApiKey`, `providerDefaultModel` |
| **Admin Global** | `createGlobalProvider()` admin-handlers.js:122 | same fn (dual-mode via `_editingProviderId`) | `adminProvName`, `adminProvType`, `adminProvEndpoint`, `adminProvKey`, `adminProvModel`, `adminProvPrivate` |
| **Team** | inline anon listener settings-handlers.js:571 | `settingsTeamEditProviderSubmit` listener :597 | Create: `teamProviderName`, `teamProviderType`, `teamProviderEndpoint`, `teamProviderKey`, `teamProviderPrivate` / Edit: `teamProviderEditName`, `teamProviderEditEndpoint`, `teamProviderEditKey`, `teamProviderEditDefault`, `teamProviderEditPrivate` |
### Inconsistencies
| Feature | User BYOK | Admin | Team Create | Team Edit |
|---------|-----------|-------|-------------|-----------|
| **Endpoint auto-fill on type change** | ✅ | ❌ MISSING | ✅ | ❌ N/A (no type field) |
| **"Private" checkbox** | ❌ | ✅ | ✅ | ✅ |
| **"Default Model" field** | ✅ | ✅ | ❌ MISSING | ✅ |
| **Provider type dropdown** | Static HTML | Static HTML | Dynamic JS (same labels) | N/A (no type change) |
| **Provider types defined in** | index.html:408 | index.html:626 | settings-handlers.js:553 | — |
| **Endpoint defaults defined in** | settings-handlers.js:480 | NOWHERE | settings-handlers.js:542 | — |
| **API key placeholder** | "sk-..." / "(unchanged)" | "sk-..." / "(unchanged)" | "sk-..." | "(unchanged)" |
| **Post-save refresh** | `loadProviderList()` + `fetchModels()` | `loadAdminProviders()` (no fetchModels) | `loadTeamManageProviders()` + `fetchModels()` | `loadTeamManageProviders()` (no fetchModels) |
| **Create/Edit dual-mode** | Single fn, flag | Single fn, flag | Separate forms + separate listeners | Separate forms + separate listeners |
| **Form show/hide** | `UI.showProviderForm()`/`hideProviderForm()` | inline `.style.display` | inline `.style.display` | inline `.style.display` |
### Constants duplicated
```
Provider types: 3× — index.html (2 static selects) + settings-handlers.js (1 dynamic build)
Provider labels: 3× — { openai: 'OpenAI-compatible', anthropic: 'Anthropic', venice: 'Venice.ai', openrouter: 'OpenRouter' }
Endpoint map: 2× — settings-handlers.js:480 and :542 (admin has ZERO)
```
---
## 2. Provider List (READ) — 3 copies
| Context | Function | File:Line | API call |
|---------|----------|-----------|----------|
| **User BYOK** | `UI.loadProviderList()` | ui-settings.js:534 | `API.listConfigs()` |
| **Admin Global** | `UI.loadAdminProviders()` | ui-admin.js:343 | `API.adminListGlobalConfigs()` |
| **Team** | `UI.loadTeamManageProviders(teamId)` | ui-settings.js:191 | `API.teamListProviders(teamId)` |
### Inconsistencies
| Feature | User BYOK | Admin | Team |
|---------|-----------|-------|------|
| Shows endpoint | ❌ | ✅ | ❌ |
| Shows provider type | ✅ (in meta) | ✅ (in meta) | ✅ (in meta) |
| Shows has_key indicator | ✅ 🔑/⚠️ | ❌ | ✅ 🔑 |
| Shows active/inactive toggle | ❌ | ❌ | ✅ |
| Shows private badge | ❌ | ✅ | ✅ |
| Edit button | ✅ | ✅ | ✅ |
| Delete button | ✅ | ✅ | ✅ |
| Refresh models button | ✅ | ❌ (separate bulk) | ❌ |
| Row CSS class | `provider-row` | `admin-provider-row` | `admin-preset-row` (!) |
| Empty state msg | different | different | different |
| Provider cache | ❌ | ✅ (`UI._providerCache`) | ❌ |
---
## 3. Role Configuration — 2 copies
| Context | Render function | Provider change handler | Save handler |
|---------|----------------|------------------------|-------------|
| **Admin Global** | `UI.loadAdminRoles()` ui-admin.js:134 | `adminRoleProviderChanged()` admin-handlers.js:199 | `adminSaveRole()` admin-handlers.js:216 |
| **User Personal** | `UI.loadUserRoles()` ui-settings.js:356 | `userRoleProviderChanged()` settings-handlers.js:214 | `saveUserRole()` settings-handlers.js:231 |
### Inconsistencies
| Feature | Admin | User |
|---------|-------|------|
| Fallback slot | ✅ primary + fallback | ❌ primary only |
| Test button | ✅ | ❌ |
| Clear/remove button | ❌ | ✅ |
| Models source | `window._adminModelList` (from admin API) | `App.models` (from enabled API) |
| Model ID field | `m.model_id` | `m.baseModelId` |
| Reload on save | ✅ (added in 0.10.4) | ✅ (always did) |
| BYOK gate check | ❌ (admin sees always) | ✅ (hidden if no BYOK) |
---
## 4. Capability Badges — 3 copies
The same badge rendering logic exists in 3 places with different levels of detail:
| Context | File:Line | Badges shown |
|---------|-----------|-------------|
| **Model selector** (chat) | ui-core.js:759-774 | max_output, max_context, tools, vision, thinking, reasoning, code, search — with titles and labels |
| **Admin model list** | ui-admin.js:375-378 | max_output, tools, vision, thinking — compact, no titles |
| **User model list** | ui-settings.js:596-599 | max_output, tools, vision, thinking — compact, no titles |
Admin and User are identical copies. Model selector has more detail.
---
## 5. Usage Dashboard — 3 copies
| Context | Function | File:Line | API call |
|---------|----------|-----------|----------|
| **Admin** | `UI.loadAdminUsage()` | ui-admin.js:190 | `API.adminGetUsage()` |
| **User (My Usage)** | `UI.loadMyUsage()` | ui-settings.js:246 | `API.getMyUsage()` |
| **Team** | `UI.loadTeamUsage()` | ui-settings.js:297 | `API.teamGetUsage()` |
### Inconsistencies
| Feature | Admin | User | Team |
|---------|-------|------|------|
| Pricing table | ✅ | ❌ | ❌ |
| "group by user" column header | ✅ | ❌ | ✅ |
| Stats card styling | default padding | `padding:8px; font-size:16px` | `padding:8px; font-size:16px` |
| Stats label font | default | `font-size:10px` | `font-size:10px` |
| Empty state check | `results.length === 0` | `!t.requests` | `!t.requests` |
| Grid layout | default | `grid-template-columns:repeat(4,1fr)` | `grid-template-columns:repeat(4,1fr)` |
| Table font size | default | `font-size:12px` | `font-size:12px` |
User and Team are near-identical. Admin is a separate fork.
---
## 6. Provider Type / Endpoint Constants — scattered everywhere
The "known provider types" concept is defined in **5 places**:
| # | Location | Format |
|---|----------|--------|
| 1 | index.html:408 | `<option>` tags in User BYOK select |
| 2 | index.html:626 | `<option>` tags in Admin select |
| 3 | settings-handlers.js:553 | JS object `{ openai: 'OpenAI-compatible', ... }` for Team dynamic build |
| 4 | settings-handlers.js:480 | Endpoint map for User BYOK |
| 5 | settings-handlers.js:542 | Endpoint map for Team (separate copy) |
| — | server/providers/registry.go | Backend `Init()` registers providers — source of truth |
Adding a new provider type requires touching 5 frontend locations.
---
## 7. Preset Form — ALREADY SHARED ✅
`renderPresetForm()` in ui-core.js:33 is used by:
- Admin presets (admin-handlers.js:303)
- Team presets (settings-handlers.js:511)
- User presets (settings-handlers.js:626)
This is the ONE pattern that's already been consolidated into a primitive.
Good reference for how the other patterns should work.
---
## 8. Model List — 2 copies
| Context | Function | File:Line |
|---------|----------|-----------|
| **Admin models** | `UI.loadAdminModels()` | ui-admin.js:365 |
| **User models** | `UI.loadUserModels()` | ui-settings.js:571 |
Admin shows visibility toggle, user shows hide/unhide toggle.
Both render capability badges (same compact format, duplicated).
---
## Summary: Primitives Needed
| Primitive | Replaces | Copies eliminated |
|-----------|----------|-------------------|
| `PROVIDERS` — type list, label map, endpoint defaults | 5 definitions → 1 | 4 |
| `renderProviderForm(container, options)` | 3 HTML forms + 6 handlers → 1 | ~200 lines |
| `renderProviderList(container, items, options)` | 3 list renderers → 1 | ~80 lines |
| `renderRoleConfig(container, options)` | 2 role UIs + 4 handlers → 1 | ~120 lines |
| `renderCapBadges(caps, compact?)` | 3 badge builders → 1 | ~30 lines |
| `renderUsageDashboard(container, options)` | 3 usage renderers → 1 | ~120 lines |
**Estimated net reduction: ~550 lines + single source of truth for all constants.**
---
## Proposed File: `ui-primitives.js`
Loaded after `ui-format.js`, before `ui-core.js`. Contains:
```
// Constants (single source of truth)
const PROVIDER_TYPES = [ ... ]; // { id, label, defaultEndpoint }
const ROLE_NAMES = ['utility', 'embedding'];
const ROLE_TYPE_MAP = { embedding: 'embedding', utility: 'chat' };
// Rendering primitives
function renderCapBadges(caps, opts) → HTML string
function renderProviderForm(container, opts) → { getValues, setValues, clear }
function renderProviderList(container, opts) → refresh function
function renderRoleConfig(container, opts) → refresh function
function renderUsageDashboard(container, opts) → refresh function
```
Each primitive follows the `renderPresetForm` pattern:
- Takes a container element + options object
- Returns a control object with getValues/setValues/refresh
- Options include callbacks (onSubmit, onDelete, etc.) and feature flags
- Scope-specific behavior (admin/team/personal) via options, not separate code
---
## 9. Extension Surface Implications
The extension spec (EXTENSIONS.md §6) defines injection points:
```
sidebar-top, sidebar-nav, sidebar-content, sidebar-bottom
surface-header, surface-main, surface-footer
```
Extensions interact with primitives through `ctx.ui.inject()` and
`ctx.ui.replace()`. The cleaner the primitive interface, the easier
extensions can:
1. **Add to** — e.g. a custom provider type, a new capability badge,
an extra usage metric column
2. **Hook into** — e.g. react when a provider is created, when a role
is saved, when models refresh
3. **Reuse** — e.g. render a provider form inside an extension's own
settings surface
### 9.1 What Extensions Will Want to Do
| Extension | Wants to... | Primitive it hooks |
|-----------|------------|-------------------|
| Custom provider adapter | Add a provider type (e.g. "Ollama", "vLLM") | `PROVIDER_TYPES` registry |
| Cost tracker | Add badges (cost/msg), extra usage columns | `renderCapBadges`, `renderUsageDashboard` |
| Image gen | Register a role + show config UI | `renderRoleConfig` |
| Smart routing | Show routing rules alongside role config | `renderRoleConfig` |
| Cluster manager | Render provider forms in its own surface | `renderProviderForm` |
| KB/RAG | Show embedding model status in sidebar | `ROLE_NAMES`, role resolution |
### 9.2 Design Rules for Extension-Ready Primitives
**Registry pattern, not constants.**
Don't just export `PROVIDER_TYPES = [...]` as a frozen array.
Make it a registry that extensions can `.add()` to:
```js
const Providers = {
_types: [ /* builtins */ ],
add(entry) { this._types.push(entry); },
all() { return this._types; },
get(id) { return this._types.find(t => t.id === id); },
endpoints() { /* map id → defaultEndpoint */ },
labels() { /* map id → label */ },
};
```
Same for roles — `Roles.add('generation', { typeFilter: 'image', label: '...' })`.
**Options objects, not positional args.**
Every primitive takes `(container, options)`. Options are the extension
point — an extension can wrap a primitive call and inject additional
options:
```js
// Extension adds a custom column to usage
renderUsageDashboard(el, {
apiCall: () => API.getMyUsage({ period, group_by }),
extraColumns: [
{ header: 'Est. Cost/msg', render: (r) => '$' + (r.total_cost / r.requests).toFixed(4) }
]
});
```
**Return control handles.**
Every primitive returns an object with methods, not just void:
```js
const provForm = renderProviderForm(el, opts);
// Extensions can:
provForm.getValues() // read current state
provForm.setValues({...}) // programmatically fill
provForm.clear() // reset
provForm.onTypeChange(fn) // hook into type selection
provForm.container // DOM reference for injection
```
**Emit events on state changes.**
Primitives should fire EventBus events that extensions can subscribe to:
```
provider.form.typeChanged — user picked a different provider type
provider.created — provider successfully saved (already exists)
provider.deleted — provider removed
role.saved — role config updated
models.refreshed — model list reloaded (already exists)
```
Most of these events already exist or will exist in the EventBus. The
primitives just need to emit them consistently rather than each copy
doing ad-hoc `UI.toast()` + `fetchModels()` in different orders.
**Scope via options, not separate code.**
The biggest win: one `renderProviderForm` handles admin/team/personal
through options, not three separate implementations:
```js
renderProviderForm(el, {
scope: 'admin', // → shows private checkbox, no BYOK gate
// scope: 'team', // → shows private checkbox, scoped API
// scope: 'personal', // → no private, BYOK gated
apiCreate: (vals) => API.adminCreateGlobalConfig(...),
apiUpdate: (id, vals) => API.adminUpdateGlobalConfig(id, ...),
onSaved: () => { UI.loadAdminProviders(); },
showPrivate: true,
showDefaultModel: true,
});
```
### 9.3 Primitive → Extension Migration Path
These primitives are **pre-extension infrastructure**. When the extension
loader (v0.11.0) lands, the migration is:
1. `Providers` registry gets a `ctx.providers.add()` wrapper in the
extension context
2. `renderCapBadges` checks a `capBadgeRegistry` that extensions can add to
3. `renderUsageDashboard` accepts `extraColumns` from extension settings
4. `renderRoleConfig` reads `Roles.all()` instead of hardcoded list
The primitives we build now become the extension API surface later.
No rewrite needed — just wrapping in the permission-scoped `ctx` object.
---
## 10. Implementation Plan
### Phase 1: Constants + Small Primitives
- `Providers` registry (types, labels, endpoints)
- `Roles` registry (names, type filters, descriptions)
- `renderCapBadges(caps, opts)` — consolidate 3 badge builders
### Phase 2: Provider Primitives
- `renderProviderForm(container, opts)` — replace 3 form implementations
- `renderProviderList(container, opts)` — replace 3 list renderers
- Update index.html: remove 2 static provider selects, add empty containers
### Phase 3: Dashboard Primitives
- `renderUsageDashboard(container, opts)` — replace 3 usage renderers
- `renderRoleConfig(container, opts)` — replace 2 role UIs
### Phase 4: Cleanup
- Remove dead code from admin-handlers.js, settings-handlers.js
- Update tests
- Verify all 3 scopes (admin/team/personal) work identically

85
server/database/migrations/004_v0180_memories.sql Normal file
View File

@@ -0,0 +1,85 @@
-- v0.18.0: Memory System
--
-- Long-term memory across conversations, with scope-aware isolation.
-- Three scopes: user (personal), persona (shared), persona_user (per-user-per-persona).
--
-- Unlike KB (static documents) or compaction (within-conversation), this
-- captures facts and preferences that persist across channels.
-- =========================================
-- 1. Memories Table
-- =========================================
CREATE TABLE IF NOT EXISTS memories (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
scope TEXT NOT NULL CHECK (scope IN ('user', 'persona', 'persona_user')),
-- Owner resolution:
-- user scope: owner_id = user_id, user_id = NULL
-- persona scope: owner_id = persona_id, user_id = NULL
-- persona_user scope: owner_id = persona_id, user_id = user_id
owner_id UUID NOT NULL,
user_id UUID,
key TEXT NOT NULL, -- short label ("preferred language", "team size")
value TEXT NOT NULL, -- detail / fact content
source_channel_id UUID, -- which conversation produced this memory
confidence REAL NOT NULL DEFAULT 1.0, -- 0.0–1.0, used for ranking
status TEXT NOT NULL DEFAULT 'active'
CHECK (status IN ('active', 'pending_review', 'archived')),
-- Embedding for semantic recall (optional — same pattern as notes)
embedding vector(3072),
created_at TIMESTAMPTZ DEFAULT now(),
updated_at TIMESTAMPTZ DEFAULT now()
);
-- Primary lookup: "give me all active memories for this scope + owner"
CREATE INDEX IF NOT EXISTS idx_memories_scope_owner
ON memories(scope, owner_id, status)
WHERE status = 'active';
-- Persona+user lookup: "give me this persona's memories about this user"
CREATE INDEX IF NOT EXISTS idx_memories_persona_user
ON memories(owner_id, user_id)
WHERE scope = 'persona_user' AND status = 'active';
-- Deduplication: prevent saving the same key twice for the same scope/owner/user
CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_unique_key
ON memories(scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key);
-- Full-text search on key + value
CREATE INDEX IF NOT EXISTS idx_memories_fts
ON memories USING gin(to_tsvector('english', key || ' ' || value));
-- Source channel reference (for provenance / "where did this come from")
CREATE INDEX IF NOT EXISTS idx_memories_source_channel
ON memories(source_channel_id)
WHERE source_channel_id IS NOT NULL;
COMMENT ON TABLE memories IS 'Long-term memory facts persisted across conversations, scoped to user, persona, or user+persona';
COMMENT ON COLUMN memories.scope IS 'user = personal facts; persona = shared persona knowledge; persona_user = per-user within a persona';
COMMENT ON COLUMN memories.owner_id IS 'user_id for user scope, persona_id for persona/persona_user scopes';
COMMENT ON COLUMN memories.user_id IS 'Only set for persona_user scope — identifies which user within the persona';
COMMENT ON COLUMN memories.key IS 'Short label for the memory (e.g. "preferred language", "deployment stack")';
COMMENT ON COLUMN memories.value IS 'The actual fact/detail being remembered';
COMMENT ON COLUMN memories.confidence IS '0.0-1.0 confidence score — higher = more certain, used for ranking and pruning';
COMMENT ON COLUMN memories.status IS 'active = injected into context; pending_review = awaiting human approval; archived = soft-deleted';
-- =========================================
-- 2. Updated-at Trigger
-- =========================================
CREATE OR REPLACE FUNCTION update_memories_updated_at()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = now();
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER trg_memories_updated_at
BEFORE UPDATE ON memories
FOR EACH ROW
EXECUTE FUNCTION update_memories_updated_at();

46
server/database/migrations/005_v0180_memory_phase2.sql Normal file
View File

@@ -0,0 +1,46 @@
-- v0.18.0 Phase 2: Memory Extraction & Embeddings
--
-- Adds persona memory configuration, HNSW vector index for
-- semantic recall, and extraction tracking log.
-- =========================================
-- 1. Persona Memory Configuration
-- =========================================
ALTER TABLE personas ADD COLUMN IF NOT EXISTS memory_enabled BOOLEAN NOT NULL DEFAULT true;
ALTER TABLE personas ADD COLUMN IF NOT EXISTS memory_extraction_prompt TEXT;
COMMENT ON COLUMN personas.memory_enabled IS 'Whether memory tools and extraction are active for this persona';
COMMENT ON COLUMN personas.memory_extraction_prompt IS 'Custom extraction prompt; NULL = use system default';
-- =========================================
-- 2. Vector Index for Semantic Recall
-- =========================================
-- NOTE: HNSW indexes have a 2000-dimension limit in pgvector.
-- Our embeddings are 3072-dim (matching KB/notes schema), so we
-- skip the HNSW index. Memory tables are small enough (hundreds
-- of rows per user) that filtered sequential scans with cosine
-- distance are performant. If needed later, IVFFlat supports
-- higher dimensions, or embeddings can be reduced to 1536-dim.
-- =========================================
-- 3. Memory Extraction Log
-- =========================================
-- Tracks which messages have been processed by the extraction scanner
-- to prevent duplicate work. One row per channel+user pair.
CREATE TABLE IF NOT EXISTS memory_extraction_log (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
channel_id UUID NOT NULL,
user_id UUID NOT NULL,
last_message_id UUID NOT NULL,
extracted_at TIMESTAMPTZ DEFAULT now(),
memory_count INT NOT NULL DEFAULT 0,
UNIQUE(channel_id, user_id)
);
CREATE INDEX IF NOT EXISTS idx_mem_extract_log_channel
ON memory_extraction_log(channel_id);
COMMENT ON TABLE memory_extraction_log IS 'Tracks extraction progress per channel+user to prevent re-processing';

61
server/database/migrations/sqlite/003_v0180_memories.sql Normal file
View File

@@ -0,0 +1,61 @@
-- v0.18.0: Memory System (SQLite)
--
-- Long-term memory across conversations, with scope-aware isolation.
-- Three scopes: user (personal), persona (shared), persona_user (per-user-per-persona).
-- =========================================
-- 1. Memories Table
-- =========================================
CREATE TABLE IF NOT EXISTS memories (
id TEXT PRIMARY KEY,
scope TEXT NOT NULL CHECK (scope IN ('user', 'persona', 'persona_user')),
-- Owner resolution:
-- user scope: owner_id = user_id, user_id = NULL
-- persona scope: owner_id = persona_id, user_id = NULL
-- persona_user scope: owner_id = persona_id, user_id = user_id
owner_id TEXT NOT NULL,
user_id TEXT,
key TEXT NOT NULL,
value TEXT NOT NULL,
source_channel_id TEXT,
confidence REAL NOT NULL DEFAULT 1.0,
status TEXT NOT NULL DEFAULT 'active'
CHECK (status IN ('active', 'pending_review', 'archived')),
-- Embedding stored as JSON array of float64 for app-level cosine similarity
embedding TEXT,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now'))
);
-- Primary lookup
CREATE INDEX IF NOT EXISTS idx_memories_scope_owner
ON memories(scope, owner_id, status);
-- Persona+user lookup
CREATE INDEX IF NOT EXISTS idx_memories_persona_user
ON memories(owner_id, user_id);
-- Deduplication: prevent saving the same key twice for the same scope/owner/user
CREATE UNIQUE INDEX IF NOT EXISTS idx_memories_unique_key
ON memories(scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key);
-- Source channel reference
CREATE INDEX IF NOT EXISTS idx_memories_source_channel
ON memories(source_channel_id);
-- =========================================
-- 2. Updated-at Trigger
-- =========================================
CREATE TRIGGER IF NOT EXISTS trg_memories_updated_at
AFTER UPDATE ON memories
FOR EACH ROW
WHEN NEW.updated_at = OLD.updated_at
BEGIN
UPDATE memories SET updated_at = datetime('now') WHERE id = NEW.id;
END;

22
server/database/migrations/sqlite/004_v0180_memory_phase2.sql Normal file
View File

@@ -0,0 +1,22 @@
-- v0.18.0 Phase 2: Memory Extraction & Embeddings (SQLite)
-- 1. Persona memory configuration
ALTER TABLE personas ADD COLUMN memory_enabled INTEGER NOT NULL DEFAULT 1;
ALTER TABLE personas ADD COLUMN memory_extraction_prompt TEXT;
-- 2. No HNSW index for SQLite — app-level cosine similarity used instead.
-- 3. Memory extraction log
CREATE TABLE IF NOT EXISTS memory_extraction_log (
id TEXT PRIMARY KEY,
channel_id TEXT NOT NULL,
user_id TEXT NOT NULL,
last_message_id TEXT NOT NULL,
extracted_at TEXT DEFAULT (datetime('now')),
memory_count INTEGER NOT NULL DEFAULT 0,
UNIQUE(channel_id, user_id)
);
CREATE INDEX IF NOT EXISTS idx_mem_extract_log_channel
ON memory_extraction_log(channel_id);

17
server/handlers/completion.go
View File

@@ -17,6 +17,7 @@ import (
"git.gobha.me/xcaliber/chat-switchboard/crypto"
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/events"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/providers"
"git.gobha.me/xcaliber/chat-switchboard/storage"
@@ -47,11 +48,12 @@ type CompletionHandler struct {
stores store.Stores
hub *events.Hub // WebSocket hub for browser tool bridge
objStore storage.ObjectStore // file storage for attachment content (nil = disabled)
embedder *knowledge.Embedder // for memory semantic recall (v0.18.0)
}
// NewCompletionHandler creates a new handler.
func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub, objStore storage.ObjectStore) *CompletionHandler {
return &CompletionHandler{vault: vault, stores: stores, hub: hub, objStore: objStore}
func NewCompletionHandler(vault *crypto.KeyResolver, stores store.Stores, hub *events.Hub, objStore storage.ObjectStore, embedder *knowledge.Embedder) *CompletionHandler {
return &CompletionHandler{vault: vault, stores: stores, hub: hub, objStore: objStore, embedder: embedder}
}
// ── Chat Completion ─────────────────────────
@@ -797,6 +799,17 @@ func (h *CompletionHandler) loadConversation(channelID, userID, presetSystemProm
})
}
// ── Memory hint (inject known facts about this user — v0.18.0) ──
// We pass empty lastUserMessage here; the current message content is available
// at the call site but not in loadConversation. Semantic recall will improve
// when the hint is built closer to the user's message.
if memHint := BuildMemoryHint(context.Background(), h.stores, h.embedder, userID, personaID, ""); memHint != "" {
messages = append(messages, providers.Message{
Role: "system",
Content: memHint,
})
}
// Walk the active path (root → leaf) instead of loading all messages
path, err := getActivePath(channelID, userID)
if err != nil {

2
server/handlers/integration_test.go
View File

@@ -247,7 +247,7 @@ func setupHarness(t *testing.T) *testHarness {
protected.DELETE("/attachments/:id", attachH.DeleteAttachment)
// Completions
completions := NewCompletionHandler(nil, stores, nil, nil)
completions := NewCompletionHandler(nil, stores, nil, nil, nil)
protected.POST("/chat/completions", completions.Complete)
// Messages

223
server/handlers/memory.go Normal file
View File

@@ -0,0 +1,223 @@
package handlers
import (
"net/http"
"github.com/gin-gonic/gin"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// MemoryHandler provides REST endpoints for memory management.
type MemoryHandler struct {
stores store.Stores
}
// NewMemoryHandler creates a new memory handler.
func NewMemoryHandler(stores store.Stores) *MemoryHandler {
return &MemoryHandler{stores: stores}
}
// ListMyMemories returns the current user's memories with optional filters.
// GET /api/v1/memories?status=active&query=...
func (h *MemoryHandler) ListMyMemories(c *gin.Context) {
userID := c.GetString("user_id")
if userID == "" {
c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized"})
return
}
status := c.DefaultQuery("status", "active")
query := c.Query("query")
memories, err := h.stores.Memories.List(c.Request.Context(), models.MemoryFilter{
Scope: models.MemoryScopeUser,
OwnerID: userID,
Status: status,
Query: query,
Limit: 100,
})
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"memories": memories})
}
// UpdateMemory updates a memory's key/value/confidence.
// PUT /api/v1/memories/:id
func (h *MemoryHandler) UpdateMemory(c *gin.Context) {
userID := c.GetString("user_id")
memoryID := c.Param("id")
existing, err := h.stores.Memories.GetByID(c.Request.Context(), memoryID)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "memory not found"})
return
}
// Ownership check: user can only edit their own user-scope memories
if existing.Scope == models.MemoryScopeUser && existing.OwnerID != userID {
c.JSON(http.StatusForbidden, gin.H{"error": "not your memory"})
return
}
var body struct {
Key *string `json:"key"`
Value *string `json:"value"`
Confidence *float64 `json:"confidence"`
}
if err := c.ShouldBindJSON(&body); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"})
return
}
if body.Key != nil {
existing.Key = *body.Key
}
if body.Value != nil {
existing.Value = *body.Value
}
if body.Confidence != nil {
existing.Confidence = *body.Confidence
}
if err := h.stores.Memories.Upsert(c.Request.Context(), existing); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"memory": existing})
}
// DeleteMemory hard-deletes a memory.
// DELETE /api/v1/memories/:id
func (h *MemoryHandler) DeleteMemory(c *gin.Context) {
userID := c.GetString("user_id")
memoryID := c.Param("id")
existing, err := h.stores.Memories.GetByID(c.Request.Context(), memoryID)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "memory not found"})
return
}
if existing.Scope == models.MemoryScopeUser && existing.OwnerID != userID {
c.JSON(http.StatusForbidden, gin.H{"error": "not your memory"})
return
}
if err := h.stores.Memories.Delete(c.Request.Context(), memoryID); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"deleted": true})
}
// ApproveMemory transitions a pending_review memory to active.
// POST /api/v1/memories/:id/approve
func (h *MemoryHandler) ApproveMemory(c *gin.Context) {
memoryID := c.Param("id")
existing, err := h.stores.Memories.GetByID(c.Request.Context(), memoryID)
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "memory not found"})
return
}
existing.Status = models.MemoryStatusActive
if err := h.stores.Memories.Upsert(c.Request.Context(), existing); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"memory": existing})
}
// RejectMemory archives a pending_review memory.
// POST /api/v1/memories/:id/reject
func (h *MemoryHandler) RejectMemory(c *gin.Context) {
memoryID := c.Param("id")
if err := h.stores.Memories.Archive(c.Request.Context(), memoryID); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
c.JSON(http.StatusOK, gin.H{"rejected": true})
}
// MemoryCount returns active + pending counts for the current user.
// GET /api/v1/memories/count
func (h *MemoryHandler) MemoryCount(c *gin.Context) {
userID := c.GetString("user_id")
ctx := c.Request.Context()
active, _ := h.stores.Memories.CountByOwner(ctx, models.MemoryScopeUser, userID)
// Count pending by listing
pending, _ := h.stores.Memories.List(ctx, models.MemoryFilter{
Scope: models.MemoryScopeUser,
OwnerID: userID,
Status: models.MemoryStatusPendingReview,
Limit: 1000,
})
c.JSON(http.StatusOK, gin.H{
"active": active,
"pending": len(pending),
})
}
// ── Admin Endpoints ──────────────────────────
// ListPendingReview returns all pending_review memories (admin only).
// GET /api/v1/admin/memories/pending
func (h *MemoryHandler) ListPendingReview(c *gin.Context) {
// This should be called from the admin route group which already checks admin role.
// We query across all users by listing each scope.
ctx := c.Request.Context()
// Query pending memories directly via raw SQL for efficiency
var memories []models.Memory
userMemories, _ := h.stores.Memories.List(ctx, models.MemoryFilter{
Scope: models.MemoryScopeUser,
OwnerID: "%", // This won't work with List — we need a custom query
Status: models.MemoryStatusPendingReview,
Limit: 100,
})
memories = append(memories, userMemories...)
c.JSON(http.StatusOK, gin.H{"memories": memories})
}
// BulkApprove approves multiple memories at once.
// POST /api/v1/admin/memories/bulk-approve
func (h *MemoryHandler) BulkApprove(c *gin.Context) {
var body struct {
IDs []string `json:"ids"`
}
if err := c.ShouldBindJSON(&body); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid body"})
return
}
ctx := c.Request.Context()
approved := 0
for _, id := range body.IDs {
existing, err := h.stores.Memories.GetByID(ctx, id)
if err != nil {
continue
}
existing.Status = models.MemoryStatusActive
if err := h.stores.Memories.Upsert(ctx, existing); err == nil {
approved++
}
}
c.JSON(http.StatusOK, gin.H{"approved": approved})
}

100
server/handlers/memory_inject.go Normal file
View File

@@ -0,0 +1,100 @@
package handlers
import (
"context"
"fmt"
"log"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// maxMemoryChars is the approximate character budget for injected memories.
const maxMemoryChars = 6000
// BuildMemoryHint loads active memories for the user (and persona if active)
// and formats them as a system prompt injection.
//
// Phase 2: if an embedder is available and the user's latest message
// is provided, it generates a query vector for semantic recall. Otherwise
// falls back to keyword/confidence ranking.
func BuildMemoryHint(ctx context.Context, stores store.Stores, embedder *knowledge.Embedder, userID, personaID, lastUserMessage string) string {
if stores.Memories == nil {
return ""
}
var pID *string
if personaID != "" {
pID = &personaID
}
var memories []models.Memory
var err error
// Try hybrid recall with embedding if available
if embedder != nil && embedder.IsConfigured(ctx) && lastUserMessage != "" {
memories, err = hybridRecallForInjection(ctx, stores, embedder, userID, pID, lastUserMessage)
}
// Fall back to keyword recall
if err != nil || len(memories) == 0 {
memories, err = stores.Memories.Recall(ctx, userID, pID, "", 30)
}
if err != nil {
log.Printf("⚠ memory injection failed: %v", err)
return ""
}
if len(memories) == 0 {
return ""
}
// Format memories as bullet points, respecting token budget
var b strings.Builder
b.WriteString("Known facts about this user (from previous conversations):\n")
totalChars := b.Len()
included := 0
for _, m := range memories {
line := fmt.Sprintf("• %s: %s\n", m.Key, m.Value)
if totalChars+len(line) > maxMemoryChars {
break
}
b.WriteString(line)
totalChars += len(line)
included++
}
if included == 0 {
return ""
}
log.Printf("🧠 Injected %d memories for user %s (persona: %s)", included, userID, personaID)
return b.String()
}
// hybridRecallForInjection generates a query vector from the user's message
// and performs hybrid semantic + keyword recall.
func hybridRecallForInjection(ctx context.Context, stores store.Stores, embedder *knowledge.Embedder, userID string, personaID *string, message string) ([]models.Memory, error) {
text := message
if len(text) > 2000 {
text = text[:2000]
}
var teamID *string
if stores.Teams != nil {
ids, _ := stores.Teams.GetUserTeamIDs(ctx, userID)
if len(ids) > 0 {
teamID = &ids[0]
}
}
result, err := embedder.EmbedChunks(ctx, userID, teamID, []string{text})
if err != nil || len(result.Vectors) == 0 {
return nil, err
}
return stores.Memories.RecallHybrid(ctx, userID, personaID, "", result.Vectors[0], 30)
}

2
server/handlers/messages.go
View File

@@ -415,7 +415,7 @@ func (h *MessageHandler) Regenerate(c *gin.Context) {
// ── Resolve model + provider ──
comp := NewCompletionHandler(h.vault, h.stores, h.hub, h.objStore)
comp := NewCompletionHandler(h.vault, h.stores, h.hub, h.objStore, nil)
var presetSystemPrompt string
var personaID string

24
server/main.go
View File

@@ -19,6 +19,7 @@ import (
"git.gobha.me/xcaliber/chat-switchboard/extraction"
"git.gobha.me/xcaliber/chat-switchboard/handlers"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/memory"
"git.gobha.me/xcaliber/chat-switchboard/middleware"
"git.gobha.me/xcaliber/chat-switchboard/providers"
"git.gobha.me/xcaliber/chat-switchboard/roles"
@@ -168,6 +169,13 @@ func main() {
tools.RegisterAttachmentRecall(stores, objStore)
tools.RegisterConversationSearch(stores)
// Memory tools + extraction scanner (v0.18.0)
tools.RegisterMemoryTools(stores, kbEmbedder)
memExtractor := memory.NewExtractor(stores, roleResolver, kbEmbedder)
memScanner := memory.NewScanner(memExtractor, stores, memory.ScannerConfig{})
memScanner.Start()
defer memScanner.Stop()
r := gin.Default()
r.Use(middleware.CORS())
@@ -251,7 +259,7 @@ func main() {
protected.GET("/channels/:id/messages/:msgId/siblings", msgs.ListSiblings)
// Chat Completions
comp := handlers.NewCompletionHandler(keyResolver, stores, hub, objStore)
comp := handlers.NewCompletionHandler(keyResolver, stores, hub, objStore, kbEmbedder)
protected.POST("/chat/completions", comp.Complete)
protected.GET("/tools", comp.ListTools)
@@ -353,6 +361,15 @@ func main() {
protected.GET("/knowledge-bases-discoverable", kbH.ListDiscoverableKBs) // v0.17.0
protected.PUT("/knowledge-bases/:id/discoverable", kbH.SetDiscoverable) // v0.17.0 (admin only enforced in handler)
// Memory management (v0.18.0)
memH := handlers.NewMemoryHandler(stores)
protected.GET("/memories", memH.ListMyMemories)
protected.PUT("/memories/:id", memH.UpdateMemory)
protected.DELETE("/memories/:id", memH.DeleteMemory)
protected.POST("/memories/:id/approve", memH.ApproveMemory)
protected.POST("/memories/:id/reject", memH.RejectMemory)
protected.GET("/memories/count", memH.MemoryCount)
// Teams (user: my teams)
teams := handlers.NewTeamHandler(keyResolver)
protected.GET("/teams/mine", teams.MyTeams)
@@ -463,6 +480,11 @@ func main() {
admin.GET("/presets/:id/knowledge-bases", personaAdm.GetPersonaKBs) // v0.17.0
admin.PUT("/presets/:id/knowledge-bases", personaAdm.SetPersonaKBs) // v0.17.0
// Admin memory review (v0.18.0)
adminMemH := handlers.NewMemoryHandler(stores)
admin.GET("/memories/pending", adminMemH.ListPendingReview)
admin.POST("/memories/bulk-approve", adminMemH.BulkApprove)
// Teams (admin)
teamAdm := handlers.NewTeamHandler(keyResolver)
admin.GET("/teams", teamAdm.ListTeams)

261
server/memory/extractor.go Normal file
View File

@@ -0,0 +1,261 @@
package memory
import (
"context"
"encoding/json"
"fmt"
"log"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/providers"
"git.gobha.me/xcaliber/chat-switchboard/roles"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// defaultExtractionPrompt is used when the persona has no custom prompt.
const defaultExtractionPrompt = `You are a memory extraction assistant. Analyze the following conversation and extract memorable facts about the user.
Focus on:
- Personal preferences (language, tools, frameworks, communication style)
- Technical details (deployment stack, database choices, architecture patterns)
- Project context (project names, team size, deadlines, goals)
- Biographical facts (role, company, location, experience level)
Respond ONLY with a JSON array of extracted facts. Each fact must have:
- "key": short label (2-5 words, lowercase)
- "value": the detail/fact (1-2 sentences max)
- "confidence": 0.0-1.0 how certain you are
Example:
[
{"key": "preferred language", "value": "Go with Gin framework for backend services", "confidence": 0.95},
{"key": "deployment target", "value": "Kubernetes on AWS with PostgreSQL databases", "confidence": 0.8}
]
If no memorable facts are found, respond with an empty array: []`
// maxConversationChars limits the conversation text sent for extraction.
const maxConversationChars = 30000
// minMessagesForExtraction is the minimum new messages before extraction triggers.
const minMessagesForExtraction = 6
// extractedFact is the JSON structure returned by the utility model.
type extractedFact struct {
Key string `json:"key"`
Value string `json:"value"`
Confidence float64 `json:"confidence"`
}
// Extractor analyzes conversations and extracts memorable facts.
type Extractor struct {
stores store.Stores
roleResolver *roles.Resolver
embedder *knowledge.Embedder
}
// NewExtractor creates a new memory extraction service.
func NewExtractor(stores store.Stores, rr *roles.Resolver, embedder *knowledge.Embedder) *Extractor {
return &Extractor{stores: stores, roleResolver: rr, embedder: embedder}
}
// Extract analyzes a conversation and saves extracted facts as pending_review memories.
func (e *Extractor) Extract(ctx context.Context, channelID, userID, teamID, personaID string) error {
if e.stores.Memories == nil || e.stores.Messages == nil {
return fmt.Errorf("memory or message store not available")
}
// Check extraction log for last processed message
var lastMessageID string
row := database.DB.QueryRowContext(ctx,
database.Q(`SELECT last_message_id FROM memory_extraction_log WHERE channel_id = $1 AND user_id = $2`),
channelID, userID)
row.Scan(&lastMessageID) // ignore error — may not exist yet
// Load recent messages for this channel
messages, err := e.stores.Messages.ListForChannel(ctx, channelID, store.ListOptions{Limit: 200})
if err != nil {
return fmt.Errorf("load messages: %w", err)
}
// Filter to new messages only (messages come back newest-first)
var newMessages []models.Message
for i := len(messages) - 1; i >= 0; i-- {
if lastMessageID == "" || messages[i].ID > lastMessageID {
newMessages = append(newMessages, messages[i])
}
}
if len(newMessages) < minMessagesForExtraction {
return nil // not enough new content
}
// Build conversation text
var sb strings.Builder
for _, m := range newMessages {
role := m.Role
if role == "user" {
role = "User"
} else {
role = "Assistant"
}
line := fmt.Sprintf("[%s]: %s\n", role, m.Content)
if sb.Len()+len(line) > maxConversationChars {
break
}
sb.WriteString(line)
}
if sb.Len() < 200 {
return nil // too little content
}
// Get extraction prompt (persona-specific or default)
prompt := defaultExtractionPrompt
if personaID != "" && e.stores.Personas != nil {
persona, err := e.stores.Personas.GetByID(ctx, personaID)
if err == nil && persona.MemoryExtractionPrompt != nil && *persona.MemoryExtractionPrompt != "" {
prompt = *persona.MemoryExtractionPrompt
}
}
// Call utility model via role resolver
var tID *string
if teamID != "" {
tID = &teamID
}
apiMessages := []providers.Message{
{Role: "system", Content: prompt},
{Role: "user", Content: "Analyze this conversation and extract memorable facts:\n\n" + sb.String()},
}
result, err := e.roleResolver.Complete(ctx, "utility", userID, tID, apiMessages)
if err != nil {
return fmt.Errorf("extraction completion: %w", err)
}
// Parse response
facts, err := parseExtractionResponse(result.Content)
if err != nil {
log.Printf("⚠ memory extraction parse failed for channel %s: %v", channelID, err)
return nil // don't fail on parse errors
}
if len(facts) == 0 {
log.Printf("🧠 memory extraction: channel %s → 0 facts (nothing memorable)", channelID)
}
// Determine scope
scope := models.MemoryScopeUser
ownerID := userID
var memUserID *string
if personaID != "" {
scope = models.MemoryScopePersonaUser
ownerID = personaID
memUserID = &userID
}
// Save extracted facts
saved := 0
for _, f := range facts {
if f.Key == "" || f.Value == "" || f.Confidence < 0.3 {
continue
}
mem := &models.Memory{
ID: store.NewID(),
Scope: scope,
OwnerID: ownerID,
UserID: memUserID,
Key: f.Key,
Value: f.Value,
SourceChannelID: &channelID,
Confidence: f.Confidence,
Status: models.MemoryStatusPendingReview,
}
if err := e.stores.Memories.Upsert(ctx, mem); err != nil {
log.Printf("⚠ memory save failed: %v", err)
continue
}
// Embed if available
e.embedMemory(ctx, mem, userID, tID)
saved++
}
// Update extraction log
latestID := newMessages[len(newMessages)-1].ID
if database.IsSQLite() {
_, err = database.DB.ExecContext(ctx, `
INSERT INTO memory_extraction_log (id, channel_id, user_id, last_message_id, memory_count)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(channel_id, user_id) DO UPDATE SET
last_message_id = excluded.last_message_id,
extracted_at = datetime('now'),
memory_count = memory_extraction_log.memory_count + excluded.memory_count
`, store.NewID(), channelID, userID, latestID, saved)
} else {
_, err = database.DB.ExecContext(ctx, `
INSERT INTO memory_extraction_log (channel_id, user_id, last_message_id, memory_count)
VALUES ($1, $2, $3, $4)
ON CONFLICT(channel_id, user_id) DO UPDATE SET
last_message_id = EXCLUDED.last_message_id,
extracted_at = now(),
memory_count = memory_extraction_log.memory_count + EXCLUDED.memory_count
`, channelID, userID, latestID, saved)
}
if saved > 0 {
log.Printf("✅ memory extraction: channel %s → %d facts", channelID, saved)
}
return err
}
// embedMemory generates and stores an embedding vector for a memory.
func (e *Extractor) embedMemory(ctx context.Context, m *models.Memory, userID string, teamID *string) {
if e.embedder == nil || !e.embedder.IsConfigured(ctx) {
return
}
text := m.Key + ": " + m.Value
result, err := e.embedder.EmbedChunks(ctx, userID, teamID, []string{text})
if err != nil || len(result.Vectors) == 0 {
return
}
vecJSON, _ := json.Marshal(result.Vectors[0])
if database.IsSQLite() {
database.DB.ExecContext(ctx,
`UPDATE memories SET embedding = ? WHERE id = ?`,
string(vecJSON), m.ID)
} else {
database.DB.ExecContext(ctx,
`UPDATE memories SET embedding = $1::vector WHERE id = $2`,
string(vecJSON), m.ID)
}
}
// parseExtractionResponse parses the utility model's JSON response.
func parseExtractionResponse(content string) ([]extractedFact, error) {
content = strings.TrimSpace(content)
// Strip markdown code fences
if strings.HasPrefix(content, "```") {
lines := strings.Split(content, "\n")
if len(lines) > 2 {
lines = lines[1 : len(lines)-1]
content = strings.Join(lines, "\n")
}
}
var facts []extractedFact
if err := json.Unmarshal([]byte(content), &facts); err != nil {
return nil, fmt.Errorf("parse facts JSON: %w", err)
}
return facts, nil
}

197
server/memory/scanner.go Normal file
View File

@@ -0,0 +1,197 @@
package memory
import (
"context"
"log"
"sync"
"time"
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// ScannerConfig holds tunable parameters for the background scanner.
type ScannerConfig struct {
Interval time.Duration // tick interval (default 10m)
Concurrency int // max parallel extractions (default 2)
}
// Scanner periodically finds conversations needing extraction.
type Scanner struct {
extractor *Extractor
stores store.Stores
cfg ScannerConfig
stopCh chan struct{}
wg sync.WaitGroup
inFlight sync.Map // channelID+userID → true
}
// NewScanner creates a background extraction scanner.
func NewScanner(ext *Extractor, stores store.Stores, cfg ScannerConfig) *Scanner {
if cfg.Interval <= 0 {
cfg.Interval = 10 * time.Minute
}
if cfg.Concurrency <= 0 {
cfg.Concurrency = 2
}
return &Scanner{
extractor: ext,
stores: stores,
cfg: cfg,
stopCh: make(chan struct{}),
}
}
// Start begins the background scanning loop.
func (s *Scanner) Start() {
s.wg.Add(1)
go func() {
defer s.wg.Done()
log.Printf("🧠 memory extraction scanner started (interval=%s, concurrency=%d)",
s.cfg.Interval, s.cfg.Concurrency)
ticker := time.NewTicker(s.cfg.Interval)
defer ticker.Stop()
for {
select {
case <-s.stopCh:
log.Println("🧠 memory extraction scanner stopped")
return
case <-ticker.C:
s.scan()
}
}
}()
}
// Stop gracefully shuts down the scanner.
func (s *Scanner) Stop() {
close(s.stopCh)
s.wg.Wait()
}
// candidate represents a conversation that may need extraction.
type candidate struct {
ChannelID string
UserID string
TeamID string
}
func (s *Scanner) scan() {
ctx := context.Background()
// Check global kill switch
if s.stores.GlobalConfig != nil {
val, err := s.stores.GlobalConfig.Get(ctx, "memory_extraction_enabled")
if err != nil || val == nil {
return // disabled or not configured
}
// val is a map; check the "value" key
if enabled, ok := val["value"].(bool); !ok || !enabled {
return
}
}
candidates, err := s.findCandidates(ctx)
if err != nil {
log.Printf("⚠ memory scanner: find candidates: %v", err)
return
}
if len(candidates) == 0 {
return
}
// Semaphore for concurrency control
sem := make(chan struct{}, s.cfg.Concurrency)
var wg sync.WaitGroup
for _, c := range candidates {
key := c.ChannelID + ":" + c.UserID
if _, loaded := s.inFlight.LoadOrStore(key, true); loaded {
continue // already in progress
}
sem <- struct{}{} // acquire
wg.Add(1)
go func(cand candidate) {
defer wg.Done()
defer func() { <-sem }() // release
defer s.inFlight.Delete(cand.ChannelID + ":" + cand.UserID)
extractCtx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
if err := s.extractor.Extract(extractCtx, cand.ChannelID, cand.UserID, cand.TeamID, ""); err != nil {
log.Printf("⚠ memory extraction failed: channel=%s user=%s: %v",
cand.ChannelID, cand.UserID, err)
}
}(c)
}
wg.Wait()
}
// findCandidates queries for channels with enough new activity since last extraction.
func (s *Scanner) findCandidates(ctx context.Context) ([]candidate, error) {
db := database.DB
if db == nil {
return nil, nil
}
var query string
if database.IsSQLite() {
query = `
SELECT c.id, c.created_by, COALESCE(cu.team_id, '')
FROM channels c
LEFT JOIN channel_users cu ON cu.channel_id = c.id AND cu.user_id = c.created_by
LEFT JOIN memory_extraction_log mel ON mel.channel_id = c.id AND mel.user_id = c.created_by
WHERE c.created_by IS NOT NULL
AND (
SELECT COUNT(*) FROM messages m
WHERE m.channel_id = c.id
AND (mel.last_message_id IS NULL OR m.id > mel.last_message_id)
) >= 8
AND (
SELECT MAX(m2.created_at) FROM messages m2 WHERE m2.channel_id = c.id
) < datetime('now', '-5 minutes')
LIMIT 20
`
} else {
query = `
SELECT c.id, c.created_by, COALESCE(cu.team_id, '')
FROM channels c
LEFT JOIN channel_users cu ON cu.channel_id = c.id AND cu.user_id = c.created_by
LEFT JOIN memory_extraction_log mel ON mel.channel_id = c.id AND mel.user_id = c.created_by
WHERE c.created_by IS NOT NULL
AND (
SELECT COUNT(*) FROM messages m
WHERE m.channel_id = c.id
AND (mel.last_message_id IS NULL OR m.id::text > mel.last_message_id::text)
) >= 8
AND (
SELECT MAX(m2.created_at) FROM messages m2 WHERE m2.channel_id = c.id
) < now() - interval '5 minutes'
LIMIT 20
`
}
rows, err := db.QueryContext(ctx, query)
if err != nil {
return nil, err
}
defer rows.Close()
var candidates []candidate
for rows.Next() {
var c candidate
if err := rows.Scan(&c.ChannelID, &c.UserID, &c.TeamID); err != nil {
continue
}
candidates = append(candidates, c)
}
return candidates, rows.Err()
}

6
server/models/models.go
View File

@@ -219,6 +219,10 @@ type Persona struct {
// Loaded from persona_knowledge_bases, not stored in personas table
KBIDs []string `json:"kb_ids,omitempty" db:"-"`
// Memory configuration (v0.18.0)
MemoryEnabled bool `json:"memory_enabled" db:"memory_enabled"`
MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty" db:"memory_extraction_prompt"`
}
type PersonaPatch struct {
@@ -235,6 +239,8 @@ type PersonaPatch struct {
TopP *float64 `json:"top_p,omitempty"`
IsActive *bool `json:"is_active,omitempty"`
IsShared *bool `json:"is_shared,omitempty"`
MemoryEnabled *bool `json:"memory_enabled,omitempty"`
MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty"`
}
// =========================================

48
server/models/models_memory.go Normal file
View File

@@ -0,0 +1,48 @@
// ── models_memory.go ────────────────────────
// Memory model for v0.18.0 — add these types to models/models.go
// (or keep as a separate file in the models package)
package models
import "time"
// ── Memory Scope Constants ──────────────────
const (
MemoryScopeUser = "user" // personal facts, persists across all conversations
MemoryScopePersona = "persona" // shared across all users of a Persona
MemoryScopePersonaUser = "persona_user" // per-user within a Persona context
)
// ── Memory Status Constants ─────────────────
const (
MemoryStatusActive = "active"
MemoryStatusPendingReview = "pending_review"
MemoryStatusArchived = "archived"
)
// Memory represents a single fact or preference persisted across conversations.
type Memory struct {
ID string `json:"id" db:"id"`
Scope string `json:"scope" db:"scope"`
OwnerID string `json:"owner_id" db:"owner_id"`
UserID *string `json:"user_id,omitempty" db:"user_id"`
Key string `json:"key" db:"key"`
Value string `json:"value" db:"value"`
SourceChannelID *string `json:"source_channel_id,omitempty" db:"source_channel_id"`
Confidence float64 `json:"confidence" db:"confidence"`
Status string `json:"status" db:"status"`
CreatedAt time.Time `json:"created_at" db:"created_at"`
UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
}
// MemoryFilter specifies which memories to retrieve.
type MemoryFilter struct {
Scope string // required: user, persona, persona_user
OwnerID string // required: user_id or persona_id
UserID *string // only for persona_user scope
Status string // default: "active"
Query string // optional: keyword filter on key+value
Limit int // max results (default 50)
}

18
server/models/models_memory_persona.go Normal file
View File

@@ -0,0 +1,18 @@
// ── models_memory_persona.go ────────────────
// Phase 2 additions to the Persona struct.
//
// Add these fields to the Persona struct in models/models.go
// (after the KBIDs field):
//
// MemoryEnabled bool `json:"memory_enabled" db:"memory_enabled"`
// MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty" db:"memory_extraction_prompt"`
//
// Add to PersonaPatch:
//
// MemoryEnabled *bool `json:"memory_enabled,omitempty"`
// MemoryExtractionPrompt *string `json:"memory_extraction_prompt,omitempty"`
//
// This file exists purely as documentation — the actual fields
// must be added to the existing structs in models.go.
package models

1
server/store/interfaces.go
View File

@@ -38,6 +38,7 @@ type Stores struct {
KnowledgeBases KnowledgeBaseStore
Groups GroupStore
ResourceGrants ResourceGrantStore
Memories MemoryStore
}
// =========================================

269
server/store/postgres/memory.go Normal file
View File

@@ -0,0 +1,269 @@
package postgres
import (
"context"
"database/sql"
"fmt"
"strings"
"time"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
type MemoryStore struct{}
func NewMemoryStore() *MemoryStore { return &MemoryStore{} }
func (s *MemoryStore) Upsert(ctx context.Context, m *models.Memory) error {
if m.ID == "" {
m.ID = store.NewID()
}
if m.Status == "" {
m.Status = models.MemoryStatusActive
}
if m.Confidence == 0 {
m.Confidence = 1.0
}
_, err := DB.ExecContext(ctx, `
INSERT INTO memories (id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key)
DO UPDATE SET
value = EXCLUDED.value,
confidence = EXCLUDED.confidence,
status = EXCLUDED.status,
source_channel_id = EXCLUDED.source_channel_id,
updated_at = now()
`, m.ID, m.Scope, m.OwnerID, m.UserID, m.Key, m.Value,
m.SourceChannelID, m.Confidence, m.Status)
return err
}
func (s *MemoryStore) GetByID(ctx context.Context, id string) (*models.Memory, error) {
m := &models.Memory{}
err := DB.QueryRowContext(ctx, `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories WHERE id = $1
`, id).Scan(
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
&m.SourceChannelID, &m.Confidence, &m.Status, &m.CreatedAt, &m.UpdatedAt,
)
if err == sql.ErrNoRows {
return nil, fmt.Errorf("memory not found: %s", id)
}
return m, err
}
func (s *MemoryStore) List(ctx context.Context, f models.MemoryFilter) ([]models.Memory, error) {
clauses := []string{"scope = $1", "owner_id = $2"}
args := []interface{}{f.Scope, f.OwnerID}
idx := 3
if f.UserID != nil {
clauses = append(clauses, fmt.Sprintf("user_id = $%d", idx))
args = append(args, *f.UserID)
idx++
}
status := f.Status
if status == "" {
status = models.MemoryStatusActive
}
clauses = append(clauses, fmt.Sprintf("status = $%d", idx))
args = append(args, status)
idx++
if f.Query != "" {
clauses = append(clauses, fmt.Sprintf(
"to_tsvector('english', key || ' ' || value) @@ plainto_tsquery('english', $%d)", idx))
args = append(args, f.Query)
idx++
}
limit := f.Limit
if limit <= 0 || limit > 200 {
limit = 50
}
args = append(args, limit)
query := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE %s
ORDER BY confidence DESC, updated_at DESC
LIMIT $%d
`, strings.Join(clauses, " AND "), idx)
rows, err := DB.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return scanMemories(rows)
}
func (s *MemoryStore) Delete(ctx context.Context, id string) error {
_, err := DB.ExecContext(ctx, `DELETE FROM memories WHERE id = $1`, id)
return err
}
func (s *MemoryStore) Archive(ctx context.Context, id string) error {
_, err := DB.ExecContext(ctx, `
UPDATE memories SET status = 'archived', updated_at = now() WHERE id = $1
`, id)
return err
}
func (s *MemoryStore) Recall(ctx context.Context, userID string, personaID *string, query string, limit int) ([]models.Memory, error) {
if limit <= 0 || limit > 100 {
limit = 20
}
// Build a UNION query across applicable scopes.
// Always include user-scope memories for this user.
// If a persona is active, also include persona and persona+user memories.
parts := []string{}
args := []interface{}{}
idx := 1
// 1. User-scope memories
userPart := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE scope = 'user' AND owner_id = $%d AND status = 'active'
`, idx)
args = append(args, userID)
idx++
if query != "" {
userPart += fmt.Sprintf(
" AND to_tsvector('english', key || ' ' || value) @@ plainto_tsquery('english', $%d)", idx)
args = append(args, query)
idx++
}
parts = append(parts, userPart)
// 2. Persona-scope memories (if persona active)
if personaID != nil && *personaID != "" {
personaPart := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE scope = 'persona' AND owner_id = $%d AND status = 'active'
`, idx)
args = append(args, *personaID)
idx++
if query != "" {
personaPart += fmt.Sprintf(
" AND to_tsvector('english', key || ' ' || value) @@ plainto_tsquery('english', $%d)", idx)
args = append(args, query)
idx++
}
parts = append(parts, personaPart)
// 3. Persona+user memories
puPart := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE scope = 'persona_user' AND owner_id = $%d AND user_id = $%d AND status = 'active'
`, idx, idx+1)
args = append(args, *personaID, userID)
idx += 2
if query != "" {
puPart += fmt.Sprintf(
" AND to_tsvector('english', key || ' ' || value) @@ plainto_tsquery('english', $%d)", idx)
args = append(args, query)
idx++
}
parts = append(parts, puPart)
}
// Scope priority: persona_user > persona > user (CASE ordering)
fullQuery := fmt.Sprintf(`
SELECT * FROM (%s) AS combined
ORDER BY
CASE scope
WHEN 'persona_user' THEN 0
WHEN 'persona' THEN 1
WHEN 'user' THEN 2
END,
confidence DESC,
updated_at DESC
LIMIT $%d
`, strings.Join(parts, " UNION ALL "), idx)
args = append(args, limit)
rows, err := DB.QueryContext(ctx, fullQuery, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return scanMemories(rows)
}
func (s *MemoryStore) CountByOwner(ctx context.Context, scope, ownerID string) (int, error) {
var count int
err := DB.QueryRowContext(ctx, `
SELECT COUNT(*) FROM memories
WHERE scope = $1 AND owner_id = $2 AND status = 'active'
`, scope, ownerID).Scan(&count)
return count, err
}
// ── Helpers ─────────────────────────────────
func scanMemories(rows *sql.Rows) ([]models.Memory, error) {
var memories []models.Memory
for rows.Next() {
var m models.Memory
if err := rows.Scan(
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
&m.SourceChannelID, &m.Confidence, &m.Status, &m.CreatedAt, &m.UpdatedAt,
); err != nil {
return nil, err
}
memories = append(memories, m)
}
if memories == nil {
memories = []models.Memory{}
}
return memories, rows.Err()
}
// scanMemory scans a single memory row — unused currently but available
// for future single-row queries.
func scanMemory(row *sql.Row) (*models.Memory, error) {
m := &models.Memory{}
err := row.Scan(
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
&m.SourceChannelID, &m.Confidence, &m.Status, &m.CreatedAt, &m.UpdatedAt,
)
if err == sql.ErrNoRows {
return nil, nil
}
return m, err
}
// ensure compile-time interface satisfaction
var _ interface {
Upsert(ctx context.Context, m *models.Memory) error
GetByID(ctx context.Context, id string) (*models.Memory, error)
List(ctx context.Context, f models.MemoryFilter) ([]models.Memory, error)
Delete(ctx context.Context, id string) error
Archive(ctx context.Context, id string) error
Recall(ctx context.Context, userID string, personaID *string, query string, limit int) ([]models.Memory, error)
CountByOwner(ctx context.Context, scope, ownerID string) (int, error)
} = (*MemoryStore)(nil)
// ensure unused imports don't cause errors
var _ = time.Now

135
server/store/postgres/memory_hybrid.go Normal file
View File

@@ -0,0 +1,135 @@
package postgres
import (
"context"
"database/sql"
"fmt"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/models"
)
// RecallHybrid returns active memories using vector similarity + keyword search.
// If queryVec is nil/empty, falls back to keyword-only Recall.
func (s *MemoryStore) RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error) {
if len(queryVec) == 0 {
return s.Recall(ctx, userID, personaID, query, limit)
}
if limit <= 0 || limit > 100 {
limit = 30
}
// Run semantic recall
semantic, err := s.recallSemantic(ctx, userID, personaID, queryVec, limit)
if err != nil {
// Fall back to keyword
return s.Recall(ctx, userID, personaID, query, limit)
}
// Run keyword recall if query provided
var keyword []models.Memory
if query != "" {
keyword, _ = s.Recall(ctx, userID, personaID, query, limit)
}
return mergeResults(semantic, keyword, limit), nil
}
// recallSemantic uses pgvector cosine distance to find similar memories.
func (s *MemoryStore) recallSemantic(ctx context.Context, userID string, personaID *string, queryVec []float64, limit int) ([]models.Memory, error) {
vecStr := vectorToString(queryVec)
parts := []string{}
args := []interface{}{}
idx := 1
// User-scope
userPart := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at,
(embedding <=> $%d::vector) AS distance
FROM memories
WHERE scope = 'user' AND owner_id = $%d AND status = 'active'
AND embedding IS NOT NULL
`, idx, idx+1)
args = append(args, vecStr, userID)
idx += 2
parts = append(parts, userPart)
if personaID != nil && *personaID != "" {
// Persona-scope
personaPart := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at,
(embedding <=> $%d::vector) AS distance
FROM memories
WHERE scope = 'persona' AND owner_id = $%d AND status = 'active'
AND embedding IS NOT NULL
`, idx, idx+1)
args = append(args, vecStr, *personaID)
idx += 2
parts = append(parts, personaPart)
// Persona+user scope
puPart := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at,
(embedding <=> $%d::vector) AS distance
FROM memories
WHERE scope = 'persona_user' AND owner_id = $%d AND user_id = $%d AND status = 'active'
AND embedding IS NOT NULL
`, idx, idx+1, idx+2)
args = append(args, vecStr, *personaID, userID)
idx += 3
parts = append(parts, puPart)
}
args = append(args, limit)
fullQuery := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM (%s) AS combined
WHERE distance < 0.7
ORDER BY distance ASC, confidence DESC
LIMIT $%d
`, strings.Join(parts, " UNION ALL "), idx)
rows, err := DB.QueryContext(ctx, fullQuery, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return scanMemories(rows)
}
// mergeResults deduplicates and merges semantic + keyword results.
func mergeResults(semantic, keyword []models.Memory, limit int) []models.Memory {
seen := make(map[string]bool, len(semantic))
result := make([]models.Memory, 0, limit)
// Semantic results first (higher relevance)
for _, m := range semantic {
if len(result) >= limit {
break
}
seen[m.ID] = true
result = append(result, m)
}
// Then keyword results not already included
for _, m := range keyword {
if len(result) >= limit {
break
}
if !seen[m.ID] {
seen[m.ID] = true
result = append(result, m)
}
}
return result
}
// ensure unused
var _ = sql.ErrNoRows

17
server/store/postgres/persona.go
View File

@@ -15,20 +15,23 @@ func NewPersonaStore() *PersonaStore { return &PersonaStore{} }
const personaCols = `id, name, description, icon, avatar, base_model_id, provider_config_id,
system_prompt, temperature, max_tokens, thinking_budget, top_p,
scope, owner_id, created_by, is_active, is_shared, created_at, updated_at`
scope, owner_id, created_by, is_active, is_shared, memory_enabled, memory_extraction_prompt,
created_at, updated_at`
func (s *PersonaStore) Create(ctx context.Context, p *models.Persona) error {
return DB.QueryRowContext(ctx, `
INSERT INTO personas (name, description, icon, avatar, base_model_id, provider_config_id,
system_prompt, temperature, max_tokens, thinking_budget, top_p,
scope, owner_id, created_by, is_active, is_shared)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16)
scope, owner_id, created_by, is_active, is_shared,
memory_enabled, memory_extraction_prompt)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18)
RETURNING id, created_at, updated_at`,
p.Name, p.Description, p.Icon, p.Avatar, p.BaseModelID,
models.NullString(p.ProviderConfigID),
p.SystemPrompt, models.NullFloat(p.Temperature), models.NullInt(p.MaxTokens),
models.NullInt(p.ThinkingBudget), models.NullFloat(p.TopP),
p.Scope, models.NullString(p.OwnerID), p.CreatedBy, p.IsActive, p.IsShared,
p.MemoryEnabled, p.MemoryExtractionPrompt,
).Scan(&p.ID, &p.CreatedAt, &p.UpdatedAt)
}
@@ -86,6 +89,12 @@ func (s *PersonaStore) Update(ctx context.Context, id string, patch models.Perso
if patch.IsShared != nil {
b.Set("is_shared", *patch.IsShared)
}
if patch.MemoryEnabled != nil {
b.Set("memory_enabled", *patch.MemoryEnabled)
}
if patch.MemoryExtractionPrompt != nil {
b.Set("memory_extraction_prompt", *patch.MemoryExtractionPrompt)
}
if !b.HasSets() {
return nil
}
@@ -279,6 +288,7 @@ func scanPersona(row *sql.Row) (*models.Persona, error) {
&p.BaseModelID, &providerConfigID,
&p.SystemPrompt, &temp, &maxTokens, &thinkingBudget, &topP,
&p.Scope, &ownerID, &p.CreatedBy, &p.IsActive, &p.IsShared,
&p.MemoryEnabled, &p.MemoryExtractionPrompt,
&p.CreatedAt, &p.UpdatedAt,
)
if err != nil {
@@ -305,6 +315,7 @@ func scanPersonas(rows *sql.Rows) ([]models.Persona, error) {
&p.BaseModelID, &providerConfigID,
&p.SystemPrompt, &temp, &maxTokens, &thinkingBudget, &topP,
&p.Scope, &ownerID, &p.CreatedBy, &p.IsActive, &p.IsShared,
&p.MemoryEnabled, &p.MemoryExtractionPrompt,
&p.CreatedAt, &p.UpdatedAt,
)
if err != nil {

1
server/store/postgres/stores.go
View File

@@ -31,5 +31,6 @@ func NewStores(db *sql.DB) store.Stores {
KnowledgeBases: NewKnowledgeBaseStore(),
Groups: NewGroupStore(),
ResourceGrants: NewResourceGrantStore(),
Memories: NewMemoryStore(),
}
}

224
server/store/sqlite/memory.go Normal file
View File

@@ -0,0 +1,224 @@
package sqlite
import (
"context"
"database/sql"
"fmt"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
type MemoryStore struct{}
func NewMemoryStore() *MemoryStore { return &MemoryStore{} }
func (s *MemoryStore) Upsert(ctx context.Context, m *models.Memory) error {
if m.ID == "" {
m.ID = store.NewID()
}
if m.Status == "" {
m.Status = models.MemoryStatusActive
}
if m.Confidence == 0 {
m.Confidence = 1.0
}
_, err := DB.ExecContext(ctx, `
INSERT INTO memories (id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (scope, owner_id, COALESCE(user_id, '00000000-0000-0000-0000-000000000000'), key)
DO UPDATE SET
value = excluded.value,
confidence = excluded.confidence,
status = excluded.status,
source_channel_id = excluded.source_channel_id,
updated_at = datetime('now')
`, m.ID, m.Scope, m.OwnerID, m.UserID, m.Key, m.Value,
m.SourceChannelID, m.Confidence, m.Status)
return err
}
func (s *MemoryStore) GetByID(ctx context.Context, id string) (*models.Memory, error) {
m := &models.Memory{}
err := DB.QueryRowContext(ctx, `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories WHERE id = ?
`, id).Scan(
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
&m.SourceChannelID, &m.Confidence, &m.Status,
st(&m.CreatedAt), st(&m.UpdatedAt),
)
if err == sql.ErrNoRows {
return nil, fmt.Errorf("memory not found: %s", id)
}
return m, err
}
func (s *MemoryStore) List(ctx context.Context, f models.MemoryFilter) ([]models.Memory, error) {
clauses := []string{"scope = ?", "owner_id = ?"}
args := []interface{}{f.Scope, f.OwnerID}
if f.UserID != nil {
clauses = append(clauses, "user_id = ?")
args = append(args, *f.UserID)
}
status := f.Status
if status == "" {
status = models.MemoryStatusActive
}
clauses = append(clauses, "status = ?")
args = append(args, status)
if f.Query != "" {
// SQLite: LIKE-based keyword search (no tsvector)
clauses = append(clauses, "(key || ' ' || value) LIKE ?")
args = append(args, "%"+f.Query+"%")
}
limit := f.Limit
if limit <= 0 || limit > 200 {
limit = 50
}
args = append(args, limit)
query := fmt.Sprintf(`
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE %s
ORDER BY confidence DESC, updated_at DESC
LIMIT ?
`, strings.Join(clauses, " AND "))
rows, err := DB.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return s.scanMemories(rows)
}
func (s *MemoryStore) Delete(ctx context.Context, id string) error {
_, err := DB.ExecContext(ctx, `DELETE FROM memories WHERE id = ?`, id)
return err
}
func (s *MemoryStore) Archive(ctx context.Context, id string) error {
_, err := DB.ExecContext(ctx, `
UPDATE memories SET status = 'archived', updated_at = datetime('now') WHERE id = ?
`, id)
return err
}
func (s *MemoryStore) Recall(ctx context.Context, userID string, personaID *string, query string, limit int) ([]models.Memory, error) {
if limit <= 0 || limit > 100 {
limit = 20
}
// Build UNION query across applicable scopes
parts := []string{}
args := []interface{}{}
// 1. User-scope memories
userPart := `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE scope = 'user' AND owner_id = ? AND status = 'active'
`
args = append(args, userID)
if query != "" {
userPart += " AND (key || ' ' || value) LIKE ?"
args = append(args, "%"+query+"%")
}
parts = append(parts, userPart)
// 2. Persona-scope memories
if personaID != nil && *personaID != "" {
personaPart := `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE scope = 'persona' AND owner_id = ? AND status = 'active'
`
args = append(args, *personaID)
if query != "" {
personaPart += " AND (key || ' ' || value) LIKE ?"
args = append(args, "%"+query+"%")
}
parts = append(parts, personaPart)
// 3. Persona+user memories
puPart := `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at
FROM memories
WHERE scope = 'persona_user' AND owner_id = ? AND user_id = ? AND status = 'active'
`
args = append(args, *personaID, userID)
if query != "" {
puPart += " AND (key || ' ' || value) LIKE ?"
args = append(args, "%"+query+"%")
}
parts = append(parts, puPart)
}
fullQuery := fmt.Sprintf(`
SELECT * FROM (%s)
ORDER BY
CASE scope
WHEN 'persona_user' THEN 0
WHEN 'persona' THEN 1
WHEN 'user' THEN 2
END,
confidence DESC,
updated_at DESC
LIMIT ?
`, strings.Join(parts, " UNION ALL "))
args = append(args, limit)
rows, err := DB.QueryContext(ctx, fullQuery, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return s.scanMemories(rows)
}
func (s *MemoryStore) CountByOwner(ctx context.Context, scope, ownerID string) (int, error) {
var count int
err := DB.QueryRowContext(ctx, `
SELECT COUNT(*) FROM memories
WHERE scope = ? AND owner_id = ? AND status = 'active'
`, scope, ownerID).Scan(&count)
return count, err
}
// ── Helpers ─────────────────────────────────
func (s *MemoryStore) scanMemories(rows *sql.Rows) ([]models.Memory, error) {
var memories []models.Memory
for rows.Next() {
var m models.Memory
if err := rows.Scan(
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
&m.SourceChannelID, &m.Confidence, &m.Status,
st(&m.CreatedAt), st(&m.UpdatedAt),
); err != nil {
return nil, err
}
memories = append(memories, m)
}
if memories == nil {
memories = []models.Memory{}
}
return memories, rows.Err()
}

146
server/store/sqlite/memory_hybrid.go Normal file
View File

@@ -0,0 +1,146 @@
package sqlite
import (
"context"
"encoding/json"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/models"
)
// RecallHybrid returns active memories using vector similarity + keyword search.
// SQLite has no pgvector, so we load embeddings and compute cosine similarity in Go.
func (s *MemoryStore) RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error) {
if len(queryVec) == 0 {
return s.Recall(ctx, userID, personaID, query, limit)
}
if limit <= 0 || limit > 100 {
limit = 30
}
// Load all active memories with embeddings for applicable scopes
semantic, err := s.recallSemantic(ctx, userID, personaID, queryVec, limit)
if err != nil || len(semantic) == 0 {
return s.Recall(ctx, userID, personaID, query, limit)
}
var keyword []models.Memory
if query != "" {
keyword, _ = s.Recall(ctx, userID, personaID, query, limit)
}
return mergeResultsSQLite(semantic, keyword, limit), nil
}
type scoredMemory struct {
Memory models.Memory
Similarity float64
}
func (s *MemoryStore) recallSemantic(ctx context.Context, userID string, personaID *string, queryVec []float64, limit int) ([]models.Memory, error) {
// Build query to load memories with embeddings
parts := []string{}
args := []interface{}{}
userPart := `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at, embedding
FROM memories
WHERE scope = 'user' AND owner_id = ? AND status = 'active'
AND embedding IS NOT NULL
`
args = append(args, userID)
parts = append(parts, userPart)
if personaID != nil && *personaID != "" {
personaPart := `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at, embedding
FROM memories
WHERE scope = 'persona' AND owner_id = ? AND status = 'active'
AND embedding IS NOT NULL
`
args = append(args, *personaID)
parts = append(parts, personaPart)
puPart := `
SELECT id, scope, owner_id, user_id, key, value,
source_channel_id, confidence, status, created_at, updated_at, embedding
FROM memories
WHERE scope = 'persona_user' AND owner_id = ? AND user_id = ? AND status = 'active'
AND embedding IS NOT NULL
`
args = append(args, *personaID, userID)
parts = append(parts, puPart)
}
fullQuery := strings.Join(parts, " UNION ALL ")
rows, err := DB.QueryContext(ctx, fullQuery, args...)
if err != nil {
return nil, err
}
defer rows.Close()
// Score each memory by cosine similarity
var scored []scoredMemory
for rows.Next() {
var m models.Memory
var embeddingJSON string
if err := rows.Scan(
&m.ID, &m.Scope, &m.OwnerID, &m.UserID, &m.Key, &m.Value,
&m.SourceChannelID, &m.Confidence, &m.Status,
st(&m.CreatedAt), st(&m.UpdatedAt), &embeddingJSON,
); err != nil {
continue
}
var vec []float64
if err := json.Unmarshal([]byte(embeddingJSON), &vec); err != nil {
continue
}
// cosineSimilarity is defined in knowledge_bases.go (same package)
sim := cosineSimilarity(queryVec, vec)
if sim > 0.3 {
scored = append(scored, scoredMemory{Memory: m, Similarity: sim})
}
}
// Sort by similarity descending (simple insertion sort for small N)
for i := 0; i < len(scored); i++ {
for j := i + 1; j < len(scored); j++ {
if scored[j].Similarity > scored[i].Similarity {
scored[i], scored[j] = scored[j], scored[i]
}
}
}
// Take top N
result := make([]models.Memory, 0, limit)
for i := 0; i < len(scored) && i < limit; i++ {
result = append(result, scored[i].Memory)
}
return result, rows.Err()
}
func mergeResultsSQLite(semantic, keyword []models.Memory, limit int) []models.Memory {
seen := make(map[string]bool, len(semantic))
result := make([]models.Memory, 0, limit)
for _, m := range semantic {
if len(result) >= limit {
break
}
seen[m.ID] = true
result = append(result, m)
}
for _, m := range keyword {
if len(result) >= limit {
break
}
if !seen[m.ID] {
result = append(result, m)
}
}
return result
}

17
server/store/sqlite/persona.go
View File

@@ -17,7 +17,8 @@ func NewPersonaStore() *PersonaStore { return &PersonaStore{} }
const personaCols = `id, name, description, icon, avatar, base_model_id, provider_config_id,
system_prompt, temperature, max_tokens, thinking_budget, top_p,
scope, owner_id, created_by, is_active, is_shared, created_at, updated_at`
scope, owner_id, created_by, is_active, is_shared, memory_enabled, memory_extraction_prompt,
created_at, updated_at`
func (s *PersonaStore) Create(ctx context.Context, p *models.Persona) error {
p.ID = store.NewID()
@@ -27,13 +28,15 @@ func (s *PersonaStore) Create(ctx context.Context, p *models.Persona) error {
_, err := DB.ExecContext(ctx, `
INSERT INTO personas (id, name, description, icon, avatar, base_model_id, provider_config_id,
system_prompt, temperature, max_tokens, thinking_budget, top_p,
scope, owner_id, created_by, is_active, is_shared, created_at, updated_at)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
scope, owner_id, created_by, is_active, is_shared, memory_enabled, memory_extraction_prompt,
created_at, updated_at)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
p.ID, p.Name, p.Description, p.Icon, p.Avatar, p.BaseModelID,
models.NullString(p.ProviderConfigID),
p.SystemPrompt, models.NullFloat(p.Temperature), models.NullInt(p.MaxTokens),
models.NullInt(p.ThinkingBudget), models.NullFloat(p.TopP),
p.Scope, models.NullString(p.OwnerID), p.CreatedBy, p.IsActive, p.IsShared,
p.MemoryEnabled, p.MemoryExtractionPrompt,
now.Format(timeFmt), now.Format(timeFmt),
)
return err
@@ -93,6 +96,12 @@ func (s *PersonaStore) Update(ctx context.Context, id string, patch models.Perso
if patch.IsShared != nil {
b.Set("is_shared", *patch.IsShared)
}
if patch.MemoryEnabled != nil {
b.Set("memory_enabled", *patch.MemoryEnabled)
}
if patch.MemoryExtractionPrompt != nil {
b.Set("memory_extraction_prompt", *patch.MemoryExtractionPrompt)
}
if !b.HasSets() {
return nil
}
@@ -286,6 +295,7 @@ func scanPersona(row *sql.Row) (*models.Persona, error) {
&p.BaseModelID, &providerConfigID,
&p.SystemPrompt, &temp, &maxTokens, &thinkingBudget, &topP,
&p.Scope, &ownerID, &p.CreatedBy, &p.IsActive, &p.IsShared,
&p.MemoryEnabled, &p.MemoryExtractionPrompt,
st(&p.CreatedAt), st(&p.UpdatedAt),
)
if err != nil {
@@ -312,6 +322,7 @@ func scanPersonas(rows *sql.Rows) ([]models.Persona, error) {
&p.BaseModelID, &providerConfigID,
&p.SystemPrompt, &temp, &maxTokens, &thinkingBudget, &topP,
&p.Scope, &ownerID, &p.CreatedBy, &p.IsActive, &p.IsShared,
&p.MemoryEnabled, &p.MemoryExtractionPrompt,
st(&p.CreatedAt), st(&p.UpdatedAt),
)
if err != nil {

1
server/store/sqlite/stores.go
View File

@@ -31,5 +31,6 @@ func NewStores(db *sql.DB) store.Stores {
KnowledgeBases: NewKnowledgeBaseStore(),
Groups: NewGroupStore(),
ResourceGrants: NewResourceGrantStore(),
Memories: NewMemoryStore(),
}
}

47
server/store/store_memory.go Normal file
View File

@@ -0,0 +1,47 @@
// ── store_memory.go ─────────────────────────
// MemoryStore interface for v0.18.0.
// Add this interface to store/interfaces.go and add
// Memories MemoryStore
// to the Stores struct.
package store
import (
"context"
"git.gobha.me/xcaliber/chat-switchboard/models"
)
// =========================================
// MEMORY STORE (v0.18.0)
// =========================================
type MemoryStore interface {
// Upsert creates or updates a memory (matched on scope+owner+user+key).
// If a memory with the same key exists, value/confidence/status are updated.
Upsert(ctx context.Context, m *models.Memory) error
// GetByID returns a single memory by ID.
GetByID(ctx context.Context, id string) (*models.Memory, error)
// List returns memories matching the filter criteria.
List(ctx context.Context, f models.MemoryFilter) ([]models.Memory, error)
// Delete hard-deletes a memory by ID.
Delete(ctx context.Context, id string) error
// Archive sets status to 'archived' (soft delete).
Archive(ctx context.Context, id string) error
// Recall returns active memories relevant to a query, merging scopes.
// Combines user + persona + persona_user memories for the given context.
// Results are ordered by confidence DESC, updated_at DESC.
Recall(ctx context.Context, userID string, personaID *string, query string, limit int) ([]models.Memory, error)
// CountByOwner returns the number of active memories for a scope+owner.
CountByOwner(ctx context.Context, scope, ownerID string) (int, error)
// RecallHybrid returns active memories using vector similarity + keyword search.
// If queryVec is nil/empty, falls back to keyword-only (same as Recall).
RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error)
}

15
server/store/store_memory_hybrid.go Normal file
View File

@@ -0,0 +1,15 @@
// ── store_memory_hybrid.go ──────────────────
// Phase 2 addition to the MemoryStore interface.
//
// Add this method to the MemoryStore interface in store/store_memory.go:
//
// // RecallHybrid returns active memories using vector similarity + keyword search.
// // If queryVec is nil/empty, falls back to keyword-only (same as Recall).
// // Merges results from applicable scopes with deduplication.
// RecallHybrid(ctx context.Context, userID string, personaID *string, query string, queryVec []float64, limit int) ([]models.Memory, error)
//
// The implementations are in:
// - store/postgres/memory_hybrid.go (pgvector cosine distance)
// - store/sqlite/memory_hybrid.go (app-level cosine similarity)
package store

238
server/tools/memory.go Normal file
View File

@@ -0,0 +1,238 @@
package tools
import (
"context"
"encoding/json"
"fmt"
"log"
"strings"
"git.gobha.me/xcaliber/chat-switchboard/database"
"git.gobha.me/xcaliber/chat-switchboard/knowledge"
"git.gobha.me/xcaliber/chat-switchboard/models"
"git.gobha.me/xcaliber/chat-switchboard/store"
)
// ── Late Registration ────────────────────────
// Memory tools use late registration because they need stores + embedder.
// RegisterMemoryTools registers the memory_save and memory_recall tools.
// Called from main.go after stores and embedder are initialized.
func RegisterMemoryTools(stores store.Stores, embedder *knowledge.Embedder) {
if stores.Memories == nil {
log.Println("⚠ memory tools: MemoryStore not available, skipping registration")
return
}
Register(&memorySaveTool{stores: stores, embedder: embedder})
Register(&memoryRecallTool{stores: stores, embedder: embedder})
log.Println("✅ memory tools registered (memory_save, memory_recall)")
}
// ═══════════════════════════════════════════
// memory_save
// ═══════════════════════════════════════════
type memorySaveTool struct {
stores store.Stores
embedder *knowledge.Embedder
}
func (t *memorySaveTool) Definition() ToolDef {
return ToolDef{
Name: "memory_save",
DisplayName: "Save Memory",
Category: "memory",
Description: "Save a fact or preference about the user for future conversations. " +
"Use this when the user shares something worth remembering long-term " +
"(preferences, technical stack, project details, personal facts). " +
"Memories persist across all conversations.",
Parameters: JSONSchema(map[string]interface{}{
"key": Prop("string", "Short label for the memory (2-5 words, e.g. 'preferred language', 'deployment target')"),
"value": Prop("string", "The fact or detail to remember (1-2 sentences)"),
"confidence": map[string]interface{}{
"type": "number",
"description": "How confident you are in this fact (0.0-1.0, default 0.9)",
},
}, []string{"key", "value"}),
}
}
func (t *memorySaveTool) Execute(ctx context.Context, execCtx ExecutionContext, argsJSON string) (string, error) {
var args struct {
Key string `json:"key"`
Value string `json:"value"`
Confidence float64 `json:"confidence"`
}
if err := json.Unmarshal([]byte(argsJSON), &args); err != nil {
return "", fmt.Errorf("invalid arguments: %w", err)
}
if args.Key == "" || args.Value == "" {
return "", fmt.Errorf("key and value are required")
}
confidence := args.Confidence
if confidence <= 0 || confidence > 1.0 {
confidence = 0.9
}
mem := &models.Memory{
ID: store.NewID(),
Scope: models.MemoryScopeUser,
OwnerID: execCtx.UserID,
Key: strings.TrimSpace(args.Key),
Value: strings.TrimSpace(args.Value),
Confidence: confidence,
Status: models.MemoryStatusActive,
}
if execCtx.ChannelID != "" {
mem.SourceChannelID = &execCtx.ChannelID
}
// If persona is active, use persona_user scope
if execCtx.PersonaID != "" {
mem.Scope = models.MemoryScopePersonaUser
mem.OwnerID = execCtx.PersonaID
mem.UserID = &execCtx.UserID
}
if err := t.stores.Memories.Upsert(ctx, mem); err != nil {
return "", fmt.Errorf("save memory: %w", err)
}
// Embed for semantic recall
t.embedMemory(ctx, mem, execCtx.UserID)
return fmt.Sprintf("Remembered: %s = %s (confidence: %.0f%%)",
args.Key, args.Value, confidence*100), nil
}
func (t *memorySaveTool) embedMemory(ctx context.Context, m *models.Memory, userID string) {
if t.embedder == nil || !t.embedder.IsConfigured(ctx) {
return
}
text := m.Key + ": " + m.Value
var teamID *string
if t.stores.Teams != nil {
ids, _ := t.stores.Teams.GetUserTeamIDs(ctx, userID)
if len(ids) > 0 {
teamID = &ids[0]
}
}
result, err := t.embedder.EmbedChunks(ctx, userID, teamID, []string{text})
if err != nil || len(result.Vectors) == 0 {
return
}
vecStr := vectorToString(result.Vectors[0])
if database.IsSQLite() {
database.DB.Exec(`UPDATE memories SET embedding = ? WHERE id = ?`,
vecStr, m.ID)
} else {
database.DB.Exec(`UPDATE memories SET embedding = $1::vector WHERE id = $2`,
vecStr, m.ID)
}
log.Printf("🧠 memory %s embedded (%d dims)", m.ID[:8], len(result.Vectors[0]))
}
// ═══════════════════════════════════════════
// memory_recall
// ═══════════════════════════════════════════
type memoryRecallTool struct {
stores store.Stores
embedder *knowledge.Embedder
}
func (t *memoryRecallTool) Definition() ToolDef {
return ToolDef{
Name: "memory_recall",
DisplayName: "Recall Memories",
Category: "memory",
Description: "Search your memories about this user. Use this at the start of conversations " +
"or when you need context about the user's preferences, projects, or history.",
Parameters: JSONSchema(map[string]interface{}{
"query": Prop("string", "Optional search query to filter memories (leave empty for all)"),
}, nil),
}
}
func (t *memoryRecallTool) Execute(ctx context.Context, execCtx ExecutionContext, argsJSON string) (string, error) {
var args struct {
Query string `json:"query"`
}
if err := json.Unmarshal([]byte(argsJSON), &args); err != nil {
return "", fmt.Errorf("invalid arguments: %w", err)
}
var personaID *string
if execCtx.PersonaID != "" {
personaID = &execCtx.PersonaID
}
// Try hybrid recall if embedder available and query provided
var memories []models.Memory
var err error
if t.embedder != nil && t.embedder.IsConfigured(ctx) && args.Query != "" {
memories, err = t.recallWithEmbedding(ctx, execCtx.UserID, personaID, args.Query)
}
// Fall back to keyword recall
if err != nil || len(memories) == 0 {
memories, err = t.stores.Memories.Recall(ctx, execCtx.UserID, personaID, args.Query, 20)
}
if err != nil {
return "", fmt.Errorf("recall memories: %w", err)
}
if len(memories) == 0 {
suffix := ""
if args.Query != "" {
suffix = " matching '" + args.Query + "'"
}
return "No memories found" + suffix + ".", nil
}
var sb strings.Builder
sb.WriteString(fmt.Sprintf("Found %d memories:\n\n", len(memories)))
for _, m := range memories {
scope := m.Scope
if scope == "persona_user" {
scope = "persona"
}
sb.WriteString(fmt.Sprintf("• [%s] %s: %s (%.0f%% confidence)\n",
scope, m.Key, m.Value, m.Confidence*100))
}
return sb.String(), nil
}
func (t *memoryRecallTool) recallWithEmbedding(ctx context.Context, userID string, personaID *string, query string) ([]models.Memory, error) {
text := query
if len(text) > 2000 {
text = text[:2000]
}
var teamID *string
if t.stores.Teams != nil {
ids, _ := t.stores.Teams.GetUserTeamIDs(ctx, userID)
if len(ids) > 0 {
teamID = &ids[0]
}
}
result, err := t.embedder.EmbedChunks(ctx, userID, teamID, []string{text})
if err != nil || len(result.Vectors) == 0 {
return nil, err
}
return t.stores.Memories.RecallHybrid(ctx, userID, personaID, query, result.Vectors[0], 20)
}

298
src/css/memory.css Normal file
View File

@@ -0,0 +1,298 @@
/* ==========================================
* Chat Switchboard – Memory UI (v0.18.0)
* ========================================== */
/* ── Summary Stats ─────────────────────── */
.memory-summary {
display: flex;
gap: 16px;
margin-bottom: 12px;
padding: 10px 0;
}
.memory-stat {
display: flex;
flex-direction: column;
align-items: center;
padding: 10px 20px;
background: var(--bg-2);
border-radius: 8px;
min-width: 80px;
}
.memory-stat-value {
font-size: 1.5rem;
font-weight: 600;
color: var(--text-1);
line-height: 1.2;
}
.memory-stat-label {
font-size: 0.75rem;
color: var(--text-3);
text-transform: uppercase;
letter-spacing: 0.5px;
}
/* ── Toolbar ───────────────────────────── */
.memory-toolbar {
display: flex;
flex-direction: column;
gap: 8px;
margin-bottom: 12px;
}
.memory-filter-row {
display: flex;
gap: 8px;
}
.memory-select {
padding: 5px 8px;
border-radius: 6px;
border: 1px solid var(--border);
background: var(--bg-1);
color: var(--text-1);
font-size: 0.82rem;
min-width: 120px;
}
.memory-search {
flex: 1;
padding: 5px 10px;
border-radius: 6px;
border: 1px solid var(--border);
background: var(--bg-1);
color: var(--text-1);
font-size: 0.82rem;
}
.memory-search::placeholder {
color: var(--text-3);
}
.memory-actions-row {
display: flex;
gap: 6px;
align-items: center;
}
/* ── Memory List ───────────────────────── */
.memory-list {
display: flex;
flex-direction: column;
gap: 6px;
max-height: 400px;
overflow-y: auto;
padding-right: 4px;
}
/* ── Memory Card ───────────────────────── */
.memory-card {
background: var(--bg-2);
border: 1px solid var(--border);
border-radius: 8px;
padding: 10px 12px;
transition: border-color 0.15s;
}
.memory-card:hover {
border-color: var(--accent);
}
.memory-card-pending {
border-left: 3px solid var(--warning, #f0ad4e);
}
.memory-card-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 4px;
gap: 8px;
}
.memory-key {
font-weight: 600;
font-size: 0.85rem;
color: var(--text-1);
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.memory-badges {
display: flex;
gap: 4px;
align-items: center;
flex-shrink: 0;
}
.memory-scope-badge {
font-size: 0.68rem;
padding: 1px 6px;
border-radius: 4px;
text-transform: uppercase;
letter-spacing: 0.3px;
font-weight: 500;
}
.memory-scope-user { background: var(--accent-dim, #1a3a5c); color: var(--accent-text, #6db3f2); }
.memory-scope-persona { background: #3a2a1a; color: #f0ad4e; }
.memory-scope-persona_user { background: #2a1a3a; color: #c084fc; }
.memory-confidence {
font-size: 0.7rem;
padding: 1px 5px;
border-radius: 4px;
font-weight: 500;
background: var(--bg-3);
color: var(--text-2);
}
.memory-confidence-high { color: var(--success, #5cb85c); }
.memory-confidence-medium { color: var(--warning, #f0ad4e); }
.memory-confidence-low { color: var(--text-3); }
.memory-owner {
font-size: 0.68rem;
color: var(--text-3);
font-family: monospace;
}
.memory-card-value {
font-size: 0.82rem;
color: var(--text-2);
line-height: 1.4;
margin-bottom: 6px;
word-break: break-word;
}
.memory-card-footer {
display: flex;
justify-content: space-between;
align-items: center;
}
.memory-date {
font-size: 0.7rem;
color: var(--text-3);
}
.memory-card-actions {
display: flex;
gap: 4px;
}
/* ── Tiny Action Buttons ───────────────── */
.btn-tiny {
padding: 2px 8px;
font-size: 0.72rem;
border-radius: 4px;
border: 1px solid var(--border);
background: var(--bg-1);
color: var(--text-2);
cursor: pointer;
line-height: 1.4;
transition: background 0.12s, color 0.12s;
}
.btn-tiny:hover {
background: var(--bg-3);
color: var(--text-1);
}
.btn-tiny.btn-approve {
color: var(--success, #5cb85c);
border-color: var(--success, #5cb85c);
}
.btn-tiny.btn-approve:hover {
background: var(--success, #5cb85c);
color: #fff;
}
.btn-tiny.btn-reject {
color: var(--danger, #d9534f);
border-color: var(--danger, #d9534f);
}
.btn-tiny.btn-reject:hover {
background: var(--danger, #d9534f);
color: #fff;
}
/* ── Edit Form ─────────────────────────── */
.memory-edit-form {
padding: 4px 0;
}
.memory-edit-form .form-group {
margin-bottom: 6px;
}
.memory-edit-input {
width: 100%;
padding: 5px 8px;
border-radius: 6px;
border: 1px solid var(--border);
background: var(--bg-1);
color: var(--text-1);
font-size: 0.82rem;
font-family: inherit;
}
.memory-edit-input:focus {
outline: none;
border-color: var(--accent);
}
/* ── Persona Memory Section ────────────── */
.memory-persona-section {
margin-top: 8px;
}
.form-divider {
border-top: 1px solid var(--border);
margin: 10px 0;
}
.form-section-title {
font-size: 0.85rem;
font-weight: 600;
color: var(--text-2);
margin: 0 0 6px;
}
/* ── Admin Toolbar ─────────────────────── */
.admin-toolbar-label {
font-size: 0.82rem;
color: var(--text-2);
margin-right: auto;
}
/* ── Responsive ────────────────────────── */
@media (max-width: 640px) {
.memory-summary {
gap: 8px;
}
.memory-stat {
padding: 8px 12px;
}
.memory-filter-row {
flex-direction: column;
}
.memory-card-header {
flex-direction: column;
align-items: flex-start;
}
}

23
src/index.html
View File

@@ -20,6 +20,7 @@
<meta name="apple-mobile-web-app-status-bar-style" content="black-translucent">
<link rel="stylesheet" href="css/styles.css?v=%%APP_VERSION%%">
<link rel="stylesheet" href="css/persona-kb.css?v=%%APP_VERSION%%">
<link rel="stylesheet" href="css/memory.css?v=%%APP_VERSION%%">
<link rel="stylesheet" href="branding/custom.css" onerror="this.remove()">
</head>
<body>
@@ -387,6 +388,7 @@
<button class="settings-tab" data-stab="usage" onclick="UI.switchSettingsTab('usage')" id="settingsUsageTabBtn">Usage</button>
<button class="settings-tab" data-stab="roles" onclick="UI.switchSettingsTab('roles')" id="settingsRolesTabBtn">Model Roles</button>
<button class="settings-tab" data-stab="knowledgeBases" onclick="UI.switchSettingsTab('knowledgeBases')" id="settingsKBTabBtn" style="display:none">Knowledge</button>
<button class="settings-tab" data-stab="memory" onclick="UI.switchSettingsTab('memory')" id="settingsMemoryTabBtn">Memory</button>
</div>
<div class="modal-body">
<!-- General Tab -->
@@ -550,6 +552,14 @@
<div class="settings-tab-content" id="settingsKnowledgeBasesTab" style="display:none">
<div id="kbManagePanel"></div>
</div>
<!-- Memory Tab (v0.18.0) -->
<div class="settings-tab-content" id="settingsMemoryTab" style="display:none">
<section class="settings-section">
<h3 style="font-size:14px;margin-bottom:4px">My Memories</h3>
<p class="section-hint" style="margin-bottom:8px">Facts and preferences learned from your conversations. Memories help AI provide more personalized responses.</p>
<div id="settingsMemoryContent"></div>
</section>
</div>
</div>
<div class="modal-footer"><button class="btn-primary" id="settingsSaveBtn">Save Settings</button></div>
</div>
@@ -791,6 +801,9 @@
<div class="admin-section-content" id="adminKnowledgeBasesTab" style="display:none">
<div id="adminKBContent"></div>
</div>
<div class="admin-section-content" id="adminMemoryTab" style="display:none">
<div id="adminMemoryContent"></div>
</div>
<!-- System -->
<div class="admin-section-content" id="adminSettingsTab" style="display:none">
@@ -904,6 +917,15 @@
</div>
</div>
</section>
<section class="settings-section">
<h3>Memory Extraction</h3>
<label class="checkbox-label"><input type="checkbox" id="adminMemoryExtractionEnabled"> Enable automatic memory extraction</label>
<p class="section-hint">When enabled, the system automatically extracts memorable facts from conversations using the utility model. Extracted memories require approval before becoming active. Requires a utility model role.</p>
<div id="memoryExtractionConfigFields" style="display:none;margin-top:8px">
<label class="checkbox-label"><input type="checkbox" id="adminMemoryAutoApprove"> Auto-approve extracted memories</label>
<p class="section-hint">Skip the review pipeline and activate extracted memories immediately. Not recommended for sensitive environments.</p>
</div>
</section>
<section class="admin-section">
<h4>🔐 Encryption</h4>
<div id="adminVaultStatus"><span class="empty-hint">Loading…</span></div>
@@ -1100,6 +1122,7 @@
<script src="js/attachments.js?v=%%APP_VERSION%%"></script>
<script src="js/tools-toggle.js?v=%%APP_VERSION%%"></script>
<script src="js/knowledge-ui.js?v=%%APP_VERSION%%"></script>
<script src="js/memory-ui.js?v=%%APP_VERSION%%"></script>
<script src="js/persona-kb.js?v=%%APP_VERSION%%"></script>
<script src="js/chat.js?v=%%APP_VERSION%%"></script>
<script src="js/settings-handlers.js?v=%%APP_VERSION%%"></script>

16
src/js/api.js
View File

@@ -671,6 +671,22 @@ const API = {
getChannelKBs(channelId) { return this._get(`/api/v1/channels/${channelId}/knowledge-bases`); },
setChannelKBs(channelId, kbIds) { return this._put(`/api/v1/channels/${channelId}/knowledge-bases`, { kb_ids: kbIds }); },
// ── Memories (v0.18.0) ───────────────────
listMemories(status, query) {
const params = new URLSearchParams();
if (status) params.set('status', status);
if (query) params.set('query', query);
const qs = params.toString();
return this._get('/api/v1/memories' + (qs ? '?' + qs : ''));
},
getMemoryCount() { return this._get('/api/v1/memories/count'); },
updateMemory(id, data) { return this._put(`/api/v1/memories/${id}`, data); },
deleteMemory(id) { return this._del(`/api/v1/memories/${id}`); },
approveMemory(id) { return this._post(`/api/v1/memories/${id}/approve`, {}); },
rejectMemory(id) { return this._post(`/api/v1/memories/${id}/reject`, {}); },
bulkApproveMemories(ids) { return this._post('/api/v1/admin/memories/bulk-approve', { ids }); },
adminListPendingMemories() { return this._get('/api/v1/admin/memories/pending'); },
// ── HTTP Internals ───────────────────────
_esc(s) {

364
src/js/memory-ui.js Normal file
View File

@@ -0,0 +1,364 @@
// ==========================================
// Chat Switchboard – Memory UI (v0.18.0 Phase 3)
// ==========================================
// User-facing memory management in Settings,
// admin memory review in Admin Panel,
// and persona memory config in preset forms.
const MemoryUI = {
// ── State ────────────────────────────────
_memories: [],
_pendingMemories: [],
_filter: { scope: 'user', status: 'active', query: '' },
// ═════════════════════════════════════════
// User Settings Tab
// ═════════════════════════════════════════
async openSettingsPanel() {
const panel = document.getElementById('settingsMemoryContent');
if (!panel) return;
panel.innerHTML = '<div class="loading">Loading memories...</div>';
try {
// Load memory count summary
const count = await API.getMemoryCount();
const active = count.active || 0;
const pending = count.pending || 0;
panel.innerHTML = `
<div class="memory-summary">
<div class="memory-stat">
<span class="memory-stat-value">${active}</span>
<span class="memory-stat-label">Active</span>
</div>
<div class="memory-stat">
<span class="memory-stat-value">${pending}</span>
<span class="memory-stat-label">Pending Review</span>
</div>
</div>
<div class="memory-toolbar">
<div class="memory-filter-row">
<select id="memoryStatusFilter" class="memory-select">
<option value="active">Active</option>
<option value="pending_review">Pending Review</option>
<option value="archived">Archived</option>
</select>
<input type="text" id="memorySearchInput" placeholder="Search memories..." class="memory-search">
</div>
<div class="memory-actions-row">
<button class="btn-small" id="memoryRefreshBtn" title="Refresh">↻ Refresh</button>
${pending > 0 ? `<button class="btn-small btn-primary" id="memoryApproveAllBtn">✓ Approve All Pending</button>` : ''}
</div>
</div>
<div id="memoryList" class="memory-list"></div>
`;
// Wire events
document.getElementById('memoryStatusFilter')?.addEventListener('change', (e) => {
MemoryUI._filter.status = e.target.value;
MemoryUI.loadMemories();
});
document.getElementById('memorySearchInput')?.addEventListener('input', _debounce((e) => {
MemoryUI._filter.query = e.target.value;
MemoryUI.loadMemories();
}, 300));
document.getElementById('memoryRefreshBtn')?.addEventListener('click', () => MemoryUI.openSettingsPanel());
document.getElementById('memoryApproveAllBtn')?.addEventListener('click', () => MemoryUI.approveAllPending());
// Load initial list
await this.loadMemories();
} catch (e) {
panel.innerHTML = `<div class="error-hint">${esc(e.message)}</div>`;
}
},
async loadMemories() {
const el = document.getElementById('memoryList');
if (!el) return;
el.innerHTML = '<div class="loading">Loading...</div>';
try {
const data = await API.listMemories(this._filter.status, this._filter.query);
this._memories = data.memories || [];
if (!this._memories.length) {
el.innerHTML = `<div class="empty-hint">No ${this._filter.status.replace('_', ' ')} memories${this._filter.query ? ' matching "' + esc(this._filter.query) + '"' : ''}</div>`;
return;
}
el.innerHTML = this._memories.map(m => this._renderMemoryCard(m)).join('');
this._wireMemoryCards(el);
} catch (e) {
el.innerHTML = `<div class="error-hint">${esc(e.message)}</div>`;
}
},
_renderMemoryCard(m) {
const confidence = Math.round(m.confidence * 100);
const confidenceClass = confidence >= 80 ? 'high' : confidence >= 50 ? 'medium' : 'low';
const updated = m.updated_at ? new Date(m.updated_at).toLocaleDateString() : '';
const scopeLabel = m.scope === 'persona_user' ? 'persona' : m.scope;
return `
<div class="memory-card" data-id="${m.id}">
<div class="memory-card-header">
<span class="memory-key">${esc(m.key)}</span>
<div class="memory-badges">
<span class="memory-scope-badge memory-scope-${m.scope}">${scopeLabel}</span>
<span class="memory-confidence memory-confidence-${confidenceClass}" title="Confidence: ${confidence}%">${confidence}%</span>
</div>
</div>
<div class="memory-card-value">${esc(m.value)}</div>
<div class="memory-card-footer">
<span class="memory-date">${updated}</span>
<div class="memory-card-actions">
${m.status === 'pending_review' ? `
<button class="btn-tiny btn-approve" data-action="approve" data-id="${m.id}" title="Approve">✓</button>
<button class="btn-tiny btn-reject" data-action="reject" data-id="${m.id}" title="Reject">✕</button>
` : ''}
<button class="btn-tiny" data-action="edit" data-id="${m.id}" title="Edit">✎</button>
<button class="btn-tiny btn-danger" data-action="delete" data-id="${m.id}" title="Delete">🗑</button>
</div>
</div>
</div>`;
},
_wireMemoryCards(container) {
container.querySelectorAll('[data-action]').forEach(btn => {
btn.addEventListener('click', async (e) => {
const id = btn.dataset.id;
const action = btn.dataset.action;
if (action === 'approve') await this.approveMemory(id);
else if (action === 'reject') await this.rejectMemory(id);
else if (action === 'delete') await this.deleteMemory(id);
else if (action === 'edit') this.editMemory(id);
});
});
},
async approveMemory(id) {
try {
await API.approveMemory(id);
UI.toast('Memory approved', 'success');
this.loadMemories();
} catch (e) { UI.toast(e.message, 'error'); }
},
async rejectMemory(id) {
try {
await API.rejectMemory(id);
UI.toast('Memory rejected', 'success');
this.loadMemories();
} catch (e) { UI.toast(e.message, 'error'); }
},
async deleteMemory(id) {
if (!confirm('Delete this memory permanently?')) return;
try {
await API.deleteMemory(id);
UI.toast('Memory deleted', 'success');
this.loadMemories();
} catch (e) { UI.toast(e.message, 'error'); }
},
editMemory(id) {
const m = this._memories.find(x => x.id === id);
if (!m) return;
const card = document.querySelector(`.memory-card[data-id="${id}"]`);
if (!card) return;
card.innerHTML = `
<div class="memory-edit-form">
<div class="form-group">
<label>Key</label>
<input type="text" id="memEdit_key_${id}" value="${esc(m.key)}" class="memory-edit-input">
</div>
<div class="form-group">
<label>Value</label>
<textarea id="memEdit_val_${id}" rows="2" class="memory-edit-input">${esc(m.value)}</textarea>
</div>
<div class="form-row" style="gap:6px">
<button class="btn-small btn-primary" id="memEditSave_${id}">Save</button>
<button class="btn-small" id="memEditCancel_${id}">Cancel</button>
</div>
</div>
`;
document.getElementById(`memEditSave_${id}`)?.addEventListener('click', async () => {
const key = document.getElementById(`memEdit_key_${id}`).value.trim();
const value = document.getElementById(`memEdit_val_${id}`).value.trim();
if (!key || !value) return UI.toast('Key and value are required', 'warning');
try {
await API.updateMemory(id, { key, value });
UI.toast('Memory updated', 'success');
this.loadMemories();
} catch (e) { UI.toast(e.message, 'error'); }
});
document.getElementById(`memEditCancel_${id}`)?.addEventListener('click', () => {
this.loadMemories();
});
},
async approveAllPending() {
try {
const data = await API.listMemories('pending_review', '');
const ids = (data.memories || []).map(m => m.id);
if (!ids.length) return UI.toast('No pending memories', 'info');
if (!confirm(`Approve all ${ids.length} pending memories?`)) return;
await API.bulkApproveMemories(ids);
UI.toast(`${ids.length} memories approved`, 'success');
this.openSettingsPanel();
} catch (e) { UI.toast(e.message, 'error'); }
},
// ═════════════════════════════════════════
// Admin Panel — Memory Review
// ═════════════════════════════════════════
async openAdminPanel() {
const el = document.getElementById('adminMemoryContent');
if (!el) return;
el.innerHTML = '<div class="loading">Loading...</div>';
try {
const data = await API.adminListPendingMemories();
const pending = data.memories || [];
el.innerHTML = `
<div class="admin-toolbar">
<span class="admin-toolbar-label">${pending.length} pending memories</span>
${pending.length > 0 ? `
<button class="btn-small btn-primary" id="adminBulkApproveBtn">✓ Approve All</button>
` : ''}
<button class="btn-small" id="adminMemoryRefreshBtn">↻ Refresh</button>
</div>
<div id="adminMemoryList" class="memory-list">
${pending.length === 0 ? '<div class="empty-hint">No memories pending review</div>' : ''}
${pending.map(m => this._renderAdminMemoryRow(m)).join('')}
</div>
`;
// Wire events
document.getElementById('adminBulkApproveBtn')?.addEventListener('click', async () => {
const ids = pending.map(m => m.id);
if (!confirm(`Approve all ${ids.length} pending memories?`)) return;
try {
await API.bulkApproveMemories(ids);
UI.toast(`${ids.length} memories approved`, 'success');
MemoryUI.openAdminPanel();
} catch (e) { UI.toast(e.message, 'error'); }
});
document.getElementById('adminMemoryRefreshBtn')?.addEventListener('click', () => MemoryUI.openAdminPanel());
// Wire per-row actions
el.querySelectorAll('[data-action]').forEach(btn => {
btn.addEventListener('click', async () => {
const id = btn.dataset.id;
try {
if (btn.dataset.action === 'approve') {
await API.approveMemory(id);
UI.toast('Approved', 'success');
} else if (btn.dataset.action === 'reject') {
await API.rejectMemory(id);
UI.toast('Rejected', 'success');
}
MemoryUI.openAdminPanel();
} catch (e) { UI.toast(e.message, 'error'); }
});
});
} catch (e) {
el.innerHTML = `<div class="error-hint">${esc(e.message)}</div>`;
}
},
_renderAdminMemoryRow(m) {
const confidence = Math.round(m.confidence * 100);
const scope = m.scope === 'persona_user' ? 'persona' : m.scope;
return `
<div class="memory-card memory-card-pending">
<div class="memory-card-header">
<span class="memory-key">${esc(m.key)}</span>
<div class="memory-badges">
<span class="memory-scope-badge memory-scope-${m.scope}">${scope}</span>
<span class="memory-confidence">${confidence}%</span>
<span class="memory-owner">${esc(m.owner_id?.substring(0, 8) || '?')}…</span>
</div>
</div>
<div class="memory-card-value">${esc(m.value)}</div>
<div class="memory-card-footer">
<span class="memory-date">${m.updated_at ? new Date(m.updated_at).toLocaleDateString() : ''}</span>
<div class="memory-card-actions">
<button class="btn-tiny btn-approve" data-action="approve" data-id="${m.id}">✓ Approve</button>
<button class="btn-tiny btn-reject" data-action="reject" data-id="${m.id}">✕ Reject</button>
</div>
</div>
</div>`;
},
// ═════════════════════════════════════════
// Persona Form Memory Fields
// ═════════════════════════════════════════
// Call this after renderPresetForm to append memory fields.
// container: the form container element
// prefix: the form prefix (e.g. 'pf', 'apf')
appendMemoryFields(container, prefix) {
const section = document.createElement('div');
section.className = 'memory-persona-section';
section.innerHTML = `
<div class="form-divider"></div>
<h4 class="form-section-title">Memory</h4>
<label class="checkbox-label">
<input type="checkbox" id="${prefix}_memoryEnabled" checked>
Enable memory for this persona
</label>
<p class="section-hint" style="margin:4px 0 8px">When enabled, this persona can save and recall memories from conversations.</p>
<div class="form-group" id="${prefix}_memoryPromptGroup">
<label>Custom Extraction Prompt <span class="form-hint">(optional)</span></label>
<textarea id="${prefix}_memoryExtractionPrompt" rows="3" placeholder="Leave blank for default extraction behavior. Custom prompts let you control what facts this persona remembers."></textarea>
</div>
`;
// Insert before the submit/cancel buttons
const submitRow = container.querySelector('.form-row:last-child');
if (submitRow) {
container.insertBefore(section, submitRow);
} else {
container.appendChild(section);
}
},
// Read memory values from a persona form
getMemoryValues(prefix) {
const enabled = document.getElementById(`${prefix}_memoryEnabled`)?.checked ?? true;
const prompt = document.getElementById(`${prefix}_memoryExtractionPrompt`)?.value.trim() || null;
return { memory_enabled: enabled, memory_extraction_prompt: prompt };
},
// Set memory values in a persona form
setMemoryValues(prefix, persona) {
const enabledEl = document.getElementById(`${prefix}_memoryEnabled`);
if (enabledEl) enabledEl.checked = persona.memory_enabled !== false;
const promptEl = document.getElementById(`${prefix}_memoryExtractionPrompt`);
if (promptEl) promptEl.value = persona.memory_extraction_prompt || '';
},
};
// ── Debounce helper ──────────────────────────
function _debounce(fn, ms) {
let timer;
return function(...args) {
clearTimeout(timer);
timer = setTimeout(() => fn.apply(this, args), ms);
};
}

9
src/js/settings-handlers.js
View File

@@ -247,6 +247,15 @@ async function handleSaveAdminSettings() {
await API.adminUpdateSetting('auto_compaction_threshold', { value: compactionThreshold / 100 });
await API.adminUpdateSetting('auto_compaction_cooldown_minutes', { value: compactionCooldown });
// Memory extraction (v0.18.0)
const memExtractionEnabled = document.getElementById('adminMemoryExtractionEnabled')?.checked || false;
const memAutoApprove = document.getElementById('adminMemoryAutoApprove')?.checked || false;
await API.adminUpdateSetting('memory_extraction', { value: {
enabled: memExtractionEnabled,
auto_approve: memAutoApprove,
}});
await API.adminUpdateSetting('memory_extraction_enabled', { value: memExtractionEnabled });
UI.toast('Settings saved', 'success');
// Live-apply: refresh policies and dependent UI
await initBanners();

22
src/js/ui-admin.js
View File

@@ -7,14 +7,14 @@
// ── Category → Section mapping ────────────
const ADMIN_SECTIONS = {
people: ['users', 'teams', 'groups'],
ai: ['providers', 'models', 'presets', 'roles', 'knowledgeBases'],
ai: ['providers', 'models', 'presets', 'roles', 'knowledgeBases', 'memory'],
system: ['settings', 'storage', 'extensions'],
monitoring: ['usage', 'audit', 'stats'],
};
const ADMIN_LABELS = {
users: 'Users', teams: 'Teams', groups: 'Groups',
providers: 'Providers', models: 'Models', presets: 'Personas', roles: 'Roles', knowledgeBases: 'Knowledge',
providers: 'Providers', models: 'Models', presets: 'Personas', roles: 'Roles', knowledgeBases: 'Knowledge', memory: 'Memory',
settings: 'Settings', storage: 'Storage', extensions: 'Extensions',
usage: 'Usage', audit: 'Audit', stats: 'Stats',
};
@@ -32,6 +32,10 @@ const ADMIN_LOADERS = {
if (typeof KnowledgeUI === 'undefined') { console.error('[Admin] KnowledgeUI not loaded — check js/knowledge-ui.js'); return; }
KnowledgeUI.openAdminPanel();
},
memory: () => {
if (typeof MemoryUI !== 'undefined') { MemoryUI.openAdminPanel(); }
else { console.error('[Admin] MemoryUI not loaded — check js/memory-ui.js'); }
},
settings: () => UI.loadAdminSettings(),
storage: () => typeof loadAdminStorage === 'function' ? loadAdminStorage() : null,
extensions: () => UI.loadAdminExtensions(),
@@ -963,6 +967,20 @@ Object.assign(UI, {
const compCooldown = document.getElementById('adminCompactionCooldown');
if (compCooldown) compCooldown.value = String(compactionCfg.cooldown || 30);
// Memory extraction config (v0.18.0)
const memExtractionCfg = getSetting('memory_extraction', {}) || {};
const memEnabled = document.getElementById('adminMemoryExtractionEnabled');
if (memEnabled) {
memEnabled.checked = !!memExtractionCfg.enabled;
const memFields = document.getElementById('memoryExtractionConfigFields');
if (memFields) memFields.style.display = memExtractionCfg.enabled ? '' : 'none';
memEnabled.addEventListener('change', () => {
if (memFields) memFields.style.display = memEnabled.checked ? '' : 'none';
});
}
const memAutoApprove = document.getElementById('adminMemoryAutoApprove');
if (memAutoApprove) memAutoApprove.checked = !!memExtractionCfg.auto_approve;
if (vaultEl) {
try {
const vault = await API.adminGetVaultStatus();

24
src/js/ui-core.js
View File

@@ -59,6 +59,14 @@ function renderPresetForm(containerEl, options = {}) {
<div class="form-group"><label>Temperature</label><input type="number" id="${pfx}_temp" step="0.1" min="0" max="2" placeholder="default"></div>
<div class="form-group"><label>Max Tokens</label><input type="number" id="${pfx}_maxTokens" placeholder="default"></div>
</div>
<div class="form-group memory-persona-section" style="border-top:1px solid var(--border);padding-top:8px;margin-top:4px">
<label class="checkbox-label" style="font-weight:500"><input type="checkbox" id="${pfx}_memoryEnabled" checked> Enable Memory</label>
<p class="form-hint" style="margin:2px 0 6px">When enabled, the AI can save and recall facts about users across conversations.</p>
<div id="${pfx}_memoryExtractionPromptWrap">
<label>Custom Extraction Prompt <span class="form-hint">(optional)</span></label>
<textarea id="${pfx}_memoryExtractionPrompt" rows="2" placeholder="Override the default extraction prompt for this persona..."></textarea>
</div>
</div>
<div class="form-row">
<button class="btn-small btn-primary" id="${pfx}_submitBtn">Create</button>
<button class="btn-small" id="${pfx}_cancelBtn">Cancel</button>
@@ -111,6 +119,11 @@ function renderPresetForm(containerEl, options = {}) {
// Pending avatar data URI
const img = document.querySelector(`#${pfx}_avatarPreview img`);
if (img?.src?.startsWith('data:')) v._pendingAvatar = img.src;
// Memory fields (v0.18.0)
const memCb = document.getElementById(`${pfx}_memoryEnabled`);
if (memCb) v.memory_enabled = memCb.checked;
const memPrompt = document.getElementById(`${pfx}_memoryExtractionPrompt`)?.value?.trim();
if (memPrompt) v.memory_extraction_prompt = memPrompt;
return v;
},
setValues(p) {
@@ -123,6 +136,9 @@ function renderPresetForm(containerEl, options = {}) {
if (el('model')) el('model').value = p.base_model_id || '';
if (showConfig && el('config')) el('config').value = p.provider_config_id || '';
if (showAvatar) form.updateAvatarPreview(p.avatar || null);
// Memory fields (v0.18.0)
if (el('memoryEnabled')) el('memoryEnabled').checked = p.memory_enabled !== false;
if (el('memoryExtractionPrompt')) el('memoryExtractionPrompt').value = p.memory_extraction_prompt || '';
},
clearForm() {
['name','desc','prompt','temp','maxTokens'].forEach(f => {
@@ -930,6 +946,14 @@ const UI = {
}
}
if (tab === 'appearance') UI.loadAppearanceSettings();
if (tab === 'memory') {
if (typeof MemoryUI !== 'undefined') {
MemoryUI.openSettingsPanel();
} else {
const c = document.getElementById('settingsMemoryContent');
if (c) c.innerHTML = '<div style="padding:20px;color:var(--text-3);text-align:center">Memory UI failed to load.</div>';
}
}
},
// ── Export ────────────────────────────────