Changeset 0.34.0 (#208)

Co-authored-by: Jeffrey Smith <jasafpro@gmail.com>
Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>
This commit is contained in:
2026-03-20 00:02:37 +00:00
committed by xcaliber
parent ed3e9363f2
commit d16bb93177
20 changed files with 5623 additions and 1 deletions

View File

@@ -1 +1 @@
0.33.0
0.34.0

View File

@@ -0,0 +1,95 @@
{{- if and .Values.backup.enabled (eq .Values.database.driver "postgres") }}
apiVersion: batch/v1
kind: CronJob
metadata:
name: {{ .Release.Name }}-backup
labels:
{{- include "switchboard.labels" . | nindent 4 }}
app.kubernetes.io/component: backup
spec:
schedule: {{ .Values.backup.schedule | quote }}
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: {{ .Values.backup.historyLimit | default 3 }}
failedJobsHistoryLimit: {{ .Values.backup.failedHistoryLimit | default 1 }}
jobTemplate:
spec:
activeDeadlineSeconds: {{ .Values.backup.activeDeadlineSeconds | default 3600 }}
template:
metadata:
labels:
app.kubernetes.io/name: {{ .Chart.Name }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/component: backup
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 12 }}
{{- end }}
restartPolicy: OnFailure
containers:
- name: backup
image: postgres:16-alpine
command:
- /bin/sh
- -c
- |
set -e
TIMESTAMP=$(date +%Y%m%d-%H%M%S)
FILENAME="switchboard-${TIMESTAMP}.sql.gz"
echo "Starting backup: ${FILENAME}"
pg_dump "${DATABASE_URL}" | gzip > "/backup/${FILENAME}"
echo "Backup written: /backup/${FILENAME} ($(du -h /backup/${FILENAME} | cut -f1))"
{{- if .Values.backup.s3.enabled }}
# Upload to S3-compatible storage
apk add --no-cache aws-cli >/dev/null 2>&1
export AWS_ACCESS_KEY_ID="${S3_ACCESS_KEY}"
export AWS_SECRET_ACCESS_KEY="${S3_SECRET_KEY}"
S3_PATH="s3://{{ .Values.backup.s3.bucket }}/{{ .Values.backup.s3.prefix }}${FILENAME}"
aws s3 cp "/backup/${FILENAME}" "${S3_PATH}" \
--endpoint-url "{{ .Values.backup.s3.endpoint }}" \
{{- if .Values.backup.s3.forcePathStyle }}
--no-verify-ssl \
{{- end }}
--region "{{ .Values.backup.s3.region }}"
echo "Uploaded to ${S3_PATH}"
{{- end }}
# Prune old local backups beyond retention
cd /backup
ls -t switchboard-*.sql.gz 2>/dev/null | tail -n +{{ add1 .Values.backup.retention }} | xargs -r rm -v
echo "Backup complete"
env:
- name: DATABASE_URL
value: {{ include "switchboard.databaseURL" . | quote }}
{{- if .Values.backup.s3.enabled }}
- name: S3_ACCESS_KEY
valueFrom:
secretKeyRef:
name: {{ include "switchboard.secretName" . }}
key: BACKUP_S3_ACCESS_KEY
optional: true
- name: S3_SECRET_KEY
valueFrom:
secretKeyRef:
name: {{ include "switchboard.secretName" . }}
key: BACKUP_S3_SECRET_KEY
optional: true
{{- end }}
{{- if .Values.backup.resources }}
resources:
{{- toYaml .Values.backup.resources | nindent 16 }}
{{- end }}
volumeMounts:
- name: backup-data
mountPath: /backup
volumes:
- name: backup-data
{{- if .Values.backup.persistence.enabled }}
persistentVolumeClaim:
claimName: {{ .Release.Name }}-backup
{{- else }}
emptyDir: {}
{{- end }}
{{- end }}

View File

@@ -0,0 +1,18 @@
{{- if and .Values.backup.enabled .Values.backup.persistence.enabled (eq .Values.database.driver "postgres") }}
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: {{ .Release.Name }}-backup
labels:
{{- include "switchboard.labels" . | nindent 4 }}
app.kubernetes.io/component: backup
spec:
accessModes:
- {{ .Values.backup.persistence.accessMode | default "ReadWriteOnce" }}
{{- if .Values.backup.persistence.storageClass }}
storageClassName: {{ .Values.backup.persistence.storageClass | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.backup.persistence.size | default "5Gi" }}
{{- end }}

View File

@@ -157,6 +157,35 @@ monitoring:
labels:
release: prometheus # match kube-prometheus-stack
# ── Backup (v0.34.0) ─────────────────────────
# Scheduled pg_dump backups (postgres only, ignored for sqlite).
backup:
enabled: false
schedule: "0 2 * * *" # daily at 02:00 UTC
retention: 7 # keep last N local backups
historyLimit: 3
failedHistoryLimit: 1
activeDeadlineSeconds: 3600
persistence:
enabled: true
storageClass: "" # use cluster default
size: 5Gi
accessMode: ReadWriteOnce
s3:
enabled: false
endpoint: ""
bucket: ""
region: us-east-1
prefix: "backups/"
forcePathStyle: true
resources:
requests:
cpu: 100m
memory: 256Mi
limits:
cpu: 500m
memory: 512Mi
# ── CORS ───────────────────────────────────
corsAllowedOrigins: "*"

217
docs/DESIGN-0.34.0.md Normal file
View File

@@ -0,0 +1,217 @@
# DESIGN-0.34.0 — Data Portability
Export your data, import it elsewhere, delete your account. Six
changesets covering user export, user import, GDPR delete, admin
team export/import, ChatGPT import, and K8s scheduled backups.
Depends on: v0.33.0 (Observability, current HEAD).
**Design decision:** One new migration (021) — indexes only, no new
tables. The export archive is a ZIP file (`.switchboard` extension)
containing a JSON manifest and per-entity JSON files. Sensitive fields
(password hashes, encrypted keys, storage keys) are stripped at export
time via entity sanitization.
**Archive format:** `.switchboard` ZIP with `format_version=1`:
```
manifest.json
data/users.json
data/channels.json
data/messages.json
data/channel_participants.json
data/channel_models.json
data/channel_cursors.json
data/notes.json
data/note_links.json
data/memories.json
data/projects.json
data/project_channels.json
data/project_knowledge_bases.json
data/project_notes.json
data/workspaces.json
data/workspace_files.json
data/folders.json
data/user_settings.json
data/notification_preferences.json
data/usage_entries.json
data/persona_groups.json
data/persona_group_members.json
data/files.json
files/{file_id}/{filename}
```
---
## What Already Existed
- **Package export** — `.pkg` ZIP format, `package_export.go`
- **Markdown export** — pandoc-based, `export.go`
- **Workspace archive** — ZIP/tar.gz with size limits, `workspace/archive.go`
- **Soft deletes** — `deleted_at` on messages
- **Audit trail** — `audit_logs` table
- **39 store interfaces** covering all entity types
---
## CS0 — Export Format + User Export
Foundation changeset. Defines the archive format and ExportStore interface.
### `server/export/format.go` (new)
Archive types: `Manifest`, `ManifestScope`, `ArchiveWriter`, `ArchiveReader`.
Size limits: 500 MB archive, 100 MB per file, 10K files max.
Key methods: `WriteManifest`, `WriteEntityJSON`, `WriteFile`,
`ReadManifest`, `ReadEntityJSON`, `FileEntries`.
### `server/export/entities.go` (new)
Per-entity sanitization functions. Export types strip sensitive fields:
- `ExportChannel` — removes `provider_config_id`
- `ExportMessage` — no changes needed
- `ExportWorkspace` — removes `root_path`, git credentials
- `ExportUsageEntry` — removes `provider_config_id`
- Users — removes `password_hash`, `encrypted_uek`, `uek_salt`
### `server/store/interfaces.go` (modified)
Added `Export ExportStore` to `Stores` struct. The `ExportStore`
interface provides 21 user-scoped read methods, 20 import methods
(returning `(imported, skipped, error)` counts), 14 team-scoped
read methods, and 4 GDPR methods.
### `server/store/{postgres,sqlite}/export.go` (new)
Full implementations for both dialects (~1400-1500 lines each).
Import uses `INSERT ON CONFLICT DO NOTHING` (PG) / `INSERT OR IGNORE`
(SQLite) for UUID-based dedup.
### `server/handlers/export_data.go` (new)
`DataExportHandler.ExportMyData``GET /api/v1/export/me`.
Streams ZIP directly to response using `zip.NewWriter(c.Writer)`.
Includes file blobs from object store.
### Migration 021
Indexes only — `idx_messages_channel_active`, `idx_channels_user_id`,
`idx_files_user_id`. PG version uses partial index
(`WHERE deleted_at IS NULL`).
---
## CS1 — User Import
### `server/handlers/import_data.go` (new)
`DataImportHandler.ImportMyData``POST /api/v1/import/me`.
Opens archive, validates manifest `format_version`, reads entities
in FK-dependency order, remaps `user_id` for cross-instance import.
Returns `{imported: {channels: N, ...}, skipped: {...}, errors: [...]}`.
Import order: Folders → Projects → Workspaces → Channels →
Participants/Models/Cursors → Messages → Notes → Note Links →
Memories → Project links → Workspace files → Files + blobs →
Settings → Persona groups.
---
## CS2 — GDPR Account Delete
### `server/handlers/gdpr.go` (new)
`GDPRHandler.DeleteMyAccount``DELETE /api/v1/me`.
Requires `{"confirm": "DELETE", "password": "..."}`. Verifies
password, prevents last-admin deletion, then:
1. Soft-deletes messages, archives channels
2. Hard-deletes: notes, memories, workspaces, files, projects, settings
3. Revokes tokens (refresh + WS tickets)
4. Anonymizes user: `deleted-user-{sha256(id)[:12]}`
5. Writes audit log entry
---
## CS3 — Admin Team Export/Import
`DataExportHandler.ExportTeam``GET /api/v1/admin/teams/:id/export`
`DataImportHandler.ImportTeam``POST /api/v1/admin/teams/:id/import`
Same archive format, scoped to team entities (channels, personas,
knowledge bases, workflows, groups, projects, resource grants).
Team import skips members whose `user_id` doesn't exist on target.
---
## CS4 — ChatGPT Import
### `server/export/chatgpt.go` (new)
Parses ChatGPT `conversations.json` format. Each conversation maps
to one channel (type=direct). The `mapping` DAG is walked depth-first
to produce ordered messages with `parent_id` and `sibling_index`.
Role mapping: `user`→user, `assistant`→assistant, `system`→system,
`tool`→tool. Content: `parts[]` joined with `\n`, non-string parts
skipped. Timestamps: `float64` epoch → `time.Time`.
`DataImportHandler.ImportChatGPT``POST /api/v1/import/chatgpt`.
Accepts multipart upload of `conversations.json` or ZIP containing it.
---
## CS5 — K8s Backup + ICD Tests
### `chart/templates/cronjob-backup.yaml` (new)
CronJob running `pg_dump | gzip` on schedule (default daily 02:00 UTC).
Optional S3 upload via AWS CLI. Local retention pruning. Gated on
`backup.enabled=true` and `database.driver=postgres`.
### `chart/templates/pvc-backup.yaml` (new)
Conditional PVC for local backup storage (when `backup.persistence.enabled`).
### `chart/values.yaml` (modified)
New `backup:` section with schedule, retention, S3 config, persistence,
and resource limits.
### ICD Tests
~30 new tests in `crud/portability.js` covering:
- Export download + archive validation (ZIP structure, manifest, entities)
- Sensitive field exclusion verification
- Import re-import (dedup validation)
- Invalid archive rejection
- ChatGPT import (valid, invalid JSON, empty array)
- ChatGPT channel/message verification
- GDPR delete flow (register → create data → delete → verify lockout)
- GDPR validation (missing confirm, wrong password)
- Admin team export/import
---
## New Routes
| Method | Path | Auth | Handler |
|--------|------|------|---------|
| GET | `/api/v1/export/me` | JWT | `DataExportHandler.ExportMyData` |
| POST | `/api/v1/import/me` | JWT | `DataImportHandler.ImportMyData` |
| POST | `/api/v1/import/chatgpt` | JWT | `DataImportHandler.ImportChatGPT` |
| DELETE | `/api/v1/me` | JWT | `GDPRHandler.DeleteMyAccount` |
| GET | `/api/v1/admin/teams/:id/export` | Admin | `DataExportHandler.ExportTeam` |
| POST | `/api/v1/admin/teams/:id/import` | Admin | `DataImportHandler.ImportTeam` |
---
## Verification
1. `cd server && go build ./...` — all changesets compile clean
2. `helm template ./chart --set backup.enabled=true` — CronJob renders
3. ICD test suite: ~615 tests (585 existing + ~30 portability)
4. Manual smoke: export → inspect ZIP → re-import → verify dedup
5. GDPR: delete → verify login fails → inspect anonymized DB record
6. Sensitive fields: no `password_hash`, `encrypted_uek`, `storage_key` in export

View File

@@ -0,0 +1,18 @@
-- ==========================================
-- Chat Switchboard — 021 Data Portability
-- ==========================================
-- v0.34.0: Indexes for data export and GDPR operations.
-- ==========================================
-- Partial index for batch message export (only non-deleted)
CREATE INDEX IF NOT EXISTS idx_messages_channel_active
ON messages (channel_id, created_at)
WHERE deleted_at IS NULL;
-- Index for user-scoped channel listing (export)
CREATE INDEX IF NOT EXISTS idx_channels_user_id
ON channels (user_id);
-- Index for file lookup by user (export)
CREATE INDEX IF NOT EXISTS idx_files_user_id
ON files (user_id);

View File

@@ -0,0 +1,15 @@
-- ==========================================
-- Chat Switchboard — 021 Data Portability
-- ==========================================
-- v0.34.0: Indexes for data export and GDPR operations.
-- SQLite version (no partial indexes).
-- ==========================================
CREATE INDEX IF NOT EXISTS idx_messages_channel_active
ON messages (channel_id, created_at);
CREATE INDEX IF NOT EXISTS idx_channels_user_id
ON channels (user_id);
CREATE INDEX IF NOT EXISTS idx_files_user_id
ON files (user_id);

244
server/export/chatgpt.go Normal file
View File

@@ -0,0 +1,244 @@
package export
// chatgpt.go — v0.34.0 CS4
//
// Parses ChatGPT conversation exports (conversations.json) and converts
// them to Switchboard channel/message models.
import (
"encoding/json"
"fmt"
"io"
"sort"
"time"
"chat-switchboard/models"
"chat-switchboard/store"
)
// ── ChatGPT format types ─────────────────────
// ChatGPTExport is the top-level conversations.json structure.
type ChatGPTExport []ChatGPTConversation
// ChatGPTConversation represents one conversation in the ChatGPT export.
type ChatGPTConversation struct {
Title string `json:"title"`
CreateTime float64 `json:"create_time"`
UpdateTime float64 `json:"update_time"`
Mapping map[string]ChatGPTMappingNode `json:"mapping"`
ConversationID string `json:"conversation_id"`
}
// ChatGPTMappingNode is a node in the conversation DAG.
type ChatGPTMappingNode struct {
ID string `json:"id"`
Parent *string `json:"parent"`
Children []string `json:"children"`
Message *ChatGPTMessage `json:"message"`
}
// ChatGPTMessage is a message within a mapping node.
type ChatGPTMessage struct {
ID string `json:"id"`
Author ChatGPTAuthor `json:"author"`
CreateTime *float64 `json:"create_time"`
Content ChatGPTContent `json:"content"`
Metadata json.RawMessage `json:"metadata"`
}
// ChatGPTAuthor identifies the message sender.
type ChatGPTAuthor struct {
Role string `json:"role"`
}
// ChatGPTContent holds the message content.
type ChatGPTContent struct {
ContentType string `json:"content_type"`
Parts []interface{} `json:"parts"`
}
// ── Parsing ──────────────────────────────────
// ParseChatGPTExport reads and parses a conversations.json file.
func ParseChatGPTExport(r io.Reader) (ChatGPTExport, error) {
var out ChatGPTExport
if err := json.NewDecoder(r).Decode(&out); err != nil {
return nil, fmt.Errorf("chatgpt: parse error: %w", err)
}
return out, nil
}
// ── Conversion ───────────────────────────────
// ChatGPTImportResult holds converted channels and messages.
type ChatGPTImportResult struct {
Channels []models.Channel
Messages []models.Message
}
// ConvertChatGPTExport converts ChatGPT conversations to Switchboard models.
func ConvertChatGPTExport(convs ChatGPTExport, userID string) *ChatGPTImportResult {
result := &ChatGPTImportResult{}
for _, conv := range convs {
ch, msgs := convertConversation(conv, userID)
if ch != nil {
result.Channels = append(result.Channels, *ch)
result.Messages = append(result.Messages, msgs...)
}
}
return result
}
func convertConversation(conv ChatGPTConversation, userID string) (*models.Channel, []models.Message) {
channelID := store.NewID()
title := conv.Title
if title == "" {
title = "Imported Conversation"
}
createdAt := floatToTime(conv.CreateTime)
updatedAt := floatToTime(conv.UpdateTime)
if updatedAt.IsZero() {
updatedAt = createdAt
}
ch := &models.Channel{
BaseModel: models.BaseModel{
ID: channelID,
CreatedAt: createdAt,
UpdatedAt: updatedAt,
},
UserID: userID,
Title: title,
Type: "direct",
}
// Walk the DAG depth-first to produce ordered messages
msgs := walkDAG(conv.Mapping, channelID, userID)
return ch, msgs
}
func walkDAG(mapping map[string]ChatGPTMappingNode, channelID, userID string) []models.Message {
// Find root nodes (no parent or parent not in mapping)
var roots []string
for id, node := range mapping {
if node.Parent == nil {
roots = append(roots, id)
} else if _, ok := mapping[*node.Parent]; !ok {
roots = append(roots, id)
}
}
var msgs []models.Message
idMap := make(map[string]string) // chatgpt node ID → switchboard message ID
var walk func(nodeID string, parentMsgID *string, siblingIdx int)
walk = func(nodeID string, parentMsgID *string, siblingIdx int) {
node, ok := mapping[nodeID]
if !ok {
return
}
if node.Message != nil && node.Message.Content.ContentType != "" {
content := extractParts(node.Message.Content.Parts)
if content != "" {
msgID := store.NewID()
idMap[nodeID] = msgID
role := mapRole(node.Message.Author.Role)
createdAt := time.Now().UTC()
if node.Message.CreateTime != nil {
createdAt = floatToTime(*node.Message.CreateTime)
}
msg := models.Message{
BaseModel: models.BaseModel{
ID: msgID,
CreatedAt: createdAt,
UpdatedAt: createdAt,
},
ChannelID: channelID,
Role: role,
Content: content,
ParentID: parentMsgID,
SiblingIndex: siblingIdx,
ParticipantType: "user",
ParticipantID: userID,
}
if role == "assistant" || role == "system" || role == "tool" {
msg.ParticipantType = role
msg.ParticipantID = ""
}
msgs = append(msgs, msg)
parentMsgID = &msgID
}
}
// Sort children for deterministic order
children := make([]string, len(node.Children))
copy(children, node.Children)
sort.Strings(children)
for i, childID := range children {
walk(childID, parentMsgID, i)
}
}
sort.Strings(roots)
for _, root := range roots {
walk(root, nil, 0)
}
return msgs
}
func mapRole(role string) string {
switch role {
case "user":
return "user"
case "assistant":
return "assistant"
case "system":
return "system"
case "tool":
return "tool"
default:
return "user"
}
}
func extractParts(parts []interface{}) string {
var texts []string
for _, part := range parts {
switch v := part.(type) {
case string:
if v != "" {
texts = append(texts, v)
}
}
}
if len(texts) == 0 {
return ""
}
result := texts[0]
for i := 1; i < len(texts); i++ {
result += "\n" + texts[i]
}
return result
}
func floatToTime(f float64) time.Time {
if f <= 0 {
return time.Time{}
}
sec := int64(f)
nsec := int64((f - float64(sec)) * 1e9)
return time.Unix(sec, nsec).UTC()
}

216
server/export/entities.go Normal file
View File

@@ -0,0 +1,216 @@
package export
// entities.go — v0.34.0 CS0
//
// Per-entity sanitization: strips sensitive, instance-specific, or
// non-portable fields before writing to the export archive. Each
// Sanitize* function returns a clean map ready for JSON marshalling.
import (
"chat-switchboard/models"
)
// ── User ─────────────────────────────────────
// SanitizeUser strips password_hash (json:"-" already), vault fields,
// and external_id. The User struct's json tags already exclude
// password_hash, so standard marshalling is safe. We nil out a few
// more fields for cross-instance portability.
func SanitizeUser(u *models.User) *models.User {
out := *u // shallow copy
out.PasswordHash = ""
out.ExternalID = nil
return &out
}
// ── Channel ──────────────────────────────────
// ExportChannel is a Channel without instance-specific provider binding.
type ExportChannel struct {
models.Channel
ProviderConfigID *string `json:"provider_config_id,omitempty"` // always nil in export
}
// SanitizeChannel strips provider_config_id.
func SanitizeChannel(ch *models.Channel) ExportChannel {
return ExportChannel{
Channel: *ch,
ProviderConfigID: nil,
}
}
// SanitizeChannels batch-sanitizes a slice.
func SanitizeChannels(chs []models.Channel) []ExportChannel {
out := make([]ExportChannel, len(chs))
for i := range chs {
chs[i].ProviderConfigID = nil
out[i] = SanitizeChannel(&chs[i])
}
return out
}
// ── Message ──────────────────────────────────
// ExportMessage is a Message without provider_config_id.
type ExportMessage struct {
models.Message
ProviderConfigID *string `json:"provider_config_id,omitempty"` // always nil
}
// SanitizeMessages strips provider_config_id from each message.
func SanitizeMessages(msgs []models.Message) []ExportMessage {
out := make([]ExportMessage, len(msgs))
for i := range msgs {
msgs[i].ProviderConfigID = nil
out[i] = ExportMessage{Message: msgs[i]}
}
return out
}
// ── Channel Model ────────────────────────────
// ExportChannelModel strips provider_config_id and persona_id.
type ExportChannelModel struct {
ID string `json:"id"`
ChannelID string `json:"channel_id"`
ModelID string `json:"model_id"`
Handle string `json:"handle,omitempty"`
DisplayName string `json:"display_name,omitempty"`
SystemPrompt string `json:"system_prompt,omitempty"`
IsDefault bool `json:"is_default"`
}
// SanitizeChannelModels strips instance-specific refs.
func SanitizeChannelModels(cms []models.ChannelModel) []ExportChannelModel {
out := make([]ExportChannelModel, len(cms))
for i, cm := range cms {
out[i] = ExportChannelModel{
ID: cm.ID,
ChannelID: cm.ChannelID,
ModelID: cm.ModelID,
Handle: cm.Handle,
DisplayName: cm.DisplayName,
SystemPrompt: cm.SystemPrompt,
IsDefault: cm.IsDefault,
}
}
return out
}
// ── User Model Settings ──────────────────────
// ExportUserModelSetting strips provider_config_id.
type ExportUserModelSetting struct {
models.UserModelSetting
ProviderConfigID *string `json:"provider_config_id,omitempty"` // always nil
}
// SanitizeUserModelSettings strips instance-specific provider binding.
func SanitizeUserModelSettings(ums []models.UserModelSetting) []ExportUserModelSetting {
out := make([]ExportUserModelSetting, len(ums))
for i := range ums {
ums[i].ProviderConfigID = nil
out[i] = ExportUserModelSetting{UserModelSetting: ums[i]}
}
return out
}
// ── Usage Entry ──────────────────────────────
// ExportUsageEntry strips provider_config_id and routing_decision.
type ExportUsageEntry struct {
ID string `json:"id"`
ChannelID *string `json:"channel_id,omitempty"`
UserID string `json:"user_id"`
ModelID string `json:"model_id"`
Role *string `json:"role,omitempty"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
CacheCreationTokens int `json:"cache_creation_tokens"`
CacheReadTokens int `json:"cache_read_tokens"`
CostInput *float64 `json:"cost_input,omitempty"`
CostOutput *float64 `json:"cost_output,omitempty"`
CreatedAt string `json:"created_at"`
}
// SanitizeUsageEntries strips instance-specific fields.
func SanitizeUsageEntries(ues []models.UsageEntry) []ExportUsageEntry {
out := make([]ExportUsageEntry, len(ues))
for i, ue := range ues {
out[i] = ExportUsageEntry{
ID: ue.ID,
ChannelID: ue.ChannelID,
UserID: ue.UserID,
ModelID: ue.ModelID,
Role: ue.Role,
PromptTokens: ue.PromptTokens,
CompletionTokens: ue.CompletionTokens,
CacheCreationTokens: ue.CacheCreationTokens,
CacheReadTokens: ue.CacheReadTokens,
CostInput: ue.CostInput,
CostOutput: ue.CostOutput,
CreatedAt: ue.CreatedAt.Format("2006-01-02T15:04:05Z07:00"),
}
}
return out
}
// ── Workspace ────────────────────────────────
// ExportWorkspace strips root_path (json:"-" already) and git credentials.
type ExportWorkspace struct {
models.Workspace
GitCredentialID *string `json:"git_credential_id,omitempty"` // always nil
}
// SanitizeWorkspaces strips git credential refs.
func SanitizeWorkspaces(ws []models.Workspace) []ExportWorkspace {
out := make([]ExportWorkspace, len(ws))
for i := range ws {
ws[i].GitCredentialID = nil
ws[i].GitLastSync = nil
out[i] = ExportWorkspace{Workspace: ws[i]}
}
return out
}
// ── Persona ──────────────────────────────────
// ExportPersona strips provider_config_id.
type ExportPersona struct {
models.Persona
ProviderConfigID *string `json:"provider_config_id,omitempty"` // always nil
}
// SanitizePersonas strips instance-specific provider binding.
func SanitizePersonas(ps []models.Persona) []ExportPersona {
out := make([]ExportPersona, len(ps))
for i := range ps {
ps[i].ProviderConfigID = nil
out[i] = ExportPersona{Persona: ps[i]}
}
return out
}
// ── Workflow ─────────────────────────────────
// SanitizeWorkflows strips webhook secrets.
func SanitizeWorkflows(wfs []models.Workflow) []models.Workflow {
out := make([]models.Workflow, len(wfs))
for i := range wfs {
out[i] = wfs[i]
out[i].WebhookSecret = ""
}
return out
}
// Note links use models.ExportNoteLink (includes source_note_id).
// See models/models.go for the type definition.
// Note, Memory, Folder, NotificationPreference,
// ChannelParticipant, ChannelCursor, Project, ProjectChannel,
// ProjectKB, ProjectNote, WorkspaceFile, PersonaGroup,
// PersonaGroupMember, KnowledgeBase, KBDocument, Group,
// GroupMember, ResourceGrant, Team, TeamMember, WorkflowVersion,
// WorkflowStage — these are exported as-is (no sensitive fields
// after query-level exclusion of embeddings).

211
server/export/format.go Normal file
View File

@@ -0,0 +1,211 @@
package export
// format.go — v0.34.0 CS0
//
// Defines the .switchboard archive format: a zip file containing a
// manifest.json and per-entity JSON files under data/. File blobs
// live under files/{id}/{filename}.
import (
"archive/zip"
"encoding/json"
"fmt"
"io"
"time"
)
// ── Constants ────────────────────────────────
const (
FormatVersion = 1
MaxExportArchiveSize = 500 * 1024 * 1024 // 500 MB total
MaxExportFileSize = 100 * 1024 * 1024 // 100 MB per file blob
MaxExportFiles = 10000
ExportContentType = "application/zip"
ExportExtension = ".switchboard"
)
// ── Manifest ─────────────────────────────────
// Manifest is the top-level metadata written to manifest.json.
type Manifest struct {
Version string `json:"version"` // server version, e.g. "0.34.0"
FormatVersion int `json:"format_version"` // always 1 for now
ExportType string `json:"export_type"` // "user", "team"
CreatedAt time.Time `json:"created_at"`
ExportedBy string `json:"exported_by"` // user ID
EntityCounts map[string]int `json:"entity_counts"`
Scope *ManifestScope `json:"scope,omitempty"`
}
// ManifestScope narrows the export to a user or team.
type ManifestScope struct {
UserID string `json:"user_id,omitempty"`
TeamID string `json:"team_id,omitempty"`
}
// ── ArchiveWriter ────────────────────────────
// ArchiveWriter wraps zip.Writer with size tracking and convenience
// methods for writing manifest and entity JSON files.
type ArchiveWriter struct {
zw *zip.Writer
written int64
maxBytes int64
}
// NewArchiveWriter creates an ArchiveWriter that streams to w.
func NewArchiveWriter(w io.Writer) *ArchiveWriter {
return &ArchiveWriter{
zw: zip.NewWriter(w),
maxBytes: MaxExportArchiveSize,
}
}
// WriteManifest serializes m as indented JSON into manifest.json.
func (aw *ArchiveWriter) WriteManifest(m *Manifest) error {
data, err := json.MarshalIndent(m, "", " ")
if err != nil {
return fmt.Errorf("export: marshal manifest: %w", err)
}
return aw.writeEntry("manifest.json", data)
}
// WriteEntityJSON serializes data as indented JSON into data/{name}.json.
func (aw *ArchiveWriter) WriteEntityJSON(name string, data interface{}) (int, error) {
buf, err := json.MarshalIndent(data, "", " ")
if err != nil {
return 0, fmt.Errorf("export: marshal %s: %w", name, err)
}
if err := aw.writeEntry("data/"+name+".json", buf); err != nil {
return 0, err
}
// Return entity count for manifest
switch v := data.(type) {
case []interface{}:
return len(v), nil
default:
// Use json.RawMessage length heuristic — caller tracks count
return 0, nil
}
}
// WriteFile streams a file blob into files/{zipPath}. Returns bytes
// written or an error if the file exceeds MaxExportFileSize.
func (aw *ArchiveWriter) WriteFile(zipPath string, r io.Reader) (int64, error) {
if aw.written+1 > aw.maxBytes {
return 0, fmt.Errorf("export: archive size limit exceeded")
}
f, err := aw.zw.Create("files/" + zipPath)
if err != nil {
return 0, fmt.Errorf("export: create zip entry %s: %w", zipPath, err)
}
lr := io.LimitReader(r, MaxExportFileSize+1)
n, err := io.Copy(f, lr)
if err != nil {
return n, fmt.Errorf("export: write file %s: %w", zipPath, err)
}
if n > MaxExportFileSize {
return n, fmt.Errorf("export: file %s exceeds %d bytes", zipPath, MaxExportFileSize)
}
aw.written += n
return n, nil
}
// Close finalizes the zip archive.
func (aw *ArchiveWriter) Close() error {
return aw.zw.Close()
}
// BytesWritten returns the approximate bytes written so far.
func (aw *ArchiveWriter) BytesWritten() int64 {
return aw.written
}
// ── ArchiveReader ────────────────────────────
// ArchiveReader wraps zip.ReadCloser with validation helpers.
type ArchiveReader struct {
zr *zip.ReadCloser
}
// OpenArchive opens and validates a .switchboard zip archive.
func OpenArchive(path string) (*ArchiveReader, error) {
zr, err := zip.OpenReader(path)
if err != nil {
return nil, fmt.Errorf("export: open archive: %w", err)
}
return &ArchiveReader{zr: zr}, nil
}
// ReadManifest reads and validates manifest.json from the archive.
func (ar *ArchiveReader) ReadManifest() (*Manifest, error) {
for _, f := range ar.zr.File {
if f.Name == "manifest.json" {
rc, err := f.Open()
if err != nil {
return nil, fmt.Errorf("export: open manifest: %w", err)
}
defer rc.Close()
var m Manifest
if err := json.NewDecoder(rc).Decode(&m); err != nil {
return nil, fmt.Errorf("export: decode manifest: %w", err)
}
if m.FormatVersion > FormatVersion {
return nil, fmt.Errorf("export: unsupported format version %d (max %d)", m.FormatVersion, FormatVersion)
}
return &m, nil
}
}
return nil, fmt.Errorf("export: manifest.json not found in archive")
}
// ReadEntityJSON reads data/{name}.json into target.
func (ar *ArchiveReader) ReadEntityJSON(name string, target interface{}) error {
path := "data/" + name + ".json"
for _, f := range ar.zr.File {
if f.Name == path {
rc, err := f.Open()
if err != nil {
return fmt.Errorf("export: open %s: %w", path, err)
}
defer rc.Close()
if err := json.NewDecoder(rc).Decode(target); err != nil {
return fmt.Errorf("export: decode %s: %w", path, err)
}
return nil
}
}
// Entity file not present — not an error (sparse archives are valid)
return nil
}
// FileEntries returns zip entries under files/.
func (ar *ArchiveReader) FileEntries() []*zip.File {
var out []*zip.File
for _, f := range ar.zr.File {
if len(f.Name) > 6 && f.Name[:6] == "files/" && !f.FileInfo().IsDir() {
out = append(out, f)
}
}
return out
}
// Close closes the underlying zip reader.
func (ar *ArchiveReader) Close() error {
return ar.zr.Close()
}
// writeEntry writes raw bytes as a zip entry, tracking total size.
func (aw *ArchiveWriter) writeEntry(name string, data []byte) error {
if aw.written+int64(len(data)) > aw.maxBytes {
return fmt.Errorf("export: archive size limit exceeded writing %s", name)
}
f, err := aw.zw.Create(name)
if err != nil {
return fmt.Errorf("export: create entry %s: %w", name, err)
}
n, err := f.Write(data)
aw.written += int64(n)
return err
}

View File

@@ -0,0 +1,565 @@
package handlers
// export_data.go — v0.34.0 CS0
//
// Data export endpoints: user data export (GDPR "download my data")
// and team data export (admin). Streams .switchboard zip archives
// directly to the HTTP response.
import (
"fmt"
"log/slog"
"net/http"
"time"
"github.com/gin-gonic/gin"
"chat-switchboard/export"
"chat-switchboard/models"
"chat-switchboard/storage"
"chat-switchboard/store"
)
// DataExportHandler serves data export endpoints.
type DataExportHandler struct {
stores store.Stores
objStore storage.ObjectStore
}
// NewDataExportHandler creates a new handler for data export operations.
func NewDataExportHandler(s store.Stores, obj storage.ObjectStore) *DataExportHandler {
return &DataExportHandler{stores: s, objStore: obj}
}
// ExportMyData streams the requesting user's data as a .switchboard zip.
// GET /api/v1/export/me
func (h *DataExportHandler) ExportMyData(c *gin.Context) {
ctx := c.Request.Context()
userID := c.GetString("user_id")
// ── Fetch user ──
user, err := h.stores.Users.GetByID(ctx, userID)
if err != nil || user == nil {
c.JSON(http.StatusNotFound, gin.H{"error": "user not found"})
return
}
// ── Fetch all user-scoped entities ──
channels, err := h.stores.Export.UserChannels(ctx, userID)
if err != nil {
slog.Error("export: fetch channels", "error", err, "user_id", userID)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export channels"})
return
}
channelIDs := make([]string, len(channels))
for i, ch := range channels {
channelIDs[i] = ch.ID
}
messages, err := h.stores.Export.UserMessages(ctx, channelIDs)
if err != nil {
slog.Error("export: fetch messages", "error", err, "user_id", userID)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export messages"})
return
}
participants, err := h.stores.Export.UserChannelParticipants(ctx, channelIDs)
if err != nil {
slog.Error("export: fetch participants", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export participants"})
return
}
channelModels, err := h.stores.Export.UserChannelModels(ctx, channelIDs)
if err != nil {
slog.Error("export: fetch channel models", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export channel models"})
return
}
cursors, err := h.stores.Export.UserChannelCursors(ctx, userID, channelIDs)
if err != nil {
slog.Error("export: fetch cursors", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export cursors"})
return
}
notes, err := h.stores.Export.UserNotes(ctx, userID)
if err != nil {
slog.Error("export: fetch notes", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export notes"})
return
}
noteIDs := make([]string, len(notes))
for i, n := range notes {
noteIDs[i] = n.ID
}
noteLinks, err := h.stores.Export.UserNoteLinks(ctx, noteIDs)
if err != nil {
slog.Error("export: fetch note links", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export note links"})
return
}
memories, err := h.stores.Export.UserMemories(ctx, userID)
if err != nil {
slog.Error("export: fetch memories", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export memories"})
return
}
projects, err := h.stores.Export.UserProjects(ctx, userID)
if err != nil {
slog.Error("export: fetch projects", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export projects"})
return
}
projectIDs := make([]string, len(projects))
for i, p := range projects {
projectIDs[i] = p.ID
}
projectChannels, err := h.stores.Export.UserProjectChannels(ctx, projectIDs)
if err != nil {
slog.Error("export: fetch project channels", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export project channels"})
return
}
projectKBs, err := h.stores.Export.UserProjectKBs(ctx, projectIDs)
if err != nil {
slog.Error("export: fetch project KBs", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export project KBs"})
return
}
projectNotes, err := h.stores.Export.UserProjectNotes(ctx, projectIDs)
if err != nil {
slog.Error("export: fetch project notes", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export project notes"})
return
}
workspaces, err := h.stores.Export.UserWorkspaces(ctx, userID)
if err != nil {
slog.Error("export: fetch workspaces", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export workspaces"})
return
}
workspaceIDs := make([]string, len(workspaces))
for i, w := range workspaces {
workspaceIDs[i] = w.ID
}
workspaceFiles, err := h.stores.Export.UserWorkspaceFiles(ctx, workspaceIDs)
if err != nil {
slog.Error("export: fetch workspace files", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export workspace files"})
return
}
files, err := h.stores.Export.UserFiles(ctx, userID)
if err != nil {
slog.Error("export: fetch files", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export files"})
return
}
folders, err := h.stores.Export.UserFolders(ctx, userID)
if err != nil {
slog.Error("export: fetch folders", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export folders"})
return
}
userModelSettings, err := h.stores.Export.UserModelSettings(ctx, userID)
if err != nil {
slog.Error("export: fetch user model settings", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export user settings"})
return
}
notifPrefs, err := h.stores.Export.UserNotifPrefs(ctx, userID)
if err != nil {
slog.Error("export: fetch notification prefs", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export notification prefs"})
return
}
usageEntries, err := h.stores.Export.UserUsageEntries(ctx, userID)
if err != nil {
slog.Error("export: fetch usage entries", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export usage entries"})
return
}
personaGroups, err := h.stores.Export.UserPersonaGroups(ctx, userID)
if err != nil {
slog.Error("export: fetch persona groups", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export persona groups"})
return
}
pgIDs := make([]string, len(personaGroups))
for i, pg := range personaGroups {
pgIDs[i] = pg.ID
}
personaGroupMembers, err := h.stores.Export.UserPersonaGroupMembers(ctx, pgIDs)
if err != nil {
slog.Error("export: fetch persona group members", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export persona group members"})
return
}
// ── Build manifest ──
sanitizedUser := export.SanitizeUser(user)
sanitizedChannels := export.SanitizeChannels(channels)
sanitizedMessages := export.SanitizeMessages(messages)
sanitizedChannelModels := export.SanitizeChannelModels(channelModels)
sanitizedUserModelSettings := export.SanitizeUserModelSettings(userModelSettings)
sanitizedUsageEntries := export.SanitizeUsageEntries(usageEntries)
sanitizedWorkspaces := export.SanitizeWorkspaces(workspaces)
counts := map[string]int{
"users": 1,
"channels": len(channels),
"messages": len(messages),
"channel_participants": len(participants),
"channel_models": len(channelModels),
"channel_cursors": len(cursors),
"notes": len(notes),
"note_links": len(noteLinks),
"memories": len(memories),
"projects": len(projects),
"project_channels": len(projectChannels),
"project_knowledge_bases": len(projectKBs),
"project_notes": len(projectNotes),
"workspaces": len(workspaces),
"workspace_files": len(workspaceFiles),
"files": len(files),
"folders": len(folders),
"user_settings": len(userModelSettings),
"notification_preferences": len(notifPrefs),
"usage_entries": len(usageEntries),
"persona_groups": len(personaGroups),
"persona_group_members": len(personaGroupMembers),
}
manifest := &export.Manifest{
Version: "0.34.0",
FormatVersion: export.FormatVersion,
ExportType: "user",
CreatedAt: time.Now().UTC(),
ExportedBy: userID,
EntityCounts: counts,
Scope: &export.ManifestScope{UserID: userID},
}
// ── Stream zip to response ──
filename := fmt.Sprintf("switchboard-export-%s%s", userID[:8], export.ExportExtension)
c.Header("Content-Type", export.ExportContentType)
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filename))
aw := export.NewArchiveWriter(c.Writer)
defer aw.Close()
if err := aw.WriteManifest(manifest); err != nil {
slog.Error("export: write manifest", "error", err)
return
}
// Write entity JSON files
writeEntity := func(name string, data interface{}) {
if _, err := aw.WriteEntityJSON(name, data); err != nil {
slog.Error("export: write entity", "name", name, "error", err)
}
}
writeEntity("users", []interface{}{sanitizedUser})
writeEntity("channels", sanitizedChannels)
writeEntity("messages", sanitizedMessages)
writeEntity("channel_participants", participants)
writeEntity("channel_models", sanitizedChannelModels)
writeEntity("channel_cursors", cursors)
writeEntity("notes", notes)
writeEntity("note_links", noteLinks)
writeEntity("memories", memories)
writeEntity("projects", projects)
writeEntity("project_channels", projectChannels)
writeEntity("project_knowledge_bases", projectKBs)
writeEntity("project_notes", projectNotes)
writeEntity("workspaces", sanitizedWorkspaces)
writeEntity("workspace_files", workspaceFiles)
writeEntity("folders", folders)
writeEntity("user_settings", sanitizedUserModelSettings)
writeEntity("notification_preferences", notifPrefs)
writeEntity("usage_entries", sanitizedUsageEntries)
writeEntity("persona_groups", personaGroups)
writeEntity("persona_group_members", personaGroupMembers)
// Write file metadata (without storage_key — already excluded by json:"-" tag)
writeEntity("files", files)
// ── Stream file blobs ──
var warnings []string
if h.objStore != nil {
fileCount := 0
for _, f := range files {
if fileCount >= export.MaxExportFiles {
warnings = append(warnings, "file blob limit reached, some files skipped")
break
}
if f.SizeBytes > export.MaxExportFileSize {
warnings = append(warnings, fmt.Sprintf("file %s too large (%d bytes), skipped", f.Filename, f.SizeBytes))
continue
}
rc, _, _, err := h.objStore.Get(ctx, f.StorageKey)
if err != nil {
warnings = append(warnings, fmt.Sprintf("file %s not found in storage, skipped", f.Filename))
continue
}
zipPath := f.ID + "/" + f.Filename
if _, err := aw.WriteFile(zipPath, rc); err != nil {
rc.Close()
warnings = append(warnings, fmt.Sprintf("file %s write error: %v, skipped", f.Filename, err))
continue
}
rc.Close()
fileCount++
}
}
if len(warnings) > 0 {
writeEntity("export_warnings", warnings)
}
// Audit log
h.stores.Audit.Log(ctx, &models.AuditEntry{
ActorID: &userID,
Action: "user.export",
ResourceType: "user",
ResourceID: userID,
Metadata: models.JSONMap{"entity_counts": counts},
})
}
// ExportTeam streams a team's data as a .switchboard zip.
// GET /api/v1/admin/teams/:id/export
func (h *DataExportHandler) ExportTeam(c *gin.Context) {
ctx := c.Request.Context()
userID := c.GetString("user_id")
teamID := c.Param("id")
// Fetch team to verify it exists
team, err := h.stores.Teams.GetByID(ctx, teamID)
if err != nil || team == nil {
c.JSON(http.StatusNotFound, gin.H{"error": "team not found"})
return
}
// Fetch all team-scoped entities
channels, err := h.stores.Export.TeamChannels(ctx, teamID)
if err != nil {
slog.Error("export: fetch team channels", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export team channels"})
return
}
channelIDs := make([]string, len(channels))
for i, ch := range channels {
channelIDs[i] = ch.ID
}
messages, err := h.stores.Export.UserMessages(ctx, channelIDs)
if err != nil {
slog.Error("export: fetch team messages", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export team messages"})
return
}
members, err := h.stores.Export.TeamMembers(ctx, teamID)
if err != nil {
slog.Error("export: fetch team members", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export team members"})
return
}
personas, err := h.stores.Export.TeamPersonas(ctx, teamID)
if err != nil {
slog.Error("export: fetch team personas", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export team personas"})
return
}
personaIDs := make([]string, len(personas))
for i, p := range personas {
personaIDs[i] = p.ID
}
personaKBs, err := h.stores.Export.TeamPersonaKBs(ctx, personaIDs)
if err != nil {
slog.Error("export: fetch persona KBs", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export persona KBs"})
return
}
kbs, err := h.stores.Export.TeamKnowledgeBases(ctx, teamID)
if err != nil {
slog.Error("export: fetch team KBs", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export team KBs"})
return
}
kbIDs := make([]string, len(kbs))
for i, kb := range kbs {
kbIDs[i] = kb.ID
}
kbDocs, err := h.stores.Export.TeamKBDocuments(ctx, kbIDs)
if err != nil {
slog.Error("export: fetch KB docs", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export KB documents"})
return
}
workflows, err := h.stores.Export.TeamWorkflows(ctx, teamID)
if err != nil {
slog.Error("export: fetch workflows", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export workflows"})
return
}
workflowIDs := make([]string, len(workflows))
for i, w := range workflows {
workflowIDs[i] = w.ID
}
workflowVersions, err := h.stores.Export.TeamWorkflowVersions(ctx, workflowIDs)
if err != nil {
slog.Error("export: fetch workflow versions", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export workflow versions"})
return
}
workflowStages, err := h.stores.Export.TeamWorkflowStages(ctx, workflowIDs)
if err != nil {
slog.Error("export: fetch workflow stages", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export workflow stages"})
return
}
groups, err := h.stores.Export.TeamGroups(ctx, teamID)
if err != nil {
slog.Error("export: fetch groups", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export groups"})
return
}
groupIDs := make([]string, len(groups))
for i, g := range groups {
groupIDs[i] = g.ID
}
groupMembers, err := h.stores.Export.TeamGroupMembers(ctx, groupIDs)
if err != nil {
slog.Error("export: fetch group members", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export group members"})
return
}
resourceGrants, err := h.stores.Export.TeamResourceGrants(ctx, teamID)
if err != nil {
slog.Error("export: fetch resource grants", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export resource grants"})
return
}
projects, err := h.stores.Export.TeamProjects(ctx, teamID)
if err != nil {
slog.Error("export: fetch team projects", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export team projects"})
return
}
// Sanitize
sanitizedChannels := export.SanitizeChannels(channels)
sanitizedMessages := export.SanitizeMessages(messages)
sanitizedPersonas := export.SanitizePersonas(personas)
sanitizedWorkflows := export.SanitizeWorkflows(workflows)
counts := map[string]int{
"team": 1,
"channels": len(channels),
"messages": len(messages),
"members": len(members),
"personas": len(personas),
"persona_kbs": len(personaKBs),
"knowledge_bases": len(kbs),
"kb_documents": len(kbDocs),
"workflows": len(workflows),
"workflow_versions": len(workflowVersions),
"workflow_stages": len(workflowStages),
"groups": len(groups),
"group_members": len(groupMembers),
"resource_grants": len(resourceGrants),
"projects": len(projects),
}
manifest := &export.Manifest{
Version: "0.34.0",
FormatVersion: export.FormatVersion,
ExportType: "team",
CreatedAt: time.Now().UTC(),
ExportedBy: userID,
EntityCounts: counts,
Scope: &export.ManifestScope{TeamID: teamID},
}
filename := fmt.Sprintf("switchboard-team-%s%s", teamID[:8], export.ExportExtension)
c.Header("Content-Type", export.ExportContentType)
c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filename))
aw := export.NewArchiveWriter(c.Writer)
defer aw.Close()
if err := aw.WriteManifest(manifest); err != nil {
slog.Error("export: write manifest", "error", err)
return
}
writeEntity := func(name string, data interface{}) {
if _, err := aw.WriteEntityJSON(name, data); err != nil {
slog.Error("export: write entity", "name", name, "error", err)
}
}
writeEntity("team", []interface{}{team})
writeEntity("channels", sanitizedChannels)
writeEntity("messages", sanitizedMessages)
writeEntity("members", members)
writeEntity("personas", sanitizedPersonas)
writeEntity("persona_knowledge_bases", personaKBs)
writeEntity("knowledge_bases", kbs)
writeEntity("kb_documents", kbDocs)
writeEntity("workflows", sanitizedWorkflows)
writeEntity("workflow_versions", workflowVersions)
writeEntity("workflow_stages", workflowStages)
writeEntity("groups", groups)
writeEntity("group_members", groupMembers)
writeEntity("resource_grants", resourceGrants)
writeEntity("projects", projects)
h.stores.Audit.Log(ctx, &models.AuditEntry{
ActorID: &userID,
Action: "team.export",
ResourceType: "team",
ResourceID: teamID,
Metadata: models.JSONMap{"entity_counts": counts},
})
}

130
server/handlers/gdpr.go Normal file
View File

@@ -0,0 +1,130 @@
package handlers
// gdpr.go — v0.34.0 CS2
//
// GDPR delete endpoint: DELETE /api/v1/me
// Allows a user to delete their own account and all associated data.
import (
"crypto/sha256"
"fmt"
"log/slog"
"net/http"
"github.com/gin-gonic/gin"
"golang.org/x/crypto/bcrypt"
"chat-switchboard/models"
"chat-switchboard/store"
)
// GDPRHandler serves account deletion endpoints.
type GDPRHandler struct {
stores store.Stores
}
// NewGDPRHandler creates a new handler for GDPR operations.
func NewGDPRHandler(s store.Stores) *GDPRHandler {
return &GDPRHandler{stores: s}
}
// deleteAccountRequest is the body for DELETE /api/v1/me.
type deleteAccountRequest struct {
Confirm string `json:"confirm"`
Password string `json:"password"`
}
// DeleteMyAccount permanently deletes the requesting user's account.
// DELETE /api/v1/me
func (h *GDPRHandler) DeleteMyAccount(c *gin.Context) {
ctx := c.Request.Context()
userID := c.GetString("user_id")
var req deleteAccountRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request body"})
return
}
// Require explicit confirmation
if req.Confirm != "DELETE" {
c.JSON(http.StatusBadRequest, gin.H{"error": "confirm field must be \"DELETE\""})
return
}
// Fetch user
user, err := h.stores.Users.GetByID(ctx, userID)
if err != nil || user == nil {
c.JSON(http.StatusNotFound, gin.H{"error": "user not found"})
return
}
// Verify password
if err := bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(req.Password)); err != nil {
c.JSON(http.StatusUnauthorized, gin.H{"error": "invalid password"})
return
}
// Prevent deleting last admin
if user.Role == "admin" {
adminCount, err := h.stores.Export.CountActiveAdmins(ctx)
if err != nil {
slog.Error("gdpr: count admins", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to check admin count"})
return
}
if adminCount <= 1 {
c.JSON(http.StatusForbidden, gin.H{"error": "cannot delete the last admin account"})
return
}
}
// Generate anonymized hash from user ID
hash := sha256.Sum256([]byte(userID))
anonHash := fmt.Sprintf("%x", hash[:6]) // 12 hex chars
// Step 1: Soft-delete/hard-delete user data
counts, err := h.stores.Export.SoftDeleteUserData(ctx, userID)
if err != nil {
slog.Error("gdpr: delete user data", "error", err, "user_id", userID)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to delete user data"})
return
}
// Step 2: Revoke tokens
if err := h.stores.Export.DeleteUserTokens(ctx, userID); err != nil {
slog.Error("gdpr: delete tokens", "error", err, "user_id", userID)
// Continue — tokens will expire naturally
}
// Step 3: Delete personal provider configs
if deleted, err := h.stores.Providers.DeletePersonalByOwner(ctx, userID); err != nil {
slog.Error("gdpr: delete provider configs", "error", err)
} else {
counts["provider_configs"] = deleted
}
// Step 4: Anonymize user record
if err := h.stores.Export.AnonymizeUser(ctx, userID, anonHash); err != nil {
slog.Error("gdpr: anonymize user", "error", err, "user_id", userID)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to anonymize user"})
return
}
// Audit log (before the user record is fully anonymized)
anonUsername := "deleted-user-" + anonHash
h.stores.Audit.Log(ctx, &models.AuditEntry{
ActorID: &userID,
Action: "user.gdpr_delete",
ResourceType: "user",
ResourceID: userID,
Metadata: models.JSONMap{"anonymized_as": anonUsername, "deleted_counts": counts},
})
slog.Info("gdpr: account deleted", "user_id", userID, "anonymized_as", anonUsername)
c.JSON(http.StatusOK, gin.H{
"message": "account deleted",
"anonymized_as": anonUsername,
})
}

View File

@@ -0,0 +1,733 @@
package handlers
// import_data.go — v0.34.0 CS1
//
// Data import endpoint: user data import from .switchboard archives.
// Reads the archive, validates manifest, and imports entities in
// FK-dependency order with UUID dedup (skip if ID exists).
import (
"fmt"
"io"
"log/slog"
"net/http"
"os"
"path/filepath"
"strings"
"github.com/gin-gonic/gin"
"chat-switchboard/export"
"chat-switchboard/models"
"chat-switchboard/storage"
"chat-switchboard/store"
)
// DataImportHandler serves data import endpoints.
type DataImportHandler struct {
stores store.Stores
objStore storage.ObjectStore
}
// NewDataImportHandler creates a new handler for data import operations.
func NewDataImportHandler(s store.Stores, obj storage.ObjectStore) *DataImportHandler {
return &DataImportHandler{stores: s, objStore: obj}
}
// ImportMyData imports user data from a .switchboard zip archive.
// POST /api/v1/import/me
func (h *DataImportHandler) ImportMyData(c *gin.Context) {
ctx := c.Request.Context()
userID := c.GetString("user_id")
// ── Receive upload ──
file, header, err := c.Request.FormFile("file")
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "missing file field"})
return
}
defer file.Close()
// Validate extension
if !strings.HasSuffix(header.Filename, export.ExportExtension) && !strings.HasSuffix(header.Filename, ".zip") {
c.JSON(http.StatusBadRequest, gin.H{"error": "file must be a .switchboard or .zip archive"})
return
}
// Size check: use MaxExportArchiveSize
if header.Size > export.MaxExportArchiveSize {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("archive too large (max %d MB)", export.MaxExportArchiveSize/(1024*1024)),
})
return
}
// Save to temp file (zip.OpenReader needs a file path)
tmp, err := os.CreateTemp("", "switchboard-import-*.zip")
if err != nil {
slog.Error("import: create temp file", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create temp file"})
return
}
defer os.Remove(tmp.Name())
defer tmp.Close()
if _, err := io.Copy(tmp, file); err != nil {
slog.Error("import: save temp file", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to save upload"})
return
}
tmp.Close()
// ── Open archive ──
ar, err := export.OpenArchive(tmp.Name())
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid archive: " + err.Error()})
return
}
defer ar.Close()
manifest, err := ar.ReadManifest()
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid manifest: " + err.Error()})
return
}
if manifest.FormatVersion > export.FormatVersion {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("unsupported format version %d (max %d)", manifest.FormatVersion, export.FormatVersion),
})
return
}
// ── Read entities from archive ──
imported := make(map[string]int)
skipped := make(map[string]int)
var errors []string
importEntity := func(name string, fn func() (int, int, error)) {
imp, skip, err := fn()
if err != nil {
errors = append(errors, fmt.Sprintf("%s: %v", name, err))
slog.Error("import: entity error", "entity", name, "error", err)
}
if imp > 0 {
imported[name] = imp
}
if skip > 0 {
skipped[name] = skip
}
}
// Helper to remap user_id on entities for cross-instance import
remapUserID := manifest.Scope != nil && manifest.Scope.UserID != "" && manifest.Scope.UserID != userID
sourceUserID := ""
if manifest.Scope != nil {
sourceUserID = manifest.Scope.UserID
}
// 1. Folders
var folders []models.Folder
if err := ar.ReadEntityJSON("folders", &folders); err == nil && len(folders) > 0 {
if remapUserID {
for i := range folders {
if folders[i].UserID == sourceUserID {
folders[i].UserID = userID
}
}
}
importEntity("folders", func() (int, int, error) {
return h.stores.Export.ImportFolders(ctx, folders)
})
}
// 2. Projects
var projects []models.Project
if err := ar.ReadEntityJSON("projects", &projects); err == nil && len(projects) > 0 {
if remapUserID {
for i := range projects {
if projects[i].OwnerID == sourceUserID {
projects[i].OwnerID = userID
}
}
}
importEntity("projects", func() (int, int, error) {
return h.stores.Export.ImportProjects(ctx, projects)
})
}
// 3. Workspaces
var workspaces []models.Workspace
if err := ar.ReadEntityJSON("workspaces", &workspaces); err == nil && len(workspaces) > 0 {
if remapUserID {
for i := range workspaces {
if workspaces[i].OwnerID == sourceUserID {
workspaces[i].OwnerID = userID
}
}
}
importEntity("workspaces", func() (int, int, error) {
return h.stores.Export.ImportWorkspaces(ctx, workspaces)
})
}
// 4. Channels
var channels []models.Channel
if err := ar.ReadEntityJSON("channels", &channels); err == nil && len(channels) > 0 {
if remapUserID {
for i := range channels {
if channels[i].UserID == sourceUserID {
channels[i].UserID = userID
}
}
}
importEntity("channels", func() (int, int, error) {
return h.stores.Export.ImportChannels(ctx, channels)
})
}
// 5. Channel participants
var participants []models.ChannelParticipant
if err := ar.ReadEntityJSON("channel_participants", &participants); err == nil && len(participants) > 0 {
if remapUserID {
for i := range participants {
if participants[i].ParticipantType == "user" && participants[i].ParticipantID == sourceUserID {
participants[i].ParticipantID = userID
}
}
}
importEntity("channel_participants", func() (int, int, error) {
return h.stores.Export.ImportChannelParticipants(ctx, participants)
})
}
// 5b. Channel models
var channelModels []models.ChannelModel
if err := ar.ReadEntityJSON("channel_models", &channelModels); err == nil && len(channelModels) > 0 {
importEntity("channel_models", func() (int, int, error) {
return h.stores.Export.ImportChannelModels(ctx, channelModels)
})
}
// 5c. Channel cursors
var cursors []models.ChannelCursor
if err := ar.ReadEntityJSON("channel_cursors", &cursors); err == nil && len(cursors) > 0 {
if remapUserID {
for i := range cursors {
if cursors[i].UserID == sourceUserID {
cursors[i].UserID = userID
}
}
}
importEntity("channel_cursors", func() (int, int, error) {
return h.stores.Export.ImportChannelCursors(ctx, cursors)
})
}
// 6. Messages (ordered by created_at ASC for parent_id integrity)
var messages []models.Message
if err := ar.ReadEntityJSON("messages", &messages); err == nil && len(messages) > 0 {
if remapUserID {
for i := range messages {
if messages[i].ParticipantType == "user" && messages[i].ParticipantID == sourceUserID {
messages[i].ParticipantID = userID
}
}
}
importEntity("messages", func() (int, int, error) {
return h.stores.Export.ImportMessages(ctx, messages)
})
}
// 7. Notes
var notes []models.Note
if err := ar.ReadEntityJSON("notes", &notes); err == nil && len(notes) > 0 {
if remapUserID {
for i := range notes {
if notes[i].UserID == sourceUserID {
notes[i].UserID = userID
}
}
}
importEntity("notes", func() (int, int, error) {
return h.stores.Export.ImportNotes(ctx, notes)
})
}
// 8. Note links
var noteLinks []models.ExportNoteLink
if err := ar.ReadEntityJSON("note_links", &noteLinks); err == nil && len(noteLinks) > 0 {
importEntity("note_links", func() (int, int, error) {
return h.stores.Export.ImportNoteLinks(ctx, noteLinks)
})
}
// 9. Memories
var memories []models.Memory
if err := ar.ReadEntityJSON("memories", &memories); err == nil && len(memories) > 0 {
if remapUserID {
for i := range memories {
if memories[i].OwnerID == sourceUserID {
memories[i].OwnerID = userID
}
}
}
importEntity("memories", func() (int, int, error) {
return h.stores.Export.ImportMemories(ctx, memories)
})
}
// 10. Project channels
var projectChannels []models.ProjectChannel
if err := ar.ReadEntityJSON("project_channels", &projectChannels); err == nil && len(projectChannels) > 0 {
importEntity("project_channels", func() (int, int, error) {
return h.stores.Export.ImportProjectChannels(ctx, projectChannels)
})
}
// 10b. Project KBs
var projectKBs []models.ProjectKB
if err := ar.ReadEntityJSON("project_knowledge_bases", &projectKBs); err == nil && len(projectKBs) > 0 {
importEntity("project_knowledge_bases", func() (int, int, error) {
return h.stores.Export.ImportProjectKBs(ctx, projectKBs)
})
}
// 10c. Project notes
var projectNotes []models.ProjectNote
if err := ar.ReadEntityJSON("project_notes", &projectNotes); err == nil && len(projectNotes) > 0 {
importEntity("project_notes", func() (int, int, error) {
return h.stores.Export.ImportProjectNotes(ctx, projectNotes)
})
}
// 11. Workspace files
var workspaceFiles []models.WorkspaceFile
if err := ar.ReadEntityJSON("workspace_files", &workspaceFiles); err == nil && len(workspaceFiles) > 0 {
importEntity("workspace_files", func() (int, int, error) {
return h.stores.Export.ImportWorkspaceFiles(ctx, workspaceFiles)
})
}
// 12. Files (metadata)
var files []models.File
if err := ar.ReadEntityJSON("files", &files); err == nil && len(files) > 0 {
if remapUserID {
for i := range files {
if files[i].UserID == sourceUserID {
files[i].UserID = userID
}
}
}
// Generate storage keys for imported files
for i := range files {
if files[i].StorageKey == "" && files[i].ChannelID != "" {
files[i].StorageKey = fmt.Sprintf("files/%s/%s_%s", files[i].ChannelID, files[i].ID, files[i].Filename)
}
}
importEntity("files", func() (int, int, error) {
return h.stores.Export.ImportFiles(ctx, files)
})
}
// 12b. File blobs from archive
if h.objStore != nil {
fileBlobs := ar.FileEntries()
blobCount := 0
for _, entry := range fileBlobs {
if blobCount >= export.MaxExportFiles {
errors = append(errors, "file blob limit reached, some files skipped")
break
}
// Extract file ID from path: files/{fileID}/{filename}
parts := strings.SplitN(strings.TrimPrefix(entry.Name, "files/"), "/", 2)
if len(parts) != 2 {
continue
}
fileID := parts[0]
filename := parts[1]
// Find matching file record for storage key
storageKey := ""
var contentType string
for _, f := range files {
if f.ID == fileID {
storageKey = f.StorageKey
contentType = f.ContentType
break
}
}
if storageKey == "" {
storageKey = fmt.Sprintf("files/imported/%s_%s", fileID, filename)
}
if contentType == "" {
contentType = "application/octet-stream"
}
rc, err := entry.Open()
if err != nil {
errors = append(errors, fmt.Sprintf("file %s: open error", filepath.Base(entry.Name)))
continue
}
if err := h.objStore.Put(ctx, storageKey, rc, int64(entry.UncompressedSize64), contentType); err != nil {
rc.Close()
errors = append(errors, fmt.Sprintf("file %s: upload error", filepath.Base(entry.Name)))
continue
}
rc.Close()
blobCount++
}
if blobCount > 0 {
imported["file_blobs"] = blobCount
}
}
// 13. Settings/Prefs
var userModelSettings []models.UserModelSetting
if err := ar.ReadEntityJSON("user_settings", &userModelSettings); err == nil && len(userModelSettings) > 0 {
if remapUserID {
for i := range userModelSettings {
if userModelSettings[i].UserID == sourceUserID {
userModelSettings[i].UserID = userID
}
}
}
importEntity("user_settings", func() (int, int, error) {
return h.stores.Export.ImportUserModelSettings(ctx, userModelSettings)
})
}
var notifPrefs []models.NotificationPreference
if err := ar.ReadEntityJSON("notification_preferences", &notifPrefs); err == nil && len(notifPrefs) > 0 {
if remapUserID {
for i := range notifPrefs {
if notifPrefs[i].UserID == sourceUserID {
notifPrefs[i].UserID = userID
}
}
}
importEntity("notification_preferences", func() (int, int, error) {
return h.stores.Export.ImportNotifPrefs(ctx, notifPrefs)
})
}
// 14. Persona groups
var personaGroups []models.PersonaGroup
if err := ar.ReadEntityJSON("persona_groups", &personaGroups); err == nil && len(personaGroups) > 0 {
if remapUserID {
for i := range personaGroups {
if personaGroups[i].OwnerID == sourceUserID {
personaGroups[i].OwnerID = userID
}
}
}
importEntity("persona_groups", func() (int, int, error) {
return h.stores.Export.ImportPersonaGroups(ctx, personaGroups)
})
}
var personaGroupMembers []models.PersonaGroupMember
if err := ar.ReadEntityJSON("persona_group_members", &personaGroupMembers); err == nil && len(personaGroupMembers) > 0 {
importEntity("persona_group_members", func() (int, int, error) {
return h.stores.Export.ImportPersonaGroupMembers(ctx, personaGroupMembers)
})
}
// ── Audit log ──
h.stores.Audit.Log(ctx, &models.AuditEntry{
ActorID: &userID,
Action: "user.import",
ResourceType: "user",
ResourceID: userID,
Metadata: models.JSONMap{"imported": imported, "skipped": skipped, "errors": errors},
})
c.JSON(http.StatusOK, gin.H{
"imported": imported,
"skipped": skipped,
"errors": errors,
})
}
// ImportTeam imports team data from a .switchboard archive.
// POST /api/v1/admin/teams/:id/import
func (h *DataImportHandler) ImportTeam(c *gin.Context) {
ctx := c.Request.Context()
userID := c.GetString("user_id")
teamID := c.Param("id")
// Verify team exists
team, err := h.stores.Teams.GetByID(ctx, teamID)
if err != nil || team == nil {
c.JSON(http.StatusNotFound, gin.H{"error": "team not found"})
return
}
// Receive upload
file, header, err := c.Request.FormFile("file")
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "missing file field"})
return
}
defer file.Close()
if header.Size > export.MaxExportArchiveSize {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("archive too large (max %d MB)", export.MaxExportArchiveSize/(1024*1024)),
})
return
}
tmp, err := os.CreateTemp("", "switchboard-team-import-*.zip")
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create temp file"})
return
}
defer os.Remove(tmp.Name())
defer tmp.Close()
if _, err := io.Copy(tmp, file); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to save upload"})
return
}
tmp.Close()
ar, err := export.OpenArchive(tmp.Name())
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid archive: " + err.Error()})
return
}
defer ar.Close()
manifest, err := ar.ReadManifest()
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid manifest: " + err.Error()})
return
}
if manifest.FormatVersion > export.FormatVersion {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("unsupported format version %d", manifest.FormatVersion),
})
return
}
imported := make(map[string]int)
skipped := make(map[string]int)
var errors []string
importEntity := func(name string, fn func() (int, int, error)) {
imp, skip, err := fn()
if err != nil {
errors = append(errors, fmt.Sprintf("%s: %v", name, err))
slog.Error("import: team entity error", "entity", name, "error", err)
}
if imp > 0 {
imported[name] = imp
}
if skip > 0 {
skipped[name] = skip
}
}
// Remap team_id on entities to target team
sourceTeamID := ""
if manifest.Scope != nil {
sourceTeamID = manifest.Scope.TeamID
}
remapTeam := sourceTeamID != "" && sourceTeamID != teamID
// Channels (remap team_id)
var channels []models.Channel
if err := ar.ReadEntityJSON("channels", &channels); err == nil && len(channels) > 0 {
if remapTeam {
for i := range channels {
if channels[i].TeamID != nil && *channels[i].TeamID == sourceTeamID {
channels[i].TeamID = &teamID
}
}
}
importEntity("channels", func() (int, int, error) {
return h.stores.Export.ImportChannels(ctx, channels)
})
}
// Messages
var messages []models.Message
if err := ar.ReadEntityJSON("messages", &messages); err == nil && len(messages) > 0 {
importEntity("messages", func() (int, int, error) {
return h.stores.Export.ImportMessages(ctx, messages)
})
}
// Projects (remap team_id)
var projects []models.Project
if err := ar.ReadEntityJSON("projects", &projects); err == nil && len(projects) > 0 {
if remapTeam {
for i := range projects {
if projects[i].TeamID != nil && *projects[i].TeamID == sourceTeamID {
projects[i].TeamID = &teamID
}
}
}
importEntity("projects", func() (int, int, error) {
return h.stores.Export.ImportProjects(ctx, projects)
})
}
// Persona groups (remap team_id)
var personaGroups []models.PersonaGroup
if err := ar.ReadEntityJSON("persona_groups", &personaGroups); err == nil && len(personaGroups) > 0 {
if remapTeam {
for i := range personaGroups {
if personaGroups[i].TeamID != nil && *personaGroups[i].TeamID == sourceTeamID {
personaGroups[i].TeamID = &teamID
}
}
}
importEntity("persona_groups", func() (int, int, error) {
return h.stores.Export.ImportPersonaGroups(ctx, personaGroups)
})
}
var personaGroupMembers []models.PersonaGroupMember
if err := ar.ReadEntityJSON("persona_group_members", &personaGroupMembers); err == nil && len(personaGroupMembers) > 0 {
importEntity("persona_group_members", func() (int, int, error) {
return h.stores.Export.ImportPersonaGroupMembers(ctx, personaGroupMembers)
})
}
// Audit log
h.stores.Audit.Log(ctx, &models.AuditEntry{
ActorID: &userID,
Action: "team.import",
ResourceType: "team",
ResourceID: teamID,
Metadata: models.JSONMap{"imported": imported, "skipped": skipped, "errors": errors},
})
c.JSON(http.StatusOK, gin.H{
"imported": imported,
"skipped": skipped,
"errors": errors,
})
}
// ImportChatGPT imports conversations from a ChatGPT export.
// POST /api/v1/import/chatgpt
func (h *DataImportHandler) ImportChatGPT(c *gin.Context) {
ctx := c.Request.Context()
userID := c.GetString("user_id")
file, header, err := c.Request.FormFile("file")
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "missing file field"})
return
}
defer file.Close()
if header.Size > export.MaxExportArchiveSize {
c.JSON(http.StatusBadRequest, gin.H{
"error": fmt.Sprintf("file too large (max %d MB)", export.MaxExportArchiveSize/(1024*1024)),
})
return
}
// Detect if it's a zip or raw JSON
var convs export.ChatGPTExport
if strings.HasSuffix(header.Filename, ".zip") {
// Save to temp and extract conversations.json
tmp, err := os.CreateTemp("", "chatgpt-import-*.zip")
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create temp file"})
return
}
defer os.Remove(tmp.Name())
defer tmp.Close()
if _, err := io.Copy(tmp, file); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to save upload"})
return
}
tmp.Close()
ar, err := export.OpenArchive(tmp.Name())
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid zip archive"})
return
}
defer ar.Close()
// Try to find conversations.json in the zip
if err := ar.ReadEntityJSON("conversations", &convs); err != nil || len(convs) == 0 {
// Try searching all files for conversations.json
for _, f := range ar.FileEntries() {
if strings.HasSuffix(f.Name, "conversations.json") {
rc, err := f.Open()
if err != nil {
continue
}
convs, err = export.ParseChatGPTExport(rc)
rc.Close()
if err == nil {
break
}
}
}
}
} else {
// Raw JSON file
convs, err = export.ParseChatGPTExport(file)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "failed to parse conversations.json: " + err.Error()})
return
}
}
if len(convs) == 0 {
c.JSON(http.StatusBadRequest, gin.H{"error": "no conversations found in upload"})
return
}
// Convert to Switchboard models
result := export.ConvertChatGPTExport(convs, userID)
// Import channels and messages
chImported, chSkipped, err := h.stores.Export.ImportChannels(ctx, result.Channels)
if err != nil {
slog.Error("chatgpt import: channels", "error", err)
}
msgImported, msgSkipped, err := h.stores.Export.ImportMessages(ctx, result.Messages)
if err != nil {
slog.Error("chatgpt import: messages", "error", err)
}
imported := map[string]int{
"channels": chImported,
"messages": msgImported,
}
skipped := map[string]int{
"channels": chSkipped,
"messages": msgSkipped,
}
h.stores.Audit.Log(ctx, &models.AuditEntry{
ActorID: &userID,
Action: "user.import_chatgpt",
ResourceType: "user",
ResourceID: userID,
Metadata: models.JSONMap{
"conversations": len(convs),
"imported": imported,
"skipped": skipped,
},
})
c.JSON(http.StatusOK, gin.H{
"conversations": len(convs),
"imported": imported,
"skipped": skipped,
})
}

View File

@@ -945,6 +945,15 @@ func main() {
exportH := handlers.NewExportHandler()
protected.POST("/export", exportH.Convert)
// Data portability (v0.34.0) — user data export/import
dataExport := handlers.NewDataExportHandler(stores, objStore)
protected.GET("/export/me", dataExport.ExportMyData)
dataImport := handlers.NewDataImportHandler(stores, objStore)
protected.POST("/import/me", dataImport.ImportMyData)
protected.POST("/import/chatgpt", dataImport.ImportChatGPT)
gdprH := handlers.NewGDPRHandler(stores)
protected.DELETE("/me", gdprH.DeleteMyAccount)
// Hook: clean up storage files when channels are deleted
handlers.SetChannelDeleteHook(fileH.CleanupChannelStorage)
@@ -1153,6 +1162,12 @@ func main() {
admin.PUT("/teams/:id/members/:memberId", teamAdm.UpdateMember)
admin.DELETE("/teams/:id/members/:memberId", teamAdm.RemoveMember)
// Team data export/import (v0.34.0)
teamExport := handlers.NewDataExportHandler(stores, objStore)
teamImport := handlers.NewDataImportHandler(stores, objStore)
admin.GET("/teams/:id/export", teamExport.ExportTeam)
admin.POST("/teams/:id/import", teamImport.ImportTeam)
// Admin broadcast (v0.28.6)
adminNotifH := handlers.NewNotificationHandler(stores, hub)
admin.POST("/notifications/broadcast", adminNotifH.Broadcast)

View File

@@ -553,6 +553,15 @@ type NoteLink struct {
IsTransclusion bool `json:"is_transclusion"`
}
// ExportNoteLink is a note_link with source_note_id included (for export).
type ExportNoteLink struct {
SourceNoteID string `json:"source_note_id" db:"source_note_id"`
TargetNoteID *string `json:"target_note_id,omitempty" db:"target_note_id"`
TargetTitle string `json:"target_title" db:"target_title"`
DisplayText string `json:"display_text,omitempty" db:"display_text"`
IsTransclusion bool `json:"is_transclusion" db:"is_transclusion"`
}
// NoteLinkResult represents a backlink — a note that links to a given note.
type NoteLinkResult struct {
SourceNoteID string `json:"id"`

View File

@@ -64,6 +64,7 @@ type Stores struct {
ExtData ExtDataStore // v0.29.2: Extension namespaced table catalog
Tickets TicketStore // v0.32.0: WS auth tickets (PG-backed for cross-pod)
RateLimits RateLimitStore // v0.32.0: Distributed rate limiting
Export ExportStore // v0.34.0: Data portability batch reads + GDPR ops
}
// =========================================
@@ -1193,6 +1194,162 @@ type PresenceStore interface {
GetStatuses(ctx context.Context, userIDs []string, threshold time.Time) (map[string]string, error)
}
// =========================================
// EXPORT STORE (v0.34.0)
// =========================================
// ExportStore provides batch-read queries for data export, import,
// and GDPR operations. These methods are optimized for bulk retrieval
// without N+1 queries. Embeddings are excluded at query level.
type ExportStore interface {
// ── User-scoped export ──
// UserChannels returns all channels owned by userID.
UserChannels(ctx context.Context, userID string) ([]models.Channel, error)
// UserMessages returns all non-deleted messages in the given channels, ordered by created_at ASC.
UserMessages(ctx context.Context, channelIDs []string) ([]models.Message, error)
// UserChannelParticipants returns participants for the given channels.
UserChannelParticipants(ctx context.Context, channelIDs []string) ([]models.ChannelParticipant, error)
// UserChannelModels returns channel_models for the given channels.
UserChannelModels(ctx context.Context, channelIDs []string) ([]models.ChannelModel, error)
// UserChannelCursors returns cursors for the user in the given channels.
UserChannelCursors(ctx context.Context, userID string, channelIDs []string) ([]models.ChannelCursor, error)
// UserNotes returns all notes owned by userID.
UserNotes(ctx context.Context, userID string) ([]models.Note, error)
// UserNoteLinks returns note_links with source_note_id for the given note IDs.
UserNoteLinks(ctx context.Context, noteIDs []string) ([]models.ExportNoteLink, error)
// UserMemories returns active memories for userID (scope=user).
UserMemories(ctx context.Context, userID string) ([]models.Memory, error)
// UserProjects returns personal-scope projects owned by userID.
UserProjects(ctx context.Context, userID string) ([]models.Project, error)
// UserProjectChannels returns project_channels for the given projects.
UserProjectChannels(ctx context.Context, projectIDs []string) ([]models.ProjectChannel, error)
// UserProjectKBs returns project_knowledge_bases for the given projects.
UserProjectKBs(ctx context.Context, projectIDs []string) ([]models.ProjectKB, error)
// UserProjectNotes returns project_notes for the given projects.
UserProjectNotes(ctx context.Context, projectIDs []string) ([]models.ProjectNote, error)
// UserWorkspaces returns workspaces owned by user (owner_type=user).
UserWorkspaces(ctx context.Context, userID string) ([]models.Workspace, error)
// UserWorkspaceFiles returns file index for the given workspaces.
UserWorkspaceFiles(ctx context.Context, workspaceIDs []string) ([]models.WorkspaceFile, error)
// UserFiles returns file metadata for files uploaded by userID.
UserFiles(ctx context.Context, userID string) ([]models.File, error)
// UserFolders returns all folders for userID.
UserFolders(ctx context.Context, userID string) ([]models.Folder, error)
// UserModelSettings returns user_model_settings for userID.
UserModelSettings(ctx context.Context, userID string) ([]models.UserModelSetting, error)
// UserNotifPrefs returns notification preferences for userID.
UserNotifPrefs(ctx context.Context, userID string) ([]models.NotificationPreference, error)
// UserUsageEntries returns usage entries for userID (last 90 days).
UserUsageEntries(ctx context.Context, userID string) ([]models.UsageEntry, error)
// UserPersonaGroups returns persona groups owned by userID.
UserPersonaGroups(ctx context.Context, userID string) ([]models.PersonaGroup, error)
// UserPersonaGroupMembers returns members for the given groups.
UserPersonaGroupMembers(ctx context.Context, groupIDs []string) ([]models.PersonaGroupMember, error)
// ── Team-scoped export ──
// TeamChannels returns all channels belonging to teamID.
TeamChannels(ctx context.Context, teamID string) ([]models.Channel, error)
// TeamMembers returns team membership records.
TeamMembers(ctx context.Context, teamID string) ([]models.TeamMember, error)
// TeamPersonas returns team-scoped personas.
TeamPersonas(ctx context.Context, teamID string) ([]models.Persona, error)
// TeamPersonaKBs returns persona_knowledge_bases for given persona IDs.
TeamPersonaKBs(ctx context.Context, personaIDs []string) ([]models.PersonaKB, error)
// TeamKnowledgeBases returns team-scoped knowledge bases.
TeamKnowledgeBases(ctx context.Context, teamID string) ([]models.KnowledgeBase, error)
// TeamKBDocuments returns KB document metadata for given KB IDs (no chunks/embeddings).
TeamKBDocuments(ctx context.Context, kbIDs []string) ([]models.KBDocument, error)
// TeamWorkflows returns team-scoped workflows.
TeamWorkflows(ctx context.Context, teamID string) ([]models.Workflow, error)
// TeamWorkflowVersions returns versions for given workflow IDs.
TeamWorkflowVersions(ctx context.Context, workflowIDs []string) ([]models.WorkflowVersion, error)
// TeamWorkflowStages returns stages for given workflow IDs.
TeamWorkflowStages(ctx context.Context, workflowIDs []string) ([]models.WorkflowStage, error)
// TeamGroups returns team-scoped groups.
TeamGroups(ctx context.Context, teamID string) ([]models.Group, error)
// TeamGroupMembers returns members for given group IDs.
TeamGroupMembers(ctx context.Context, groupIDs []string) ([]models.GroupMember, error)
// TeamResourceGrants returns resource grants for team resources.
TeamResourceGrants(ctx context.Context, teamID string) ([]models.ResourceGrant, error)
// TeamProjects returns team-scoped projects.
TeamProjects(ctx context.Context, teamID string) ([]models.Project, error)
// ── Import (CS1) ──
// Each ImportXxx method inserts entities preserving their original IDs.
// Uses INSERT ON CONFLICT DO NOTHING (PG) / INSERT OR IGNORE (SQLite)
// to skip duplicates. Returns (imported, skipped) counts.
ImportFolders(ctx context.Context, folders []models.Folder) (int, int, error)
ImportProjects(ctx context.Context, projects []models.Project) (int, int, error)
ImportWorkspaces(ctx context.Context, workspaces []models.Workspace) (int, int, error)
ImportChannels(ctx context.Context, channels []models.Channel) (int, int, error)
ImportChannelParticipants(ctx context.Context, cps []models.ChannelParticipant) (int, int, error)
ImportChannelModels(ctx context.Context, cms []models.ChannelModel) (int, int, error)
ImportChannelCursors(ctx context.Context, cursors []models.ChannelCursor) (int, int, error)
ImportMessages(ctx context.Context, msgs []models.Message) (int, int, error)
ImportNotes(ctx context.Context, notes []models.Note) (int, int, error)
ImportNoteLinks(ctx context.Context, links []models.ExportNoteLink) (int, int, error)
ImportMemories(ctx context.Context, mems []models.Memory) (int, int, error)
ImportProjectChannels(ctx context.Context, pcs []models.ProjectChannel) (int, int, error)
ImportProjectKBs(ctx context.Context, pks []models.ProjectKB) (int, int, error)
ImportProjectNotes(ctx context.Context, pns []models.ProjectNote) (int, int, error)
ImportWorkspaceFiles(ctx context.Context, wfs []models.WorkspaceFile) (int, int, error)
ImportFiles(ctx context.Context, files []models.File) (int, int, error)
ImportUserModelSettings(ctx context.Context, ums []models.UserModelSetting) (int, int, error)
ImportNotifPrefs(ctx context.Context, nps []models.NotificationPreference) (int, int, error)
ImportPersonaGroups(ctx context.Context, pgs []models.PersonaGroup) (int, int, error)
ImportPersonaGroupMembers(ctx context.Context, pgms []models.PersonaGroupMember) (int, int, error)
// ── GDPR ──
// CountActiveAdmins returns the number of active admin users.
CountActiveAdmins(ctx context.Context) (int, error)
// AnonymizeUser replaces PII with "deleted-user-{hash}" placeholders.
AnonymizeUser(ctx context.Context, userID string, anonHash string) error
// SoftDeleteUserData cascade-deletes/soft-deletes all user-owned data.
// Returns entity counts by type for the audit trail.
SoftDeleteUserData(ctx context.Context, userID string) (map[string]int64, error)
// DeleteUserTokens revokes all refresh tokens and WS tickets.
DeleteUserTokens(ctx context.Context, userID string) error
}
// =========================================
// SHARED TYPES
// =========================================

File diff suppressed because it is too large Load Diff

View File

@@ -50,5 +50,6 @@ func NewStores(db *sql.DB) store.Stores {
ExtData: NewExtDataStore(db),
Tickets: NewTicketStore(),
RateLimits: NewRateLimitStore(),
Export: NewExportStore(),
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -50,5 +50,6 @@ func NewStores(db *sql.DB) store.Stores {
ExtData: NewExtDataStore(),
Tickets: NewTicketStore(),
RateLimits: NewRateLimitStore(),
Export: NewExportStore(),
}
}