Changeset 0.34.0 (#208)

Co-authored-by: Jeffrey Smith <jasafpro@gmail.com>
Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>

server/handlers/export_data.go

@@ -0,0 +1,565 @@
+package handlers
+
+// export_data.go — v0.34.0 CS0
+//
+// Data export endpoints: user data export (GDPR "download my data")
+// and team data export (admin). Streams .switchboard zip archives
+// directly to the HTTP response.
+
+import (
+	"fmt"
+	"log/slog"
+	"net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+
+	"chat-switchboard/export"
+	"chat-switchboard/models"
+	"chat-switchboard/storage"
+	"chat-switchboard/store"
+)
+
+// DataExportHandler serves data export endpoints.
+type DataExportHandler struct {
+	stores   store.Stores
+	objStore storage.ObjectStore
+}
+
+// NewDataExportHandler creates a new handler for data export operations.
+func NewDataExportHandler(s store.Stores, obj storage.ObjectStore) *DataExportHandler {
+	return &DataExportHandler{stores: s, objStore: obj}
+}
+
+// ExportMyData streams the requesting user's data as a .switchboard zip.
+// GET /api/v1/export/me
+func (h *DataExportHandler) ExportMyData(c *gin.Context) {
+	ctx := c.Request.Context()
+	userID := c.GetString("user_id")
+
+	// ── Fetch user ──
+	user, err := h.stores.Users.GetByID(ctx, userID)
+	if err != nil || user == nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "user not found"})
+		return
+	}
+
+	// ── Fetch all user-scoped entities ──
+	channels, err := h.stores.Export.UserChannels(ctx, userID)
+	if err != nil {
+		slog.Error("export: fetch channels", "error", err, "user_id", userID)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export channels"})
+		return
+	}
+
+	channelIDs := make([]string, len(channels))
+	for i, ch := range channels {
+		channelIDs[i] = ch.ID
+	}
+
+	messages, err := h.stores.Export.UserMessages(ctx, channelIDs)
+	if err != nil {
+		slog.Error("export: fetch messages", "error", err, "user_id", userID)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export messages"})
+		return
+	}
+
+	participants, err := h.stores.Export.UserChannelParticipants(ctx, channelIDs)
+	if err != nil {
+		slog.Error("export: fetch participants", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export participants"})
+		return
+	}
+
+	channelModels, err := h.stores.Export.UserChannelModels(ctx, channelIDs)
+	if err != nil {
+		slog.Error("export: fetch channel models", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export channel models"})
+		return
+	}
+
+	cursors, err := h.stores.Export.UserChannelCursors(ctx, userID, channelIDs)
+	if err != nil {
+		slog.Error("export: fetch cursors", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export cursors"})
+		return
+	}
+
+	notes, err := h.stores.Export.UserNotes(ctx, userID)
+	if err != nil {
+		slog.Error("export: fetch notes", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export notes"})
+		return
+	}
+
+	noteIDs := make([]string, len(notes))
+	for i, n := range notes {
+		noteIDs[i] = n.ID
+	}
+
+	noteLinks, err := h.stores.Export.UserNoteLinks(ctx, noteIDs)
+	if err != nil {
+		slog.Error("export: fetch note links", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export note links"})
+		return
+	}
+
+	memories, err := h.stores.Export.UserMemories(ctx, userID)
+	if err != nil {
+		slog.Error("export: fetch memories", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export memories"})
+		return
+	}
+
+	projects, err := h.stores.Export.UserProjects(ctx, userID)
+	if err != nil {
+		slog.Error("export: fetch projects", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export projects"})
+		return
+	}
+
+	projectIDs := make([]string, len(projects))
+	for i, p := range projects {
+		projectIDs[i] = p.ID
+	}
+
+	projectChannels, err := h.stores.Export.UserProjectChannels(ctx, projectIDs)
+	if err != nil {
+		slog.Error("export: fetch project channels", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export project channels"})
+		return
+	}
+
+	projectKBs, err := h.stores.Export.UserProjectKBs(ctx, projectIDs)
+	if err != nil {
+		slog.Error("export: fetch project KBs", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export project KBs"})
+		return
+	}
+
+	projectNotes, err := h.stores.Export.UserProjectNotes(ctx, projectIDs)
+	if err != nil {
+		slog.Error("export: fetch project notes", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export project notes"})
+		return
+	}
+
+	workspaces, err := h.stores.Export.UserWorkspaces(ctx, userID)
+	if err != nil {
+		slog.Error("export: fetch workspaces", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export workspaces"})
+		return
+	}
+
+	workspaceIDs := make([]string, len(workspaces))
+	for i, w := range workspaces {
+		workspaceIDs[i] = w.ID
+	}
+
+	workspaceFiles, err := h.stores.Export.UserWorkspaceFiles(ctx, workspaceIDs)
+	if err != nil {
+		slog.Error("export: fetch workspace files", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export workspace files"})
+		return
+	}
+
+	files, err := h.stores.Export.UserFiles(ctx, userID)
+	if err != nil {
+		slog.Error("export: fetch files", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export files"})
+		return
+	}
+
+	folders, err := h.stores.Export.UserFolders(ctx, userID)
+	if err != nil {
+		slog.Error("export: fetch folders", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export folders"})
+		return
+	}
+
+	userModelSettings, err := h.stores.Export.UserModelSettings(ctx, userID)
+	if err != nil {
+		slog.Error("export: fetch user model settings", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export user settings"})
+		return
+	}
+
+	notifPrefs, err := h.stores.Export.UserNotifPrefs(ctx, userID)
+	if err != nil {
+		slog.Error("export: fetch notification prefs", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export notification prefs"})
+		return
+	}
+
+	usageEntries, err := h.stores.Export.UserUsageEntries(ctx, userID)
+	if err != nil {
+		slog.Error("export: fetch usage entries", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export usage entries"})
+		return
+	}
+
+	personaGroups, err := h.stores.Export.UserPersonaGroups(ctx, userID)
+	if err != nil {
+		slog.Error("export: fetch persona groups", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export persona groups"})
+		return
+	}
+
+	pgIDs := make([]string, len(personaGroups))
+	for i, pg := range personaGroups {
+		pgIDs[i] = pg.ID
+	}
+
+	personaGroupMembers, err := h.stores.Export.UserPersonaGroupMembers(ctx, pgIDs)
+	if err != nil {
+		slog.Error("export: fetch persona group members", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export persona group members"})
+		return
+	}
+
+	// ── Build manifest ──
+	sanitizedUser := export.SanitizeUser(user)
+	sanitizedChannels := export.SanitizeChannels(channels)
+	sanitizedMessages := export.SanitizeMessages(messages)
+	sanitizedChannelModels := export.SanitizeChannelModels(channelModels)
+	sanitizedUserModelSettings := export.SanitizeUserModelSettings(userModelSettings)
+	sanitizedUsageEntries := export.SanitizeUsageEntries(usageEntries)
+	sanitizedWorkspaces := export.SanitizeWorkspaces(workspaces)
+
+	counts := map[string]int{
+		"users":                    1,
+		"channels":                 len(channels),
+		"messages":                 len(messages),
+		"channel_participants":     len(participants),
+		"channel_models":           len(channelModels),
+		"channel_cursors":          len(cursors),
+		"notes":                    len(notes),
+		"note_links":               len(noteLinks),
+		"memories":                 len(memories),
+		"projects":                 len(projects),
+		"project_channels":         len(projectChannels),
+		"project_knowledge_bases":  len(projectKBs),
+		"project_notes":            len(projectNotes),
+		"workspaces":               len(workspaces),
+		"workspace_files":          len(workspaceFiles),
+		"files":                    len(files),
+		"folders":                  len(folders),
+		"user_settings":            len(userModelSettings),
+		"notification_preferences": len(notifPrefs),
+		"usage_entries":            len(usageEntries),
+		"persona_groups":           len(personaGroups),
+		"persona_group_members":    len(personaGroupMembers),
+	}
+
+	manifest := &export.Manifest{
+		Version:       "0.34.0",
+		FormatVersion: export.FormatVersion,
+		ExportType:    "user",
+		CreatedAt:     time.Now().UTC(),
+		ExportedBy:    userID,
+		EntityCounts:  counts,
+		Scope:         &export.ManifestScope{UserID: userID},
+	}
+
+	// ── Stream zip to response ──
+	filename := fmt.Sprintf("switchboard-export-%s%s", userID[:8], export.ExportExtension)
+	c.Header("Content-Type", export.ExportContentType)
+	c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filename))
+
+	aw := export.NewArchiveWriter(c.Writer)
+	defer aw.Close()
+
+	if err := aw.WriteManifest(manifest); err != nil {
+		slog.Error("export: write manifest", "error", err)
+		return
+	}
+
+	// Write entity JSON files
+	writeEntity := func(name string, data interface{}) {
+		if _, err := aw.WriteEntityJSON(name, data); err != nil {
+			slog.Error("export: write entity", "name", name, "error", err)
+		}
+	}
+
+	writeEntity("users", []interface{}{sanitizedUser})
+	writeEntity("channels", sanitizedChannels)
+	writeEntity("messages", sanitizedMessages)
+	writeEntity("channel_participants", participants)
+	writeEntity("channel_models", sanitizedChannelModels)
+	writeEntity("channel_cursors", cursors)
+	writeEntity("notes", notes)
+	writeEntity("note_links", noteLinks)
+	writeEntity("memories", memories)
+	writeEntity("projects", projects)
+	writeEntity("project_channels", projectChannels)
+	writeEntity("project_knowledge_bases", projectKBs)
+	writeEntity("project_notes", projectNotes)
+	writeEntity("workspaces", sanitizedWorkspaces)
+	writeEntity("workspace_files", workspaceFiles)
+	writeEntity("folders", folders)
+	writeEntity("user_settings", sanitizedUserModelSettings)
+	writeEntity("notification_preferences", notifPrefs)
+	writeEntity("usage_entries", sanitizedUsageEntries)
+	writeEntity("persona_groups", personaGroups)
+	writeEntity("persona_group_members", personaGroupMembers)
+
+	// Write file metadata (without storage_key — already excluded by json:"-" tag)
+	writeEntity("files", files)
+
+	// ── Stream file blobs ──
+	var warnings []string
+	if h.objStore != nil {
+		fileCount := 0
+		for _, f := range files {
+			if fileCount >= export.MaxExportFiles {
+				warnings = append(warnings, "file blob limit reached, some files skipped")
+				break
+			}
+			if f.SizeBytes > export.MaxExportFileSize {
+				warnings = append(warnings, fmt.Sprintf("file %s too large (%d bytes), skipped", f.Filename, f.SizeBytes))
+				continue
+			}
+			rc, _, _, err := h.objStore.Get(ctx, f.StorageKey)
+			if err != nil {
+				warnings = append(warnings, fmt.Sprintf("file %s not found in storage, skipped", f.Filename))
+				continue
+			}
+			zipPath := f.ID + "/" + f.Filename
+			if _, err := aw.WriteFile(zipPath, rc); err != nil {
+				rc.Close()
+				warnings = append(warnings, fmt.Sprintf("file %s write error: %v, skipped", f.Filename, err))
+				continue
+			}
+			rc.Close()
+			fileCount++
+		}
+	}
+
+	if len(warnings) > 0 {
+		writeEntity("export_warnings", warnings)
+	}
+
+	// Audit log
+	h.stores.Audit.Log(ctx, &models.AuditEntry{
+		ActorID:      &userID,
+		Action:       "user.export",
+		ResourceType: "user",
+		ResourceID:   userID,
+		Metadata:     models.JSONMap{"entity_counts": counts},
+	})
+}
+
+// ExportTeam streams a team's data as a .switchboard zip.
+// GET /api/v1/admin/teams/:id/export
+func (h *DataExportHandler) ExportTeam(c *gin.Context) {
+	ctx := c.Request.Context()
+	userID := c.GetString("user_id")
+	teamID := c.Param("id")
+
+	// Fetch team to verify it exists
+	team, err := h.stores.Teams.GetByID(ctx, teamID)
+	if err != nil || team == nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "team not found"})
+		return
+	}
+
+	// Fetch all team-scoped entities
+	channels, err := h.stores.Export.TeamChannels(ctx, teamID)
+	if err != nil {
+		slog.Error("export: fetch team channels", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export team channels"})
+		return
+	}
+
+	channelIDs := make([]string, len(channels))
+	for i, ch := range channels {
+		channelIDs[i] = ch.ID
+	}
+
+	messages, err := h.stores.Export.UserMessages(ctx, channelIDs)
+	if err != nil {
+		slog.Error("export: fetch team messages", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export team messages"})
+		return
+	}
+
+	members, err := h.stores.Export.TeamMembers(ctx, teamID)
+	if err != nil {
+		slog.Error("export: fetch team members", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export team members"})
+		return
+	}
+
+	personas, err := h.stores.Export.TeamPersonas(ctx, teamID)
+	if err != nil {
+		slog.Error("export: fetch team personas", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export team personas"})
+		return
+	}
+
+	personaIDs := make([]string, len(personas))
+	for i, p := range personas {
+		personaIDs[i] = p.ID
+	}
+
+	personaKBs, err := h.stores.Export.TeamPersonaKBs(ctx, personaIDs)
+	if err != nil {
+		slog.Error("export: fetch persona KBs", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export persona KBs"})
+		return
+	}
+
+	kbs, err := h.stores.Export.TeamKnowledgeBases(ctx, teamID)
+	if err != nil {
+		slog.Error("export: fetch team KBs", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export team KBs"})
+		return
+	}
+
+	kbIDs := make([]string, len(kbs))
+	for i, kb := range kbs {
+		kbIDs[i] = kb.ID
+	}
+
+	kbDocs, err := h.stores.Export.TeamKBDocuments(ctx, kbIDs)
+	if err != nil {
+		slog.Error("export: fetch KB docs", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export KB documents"})
+		return
+	}
+
+	workflows, err := h.stores.Export.TeamWorkflows(ctx, teamID)
+	if err != nil {
+		slog.Error("export: fetch workflows", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export workflows"})
+		return
+	}
+
+	workflowIDs := make([]string, len(workflows))
+	for i, w := range workflows {
+		workflowIDs[i] = w.ID
+	}
+
+	workflowVersions, err := h.stores.Export.TeamWorkflowVersions(ctx, workflowIDs)
+	if err != nil {
+		slog.Error("export: fetch workflow versions", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export workflow versions"})
+		return
+	}
+
+	workflowStages, err := h.stores.Export.TeamWorkflowStages(ctx, workflowIDs)
+	if err != nil {
+		slog.Error("export: fetch workflow stages", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export workflow stages"})
+		return
+	}
+
+	groups, err := h.stores.Export.TeamGroups(ctx, teamID)
+	if err != nil {
+		slog.Error("export: fetch groups", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export groups"})
+		return
+	}
+
+	groupIDs := make([]string, len(groups))
+	for i, g := range groups {
+		groupIDs[i] = g.ID
+	}
+
+	groupMembers, err := h.stores.Export.TeamGroupMembers(ctx, groupIDs)
+	if err != nil {
+		slog.Error("export: fetch group members", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export group members"})
+		return
+	}
+
+	resourceGrants, err := h.stores.Export.TeamResourceGrants(ctx, teamID)
+	if err != nil {
+		slog.Error("export: fetch resource grants", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export resource grants"})
+		return
+	}
+
+	projects, err := h.stores.Export.TeamProjects(ctx, teamID)
+	if err != nil {
+		slog.Error("export: fetch team projects", "error", err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to export team projects"})
+		return
+	}
+
+	// Sanitize
+	sanitizedChannels := export.SanitizeChannels(channels)
+	sanitizedMessages := export.SanitizeMessages(messages)
+	sanitizedPersonas := export.SanitizePersonas(personas)
+	sanitizedWorkflows := export.SanitizeWorkflows(workflows)
+
+	counts := map[string]int{
+		"team":                 1,
+		"channels":             len(channels),
+		"messages":             len(messages),
+		"members":              len(members),
+		"personas":             len(personas),
+		"persona_kbs":          len(personaKBs),
+		"knowledge_bases":      len(kbs),
+		"kb_documents":         len(kbDocs),
+		"workflows":            len(workflows),
+		"workflow_versions":    len(workflowVersions),
+		"workflow_stages":      len(workflowStages),
+		"groups":               len(groups),
+		"group_members":        len(groupMembers),
+		"resource_grants":      len(resourceGrants),
+		"projects":             len(projects),
+	}
+
+	manifest := &export.Manifest{
+		Version:       "0.34.0",
+		FormatVersion: export.FormatVersion,
+		ExportType:    "team",
+		CreatedAt:     time.Now().UTC(),
+		ExportedBy:    userID,
+		EntityCounts:  counts,
+		Scope:         &export.ManifestScope{TeamID: teamID},
+	}
+
+	filename := fmt.Sprintf("switchboard-team-%s%s", teamID[:8], export.ExportExtension)
+	c.Header("Content-Type", export.ExportContentType)
+	c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filename))
+
+	aw := export.NewArchiveWriter(c.Writer)
+	defer aw.Close()
+
+	if err := aw.WriteManifest(manifest); err != nil {
+		slog.Error("export: write manifest", "error", err)
+		return
+	}
+
+	writeEntity := func(name string, data interface{}) {
+		if _, err := aw.WriteEntityJSON(name, data); err != nil {
+			slog.Error("export: write entity", "name", name, "error", err)
+		}
+	}
+
+	writeEntity("team", []interface{}{team})
+	writeEntity("channels", sanitizedChannels)
+	writeEntity("messages", sanitizedMessages)
+	writeEntity("members", members)
+	writeEntity("personas", sanitizedPersonas)
+	writeEntity("persona_knowledge_bases", personaKBs)
+	writeEntity("knowledge_bases", kbs)
+	writeEntity("kb_documents", kbDocs)
+	writeEntity("workflows", sanitizedWorkflows)
+	writeEntity("workflow_versions", workflowVersions)
+	writeEntity("workflow_stages", workflowStages)
+	writeEntity("groups", groups)
+	writeEntity("group_members", groupMembers)
+	writeEntity("resource_grants", resourceGrants)
+	writeEntity("projects", projects)
+
+	h.stores.Audit.Log(ctx, &models.AuditEntry{
+		ActorID:      &userID,
+		Action:       "team.export",
+		ResourceType: "team",
+		ResourceID:   teamID,
+		Metadata:     models.JSONMap{"entity_counts": counts},
+	})
+}