package handlers // ext_db_schema.go // // DDL generation and lifecycle management for extension-owned database tables. // Tables are namespaced as ext_{pkg_slug}_{logical_name} and tracked in the // ext_data_tables catalog. CreateExtTables is called on install; // DropExtTables is called on uninstall. import ( "context" "database/sql" "fmt" "log" "regexp" "strconv" "strings" "armature/store" ) // vectorDimRegex matches manifest type strings like "vector(384)". var vectorDimRegex = regexp.MustCompile(`^vector\((\d+)\)$`) // validSchemaIdentifier matches safe SQL identifiers for table and column names. var validSchemaIdentifier = regexp.MustCompile(`^[a-z][a-z0-9_]{0,62}$`) // TableDef describes a single extension-owned table as declared in a manifest. type TableDef struct { Columns map[string]string // colName → manifest type ("text","int","real","bool","timestamp","vector(N)") Indexes [][]string // each inner slice is an ordered list of column names for one index } // parseVectorDim extracts the dimension N from a "vector(N)" type string. // Returns (N, true) for valid dimensions 1..4096, or (0, false) otherwise. func parseVectorDim(typStr string) (int, bool) { m := vectorDimRegex.FindStringSubmatch(strings.ToLower(typStr)) if m == nil { return 0, false } n, err := strconv.Atoi(m[1]) if err != nil || n < 1 || n > 4096 { return 0, false } return n, true } // ParseDBTables extracts the "db_tables" key from a package manifest map. // Returns the map of logicalName→TableDef and ok=true when the key is present // and produces at least one valid table definition. func ParseDBTables(manifest map[string]any) (map[string]TableDef, bool) { raw, ok := manifest["db_tables"] if !ok { return nil, false } tablesRaw, ok := raw.(map[string]any) if !ok || len(tablesRaw) == 0 { return nil, false } tables := make(map[string]TableDef, len(tablesRaw)) for name, defRaw := range tablesRaw { if !validSchemaIdentifier.MatchString(name) { log.Printf("[ext-db] skipping invalid table name %q", name) continue } defMap, ok := defRaw.(map[string]any) if !ok { continue } td := TableDef{Columns: map[string]string{}} // Parse columns map. if colsRaw, ok := defMap["columns"].(map[string]any); ok { for col, typ := range colsRaw { if !validSchemaIdentifier.MatchString(col) { log.Printf("[ext-db] skipping invalid column name %q in table %q", col, name) continue } if t, ok := typ.(string); ok { td.Columns[col] = t } } } // Parse indexes array. if idxsRaw, ok := defMap["indexes"].([]any); ok { for _, idxRaw := range idxsRaw { if idxArr, ok := idxRaw.([]any); ok { var cols []string for _, c := range idxArr { if s, ok := c.(string); ok && validSchemaIdentifier.MatchString(s) { cols = append(cols, s) } } if len(cols) > 0 { td.Indexes = append(td.Indexes, cols) } } } } tables[name] = td } if len(tables) == 0 { return nil, false } return tables, true } // mapColType maps a manifest type string to a SQL column type for the given dialect. // hasPgvector indicates whether the pgvector extension is available on Postgres. func mapColType(typStr string, isPostgres bool, hasPgvector bool) string { lower := strings.ToLower(typStr) // Check for vector type first. if _, ok := parseVectorDim(lower); ok { if isPostgres && hasPgvector { return lower // native pgvector type, e.g. "vector(384)" } if isPostgres { return "JSONB" // structured fallback without pgvector } return "TEXT" // SQLite fallback — JSON string } switch lower { case "int", "integer": return "INTEGER" case "real", "float": return "REAL" case "bool", "boolean": if isPostgres { return "BOOLEAN" } return "INTEGER" case "timestamp": if isPostgres { return "TIMESTAMPTZ" } return "TEXT" default: // "text" and anything unrecognized return "TEXT" } } // extPhysicalTable builds the namespaced physical table name for a package's logical table. // e.g. packageID="my-ext", logicalName="logs" → "ext_my_ext_logs" func extPhysicalTable(packageID, logicalName string) string { slug := strings.ReplaceAll(packageID, "-", "_") return "ext_" + slug + "_" + logicalName } // createdAtColDef returns the dialect-correct DDL fragment for the created_at column. func createdAtColDef(isPostgres bool) string { if isPostgres { return "created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()" } return "created_at TEXT NOT NULL DEFAULT (datetime('now'))" } // CreateExtTables generates dialect-correct DDL and executes it for all tables // in the provided map. Each successfully created table is registered in the // ext_data_tables catalog via stores.ExtData. // // hasPgvector enables native pgvector column types and HNSW indexes when true. // // Errors from individual table creation are returned immediately (fail-fast). // Index creation failures are logged but non-fatal. // Catalog registration failures are logged but non-fatal. func CreateExtTables( ctx context.Context, db *sql.DB, isPostgres bool, stores store.Stores, packageID string, tables map[string]TableDef, hasPgvector bool, ) error { if db == nil { return nil } for logicalName, td := range tables { physical := extPhysicalTable(packageID, logicalName) // Build column list: id first, user columns, created_at last. cols := []string{"id TEXT PRIMARY KEY"} for col, typ := range td.Columns { cols = append(cols, col+" "+mapColType(typ, isPostgres, hasPgvector)) } cols = append(cols, createdAtColDef(isPostgres)) ddl := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (\n\t%s\n)", physical, strings.Join(cols, ",\n\t")) if _, err := db.ExecContext(ctx, ddl); err != nil { return fmt.Errorf("create ext table %s: %w", physical, err) } // Create declared indexes. for i, idxCols := range td.Indexes { idxName := fmt.Sprintf("idx_%s_%d", physical, i) idxDDL := fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (%s)", idxName, physical, strings.Join(idxCols, ", ")) if _, err := db.ExecContext(ctx, idxDDL); err != nil { log.Printf("[ext-db] index create failed (%s): %v", idxName, err) } } // Create HNSW indexes for vector columns when pgvector is available. if isPostgres && hasPgvector { for col, typ := range td.Columns { if _, ok := parseVectorDim(strings.ToLower(typ)); ok { hnswName := fmt.Sprintf("idx_%s_%s_hnsw", physical, col) hnswDDL := fmt.Sprintf( "CREATE INDEX IF NOT EXISTS %s ON %s USING hnsw (%s vector_cosine_ops)", hnswName, physical, col) if _, err := db.ExecContext(ctx, hnswDDL); err != nil { log.Printf("[ext-db] hnsw index create failed (%s): %v", hnswName, err) } } } } // Record logical name in catalog. if stores.ExtData != nil { if err := stores.ExtData.RegisterTable(ctx, packageID, logicalName); err != nil { log.Printf("[ext-db] catalog register failed (%s/%s): %v", packageID, logicalName, err) } } log.Printf("[ext-db] created table %s for package %s", physical, packageID) } return nil } // ListExtColumns returns the set of column names for a physical table. // Uses PRAGMA table_info on SQLite and information_schema on Postgres. func ListExtColumns(ctx context.Context, db *sql.DB, isPostgres bool, physicalTable string) (map[string]bool, error) { cols := make(map[string]bool) var rows *sql.Rows var err error if isPostgres { rows, err = db.QueryContext(ctx, `SELECT column_name FROM information_schema.columns WHERE table_name = $1 AND table_schema = 'public'`, physicalTable) } else { rows, err = db.QueryContext(ctx, fmt.Sprintf("PRAGMA table_info(%s)", physicalTable)) } if err != nil { return nil, fmt.Errorf("list columns for %s: %w", physicalTable, err) } defer rows.Close() for rows.Next() { if isPostgres { var name string if err := rows.Scan(&name); err != nil { continue } cols[name] = true } else { // PRAGMA table_info returns: cid, name, type, notnull, dflt_value, pk var cid int var name, colType string var notnull int var dfltValue sql.NullString var pk int if err := rows.Scan(&cid, &name, &colType, ¬null, &dfltValue, &pk); err != nil { continue } cols[name] = true } } return cols, rows.Err() } // MigrateExtTables applies additive-only schema changes for a package update. // For each table in the new manifest: // - New table: CREATE TABLE (full creation, same as CreateExtTables) // - Existing table: ALTER TABLE ADD COLUMN for each new column // // Indexes are applied idempotently with CREATE INDEX IF NOT EXISTS. // New tables are registered in the ext_data_tables catalog. // Returns a log of changes applied (for API response). // // No destructive changes: columns present in DB but absent from manifest are NOT dropped. func MigrateExtTables( ctx context.Context, db *sql.DB, isPostgres bool, stores store.Stores, packageID string, newTables map[string]TableDef, hasPgvector bool, ) ([]string, error) { if db == nil { return nil, nil } // Get existing registered tables for this package existingTableNames := make(map[string]bool) if stores.ExtData != nil { names, err := stores.ExtData.ListTables(ctx, packageID) if err != nil { return nil, fmt.Errorf("list existing tables for %s: %w", packageID, err) } for _, n := range names { existingTableNames[n] = true } } var changes []string for logicalName, td := range newTables { physical := extPhysicalTable(packageID, logicalName) if !existingTableNames[logicalName] { // New table — full creation if err := CreateExtTables(ctx, db, isPostgres, stores, packageID, map[string]TableDef{logicalName: td}, hasPgvector); err != nil { return changes, fmt.Errorf("create new table %s: %w", physical, err) } changes = append(changes, fmt.Sprintf("created table %s", physical)) continue } // Existing table — diff columns and add missing ones existingCols, err := ListExtColumns(ctx, db, isPostgres, physical) if err != nil { return changes, err } for col, typ := range td.Columns { if existingCols[col] { continue } sqlType := mapColType(typ, isPostgres, hasPgvector) alterDDL := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", physical, col, sqlType) if _, err := db.ExecContext(ctx, alterDDL); err != nil { return changes, fmt.Errorf("add column %s.%s: %w", physical, col, err) } changes = append(changes, fmt.Sprintf("added column '%s' to %s", col, physical)) log.Printf("[ext-db] added column %s (%s) to %s", col, sqlType, physical) } // Re-apply indexes idempotently for i, idxCols := range td.Indexes { idxName := fmt.Sprintf("idx_%s_%d", physical, i) idxDDL := fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s (%s)", idxName, physical, strings.Join(idxCols, ", ")) if _, err := db.ExecContext(ctx, idxDDL); err != nil { log.Printf("[ext-db] index create failed (%s): %v", idxName, err) } } } return changes, nil } // DropExtTables drops all physical tables registered for a package and removes // their entries from the ext_data_tables catalog. // // Table drop errors are logged but non-fatal; the catalog is cleared regardless. func DropExtTables( ctx context.Context, db *sql.DB, stores store.Stores, packageID string, ) error { if db == nil || stores.ExtData == nil { return nil } tables, err := stores.ExtData.ListTables(ctx, packageID) if err != nil { return fmt.Errorf("list ext tables for %s: %w", packageID, err) } for _, logicalName := range tables { physical := extPhysicalTable(packageID, logicalName) if _, err := db.ExecContext(ctx, "DROP TABLE IF EXISTS "+physical); err != nil { log.Printf("[ext-db] drop table %s failed: %v", physical, err) } else { log.Printf("[ext-db] dropped table %s for package %s", physical, packageID) } } return stores.ExtData.DeletePackageTables(ctx, packageID) }