Feat v0.8.3 vector column (#70)

Co-authored-by: Jeffrey Smith <jasafpro@gmail.com>
Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>

server/handlers/ext_db_schema.go

@@ -13,20 +13,38 @@ import (
 	"fmt"
 	"log"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"armature/store"
 )
 
+// vectorDimRegex matches manifest type strings like "vector(384)".
+var vectorDimRegex = regexp.MustCompile(`^vector\((\d+)\)$`)
+
 // validSchemaIdentifier matches safe SQL identifiers for table and column names.
 var validSchemaIdentifier = regexp.MustCompile(`^[a-z][a-z0-9_]{0,62}$`)
 
 // TableDef describes a single extension-owned table as declared in a manifest.
 type TableDef struct {
-	Columns map[string]string // colName → manifest type ("text","int","real","bool","timestamp")
+	Columns map[string]string // colName → manifest type ("text","int","real","bool","timestamp","vector(N)")
 	Indexes [][]string        // each inner slice is an ordered list of column names for one index
 }
 
+// parseVectorDim extracts the dimension N from a "vector(N)" type string.
+// Returns (N, true) for valid dimensions 1..4096, or (0, false) otherwise.
+func parseVectorDim(typStr string) (int, bool) {
+	m := vectorDimRegex.FindStringSubmatch(strings.ToLower(typStr))
+	if m == nil {
+		return 0, false
+	}
+	n, err := strconv.Atoi(m[1])
+	if err != nil || n < 1 || n > 4096 {
+		return 0, false
+	}
+	return n, true
+}
+
 // ParseDBTables extracts the "db_tables" key from a package manifest map.
 // Returns the map of logicalName→TableDef and ok=true when the key is present
 // and produces at least one valid table definition.
@@ -93,8 +111,22 @@ func ParseDBTables(manifest map[string]any) (map[string]TableDef, bool) {
 }
 
 // mapColType maps a manifest type string to a SQL column type for the given dialect.
-func mapColType(typStr string, isPostgres bool) string {
-	switch strings.ToLower(typStr) {
+// hasPgvector indicates whether the pgvector extension is available on Postgres.
+func mapColType(typStr string, isPostgres bool, hasPgvector bool) string {
+	lower := strings.ToLower(typStr)
+
+	// Check for vector type first.
+	if _, ok := parseVectorDim(lower); ok {
+		if isPostgres && hasPgvector {
+			return lower // native pgvector type, e.g. "vector(384)"
+		}
+		if isPostgres {
+			return "JSONB" // structured fallback without pgvector
+		}
+		return "TEXT" // SQLite fallback — JSON string
+	}
+
+	switch lower {
 	case "int", "integer":
 		return "INTEGER"
 	case "real", "float":
@@ -133,6 +165,8 @@ func createdAtColDef(isPostgres bool) string {
 // in the provided map.  Each successfully created table is registered in the
 // ext_data_tables catalog via stores.ExtData.
 //
+// hasPgvector enables native pgvector column types and HNSW indexes when true.
+//
 // Errors from individual table creation are returned immediately (fail-fast).
 // Index creation failures are logged but non-fatal.
 // Catalog registration failures are logged but non-fatal.
@@ -143,6 +177,7 @@ func CreateExtTables(
 	stores store.Stores,
 	packageID string,
 	tables map[string]TableDef,
+	hasPgvector bool,
 ) error {
 	if db == nil {
 		return nil
@@ -153,7 +188,7 @@ func CreateExtTables(
 		// Build column list: id first, user columns, created_at last.
 		cols := []string{"id TEXT PRIMARY KEY"}
 		for col, typ := range td.Columns {
-			cols = append(cols, col+" "+mapColType(typ, isPostgres))
+			cols = append(cols, col+" "+mapColType(typ, isPostgres, hasPgvector))
 		}
 		cols = append(cols, createdAtColDef(isPostgres))
 
@@ -174,6 +209,21 @@ func CreateExtTables(
 			}
 		}
 
+		// Create HNSW indexes for vector columns when pgvector is available.
+		if isPostgres && hasPgvector {
+			for col, typ := range td.Columns {
+				if _, ok := parseVectorDim(strings.ToLower(typ)); ok {
+					hnswName := fmt.Sprintf("idx_%s_%s_hnsw", physical, col)
+					hnswDDL := fmt.Sprintf(
+						"CREATE INDEX IF NOT EXISTS %s ON %s USING hnsw (%s vector_cosine_ops)",
+						hnswName, physical, col)
+					if _, err := db.ExecContext(ctx, hnswDDL); err != nil {
+						log.Printf("[ext-db] hnsw index create failed (%s): %v", hnswName, err)
+					}
+				}
+			}
+		}
+
 		// Record logical name in catalog.
 		if stores.ExtData != nil {
 			if err := stores.ExtData.RegisterTable(ctx, packageID, logicalName); err != nil {
@@ -245,6 +295,7 @@ func MigrateExtTables(
 	stores store.Stores,
 	packageID string,
 	newTables map[string]TableDef,
+	hasPgvector bool,
 ) ([]string, error) {
 	if db == nil {
 		return nil, nil
@@ -270,7 +321,7 @@ func MigrateExtTables(
 		if !existingTableNames[logicalName] {
 			// New table — full creation
 			if err := CreateExtTables(ctx, db, isPostgres, stores, packageID,
-				map[string]TableDef{logicalName: td}); err != nil {
+				map[string]TableDef{logicalName: td}, hasPgvector); err != nil {
 				return changes, fmt.Errorf("create new table %s: %w", physical, err)
 			}
 			changes = append(changes, fmt.Sprintf("created table %s", physical))
@@ -287,7 +338,7 @@ func MigrateExtTables(
 			if existingCols[col] {
 				continue
 			}
-			sqlType := mapColType(typ, isPostgres)
+			sqlType := mapColType(typ, isPostgres, hasPgvector)
 			alterDDL := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", physical, col, sqlType)
 			if _, err := db.ExecContext(ctx, alterDDL); err != nil {
 				return changes, fmt.Errorf("add column %s.%s: %w", physical, col, err)