Feat v0.8.3 vector column (#70)

Co-authored-by: Jeffrey Smith <jasafpro@gmail.com>
Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>

server/handlers/ext_db_schema.go

@@ -13,20 +13,38 @@ import (
 	"fmt"
 	"log"
 	"regexp"
+	"strconv"
 	"strings"
 
 	"armature/store"
 )
 
+// vectorDimRegex matches manifest type strings like "vector(384)".
+var vectorDimRegex = regexp.MustCompile(`^vector\((\d+)\)$`)
+
 // validSchemaIdentifier matches safe SQL identifiers for table and column names.
 var validSchemaIdentifier = regexp.MustCompile(`^[a-z][a-z0-9_]{0,62}$`)
 
 // TableDef describes a single extension-owned table as declared in a manifest.
 type TableDef struct {
-	Columns map[string]string // colName → manifest type ("text","int","real","bool","timestamp")
+	Columns map[string]string // colName → manifest type ("text","int","real","bool","timestamp","vector(N)")
 	Indexes [][]string        // each inner slice is an ordered list of column names for one index
 }
 
+// parseVectorDim extracts the dimension N from a "vector(N)" type string.
+// Returns (N, true) for valid dimensions 1..4096, or (0, false) otherwise.
+func parseVectorDim(typStr string) (int, bool) {
+	m := vectorDimRegex.FindStringSubmatch(strings.ToLower(typStr))
+	if m == nil {
+		return 0, false
+	}
+	n, err := strconv.Atoi(m[1])
+	if err != nil || n < 1 || n > 4096 {
+		return 0, false
+	}
+	return n, true
+}
+
 // ParseDBTables extracts the "db_tables" key from a package manifest map.
 // Returns the map of logicalName→TableDef and ok=true when the key is present
 // and produces at least one valid table definition.
@@ -93,8 +111,22 @@ func ParseDBTables(manifest map[string]any) (map[string]TableDef, bool) {
 }
 
 // mapColType maps a manifest type string to a SQL column type for the given dialect.
-func mapColType(typStr string, isPostgres bool) string {
-	switch strings.ToLower(typStr) {
+// hasPgvector indicates whether the pgvector extension is available on Postgres.
+func mapColType(typStr string, isPostgres bool, hasPgvector bool) string {
+	lower := strings.ToLower(typStr)
+
+	// Check for vector type first.
+	if _, ok := parseVectorDim(lower); ok {
+		if isPostgres && hasPgvector {
+			return lower // native pgvector type, e.g. "vector(384)"
+		}
+		if isPostgres {
+			return "JSONB" // structured fallback without pgvector
+		}
+		return "TEXT" // SQLite fallback — JSON string
+	}
+
+	switch lower {
 	case "int", "integer":
 		return "INTEGER"
 	case "real", "float":
@@ -133,6 +165,8 @@ func createdAtColDef(isPostgres bool) string {
 // in the provided map.  Each successfully created table is registered in the
 // ext_data_tables catalog via stores.ExtData.
 //
+// hasPgvector enables native pgvector column types and HNSW indexes when true.
+//
 // Errors from individual table creation are returned immediately (fail-fast).
 // Index creation failures are logged but non-fatal.
 // Catalog registration failures are logged but non-fatal.
@@ -143,6 +177,7 @@ func CreateExtTables(
 	stores store.Stores,
 	packageID string,
 	tables map[string]TableDef,
+	hasPgvector bool,
 ) error {
 	if db == nil {
 		return nil
@@ -153,7 +188,7 @@ func CreateExtTables(
 		// Build column list: id first, user columns, created_at last.
 		cols := []string{"id TEXT PRIMARY KEY"}
 		for col, typ := range td.Columns {
-			cols = append(cols, col+" "+mapColType(typ, isPostgres))
+			cols = append(cols, col+" "+mapColType(typ, isPostgres, hasPgvector))
 		}
 		cols = append(cols, createdAtColDef(isPostgres))
 
@@ -174,6 +209,21 @@ func CreateExtTables(
 			}
 		}
 
+		// Create HNSW indexes for vector columns when pgvector is available.
+		if isPostgres && hasPgvector {
+			for col, typ := range td.Columns {
+				if _, ok := parseVectorDim(strings.ToLower(typ)); ok {
+					hnswName := fmt.Sprintf("idx_%s_%s_hnsw", physical, col)
+					hnswDDL := fmt.Sprintf(
+						"CREATE INDEX IF NOT EXISTS %s ON %s USING hnsw (%s vector_cosine_ops)",
+						hnswName, physical, col)
+					if _, err := db.ExecContext(ctx, hnswDDL); err != nil {
+						log.Printf("[ext-db] hnsw index create failed (%s): %v", hnswName, err)
+					}
+				}
+			}
+		}
+
 		// Record logical name in catalog.
 		if stores.ExtData != nil {
 			if err := stores.ExtData.RegisterTable(ctx, packageID, logicalName); err != nil {
@@ -245,6 +295,7 @@ func MigrateExtTables(
 	stores store.Stores,
 	packageID string,
 	newTables map[string]TableDef,
+	hasPgvector bool,
 ) ([]string, error) {
 	if db == nil {
 		return nil, nil
@@ -270,7 +321,7 @@ func MigrateExtTables(
 		if !existingTableNames[logicalName] {
 			// New table — full creation
 			if err := CreateExtTables(ctx, db, isPostgres, stores, packageID,
-				map[string]TableDef{logicalName: td}); err != nil {
+				map[string]TableDef{logicalName: td}, hasPgvector); err != nil {
 				return changes, fmt.Errorf("create new table %s: %w", physical, err)
 			}
 			changes = append(changes, fmt.Sprintf("created table %s", physical))
@@ -287,7 +338,7 @@ func MigrateExtTables(
 			if existingCols[col] {
 				continue
 			}
-			sqlType := mapColType(typ, isPostgres)
+			sqlType := mapColType(typ, isPostgres, hasPgvector)
 			alterDDL := fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s %s", physical, col, sqlType)
 			if _, err := db.ExecContext(ctx, alterDDL); err != nil {
 				return changes, fmt.Errorf("add column %s.%s: %w", physical, col, err)

server/handlers/ext_db_schema_test.go

@@ -145,7 +145,7 @@ func TestMapColType_SQLite(t *testing.T) {
 		{"unknown", "TEXT"},
 	}
 	for _, c := range cases {
-		got := mapColType(c.in, false)
+		got := mapColType(c.in, false, false)
 		if got != c.want {
 			t.Errorf("mapColType(%q, sqlite) = %q, want %q", c.in, got, c.want)
 		}
@@ -153,13 +153,13 @@ func TestMapColType_SQLite(t *testing.T) {
 }
 
 func TestMapColType_Postgres(t *testing.T) {
-	if mapColType("bool", true) != "BOOLEAN" {
+	if mapColType("bool", true, false) != "BOOLEAN" {
 		t.Error("expected BOOLEAN for bool in postgres")
 	}
-	if mapColType("timestamp", true) != "TIMESTAMPTZ" {
+	if mapColType("timestamp", true, false) != "TIMESTAMPTZ" {
 		t.Error("expected TIMESTAMPTZ for timestamp in postgres")
 	}
-	if mapColType("int", true) != "INTEGER" {
+	if mapColType("int", true, false) != "INTEGER" {
 		t.Error("expected INTEGER for int in postgres")
 	}
 }
@@ -189,7 +189,7 @@ func TestCreateExtTables_CreatesTable(t *testing.T) {
 			Columns: map[string]string{"message": "text", "count": "int"},
 		},
 	}
-	if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "test-pkg", tables); err != nil {
+	if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "test-pkg", tables, false); err != nil {
 		t.Fatalf("CreateExtTables: %v", err)
 	}
 
@@ -212,7 +212,7 @@ func TestCreateExtTables_WithIndexes(t *testing.T) {
 			Indexes: [][]string{{"user_id"}, {"user_id", "kind"}},
 		},
 	}
-	if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "idx-pkg", tables); err != nil {
+	if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "idx-pkg", tables, false); err != nil {
 		t.Fatalf("CreateExtTables: %v", err)
 	}
 
@@ -243,7 +243,7 @@ func TestCreateExtTables_CatalogRegistration(t *testing.T) {
 		"logs":   {Columns: map[string]string{"msg": "text"}},
 		"errors": {Columns: map[string]string{"detail": "text"}},
 	}
-	if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "cat-pkg", tables); err != nil {
+	if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "cat-pkg", tables, false); err != nil {
 		t.Fatalf("CreateExtTables: %v", err)
 	}
 
@@ -259,7 +259,7 @@ func TestCreateExtTables_NilDBIsNoop(t *testing.T) {
 	// Should not panic or error.
 	err := CreateExtTables(ctx, nil, false, storesWithExtData(m), "pkg", map[string]TableDef{
 		"t": {Columns: map[string]string{"x": "text"}},
-	})
+	}, false)
 	if err != nil {
 		t.Errorf("expected nil error for nil db, got: %v", err)
 	}
@@ -276,7 +276,7 @@ func TestDropExtTables_DropsAndClearsCatalog(t *testing.T) {
 	tables := map[string]TableDef{
 		"items": {Columns: map[string]string{"name": "text"}},
 	}
-	if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "drop-pkg", tables); err != nil {
+	if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "drop-pkg", tables, false); err != nil {
 		t.Fatalf("CreateExtTables: %v", err)
 	}
 
@@ -309,7 +309,7 @@ func TestDropExtTables_MultipleTables(t *testing.T) {
 		"alpha": {Columns: map[string]string{"v": "text"}},
 		"beta":  {Columns: map[string]string{"v": "text"}},
 	}
-	CreateExtTables(ctx, db, false, storesWithExtData(m), "multi-pkg", tables)
+	CreateExtTables(ctx, db, false, storesWithExtData(m), "multi-pkg", tables, false)
 	DropExtTables(ctx, db, storesWithExtData(m), "multi-pkg")
 
 	for _, name := range []string{"ext_multi_pkg_alpha", "ext_multi_pkg_beta"} {
@@ -331,7 +331,7 @@ func TestCreateExtTables_DDLContainsAutoColumns(t *testing.T) {
 	ctx := context.Background()
 
 	tables := map[string]TableDef{"empty": {Columns: map[string]string{}}}
-	if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "auto-pkg", tables); err != nil {
+	if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "auto-pkg", tables, false); err != nil {
 		t.Fatalf("CreateExtTables: %v", err)
 	}
 
@@ -348,3 +348,77 @@ func TestCreateExtTables_DDLContainsAutoColumns(t *testing.T) {
 		t.Error("expected created_at to be populated by default")
 	}
 }
+
+// ── parseVectorDim ──────────────────────────────────────────────────────────
+
+func TestParseVectorDim(t *testing.T) {
+	cases := []struct {
+		input string
+		dim   int
+		ok    bool
+	}{
+		{"vector(384)", 384, true},
+		{"vector(1)", 1, true},
+		{"vector(4096)", 4096, true},
+		{"Vector(128)", 128, true}, // case insensitive
+		{"vector(0)", 0, false},
+		{"vector(5000)", 0, false},   // exceeds 4096
+		{"vector()", 0, false},
+		{"vector(abc)", 0, false},
+		{"text", 0, false},
+		{"vector", 0, false},
+	}
+	for _, c := range cases {
+		dim, ok := parseVectorDim(c.input)
+		if ok != c.ok || dim != c.dim {
+			t.Errorf("parseVectorDim(%q) = (%d, %v), want (%d, %v)", c.input, dim, ok, c.dim, c.ok)
+		}
+	}
+}
+
+// ── mapColType vector ───────────────────────────────────────────────────────
+
+func TestMapColType_VectorSQLite(t *testing.T) {
+	got := mapColType("vector(384)", false, false)
+	if got != "TEXT" {
+		t.Errorf("vector on SQLite: got %q, want TEXT", got)
+	}
+}
+
+func TestMapColType_VectorPgNoPgvector(t *testing.T) {
+	got := mapColType("vector(384)", true, false)
+	if got != "JSONB" {
+		t.Errorf("vector on PG without pgvector: got %q, want JSONB", got)
+	}
+}
+
+func TestMapColType_VectorPgvector(t *testing.T) {
+	got := mapColType("vector(384)", true, true)
+	if got != "vector(384)" {
+		t.Errorf("vector on PG with pgvector: got %q, want vector(384)", got)
+	}
+}
+
+// ── CreateExtTables with vector column ──────────────────────────────────────
+
+func TestCreateExtTables_VectorColumn(t *testing.T) {
+	db := newSchemaTestDB(t)
+	m := newMemExtDataStore()
+	ctx := context.Background()
+
+	tables := map[string]TableDef{
+		"docs": {
+			Columns: map[string]string{"title": "text", "embedding": "vector(384)"},
+		},
+	}
+	if err := CreateExtTables(ctx, db, false, storesWithExtData(m), "vec-pkg", tables, false); err != nil {
+		t.Fatalf("CreateExtTables: %v", err)
+	}
+
+	// Verify table exists and embedding is TEXT (SQLite fallback).
+	_, err := db.ExecContext(ctx,
+		`INSERT INTO ext_vec_pkg_docs (id, title, embedding) VALUES ('1', 'test', '[0.1,0.2]')`)
+	if err != nil {
+		t.Errorf("table not created or wrong schema: %v", err)
+	}
+}

server/handlers/extensions.go

@@ -17,13 +17,19 @@ import (
 
 // ExtensionHandler serves extension management endpoints.
 type ExtensionHandler struct {
-	stores store.Stores
+	stores       store.Stores
+	capabilities map[string]bool
 }
 
 func NewExtensionHandler(stores store.Stores) *ExtensionHandler {
 	return &ExtensionHandler{stores: stores}
 }
 
+// SetCapabilities stores the detected environment capabilities map.
+func (h *ExtensionHandler) SetCapabilities(caps map[string]bool) {
+	h.capabilities = caps
+}
+
 // validTiers is the set of accepted extension tier values.
 var validTiers = map[string]bool{
 	models.ExtTierBrowser:  true,
@@ -231,7 +237,7 @@ func (h *ExtensionHandler) AdminInstallExtension(c *gin.Context) {
 	SyncManifestPermissions(c, h.stores, pkg.ID, manifestMap)
 
 	if tables, ok := ParseDBTables(manifestMap); ok {
-		if err := CreateExtTables(c.Request.Context(), database.DB, database.IsPostgres(), h.stores, pkg.ID, tables); err != nil {
+		if err := CreateExtTables(c.Request.Context(), database.DB, database.IsPostgres(), h.stores, pkg.ID, tables, h.capabilities["pgvector"]); err != nil {
 			log.Printf("[ext-db] schema create failed for %s: %v", pkg.ID, err)
 		}
 	}

server/handlers/packages.go

@@ -553,7 +553,7 @@ func (h *PackageHandler) registerPackage(c *gin.Context, pkgID string, mInfo *Ma
 // and runs schema migrations.
 func (h *PackageHandler) applySchemaAndPermissions(c *gin.Context, pkgID string, mInfo *ManifestInfo, manifest map[string]any, existing *store.PackageRegistration) error {
 	if tables, ok := ParseDBTables(manifest); ok {
-		if err := CreateExtTables(c.Request.Context(), database.DB, database.IsPostgres(), h.stores, pkgID, tables); err != nil {
+		if err := CreateExtTables(c.Request.Context(), database.DB, database.IsPostgres(), h.stores, pkgID, tables, h.capabilities["pgvector"]); err != nil {
 			log.Printf("[ext-db] schema create failed for %s: %v", pkgID, err)
 		}
 	}
@@ -1047,7 +1047,7 @@ func (h *PackageHandler) UpdatePackage(c *gin.Context) {
 	// 6. Additive schema migration
 	var schemaChanges []string
 	if tables, ok := ParseDBTables(manifest); ok {
-		changes, err := MigrateExtTables(ctx, database.DB, database.IsPostgres(), h.stores, pkgID, tables)
+		changes, err := MigrateExtTables(ctx, database.DB, database.IsPostgres(), h.stores, pkgID, tables, h.capabilities["pgvector"])
 		if err != nil {
 			c.JSON(http.StatusInternalServerError, gin.H{"error": "schema migration failed: " + err.Error()})
 			return

server/handlers/packages_bundled.go

@@ -196,7 +196,7 @@ func installBundledPackage(pkgPath, packagesDir string, stores store.Stores, run
 
 	// Create namespaced DB tables declared in the manifest
 	if tables, ok := ParseDBTables(manifest); ok {
-		if err := CreateExtTables(ctx, database.DB, database.IsPostgres(), stores, pkgID, tables); err != nil {
+		if err := CreateExtTables(ctx, database.DB, database.IsPostgres(), stores, pkgID, tables, detectedCaps["pgvector"]); err != nil {
 			log.Printf("[bundled] [ext-db] schema create failed for %s: %v", pkgID, err)
 		}
 	}

server/handlers/packages_update_test.go

@@ -267,7 +267,7 @@ func TestUpdatePackage_SchemaAddColumn(t *testing.T) {
 			},
 		},
 	})
-	CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "schema-pkg", tables)
+	CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "schema-pkg", tables, false)
 
 	// Insert a row to verify data preservation
 	physical := extPhysicalTable("schema-pkg", "items")

server/handlers/upgrade_test.go

@@ -42,7 +42,7 @@ func TestUpgrade_SchemaAddIndex(t *testing.T) {
 			},
 		},
 	})
-	CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "idx-pkg", tables)
+	CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "idx-pkg", tables, false)
 
 	// Insert a row
 	physical := extPhysicalTable("idx-pkg", "events")
@@ -116,7 +116,7 @@ func TestUpgrade_SchemaAddColumnIdempotent(t *testing.T) {
 			},
 		},
 	})
-	CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "idem-pkg", tables)
+	CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "idem-pkg", tables, false)
 
 	// Insert a row
 	physical := extPhysicalTable("idem-pkg", "items")
@@ -183,7 +183,7 @@ func TestUpgrade_SchemaMultiColumnAdd(t *testing.T) {
 			},
 		},
 	})
-	CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "multi-pkg", tables)
+	CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "multi-pkg", tables, false)
 
 	// Insert a row
 	physical := extPhysicalTable("multi-pkg", "records")
@@ -521,7 +521,7 @@ func TestUpgrade_MigrateExtTables_NewTable(t *testing.T) {
 		},
 	}
 
-	changes, err := MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-new", newTables)
+	changes, err := MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-new", newTables, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -559,7 +559,7 @@ func TestUpgrade_MigrateExtTables_AddColumnPreservesRows(t *testing.T) {
 			},
 		},
 	})
-	CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-col", tables)
+	CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-col", tables, false)
 
 	physical := extPhysicalTable("migrate-col", "records")
 	_, err := database.TestDB.ExecContext(ctx,
@@ -574,7 +574,7 @@ func TestUpgrade_MigrateExtTables_AddColumnPreservesRows(t *testing.T) {
 			Columns: map[string]string{"title": "text", "status": "text"},
 		},
 	}
-	changes, err := MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-col", newTables)
+	changes, err := MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-col", newTables, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -626,7 +626,7 @@ func TestUpgrade_MigrateExtTables_IndexIdempotent(t *testing.T) {
 			},
 		},
 	})
-	CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-idx", tables)
+	CreateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-idx", tables, false)
 
 	// Migrate with same index — should not fail
 	newTables := map[string]TableDef{
@@ -635,13 +635,13 @@ func TestUpgrade_MigrateExtTables_IndexIdempotent(t *testing.T) {
 			Indexes: [][]string{{"category"}},
 		},
 	}
-	_, err := MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-idx", newTables)
+	_, err := MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-idx", newTables, false)
 	if err != nil {
 		t.Fatal("idempotent index migration should not fail:", err)
 	}
 
 	// Run again — still no error
-	_, err = MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-idx", newTables)
+	_, err = MigrateExtTables(ctx, database.TestDB, database.IsPostgres(), stores, "migrate-idx", newTables, false)
 	if err != nil {
 		t.Fatal("second idempotent index migration should not fail:", err)
 	}