Changeset 0.28.8 (#194)
This commit is contained in:
@@ -25,50 +25,88 @@ const catalogColsMC = `mc.id, mc.provider_config_id, mc.model_id, mc.display_nam
|
||||
|
||||
// UpsertFromSync bulk-inserts or updates catalog entries from a provider API fetch.
|
||||
// New models default to 'disabled' visibility (secure by default).
|
||||
//
|
||||
// Runs in a single transaction: one SELECT to identify existing models,
|
||||
// then N upserts via a prepared INSERT ON CONFLICT statement, one COMMIT.
|
||||
// Previously this was N×2 auto-committed queries (SELECT + INSERT/UPDATE per model),
|
||||
// which caused 300+ round-trips and WAL flushes on a typical Venice sync.
|
||||
func (s *CatalogStore) UpsertFromSync(ctx context.Context, providerConfigID string, entries []store.CatalogSyncEntry) (added, updated int, err error) {
|
||||
if len(entries) == 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
tx, err := DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback() // no-op after commit
|
||||
|
||||
// Pre-fetch existing model_ids for this provider to track added vs updated.
|
||||
existing := make(map[string]bool)
|
||||
rows, err := tx.QueryContext(ctx,
|
||||
"SELECT model_id FROM model_catalog WHERE provider_config_id = $1",
|
||||
providerConfigID)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("list existing: %w", err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var mid string
|
||||
if err := rows.Scan(&mid); err != nil {
|
||||
rows.Close()
|
||||
return 0, 0, fmt.Errorf("scan existing: %w", err)
|
||||
}
|
||||
existing[mid] = true
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, 0, fmt.Errorf("list existing: %w", err)
|
||||
}
|
||||
|
||||
// Prepare the upsert statement once, execute N times.
|
||||
// INSERT creates new rows with visibility='disabled' (secure by default).
|
||||
// ON CONFLICT updates metadata but preserves visibility (admin controls that).
|
||||
stmt, err := tx.PrepareContext(ctx, `
|
||||
INSERT INTO model_catalog (provider_config_id, model_id, display_name,
|
||||
model_type, capabilities, pricing, visibility, last_synced_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'disabled', $7)
|
||||
ON CONFLICT (provider_config_id, model_id) DO UPDATE SET
|
||||
display_name = EXCLUDED.display_name,
|
||||
model_type = EXCLUDED.model_type,
|
||||
capabilities = EXCLUDED.capabilities,
|
||||
pricing = EXCLUDED.pricing,
|
||||
last_synced_at = EXCLUDED.last_synced_at`)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("prepare upsert: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
now := time.Now()
|
||||
for _, e := range entries {
|
||||
capsJSON := ToJSON(e.Capabilities)
|
||||
pricingJSON := ToJSON(e.Pricing) // nil → "{}", always valid JSONB
|
||||
pricingJSON := ToJSON(e.Pricing)
|
||||
|
||||
// Normalize model type: empty → "chat" (the default)
|
||||
modelType := e.ModelType
|
||||
if modelType == "" {
|
||||
modelType = "chat"
|
||||
}
|
||||
|
||||
var existingID string
|
||||
err := DB.QueryRowContext(ctx,
|
||||
"SELECT id FROM model_catalog WHERE provider_config_id = $1 AND model_id = $2",
|
||||
providerConfigID, e.ModelID,
|
||||
).Scan(&existingID)
|
||||
if _, err := stmt.ExecContext(ctx,
|
||||
providerConfigID, e.ModelID, e.DisplayName,
|
||||
modelType, capsJSON, pricingJSON, now,
|
||||
); err != nil {
|
||||
return added, updated, fmt.Errorf("upsert %s: %w", e.ModelID, err)
|
||||
}
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
// Insert new (disabled by default)
|
||||
_, err = DB.ExecContext(ctx, `
|
||||
INSERT INTO model_catalog (provider_config_id, model_id, display_name, model_type,
|
||||
capabilities, pricing, visibility, last_synced_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, 'disabled', $7)`,
|
||||
providerConfigID, e.ModelID, e.DisplayName, modelType, capsJSON, pricingJSON, now)
|
||||
if err != nil {
|
||||
return added, updated, fmt.Errorf("insert %s: %w", e.ModelID, err)
|
||||
}
|
||||
added++
|
||||
} else if err == nil {
|
||||
// Update existing (preserve visibility)
|
||||
_, err = DB.ExecContext(ctx, `
|
||||
UPDATE model_catalog SET display_name = $1, model_type = $2, capabilities = $3,
|
||||
pricing = $4, last_synced_at = $5
|
||||
WHERE id = $6`,
|
||||
e.DisplayName, modelType, capsJSON, pricingJSON, now, existingID)
|
||||
if err != nil {
|
||||
return added, updated, fmt.Errorf("update %s: %w", e.ModelID, err)
|
||||
}
|
||||
if existing[e.ModelID] {
|
||||
updated++
|
||||
} else {
|
||||
return added, updated, fmt.Errorf("check %s: %w", e.ModelID, err)
|
||||
added++
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, 0, fmt.Errorf("commit: %w", err)
|
||||
}
|
||||
return added, updated, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -25,50 +25,87 @@ const catalogColsMC = `mc.id, mc.provider_config_id, mc.model_id, mc.display_nam
|
||||
|
||||
// UpsertFromSync bulk-inserts or updates catalog entries from a provider API fetch.
|
||||
// New models default to 'disabled' visibility (secure by default).
|
||||
//
|
||||
// Runs in a single transaction: one SELECT to identify existing models,
|
||||
// then N upserts via a prepared INSERT ON CONFLICT statement, one COMMIT.
|
||||
// SQLite with WAL benefits enormously — one journal cycle instead of N.
|
||||
func (s *CatalogStore) UpsertFromSync(ctx context.Context, providerConfigID string, entries []store.CatalogSyncEntry) (added, updated int, err error) {
|
||||
now := time.Now()
|
||||
if len(entries) == 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
tx, err := DB.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("begin tx: %w", err)
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
// Pre-fetch existing model_ids for this provider to track added vs updated.
|
||||
existing := make(map[string]bool)
|
||||
rows, err := tx.QueryContext(ctx,
|
||||
"SELECT model_id FROM model_catalog WHERE provider_config_id = ?",
|
||||
providerConfigID)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("list existing: %w", err)
|
||||
}
|
||||
for rows.Next() {
|
||||
var mid string
|
||||
if err := rows.Scan(&mid); err != nil {
|
||||
rows.Close()
|
||||
return 0, 0, fmt.Errorf("scan existing: %w", err)
|
||||
}
|
||||
existing[mid] = true
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, 0, fmt.Errorf("list existing: %w", err)
|
||||
}
|
||||
|
||||
// Prepare the upsert statement once, execute N times.
|
||||
// SQLite needs an explicit id value (TEXT PK, no auto-generation).
|
||||
// On conflict the id is preserved — only metadata columns update.
|
||||
stmt, err := tx.PrepareContext(ctx, `
|
||||
INSERT INTO model_catalog (id, provider_config_id, model_id, display_name,
|
||||
model_type, capabilities, pricing, visibility, last_synced_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, 'disabled', ?)
|
||||
ON CONFLICT (provider_config_id, model_id) DO UPDATE SET
|
||||
display_name = excluded.display_name,
|
||||
model_type = excluded.model_type,
|
||||
capabilities = excluded.capabilities,
|
||||
pricing = excluded.pricing,
|
||||
last_synced_at = excluded.last_synced_at`)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("prepare upsert: %w", err)
|
||||
}
|
||||
defer stmt.Close()
|
||||
|
||||
now := time.Now().UTC().Format("2006-01-02 15:04:05")
|
||||
for _, e := range entries {
|
||||
capsJSON := ToJSON(e.Capabilities)
|
||||
pricingJSON := ToJSON(e.Pricing) // nil → "{}", always valid JSONB
|
||||
pricingJSON := ToJSON(e.Pricing)
|
||||
|
||||
// Normalize model type: empty → "chat" (the default)
|
||||
modelType := e.ModelType
|
||||
if modelType == "" {
|
||||
modelType = "chat"
|
||||
}
|
||||
|
||||
var existingID string
|
||||
err := DB.QueryRowContext(ctx,
|
||||
"SELECT id FROM model_catalog WHERE provider_config_id = ? AND model_id = ?",
|
||||
providerConfigID, e.ModelID,
|
||||
).Scan(&existingID)
|
||||
if _, err := stmt.ExecContext(ctx,
|
||||
store.NewID(), providerConfigID, e.ModelID, e.DisplayName,
|
||||
modelType, capsJSON, pricingJSON, now,
|
||||
); err != nil {
|
||||
return added, updated, fmt.Errorf("upsert %s: %w", e.ModelID, err)
|
||||
}
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
// Insert new (disabled by default)
|
||||
_, err = DB.ExecContext(ctx, `
|
||||
INSERT INTO model_catalog (id, provider_config_id, model_id, display_name, model_type,
|
||||
capabilities, pricing, visibility, last_synced_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, 'disabled', ?)`,
|
||||
store.NewID(), providerConfigID, e.ModelID, e.DisplayName, modelType, capsJSON, pricingJSON, now)
|
||||
if err != nil {
|
||||
return added, updated, fmt.Errorf("insert %s: %w", e.ModelID, err)
|
||||
}
|
||||
added++
|
||||
} else if err == nil {
|
||||
// Update existing (preserve visibility)
|
||||
_, err = DB.ExecContext(ctx, `
|
||||
UPDATE model_catalog SET display_name = ?, model_type = ?, capabilities = ?,
|
||||
pricing = ?, last_synced_at = ?
|
||||
WHERE id = ?`,
|
||||
e.DisplayName, modelType, capsJSON, pricingJSON, now, existingID)
|
||||
if err != nil {
|
||||
return added, updated, fmt.Errorf("update %s: %w", e.ModelID, err)
|
||||
}
|
||||
if existing[e.ModelID] {
|
||||
updated++
|
||||
} else {
|
||||
return added, updated, fmt.Errorf("check %s: %w", e.ModelID, err)
|
||||
added++
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, 0, fmt.Errorf("commit: %w", err)
|
||||
}
|
||||
return added, updated, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user