switchboard/core
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Releases 9 Wiki Activity

Feat v0.7.1 surface runner framework (#55)
All checks were successful
CI/CD / detect-changes (push) Successful in 3s
Details
CI/CD / test-frontend (push) Successful in 6s
Details
CI/CD / test-go-pg (push) Successful in 2m51s
Details
CI/CD / test-sqlite (push) Successful in 3m2s
Details
CI/CD / build-and-deploy (push) Successful in 1m26s
Details

Browse Source 
Co-authored-by: Jeffrey Smith <jasafpro@gmail.com>
Co-committed-by: Jeffrey Smith <jasafpro@gmail.com>
This commit was merged in pull request #55.
This commit is contained in:
Jeffrey Smith
2026-04-01 23:01:38 +00:00
committed by xcaliber
parent e916ed41ea
commit 829caa3b20
59 changed files with 2509 additions and 6525 deletions
Download Patch File Download Diff File Expand all files Collapse all files

256
packages/icd-test-runner/js/tier-providers.js
View File

@@ -6,6 +6,7 @@
'use strict';
var T = window.ICD;
if (!T) return;
if (!window.sw || !window.sw.testing) return;
var PROVIDER_DEFAULTS = {
openai: 'https://api.openai.com/v1',
@@ -26,7 +27,7 @@
return await T.apiPost(path, body);
} catch (e) {
if (retries > 0 && (e.status === 502 || e.status === 504)) {
console.log(' ⟳ got ' + e.status + ' from ' + path + ', retrying in 2s…');
console.log(' got ' + e.status + ' from ' + path + ', retrying in 2s...');
await new Promise(function (r) { setTimeout(r, 2000); });
return await apiPostRetry(path, body, retries - 1);
}
@@ -130,10 +131,10 @@
return chatModels[0];
};
T.runProviders = async function () {
sw.testing.suite('icd/providers', async function (s) {
if (!T.providerSetup.configured || !T.providerSetup.apiKey) {
await T.test('provider', 'setup', 'SKIP — no provider configured', async function () {
throw new Error('Configure a provider (type + API key) in the panel above, then run again');
s.test('setup: provider required', async function (t) {
t.skip('Configure a provider (type + API key) in the panel above, then run again');
});
return;
}
@@ -157,24 +158,29 @@
// ════════════════════════════════════════════════════════════
if (T.user.role === 'admin') {
await T.test('provider', 'global', 'POST /admin/configs (create)', async function () {
var d = await T.apiPost('/admin/configs', {
name: tag + '-global',
provider: prov,
endpoint: endpoint,
api_key: key,
is_private: false,
config: {},
headers: {},
settings: {}
});
T.assertHasKey(d, 'id', 'global-config');
globalConfigId = d.id;
T.registerCleanup(function () { if (globalConfigId) return T.safeDelete('/admin/configs/' + globalConfigId); });
s.test('global: POST /admin/configs (create)', async function (t) {
var cleanup = [];
try {
var d = await T.apiPost('/admin/configs', {
name: tag + '-global',
provider: prov,
endpoint: endpoint,
api_key: key,
is_private: false,
config: {},
headers: {},
settings: {}
});
T.assertHasKey(d, 'id', 'global-config');
globalConfigId = d.id;
cleanup.push(function () { if (globalConfigId) return T.safeDelete('/admin/configs/' + globalConfigId); });
} finally {
for (var i = 0; i < cleanup.length; i++) { try { await cleanup[i](); } catch (e) { /* ok */ } }
}
});
if (globalConfigId) {
await T.test('provider', 'global', 'GET /admin/configs/:id (read)', async function () {
s.test('global: GET /admin/configs/:id (read)', async function (t) {
var d = await T.apiGet('/admin/configs');
var configs = d.configs || d.data || d;
T.assert(Array.isArray(configs), 'expected configs array');
@@ -185,12 +191,12 @@
T.assert(!found.api_key, 'api_key should be redacted');
});
await T.test('provider', 'global', 'PUT /admin/configs/:id (update name)', async function () {
s.test('global: PUT /admin/configs/:id (update name)', async function (t) {
var d = await T.apiPut('/admin/configs/' + globalConfigId, { name: tag + '-global-renamed' });
T.assert(typeof d === 'object', 'expected object');
});
await T.test('provider', 'global', 'POST /admin/models/fetch (sync catalog)', async function () {
s.test('global: POST /admin/models/fetch (sync catalog)', async function (t) {
var d = await apiPostRetry('/admin/models/fetch', { provider_config_id: globalConfigId });
T.assert(typeof d === 'object', 'expected object');
T.assert(d.total !== undefined || d.added !== undefined || d.message,
@@ -198,7 +204,7 @@
});
// Newly synced models may default to visibility=disabled — enable them
await T.test('provider', 'global', 'PUT /admin/models/bulk (enable visibility)', async function () {
s.test('global: PUT /admin/models/bulk (enable visibility)', async function (t) {
var d = await T.apiPut('/admin/models/bulk', {
provider_config_id: globalConfigId,
visibility: 'enabled'
@@ -206,7 +212,7 @@
T.assert(typeof d === 'object', 'expected object');
});
await T.test('provider', 'global', 'GET /models/enabled (global visible)', async function () {
s.test('global: GET /models/enabled (global visible)', async function (t) {
var d = await T.apiGet('/models/enabled');
var models = d.models || d.data || d;
T.assert(Array.isArray(models), 'expected models array');
@@ -227,23 +233,28 @@
if (globalModel) {
var globalChannelId = null;
await T.test('provider', 'global', 'completion via global provider', async function () {
// Create channel, run completion, verify
var ch = await T.apiPost('/channels', {
title: tag + '-global-test',
type: 'direct',
model: (globalModel.model_id || globalModel.id),
provider_config_id: globalConfigId
});
globalChannelId = ch.id;
T.registerCleanup(function () { if (globalChannelId) return T.safeDelete('/channels/' + globalChannelId); });
s.test('global: completion via global provider', async function (t) {
var cleanup = [];
try {
// Create channel, run completion, verify
var ch = await T.apiPost('/channels', {
title: tag + '-global-test',
type: 'direct',
model: (globalModel.model_id || globalModel.id),
provider_config_id: globalConfigId
});
globalChannelId = ch.id;
cleanup.push(function () { if (globalChannelId) return T.safeDelete('/channels/' + globalChannelId); });
var result = await T.testCompletion(globalChannelId, (globalModel.model_id || globalModel.id), globalConfigId, 'global');
T.assert(result.content.length > 0, 'completion produced no content');
var result = await T.testCompletion(globalChannelId, (globalModel.model_id || globalModel.id), globalConfigId, 'global');
T.assert(result.content.length > 0, 'completion produced no content');
} finally {
for (var i = 0; i < cleanup.length; i++) { try { await cleanup[i](); } catch (e) { /* ok */ } }
}
});
if (globalChannelId) {
await T.test('provider', 'global', 'GET /channels/:id/path (message persisted)', async function () {
s.test('global: GET /channels/:id/path (message persisted)', async function (t) {
var d = await T.apiGet('/channels/' + globalChannelId + '/path');
var msgs = d.messages || d.data || d;
T.assert(Array.isArray(msgs), 'expected messages array');
@@ -253,7 +264,7 @@
T.assert(roles.indexOf('assistant') !== -1, 'missing assistant message');
});
await T.test('provider', 'global', 'DELETE test channel', async function () {
s.test('global: DELETE test channel', async function (t) {
await T.safeDelete('/channels/' + globalChannelId);
globalChannelId = null;
});
@@ -267,32 +278,42 @@
// ════════════════════════════════════════════════════════════
if (T.user.role === 'admin') {
await T.test('provider', 'team', 'POST /admin/teams (create test team)', async function () {
var d = await T.apiPost('/admin/teams', { name: tag + '-team', description: 'Provider test team' });
teamId = d.id;
T.registerCleanup(function () { if (teamId) return T.safeDelete('/admin/teams/' + teamId); });
// Add self as team admin
try { await T.apiPost('/admin/teams/' + teamId + '/members', { user_id: T.user.id, role: 'admin' }); } catch (e) { /* auto-added */ }
s.test('team: POST /admin/teams (create test team)', async function (t) {
var cleanup = [];
try {
var d = await T.apiPost('/admin/teams', { name: tag + '-team', description: 'Provider test team' });
teamId = d.id;
cleanup.push(function () { if (teamId) return T.safeDelete('/admin/teams/' + teamId); });
// Add self as team admin
try { await T.apiPost('/admin/teams/' + teamId + '/members', { user_id: T.user.id, role: 'admin' }); } catch (e) { /* auto-added */ }
} finally {
for (var i = 0; i < cleanup.length; i++) { try { await cleanup[i](); } catch (e) { /* ok */ } }
}
});
if (teamId) {
await T.test('provider', 'team', 'POST /teams/:id/providers (create)', async function () {
var d = await T.apiPost('/teams/' + teamId + '/providers', {
name: tag + '-team-prov',
provider: prov,
endpoint: endpoint,
api_key: key,
config: {},
headers: {},
settings: {}
});
T.assertHasKey(d, 'id', 'team-config');
teamConfigId = d.id;
T.registerCleanup(function () { if (teamConfigId) return T.safeDelete('/teams/' + teamId + '/providers/' + teamConfigId); });
s.test('team: POST /teams/:id/providers (create)', async function (t) {
var cleanup = [];
try {
var d = await T.apiPost('/teams/' + teamId + '/providers', {
name: tag + '-team-prov',
provider: prov,
endpoint: endpoint,
api_key: key,
config: {},
headers: {},
settings: {}
});
T.assertHasKey(d, 'id', 'team-config');
teamConfigId = d.id;
cleanup.push(function () { if (teamConfigId) return T.safeDelete('/teams/' + teamId + '/providers/' + teamConfigId); });
} finally {
for (var i = 0; i < cleanup.length; i++) { try { await cleanup[i](); } catch (e) { /* ok */ } }
}
});
if (teamConfigId) {
await T.test('provider', 'team', 'GET /teams/:id/providers (list)', async function () {
s.test('team: GET /teams/:id/providers (list)', async function (t) {
var d = await T.apiGet('/teams/' + teamId + '/providers');
var configs = d.configs || d.data || d.providers || (Array.isArray(d) ? d : null);
T.assert(Array.isArray(configs), 'expected configs array, got keys: ' + Object.keys(d).join(', '));
@@ -300,12 +321,12 @@
T.assert(found, 'team config not in list');
});
await T.test('provider', 'team', 'PUT /teams/:id/providers/:id (update)', async function () {
s.test('team: PUT /teams/:id/providers/:id (update)', async function (t) {
var d = await T.apiPut('/teams/' + teamId + '/providers/' + teamConfigId, { name: tag + '-team-renamed' });
T.assert(typeof d === 'object', 'expected object');
});
await T.test('provider', 'team', 'GET /teams/:id/providers/:id/models (fetch)', async function () {
s.test('team: GET /teams/:id/providers/:id/models (fetch)', async function (t) {
var d = await T.apiGet('/teams/' + teamId + '/providers/' + teamConfigId + '/models');
var models = d.models || d.data || d;
T.assert(Array.isArray(models), 'expected models array');
@@ -316,7 +337,7 @@
// If no models from team-specific fetch, try /models/enabled filtered
if (!teamModel) {
await T.test('provider', 'team', 'GET /models/enabled (team model fallback)', async function () {
s.test('team: GET /models/enabled (team model fallback)', async function (t) {
var d = await T.apiGet('/models/enabled');
var models = d.models || d.data || d;
var ours = models.filter(function (m) { return m.provider_config_id === teamConfigId; });
@@ -328,35 +349,40 @@
if (teamModel) {
var teamChannelId = null;
await T.test('provider', 'team', 'completion via team provider', async function () {
var ch = await T.apiPost('/channels', {
title: tag + '-team-test',
type: 'direct',
model: (teamModel.model_id || teamModel.id),
provider_config_id: teamConfigId
});
teamChannelId = ch.id;
T.registerCleanup(function () { if (teamChannelId) return T.safeDelete('/channels/' + teamChannelId); });
s.test('team: completion via team provider', async function (t) {
var cleanup = [];
try {
var ch = await T.apiPost('/channels', {
title: tag + '-team-test',
type: 'direct',
model: (teamModel.model_id || teamModel.id),
provider_config_id: teamConfigId
});
teamChannelId = ch.id;
cleanup.push(function () { if (teamChannelId) return T.safeDelete('/channels/' + teamChannelId); });
var result = await T.testCompletion(teamChannelId, (teamModel.model_id || teamModel.id), teamConfigId, 'team');
T.assert(result.content.length > 0, 'completion produced no content');
var result = await T.testCompletion(teamChannelId, (teamModel.model_id || teamModel.id), teamConfigId, 'team');
T.assert(result.content.length > 0, 'completion produced no content');
} finally {
for (var i = 0; i < cleanup.length; i++) { try { await cleanup[i](); } catch (e) { /* ok */ } }
}
});
if (teamChannelId) {
await T.test('provider', 'team', 'DELETE test channel', async function () {
s.test('team: DELETE test channel', async function (t) {
await T.safeDelete('/channels/' + teamChannelId);
teamChannelId = null;
});
}
}
await T.test('provider', 'team', 'DELETE /teams/:id/providers/:id', async function () {
s.test('team: DELETE /teams/:id/providers/:id', async function (t) {
await T.safeDelete('/teams/' + teamId + '/providers/' + teamConfigId);
teamConfigId = null;
});
}
await T.test('provider', 'team', 'DELETE /admin/teams/:id (cleanup)', async function () {
s.test('team: DELETE /admin/teams/:id (cleanup)', async function (t) {
await T.safeDelete('/admin/teams/' + teamId);
teamId = null;
});
@@ -371,7 +397,7 @@
var byokPolicyWas = null; // null = don't restore
if (T.user.role === 'admin') {
await T.test('provider', 'byok', 'enable allow_user_byok policy', async function () {
s.test('byok: enable allow_user_byok policy', async function (t) {
// Read current policy value
try {
var pub = await T.apiGet('/settings/public');
@@ -384,23 +410,28 @@
});
}
await T.test('provider', 'byok', 'POST /api-configs (create)', async function () {
var d = await T.apiPost('/api-configs', {
name: tag + '-byok',
provider: prov,
endpoint: endpoint,
api_key: key,
config: {},
headers: {},
settings: {}
});
T.assertHasKey(d, 'id', 'byok-config');
byokConfigId = d.id;
T.registerCleanup(function () { if (byokConfigId) return T.safeDelete('/api-configs/' + byokConfigId); });
s.test('byok: POST /api-configs (create)', async function (t) {
var cleanup = [];
try {
var d = await T.apiPost('/api-configs', {
name: tag + '-byok',
provider: prov,
endpoint: endpoint,
api_key: key,
config: {},
headers: {},
settings: {}
});
T.assertHasKey(d, 'id', 'byok-config');
byokConfigId = d.id;
cleanup.push(function () { if (byokConfigId) return T.safeDelete('/api-configs/' + byokConfigId); });
} finally {
for (var i = 0; i < cleanup.length; i++) { try { await cleanup[i](); } catch (e) { /* ok */ } }
}
});
if (byokConfigId) {
await T.test('provider', 'byok', 'GET /api-configs/:id (read — key redacted)', async function () {
s.test('byok: GET /api-configs/:id (read — key redacted)', async function (t) {
var d = await T.apiGet('/api-configs/' + byokConfigId);
T.assertHasKey(d, 'id', 'byok-config');
// has_key may not be present on single-config GET (only on list) — check if present
@@ -408,17 +439,17 @@
T.assert(!d.api_key, 'api_key should be redacted in response');
});
await T.test('provider', 'byok', 'PUT /api-configs/:id (update name)', async function () {
s.test('byok: PUT /api-configs/:id (update name)', async function (t) {
var d = await T.apiPut('/api-configs/' + byokConfigId, { name: tag + '-byok-renamed' });
T.assert(typeof d === 'object', 'expected object');
});
await T.test('provider', 'byok', 'POST /api-configs/:id/models/fetch (sync)', async function () {
s.test('byok: POST /api-configs/:id/models/fetch (sync)', async function (t) {
var d = await apiPostRetry('/api-configs/' + byokConfigId + '/models/fetch', {});
T.assert(typeof d === 'object', 'expected object');
});
await T.test('provider', 'byok', 'GET /api-configs/:id/models (list)', async function () {
s.test('byok: GET /api-configs/:id/models (list)', async function (t) {
var d = await T.apiGet('/api-configs/' + byokConfigId + '/models');
var models = d.models || d.data || d;
T.assert(Array.isArray(models), 'expected models array');
@@ -430,22 +461,27 @@
if (byokModel) {
var byokChannelId = null;
await T.test('provider', 'byok', 'completion via BYOK provider', async function () {
var ch = await T.apiPost('/channels', {
title: tag + '-byok-test',
type: 'direct',
model: (byokModel.model_id || byokModel.id),
provider_config_id: byokConfigId
});
byokChannelId = ch.id;
T.registerCleanup(function () { if (byokChannelId) return T.safeDelete('/channels/' + byokChannelId); });
s.test('byok: completion via BYOK provider', async function (t) {
var cleanup = [];
try {
var ch = await T.apiPost('/channels', {
title: tag + '-byok-test',
type: 'direct',
model: (byokModel.model_id || byokModel.id),
provider_config_id: byokConfigId
});
byokChannelId = ch.id;
cleanup.push(function () { if (byokChannelId) return T.safeDelete('/channels/' + byokChannelId); });
var result = await T.testCompletion(byokChannelId, (byokModel.model_id || byokModel.id), byokConfigId, 'byok');
T.assert(result.content.length > 0, 'completion produced no content');
var result = await T.testCompletion(byokChannelId, (byokModel.model_id || byokModel.id), byokConfigId, 'byok');
T.assert(result.content.length > 0, 'completion produced no content');
} finally {
for (var i = 0; i < cleanup.length; i++) { try { await cleanup[i](); } catch (e) { /* ok */ } }
}
});
if (byokChannelId) {
await T.test('provider', 'byok', 'GET /channels/:id/path (message persisted)', async function () {
s.test('byok: GET /channels/:id/path (message persisted)', async function (t) {
var d = await T.apiGet('/channels/' + byokChannelId + '/path');
var msgs = d.messages || d.data || d;
T.assert(Array.isArray(msgs), 'expected messages array');
@@ -453,7 +489,7 @@
T.assert(roles.indexOf('assistant') !== -1, 'missing assistant response');
});
await T.test('provider', 'byok', 'DELETE test channel', async function () {
s.test('byok: DELETE test channel', async function (t) {
await T.safeDelete('/channels/' + byokChannelId);
byokChannelId = null;
});
@@ -462,7 +498,7 @@
// ── Scoping isolation: BYOK should NOT appear in admin/configs ──
if (T.user.role === 'admin') {
await T.test('provider', 'isolation', 'BYOK not in /admin/configs', async function () {
s.test('isolation: BYOK not in /admin/configs', async function (t) {
var d = await T.apiGet('/admin/configs');
var configs = d.configs || d.data || d;
if (Array.isArray(configs)) {
@@ -472,13 +508,13 @@
});
}
await T.test('provider', 'byok', 'DELETE /api-configs/:id', async function () {
s.test('byok: DELETE /api-configs/:id', async function (t) {
await T.safeDelete('/api-configs/' + byokConfigId);
byokConfigId = null;
});
// After delete, should not appear in list
await T.test('provider', 'byok', 'GET /api-configs (deleted config gone)', async function () {
s.test('byok: GET /api-configs (deleted config gone)', async function (t) {
var d = await T.apiGet('/api-configs');
var configs = d.configs || d.data || d;
if (Array.isArray(configs)) {
@@ -490,9 +526,9 @@
// Restore allow_user_byok to original value
if (byokPolicyWas !== null && T.user.role === 'admin') {
await T.test('provider', 'byok', 'restore allow_user_byok policy', async function () {
s.test('byok: restore allow_user_byok policy', async function (t) {
await T.apiPut('/admin/settings/allow_user_byok', { value: byokPolicyWas || 'false' });
});
}
};
});
})();