Changeset 0.28.4 (#190)
This commit is contained in:
471
surfaces/icd-test-runner/js/tier-providers.js
Normal file
471
surfaces/icd-test-runner/js/tier-providers.js
Normal file
@@ -0,0 +1,471 @@
|
||||
/**
|
||||
* ICD Test Runner — Provider Tier
|
||||
* Three-tier provider CRUD (global → team → BYOK) with live SSE completions.
|
||||
*/
|
||||
(function () {
|
||||
'use strict';
|
||||
var T = window.ICD;
|
||||
if (!T) return;
|
||||
|
||||
var PROVIDER_DEFAULTS = {
|
||||
openai: 'https://api.openai.com/v1',
|
||||
anthropic: 'https://api.anthropic.com',
|
||||
venice: 'https://api.venice.ai/api/v1',
|
||||
openrouter: 'https://openrouter.ai/api/v1'
|
||||
};
|
||||
T.PROVIDER_DEFAULTS = PROVIDER_DEFAULTS;
|
||||
|
||||
// ─── SSE Completion Helper ──────────────────────────────────
|
||||
|
||||
T.testCompletion = async function (channelId, model, providerConfigId, label) {
|
||||
var token = await T.getAuthToken();
|
||||
var resp = await fetch(T.base + '/api/v1/chat/completions', {
|
||||
method: 'POST',
|
||||
headers: { 'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json' },
|
||||
credentials: 'same-origin',
|
||||
body: JSON.stringify({
|
||||
channel_id: channelId,
|
||||
content: 'Reply with exactly: ICD_TEST_OK',
|
||||
model: model,
|
||||
provider_config_id: providerConfigId,
|
||||
tools_enabled: false
|
||||
})
|
||||
});
|
||||
|
||||
T.assert(resp.ok, label + ': completion returned HTTP ' + resp.status);
|
||||
T.assert(resp.headers.get('content-type').indexOf('text/event-stream') !== -1,
|
||||
label + ': expected SSE, got ' + resp.headers.get('content-type'));
|
||||
|
||||
var reader = resp.body.getReader();
|
||||
var decoder = new TextDecoder();
|
||||
var content = '';
|
||||
var gotFinish = false;
|
||||
var finishData = null;
|
||||
var buf = '';
|
||||
|
||||
while (true) {
|
||||
var chunk = await reader.read();
|
||||
if (chunk.done) break;
|
||||
buf += decoder.decode(chunk.value, { stream: true });
|
||||
var lines = buf.split('\n');
|
||||
buf = lines.pop();
|
||||
for (var i = 0; i < lines.length; i++) {
|
||||
var line = lines[i].trim();
|
||||
if (!line) continue;
|
||||
if (line === 'data: [DONE]') { gotFinish = true; continue; }
|
||||
if (line.indexOf('event:') === 0) continue;
|
||||
if (line.indexOf('data:') === 0) {
|
||||
var raw = line.slice(5).trim();
|
||||
try {
|
||||
var evt = JSON.parse(raw);
|
||||
// Handler emits OpenAI-format: choices[0].delta.content
|
||||
if (evt.choices && evt.choices[0] && evt.choices[0].delta && evt.choices[0].delta.content) {
|
||||
content += evt.choices[0].delta.content;
|
||||
} else if (evt.content) {
|
||||
content += evt.content; // fallback for non-OpenAI format
|
||||
}
|
||||
if (evt.message_id) { gotFinish = true; finishData = evt; }
|
||||
} catch (e) { /* skip */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
T.assert(content.length > 0, label + ': no content in stream');
|
||||
T.assert(gotFinish, label + ': no finish event');
|
||||
return { content: content, finish: finishData };
|
||||
};
|
||||
|
||||
// ─── Model Selection ────────────────────────────────────────
|
||||
|
||||
T.pickCheapestChat = function (models) {
|
||||
var skipPatterns = ['embed', 'image', 'dall-e', 'tts', 'whisper', 'moderation'];
|
||||
var chatModels = models.filter(function (m) {
|
||||
var t = m.model_type || m.type || '';
|
||||
if (t === 'embedding' || t === 'image') return false;
|
||||
// When type is missing, exclude by model ID pattern
|
||||
var mid = (m.model_id || m.id || '').toLowerCase();
|
||||
for (var i = 0; i < skipPatterns.length; i++) {
|
||||
if (mid.indexOf(skipPatterns[i]) !== -1) return false;
|
||||
}
|
||||
return (t === 'chat' || t === 'text' || t === '');
|
||||
});
|
||||
if (chatModels.length === 0) return models[0] || null;
|
||||
chatModels.sort(function (a, b) {
|
||||
return (a.output_price_per_m || a.output_price || 999) - (b.output_price_per_m || b.output_price || 999);
|
||||
});
|
||||
var cheapNames = ['mini', 'haiku', 'flash', 'fast', 'nano', 'lite', 'small'];
|
||||
for (var i = 0; i < chatModels.length; i++) {
|
||||
var mid = (chatModels[i].model_id || chatModels[i].id || '').toLowerCase();
|
||||
for (var j = 0; j < cheapNames.length; j++) {
|
||||
if (mid.indexOf(cheapNames[j]) !== -1) return chatModels[i];
|
||||
}
|
||||
}
|
||||
return chatModels[0];
|
||||
};
|
||||
|
||||
T.runProviders = async function () {
|
||||
if (!T.providerSetup.configured || !T.providerSetup.apiKey) {
|
||||
await T.test('provider', 'setup', 'SKIP — no provider configured', async function () {
|
||||
throw new Error('Configure a provider (type + API key) in the panel above, then run again');
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
var tag = 'icd-prov-' + Date.now();
|
||||
var prov = T.providerSetup.provider;
|
||||
var key = T.providerSetup.apiKey;
|
||||
var endpoint = T.providerSetup.endpoint || PROVIDER_DEFAULTS[prov] || '';
|
||||
|
||||
// Track IDs for scoping assertions
|
||||
var globalConfigId = null;
|
||||
var globalModel = null;
|
||||
var teamId = null;
|
||||
var teamConfigId = null;
|
||||
var teamModel = null;
|
||||
var byokConfigId = null;
|
||||
var byokModel = null;
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// Tier 1: Global (admin)
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
if (T.user.role === 'admin') {
|
||||
await T.test('provider', 'global', 'POST /admin/configs (create)', async function () {
|
||||
var d = await T.apiPost('/admin/configs', {
|
||||
name: tag + '-global',
|
||||
provider: prov,
|
||||
endpoint: endpoint,
|
||||
api_key: key,
|
||||
is_private: false,
|
||||
config: {},
|
||||
headers: {},
|
||||
settings: {}
|
||||
});
|
||||
T.assertHasKey(d, 'id', 'global-config');
|
||||
globalConfigId = d.id;
|
||||
T.registerCleanup(function () { if (globalConfigId) return T.safeDelete('/admin/configs/' + globalConfigId); });
|
||||
});
|
||||
|
||||
if (globalConfigId) {
|
||||
await T.test('provider', 'global', 'GET /admin/configs/:id (read)', async function () {
|
||||
var d = await T.apiGet('/admin/configs');
|
||||
var configs = d.configs || d.data || d;
|
||||
T.assert(Array.isArray(configs), 'expected configs array');
|
||||
var found = configs.find(function (c) { return c.id === globalConfigId; });
|
||||
T.assert(found, 'created config not in list');
|
||||
T.assert(found.has_key === true, 'has_key should be true');
|
||||
// Key must be redacted
|
||||
T.assert(!found.api_key, 'api_key should be redacted');
|
||||
});
|
||||
|
||||
await T.test('provider', 'global', 'PUT /admin/configs/:id (update name)', async function () {
|
||||
var d = await T.apiPut('/admin/configs/' + globalConfigId, { name: tag + '-global-renamed' });
|
||||
T.assert(typeof d === 'object', 'expected object');
|
||||
});
|
||||
|
||||
await T.test('provider', 'global', 'POST /admin/models/fetch (sync catalog)', async function () {
|
||||
var d = await T.apiPost('/admin/models/fetch', { provider_config_id: globalConfigId });
|
||||
T.assert(typeof d === 'object', 'expected object');
|
||||
T.assert(d.total !== undefined || d.added !== undefined || d.message,
|
||||
'expected sync result, got keys: ' + Object.keys(d).join(', '));
|
||||
});
|
||||
|
||||
// Newly synced models may default to visibility=disabled — enable them
|
||||
await T.test('provider', 'global', 'PUT /admin/models/bulk (enable visibility)', async function () {
|
||||
var d = await T.apiPut('/admin/models/bulk', {
|
||||
provider_config_id: globalConfigId,
|
||||
visibility: 'enabled'
|
||||
});
|
||||
T.assert(typeof d === 'object', 'expected object');
|
||||
});
|
||||
|
||||
await T.test('provider', 'global', 'GET /models/enabled (global visible)', async function () {
|
||||
var d = await T.apiGet('/models/enabled');
|
||||
var models = d.models || d.data || d;
|
||||
T.assert(Array.isArray(models), 'expected models array');
|
||||
var ours = models.filter(function (m) { return m.provider_config_id === globalConfigId; });
|
||||
if (ours.length === 0) {
|
||||
// Fall back to admin catalog — models may be visible there even if not in user endpoint
|
||||
var catalog = await T.apiGet('/admin/models');
|
||||
var allModels = catalog.models || catalog.data || catalog;
|
||||
if (Array.isArray(allModels)) {
|
||||
ours = allModels.filter(function (m) { return m.provider_config_id === globalConfigId; });
|
||||
}
|
||||
}
|
||||
T.assert(ours.length > 0, 'no models visible from global config ' + globalConfigId);
|
||||
globalModel = T.pickCheapestChat(ours);
|
||||
T.assert(globalModel, 'no chat model found in global config');
|
||||
});
|
||||
|
||||
if (globalModel) {
|
||||
var globalChannelId = null;
|
||||
|
||||
await T.test('provider', 'global', 'completion via global provider', async function () {
|
||||
// Create channel, run completion, verify
|
||||
var ch = await T.apiPost('/channels', {
|
||||
title: tag + '-global-test',
|
||||
type: 'direct',
|
||||
model: (globalModel.model_id || globalModel.id),
|
||||
provider_config_id: globalConfigId
|
||||
});
|
||||
globalChannelId = ch.id;
|
||||
T.registerCleanup(function () { if (globalChannelId) return T.safeDelete('/channels/' + globalChannelId); });
|
||||
|
||||
var result = await T.testCompletion(globalChannelId, (globalModel.model_id || globalModel.id), globalConfigId, 'global');
|
||||
T.assert(result.content.length > 0, 'completion produced no content');
|
||||
});
|
||||
|
||||
if (globalChannelId) {
|
||||
await T.test('provider', 'global', 'GET /channels/:id/path (message persisted)', async function () {
|
||||
var d = await T.apiGet('/channels/' + globalChannelId + '/path');
|
||||
var msgs = d.messages || d.data || d;
|
||||
T.assert(Array.isArray(msgs), 'expected messages array');
|
||||
// Should have user + assistant messages
|
||||
var roles = msgs.map(function (m) { return m.role; });
|
||||
T.assert(roles.indexOf('user') !== -1, 'missing user message');
|
||||
T.assert(roles.indexOf('assistant') !== -1, 'missing assistant message');
|
||||
});
|
||||
|
||||
await T.test('provider', 'global', 'DELETE test channel', async function () {
|
||||
await T.safeDelete('/channels/' + globalChannelId);
|
||||
globalChannelId = null;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// Tier 2: Team provider
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
if (T.user.role === 'admin') {
|
||||
await T.test('provider', 'team', 'POST /admin/teams (create test team)', async function () {
|
||||
var d = await T.apiPost('/admin/teams', { name: tag + '-team', description: 'Provider test team' });
|
||||
teamId = d.id;
|
||||
T.registerCleanup(function () { if (teamId) return T.safeDelete('/admin/teams/' + teamId); });
|
||||
// Add self as team admin
|
||||
try { await T.apiPost('/admin/teams/' + teamId + '/members', { user_id: T.user.id, role: 'admin' }); } catch (e) { /* auto-added */ }
|
||||
});
|
||||
|
||||
if (teamId) {
|
||||
await T.test('provider', 'team', 'POST /teams/:id/providers (create)', async function () {
|
||||
var d = await T.apiPost('/teams/' + teamId + '/providers', {
|
||||
name: tag + '-team-prov',
|
||||
provider: prov,
|
||||
endpoint: endpoint,
|
||||
api_key: key,
|
||||
config: {},
|
||||
headers: {},
|
||||
settings: {}
|
||||
});
|
||||
T.assertHasKey(d, 'id', 'team-config');
|
||||
teamConfigId = d.id;
|
||||
T.registerCleanup(function () { if (teamConfigId) return T.safeDelete('/teams/' + teamId + '/providers/' + teamConfigId); });
|
||||
});
|
||||
|
||||
if (teamConfigId) {
|
||||
await T.test('provider', 'team', 'GET /teams/:id/providers (list)', async function () {
|
||||
var d = await T.apiGet('/teams/' + teamId + '/providers');
|
||||
var configs = d.configs || d.data || d.providers || (Array.isArray(d) ? d : null);
|
||||
T.assert(Array.isArray(configs), 'expected configs array, got keys: ' + Object.keys(d).join(', '));
|
||||
var found = configs.find(function (c) { return c.id === teamConfigId; });
|
||||
T.assert(found, 'team config not in list');
|
||||
});
|
||||
|
||||
await T.test('provider', 'team', 'PUT /teams/:id/providers/:id (update)', async function () {
|
||||
var d = await T.apiPut('/teams/' + teamId + '/providers/' + teamConfigId, { name: tag + '-team-renamed' });
|
||||
T.assert(typeof d === 'object', 'expected object');
|
||||
});
|
||||
|
||||
await T.test('provider', 'team', 'GET /teams/:id/providers/:id/models (fetch)', async function () {
|
||||
var d = await T.apiGet('/teams/' + teamId + '/providers/' + teamConfigId + '/models');
|
||||
var models = d.models || d.data || d;
|
||||
T.assert(Array.isArray(models), 'expected models array');
|
||||
if (models.length > 0) {
|
||||
teamModel = T.pickCheapestChat(models);
|
||||
}
|
||||
});
|
||||
|
||||
// If no models from team-specific fetch, try /models/enabled filtered
|
||||
if (!teamModel) {
|
||||
await T.test('provider', 'team', 'GET /models/enabled (team model fallback)', async function () {
|
||||
var d = await T.apiGet('/models/enabled');
|
||||
var models = d.models || d.data || d;
|
||||
var ours = models.filter(function (m) { return m.provider_config_id === teamConfigId; });
|
||||
if (ours.length > 0) teamModel = T.pickCheapestChat(ours);
|
||||
T.assert(teamModel, 'no chat model found for team config');
|
||||
});
|
||||
}
|
||||
|
||||
if (teamModel) {
|
||||
var teamChannelId = null;
|
||||
|
||||
await T.test('provider', 'team', 'completion via team provider', async function () {
|
||||
var ch = await T.apiPost('/channels', {
|
||||
title: tag + '-team-test',
|
||||
type: 'direct',
|
||||
model: (teamModel.model_id || teamModel.id),
|
||||
provider_config_id: teamConfigId
|
||||
});
|
||||
teamChannelId = ch.id;
|
||||
T.registerCleanup(function () { if (teamChannelId) return T.safeDelete('/channels/' + teamChannelId); });
|
||||
|
||||
var result = await T.testCompletion(teamChannelId, (teamModel.model_id || teamModel.id), teamConfigId, 'team');
|
||||
T.assert(result.content.length > 0, 'completion produced no content');
|
||||
});
|
||||
|
||||
if (teamChannelId) {
|
||||
await T.test('provider', 'team', 'DELETE test channel', async function () {
|
||||
await T.safeDelete('/channels/' + teamChannelId);
|
||||
teamChannelId = null;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await T.test('provider', 'team', 'DELETE /teams/:id/providers/:id', async function () {
|
||||
await T.safeDelete('/teams/' + teamId + '/providers/' + teamConfigId);
|
||||
teamConfigId = null;
|
||||
});
|
||||
}
|
||||
|
||||
await T.test('provider', 'team', 'DELETE /admin/teams/:id (cleanup)', async function () {
|
||||
await T.safeDelete('/admin/teams/' + teamId);
|
||||
teamId = null;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ════════════════════════════════════════════════════════════
|
||||
// Tier 3: Personal BYOK
|
||||
// ════════════════════════════════════════════════════════════
|
||||
|
||||
// BYOK requires allow_user_byok policy — enable it if admin, skip if not
|
||||
var byokPolicyWas = null; // null = don't restore
|
||||
|
||||
if (T.user.role === 'admin') {
|
||||
await T.test('provider', 'byok', 'enable allow_user_byok policy', async function () {
|
||||
// Read current policy value
|
||||
try {
|
||||
var pub = await T.apiGet('/settings/public');
|
||||
var policies = pub.policies || {};
|
||||
byokPolicyWas = policies.allow_user_byok;
|
||||
} catch (e) { /* can't read — will try to set anyway */ }
|
||||
|
||||
// Enable BYOK
|
||||
await T.apiPut('/admin/settings/allow_user_byok', { value: 'true' });
|
||||
});
|
||||
}
|
||||
|
||||
await T.test('provider', 'byok', 'POST /api-configs (create)', async function () {
|
||||
var d = await T.apiPost('/api-configs', {
|
||||
name: tag + '-byok',
|
||||
provider: prov,
|
||||
endpoint: endpoint,
|
||||
api_key: key,
|
||||
config: {},
|
||||
headers: {},
|
||||
settings: {}
|
||||
});
|
||||
T.assertHasKey(d, 'id', 'byok-config');
|
||||
byokConfigId = d.id;
|
||||
T.registerCleanup(function () { if (byokConfigId) return T.safeDelete('/api-configs/' + byokConfigId); });
|
||||
});
|
||||
|
||||
if (byokConfigId) {
|
||||
await T.test('provider', 'byok', 'GET /api-configs/:id (read — key redacted)', async function () {
|
||||
var d = await T.apiGet('/api-configs/' + byokConfigId);
|
||||
T.assertHasKey(d, 'id', 'byok-config');
|
||||
// has_key may not be present on single-config GET (only on list) — check if present
|
||||
if (d.has_key !== undefined) T.assert(d.has_key === true, 'has_key should be true');
|
||||
T.assert(!d.api_key, 'api_key should be redacted in response');
|
||||
});
|
||||
|
||||
await T.test('provider', 'byok', 'PUT /api-configs/:id (update name)', async function () {
|
||||
var d = await T.apiPut('/api-configs/' + byokConfigId, { name: tag + '-byok-renamed' });
|
||||
T.assert(typeof d === 'object', 'expected object');
|
||||
});
|
||||
|
||||
await T.test('provider', 'byok', 'POST /api-configs/:id/models/fetch (sync)', async function () {
|
||||
var d = await T.apiPost('/api-configs/' + byokConfigId + '/models/fetch', {});
|
||||
T.assert(typeof d === 'object', 'expected object');
|
||||
});
|
||||
|
||||
await T.test('provider', 'byok', 'GET /api-configs/:id/models (list)', async function () {
|
||||
var d = await T.apiGet('/api-configs/' + byokConfigId + '/models');
|
||||
var models = d.models || d.data || d;
|
||||
T.assert(Array.isArray(models), 'expected models array');
|
||||
T.assert(models.length > 0, 'no models returned after fetch');
|
||||
byokModel = T.pickCheapestChat(models);
|
||||
T.assert(byokModel, 'no chat model found in BYOK config');
|
||||
});
|
||||
|
||||
if (byokModel) {
|
||||
var byokChannelId = null;
|
||||
|
||||
await T.test('provider', 'byok', 'completion via BYOK provider', async function () {
|
||||
var ch = await T.apiPost('/channels', {
|
||||
title: tag + '-byok-test',
|
||||
type: 'direct',
|
||||
model: (byokModel.model_id || byokModel.id),
|
||||
provider_config_id: byokConfigId
|
||||
});
|
||||
byokChannelId = ch.id;
|
||||
T.registerCleanup(function () { if (byokChannelId) return T.safeDelete('/channels/' + byokChannelId); });
|
||||
|
||||
var result = await T.testCompletion(byokChannelId, (byokModel.model_id || byokModel.id), byokConfigId, 'byok');
|
||||
T.assert(result.content.length > 0, 'completion produced no content');
|
||||
});
|
||||
|
||||
if (byokChannelId) {
|
||||
await T.test('provider', 'byok', 'GET /channels/:id/path (message persisted)', async function () {
|
||||
var d = await T.apiGet('/channels/' + byokChannelId + '/path');
|
||||
var msgs = d.messages || d.data || d;
|
||||
T.assert(Array.isArray(msgs), 'expected messages array');
|
||||
var roles = msgs.map(function (m) { return m.role; });
|
||||
T.assert(roles.indexOf('assistant') !== -1, 'missing assistant response');
|
||||
});
|
||||
|
||||
await T.test('provider', 'byok', 'DELETE test channel', async function () {
|
||||
await T.safeDelete('/channels/' + byokChannelId);
|
||||
byokChannelId = null;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ── Scoping isolation: BYOK should NOT appear in admin/configs ──
|
||||
if (T.user.role === 'admin') {
|
||||
await T.test('provider', 'isolation', 'BYOK not in /admin/configs', async function () {
|
||||
var d = await T.apiGet('/admin/configs');
|
||||
var configs = d.configs || d.data || d;
|
||||
if (Array.isArray(configs)) {
|
||||
var leaked = configs.find(function (c) { return c.id === byokConfigId; });
|
||||
T.assert(!leaked, 'BYOK config leaked into global admin configs list');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
await T.test('provider', 'byok', 'DELETE /api-configs/:id', async function () {
|
||||
await T.safeDelete('/api-configs/' + byokConfigId);
|
||||
byokConfigId = null;
|
||||
});
|
||||
|
||||
// After delete, should not appear in list
|
||||
await T.test('provider', 'byok', 'GET /api-configs (deleted config gone)', async function () {
|
||||
var d = await T.apiGet('/api-configs');
|
||||
var configs = d.configs || d.data || d;
|
||||
if (Array.isArray(configs)) {
|
||||
var ghost = configs.find(function (c) { return c.name && c.name.indexOf(tag) !== -1; });
|
||||
T.assert(!ghost, 'deleted BYOK config still in list');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Restore allow_user_byok to original value
|
||||
if (byokPolicyWas !== null && T.user.role === 'admin') {
|
||||
await T.test('provider', 'byok', 'restore allow_user_byok policy', async function () {
|
||||
await T.apiPut('/admin/settings/allow_user_byok', { value: byokPolicyWas || 'false' });
|
||||
});
|
||||
}
|
||||
};
|
||||
})();
|
||||
Reference in New Issue
Block a user