499 lines
22 KiB
JavaScript
499 lines
22 KiB
JavaScript
/**
|
|
* ICD Test Runner — Provider Tier
|
|
* Three-tier provider CRUD (global → team → BYOK) with live SSE completions.
|
|
*/
|
|
(function () {
|
|
'use strict';
|
|
var T = window.ICD;
|
|
if (!T) return;
|
|
|
|
var PROVIDER_DEFAULTS = {
|
|
openai: 'https://api.openai.com/v1',
|
|
anthropic: 'https://api.anthropic.com',
|
|
venice: 'https://api.venice.ai/api/v1',
|
|
openrouter: 'https://openrouter.ai/api/v1'
|
|
};
|
|
T.PROVIDER_DEFAULTS = PROVIDER_DEFAULTS;
|
|
|
|
// ─── Retry Helper (upstream timeout resilience) ────────────
|
|
|
|
// apiPostRetry retries a POST once on 502/504. Slow upstream providers
|
|
// (especially Venice) can exceed the reverse proxy timeout on cold starts.
|
|
// A single retry with a short backoff is enough to ride out transient timeouts.
|
|
async function apiPostRetry(path, body, retries) {
|
|
if (retries === undefined) retries = 1;
|
|
try {
|
|
return await T.apiPost(path, body);
|
|
} catch (e) {
|
|
if (retries > 0 && (e.status === 502 || e.status === 504)) {
|
|
console.log(' ⟳ got ' + e.status + ' from ' + path + ', retrying in 2s…');
|
|
await new Promise(function (r) { setTimeout(r, 2000); });
|
|
return await apiPostRetry(path, body, retries - 1);
|
|
}
|
|
throw e;
|
|
}
|
|
}
|
|
|
|
// ─── SSE Completion Helper ──────────────────────────────────
|
|
|
|
T.testCompletion = async function (channelId, model, providerConfigId, label) {
|
|
var token = await T.getAuthToken();
|
|
var resp = await fetch(T.base + '/api/v1/chat/completions', {
|
|
method: 'POST',
|
|
headers: { 'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json' },
|
|
credentials: 'same-origin',
|
|
body: JSON.stringify({
|
|
channel_id: channelId,
|
|
content: 'Reply with exactly: ICD_TEST_OK',
|
|
model: model,
|
|
provider_config_id: providerConfigId,
|
|
tools_enabled: false
|
|
})
|
|
});
|
|
|
|
T.assert(resp.ok, label + ': completion returned HTTP ' + resp.status);
|
|
T.assert(resp.headers.get('content-type').indexOf('text/event-stream') !== -1,
|
|
label + ': expected SSE, got ' + resp.headers.get('content-type'));
|
|
|
|
var reader = resp.body.getReader();
|
|
var decoder = new TextDecoder();
|
|
var content = '';
|
|
var gotFinish = false;
|
|
var finishData = null;
|
|
var buf = '';
|
|
|
|
while (true) {
|
|
var chunk = await reader.read();
|
|
if (chunk.done) break;
|
|
buf += decoder.decode(chunk.value, { stream: true });
|
|
var lines = buf.split('\n');
|
|
buf = lines.pop();
|
|
for (var i = 0; i < lines.length; i++) {
|
|
var line = lines[i].trim();
|
|
if (!line) continue;
|
|
if (line === 'data: [DONE]') { gotFinish = true; continue; }
|
|
if (line.indexOf('event:') === 0) continue;
|
|
if (line.indexOf('data:') === 0) {
|
|
var raw = line.slice(5).trim();
|
|
try {
|
|
var evt = JSON.parse(raw);
|
|
// Handler emits OpenAI-format: choices[0].delta.content
|
|
// v0.31.2: Also capture reasoning_content (DeepSeek, etc.)
|
|
if (evt.choices && evt.choices[0] && evt.choices[0].delta) {
|
|
if (evt.choices[0].delta.content) {
|
|
content += evt.choices[0].delta.content;
|
|
}
|
|
if (evt.choices[0].delta.reasoning_content) {
|
|
content += evt.choices[0].delta.reasoning_content;
|
|
}
|
|
} else if (evt.content) {
|
|
content += evt.content; // fallback for non-OpenAI format
|
|
}
|
|
if (evt.message_id) { gotFinish = true; finishData = evt; }
|
|
} catch (e) { /* skip */ }
|
|
}
|
|
}
|
|
}
|
|
|
|
T.assert(content.length > 0, label + ': no content in stream');
|
|
T.assert(gotFinish, label + ': no finish event');
|
|
return { content: content, finish: finishData };
|
|
};
|
|
|
|
// ─── Model Selection ────────────────────────────────────────
|
|
|
|
T.pickCheapestChat = function (models) {
|
|
var skipPatterns = ['embed', 'image', 'dall-e', 'tts', 'whisper', 'moderation'];
|
|
var chatModels = models.filter(function (m) {
|
|
// v0.31.2: Skip E2EE models — require client-side decryption we don't support
|
|
if (m.supportsE2EE || m.supports_e2ee) return false;
|
|
var t = m.model_type || m.type || '';
|
|
if (t === 'embedding' || t === 'image') return false;
|
|
// When type is missing, exclude by model ID pattern
|
|
var mid = (m.model_id || m.id || '').toLowerCase();
|
|
for (var i = 0; i < skipPatterns.length; i++) {
|
|
if (mid.indexOf(skipPatterns[i]) !== -1) return false;
|
|
}
|
|
return (t === 'chat' || t === 'text' || t === '');
|
|
});
|
|
if (chatModels.length === 0) return models[0] || null;
|
|
chatModels.sort(function (a, b) {
|
|
return (a.output_price_per_m || a.output_price || 999) - (b.output_price_per_m || b.output_price || 999);
|
|
});
|
|
var cheapNames = ['mini', 'haiku', 'flash', 'fast', 'nano', 'lite', 'small'];
|
|
for (var i = 0; i < chatModels.length; i++) {
|
|
var mid = (chatModels[i].model_id || chatModels[i].id || '').toLowerCase();
|
|
for (var j = 0; j < cheapNames.length; j++) {
|
|
if (mid.indexOf(cheapNames[j]) !== -1) return chatModels[i];
|
|
}
|
|
}
|
|
return chatModels[0];
|
|
};
|
|
|
|
T.runProviders = async function () {
|
|
if (!T.providerSetup.configured || !T.providerSetup.apiKey) {
|
|
await T.test('provider', 'setup', 'SKIP — no provider configured', async function () {
|
|
throw new Error('Configure a provider (type + API key) in the panel above, then run again');
|
|
});
|
|
return;
|
|
}
|
|
|
|
var tag = 'icd-prov-' + Date.now();
|
|
var prov = T.providerSetup.provider;
|
|
var key = T.providerSetup.apiKey;
|
|
var endpoint = T.providerSetup.endpoint || PROVIDER_DEFAULTS[prov] || '';
|
|
|
|
// Track IDs for scoping assertions
|
|
var globalConfigId = null;
|
|
var globalModel = null;
|
|
var teamId = null;
|
|
var teamConfigId = null;
|
|
var teamModel = null;
|
|
var byokConfigId = null;
|
|
var byokModel = null;
|
|
|
|
// ════════════════════════════════════════════════════════════
|
|
// Tier 1: Global (admin)
|
|
// ════════════════════════════════════════════════════════════
|
|
|
|
if (T.user.role === 'admin') {
|
|
await T.test('provider', 'global', 'POST /admin/configs (create)', async function () {
|
|
var d = await T.apiPost('/admin/configs', {
|
|
name: tag + '-global',
|
|
provider: prov,
|
|
endpoint: endpoint,
|
|
api_key: key,
|
|
is_private: false,
|
|
config: {},
|
|
headers: {},
|
|
settings: {}
|
|
});
|
|
T.assertHasKey(d, 'id', 'global-config');
|
|
globalConfigId = d.id;
|
|
T.registerCleanup(function () { if (globalConfigId) return T.safeDelete('/admin/configs/' + globalConfigId); });
|
|
});
|
|
|
|
if (globalConfigId) {
|
|
await T.test('provider', 'global', 'GET /admin/configs/:id (read)', async function () {
|
|
var d = await T.apiGet('/admin/configs');
|
|
var configs = d.configs || d.data || d;
|
|
T.assert(Array.isArray(configs), 'expected configs array');
|
|
var found = configs.find(function (c) { return c.id === globalConfigId; });
|
|
T.assert(found, 'created config not in list');
|
|
T.assert(found.has_key === true, 'has_key should be true');
|
|
// Key must be redacted
|
|
T.assert(!found.api_key, 'api_key should be redacted');
|
|
});
|
|
|
|
await T.test('provider', 'global', 'PUT /admin/configs/:id (update name)', async function () {
|
|
var d = await T.apiPut('/admin/configs/' + globalConfigId, { name: tag + '-global-renamed' });
|
|
T.assert(typeof d === 'object', 'expected object');
|
|
});
|
|
|
|
await T.test('provider', 'global', 'POST /admin/models/fetch (sync catalog)', async function () {
|
|
var d = await apiPostRetry('/admin/models/fetch', { provider_config_id: globalConfigId });
|
|
T.assert(typeof d === 'object', 'expected object');
|
|
T.assert(d.total !== undefined || d.added !== undefined || d.message,
|
|
'expected sync result, got keys: ' + Object.keys(d).join(', '));
|
|
});
|
|
|
|
// Newly synced models may default to visibility=disabled — enable them
|
|
await T.test('provider', 'global', 'PUT /admin/models/bulk (enable visibility)', async function () {
|
|
var d = await T.apiPut('/admin/models/bulk', {
|
|
provider_config_id: globalConfigId,
|
|
visibility: 'enabled'
|
|
});
|
|
T.assert(typeof d === 'object', 'expected object');
|
|
});
|
|
|
|
await T.test('provider', 'global', 'GET /models/enabled (global visible)', async function () {
|
|
var d = await T.apiGet('/models/enabled');
|
|
var models = d.models || d.data || d;
|
|
T.assert(Array.isArray(models), 'expected models array');
|
|
var ours = models.filter(function (m) { return m.provider_config_id === globalConfigId; });
|
|
if (ours.length === 0) {
|
|
// Fall back to admin catalog — models may be visible there even if not in user endpoint
|
|
var catalog = await T.apiGet('/admin/models');
|
|
var allModels = catalog.models || catalog.data || catalog;
|
|
if (Array.isArray(allModels)) {
|
|
ours = allModels.filter(function (m) { return m.provider_config_id === globalConfigId; });
|
|
}
|
|
}
|
|
T.assert(ours.length > 0, 'no models visible from global config ' + globalConfigId);
|
|
globalModel = T.pickCheapestChat(ours);
|
|
T.assert(globalModel, 'no chat model found in global config');
|
|
});
|
|
|
|
if (globalModel) {
|
|
var globalChannelId = null;
|
|
|
|
await T.test('provider', 'global', 'completion via global provider', async function () {
|
|
// Create channel, run completion, verify
|
|
var ch = await T.apiPost('/channels', {
|
|
title: tag + '-global-test',
|
|
type: 'direct',
|
|
model: (globalModel.model_id || globalModel.id),
|
|
provider_config_id: globalConfigId
|
|
});
|
|
globalChannelId = ch.id;
|
|
T.registerCleanup(function () { if (globalChannelId) return T.safeDelete('/channels/' + globalChannelId); });
|
|
|
|
var result = await T.testCompletion(globalChannelId, (globalModel.model_id || globalModel.id), globalConfigId, 'global');
|
|
T.assert(result.content.length > 0, 'completion produced no content');
|
|
});
|
|
|
|
if (globalChannelId) {
|
|
await T.test('provider', 'global', 'GET /channels/:id/path (message persisted)', async function () {
|
|
var d = await T.apiGet('/channels/' + globalChannelId + '/path');
|
|
var msgs = d.messages || d.data || d;
|
|
T.assert(Array.isArray(msgs), 'expected messages array');
|
|
// Should have user + assistant messages
|
|
var roles = msgs.map(function (m) { return m.role; });
|
|
T.assert(roles.indexOf('user') !== -1, 'missing user message');
|
|
T.assert(roles.indexOf('assistant') !== -1, 'missing assistant message');
|
|
});
|
|
|
|
await T.test('provider', 'global', 'DELETE test channel', async function () {
|
|
await T.safeDelete('/channels/' + globalChannelId);
|
|
globalChannelId = null;
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════
|
|
// Tier 2: Team provider
|
|
// ════════════════════════════════════════════════════════════
|
|
|
|
if (T.user.role === 'admin') {
|
|
await T.test('provider', 'team', 'POST /admin/teams (create test team)', async function () {
|
|
var d = await T.apiPost('/admin/teams', { name: tag + '-team', description: 'Provider test team' });
|
|
teamId = d.id;
|
|
T.registerCleanup(function () { if (teamId) return T.safeDelete('/admin/teams/' + teamId); });
|
|
// Add self as team admin
|
|
try { await T.apiPost('/admin/teams/' + teamId + '/members', { user_id: T.user.id, role: 'admin' }); } catch (e) { /* auto-added */ }
|
|
});
|
|
|
|
if (teamId) {
|
|
await T.test('provider', 'team', 'POST /teams/:id/providers (create)', async function () {
|
|
var d = await T.apiPost('/teams/' + teamId + '/providers', {
|
|
name: tag + '-team-prov',
|
|
provider: prov,
|
|
endpoint: endpoint,
|
|
api_key: key,
|
|
config: {},
|
|
headers: {},
|
|
settings: {}
|
|
});
|
|
T.assertHasKey(d, 'id', 'team-config');
|
|
teamConfigId = d.id;
|
|
T.registerCleanup(function () { if (teamConfigId) return T.safeDelete('/teams/' + teamId + '/providers/' + teamConfigId); });
|
|
});
|
|
|
|
if (teamConfigId) {
|
|
await T.test('provider', 'team', 'GET /teams/:id/providers (list)', async function () {
|
|
var d = await T.apiGet('/teams/' + teamId + '/providers');
|
|
var configs = d.configs || d.data || d.providers || (Array.isArray(d) ? d : null);
|
|
T.assert(Array.isArray(configs), 'expected configs array, got keys: ' + Object.keys(d).join(', '));
|
|
var found = configs.find(function (c) { return c.id === teamConfigId; });
|
|
T.assert(found, 'team config not in list');
|
|
});
|
|
|
|
await T.test('provider', 'team', 'PUT /teams/:id/providers/:id (update)', async function () {
|
|
var d = await T.apiPut('/teams/' + teamId + '/providers/' + teamConfigId, { name: tag + '-team-renamed' });
|
|
T.assert(typeof d === 'object', 'expected object');
|
|
});
|
|
|
|
await T.test('provider', 'team', 'GET /teams/:id/providers/:id/models (fetch)', async function () {
|
|
var d = await T.apiGet('/teams/' + teamId + '/providers/' + teamConfigId + '/models');
|
|
var models = d.models || d.data || d;
|
|
T.assert(Array.isArray(models), 'expected models array');
|
|
if (models.length > 0) {
|
|
teamModel = T.pickCheapestChat(models);
|
|
}
|
|
});
|
|
|
|
// If no models from team-specific fetch, try /models/enabled filtered
|
|
if (!teamModel) {
|
|
await T.test('provider', 'team', 'GET /models/enabled (team model fallback)', async function () {
|
|
var d = await T.apiGet('/models/enabled');
|
|
var models = d.models || d.data || d;
|
|
var ours = models.filter(function (m) { return m.provider_config_id === teamConfigId; });
|
|
if (ours.length > 0) teamModel = T.pickCheapestChat(ours);
|
|
T.assert(teamModel, 'no chat model found for team config');
|
|
});
|
|
}
|
|
|
|
if (teamModel) {
|
|
var teamChannelId = null;
|
|
|
|
await T.test('provider', 'team', 'completion via team provider', async function () {
|
|
var ch = await T.apiPost('/channels', {
|
|
title: tag + '-team-test',
|
|
type: 'direct',
|
|
model: (teamModel.model_id || teamModel.id),
|
|
provider_config_id: teamConfigId
|
|
});
|
|
teamChannelId = ch.id;
|
|
T.registerCleanup(function () { if (teamChannelId) return T.safeDelete('/channels/' + teamChannelId); });
|
|
|
|
var result = await T.testCompletion(teamChannelId, (teamModel.model_id || teamModel.id), teamConfigId, 'team');
|
|
T.assert(result.content.length > 0, 'completion produced no content');
|
|
});
|
|
|
|
if (teamChannelId) {
|
|
await T.test('provider', 'team', 'DELETE test channel', async function () {
|
|
await T.safeDelete('/channels/' + teamChannelId);
|
|
teamChannelId = null;
|
|
});
|
|
}
|
|
}
|
|
|
|
await T.test('provider', 'team', 'DELETE /teams/:id/providers/:id', async function () {
|
|
await T.safeDelete('/teams/' + teamId + '/providers/' + teamConfigId);
|
|
teamConfigId = null;
|
|
});
|
|
}
|
|
|
|
await T.test('provider', 'team', 'DELETE /admin/teams/:id (cleanup)', async function () {
|
|
await T.safeDelete('/admin/teams/' + teamId);
|
|
teamId = null;
|
|
});
|
|
}
|
|
}
|
|
|
|
// ════════════════════════════════════════════════════════════
|
|
// Tier 3: Personal BYOK
|
|
// ════════════════════════════════════════════════════════════
|
|
|
|
// BYOK requires allow_user_byok policy — enable it if admin, skip if not
|
|
var byokPolicyWas = null; // null = don't restore
|
|
|
|
if (T.user.role === 'admin') {
|
|
await T.test('provider', 'byok', 'enable allow_user_byok policy', async function () {
|
|
// Read current policy value
|
|
try {
|
|
var pub = await T.apiGet('/settings/public');
|
|
var policies = pub.policies || {};
|
|
byokPolicyWas = policies.allow_user_byok;
|
|
} catch (e) { /* can't read — will try to set anyway */ }
|
|
|
|
// Enable BYOK
|
|
await T.apiPut('/admin/settings/allow_user_byok', { value: 'true' });
|
|
});
|
|
}
|
|
|
|
await T.test('provider', 'byok', 'POST /api-configs (create)', async function () {
|
|
var d = await T.apiPost('/api-configs', {
|
|
name: tag + '-byok',
|
|
provider: prov,
|
|
endpoint: endpoint,
|
|
api_key: key,
|
|
config: {},
|
|
headers: {},
|
|
settings: {}
|
|
});
|
|
T.assertHasKey(d, 'id', 'byok-config');
|
|
byokConfigId = d.id;
|
|
T.registerCleanup(function () { if (byokConfigId) return T.safeDelete('/api-configs/' + byokConfigId); });
|
|
});
|
|
|
|
if (byokConfigId) {
|
|
await T.test('provider', 'byok', 'GET /api-configs/:id (read — key redacted)', async function () {
|
|
var d = await T.apiGet('/api-configs/' + byokConfigId);
|
|
T.assertHasKey(d, 'id', 'byok-config');
|
|
// has_key may not be present on single-config GET (only on list) — check if present
|
|
if (d.has_key !== undefined) T.assert(d.has_key === true, 'has_key should be true');
|
|
T.assert(!d.api_key, 'api_key should be redacted in response');
|
|
});
|
|
|
|
await T.test('provider', 'byok', 'PUT /api-configs/:id (update name)', async function () {
|
|
var d = await T.apiPut('/api-configs/' + byokConfigId, { name: tag + '-byok-renamed' });
|
|
T.assert(typeof d === 'object', 'expected object');
|
|
});
|
|
|
|
await T.test('provider', 'byok', 'POST /api-configs/:id/models/fetch (sync)', async function () {
|
|
var d = await apiPostRetry('/api-configs/' + byokConfigId + '/models/fetch', {});
|
|
T.assert(typeof d === 'object', 'expected object');
|
|
});
|
|
|
|
await T.test('provider', 'byok', 'GET /api-configs/:id/models (list)', async function () {
|
|
var d = await T.apiGet('/api-configs/' + byokConfigId + '/models');
|
|
var models = d.models || d.data || d;
|
|
T.assert(Array.isArray(models), 'expected models array');
|
|
T.assert(models.length > 0, 'no models returned after fetch');
|
|
byokModel = T.pickCheapestChat(models);
|
|
T.assert(byokModel, 'no chat model found in BYOK config');
|
|
});
|
|
|
|
if (byokModel) {
|
|
var byokChannelId = null;
|
|
|
|
await T.test('provider', 'byok', 'completion via BYOK provider', async function () {
|
|
var ch = await T.apiPost('/channels', {
|
|
title: tag + '-byok-test',
|
|
type: 'direct',
|
|
model: (byokModel.model_id || byokModel.id),
|
|
provider_config_id: byokConfigId
|
|
});
|
|
byokChannelId = ch.id;
|
|
T.registerCleanup(function () { if (byokChannelId) return T.safeDelete('/channels/' + byokChannelId); });
|
|
|
|
var result = await T.testCompletion(byokChannelId, (byokModel.model_id || byokModel.id), byokConfigId, 'byok');
|
|
T.assert(result.content.length > 0, 'completion produced no content');
|
|
});
|
|
|
|
if (byokChannelId) {
|
|
await T.test('provider', 'byok', 'GET /channels/:id/path (message persisted)', async function () {
|
|
var d = await T.apiGet('/channels/' + byokChannelId + '/path');
|
|
var msgs = d.messages || d.data || d;
|
|
T.assert(Array.isArray(msgs), 'expected messages array');
|
|
var roles = msgs.map(function (m) { return m.role; });
|
|
T.assert(roles.indexOf('assistant') !== -1, 'missing assistant response');
|
|
});
|
|
|
|
await T.test('provider', 'byok', 'DELETE test channel', async function () {
|
|
await T.safeDelete('/channels/' + byokChannelId);
|
|
byokChannelId = null;
|
|
});
|
|
}
|
|
}
|
|
|
|
// ── Scoping isolation: BYOK should NOT appear in admin/configs ──
|
|
if (T.user.role === 'admin') {
|
|
await T.test('provider', 'isolation', 'BYOK not in /admin/configs', async function () {
|
|
var d = await T.apiGet('/admin/configs');
|
|
var configs = d.configs || d.data || d;
|
|
if (Array.isArray(configs)) {
|
|
var leaked = configs.find(function (c) { return c.id === byokConfigId; });
|
|
T.assert(!leaked, 'BYOK config leaked into global admin configs list');
|
|
}
|
|
});
|
|
}
|
|
|
|
await T.test('provider', 'byok', 'DELETE /api-configs/:id', async function () {
|
|
await T.safeDelete('/api-configs/' + byokConfigId);
|
|
byokConfigId = null;
|
|
});
|
|
|
|
// After delete, should not appear in list
|
|
await T.test('provider', 'byok', 'GET /api-configs (deleted config gone)', async function () {
|
|
var d = await T.apiGet('/api-configs');
|
|
var configs = d.configs || d.data || d;
|
|
if (Array.isArray(configs)) {
|
|
var ghost = configs.find(function (c) { return c.name && c.name.indexOf(tag) !== -1; });
|
|
T.assert(!ghost, 'deleted BYOK config still in list');
|
|
}
|
|
});
|
|
}
|
|
|
|
// Restore allow_user_byok to original value
|
|
if (byokPolicyWas !== null && T.user.role === 'admin') {
|
|
await T.test('provider', 'byok', 'restore allow_user_byok policy', async function () {
|
|
await T.apiPut('/admin/settings/allow_user_byok', { value: byokPolicyWas || 'false' });
|
|
});
|
|
}
|
|
};
|
|
})();
|