/** * ICD Test Runner — Provider Tier * Three-tier provider CRUD (global → team → BYOK) with live SSE completions. */ (function () { 'use strict'; var T = window.ICD; if (!T) return; var PROVIDER_DEFAULTS = { openai: 'https://api.openai.com/v1', anthropic: 'https://api.anthropic.com', venice: 'https://api.venice.ai/api/v1', openrouter: 'https://openrouter.ai/api/v1' }; T.PROVIDER_DEFAULTS = PROVIDER_DEFAULTS; // ─── Retry Helper (upstream timeout resilience) ──────────── // apiPostRetry retries a POST once on 502/504. Slow upstream providers // (especially Venice) can exceed the reverse proxy timeout on cold starts. // A single retry with a short backoff is enough to ride out transient timeouts. async function apiPostRetry(path, body, retries) { if (retries === undefined) retries = 1; try { return await T.apiPost(path, body); } catch (e) { if (retries > 0 && (e.status === 502 || e.status === 504)) { console.log(' ⟳ got ' + e.status + ' from ' + path + ', retrying in 2s…'); await new Promise(function (r) { setTimeout(r, 2000); }); return await apiPostRetry(path, body, retries - 1); } throw e; } } // ─── SSE Completion Helper ────────────────────────────────── T.testCompletion = async function (channelId, model, providerConfigId, label) { var token = await T.getAuthToken(); var resp = await fetch(T.base + '/api/v1/chat/completions', { method: 'POST', headers: { 'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json' }, credentials: 'same-origin', body: JSON.stringify({ channel_id: channelId, content: 'Reply with exactly: ICD_TEST_OK', model: model, provider_config_id: providerConfigId, tools_enabled: false }) }); T.assert(resp.ok, label + ': completion returned HTTP ' + resp.status); T.assert(resp.headers.get('content-type').indexOf('text/event-stream') !== -1, label + ': expected SSE, got ' + resp.headers.get('content-type')); var reader = resp.body.getReader(); var decoder = new TextDecoder(); var content = ''; var gotFinish = false; var finishData = null; var buf = ''; while (true) { var chunk = await reader.read(); if (chunk.done) break; buf += decoder.decode(chunk.value, { stream: true }); var lines = buf.split('\n'); buf = lines.pop(); for (var i = 0; i < lines.length; i++) { var line = lines[i].trim(); if (!line) continue; if (line === 'data: [DONE]') { gotFinish = true; continue; } if (line.indexOf('event:') === 0) continue; if (line.indexOf('data:') === 0) { var raw = line.slice(5).trim(); try { var evt = JSON.parse(raw); // Handler emits OpenAI-format: choices[0].delta.content if (evt.choices && evt.choices[0] && evt.choices[0].delta && evt.choices[0].delta.content) { content += evt.choices[0].delta.content; } else if (evt.content) { content += evt.content; // fallback for non-OpenAI format } if (evt.message_id) { gotFinish = true; finishData = evt; } } catch (e) { /* skip */ } } } } T.assert(content.length > 0, label + ': no content in stream'); T.assert(gotFinish, label + ': no finish event'); return { content: content, finish: finishData }; }; // ─── Model Selection ──────────────────────────────────────── T.pickCheapestChat = function (models) { var skipPatterns = ['embed', 'image', 'dall-e', 'tts', 'whisper', 'moderation']; var chatModels = models.filter(function (m) { var t = m.model_type || m.type || ''; if (t === 'embedding' || t === 'image') return false; // When type is missing, exclude by model ID pattern var mid = (m.model_id || m.id || '').toLowerCase(); for (var i = 0; i < skipPatterns.length; i++) { if (mid.indexOf(skipPatterns[i]) !== -1) return false; } return (t === 'chat' || t === 'text' || t === ''); }); if (chatModels.length === 0) return models[0] || null; chatModels.sort(function (a, b) { return (a.output_price_per_m || a.output_price || 999) - (b.output_price_per_m || b.output_price || 999); }); var cheapNames = ['mini', 'haiku', 'flash', 'fast', 'nano', 'lite', 'small']; for (var i = 0; i < chatModels.length; i++) { var mid = (chatModels[i].model_id || chatModels[i].id || '').toLowerCase(); for (var j = 0; j < cheapNames.length; j++) { if (mid.indexOf(cheapNames[j]) !== -1) return chatModels[i]; } } return chatModels[0]; }; T.runProviders = async function () { if (!T.providerSetup.configured || !T.providerSetup.apiKey) { await T.test('provider', 'setup', 'SKIP — no provider configured', async function () { throw new Error('Configure a provider (type + API key) in the panel above, then run again'); }); return; } var tag = 'icd-prov-' + Date.now(); var prov = T.providerSetup.provider; var key = T.providerSetup.apiKey; var endpoint = T.providerSetup.endpoint || PROVIDER_DEFAULTS[prov] || ''; // Track IDs for scoping assertions var globalConfigId = null; var globalModel = null; var teamId = null; var teamConfigId = null; var teamModel = null; var byokConfigId = null; var byokModel = null; // ════════════════════════════════════════════════════════════ // Tier 1: Global (admin) // ════════════════════════════════════════════════════════════ if (T.user.role === 'admin') { await T.test('provider', 'global', 'POST /admin/configs (create)', async function () { var d = await T.apiPost('/admin/configs', { name: tag + '-global', provider: prov, endpoint: endpoint, api_key: key, is_private: false, config: {}, headers: {}, settings: {} }); T.assertHasKey(d, 'id', 'global-config'); globalConfigId = d.id; T.registerCleanup(function () { if (globalConfigId) return T.safeDelete('/admin/configs/' + globalConfigId); }); }); if (globalConfigId) { await T.test('provider', 'global', 'GET /admin/configs/:id (read)', async function () { var d = await T.apiGet('/admin/configs'); var configs = d.configs || d.data || d; T.assert(Array.isArray(configs), 'expected configs array'); var found = configs.find(function (c) { return c.id === globalConfigId; }); T.assert(found, 'created config not in list'); T.assert(found.has_key === true, 'has_key should be true'); // Key must be redacted T.assert(!found.api_key, 'api_key should be redacted'); }); await T.test('provider', 'global', 'PUT /admin/configs/:id (update name)', async function () { var d = await T.apiPut('/admin/configs/' + globalConfigId, { name: tag + '-global-renamed' }); T.assert(typeof d === 'object', 'expected object'); }); await T.test('provider', 'global', 'POST /admin/models/fetch (sync catalog)', async function () { var d = await apiPostRetry('/admin/models/fetch', { provider_config_id: globalConfigId }); T.assert(typeof d === 'object', 'expected object'); T.assert(d.total !== undefined || d.added !== undefined || d.message, 'expected sync result, got keys: ' + Object.keys(d).join(', ')); }); // Newly synced models may default to visibility=disabled — enable them await T.test('provider', 'global', 'PUT /admin/models/bulk (enable visibility)', async function () { var d = await T.apiPut('/admin/models/bulk', { provider_config_id: globalConfigId, visibility: 'enabled' }); T.assert(typeof d === 'object', 'expected object'); }); await T.test('provider', 'global', 'GET /models/enabled (global visible)', async function () { var d = await T.apiGet('/models/enabled'); var models = d.models || d.data || d; T.assert(Array.isArray(models), 'expected models array'); var ours = models.filter(function (m) { return m.provider_config_id === globalConfigId; }); if (ours.length === 0) { // Fall back to admin catalog — models may be visible there even if not in user endpoint var catalog = await T.apiGet('/admin/models'); var allModels = catalog.models || catalog.data || catalog; if (Array.isArray(allModels)) { ours = allModels.filter(function (m) { return m.provider_config_id === globalConfigId; }); } } T.assert(ours.length > 0, 'no models visible from global config ' + globalConfigId); globalModel = T.pickCheapestChat(ours); T.assert(globalModel, 'no chat model found in global config'); }); if (globalModel) { var globalChannelId = null; await T.test('provider', 'global', 'completion via global provider', async function () { // Create channel, run completion, verify var ch = await T.apiPost('/channels', { title: tag + '-global-test', type: 'direct', model: (globalModel.model_id || globalModel.id), provider_config_id: globalConfigId }); globalChannelId = ch.id; T.registerCleanup(function () { if (globalChannelId) return T.safeDelete('/channels/' + globalChannelId); }); var result = await T.testCompletion(globalChannelId, (globalModel.model_id || globalModel.id), globalConfigId, 'global'); T.assert(result.content.length > 0, 'completion produced no content'); }); if (globalChannelId) { await T.test('provider', 'global', 'GET /channels/:id/path (message persisted)', async function () { var d = await T.apiGet('/channels/' + globalChannelId + '/path'); var msgs = d.messages || d.data || d; T.assert(Array.isArray(msgs), 'expected messages array'); // Should have user + assistant messages var roles = msgs.map(function (m) { return m.role; }); T.assert(roles.indexOf('user') !== -1, 'missing user message'); T.assert(roles.indexOf('assistant') !== -1, 'missing assistant message'); }); await T.test('provider', 'global', 'DELETE test channel', async function () { await T.safeDelete('/channels/' + globalChannelId); globalChannelId = null; }); } } } } // ════════════════════════════════════════════════════════════ // Tier 2: Team provider // ════════════════════════════════════════════════════════════ if (T.user.role === 'admin') { await T.test('provider', 'team', 'POST /admin/teams (create test team)', async function () { var d = await T.apiPost('/admin/teams', { name: tag + '-team', description: 'Provider test team' }); teamId = d.id; T.registerCleanup(function () { if (teamId) return T.safeDelete('/admin/teams/' + teamId); }); // Add self as team admin try { await T.apiPost('/admin/teams/' + teamId + '/members', { user_id: T.user.id, role: 'admin' }); } catch (e) { /* auto-added */ } }); if (teamId) { await T.test('provider', 'team', 'POST /teams/:id/providers (create)', async function () { var d = await T.apiPost('/teams/' + teamId + '/providers', { name: tag + '-team-prov', provider: prov, endpoint: endpoint, api_key: key, config: {}, headers: {}, settings: {} }); T.assertHasKey(d, 'id', 'team-config'); teamConfigId = d.id; T.registerCleanup(function () { if (teamConfigId) return T.safeDelete('/teams/' + teamId + '/providers/' + teamConfigId); }); }); if (teamConfigId) { await T.test('provider', 'team', 'GET /teams/:id/providers (list)', async function () { var d = await T.apiGet('/teams/' + teamId + '/providers'); var configs = d.configs || d.data || d.providers || (Array.isArray(d) ? d : null); T.assert(Array.isArray(configs), 'expected configs array, got keys: ' + Object.keys(d).join(', ')); var found = configs.find(function (c) { return c.id === teamConfigId; }); T.assert(found, 'team config not in list'); }); await T.test('provider', 'team', 'PUT /teams/:id/providers/:id (update)', async function () { var d = await T.apiPut('/teams/' + teamId + '/providers/' + teamConfigId, { name: tag + '-team-renamed' }); T.assert(typeof d === 'object', 'expected object'); }); await T.test('provider', 'team', 'GET /teams/:id/providers/:id/models (fetch)', async function () { var d = await T.apiGet('/teams/' + teamId + '/providers/' + teamConfigId + '/models'); var models = d.models || d.data || d; T.assert(Array.isArray(models), 'expected models array'); if (models.length > 0) { teamModel = T.pickCheapestChat(models); } }); // If no models from team-specific fetch, try /models/enabled filtered if (!teamModel) { await T.test('provider', 'team', 'GET /models/enabled (team model fallback)', async function () { var d = await T.apiGet('/models/enabled'); var models = d.models || d.data || d; var ours = models.filter(function (m) { return m.provider_config_id === teamConfigId; }); if (ours.length > 0) teamModel = T.pickCheapestChat(ours); T.assert(teamModel, 'no chat model found for team config'); }); } if (teamModel) { var teamChannelId = null; await T.test('provider', 'team', 'completion via team provider', async function () { var ch = await T.apiPost('/channels', { title: tag + '-team-test', type: 'direct', model: (teamModel.model_id || teamModel.id), provider_config_id: teamConfigId }); teamChannelId = ch.id; T.registerCleanup(function () { if (teamChannelId) return T.safeDelete('/channels/' + teamChannelId); }); var result = await T.testCompletion(teamChannelId, (teamModel.model_id || teamModel.id), teamConfigId, 'team'); T.assert(result.content.length > 0, 'completion produced no content'); }); if (teamChannelId) { await T.test('provider', 'team', 'DELETE test channel', async function () { await T.safeDelete('/channels/' + teamChannelId); teamChannelId = null; }); } } await T.test('provider', 'team', 'DELETE /teams/:id/providers/:id', async function () { await T.safeDelete('/teams/' + teamId + '/providers/' + teamConfigId); teamConfigId = null; }); } await T.test('provider', 'team', 'DELETE /admin/teams/:id (cleanup)', async function () { await T.safeDelete('/admin/teams/' + teamId); teamId = null; }); } } // ════════════════════════════════════════════════════════════ // Tier 3: Personal BYOK // ════════════════════════════════════════════════════════════ // BYOK requires allow_user_byok policy — enable it if admin, skip if not var byokPolicyWas = null; // null = don't restore if (T.user.role === 'admin') { await T.test('provider', 'byok', 'enable allow_user_byok policy', async function () { // Read current policy value try { var pub = await T.apiGet('/settings/public'); var policies = pub.policies || {}; byokPolicyWas = policies.allow_user_byok; } catch (e) { /* can't read — will try to set anyway */ } // Enable BYOK await T.apiPut('/admin/settings/allow_user_byok', { value: 'true' }); }); } await T.test('provider', 'byok', 'POST /api-configs (create)', async function () { var d = await T.apiPost('/api-configs', { name: tag + '-byok', provider: prov, endpoint: endpoint, api_key: key, config: {}, headers: {}, settings: {} }); T.assertHasKey(d, 'id', 'byok-config'); byokConfigId = d.id; T.registerCleanup(function () { if (byokConfigId) return T.safeDelete('/api-configs/' + byokConfigId); }); }); if (byokConfigId) { await T.test('provider', 'byok', 'GET /api-configs/:id (read — key redacted)', async function () { var d = await T.apiGet('/api-configs/' + byokConfigId); T.assertHasKey(d, 'id', 'byok-config'); // has_key may not be present on single-config GET (only on list) — check if present if (d.has_key !== undefined) T.assert(d.has_key === true, 'has_key should be true'); T.assert(!d.api_key, 'api_key should be redacted in response'); }); await T.test('provider', 'byok', 'PUT /api-configs/:id (update name)', async function () { var d = await T.apiPut('/api-configs/' + byokConfigId, { name: tag + '-byok-renamed' }); T.assert(typeof d === 'object', 'expected object'); }); await T.test('provider', 'byok', 'POST /api-configs/:id/models/fetch (sync)', async function () { var d = await apiPostRetry('/api-configs/' + byokConfigId + '/models/fetch', {}); T.assert(typeof d === 'object', 'expected object'); }); await T.test('provider', 'byok', 'GET /api-configs/:id/models (list)', async function () { var d = await T.apiGet('/api-configs/' + byokConfigId + '/models'); var models = d.models || d.data || d; T.assert(Array.isArray(models), 'expected models array'); T.assert(models.length > 0, 'no models returned after fetch'); byokModel = T.pickCheapestChat(models); T.assert(byokModel, 'no chat model found in BYOK config'); }); if (byokModel) { var byokChannelId = null; await T.test('provider', 'byok', 'completion via BYOK provider', async function () { var ch = await T.apiPost('/channels', { title: tag + '-byok-test', type: 'direct', model: (byokModel.model_id || byokModel.id), provider_config_id: byokConfigId }); byokChannelId = ch.id; T.registerCleanup(function () { if (byokChannelId) return T.safeDelete('/channels/' + byokChannelId); }); var result = await T.testCompletion(byokChannelId, (byokModel.model_id || byokModel.id), byokConfigId, 'byok'); T.assert(result.content.length > 0, 'completion produced no content'); }); if (byokChannelId) { await T.test('provider', 'byok', 'GET /channels/:id/path (message persisted)', async function () { var d = await T.apiGet('/channels/' + byokChannelId + '/path'); var msgs = d.messages || d.data || d; T.assert(Array.isArray(msgs), 'expected messages array'); var roles = msgs.map(function (m) { return m.role; }); T.assert(roles.indexOf('assistant') !== -1, 'missing assistant response'); }); await T.test('provider', 'byok', 'DELETE test channel', async function () { await T.safeDelete('/channels/' + byokChannelId); byokChannelId = null; }); } } // ── Scoping isolation: BYOK should NOT appear in admin/configs ── if (T.user.role === 'admin') { await T.test('provider', 'isolation', 'BYOK not in /admin/configs', async function () { var d = await T.apiGet('/admin/configs'); var configs = d.configs || d.data || d; if (Array.isArray(configs)) { var leaked = configs.find(function (c) { return c.id === byokConfigId; }); T.assert(!leaked, 'BYOK config leaked into global admin configs list'); } }); } await T.test('provider', 'byok', 'DELETE /api-configs/:id', async function () { await T.safeDelete('/api-configs/' + byokConfigId); byokConfigId = null; }); // After delete, should not appear in list await T.test('provider', 'byok', 'GET /api-configs (deleted config gone)', async function () { var d = await T.apiGet('/api-configs'); var configs = d.configs || d.data || d; if (Array.isArray(configs)) { var ghost = configs.find(function (c) { return c.name && c.name.indexOf(tag) !== -1; }); T.assert(!ghost, 'deleted BYOK config still in list'); } }); } // Restore allow_user_byok to original value if (byokPolicyWas !== null && T.user.role === 'admin') { await T.test('provider', 'byok', 'restore allow_user_byok policy', async function () { await T.apiPut('/admin/settings/allow_user_byok', { value: byokPolicyWas || 'false' }); }); } }; })();