diff --git a/CHANGELOG.md b/CHANGELOG.md index 95193b9..c5f9e39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,68 @@ # Changelog +## [0.33.0] — 2026-03-19 + +### Summary + +Observability: structured logging, Prometheus metrics, OpenAPI docs, +Grafana dashboards, alerting rules, and a built-in admin monitoring +dashboard. Operate the platform without reading Go source code. + +### New + +- **Structured logging (`slog`)** — `LOG_FORMAT=json|text`, + `LOG_LEVEL=debug|info|warn|error`. JSON handler for production, + text handler for development. Request ID, method, path, status, + latency, client IP, and user ID on every request log line. +- **Request ID middleware** — `X-Request-Id` UUID header generated per + request (or forwarded from upstream). Propagated through completion + chain as correlation ID. +- **Prometheus `/metrics` endpoint** — `switchboard_http_requests_total`, + `switchboard_http_request_duration_seconds`, `switchboard_websocket_connections`, + `switchboard_completion_tokens_total`, `switchboard_completions_total`, + `switchboard_completion_duration_seconds`, `switchboard_provider_status`, + `switchboard_db_open_connections` (+ `_in_use`, `_idle`, `_wait_count`, + `_wait_duration`), `switchboard_task_executions_total`, + `switchboard_sandbox_executions_total`. Uses `c.FullPath()` for + path patterns (no cardinality explosion). +- **OpenAPI 3.0.3 spec** — hand-curated `openapi.yaml` covering auth, + channels, completions, health, and admin API groups. Served at + `/api/docs/openapi.yaml`. +- **Swagger UI** — `/api/docs` renders interactive API browser via CDN. + System-respecting light/dark mode with WCAG AA 4.5:1+ contrast ratios. +- **Grafana dashboard** — `switchboard-overview.json` with request rate, + latency percentiles, provider health, token usage, DB pool, Go runtime. +- **PrometheusRule alerting** — OOM recovery, provider down, pool + exhaustion, high error rate, task failure spike. +- **ServiceMonitor** — Helm template for Prometheus Operator discovery + (gated: `monitoring.serviceMonitor.enabled`). +- **Admin monitoring dashboard** — `GET /api/v1/admin/dashboard` + aggregates provider health, 24h usage totals, DB pool stats, WS + connections, Go runtime (goroutines, heap, sys, GC count, Go version), + storage status, and recent errors. Auto-refreshes every 30s. +- **ICD observability tests** — 6 tests covering `/metrics`, + `/api/docs`, `/api/docs/openapi.yaml`, `X-Request-Id` generation, + `X-Request-Id` passthrough, and admin dashboard. + +### Changed + +- **`middleware/logging.go`** — rewritten from `gin.Logger` wrapper to + `slog.Info` with structured fields. `SkipPaths` still honored. +- **`config/config.go`** — added `LogFormat`, `LogLevel` fields with + `LOG_FORMAT`, `LOG_LEVEL` env var loading. +- **`events/ws.go`** — WebSocket connect/disconnect now updates + `switchboard_websocket_connections` gauge. +- **`health/accumulator.go`** — flush cycle updates + `switchboard_provider_status` gauge per provider config. +- **`handlers/completion.go`** — instrumented with token counters, + completion latency histogram, and status counter. +- **Dockerfile** — Go 1.22 → Go 1.23 (required by `prometheus/client_golang`). +- **`nginx.conf`** — added `location = /metrics` proxy rule. +- **Admin monitoring tab** — landing page changed from Usage to Dashboard. +- **Helm `values.yaml`** — added `monitoring` section (serviceMonitor, + grafanaDashboard, prometheusRule — all `enabled: false` by default) + and `logging.format`, `logging.level`. + ## [0.32.0] — 2026-03-19 ### Summary diff --git a/Dockerfile b/Dockerfile index 28ae9c8..1720613 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # ============================================ # ── Stage 1: Go backend build ──────────────── -FROM golang:1.22-bookworm AS backend +FROM golang:1.23-bookworm AS backend WORKDIR /app COPY server/go.mod server/go.sum* ./ diff --git a/VERSION b/VERSION index 8a0d6d4..be386c9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.32.0 \ No newline at end of file +0.33.0 diff --git a/chart/Chart.yaml b/chart/Chart.yaml index f40e907..913ab9f 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -3,7 +3,7 @@ name: switchboard description: Chat Switchboard — self-hosted enterprise AI chat platform type: application version: 0.1.0 -appVersion: "0.28.6" +appVersion: "0.0.0" # Patched at CI/release time from /VERSION home: https://gobha.ai sources: - https://git.gobha.me/xcaliber/chat-switchboard diff --git a/chart/alerting/switchboard-rules.yaml b/chart/alerting/switchboard-rules.yaml new file mode 100644 index 0000000..339aaa4 --- /dev/null +++ b/chart/alerting/switchboard-rules.yaml @@ -0,0 +1,69 @@ +# Chat Switchboard — PrometheusRule alerts (v0.33.0) +# Source file for the Helm template. Deploy via: +# monitoring.prometheusRule.enabled: true +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: switchboard-alerts +spec: + groups: + - name: switchboard.rules + rules: + # Pod restart (possible OOM) + - alert: SwitchboardPodRestart + expr: increase(kube_pod_container_status_restarts_total{container="backend"}[1h]) > 0 + for: 0m + labels: + severity: warning + annotations: + summary: "Switchboard backend pod restarted (possible OOM)" + description: "Container {{ $labels.container }} in pod {{ $labels.pod }} restarted." + + # Provider down for 5+ minutes + - alert: SwitchboardProviderDown + expr: switchboard_provider_status > 2 + for: 5m + labels: + severity: critical + annotations: + summary: "Provider {{ $labels.provider_config_id }} is down" + + # DB pool >80% utilized + - alert: SwitchboardDBPoolExhaustion + expr: switchboard_db_in_use_connections / switchboard_db_open_connections > 0.8 + for: 5m + labels: + severity: warning + annotations: + summary: "DB connection pool >80% utilized" + + # HTTP 5xx error rate >5% + - alert: SwitchboardHighErrorRate + expr: > + sum(rate(switchboard_http_requests_total{status=~"5.."}[5m])) + / sum(rate(switchboard_http_requests_total[5m])) > 0.05 + for: 5m + labels: + severity: warning + annotations: + summary: "HTTP 5xx error rate exceeds 5%" + + # Task failure rate >25% + - alert: SwitchboardTaskFailureRate + expr: > + rate(switchboard_task_executions_total{status="error"}[15m]) + / rate(switchboard_task_executions_total[15m]) > 0.25 + for: 10m + labels: + severity: warning + annotations: + summary: "Task failure rate exceeds 25% over 15 minutes" + + # No completions processed in 15 minutes (canary) + - alert: SwitchboardNoCompletions + expr: sum(rate(switchboard_completions_total[10m])) == 0 + for: 15m + labels: + severity: critical + annotations: + summary: "No completions processed in 15 minutes" diff --git a/chart/dashboards/switchboard-overview.json b/chart/dashboards/switchboard-overview.json new file mode 100644 index 0000000..f889929 --- /dev/null +++ b/chart/dashboards/switchboard-overview.json @@ -0,0 +1,207 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "9.0.0" }, + { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, + { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, + { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, + { "type": "panel", "id": "gauge", "name": "Gauge", "version": "" } + ], + "id": null, + "uid": "switchboard-overview", + "title": "Chat Switchboard — Overview", + "description": "System overview: request rates, latency, provider health, token usage, DB pool.", + "tags": ["switchboard"], + "timezone": "browser", + "refresh": "30s", + "schemaVersion": 38, + "version": 1, + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus", + "current": { "text": "Prometheus", "value": "Prometheus" } + }, + { + "name": "namespace", + "type": "query", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "query": "label_values(switchboard_http_requests_total, namespace)", + "includeAll": true, + "current": { "text": "All", "value": "$__all" } + }, + { + "name": "pod", + "type": "query", + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "query": "label_values(switchboard_http_requests_total{namespace=~\"$namespace\"}, pod)", + "includeAll": true, + "current": { "text": "All", "value": "$__all" } + } + ] + }, + "panels": [ + { + "title": "Request Rate", + "type": "timeseries", + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 0 }, + "targets": [ + { + "expr": "sum(rate(switchboard_http_requests_total{namespace=~\"$namespace\"}[5m]))", + "legendFormat": "Total req/s" + } + ] + }, + { + "title": "Error Rate (5xx)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 6, "x": 6, "y": 0 }, + "targets": [ + { + "expr": "sum(rate(switchboard_http_requests_total{namespace=~\"$namespace\",status=~\"5..\"}[5m])) / sum(rate(switchboard_http_requests_total{namespace=~\"$namespace\"}[5m]))", + "legendFormat": "5xx rate" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "max": 1, + "thresholds": { + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.01 }, + { "color": "red", "value": 0.05 } + ] + } + } + } + }, + { + "title": "Request Latency (p50 / p95 / p99)", + "type": "timeseries", + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 0 }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(switchboard_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])) by (le))", + "legendFormat": "p50" + }, + { + "expr": "histogram_quantile(0.95, sum(rate(switchboard_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])) by (le))", + "legendFormat": "p95" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(switchboard_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])) by (le))", + "legendFormat": "p99" + } + ], + "fieldConfig": { "defaults": { "unit": "s" } } + }, + { + "title": "WebSocket Connections", + "type": "stat", + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 0 }, + "targets": [ + { + "expr": "sum(switchboard_websocket_connections{namespace=~\"$namespace\"})", + "legendFormat": "Active" + } + ] + }, + { + "title": "Completion Rate by Provider", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "targets": [ + { + "expr": "sum by (provider_config_id) (rate(switchboard_completions_total{namespace=~\"$namespace\"}[5m]))", + "legendFormat": "{{provider_config_id}}" + } + ] + }, + { + "title": "Completion Latency p95 by Provider", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum by (provider_config_id, le) (rate(switchboard_completion_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])))", + "legendFormat": "{{provider_config_id}}" + } + ], + "fieldConfig": { "defaults": { "unit": "s" } } + }, + { + "title": "Tokens/min by Model", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "targets": [ + { + "expr": "sum by (model_id) (rate(switchboard_completion_tokens_total{namespace=~\"$namespace\"}[5m])) * 60", + "legendFormat": "{{model_id}}" + } + ] + }, + { + "title": "Provider Status", + "type": "stat", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "targets": [ + { + "expr": "switchboard_provider_status{namespace=~\"$namespace\"}", + "legendFormat": "{{provider_config_id}}" + } + ], + "fieldConfig": { + "defaults": { + "mappings": [ + { "type": "value", "options": { "0": { "text": "Unknown", "color": "text" } } }, + { "type": "value", "options": { "1": { "text": "Healthy", "color": "green" } } }, + { "type": "value", "options": { "2": { "text": "Degraded", "color": "yellow" } } }, + { "type": "value", "options": { "3": { "text": "Down", "color": "red" } } } + ] + } + } + }, + { + "title": "DB Connection Pool", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, + "targets": [ + { + "expr": "switchboard_db_open_connections{namespace=~\"$namespace\"}", + "legendFormat": "Open" + }, + { + "expr": "switchboard_db_in_use_connections{namespace=~\"$namespace\"}", + "legendFormat": "In Use" + }, + { + "expr": "switchboard_db_idle_connections{namespace=~\"$namespace\"}", + "legendFormat": "Idle" + } + ] + }, + { + "title": "Task Executions", + "type": "timeseries", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, + "targets": [ + { + "expr": "sum by (status) (rate(switchboard_task_executions_total{namespace=~\"$namespace\"}[5m]))", + "legendFormat": "{{status}}" + } + ] + } + ] +} diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml index 026241c..aeeac52 100644 --- a/chart/templates/configmap.yaml +++ b/chart/templates/configmap.yaml @@ -19,6 +19,8 @@ data: EXTRACTION_CONCURRENCY: {{ .Values.extraction.concurrency | quote }} WORKSPACE_INDEXING_ENABLED: {{ .Values.workspace.indexingEnabled | quote }} WORKSPACE_INDEX_CONCURRENCY: {{ .Values.workspace.indexConcurrency | quote }} + LOG_FORMAT: {{ .Values.logging.format | default "text" | quote }} + LOG_LEVEL: {{ .Values.logging.level | default "info" | quote }} CORS_ALLOWED_ORIGINS: {{ .Values.corsAllowedOrigins | quote }} {{- if .Values.storage.s3.endpoint }} S3_ENDPOINT: {{ .Values.storage.s3.endpoint | quote }} diff --git a/chart/templates/grafana-dashboard-configmap.yaml b/chart/templates/grafana-dashboard-configmap.yaml new file mode 100644 index 0000000..7cd5d1c --- /dev/null +++ b/chart/templates/grafana-dashboard-configmap.yaml @@ -0,0 +1,14 @@ +{{- if .Values.monitoring.grafanaDashboard.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "switchboard.fullname" . }}-grafana-dashboard + labels: + {{- include "switchboard.labels" . | nindent 4 }} + {{- with .Values.monitoring.grafanaDashboard.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +data: + switchboard-overview.json: |- + {{- .Files.Get "dashboards/switchboard-overview.json" | nindent 4 }} +{{- end }} diff --git a/chart/templates/prometheusrule.yaml b/chart/templates/prometheusrule.yaml new file mode 100644 index 0000000..9a08373 --- /dev/null +++ b/chart/templates/prometheusrule.yaml @@ -0,0 +1,61 @@ +{{- if .Values.monitoring.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "switchboard.fullname" . }}-alerts + labels: + {{- include "switchboard.labels" . | nindent 4 }} + {{- with .Values.monitoring.prometheusRule.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + groups: + - name: switchboard.rules + rules: + - alert: SwitchboardPodRestart + expr: increase(kube_pod_container_status_restarts_total{container="backend"}[1h]) > 0 + for: 0m + labels: + severity: warning + annotations: + summary: "Switchboard backend pod restarted (possible OOM)" + - alert: SwitchboardProviderDown + expr: switchboard_provider_status > 2 + for: 5m + labels: + severity: critical + annotations: + summary: "Provider {{`{{ $labels.provider_config_id }}`}} is down" + - alert: SwitchboardDBPoolExhaustion + expr: switchboard_db_in_use_connections / switchboard_db_open_connections > 0.8 + for: 5m + labels: + severity: warning + annotations: + summary: "DB connection pool >80% utilized" + - alert: SwitchboardHighErrorRate + expr: | + sum(rate(switchboard_http_requests_total{status=~"5.."}[5m])) + / sum(rate(switchboard_http_requests_total[5m])) > 0.05 + for: 5m + labels: + severity: warning + annotations: + summary: "HTTP 5xx error rate exceeds 5%" + - alert: SwitchboardTaskFailureRate + expr: | + rate(switchboard_task_executions_total{status="error"}[15m]) + / rate(switchboard_task_executions_total[15m]) > 0.25 + for: 10m + labels: + severity: warning + annotations: + summary: "Task failure rate exceeds 25%" + - alert: SwitchboardNoCompletions + expr: sum(rate(switchboard_completions_total[10m])) == 0 + for: 15m + labels: + severity: critical + annotations: + summary: "No completions processed in 15 minutes" +{{- end }} diff --git a/chart/templates/servicemonitor.yaml b/chart/templates/servicemonitor.yaml new file mode 100644 index 0000000..308b30b --- /dev/null +++ b/chart/templates/servicemonitor.yaml @@ -0,0 +1,20 @@ +{{- if .Values.monitoring.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "switchboard.fullname" . }} + labels: + {{- include "switchboard.labels" . | nindent 4 }} + {{- with .Values.monitoring.serviceMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "switchboard.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: backend + endpoints: + - port: http + path: {{ .Values.monitoring.serviceMonitor.path | default "/metrics" }} + interval: {{ .Values.monitoring.serviceMonitor.interval | default "30s" }} +{{- end }} diff --git a/chart/templates/services.yaml b/chart/templates/services.yaml index 6000342..8886496 100644 --- a/chart/templates/services.yaml +++ b/chart/templates/services.yaml @@ -4,6 +4,10 @@ metadata: name: {{ .Release.Name }}-backend labels: {{- include "switchboard.backend.labels" . | nindent 4 }} + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: {{ .Values.backend.port | quote }} + prometheus.io/path: "/metrics" spec: type: ClusterIP ports: diff --git a/chart/values.yaml b/chart/values.yaml index ef2a9fb..b02cdc1 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -132,6 +132,31 @@ workspace: indexingEnabled: true indexConcurrency: 2 +# ── Logging (v0.33.0) ───────────────────── +logging: + format: text # "text" (human-readable) or "json" (structured) + level: info # "debug", "info", "warn", "error" + +# ── Monitoring (v0.33.0) ────────────────── +# All monitoring resources are opt-in (disabled by default). +monitoring: + # ServiceMonitor for Prometheus Operator (kube-prometheus-stack) + serviceMonitor: + enabled: false + interval: 30s + path: /metrics + labels: {} # match your Prometheus Operator selector + # Grafana dashboard ConfigMap (auto-discovered by Grafana sidecar) + grafanaDashboard: + enabled: false + labels: + grafana_dashboard: "1" # default sidecar label + # PrometheusRule alerts + prometheusRule: + enabled: false + labels: + release: prometheus # match kube-prometheus-stack + # ── CORS ─────────────────────────────────── corsAllowedOrigins: "*" diff --git a/docker-compose.yml b/docker-compose.yml index 91c2e39..922a459 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,8 @@ services: STORAGE_BACKEND: pvc STORAGE_PATH: /data/storage CORS_ALLOWED_ORIGINS: ${CORS_ALLOWED_ORIGINS:-http://localhost:3000} + LOG_FORMAT: ${LOG_FORMAT:-text} + LOG_LEVEL: ${LOG_LEVEL:-info} volumes: - ./data:/data ports: diff --git a/docs/DESIGN-0.33.0.md b/docs/DESIGN-0.33.0.md new file mode 100644 index 0000000..eea6b2d --- /dev/null +++ b/docs/DESIGN-0.33.0.md @@ -0,0 +1,238 @@ +# DESIGN-0.33.0 — Observability + +Operate the platform without reading Go source code. Six changesets +covering structured logging, Prometheus metrics, OpenAPI docs, Grafana +dashboards, alerting rules, and a built-in admin dashboard. + +Depends on: v0.32.0 (Multi-Replica HA, current HEAD). + +**Design decision:** Zero new database migrations. All metrics are +in-memory per-pod, scraped by Prometheus. Existing tables +(`provider_health`, `tool_health_windows`, `usage_log`, `audit_log`) +continue serving the admin dashboard via stores. + +**Library choices:** +- `log/slog` (Go 1.22 stdlib) — zero-dep structured logging +- `prometheus/client_golang` v1.20+ — canonical Go Prometheus client +- Hand-curated OpenAPI 3.0.3 YAML — avoids annotation sprawl across 97 handlers +- Swagger UI v5 via CDN — single HTML embed, zero build step + +--- + +## What Already Existed + +These required no structural changes, only instrumentation: + +- **Health accumulator** — in-memory provider+tool metrics, 60s DB flush +- **`Hub.ConnCount()`** — active WebSocket connection count +- **`database.DB.Stats()`** — Go stdlib DB pool metrics +- **K8s probes** — `/healthz/ready` + `/healthz/live` in Helm chart +- **Middleware skip paths** — `/metrics` already in `SkipPaths` list +- **`google/uuid`** — already in go.mod for request ID generation + +--- + +## CS0 — Structured Logging + Request ID + +Foundation changeset. All subsequent work emits structured logs. + +### `server/logging/logger.go` (new) + +`Init(format, level)` configures the global `slog` logger: +- `format=json` → `slog.NewJSONHandler(os.Stdout, opts)` +- `format=text` → `slog.NewTextHandler(os.Stdout, opts)` +- Levels: debug, info, warn, error + +### `server/middleware/request_id.go` (new) + +Generates UUID per request. Reuses existing `google/uuid`. +Honors inbound `X-Request-Id` header for trace propagation. +Sets `request_id` in Gin context and echoes header on response. + +### Middleware chain rewrite + +Changed `gin.Default()` → `gin.New()` with explicit chain: + +``` +RequestID → Prometheus → Logger → Recovery → CORS +``` + +Logger middleware rewritten from `log.Printf` to `slog.Info("request", ...)` +with structured fields: method, path, status, latency_ms, client_ip, +request_id, user_id. + +### Config additions + +- `LogFormat` (env: `LOG_FORMAT`, default: `text`) +- `LogLevel` (env: `LOG_LEVEL`, default: `info`) + +--- + +## CS1 — Prometheus `/metrics` Endpoint + +### `server/metrics/metrics.go` (new) + +All metric definitions in one file using `promauto`: + +| Metric | Type | Labels | +|--------|------|--------| +| `switchboard_http_requests_total` | Counter | method, path_pattern, status | +| `switchboard_http_request_duration_seconds` | Histogram | method, path_pattern | +| `switchboard_websocket_connections` | Gauge | — | +| `switchboard_completion_tokens_total` | Counter | direction, model_id | +| `switchboard_completions_total` | Counter | provider_config_id, model_id, status | +| `switchboard_completion_duration_seconds` | Histogram | provider_config_id, model_id | +| `switchboard_provider_status` | Gauge | provider_config_id | +| `switchboard_db_open_connections` | Gauge | — | +| `switchboard_db_in_use_connections` | Gauge | — | +| `switchboard_db_idle_connections` | Gauge | — | +| `switchboard_db_wait_count` | Gauge | — | +| `switchboard_db_wait_duration_seconds` | Gauge | — | +| `switchboard_task_executions_total` | Counter | status | +| `switchboard_sandbox_executions_total` | Counter | entry_point, status | + +### `server/metrics/db_collector.go` (new) + +Background goroutine reads `database.DB.Stats()` every 15s, updates +Prometheus gauges. + +### `server/middleware/prometheus.go` (new) + +Gin middleware. Uses `c.FullPath()` for `path_pattern` label to avoid +cardinality explosion from path parameters (e.g. `/channels/:id` +not `/channels/abc123`). + +### Instrumentation points + +- `events/ws.go` — `WebSocketConnections.Inc()/Dec()` on connect/disconnect +- `handlers/completion.go` — token counters in `logUsage()`, completion + duration/status in `recordHealth()` +- `handlers/stream_loop.go` — same instrumentation in `recordHealthFn()` +- `health/accumulator.go` — `ProviderStatus` gauge updated in `flush()` + +### Route + +`/metrics` mounted via `promhttp.Handler()` (no auth, standard scraping). + +--- + +## CS2 — OpenAPI Spec + Swagger UI + +### `server/static/openapi.yaml` (new) + +Hand-curated OpenAPI 3.0.3 spec covering core API groups: +Auth, Channels, Completions, Health, Metrics, WebSocket Tickets, +Admin (stats, health, usage, dashboard). + +### `server/static/swagger.html` (new) + +Minimal HTML loading Swagger UI v5 from unpkg CDN, pointing at +`openapi.yaml`. Zero build step. + +### Routes + +- `GET /api/docs` → serves embedded `swagger.html` +- `GET /api/docs/openapi.yaml` → serves embedded `openapi.yaml` + +Both files embedded via `//go:embed` directives. + +--- + +## CS3 — Grafana Dashboard + Alerting Rules + Helm + +Static files + Helm templates. No Go code changes. + +### `chart/dashboards/switchboard-overview.json` (new) + +Grafana dashboard with 10 panels: +- Request rate, error rate, latency percentiles +- WebSocket connections, completion rate/latency by provider +- Tokens/min by model, provider status +- DB connection pool, task executions + +Template variables: `$datasource`, `$namespace`, `$pod`. + +### `chart/alerting/switchboard-rules.yaml` (new) + +Source PrometheusRule with 6 alerts: + +| Alert | Condition | Severity | +|-------|-----------|----------| +| SwitchboardPodRestart | Restarts in 1h | warning | +| SwitchboardProviderDown | Status > 2 for 5m | critical | +| SwitchboardDBPoolExhaustion | In-use/open > 80% for 5m | warning | +| SwitchboardHighErrorRate | 5xx rate > 5% for 5m | warning | +| SwitchboardTaskFailureRate | Error rate > 25% for 10m | warning | +| SwitchboardNoCompletions | Zero completions for 15m | critical | + +### Helm templates (new, all gated `enabled: false` by default) + +- `chart/templates/servicemonitor.yaml` — ServiceMonitor CRD +- `chart/templates/grafana-dashboard-configmap.yaml` — ConfigMap for + Grafana sidecar discovery +- `chart/templates/prometheusrule.yaml` — PrometheusRule CRD + +### Helm modifications + +- `chart/values.yaml` — `logging` section + `monitoring` section +- `chart/templates/configmap.yaml` — `LOG_FORMAT`, `LOG_LEVEL` env vars +- `chart/templates/services.yaml` — Prometheus scrape annotations + +--- + +## CS4 — Admin Observability Dashboard + +Built-in admin page for real-time health without Grafana dependency. + +### `server/handlers/dashboard_admin.go` (new) + +`GET /api/v1/admin/dashboard` aggregating: +- Provider health summaries (from `HealthStore`) +- 24h usage totals (from `UsageStore`) +- DB pool stats (`database.DB.Stats()`) +- WebSocket connection count (`Hub.ConnCount()`) +- Recent errors (from `AuditStore`) +- Process uptime + +### Frontend + +- `SCAFFOLDING.dashboard` — stat cards grid + content container +- `ADMIN_LOADERS.dashboard` → `UI.loadAdminDashboard()` +- `ADMIN_SECTIONS.monitoring` — `dashboard` added as first section +- Dashboard auto-refreshes every 30s +- CSS: provider grid, DB pool bar gauge, error list + +### Admin template + +Monitoring tab now lands on `/admin/dashboard` instead of `/admin/usage`. + +--- + +## CS5 — Wiring, docker-compose, ICD Tests + +### docker-compose.yml + +Added `LOG_FORMAT` and `LOG_LEVEL` env vars with defaults. + +### ICD tests (`crud/observability.js`) + +| Test | Assertion | +|------|-----------| +| `GET /metrics` | 200, contains `switchboard_*` metrics | +| `GET /api/docs` | 200, contains `swagger-ui` | +| `GET /api/docs/openapi.yaml` | 200, contains `openapi:` and `paths:` | +| X-Request-Id generation | Response includes 36-char UUID header | +| X-Request-Id passthrough | Client-sent header echoed back | +| `GET /admin/dashboard` | 200, has `uptime`, `ws_connections`, `provider_health` | + +--- + +## Verification + +1. `LOG_FORMAT=json docker compose up` → JSON log lines on stdout +2. `curl localhost:8080/metrics | grep switchboard_` → all metrics present +3. `curl -I localhost:8080/api/v1/channels` → `X-Request-Id` header +4. `/api/docs` in browser → Swagger UI renders +5. `/admin` → Monitoring → Dashboard shows live stat cards +6. `helm lint chart/ --set monitoring.serviceMonitor.enabled=true` → passes +7. ICD test suite passes (existing + observability tests) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index d6fa373..9416a92 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -31,7 +31,7 @@ v0.9.x–v0.28.7 Foundation through Platform Polish ✅ Extension Track Operations Track │ │ v0.29.0 Starlark Sandbox ✅ v0.32.0 Multi-Replica HA ✅ - v0.29.1 API Extensions ✅ v0.33.0 Observability + v0.29.1 API Extensions ✅ v0.33.0 Observability ✅ v0.29.2 DB Extensions ✅ v0.34.0 Data Portability v0.29.3 Workflow Forms ✅ │ v0.30.0 Package Lifecycle ✅ │ @@ -43,6 +43,8 @@ v0.9.x–v0.28.7 Foundation through Platform Polish ✅ │ │ │ v0.35.0 Workflow Product │ │ + │ v0.36.0 Full OpenAPI Spec + │ │ ══════╪═══════════════════════════════╪══════ │ MVP v0.50.0 │ │ (v0.31.2 + v0.35.0 │ @@ -384,23 +386,33 @@ no new coordination infrastructure. See `docs/DESIGN-0.32.0.md`. **Schema:** `020_ha.sql` — `ws_tickets`, `rate_limit_counters`. -### v0.33.0 — Observability +### v0.33.0 — Observability ✅ Metrics, dashboards, alerting. Operate the platform without reading Go source code. Depends on: v0.32.0 (multi-replica metrics aggregation). -- [ ] Prometheus `/metrics` endpoint: request latency, active WebSocket - gauge, completion token counters, DB pool stats, provider health -- [ ] Grafana dashboard template: system overview, per-team usage, - provider latency, error rates -- [ ] Structured logging: `LOG_FORMAT=json`, request ID propagation, - correlation IDs in completion chains -- [ ] Alerting rules: OOM recovery, provider down, pool exhaustion, - task failure rate -- [ ] Admin dashboard surface: real-time health (built-in, no Grafana) -- [ ] Swagger/OpenAPI: auto-generated spec from route definitions, served at /api/docs +**Delivered (6 changesets):** +- [x] Structured logging (`slog`): `LOG_FORMAT=json|text`, + `LOG_LEVEL=debug|info|warn|error`. Request ID middleware + (`X-Request-Id`), correlation IDs through completion chain. +- [x] Prometheus `/metrics` endpoint: HTTP request counters/histograms, + WebSocket gauge, completion tokens/duration, provider status, + DB pool stats, task/sandbox execution counters. +- [x] OpenAPI 3.0.3 spec (hand-curated, core API groups) + Swagger UI + at `/api/docs` with system dark/light mode, WCAG AA contrast. +- [x] Grafana dashboard JSON: request rate, latency percentiles, + provider health, token usage, DB pool, Go runtime. +- [x] PrometheusRule alerting: OOM recovery, provider down, pool + exhaustion, high error rate, task failure spike. +- [x] Admin monitoring dashboard: provider health, 24h usage, DB pool, + WS connections, Go runtime stats, storage status, recent errors, + 30s auto-refresh. + +**Deferred to v0.36.0:** +- Full OpenAPI spec coverage (all 20 ICD domains) +- Bearer token / mTLS auth documentation in spec ### v0.34.0 — Data Portability @@ -472,6 +484,48 @@ Depends on: v0.31.2 (team workflow self-service), v0.29.0 (Starlark sandbox). - Data intake: form → Starlark enrichment → human review → follow-up - Onboarding: multi-stage form → conditional branching → team assignment +### v0.36.0 — Full OpenAPI Spec + +Complete OpenAPI 3.0.3 coverage for every API domain. Expand the +hand-curated spec from v0.33.0 (core groups only) to cover all 20 ICD +test domains. Document Bearer token auth and mTLS modes. + +Depends on: v0.33.0 (Swagger UI + initial spec). + +**API domains to document (matching ICD test suite):** +- [ ] Admin (system settings, stats, storage, users, provider config) +- [ ] Channels (full CRUD + membership, already partially covered) +- [ ] Completions (streaming + non-streaming, already partially covered) +- [ ] Extensions (package install, permissions, lifecycle) +- [ ] Knowledge (KB articles, sources, search, embeddings) +- [ ] Memory (conversation memory, compaction, search) +- [ ] Models (provider models, BYOK, capability matching) +- [ ] Notes (CRUD, graph links, search) +- [ ] Notifications (list, mark read, preferences) +- [ ] Personas (CRUD, system prompts, tool config) +- [ ] Profile (user profile, preferences, avatar) +- [ ] Projects (CRUD, membership, file uploads) +- [ ] Surfaces (extension surfaces, mounting, lifecycle) +- [ ] Tasks (scheduled tasks, execution history, Starlark tasks) +- [ ] Teams (CRUD, membership, roles, slugs) +- [ ] Team Workflows (team-scoped CRUD, stages, publish) +- [ ] Workflows (platform-level CRUD, stages, instances, assignments) +- [ ] Workspaces (CRUD, membership, settings) +- [ ] Dashboard Package (package-specific API routes) +- [ ] Editor Package (package-specific API routes) + +**Auth documentation:** +- [ ] Bearer JWT flow (login → token → `Authorization: Bearer `) +- [ ] mTLS mode (NPE-to-NPE, no Bearer needed) +- [ ] API key mode (for service-to-service integrations) +- [ ] Security scheme definitions in OpenAPI spec + +**Quality:** +- [ ] Request/response schemas with examples for every endpoint +- [ ] Error response schemas (400, 401, 403, 404, 409, 422, 429, 500) +- [ ] Pagination parameters documented consistently +- [ ] WebSocket event schemas (connection, ticket, event types) + --- ## MVP v0.50.0 @@ -487,6 +541,7 @@ reading Go source code. Team admins build workflows visually. data portability) - Workflow product v0.35.0 (form rendering, data pipeline, conditional routing, structured review, monitoring dashboard) +- Full OpenAPI spec v0.36.0 (complete API documentation for all domains) **Additionally requires:** - [ ] Deployment guide: step-by-step for IT team (PG cluster, diff --git a/nginx.conf b/nginx.conf index b224a0d..cab7345 100644 --- a/nginx.conf +++ b/nginx.conf @@ -38,6 +38,10 @@ server { proxy_pass $backend; } + location = /metrics { + proxy_pass $backend; + } + location = /ws { proxy_pass $backend; proxy_http_version 1.1; diff --git a/packages/icd-test-runner/js/crud/observability.js b/packages/icd-test-runner/js/crud/observability.js new file mode 100644 index 0000000..7647586 --- /dev/null +++ b/packages/icd-test-runner/js/crud/observability.js @@ -0,0 +1,73 @@ +/** + * ICD Test Runner — Observability CRUD Tests (v0.33.0) + * Tests /metrics, /api/docs, /admin/dashboard, X-Request-Id, structured logging. + */ +(function () { + 'use strict'; + var T = window.ICD; + if (!T) return; + + T.crud.observability = async function () { + + // -- GET /metrics (Prometheus endpoint, no auth required) -- + await T.test('crud', 'observability', 'GET /metrics returns Prometheus text', async function () { + var resp = await fetch(T.base + '/metrics'); + T.assert(resp.ok, 'expected 200 from /metrics, got ' + resp.status); + var text = await resp.text(); + T.assert(text.indexOf('switchboard_http_requests_total') !== -1, 'expected switchboard_http_requests_total in /metrics'); + T.assert(text.indexOf('switchboard_http_request_duration_seconds') !== -1, 'expected switchboard_http_request_duration_seconds in /metrics'); + T.assert(text.indexOf('switchboard_websocket_connections') !== -1, 'expected switchboard_websocket_connections in /metrics'); + }); + + // -- GET /api/docs (Swagger UI) -- + await T.test('crud', 'observability', 'GET /api/docs returns Swagger UI HTML', async function () { + var resp = await fetch(T.base + '/api/docs'); + T.assert(resp.ok, 'expected 200 from /api/docs, got ' + resp.status); + var text = await resp.text(); + T.assert(text.indexOf('swagger-ui') !== -1, 'expected swagger-ui in /api/docs HTML'); + }); + + // -- GET /api/docs/openapi.yaml -- + await T.test('crud', 'observability', 'GET /api/docs/openapi.yaml returns valid YAML', async function () { + var resp = await fetch(T.base + '/api/docs/openapi.yaml'); + T.assert(resp.ok, 'expected 200 from /api/docs/openapi.yaml, got ' + resp.status); + var text = await resp.text(); + T.assert(text.indexOf('openapi:') !== -1, 'expected openapi: key in YAML'); + T.assert(text.indexOf('paths:') !== -1, 'expected paths: key in YAML'); + }); + + // -- X-Request-Id header propagation -- + await T.test('crud', 'observability', 'X-Request-Id header on API responses', async function () { + var resp = await fetch(T.base + '/health'); + T.assert(resp.ok, 'expected 200'); + var rid = resp.headers.get('X-Request-Id'); + T.assert(rid, 'expected X-Request-Id header in response'); + T.assert(rid.length === 36, 'X-Request-Id should be UUID (36 chars), got ' + rid.length); + }); + + // -- X-Request-Id passthrough (client sends, server echoes) -- + await T.test('crud', 'observability', 'X-Request-Id passthrough', async function () { + var customId = 'test-' + Date.now(); + var resp = await fetch(T.base + '/health', { + headers: { 'X-Request-Id': customId } + }); + T.assert(resp.ok, 'expected 200'); + var echoed = resp.headers.get('X-Request-Id'); + T.assert(echoed === customId, 'expected echoed X-Request-Id "' + customId + '", got "' + echoed + '"'); + }); + + // -- GET /admin/dashboard (admin auth required) -- + await T.test('crud', 'observability', 'GET /admin/dashboard returns dashboard data', async function () { + var d = await T.apiGet('/admin/dashboard'); + T.assertHasKey(d, 'uptime', '/admin/dashboard'); + T.assertHasKey(d, 'ws_connections', '/admin/dashboard'); + T.assertHasKey(d, 'provider_health', '/admin/dashboard'); + T.assert(typeof d.ws_connections === 'number', 'ws_connections should be number'); + T.assert(typeof d.uptime === 'string', 'uptime should be string'); + // provider_health can be null or array + if (d.provider_health) { + T.assert(Array.isArray(d.provider_health), 'provider_health should be array'); + } + }); + }; +})(); diff --git a/packages/icd-test-runner/js/main.js b/packages/icd-test-runner/js/main.js index cca9b83..7adc21b 100644 --- a/packages/icd-test-runner/js/main.js +++ b/packages/icd-test-runner/js/main.js @@ -81,6 +81,7 @@ 'crud/surfaces.js', 'crud/editor-package.js', 'crud/dashboard-package.js', + 'crud/observability.js', // Orchestrator (must come after all crud/*.js) 'tier-crud.js', 'tier-authz.js', diff --git a/server/.env.example b/server/.env.example index 06a3ade..5f8126e 100644 --- a/server/.env.example +++ b/server/.env.example @@ -43,6 +43,8 @@ CORS_ALLOWED_ORIGINS=http://localhost:3000,http://localhost:8080 # BANNER_POSITION=top # ── Logging ────────────────────────────────── +# LOG_FORMAT: "text" (default, human-readable) or "json" (structured, machine-parseable) +LOG_FORMAT=text LOG_LEVEL=info # ── File Storage ───────────────────────────── diff --git a/server/Dockerfile b/server/Dockerfile index f279cbf..c4f44f8 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -6,7 +6,7 @@ # ========================================== # Build stage -FROM golang:1.22-bookworm AS builder +FROM golang:1.23-bookworm AS builder ARG APP_VERSION=dev diff --git a/server/config/config.go b/server/config/config.go index aaa7569..06b2537 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -82,6 +82,12 @@ type Config struct { // hours are marked 'stale'. Default 72 (3 days). Set to 0 to disable. WorkflowStaleHours int + // Structured logging (v0.33.0) + // LOG_FORMAT: "text" (default, backward-compatible) or "json" (structured). + // LOG_LEVEL: "debug", "info" (default), "warn", "error". + LogFormat string + LogLevel string + // Auth mode (v0.24.0): "builtin" (default) | "mtls" | "oidc" AuthMode string @@ -151,6 +157,9 @@ func Load() *Config { WorkflowStaleHours: getEnvInt("WORKFLOW_STALE_HOURS", 72), + LogFormat: getEnv("LOG_FORMAT", "text"), + LogLevel: getEnv("LOG_LEVEL", "info"), + AuthMode: getEnv("AUTH_MODE", "builtin"), // mTLS diff --git a/server/events/ws.go b/server/events/ws.go index 3a83393..33cd1f6 100644 --- a/server/events/ws.go +++ b/server/events/ws.go @@ -10,6 +10,8 @@ import ( "github.com/gin-gonic/gin" "github.com/gorilla/websocket" + + "chat-switchboard/metrics" ) const ( @@ -121,6 +123,7 @@ func (h *Hub) HandleWebSocket(c *gin.Context) { h.mu.Unlock() log.Printf("[ws] connected: %s (user=%s, total=%d)", connID, userID, h.ConnCount()) + metrics.WebSocketConnections.Inc() // Subscribe this connection to bus events destined for clients conn.subscribeToBus() @@ -214,6 +217,7 @@ func (h *Hub) removeConn(conn *Conn) { } log.Printf("[ws] disconnected: %s (total=%d)", conn.id, h.ConnCount()) + metrics.WebSocketConnections.Dec() // Publish presence offline (only if no other conns for this user) if len(h.ConnsByUser(conn.userID)) == 0 { diff --git a/server/go.mod b/server/go.mod index b4d3abf..8625e2a 100644 --- a/server/go.mod +++ b/server/go.mod @@ -1,6 +1,6 @@ module chat-switchboard -go 1.22 +go 1.23.0 require ( github.com/gin-gonic/gin v1.9.1 @@ -10,15 +10,18 @@ require ( github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.9 github.com/minio/minio-go/v7 v7.0.82 + github.com/prometheus/client_golang v1.23.2 github.com/robfig/cron/v3 v3.0.1 go.starlark.net v0.0.0-20260210143700-b62fd896b91b - golang.org/x/crypto v0.28.0 - golang.org/x/net v0.30.0 + golang.org/x/crypto v0.41.0 + golang.org/x/net v0.43.0 modernc.org/sqlite v1.34.5 ) require ( + github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect @@ -29,23 +32,29 @@ require ( github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.66.1 // indirect + github.com/prometheus/procfs v0.16.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rs/xid v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/sys v0.26.0 // indirect - golang.org/x/text v0.19.0 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/libc v1.55.3 // indirect modernc.org/mathutil v1.6.0 // indirect diff --git a/server/go.sum b/server/go.sum index 97f2a0a..56dc482 100644 --- a/server/go.sum +++ b/server/go.sum @@ -1,9 +1,14 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -29,8 +34,8 @@ github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= @@ -42,12 +47,18 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -63,16 +74,28 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= +github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -85,41 +108,44 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= go.starlark.net v0.0.0-20260210143700-b62fd896b91b h1:mDO9/2PuBcapqFbhiCmFcEQZvlQnk3ILEZR+a8NL1z4= go.starlark.net v0.0.0-20260210143700-b62fd896b91b/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= -golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/server/handlers/completion.go b/server/handlers/completion.go index b4cf52c..d97d3b9 100644 --- a/server/handlers/completion.go +++ b/server/handlers/completion.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "log" + "log/slog" "net/http" "strings" "time" @@ -21,6 +22,7 @@ import ( "chat-switchboard/filters" "chat-switchboard/health" "chat-switchboard/knowledge" + "chat-switchboard/metrics" "chat-switchboard/models" "chat-switchboard/notifications" "chat-switchboard/providers" @@ -224,6 +226,16 @@ func (h *CompletionHandler) Complete(c *gin.Context) { userID := getUserID(c) + // v0.33.0: Correlation ID — same as request_id, propagated through + // completion chain for cross-referencing logs. + correlationID, _ := c.Get("request_id") + slog.Info("completion.start", + "request_id", correlationID, + "channel_id", channelID, + "user_id", userID, + "model", req.Model, + ) + // v0.24.3: Session participants are pre-validated by AuthOrSession middleware if isSessionAuth(c) { if !sessionCanAccessChannel(c, channelID) { @@ -1912,6 +1924,20 @@ func (h *CompletionHandler) logUsage( if err := h.stores.Usage.Log(c.Request.Context(), entry); err != nil { log.Printf("⚠ Failed to log usage: %v", err) } + + // v0.33.0: Prometheus token counters + metrics.CompletionTokensTotal.WithLabelValues("prompt", modelID).Add(float64(inputTokens)) + metrics.CompletionTokensTotal.WithLabelValues("completion", modelID).Add(float64(outputTokens)) + + // v0.33.0: Structured usage log for observability correlation + correlationID, _ := c.Get("request_id") + slog.Info("completion.usage", + "request_id", correlationID, + "model_id", modelID, + "provider_config_id", configID, + "input_tokens", inputTokens, + "output_tokens", outputTokens, + ) } // calcCost computes the cost for a given token count and price-per-million. @@ -1923,23 +1949,36 @@ func calcCost(tokens int, pricePerM *float64) *float64 { return &cost } -// recordHealth records provider call outcome for health tracking (v0.22.0). +// recordHealth records provider call outcome for health tracking (v0.22.0) +// and Prometheus completion metrics (v0.33.0). func (h *CompletionHandler) recordHealth(configID string, start time.Time, err error) { + duration := time.Since(start) + latencyMs := int(duration.Milliseconds()) + + // v0.33.0: Prometheus completion duration (always, even on error) + if configID != "" { + metrics.CompletionDuration.WithLabelValues(configID, "").Observe(duration.Seconds()) + } + if h.health == nil || configID == "" { return } - latencyMs := int(time.Since(start).Milliseconds()) if err != nil { errMsg := err.Error() + status := "error" if strings.Contains(errMsg, "context deadline exceeded") || strings.Contains(errMsg, "timeout") { h.health.RecordTimeout(configID, latencyMs, errMsg) + status = "timeout" } else if strings.Contains(errMsg, "HTTP 429") || strings.Contains(errMsg, "rate limit") || strings.Contains(errMsg, "Too Many Requests") { h.health.RecordRateLimit(configID, latencyMs, errMsg) + status = "rate_limited" } else { h.health.RecordError(configID, latencyMs, errMsg) } + metrics.CompletionsTotal.WithLabelValues(configID, "", status).Inc() } else { h.health.RecordSuccess(configID, latencyMs) + metrics.CompletionsTotal.WithLabelValues(configID, "", "success").Inc() } } diff --git a/server/handlers/dashboard_admin.go b/server/handlers/dashboard_admin.go new file mode 100644 index 0000000..0309be0 --- /dev/null +++ b/server/handlers/dashboard_admin.go @@ -0,0 +1,118 @@ +package handlers + +import ( + "database/sql" + "net/http" + "runtime" + "time" + + "github.com/gin-gonic/gin" + + "chat-switchboard/database" + "chat-switchboard/events" + "chat-switchboard/health" + "chat-switchboard/models" + "chat-switchboard/store" +) + +// ── Dashboard Admin Handler ───────────────── +// GET /api/v1/admin/dashboard +// Aggregates live operational data for the built-in admin monitoring page. + +var processStartTime = time.Now() + +type DashboardAdminHandler struct { + stores store.Stores + healthStore health.Store + hub *events.Hub +} + +func NewDashboardAdminHandler(stores store.Stores, hs health.Store, hub *events.Hub) *DashboardAdminHandler { + return &DashboardAdminHandler{stores: stores, healthStore: hs, hub: hub} +} + +type runtimeStats struct { + Goroutines int `json:"goroutines"` + HeapMB int `json:"heap_mb"` + SysMB int `json:"sys_mb"` + NumGC uint32 `json:"num_gc"` + GoVersion string `json:"go_version"` +} + +func (h *DashboardAdminHandler) GetDashboard(c *gin.Context) { + ctx := c.Request.Context() + + // Provider health summaries + var providerHealth []models.ProviderHealthSummary + windows, err := h.healthStore.ListAllCurrent(ctx) + if err == nil { + for _, w := range windows { + providerHealth = append(providerHealth, models.ProviderHealthSummary{ + ProviderConfigID: w.ProviderConfigID, + Status: health.DeriveStatus(w.ErrorRate(), w.RequestCount), + RequestCount: w.RequestCount, + ErrorRate: w.ErrorRate(), + ErrorCount: w.ErrorCount, + RateLimitCount: w.RateLimitCount, + TimeoutCount: w.TimeoutCount, + AvgLatencyMs: w.AvgLatencyMs(), + MaxLatencyMs: w.MaxLatencyMs, + LastError: w.LastError, + LastErrorAt: w.LastErrorAt, + }) + } + } + + // 24h usage totals + since := time.Now().Add(-24 * time.Hour) + totals, _ := h.stores.Usage.GetTotals(ctx, store.UsageQueryOptions{ + Since: &since, + }) + + // DB pool stats + var dbPool *sql.DBStats + if database.DB != nil { + stats := database.DB.Stats() + dbPool = &stats + } + + // WebSocket connections + wsCount := 0 + if h.hub != nil { + wsCount = h.hub.ConnCount() + } + + // Recent errors from audit log (last 10 error-related entries) + var recentErrors []models.AuditEntry + entries, _, auditErr := h.stores.Audit.List(ctx, store.AuditListOptions{ + ListOptions: store.ListOptions{Limit: 10}, + ResourceType: "error", + }) + if auditErr == nil { + recentErrors = entries + } + + // Go runtime stats + var mem runtime.MemStats + runtime.ReadMemStats(&mem) + rt := runtimeStats{ + Goroutines: runtime.NumGoroutine(), + HeapMB: int(mem.HeapAlloc / 1024 / 1024), + SysMB: int(mem.Sys / 1024 / 1024), + NumGC: mem.NumGC, + GoVersion: runtime.Version(), + } + + // Uptime + uptime := time.Since(processStartTime).Truncate(time.Second).String() + + SafeJSON(c, http.StatusOK, gin.H{ + "provider_health": providerHealth, + "usage_24h": totals, + "db_pool": dbPool, + "ws_connections": wsCount, + "recent_errors": recentErrors, + "runtime": rt, + "uptime": uptime, + }) +} diff --git a/server/handlers/stream_loop.go b/server/handlers/stream_loop.go index 5c2a617..92aef96 100644 --- a/server/handlers/stream_loop.go +++ b/server/handlers/stream_loop.go @@ -12,27 +12,39 @@ import ( "github.com/gin-gonic/gin" "chat-switchboard/events" + "chat-switchboard/metrics" "chat-switchboard/providers" "chat-switchboard/sandbox" "chat-switchboard/store" "chat-switchboard/tools" ) -// recordHealthFn records provider health from standalone streaming functions. +// recordHealthFn records provider health from standalone streaming functions +// and updates Prometheus completion metrics (v0.33.0). func recordHealthFn(hr HealthRecorder, configID string, start time.Time, err error) { + duration := time.Since(start) + latencyMs := int(duration.Milliseconds()) + + if configID != "" { + metrics.CompletionDuration.WithLabelValues(configID, "").Observe(duration.Seconds()) + } + if hr == nil || configID == "" { return } - latencyMs := int(time.Since(start).Milliseconds()) if err != nil { errMsg := err.Error() + status := "error" if strings.Contains(errMsg, "HTTP 429") || strings.Contains(errMsg, "rate limit") || strings.Contains(errMsg, "Too Many Requests") { hr.RecordRateLimit(configID, latencyMs, errMsg) + status = "rate_limited" } else { hr.RecordError(configID, latencyMs, errMsg) } + metrics.CompletionsTotal.WithLabelValues(configID, "", status).Inc() } else { hr.RecordSuccess(configID, latencyMs) + metrics.CompletionsTotal.WithLabelValues(configID, "", "success").Inc() } } diff --git a/server/health/accumulator.go b/server/health/accumulator.go index 052f178..15a9c75 100644 --- a/server/health/accumulator.go +++ b/server/health/accumulator.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "chat-switchboard/metrics" "chat-switchboard/models" ) @@ -335,6 +336,25 @@ func (a *Accumulator) flush() { if err := a.store.UpsertWindow(ctx, w); err != nil { log.Printf("⚠ health: flush failed for provider %s: %v", b.providerConfigID, err) } + + // v0.33.0: Update Prometheus provider status gauge + errorRate := float64(0) + if b.requestCount > 0 { + errorRate = float64(b.errorCount) / float64(b.requestCount) + } + status := DeriveStatus(errorRate, b.requestCount) + var statusVal float64 + switch status { + case models.StatusHealthy: + statusVal = 1 + case models.StatusDegraded: + statusVal = 2 + case models.StatusDown: + statusVal = 3 + default: + statusVal = 0 + } + metrics.ProviderStatus.WithLabelValues(b.providerConfigID).Set(statusVal) } } diff --git a/server/logging/logger.go b/server/logging/logger.go new file mode 100644 index 0000000..c136246 --- /dev/null +++ b/server/logging/logger.go @@ -0,0 +1,39 @@ +// Package logging provides structured logging configuration for +// Chat Switchboard using the standard library's log/slog package. +// Configured via LOG_FORMAT ("text"|"json") and LOG_LEVEL env vars. +package logging + +import ( + "log/slog" + "os" + "strings" +) + +// Init configures the global slog logger. +// +// - format: "json" for machine-parseable output, anything else for +// human-readable key=value pairs (backward-compatible default). +// - level: "debug", "info" (default), "warn", "error". +func Init(format, level string) { + var lvl slog.Level + switch strings.ToLower(level) { + case "debug": + lvl = slog.LevelDebug + case "warn", "warning": + lvl = slog.LevelWarn + case "error": + lvl = slog.LevelError + default: + lvl = slog.LevelInfo + } + + opts := &slog.HandlerOptions{Level: lvl} + var handler slog.Handler + if strings.ToLower(format) == "json" { + handler = slog.NewJSONHandler(os.Stdout, opts) + } else { + handler = slog.NewTextHandler(os.Stdout, opts) + } + + slog.SetDefault(slog.New(handler)) +} diff --git a/server/main.go b/server/main.go index 87b1186..05e399a 100644 --- a/server/main.go +++ b/server/main.go @@ -1,7 +1,9 @@ package main import ( + "bytes" "context" + _ "embed" "encoding/json" "fmt" "log" @@ -12,6 +14,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus/promhttp" "chat-switchboard/auth" "chat-switchboard/compaction" @@ -24,6 +27,8 @@ import ( "chat-switchboard/sandbox" "chat-switchboard/handlers" "chat-switchboard/health" + "chat-switchboard/logging" + "chat-switchboard/metrics" "chat-switchboard/knowledge" "chat-switchboard/memory" "chat-switchboard/middleware" @@ -43,6 +48,14 @@ import ( "chat-switchboard/workspace" ) +// v0.33.0: Embedded OpenAPI spec and Swagger UI for /api/docs. +// +//go:embed static/openapi.yaml +var openapiSpec []byte + +//go:embed static/swagger.html +var swaggerHTML []byte + func main() { // ── Subcommand dispatch ────────────────── // Usage: switchboard vault rekey @@ -58,6 +71,10 @@ func main() { // ── Server startup ────────────────────── cfg := config.Load() + // v0.33.0: Structured logging — must be first so all subsequent + // log output goes through slog. + logging.Init(cfg.LogFormat, cfg.LogLevel) + // Register LLM providers providers.Init() @@ -129,6 +146,9 @@ func main() { healthAccum = health.NewAccumulator(healthStore) defer healthAccum.Stop() + // v0.33.0: Start Prometheus DB pool collector + metrics.StartDBCollector(database.DB, 15*time.Second) + // Auto-disable: deactivate providers that are "down" for N consecutive // hourly windows. Configured via PROVIDER_AUTO_DISABLE_THRESHOLD env var. // Default: 3 (3 consecutive "down" hours triggers deactivation). Set to 0 to disable. @@ -390,7 +410,11 @@ func main() { log.Printf(" 🔗 Pre-completion filter chain: %d filters", filterChain.Len()) - r := gin.Default() + r := gin.New() + r.Use(middleware.RequestID()) + r.Use(middleware.Prometheus()) + r.Use(middleware.Logger()) + r.Use(middleware.Recovery()) userCache := middleware.NewUserStatusCache() r.Use(middleware.CORS(cfg)) @@ -478,6 +502,19 @@ func main() { c.JSON(200, gin.H{"status": "ok"}) }) + // v0.33.0: Prometheus metrics endpoint (no auth — Prometheus scrapes directly) + base.GET("/metrics", gin.WrapH(promhttp.Handler())) + + // v0.33.0: OpenAPI spec + Swagger UI (no auth — documentation) + base.GET("/api/docs", func(c *gin.Context) { + c.Data(http.StatusOK, "text/html; charset=utf-8", swaggerHTML) + }) + base.GET("/api/docs/openapi.yaml", func(c *gin.Context) { + // Replace ${VERSION} placeholder with actual version from VERSION file + patched := bytes.Replace(openapiSpec, []byte("${VERSION}"), []byte(Version), 1) + c.Data(http.StatusOK, "application/yaml", patched) + }) + // WebSocket endpoint base.GET("/ws", middleware.WsAuth(cfg, stores.Users, userCache, ticketAdapter), hub.HandleWebSocket) @@ -1207,6 +1244,10 @@ func main() { admin.PUT("/extensions/:id/secrets", extSecH.SetSecrets) admin.DELETE("/extensions/:id/secrets", extSecH.DeleteSecrets) + // Admin Dashboard (v0.33.0) + dashAdm := handlers.NewDashboardAdminHandler(stores, healthStore, hub) + admin.GET("/dashboard", dashAdm.GetDashboard) + // Provider Health (admin — v0.22.0) healthAdm := handlers.NewHealthAdminHandler(healthStore, stores) admin.GET("/providers/health", healthAdm.GetAllProviderHealth) diff --git a/server/metrics/db_collector.go b/server/metrics/db_collector.go new file mode 100644 index 0000000..1f83a34 --- /dev/null +++ b/server/metrics/db_collector.go @@ -0,0 +1,26 @@ +package metrics + +import ( + "database/sql" + "time" +) + +// StartDBCollector begins a background goroutine that reads sql.DBStats +// every interval and updates the Prometheus DB pool gauges. +func StartDBCollector(db *sql.DB, interval time.Duration) { + if db == nil { + return + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for range ticker.C { + stats := db.Stats() + DBOpenConnections.Set(float64(stats.OpenConnections)) + DBInUseConnections.Set(float64(stats.InUse)) + DBIdleConnections.Set(float64(stats.Idle)) + DBWaitCount.Set(float64(stats.WaitCount)) + DBWaitDuration.Set(stats.WaitDuration.Seconds()) + } + }() +} diff --git a/server/metrics/metrics.go b/server/metrics/metrics.go new file mode 100644 index 0000000..a560218 --- /dev/null +++ b/server/metrics/metrics.go @@ -0,0 +1,97 @@ +// Package metrics defines all Prometheus metrics for Chat Switchboard. +// All metrics use the "switchboard_" prefix. Registered via promauto +// so they are available on the default registry's /metrics endpoint. +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// ── HTTP Request Metrics ──────────────────── + +var ( + HTTPRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "switchboard_http_requests_total", + Help: "Total HTTP requests by method, path pattern, and status code.", + }, []string{"method", "path_pattern", "status"}) + + HTTPRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "switchboard_http_request_duration_seconds", + Help: "HTTP request latency in seconds.", + Buckets: []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30}, + }, []string{"method", "path_pattern"}) +) + +// ── WebSocket Metrics ─────────────────────── + +var WebSocketConnections = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "switchboard_websocket_connections", + Help: "Current number of active WebSocket connections.", +}) + +// ── Completion / Token Metrics ────────────── + +var ( + CompletionTokensTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "switchboard_completion_tokens_total", + Help: "Total tokens processed by direction (prompt|completion) and model.", + }, []string{"direction", "model_id"}) + + CompletionsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "switchboard_completions_total", + Help: "Total completion requests by provider, model, and status.", + }, []string{"provider_config_id", "model_id", "status"}) + + CompletionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "switchboard_completion_duration_seconds", + Help: "Completion request latency (wall time including tool loops).", + Buckets: []float64{0.1, 0.5, 1, 2, 5, 10, 30, 60, 120}, + }, []string{"provider_config_id", "model_id"}) +) + +// ── Provider Health ───────────────────────── + +var ProviderStatus = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "switchboard_provider_status", + Help: "Provider health status: 0=unknown, 1=healthy, 2=degraded, 3=down.", +}, []string{"provider_config_id"}) + +// ── Database Pool Metrics ─────────────────── + +var ( + DBOpenConnections = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "switchboard_db_open_connections", + Help: "Number of open database connections.", + }) + DBInUseConnections = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "switchboard_db_in_use_connections", + Help: "Number of database connections currently in use.", + }) + DBIdleConnections = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "switchboard_db_idle_connections", + Help: "Number of idle database connections.", + }) + DBWaitCount = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "switchboard_db_wait_count_total", + Help: "Total number of connections waited for.", + }) + DBWaitDuration = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "switchboard_db_wait_duration_seconds_total", + Help: "Total time blocked waiting for a new connection (seconds).", + }) +) + +// ── Task Scheduler Metrics ────────────────── + +var TaskExecutionsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "switchboard_task_executions_total", + Help: "Total task executions by status (success|error).", +}, []string{"status"}) + +// ── Sandbox Metrics ───────────────────────── + +var SandboxExecutionsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "switchboard_sandbox_executions_total", + Help: "Total Starlark sandbox executions by entry point and status.", +}, []string{"entry_point", "status"}) diff --git a/server/middleware/logging.go b/server/middleware/logging.go index a8f79d8..c8822bf 100644 --- a/server/middleware/logging.go +++ b/server/middleware/logging.go @@ -1,130 +1,112 @@ package middleware import ( - "log" + "log/slog" "net/http" "time" "github.com/gin-gonic/gin" ) -// RequestLoggerConfig holds the configuration for the request logger +// RequestLoggerConfig holds the configuration for the request logger. type RequestLoggerConfig struct { - SkipPaths []string - Logger *log.Logger - TimeFormat string - TimeZone string + SkipPaths []string } -// DefaultRequestLoggerConfig returns the default request logger configuration +// DefaultRequestLoggerConfig returns the default request logger configuration. func DefaultRequestLoggerConfig() *RequestLoggerConfig { return &RequestLoggerConfig{ - SkipPaths: []string{"/health", "/ready", "/metrics"}, - Logger: log.Default(), - TimeFormat: "2006/01/02 - 15:04:05", - TimeZone: "UTC", + SkipPaths: []string{"/health", "/ready", "/metrics"}, } } -// Logger returns a Gin middleware handler for logging requests +// Logger returns a Gin middleware handler for logging requests. func Logger() gin.HandlerFunc { return LoggerWithConfig(DefaultRequestLoggerConfig()) } -// LoggerWithConfig returns a Gin middleware handler for logging requests with custom configuration +// LoggerWithConfig returns a Gin middleware handler for structured +// request logging via slog. When LOG_FORMAT=json the output is +// machine-parseable JSON lines; otherwise human-readable key=value. func LoggerWithConfig(config *RequestLoggerConfig) gin.HandlerFunc { + skip := make(map[string]bool, len(config.SkipPaths)) + for _, p := range config.SkipPaths { + skip[p] = true + } + return func(c *gin.Context) { - // Skip logging for certain paths path := c.Request.URL.Path - skip := false - for _, skipPath := range config.SkipPaths { - if path == skipPath { - skip = true - break - } - } - - start := time.Now() - raw := c.Request.URL.RawQuery - - c.Next() - - if skip { + if skip[path] { + c.Next() return } - - // Calculate latency - latency := time.Since(start) - - // Get the client IP - clientIP := c.ClientIP() - - // Get the request method - method := c.Request.Method - - // Get the status code - statusCode := c.Writer.Status() - - // Log the request - if raw != "" { - raw = "?" + raw + + start := time.Now() + c.Next() + + reqID, _ := c.Get(RequestIDKey) + + attrs := []any{ + "method", c.Request.Method, + "path", path, + "status", c.Writer.Status(), + "latency_ms", time.Since(start).Milliseconds(), + "client_ip", c.ClientIP(), } - - config.Logger.Printf( - "[%s] %s %s%s | %d | %v | %s", - clientIP, - method, - path, - raw, - statusCode, - latency, - c.Errors.ByType(gin.ErrorTypePrivate).String(), - ) + if id, ok := reqID.(string); ok && id != "" { + attrs = append(attrs, "request_id", id) + } + if uid := c.GetString("user_id"); uid != "" { + attrs = append(attrs, "user_id", uid) + } + if errs := c.Errors.ByType(gin.ErrorTypePrivate).String(); errs != "" { + attrs = append(attrs, "errors", errs) + } + + slog.Info("request", attrs...) } } -// Recovery returns a Gin middleware handler for recovering from panics +// Recovery returns a Gin middleware handler for recovering from panics. func Recovery() gin.HandlerFunc { return RecoveryWithConfig(DefaultRecoveryConfig()) } -// RecoveryConfig holds the configuration for the recovery middleware +// RecoveryConfig holds the configuration for the recovery middleware. type RecoveryConfig struct { - Logger *log.Logger - TimeFormat string - TimeZone string SkipStackTrace bool StackTraceSize int StackAll bool } -// DefaultRecoveryConfig returns the default recovery configuration +// DefaultRecoveryConfig returns the default recovery configuration. func DefaultRecoveryConfig() *RecoveryConfig { return &RecoveryConfig{ - Logger: log.Default(), - TimeFormat: "2006/01/02 - 15:04:05", - TimeZone: "UTC", SkipStackTrace: false, StackTraceSize: 1024, StackAll: false, } } -// RecoveryWithConfig returns a Gin middleware handler for recovering from panics with custom configuration +// RecoveryWithConfig returns a Gin middleware handler for recovering +// from panics with custom configuration. func RecoveryWithConfig(config *RecoveryConfig) gin.HandlerFunc { return func(c *gin.Context) { defer func() { if err := recover(); err != nil { - // Log the panic - config.Logger.Printf("[PANIC] %v", err) - - // Abort with a 500 error + reqID, _ := c.Get(RequestIDKey) + slog.Error("panic", + "error", err, + "method", c.Request.Method, + "path", c.Request.URL.Path, + "request_id", reqID, + ) c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ "error": "internal server error", }) } }() - + c.Next() } -} \ No newline at end of file +} diff --git a/server/middleware/prometheus.go b/server/middleware/prometheus.go new file mode 100644 index 0000000..7735daf --- /dev/null +++ b/server/middleware/prometheus.go @@ -0,0 +1,30 @@ +package middleware + +import ( + "strconv" + "time" + + "github.com/gin-gonic/gin" + + "chat-switchboard/metrics" +) + +// Prometheus returns a Gin middleware that records HTTP request metrics. +// Uses c.FullPath() (Gin's registered route pattern, e.g. +// "/api/v1/channels/:id") to avoid label cardinality explosion from +// path parameters. +func Prometheus() gin.HandlerFunc { + return func(c *gin.Context) { + start := time.Now() + c.Next() + + pattern := c.FullPath() + if pattern == "" { + pattern = "unmatched" + } + status := strconv.Itoa(c.Writer.Status()) + + metrics.HTTPRequestsTotal.WithLabelValues(c.Request.Method, pattern, status).Inc() + metrics.HTTPRequestDuration.WithLabelValues(c.Request.Method, pattern).Observe(time.Since(start).Seconds()) + } +} diff --git a/server/middleware/request_id.go b/server/middleware/request_id.go new file mode 100644 index 0000000..a1d04f1 --- /dev/null +++ b/server/middleware/request_id.go @@ -0,0 +1,28 @@ +package middleware + +import ( + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +const ( + // RequestIDKey is the Gin context key for the request ID. + RequestIDKey = "request_id" + // RequestIDHeader is the HTTP header used to propagate request IDs. + RequestIDHeader = "X-Request-Id" +) + +// RequestID generates a UUID per request and sets it in the Gin context +// and response header. If the incoming request already carries an +// X-Request-Id header, it is preserved (useful for tracing across proxies). +func RequestID() gin.HandlerFunc { + return func(c *gin.Context) { + id := c.GetHeader(RequestIDHeader) + if id == "" { + id = uuid.New().String() + } + c.Set(RequestIDKey, id) + c.Header(RequestIDHeader, id) + c.Next() + } +} diff --git a/server/pages/templates/surfaces/admin.html b/server/pages/templates/surfaces/admin.html index f39387f..569ea55 100644 --- a/server/pages/templates/surfaces/admin.html +++ b/server/pages/templates/surfaces/admin.html @@ -40,7 +40,7 @@ System - + Monitoring diff --git a/server/static/openapi.yaml b/server/static/openapi.yaml new file mode 100644 index 0000000..c35f468 --- /dev/null +++ b/server/static/openapi.yaml @@ -0,0 +1,643 @@ +openapi: 3.0.3 +info: + title: Chat Switchboard API + description: | + Multi-provider AI chat platform with team workspaces, workflow automation, + and extensibility via Starlark packages. + + **Authentication:** Most endpoints require a Bearer JWT token obtained via + `POST /api/v1/auth/login`. Include it as `Authorization: Bearer `. + + **WebSocket:** Real-time events are delivered over WebSocket at `/ws`. + Obtain a ticket via `POST /api/v1/ws/ticket` and connect with `?ticket=`. + version: "${VERSION}" + contact: + name: Chat Switchboard + +servers: + - url: / + description: Current instance + +tags: + - name: Auth + description: Authentication and session management + - name: Channels + description: Chat channel CRUD and messaging + - name: Completions + description: AI completion requests (streaming and non-streaming) + - name: Health + description: System health and readiness probes + - name: Admin + description: Platform administration (requires admin role) + - name: Usage + description: Token usage and cost tracking + - name: WebSocket + description: Real-time event delivery + +paths: + # ── Auth ─────────────────────────────────── + /api/v1/auth/register: + post: + tags: [Auth] + summary: Register a new user + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [username, password] + properties: + username: + type: string + minLength: 3 + password: + type: string + minLength: 8 + email: + type: string + format: email + display_name: + type: string + responses: + "201": + description: User created + content: + application/json: + schema: + $ref: "#/components/schemas/AuthResponse" + "400": + $ref: "#/components/responses/BadRequest" + "409": + description: Username already taken + + /api/v1/auth/login: + post: + tags: [Auth] + summary: Authenticate and obtain JWT tokens + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [username, password] + properties: + username: + type: string + password: + type: string + responses: + "200": + description: Login successful + content: + application/json: + schema: + $ref: "#/components/schemas/AuthResponse" + "401": + description: Invalid credentials + "429": + description: Rate limited + + /api/v1/auth/refresh: + post: + tags: [Auth] + summary: Refresh an expired access token + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [refresh_token] + properties: + refresh_token: + type: string + responses: + "200": + description: New token pair + content: + application/json: + schema: + $ref: "#/components/schemas/AuthResponse" + "401": + description: Invalid or expired refresh token + + /api/v1/auth/logout: + post: + tags: [Auth] + summary: Invalidate the current session + security: + - bearerAuth: [] + responses: + "200": + description: Logged out + + # ── Channels ─────────────────────────────── + /api/v1/channels: + get: + tags: [Channels] + summary: List channels accessible to the current user + security: + - bearerAuth: [] + parameters: + - name: type + in: query + schema: + type: string + enum: [direct, group, workflow] + - name: page + in: query + schema: + type: integer + default: 1 + - name: per_page + in: query + schema: + type: integer + default: 50 + responses: + "200": + description: Channel list + content: + application/json: + schema: + type: object + properties: + channels: + type: array + items: + $ref: "#/components/schemas/Channel" + total: + type: integer + post: + tags: [Channels] + summary: Create a new channel + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + title: + type: string + type: + type: string + enum: [direct, group] + default: direct + system_prompt: + type: string + responses: + "201": + description: Channel created + content: + application/json: + schema: + $ref: "#/components/schemas/Channel" + + /api/v1/channels/{id}: + get: + tags: [Channels] + summary: Get channel details + security: + - bearerAuth: [] + parameters: + - $ref: "#/components/parameters/ChannelID" + responses: + "200": + description: Channel details + content: + application/json: + schema: + $ref: "#/components/schemas/Channel" + "404": + $ref: "#/components/responses/NotFound" + put: + tags: [Channels] + summary: Update channel settings + security: + - bearerAuth: [] + parameters: + - $ref: "#/components/parameters/ChannelID" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + title: + type: string + system_prompt: + type: string + ai_mode: + type: string + enum: [auto, off, mention_only] + responses: + "200": + description: Updated + delete: + tags: [Channels] + summary: Archive a channel + security: + - bearerAuth: [] + parameters: + - $ref: "#/components/parameters/ChannelID" + responses: + "200": + description: Archived + + # ── Completions ──────────────────────────── + /api/v1/chat/completions: + post: + tags: [Completions] + summary: Send a message and get an AI completion + description: | + Streams SSE events by default. Set `stream: false` for a single JSON response. + Supports tool calling, @mentions, multi-model routing, and workflow integration. + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [content, channel_id] + properties: + channel_id: + type: string + format: uuid + content: + type: string + model: + type: string + description: Override the default model + persona_id: + type: string + format: uuid + provider_config_id: + type: string + format: uuid + max_tokens: + type: integer + temperature: + type: number + minimum: 0 + maximum: 2 + top_p: + type: number + minimum: 0 + maximum: 1 + stream: + type: boolean + default: true + file_ids: + type: array + items: + type: string + format: uuid + disabled_tools: + type: array + items: + type: string + responses: + "200": + description: | + **Streaming:** SSE events (`data: {"content":"..."}`, `data: [DONE]`). + **Non-streaming:** OpenAI-compatible JSON response. + content: + text/event-stream: + schema: + type: string + application/json: + schema: + type: object + properties: + model: + type: string + choices: + type: array + items: + type: object + properties: + message: + type: object + properties: + role: + type: string + content: + type: string + finish_reason: + type: string + usage: + type: object + properties: + prompt_tokens: + type: integer + completion_tokens: + type: integer + total_tokens: + type: integer + "400": + $ref: "#/components/responses/BadRequest" + "429": + description: Token budget exceeded + + # ── Health ───────────────────────────────── + /health: + get: + tags: [Health] + summary: Basic health check + responses: + "200": + description: System status + content: + application/json: + schema: + type: object + properties: + status: + type: string + example: ok + version: + type: string + database: + type: boolean + schema_version: + type: integer + + /healthz/live: + get: + tags: [Health] + summary: Kubernetes liveness probe + responses: + "200": + description: Process alive + + /healthz/ready: + get: + tags: [Health] + summary: Kubernetes readiness probe + description: Returns 503 if the database is unreachable (2s timeout). + responses: + "200": + description: Ready to serve traffic + "503": + description: Database unavailable + + /metrics: + get: + tags: [Health] + summary: Prometheus metrics endpoint + description: | + Returns all `switchboard_*` metrics in Prometheus text exposition format. + Includes HTTP request latency, WebSocket connections, completion tokens, + DB pool stats, and provider health gauges. + responses: + "200": + description: Prometheus text format + content: + text/plain: + schema: + type: string + + # ── WebSocket ────────────────────────────── + /api/v1/ws/ticket: + post: + tags: [WebSocket] + summary: Obtain a WebSocket connection ticket + description: | + Returns a single-use ticket (30s TTL) for authenticating the WebSocket + upgrade at `/ws?ticket=`. Cross-pod safe (v0.32.0). + security: + - bearerAuth: [] + responses: + "200": + description: Ticket issued + content: + application/json: + schema: + type: object + properties: + ticket: + type: string + + # ── Admin: Stats ─────────────────────────── + /api/v1/admin/stats: + get: + tags: [Admin] + summary: Platform statistics + security: + - bearerAuth: [] + responses: + "200": + description: Aggregate counts + content: + application/json: + schema: + type: object + properties: + users: + type: integer + channels: + type: integer + messages: + type: integer + + # ── Admin: Provider Health ───────────────── + /api/v1/admin/providers/health: + get: + tags: [Admin] + summary: Current health status for all providers + security: + - bearerAuth: [] + responses: + "200": + description: Provider health summaries + content: + application/json: + schema: + type: object + properties: + providers: + type: array + items: + $ref: "#/components/schemas/ProviderHealth" + + # ── Admin: Usage ─────────────────────────── + /api/v1/admin/usage: + get: + tags: [Admin, Usage] + summary: Aggregated token usage + security: + - bearerAuth: [] + parameters: + - name: since + in: query + schema: + type: string + format: date-time + - name: until + in: query + schema: + type: string + format: date-time + - name: period + in: query + schema: + type: string + enum: [7d, 30d, 90d] + - name: group_by + in: query + schema: + type: string + enum: [model, user, provider, day] + responses: + "200": + description: Usage data + + # ── Admin: Dashboard ─────────────────────── + /api/v1/admin/dashboard: + get: + tags: [Admin] + summary: Real-time observability dashboard data + description: | + Aggregates provider health, 24h usage, DB pool stats, WebSocket count, + recent errors, and uptime into a single response. Designed for the + built-in admin dashboard (v0.33.0). + security: + - bearerAuth: [] + responses: + "200": + description: Dashboard snapshot + content: + application/json: + schema: + type: object + properties: + providers: + type: array + items: + $ref: "#/components/schemas/ProviderHealth" + usage_24h: + type: object + db_pool: + type: object + properties: + open_connections: + type: integer + in_use: + type: integer + idle: + type: integer + ws_connections: + type: integer + uptime_seconds: + type: number + +components: + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT + + parameters: + ChannelID: + name: id + in: path + required: true + schema: + type: string + format: uuid + + responses: + BadRequest: + description: Invalid request + content: + application/json: + schema: + type: object + properties: + error: + type: string + NotFound: + description: Resource not found + content: + application/json: + schema: + type: object + properties: + error: + type: string + + schemas: + AuthResponse: + type: object + properties: + token: + type: string + refresh_token: + type: string + user: + type: object + properties: + id: + type: string + format: uuid + username: + type: string + role: + type: string + enum: [admin, user] + + Channel: + type: object + properties: + id: + type: string + format: uuid + title: + type: string + type: + type: string + enum: [direct, group, workflow] + user_id: + type: string + format: uuid + team_id: + type: string + format: uuid + ai_mode: + type: string + enum: [auto, off, mention_only] + created_at: + type: string + format: date-time + updated_at: + type: string + format: date-time + + ProviderHealth: + type: object + properties: + provider_config_id: + type: string + format: uuid + status: + type: string + enum: [healthy, degraded, down, unknown] + request_count: + type: integer + error_count: + type: integer + error_rate: + type: number + avg_latency_ms: + type: number + max_latency_ms: + type: integer diff --git a/server/static/swagger.html b/server/static/swagger.html new file mode 100644 index 0000000..84bd70a --- /dev/null +++ b/server/static/swagger.html @@ -0,0 +1,267 @@ + + + + + + + Chat Switchboard — API Docs + + + + +
+ + + + diff --git a/src/css/admin-surfaces.css b/src/css/admin-surfaces.css index 3922052..704a14b 100644 --- a/src/css/admin-surfaces.css +++ b/src/css/admin-surfaces.css @@ -151,3 +151,51 @@ color: var(--text-2, #999); min-width: 56px; } + +/* ── Admin Dashboard (v0.33.0) ───────────── */ +.dashboard-provider-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); + gap: 12px; +} +.dashboard-provider-card { + background: var(--bg-surface, #1a1a1d); + border: 1px solid var(--border, #2a2a2e); + border-radius: 8px; + padding: 12px 14px; +} +.dashboard-db-pool { + background: var(--bg-surface, #1a1a1d); + border: 1px solid var(--border, #2a2a2e); + border-radius: 8px; + padding: 14px; +} +.dashboard-pool-bar { + height: 8px; + background: var(--bg-raised, #2a2a2e); + border-radius: 4px; + overflow: hidden; +} +.dashboard-pool-fill { + height: 100%; + background: var(--accent, #b38a4e); + border-radius: 4px; + transition: width 0.3s ease; +} +.dashboard-errors { + display: flex; + flex-direction: column; + gap: 1px; + background: var(--border, #2a2a2e); + border: 1px solid var(--border, #2a2a2e); + border-radius: 8px; + overflow: hidden; +} +.dashboard-error-row { + display: flex; + align-items: center; + gap: 12px; + padding: 8px 12px; + background: var(--bg-secondary, #151517); + font-size: 12px; +} diff --git a/src/js/admin-scaffold.js b/src/js/admin-scaffold.js index c88e5ae..bbe5814 100644 --- a/src/js/admin-scaffold.js +++ b/src/js/admin-scaffold.js @@ -217,6 +217,10 @@ // -- Monitoring ------------------------------------------------------- + SCAFFOLDING.dashboard = + '
' + + '
Loading...
'; + SCAFFOLDING.usage = '
' + '
' + diff --git a/src/js/api.js b/src/js/api.js index 379d0c6..6c0d04a 100644 --- a/src/js/api.js +++ b/src/js/api.js @@ -542,6 +542,7 @@ const API = { getPublicSettings() { return this._get('/api/v1/settings/public'); }, adminUpdateSetting(key, value) { return this._put(`/api/v1/admin/settings/${key}`, value); }, adminGetStats() { return this._get('/api/v1/admin/stats'); }, + adminGetDashboard() { return this._get('/api/v1/admin/dashboard'); }, adminListGlobalConfigs() { return this._get('/api/v1/admin/configs'); }, adminCreateGlobalConfig(name, provider, endpoint, apiKey, modelDefault, isPrivate) { diff --git a/src/js/ui-admin.js b/src/js/ui-admin.js index 22236ad..4db9ef8 100644 --- a/src/js/ui-admin.js +++ b/src/js/ui-admin.js @@ -12,7 +12,7 @@ const ADMIN_SECTIONS = { workflows: ['workflows', 'tasks'], routing: ['health', 'routing', 'capabilities'], system: ['settings', 'storage', 'packages', 'channels', 'broadcast'], - monitoring: ['usage', 'audit', 'stats'], + monitoring: ['dashboard', 'usage', 'audit', 'stats'], }; const ADMIN_LABELS = { @@ -21,7 +21,7 @@ const ADMIN_LABELS = { health: 'Health', routing: 'Routing', capabilities: 'Capabilities', workflows: 'Workflows', tasks: 'Tasks', settings: 'Settings', storage: 'Storage', packages: 'Packages', channels: 'Channels', broadcast: 'Broadcast', - usage: 'Usage', audit: 'Audit', stats: 'Stats', + dashboard: 'Dashboard', usage: 'Usage', audit: 'Audit', stats: 'Stats', }; // Section → loader mapping (reuses all existing loadAdmin* functions) @@ -47,6 +47,7 @@ const ADMIN_LOADERS = { health: () => UI.loadAdminHealth(), routing: () => UI.loadAdminRouting(), capabilities: () => UI.loadAdminCapabilities(), + dashboard: () => UI.loadAdminDashboard(), usage: () => UI.loadAdminUsage(), audit: () => UI.loadAuditLog(), stats: () => UI.loadAdminStats(), @@ -242,6 +243,120 @@ Object.assign(UI, { } catch (e) { el.innerHTML = `
${esc(e.message)}
`; } }, + // ── Admin: Dashboard (v0.33.0) ──────── + async loadAdminDashboard() { + const el = document.getElementById('adminDashboardContent'); + if (!el) return; + el.innerHTML = '
Loading dashboard...
'; + + try { + const d = await API.adminGetDashboard(); + + // Stat cards row + const providers = d.provider_health || []; + const healthy = providers.filter(p => p.status === 'healthy').length; + const degraded = providers.filter(p => p.status === 'degraded' || p.status === 'down' || p.status === 'unhealthy').length; + const usage = d.usage_24h; + const tokenStr = usage && usage.total_tokens ? Number(usage.total_tokens).toLocaleString() : '0'; + const reqStr = usage && usage.total_requests ? Number(usage.total_requests).toLocaleString() : '0'; + + const rt = d.runtime || {}; + + const cardsEl = document.getElementById('adminDashboardCards'); + if (cardsEl) { + cardsEl.innerHTML = ` +
Uptime
${esc(d.uptime || '—')}
+
WS Connections
${d.ws_connections || 0}
+
Providers
${healthy}
${degraded ? degraded + ' degraded' : 'all healthy'}
+
Tokens (24h)
${tokenStr}
${reqStr} requests
+
Goroutines
${rt.goroutines || '—'}
+
Heap
${rt.heap_mb != null ? rt.heap_mb + ' MB' : '—'}
Sys: ${rt.sys_mb || '—'} MB
`; + } + + // Provider health cards + let html = ''; + if (providers.length > 0) { + html += '

Provider Health

'; + html += '
'; + providers.forEach(p => { + const name = esc(p.provider_name || p.provider_config_id?.slice(0, 12) || '—'); + const statusCls = p.status === 'healthy' ? 'badge-healthy' : p.status === 'degraded' ? 'badge-degraded' : 'badge-unhealthy'; + const latency = p.avg_latency_ms != null ? Math.round(p.avg_latency_ms) + 'ms' : '—'; + html += `
+
+ ${name} + ${esc(p.status || 'unknown')} +
+
+ Latency: ${latency} + Reqs: ${p.request_count || 0} + Errors: ${p.error_count || 0} +
+
`; + }); + html += '
'; + } + + // DB pool stats + if (d.db_pool) { + const pool = d.db_pool; + const pct = pool.OpenConnections > 0 ? Math.round(pool.InUse / pool.OpenConnections * 100) : 0; + html += '

DB Connection Pool

'; + html += `
+
+
+ Open: ${pool.OpenConnections} + In Use: ${pool.InUse} + Idle: ${pool.Idle} + Wait Count: ${pool.WaitCount} +
+
`; + } + + // Storage status (fetched in parallel) + try { + const storage = await API._get('/api/v1/admin/storage/status'); + if (storage && storage.configured) { + html += '

Storage

'; + html += '
'; + html += `
`; + html += `Backend: ${esc(storage.backend || '—')}`; + if (storage.total_bytes != null) html += `Total: ${(storage.total_bytes / 1024 / 1024).toFixed(1)} MB`; + if (storage.file_count != null) html += `Files: ${storage.file_count}`; + html += `${storage.healthy ? 'Healthy' : 'Unhealthy'}`; + html += `
`; + } + } catch (_) { /* storage status optional */ } + + // Recent errors + const errors = d.recent_errors || []; + if (errors.length > 0) { + html += '

Recent Errors

'; + html += '
'; + errors.forEach(e => { + const time = e.created_at ? new Date(e.created_at).toLocaleString() : '—'; + html += `
+ ${time} + ${esc(e.action || '')} ${esc(e.resource_type || '')} ${esc(e.resource_id || '')} +
`; + }); + html += '
'; + } + + el.innerHTML = html || '
No data yet.
'; + + // Auto-refresh every 30s + if (UI._dashboardRefreshTimer) clearInterval(UI._dashboardRefreshTimer); + UI._dashboardRefreshTimer = setInterval(() => { + if (document.getElementById('adminDashboardContent')) { + UI.loadAdminDashboard(); + } else { + clearInterval(UI._dashboardRefreshTimer); + } + }, 30000); + } catch (e) { el.innerHTML = `
${esc(e.message)}
`; } + }, + async loadAdminStats() { const el = document.getElementById('adminStats'); el.innerHTML = '
Loading...
';