switchboard/core
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Releases 9 Wiki Activity

Changeset 0.10.0 (#56)

Browse Source
This commit is contained in:
xcaliber
2026-02-24 14:50:53 +00:00
parent cdfd69bad3
commit ea03f956ca
31 changed files with 3303 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
server/providers/anthropic.go
View File

@@ -38,10 +38,12 @@ func (p *AnthropicProvider) ChatCompletion(ctx context.Context, cfg ProviderConf
}
result := &CompletionResponse{
Model: resp.Model,
FinishReason: resp.StopReason,
InputTokens: resp.Usage.InputTokens,
OutputTokens: resp.Usage.OutputTokens,
Model: resp.Model,
FinishReason: resp.StopReason,
InputTokens: resp.Usage.InputTokens,
OutputTokens: resp.Usage.OutputTokens,
CacheCreationTokens: resp.Usage.CacheCreationInputTokens,
CacheReadTokens: resp.Usage.CacheReadInputTokens,
}
// Extract text and tool_use blocks
@@ -89,6 +91,9 @@ func (p *AnthropicProvider) StreamCompletion(ctx context.Context, cfg ProviderCo
var toolCalls []ToolCall
var currentToolIdx int = -1
// Usage accumulates across stream events
var inputTokens, outputTokens, cacheCreation, cacheRead int
scanner := bufio.NewScanner(body)
for scanner.Scan() {
line := scanner.Text()
@@ -105,6 +110,14 @@ func (p *AnthropicProvider) StreamCompletion(ctx context.Context, cfg ProviderCo
}
switch event.Type {
case "message_start":
// Capture input token counts (including cache)
if event.Message != nil {
inputTokens = event.Message.Usage.InputTokens
cacheCreation = event.Message.Usage.CacheCreationInputTokens
cacheRead = event.Message.Usage.CacheReadInputTokens
}
case "content_block_start":
if event.ContentBlock != nil && event.ContentBlock.Type == "tool_use" {
currentToolIdx++
@@ -126,10 +139,19 @@ func (p *AnthropicProvider) StreamCompletion(ctx context.Context, cfg ProviderCo
}
case "message_delta":
// Capture output tokens
if event.Usage != nil {
outputTokens = event.Usage.OutputTokens
}
stopReason := event.Delta.StopReason
ev := StreamEvent{
Done: true,
FinishReason: stopReason,
Done: true,
FinishReason: stopReason,
InputTokens: inputTokens,
OutputTokens: outputTokens,
CacheCreationTokens: cacheCreation,
CacheReadTokens: cacheRead,
}
if stopReason == "tool_use" {
ev.FinishReason = "tool_calls"
@@ -181,6 +203,13 @@ func (p *AnthropicProvider) ListModels(_ context.Context, _ ProviderConfig) ([]M
return out, nil
}
// ── Embeddings ─────────────────────────────
// Embed is not supported by the Anthropic API.
func (p *AnthropicProvider) Embed(_ context.Context, _ ProviderConfig, _ EmbeddingRequest) (*EmbeddingResponse, error) {
return nil, ErrNotSupported
}
// ── HTTP Layer ──────────────────────────────
func (p *AnthropicProvider) doRequest(ctx context.Context, cfg ProviderConfig, req CompletionRequest) (io.ReadCloser, error) {
@@ -366,20 +395,30 @@ type anthropicResponse struct {
Name string `json:"name,omitempty"`
Input json.RawMessage `json:"input,omitempty"`
} `json:"content"`
Usage struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
} `json:"usage"`
Usage anthropicUsage `json:"usage"`
}
type anthropicUsage struct {
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheCreationInputTokens int `json:"cache_creation_input_tokens"`
CacheReadInputTokens int `json:"cache_read_input_tokens"`
}
type anthropicStreamEvent struct {
Type string `json:"type"`
Type string `json:"type"`
Message *struct {
Usage anthropicUsage `json:"usage"`
} `json:"message,omitempty"`
Delta struct {
Type string `json:"type"`
Text string `json:"text"`
StopReason string `json:"stop_reason"`
PartialJSON string `json:"partial_json"`
} `json:"delta"`
Usage *struct {
OutputTokens int `json:"output_tokens"`
} `json:"usage,omitempty"`
ContentBlock *struct {
Type string `json:"type"`
ID string `json:"id"`

172
server/providers/openai.go
View File

@@ -45,6 +45,11 @@ func (p *OpenAIProvider) ChatCompletion(ctx context.Context, cfg ProviderConfig,
OutputTokens: resp.Usage.CompletionTokens,
}
// Cache tokens (OpenAI, Venice report these in prompt_tokens_details)
if resp.Usage.PromptTokensDetails != nil {
result.CacheReadTokens = resp.Usage.PromptTokensDetails.CachedTokens
}
// Extract tool calls if present
if len(resp.Choices[0].Message.ToolCalls) > 0 {
for _, tc := range resp.Choices[0].Message.ToolCalls {
@@ -84,6 +89,13 @@ func (p *OpenAIProvider) StreamCompletion(ctx context.Context, cfg ProviderConfi
// Accumulate tool calls across stream chunks
toolCallMap := map[int]*ToolCall{} // index → accumulated call
// Usage arrives in a separate chunk (or on the final chunk).
// OpenAI protocol order: content → finish_reason → usage → [DONE]
// The usage chunk has choices=[] so we must hold the finish event
// until usage arrives, otherwise tokens are always 0.
var streamUsage *openaiUsage
var pendingFinish *StreamEvent // deferred until usage arrives
scanner := bufio.NewScanner(body)
for scanner.Scan() {
line := scanner.Text()
@@ -94,7 +106,12 @@ func (p *OpenAIProvider) StreamCompletion(ctx context.Context, cfg ProviderConfi
data := strings.TrimPrefix(line, "data: ")
if data == "[DONE]" {
ch <- StreamEvent{Done: true}
// Flush any pending finish event (usage may or may not have arrived)
if pendingFinish != nil {
ch <- *pendingFinish
} else {
ch <- StreamEvent{Done: true}
}
return
}
@@ -103,6 +120,21 @@ func (p *OpenAIProvider) StreamCompletion(ctx context.Context, cfg ProviderConfi
continue
}
// Capture usage — may arrive in a separate chunk with empty choices
if chunk.Usage != nil {
streamUsage = chunk.Usage
// If finish already arrived, attach usage and flush immediately
if pendingFinish != nil {
pendingFinish.InputTokens = streamUsage.PromptTokens
pendingFinish.OutputTokens = streamUsage.CompletionTokens
if streamUsage.PromptTokensDetails != nil {
pendingFinish.CacheReadTokens = streamUsage.PromptTokensDetails.CachedTokens
}
ch <- *pendingFinish
return // Done — [DONE] will be handled by defer body.Close()
}
}
if len(chunk.Choices) == 0 {
continue
}
@@ -146,7 +178,32 @@ func (p *OpenAIProvider) StreamCompletion(ctx context.Context, cfg ProviderConfi
for _, tc := range toolCallMap {
ev.ToolCalls = append(ev.ToolCalls, *tc)
}
// Tool calls need to flush immediately — the tool loop
// needs the event to start executing tools
if streamUsage != nil {
ev.InputTokens = streamUsage.PromptTokens
ev.OutputTokens = streamUsage.CompletionTokens
if streamUsage.PromptTokensDetails != nil {
ev.CacheReadTokens = streamUsage.PromptTokensDetails.CachedTokens
}
}
ch <- ev
continue
}
// Normal finish — defer until usage chunk arrives
if streamUsage != nil {
// Usage already captured (rare: same chunk or earlier)
ev.InputTokens = streamUsage.PromptTokens
ev.OutputTokens = streamUsage.CompletionTokens
if streamUsage.PromptTokensDetails != nil {
ev.CacheReadTokens = streamUsage.PromptTokensDetails.CachedTokens
}
ch <- ev
continue
}
// Hold — usage chunk hasn't arrived yet
pendingFinish = &ev
continue
}
ch <- ev
@@ -154,6 +211,9 @@ func (p *OpenAIProvider) StreamCompletion(ctx context.Context, cfg ProviderConfi
if err := scanner.Err(); err != nil {
ch <- StreamEvent{Error: err}
} else if pendingFinish != nil {
// Scanner ended without [DONE] — flush pending finish
ch <- *pendingFinish
}
}()
@@ -210,6 +270,59 @@ func (p *OpenAIProvider) ListModels(ctx context.Context, cfg ProviderConfig) ([]
return out, nil
}
// ── Embeddings ─────────────────────────────
func (p *OpenAIProvider) Embed(ctx context.Context, cfg ProviderConfig, req EmbeddingRequest) (*EmbeddingResponse, error) {
endpoint := strings.TrimSuffix(cfg.Endpoint, "/") + "/embeddings"
oaiReq := openaiEmbeddingRequest{
Model: req.Model,
Input: req.Input,
}
body, err := json.Marshal(oaiReq)
if err != nil {
return nil, fmt.Errorf("marshal embedding request: %w", err)
}
httpReq, err := http.NewRequestWithContext(ctx, "POST", endpoint, bytes.NewReader(body))
if err != nil {
return nil, err
}
httpReq.Header.Set("Content-Type", "application/json")
if cfg.APIKey != "" {
httpReq.Header.Set("Authorization", "Bearer "+cfg.APIKey)
}
for k, v := range cfg.CustomHeaders {
httpReq.Header.Set(k, v)
}
resp, err := http.DefaultClient.Do(httpReq)
if err != nil {
return nil, fmt.Errorf("embedding request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("embedding error: HTTP %d: %s", resp.StatusCode, string(b))
}
var oaiResp openaiEmbeddingResponse
if err := json.NewDecoder(resp.Body).Decode(&oaiResp); err != nil {
return nil, fmt.Errorf("decode embedding response: %w", err)
}
result := &EmbeddingResponse{
Model: oaiResp.Model,
InputTokens: oaiResp.Usage.PromptTokens,
Embeddings: make([][]float64, len(oaiResp.Data)),
}
for i, d := range oaiResp.Data {
result.Embeddings[i] = d.Embedding
}
return result, nil
}
// ── HTTP Layer ──────────────────────────────
func (p *OpenAIProvider) doRequest(ctx context.Context, cfg ProviderConfig, req CompletionRequest) (io.ReadCloser, error) {
@@ -221,6 +334,9 @@ func (p *OpenAIProvider) doRequest(ctx context.Context, cfg ProviderConfig, req
Stream: req.Stream,
Messages: make([]openaiMessage, 0, len(req.Messages)),
}
if req.Stream {
oaiReq.StreamOptions = &openaiStreamOptions{IncludeUsage: true}
}
if req.MaxTokens > 0 {
oaiReq.MaxTokens = req.MaxTokens
}
@@ -339,13 +455,18 @@ type openaiToolDef struct {
}
type openaiChatRequest struct {
Model string `json:"model"`
Messages []openaiMessage `json:"messages"`
MaxTokens int `json:"max_tokens,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
Stream bool `json:"stream"`
Tools []openaiToolDef `json:"tools,omitempty"`
Model string `json:"model"`
Messages []openaiMessage `json:"messages"`
MaxTokens int `json:"max_tokens,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
Stream bool `json:"stream"`
StreamOptions *openaiStreamOptions `json:"stream_options,omitempty"`
Tools []openaiToolDef `json:"tools,omitempty"`
}
type openaiStreamOptions struct {
IncludeUsage bool `json:"include_usage"`
}
type openaiChatResponse struct {
@@ -354,10 +475,18 @@ type openaiChatResponse struct {
Message openaiMessage `json:"message"`
FinishReason string `json:"finish_reason"`
} `json:"choices"`
Usage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
} `json:"usage"`
Usage openaiUsage `json:"usage"`
}
type openaiUsage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
PromptTokensDetails *struct {
CachedTokens int `json:"cached_tokens"`
} `json:"prompt_tokens_details,omitempty"`
CompletionTokensDetails *struct {
ReasoningTokens int `json:"reasoning_tokens"`
} `json:"completion_tokens_details,omitempty"`
}
type openaiStreamChunk struct {
@@ -370,6 +499,7 @@ type openaiStreamChunk struct {
} `json:"delta"`
FinishReason string `json:"finish_reason"`
} `json:"choices"`
Usage *openaiUsage `json:"usage,omitempty"`
}
type openaiModelsResponse struct {
@@ -379,3 +509,21 @@ type openaiModelsResponse struct {
ContextLength int `json:"context_length,omitempty"` // Ollama, OpenRouter
} `json:"data"`
}
// ── Embedding Wire Types ──────────────────
type openaiEmbeddingRequest struct {
Model string `json:"model"`
Input []string `json:"input"`
}
type openaiEmbeddingResponse struct {
Data []struct {
Embedding []float64 `json:"embedding"`
Index int `json:"index"`
} `json:"data"`
Model string `json:"model"`
Usage struct {
PromptTokens int `json:"prompt_tokens"`
} `json:"usage"`
}

5
server/providers/openrouter.go
View File

@@ -34,6 +34,11 @@ func (p *OpenRouterProvider) StreamCompletion(ctx context.Context, cfg ProviderC
return oai.StreamCompletion(ctx, cfg, req)
}
func (p *OpenRouterProvider) Embed(ctx context.Context, cfg ProviderConfig, req EmbeddingRequest) (*EmbeddingResponse, error) {
oai := &OpenAIProvider{}
return oai.Embed(ctx, cfg, req)
}
// ── List Models ─────────────────────────────
// OpenRouter /v1/models returns architecture, pricing, context_length.

56
server/providers/provider.go
View File

@@ -3,10 +3,15 @@ package providers
import (
"context"
"encoding/json"
"errors"
"git.gobha.me/xcaliber/chat-switchboard/models"
)
// ── Errors ─────────────────────────────────
var ErrNotSupported = errors.New("operation not supported by this provider")
// ── Provider Interface ──────────────────────
// Provider is the contract for LLM API adapters.
@@ -23,6 +28,10 @@ type Provider interface {
// ListModels returns available models from this provider.
ListModels(ctx context.Context, cfg ProviderConfig) ([]Model, error)
// Embed generates embeddings for the given input texts.
// Returns ErrNotSupported if the provider has no embedding endpoint.
Embed(ctx context.Context, cfg ProviderConfig, req EmbeddingRequest) (*EmbeddingResponse, error)
}
// ── Configuration ───────────────────────────
@@ -88,23 +97,44 @@ type CompletionRequest struct {
// CompletionResponse is the normalized non-streaming response.
type CompletionResponse struct {
Content string `json:"content"`
Model string `json:"model"`
FinishReason string `json:"finish_reason"`
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"` // set when finish_reason is "tool_calls"
Content string `json:"content"`
Model string `json:"model"`
FinishReason string `json:"finish_reason"`
InputTokens int `json:"input_tokens"`
OutputTokens int `json:"output_tokens"`
CacheCreationTokens int `json:"cache_creation_tokens,omitempty"`
CacheReadTokens int `json:"cache_read_tokens,omitempty"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"` // set when finish_reason is "tool_calls"
}
// StreamEvent is a single chunk from a streaming response.
type StreamEvent struct {
Delta string `json:"delta,omitempty"`
Reasoning string `json:"reasoning,omitempty"`
Done bool `json:"done,omitempty"`
FinishReason string `json:"finish_reason,omitempty"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
Model string `json:"model,omitempty"`
Error error `json:"-"`
Delta string `json:"delta,omitempty"`
Reasoning string `json:"reasoning,omitempty"`
Done bool `json:"done,omitempty"`
FinishReason string `json:"finish_reason,omitempty"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
Model string `json:"model,omitempty"`
Error error `json:"-"`
InputTokens int `json:"input_tokens,omitempty"`
OutputTokens int `json:"output_tokens,omitempty"`
CacheCreationTokens int `json:"cache_creation_tokens,omitempty"`
CacheReadTokens int `json:"cache_read_tokens,omitempty"`
}
// ── Embedding Types ────────────────────────
// EmbeddingRequest is the normalized embedding request.
type EmbeddingRequest struct {
Model string `json:"model"`
Input []string `json:"input"` // batch of texts to embed
}
// EmbeddingResponse is the normalized embedding response.
type EmbeddingResponse struct {
Embeddings [][]float64 `json:"embeddings"`
Model string `json:"model"`
InputTokens int `json:"input_tokens"`
}
// Model represents an available model from a provider, with capabilities.

5
server/providers/venice.go
View File

@@ -32,6 +32,11 @@ func (p *VeniceProvider) StreamCompletion(ctx context.Context, cfg ProviderConfi
return oai.StreamCompletion(ctx, cfg, req)
}
func (p *VeniceProvider) Embed(ctx context.Context, cfg ProviderConfig, req EmbeddingRequest) (*EmbeddingResponse, error) {
oai := &OpenAIProvider{}
return oai.Embed(ctx, cfg, req)
}
// ── List Models ─────────────────────────────
// Venice /v1/models returns model_spec with capabilities, pricing,
// context tokens, and traits. We parse all of it.