switchboard/core
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Releases 9 Wiki Activity

Changeset 0.10.0 (#56)

Browse Source
This commit is contained in:
xcaliber
2026-02-24 14:50:53 +00:00
parent cdfd69bad3
commit ea03f956ca
31 changed files with 3303 additions and 167 deletions
Download Patch File Download Diff File Expand all files Collapse all files

172
server/providers/openai.go
View File

@@ -45,6 +45,11 @@ func (p *OpenAIProvider) ChatCompletion(ctx context.Context, cfg ProviderConfig,
OutputTokens: resp.Usage.CompletionTokens,
}
// Cache tokens (OpenAI, Venice report these in prompt_tokens_details)
if resp.Usage.PromptTokensDetails != nil {
result.CacheReadTokens = resp.Usage.PromptTokensDetails.CachedTokens
}
// Extract tool calls if present
if len(resp.Choices[0].Message.ToolCalls) > 0 {
for _, tc := range resp.Choices[0].Message.ToolCalls {
@@ -84,6 +89,13 @@ func (p *OpenAIProvider) StreamCompletion(ctx context.Context, cfg ProviderConfi
// Accumulate tool calls across stream chunks
toolCallMap := map[int]*ToolCall{} // index → accumulated call
// Usage arrives in a separate chunk (or on the final chunk).
// OpenAI protocol order: content → finish_reason → usage → [DONE]
// The usage chunk has choices=[] so we must hold the finish event
// until usage arrives, otherwise tokens are always 0.
var streamUsage *openaiUsage
var pendingFinish *StreamEvent // deferred until usage arrives
scanner := bufio.NewScanner(body)
for scanner.Scan() {
line := scanner.Text()
@@ -94,7 +106,12 @@ func (p *OpenAIProvider) StreamCompletion(ctx context.Context, cfg ProviderConfi
data := strings.TrimPrefix(line, "data: ")
if data == "[DONE]" {
ch <- StreamEvent{Done: true}
// Flush any pending finish event (usage may or may not have arrived)
if pendingFinish != nil {
ch <- *pendingFinish
} else {
ch <- StreamEvent{Done: true}
}
return
}
@@ -103,6 +120,21 @@ func (p *OpenAIProvider) StreamCompletion(ctx context.Context, cfg ProviderConfi
continue
}
// Capture usage — may arrive in a separate chunk with empty choices
if chunk.Usage != nil {
streamUsage = chunk.Usage
// If finish already arrived, attach usage and flush immediately
if pendingFinish != nil {
pendingFinish.InputTokens = streamUsage.PromptTokens
pendingFinish.OutputTokens = streamUsage.CompletionTokens
if streamUsage.PromptTokensDetails != nil {
pendingFinish.CacheReadTokens = streamUsage.PromptTokensDetails.CachedTokens
}
ch <- *pendingFinish
return // Done — [DONE] will be handled by defer body.Close()
}
}
if len(chunk.Choices) == 0 {
continue
}
@@ -146,7 +178,32 @@ func (p *OpenAIProvider) StreamCompletion(ctx context.Context, cfg ProviderConfi
for _, tc := range toolCallMap {
ev.ToolCalls = append(ev.ToolCalls, *tc)
}
// Tool calls need to flush immediately — the tool loop
// needs the event to start executing tools
if streamUsage != nil {
ev.InputTokens = streamUsage.PromptTokens
ev.OutputTokens = streamUsage.CompletionTokens
if streamUsage.PromptTokensDetails != nil {
ev.CacheReadTokens = streamUsage.PromptTokensDetails.CachedTokens
}
}
ch <- ev
continue
}
// Normal finish — defer until usage chunk arrives
if streamUsage != nil {
// Usage already captured (rare: same chunk or earlier)
ev.InputTokens = streamUsage.PromptTokens
ev.OutputTokens = streamUsage.CompletionTokens
if streamUsage.PromptTokensDetails != nil {
ev.CacheReadTokens = streamUsage.PromptTokensDetails.CachedTokens
}
ch <- ev
continue
}
// Hold — usage chunk hasn't arrived yet
pendingFinish = &ev
continue
}
ch <- ev
@@ -154,6 +211,9 @@ func (p *OpenAIProvider) StreamCompletion(ctx context.Context, cfg ProviderConfi
if err := scanner.Err(); err != nil {
ch <- StreamEvent{Error: err}
} else if pendingFinish != nil {
// Scanner ended without [DONE] — flush pending finish
ch <- *pendingFinish
}
}()
@@ -210,6 +270,59 @@ func (p *OpenAIProvider) ListModels(ctx context.Context, cfg ProviderConfig) ([]
return out, nil
}
// ── Embeddings ─────────────────────────────
func (p *OpenAIProvider) Embed(ctx context.Context, cfg ProviderConfig, req EmbeddingRequest) (*EmbeddingResponse, error) {
endpoint := strings.TrimSuffix(cfg.Endpoint, "/") + "/embeddings"
oaiReq := openaiEmbeddingRequest{
Model: req.Model,
Input: req.Input,
}
body, err := json.Marshal(oaiReq)
if err != nil {
return nil, fmt.Errorf("marshal embedding request: %w", err)
}
httpReq, err := http.NewRequestWithContext(ctx, "POST", endpoint, bytes.NewReader(body))
if err != nil {
return nil, err
}
httpReq.Header.Set("Content-Type", "application/json")
if cfg.APIKey != "" {
httpReq.Header.Set("Authorization", "Bearer "+cfg.APIKey)
}
for k, v := range cfg.CustomHeaders {
httpReq.Header.Set(k, v)
}
resp, err := http.DefaultClient.Do(httpReq)
if err != nil {
return nil, fmt.Errorf("embedding request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body)
return nil, fmt.Errorf("embedding error: HTTP %d: %s", resp.StatusCode, string(b))
}
var oaiResp openaiEmbeddingResponse
if err := json.NewDecoder(resp.Body).Decode(&oaiResp); err != nil {
return nil, fmt.Errorf("decode embedding response: %w", err)
}
result := &EmbeddingResponse{
Model: oaiResp.Model,
InputTokens: oaiResp.Usage.PromptTokens,
Embeddings: make([][]float64, len(oaiResp.Data)),
}
for i, d := range oaiResp.Data {
result.Embeddings[i] = d.Embedding
}
return result, nil
}
// ── HTTP Layer ──────────────────────────────
func (p *OpenAIProvider) doRequest(ctx context.Context, cfg ProviderConfig, req CompletionRequest) (io.ReadCloser, error) {
@@ -221,6 +334,9 @@ func (p *OpenAIProvider) doRequest(ctx context.Context, cfg ProviderConfig, req
Stream: req.Stream,
Messages: make([]openaiMessage, 0, len(req.Messages)),
}
if req.Stream {
oaiReq.StreamOptions = &openaiStreamOptions{IncludeUsage: true}
}
if req.MaxTokens > 0 {
oaiReq.MaxTokens = req.MaxTokens
}
@@ -339,13 +455,18 @@ type openaiToolDef struct {
}
type openaiChatRequest struct {
Model string `json:"model"`
Messages []openaiMessage `json:"messages"`
MaxTokens int `json:"max_tokens,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
Stream bool `json:"stream"`
Tools []openaiToolDef `json:"tools,omitempty"`
Model string `json:"model"`
Messages []openaiMessage `json:"messages"`
MaxTokens int `json:"max_tokens,omitempty"`
Temperature *float64 `json:"temperature,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
Stream bool `json:"stream"`
StreamOptions *openaiStreamOptions `json:"stream_options,omitempty"`
Tools []openaiToolDef `json:"tools,omitempty"`
}
type openaiStreamOptions struct {
IncludeUsage bool `json:"include_usage"`
}
type openaiChatResponse struct {
@@ -354,10 +475,18 @@ type openaiChatResponse struct {
Message openaiMessage `json:"message"`
FinishReason string `json:"finish_reason"`
} `json:"choices"`
Usage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
} `json:"usage"`
Usage openaiUsage `json:"usage"`
}
type openaiUsage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
PromptTokensDetails *struct {
CachedTokens int `json:"cached_tokens"`
} `json:"prompt_tokens_details,omitempty"`
CompletionTokensDetails *struct {
ReasoningTokens int `json:"reasoning_tokens"`
} `json:"completion_tokens_details,omitempty"`
}
type openaiStreamChunk struct {
@@ -370,6 +499,7 @@ type openaiStreamChunk struct {
} `json:"delta"`
FinishReason string `json:"finish_reason"`
} `json:"choices"`
Usage *openaiUsage `json:"usage,omitempty"`
}
type openaiModelsResponse struct {
@@ -379,3 +509,21 @@ type openaiModelsResponse struct {
ContextLength int `json:"context_length,omitempty"` // Ollama, OpenRouter
} `json:"data"`
}
// ── Embedding Wire Types ──────────────────
type openaiEmbeddingRequest struct {
Model string `json:"model"`
Input []string `json:"input"`
}
type openaiEmbeddingResponse struct {
Data []struct {
Embedding []float64 `json:"embedding"`
Index int `json:"index"`
} `json:"data"`
Model string `json:"model"`
Usage struct {
PromptTokens int `json:"prompt_tokens"`
} `json:"usage"`
}