diff options
Diffstat (limited to '')
| -rw-r--r-- | internal/provider/forwarder.go | 24 |
1 files changed, 20 insertions, 4 deletions
diff --git a/internal/provider/forwarder.go b/internal/provider/forwarder.go index d7d850d..f816f8a 100644 --- a/internal/provider/forwarder.go +++ b/internal/provider/forwarder.go @@ -19,9 +19,10 @@ import ( ) type Result struct { - Response *http.Response - Route domain.Route - Attempts int + Response *http.Response + Route domain.Route + Attempts int + AttemptStartedAt time.Time } type Forwarder struct { @@ -83,7 +84,7 @@ func (f *Forwarder) Forward(ctx context.Context, protocol domain.Protocol, reque lastErr = fmt.Errorf("upstream %s returned %d", route.Provider.ID, response.StatusCode) continue } - return Result{Response: response, Route: route, Attempts: attempts}, nil + return Result{Response: response, Route: route, Attempts: attempts, AttemptStartedAt: attemptStarted}, nil } if lastErr == nil { lastErr = errors.New("all upstream routes failed") @@ -99,6 +100,19 @@ func (f *Forwarder) observe(modelID string, route domain.Route, statusCode int, providerhealth.Observation{StatusCode: statusCode, Latency: latency, Failed: failed}) } +// ObserveTTFT feeds the first user-visible output latency into adaptive route +// selection. It is deliberately separate from the header/circuit observation +// because streaming TTFT is only known after response forwarding begins. +func (f *Forwarder) ObserveTTFT(modelID string, route domain.Route, latency time.Duration) { + if f.health == nil || latency <= 0 { + return + } + f.health.ObserveTTFT(providerhealth.RouteKey{ModelID: modelID, ProviderID: route.Provider.ID, WireAPI: route.Provider.EffectiveWireAPI()}, latency) + if f.metrics != nil { + f.metrics.UpstreamTTFT(latency) + } +} + func rewriteModel(body []byte, upstreamModel string) ([]byte, error) { return rewriteRequest(body, upstreamModel, domain.ProtocolOpenAI) } @@ -141,6 +155,8 @@ func rewriteRequestWithWireAPI(body []byte, upstreamModel string, protocol domai func endpointURL(provider domain.Provider, _ domain.Protocol) string { baseURL := strings.TrimRight(provider.BaseURL, "/") switch provider.EffectiveWireAPI() { + case "embeddings": + return baseURL + "/embeddings" case "responses": return baseURL + "/responses" case "messages": |