package providerhealth import ( "sort" "sync" "time" ) const recentWindow = 100 type RouteKey struct { ModelID string ProviderID string WireAPI string } type Observation struct { StatusCode int Latency time.Duration Failed bool ObservedAt time.Time } type Status struct { ModelID string `json:"model_id"` ProviderID string `json:"provider_id"` WireAPI string `json:"wire_api"` State string `json:"state"` Attempts uint64 `json:"attempts"` RecentSamples int `json:"recent_samples"` AvailabilityPercent float64 `json:"availability_percent"` HeaderLatencyEWMA int64 `json:"header_latency_ewma_ms"` ConsecutiveFailures uint64 `json:"consecutive_failures"` LastStatusCode int `json:"last_status_code,omitempty"` LastObservedAt *time.Time `json:"last_observed_at,omitempty"` LastHealthyAt *time.Time `json:"last_healthy_at,omitempty"` CircuitOpenUntil *time.Time `json:"circuit_open_until,omitempty"` } type Options struct { FailureThreshold uint64 OpenDuration time.Duration Now func() time.Time } type Tracker struct { states sync.Map failureThreshold uint64 openDuration time.Duration now func() time.Time } type routeState struct { mu sync.RWMutex attempts uint64 consecutiveFailures uint64 lastStatusCode int lastObservedAt time.Time lastHealthyAt time.Time openUntil time.Time headerLatencyEWMA float64 recent [recentWindow]bool recentCount int recentPosition int recentHealthy int } func New(options Options) *Tracker { if options.FailureThreshold == 0 { options.FailureThreshold = 3 } if options.OpenDuration <= 0 { options.OpenDuration = 30 * time.Second } if options.Now == nil { options.Now = time.Now } return &Tracker{failureThreshold: options.FailureThreshold, openDuration: options.OpenDuration, now: options.Now} } func (t *Tracker) Observe(key RouteKey, observation Observation) { if t == nil || key.ProviderID == "" { return } if observation.ObservedAt.IsZero() { observation.ObservedAt = t.now() } value, _ := t.states.LoadOrStore(key, &routeState{}) state := value.(*routeState) state.mu.Lock() defer state.mu.Unlock() state.attempts++ state.lastStatusCode = observation.StatusCode state.lastObservedAt = observation.ObservedAt if observation.Latency > 0 { latency := float64(observation.Latency.Milliseconds()) if latency < 1 { latency = 1 } if state.headerLatencyEWMA == 0 { state.headerLatencyEWMA = latency } else { state.headerLatencyEWMA = state.headerLatencyEWMA*0.8 + latency*0.2 } } state.addRecent(!observation.Failed) if observation.Failed { state.consecutiveFailures++ if state.consecutiveFailures >= t.failureThreshold { state.openUntil = observation.ObservedAt.Add(t.openDuration) } return } state.consecutiveFailures = 0 state.openUntil = time.Time{} state.lastHealthyAt = observation.ObservedAt } func (s *routeState) addRecent(healthy bool) { if s.recentCount == recentWindow { if s.recent[s.recentPosition] { s.recentHealthy-- } } else { s.recentCount++ } s.recent[s.recentPosition] = healthy if healthy { s.recentHealthy++ } s.recentPosition = (s.recentPosition + 1) % recentWindow } func (t *Tracker) CircuitOpen(key RouteKey) bool { if t == nil { return false } value, ok := t.states.Load(key) if !ok { return false } state := value.(*routeState) state.mu.RLock() defer state.mu.RUnlock() return state.openUntil.After(t.now()) } func (t *Tracker) Snapshot() []Status { if t == nil { return []Status{} } now := t.now() result := make([]Status, 0) t.states.Range(func(rawKey, rawState any) bool { key := rawKey.(RouteKey) state := rawState.(*routeState) state.mu.RLock() item := statusFromState(key, state, now) state.mu.RUnlock() result = append(result, item) return true }) sort.Slice(result, func(i, j int) bool { if result[i].ModelID != result[j].ModelID { return result[i].ModelID < result[j].ModelID } return result[i].ProviderID < result[j].ProviderID }) return result } func statusFromState(key RouteKey, state *routeState, now time.Time) Status { item := Status{ModelID: key.ModelID, ProviderID: key.ProviderID, WireAPI: key.WireAPI, Attempts: state.attempts, RecentSamples: state.recentCount, HeaderLatencyEWMA: int64(state.headerLatencyEWMA + 0.5), ConsecutiveFailures: state.consecutiveFailures, LastStatusCode: state.lastStatusCode} if state.recentCount > 0 { item.AvailabilityPercent = float64(state.recentHealthy) / float64(state.recentCount) * 100 } if !state.lastObservedAt.IsZero() { value := state.lastObservedAt item.LastObservedAt = &value } if !state.lastHealthyAt.IsZero() { value := state.lastHealthyAt item.LastHealthyAt = &value } if state.openUntil.After(now) { value := state.openUntil item.CircuitOpenUntil = &value item.State = "open" } else if state.attempts == 0 { item.State = "unknown" } else if state.consecutiveFailures > 0 || (state.recentCount >= 5 && item.AvailabilityPercent < 95) { item.State = "degraded" } else { item.State = "healthy" } return item }