1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
|
package providerhealth
import (
"sync"
"testing"
"time"
)
func TestTrackerOpensAndRecoversCircuit(t *testing.T) {
now := time.Date(2026, time.August, 6, 0, 0, 0, 0, time.UTC)
tracker := New(Options{FailureThreshold: 3, OpenDuration: 30 * time.Second, Now: func() time.Time { return now }})
key := RouteKey{ModelID: "openai/test", ProviderID: "primary", WireAPI: "responses"}
for _, status := range []int{503, 429, 504} {
tracker.Observe(key, Observation{StatusCode: status, Latency: 100 * time.Millisecond, Failed: true})
}
if !tracker.CircuitOpen(key) {
t.Fatal("circuit did not open after consecutive retryable failures")
}
snapshot := tracker.Snapshot()
if len(snapshot) != 1 || snapshot[0].State != "open" || snapshot[0].AvailabilityPercent != 0 || snapshot[0].CircuitOpenUntil == nil {
t.Fatalf("unexpected open snapshot: %+v", snapshot)
}
now = now.Add(31 * time.Second)
if tracker.CircuitOpen(key) {
t.Fatal("circuit did not permit a recovery attempt after cooldown")
}
tracker.Observe(key, Observation{StatusCode: 200, Latency: 50 * time.Millisecond})
snapshot = tracker.Snapshot()
if snapshot[0].State != "healthy" || snapshot[0].ConsecutiveFailures != 0 || snapshot[0].LastHealthyAt == nil || snapshot[0].HeaderLatencyEWMA < 50 {
t.Fatalf("unexpected recovered snapshot: %+v", snapshot[0])
}
}
func TestTrackerConcurrentObservations(t *testing.T) {
tracker := New(Options{FailureThreshold: 1000})
key := RouteKey{ModelID: "model", ProviderID: "provider", WireAPI: "chat_completions"}
var group sync.WaitGroup
for index := range 200 {
group.Add(1)
go func(failed bool) {
defer group.Done()
tracker.Observe(key, Observation{StatusCode: 200, Latency: time.Millisecond, Failed: failed})
}(index%2 == 0)
}
group.Wait()
snapshot := tracker.Snapshot()
if len(snapshot) != 1 || snapshot[0].Attempts != 200 || snapshot[0].RecentSamples != recentWindow || snapshot[0].AvailabilityPercent < 0 || snapshot[0].AvailabilityPercent > 100 {
t.Fatalf("unexpected concurrent snapshot: %+v", snapshot)
}
}
|
