1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
|
package operations
import (
"context"
"encoding/json"
"net/http"
"time"
"aigw/internal/billing"
"aigw/internal/controlplane"
"aigw/internal/telemetry"
)
type Handler struct {
Store *controlplane.Store
Manager *controlplane.Manager
Billing *billing.Service
Metrics *telemetry.Metrics
MaxSnapshotAge time.Duration
}
func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/healthz" {
write(w, http.StatusOK, map[string]any{"status": "alive"})
return
}
if r.URL.Path == "/metrics" && h.Metrics != nil {
h.Metrics.ServeHTTP(w, r)
return
}
if r.URL.Path != "/readyz" {
http.NotFound(w, r)
return
}
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
defer cancel()
checks := map[string]any{}
ready := true
if h.Store != nil {
if err := h.Store.Ping(ctx); err != nil {
checks["postgres"] = map[string]any{"status": "failed", "error": err.Error()}
ready = false
} else {
checks["postgres"] = map[string]any{"status": "ok"}
}
}
if h.Manager != nil {
healthyAt := h.Manager.LastHealthyAt()
if healthyAt.IsZero() {
healthyAt = h.Manager.LastReloadAt()
}
age := time.Since(healthyAt)
snapshotOK := h.Manager.Loaded() && age <= h.MaxSnapshotAge
checks["snapshot"] = map[string]any{"status": status(snapshotOK), "generation": h.Manager.Generation(), "age_seconds": int64(age.Seconds())}
if !snapshotOK {
ready = false
}
redisStatus := "disabled"
if h.Manager.RedisConfigured() {
redisStatus = "degraded"
if h.Manager.RedisConnected() {
redisStatus = "ok"
}
}
checks["redis"] = map[string]any{"status": redisStatus, "required": false}
}
if h.Billing != nil {
if err := h.Billing.Ping(ctx); err != nil {
checks["billing_postgres"] = map[string]any{"status": "failed", "error": err.Error()}
ready = false
} else {
checks["billing_postgres"] = map[string]any{"status": "ok"}
}
queue, err := h.Billing.SettlementQueueStatus(ctx)
if err != nil {
checks["settlement_queue"] = map[string]any{"status": "failed", "error": err.Error()}
ready = false
} else {
backlog := queue.AwaitingEvent + queue.Pending + queue.Processing + queue.Retrying
queueOK := queue.SpoolRecords == 0
if queue.OldestPending != nil && time.Since(*queue.OldestPending) > 15*time.Minute {
queueOK = false
}
if !queueOK {
ready = false
}
checks["settlement_queue"] = map[string]any{"status": status(queueOK), "backlog": backlog, "spool_records": queue.SpoolRecords, "oldest_pending": queue.OldestPending}
if h.Metrics != nil {
h.Metrics.SetSettlementQueue(backlog, queue.SpoolRecords)
}
billingHealth, err := h.Billing.OperationalStatus(ctx)
if err != nil {
checks["billing_operations"] = map[string]any{"status": "failed", "error": err.Error()}
ready = false
} else {
billingOK := billingHealth.Ready(time.Now().UTC())
checks["billing_operations"] = map[string]any{"status": status(billingOK), "details": billingHealth}
if !billingOK {
ready = false
}
if h.Metrics != nil {
h.Metrics.SetStripeOperations(billingHealth)
}
}
}
if h.Store != nil {
mail, err := h.Store.MailQueueStatus(ctx)
if err != nil {
checks["mail_queue"] = map[string]any{"status": "failed", "error": err.Error()}
ready = false
} else {
mailOK := mail.Failed == 0
if mail.OldestPending != nil && time.Since(*mail.OldestPending) > 10*time.Minute {
mailOK = false
}
checks["mail_queue"] = map[string]any{"status": status(mailOK), "details": mail}
if !mailOK {
ready = false
}
}
}
}
if h.Metrics != nil {
h.Metrics.SetReady(ready)
}
code := http.StatusOK
if !ready {
code = http.StatusServiceUnavailable
}
write(w, code, map[string]any{"status": status(ready), "checks": checks})
}
func status(ok bool) string {
if ok {
return "ok"
}
return "failed"
}
func write(w http.ResponseWriter, code int, value any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
_ = json.NewEncoder(w).Encode(value)
}
|
