feat: harden prepaid billing and commercial operations

1 files changed, 143 insertions, 0 deletions

diff --git a/internal/operations/operations.go b/internal/operations/operations.go
new file mode 100644
index 0000000..e566488
--- /dev/null
+++ b/internal/operations/operations.go
@@ -0,0 +1,143 @@
+package operations
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"aigw/internal/billing"
+	"aigw/internal/controlplane"
+	"aigw/internal/telemetry"
+)
+
+type Handler struct {
+	Store          *controlplane.Store
+	Manager        *controlplane.Manager
+	Billing        *billing.Service
+	Metrics        *telemetry.Metrics
+	MaxSnapshotAge time.Duration
+}
+
+func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Path == "/healthz" {
+		write(w, http.StatusOK, map[string]any{"status": "alive"})
+		return
+	}
+	if r.URL.Path == "/metrics" && h.Metrics != nil {
+		h.Metrics.ServeHTTP(w, r)
+		return
+	}
+	if r.URL.Path != "/readyz" {
+		http.NotFound(w, r)
+		return
+	}
+	ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
+	defer cancel()
+	checks := map[string]any{}
+	ready := true
+	if h.Store != nil {
+		if err := h.Store.Ping(ctx); err != nil {
+			checks["postgres"] = map[string]any{"status": "failed", "error": err.Error()}
+			ready = false
+		} else {
+			checks["postgres"] = map[string]any{"status": "ok"}
+		}
+	}
+	if h.Manager != nil {
+		healthyAt := h.Manager.LastHealthyAt()
+		if healthyAt.IsZero() {
+			healthyAt = h.Manager.LastReloadAt()
+		}
+		age := time.Since(healthyAt)
+		snapshotOK := h.Manager.Loaded() && age <= h.MaxSnapshotAge
+		checks["snapshot"] = map[string]any{"status": status(snapshotOK), "generation": h.Manager.Generation(), "age_seconds": int64(age.Seconds())}
+		if !snapshotOK {
+			ready = false
+		}
+		redisStatus := "disabled"
+		if h.Manager.RedisConfigured() {
+			redisStatus = "degraded"
+			if h.Manager.RedisConnected() {
+				redisStatus = "ok"
+			}
+		}
+		checks["redis"] = map[string]any{"status": redisStatus, "required": false}
+	}
+	if h.Billing != nil {
+		if err := h.Billing.Ping(ctx); err != nil {
+			checks["billing_postgres"] = map[string]any{"status": "failed", "error": err.Error()}
+			ready = false
+		} else {
+			checks["billing_postgres"] = map[string]any{"status": "ok"}
+		}
+		queue, err := h.Billing.SettlementQueueStatus(ctx)
+		if err != nil {
+			checks["settlement_queue"] = map[string]any{"status": "failed", "error": err.Error()}
+			ready = false
+		} else {
+			backlog := queue.AwaitingEvent + queue.Pending + queue.Processing + queue.Retrying
+			queueOK := queue.SpoolRecords == 0
+			if queue.OldestPending != nil && time.Since(*queue.OldestPending) > 15*time.Minute {
+				queueOK = false
+			}
+			if !queueOK {
+				ready = false
+			}
+			checks["settlement_queue"] = map[string]any{"status": status(queueOK), "backlog": backlog, "spool_records": queue.SpoolRecords, "oldest_pending": queue.OldestPending}
+			if h.Metrics != nil {
+				h.Metrics.SetSettlementQueue(backlog, queue.SpoolRecords)
+			}
+			billingHealth, err := h.Billing.OperationalStatus(ctx)
+			if err != nil {
+				checks["billing_operations"] = map[string]any{"status": "failed", "error": err.Error()}
+				ready = false
+			} else {
+				billingOK := billingHealth.Ready(time.Now().UTC())
+				checks["billing_operations"] = map[string]any{"status": status(billingOK), "details": billingHealth}
+				if !billingOK {
+					ready = false
+				}
+				if h.Metrics != nil {
+					h.Metrics.SetStripeOperations(billingHealth)
+				}
+			}
+		}
+		if h.Store != nil {
+			mail, err := h.Store.MailQueueStatus(ctx)
+			if err != nil {
+				checks["mail_queue"] = map[string]any{"status": "failed", "error": err.Error()}
+				ready = false
+			} else {
+				mailOK := mail.Failed == 0
+				if mail.OldestPending != nil && time.Since(*mail.OldestPending) > 10*time.Minute {
+					mailOK = false
+				}
+				checks["mail_queue"] = map[string]any{"status": status(mailOK), "details": mail}
+				if !mailOK {
+					ready = false
+				}
+			}
+		}
+	}
+	if h.Metrics != nil {
+		h.Metrics.SetReady(ready)
+	}
+	code := http.StatusOK
+	if !ready {
+		code = http.StatusServiceUnavailable
+	}
+	write(w, code, map[string]any{"status": status(ready), "checks": checks})
+}
+
+func status(ok bool) string {
+	if ok {
+		return "ok"
+	}
+	return "failed"
+}
+func write(w http.ResponseWriter, code int, value any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(code)
+	_ = json.NewEncoder(w).Encode(value)
+}