feat: complete commercial developer workflowspublish-commercial-control-plane

Add tenant-safe usage observability, prepaid billing controls, API key lifecycle management, Embeddings metering, configurable billing alerts, and resilient provider health propagation. Harden Stripe failure handling, migrations, readiness, and the authenticated control-plane UI with end-to-end verification evidence.

1 files changed, 91 insertions, 0 deletions

diff --git a/internal/providerhealth/prober_test.go b/internal/providerhealth/prober_test.go
new file mode 100644
index 0000000..69b6e80
--- /dev/null
+++ b/internal/providerhealth/prober_test.go
@@ -0,0 +1,91 @@
+package providerhealth
+
+import (
+	"context"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"aigw/internal/catalog"
+	"aigw/internal/domain"
+)
+
+type probeMetricCounter struct {
+	total  atomic.Int64
+	failed atomic.Int64
+}
+
+func (m *probeMetricCounter) ProviderProbe(success bool) {
+	m.total.Add(1)
+	if !success {
+		m.failed.Add(1)
+	}
+}
+
+func TestProberAuthenticatesDeduplicatesAndOpensRouteCircuits(t *testing.T) {
+	var calls atomic.Int64
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		calls.Add(1)
+		if r.URL.Path != "/v1/models" || r.Header.Get("Authorization") != "Bearer secret" {
+			t.Errorf("unexpected probe path=%s auth=%q", r.URL.Path, r.Header.Get("Authorization"))
+		}
+		http.Error(w, "unavailable", http.StatusServiceUnavailable)
+	}))
+	defer server.Close()
+
+	provider := domain.Provider{ID: "provider", Protocol: domain.ProtocolOpenAI, WireAPI: "embeddings", BaseURL: server.URL + "/v1", APIKey: "secret"}
+	model := domain.Model{ID: "public/model", Routes: []domain.Route{{Provider: provider, UpstreamModel: "one"}, {Provider: provider, UpstreamModel: "two"}}}
+	tracker := New(Options{FailureThreshold: 1, OpenDuration: time.Minute})
+	metrics := &probeMetricCounter{}
+	prober := NewProber(ProbeOptions{Enabled: true, Catalog: catalog.NewModels([]domain.Model{model}), Tracker: tracker,
+		Metrics: metrics, Timeout: time.Second, Logger: slog.New(slog.NewTextHandler(io.Discard, nil))})
+	prober.ProbeOnce(context.Background())
+
+	key := RouteKey{ModelID: model.ID, ProviderID: provider.ID, WireAPI: provider.WireAPI}
+	if calls.Load() != 1 || metrics.total.Load() != 1 || metrics.failed.Load() != 1 || !tracker.CircuitOpen(key) {
+		t.Fatalf("calls=%d total=%d failed=%d open=%v", calls.Load(), metrics.total.Load(), metrics.failed.Load(), tracker.CircuitOpen(key))
+	}
+	status := tracker.Snapshot()[0]
+	if status.ActiveProbes != 1 || status.LastProbeAt == nil || status.LastStatusCode != http.StatusServiceUnavailable {
+		t.Fatalf("unexpected probe status: %+v", status)
+	}
+}
+
+func TestProberRejectsHTMLSuccessResponse(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		_, _ = io.WriteString(w, "<html>provider console</html>")
+	}))
+	defer server.Close()
+	provider := domain.Provider{ID: "provider", Protocol: domain.ProtocolOpenAI, BaseURL: server.URL, APIKey: "secret"}
+	model := domain.Model{ID: "model", Routes: []domain.Route{{Provider: provider, UpstreamModel: "model"}}}
+	tracker := New(Options{FailureThreshold: 1})
+	NewProber(ProbeOptions{Enabled: true, Catalog: catalog.NewModels([]domain.Model{model}), Tracker: tracker, Timeout: time.Second,
+		Logger: slog.New(slog.NewTextHandler(io.Discard, nil))}).ProbeOnce(context.Background())
+	if !tracker.CircuitOpen(RouteKey{ModelID: model.ID, ProviderID: provider.ID, WireAPI: "chat_completions"}) {
+		t.Fatal("HTML success response must not be treated as a healthy API probe")
+	}
+}
+
+func TestProberUsesAnthropicAuthentication(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("x-api-key") != "anthropic-secret" || r.Header.Get("anthropic-version") == "" {
+			t.Errorf("unexpected Anthropic headers")
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.WriteString(w, `{"data":[]}`)
+	}))
+	defer server.Close()
+	provider := domain.Provider{ID: "anthropic", Protocol: domain.ProtocolAnthropic, WireAPI: "messages", BaseURL: server.URL + "/v1", APIKey: "anthropic-secret"}
+	model := domain.Model{ID: "anthropic/model", Routes: []domain.Route{{Provider: provider, UpstreamModel: "model"}}}
+	tracker := New(Options{})
+	NewProber(ProbeOptions{Enabled: true, Catalog: catalog.NewModels([]domain.Model{model}), Tracker: tracker, Timeout: time.Second}).ProbeOnce(context.Background())
+	status := tracker.Snapshot()[0]
+	if status.State != "healthy" || status.ActiveProbes != 1 {
+		t.Fatalf("unexpected status: %+v", status)
+	}
+}