feat: complete commercial developer workflowspublish-commercial-control-plane

Add tenant-safe usage observability, prepaid billing controls, API key lifecycle management, Embeddings metering, configurable billing alerts, and resilient provider health propagation. Harden Stripe failure handling, migrations, readiness, and the authenticated control-plane UI with end-to-end verification evidence.

1 files changed, 54 insertions, 21 deletions

diff --git a/internal/controlplane/preferences.go b/internal/controlplane/preferences.go
index 73bcce6..2f80ad5 100644
--- a/internal/controlplane/preferences.go
+++ b/internal/controlplane/preferences.go
@@ -10,22 +10,37 @@ import (
 	"github.com/jackc/pgx/v5"
 )
 
-const maxLowBalanceThresholdMicros int64 = 1_000_000_000_000_000
+const maxAlertThresholdMicros int64 = 1_000_000_000_000_000
 
-func (s *Store) GetTenantPreferences(ctx context.Context, tenantID string, defaultThresholdMicros int64) (TenantPreferences, error) {
-	if defaultThresholdMicros <= 0 {
-		defaultThresholdMicros = 5_000_000
+func normalizeBillingPreferenceDefaults(defaults BillingPreferenceDefaults) BillingPreferenceDefaults {
+	if defaults.LowBalanceThresholdMicros <= 0 {
+		defaults.LowBalanceThresholdMicros = 5_000_000
 	}
-	result := TenantPreferences{TenantID: strings.TrimSpace(tenantID), LowBalanceEnabled: true, LowBalanceThresholdMicros: defaultThresholdMicros}
+	if defaults.SpendAnomalyMultiplier < 2 {
+		defaults.SpendAnomalyMultiplier = 3
+	}
+	if defaults.SpendAnomalyMinMicros < 0 {
+		defaults.SpendAnomalyMinMicros = 10_000_000
+	}
+	return defaults
+}
+
+func (s *Store) GetTenantPreferences(ctx context.Context, tenantID string, defaults BillingPreferenceDefaults) (TenantPreferences, error) {
+	defaults = normalizeBillingPreferenceDefaults(defaults)
+	result := TenantPreferences{TenantID: strings.TrimSpace(tenantID), LowBalanceEnabled: true,
+		LowBalanceThresholdMicros: defaults.LowBalanceThresholdMicros, SpendAnomalyEnabled: true,
+		SpendAnomalyMultiplier: defaults.SpendAnomalyMultiplier, SpendAnomalyMinMicros: defaults.SpendAnomalyMinMicros}
 	if result.TenantID == "" {
 		return result, nil
 	}
 	var defaultModel, fallbackModel *string
 	var updatedAt time.Time
 	err := s.db.QueryRow(ctx, `
-		SELECT default_model, fallback_model, low_balance_enabled, low_balance_threshold_micros, updated_at
-		FROM tenant_preferences WHERE tenant_id=$1`, result.TenantID).Scan(
-		&defaultModel, &fallbackModel, &result.LowBalanceEnabled, &result.LowBalanceThresholdMicros, &updatedAt)
+		SELECT default_model, fallback_model, low_balance_enabled, low_balance_threshold_micros,
+			spend_anomaly_enabled, COALESCE(spend_anomaly_multiplier,$2), COALESCE(spend_anomaly_min_micros,$3), updated_at
+		FROM tenant_preferences WHERE tenant_id=$1`, result.TenantID, defaults.SpendAnomalyMultiplier, defaults.SpendAnomalyMinMicros).Scan(
+		&defaultModel, &fallbackModel, &result.LowBalanceEnabled, &result.LowBalanceThresholdMicros,
+		&result.SpendAnomalyEnabled, &result.SpendAnomalyMultiplier, &result.SpendAnomalyMinMicros, &updatedAt)
 	if errors.Is(err, pgx.ErrNoRows) {
 		return result, nil
 	}
@@ -42,10 +57,11 @@ func (s *Store) GetTenantPreferences(ctx context.Context, tenantID string, defau
 	return result, nil
 }
 
-func (s *Store) SetDeveloperPreferences(ctx context.Context, input SetDeveloperPreferencesInput) (TenantPreferences, error) {
+func (s *Store) SetDeveloperPreferences(ctx context.Context, input SetDeveloperPreferencesInput, defaults BillingPreferenceDefaults) (TenantPreferences, error) {
 	input.TenantID = strings.TrimSpace(input.TenantID)
 	input.DefaultModel = strings.TrimSpace(input.DefaultModel)
 	input.FallbackModel = strings.TrimSpace(input.FallbackModel)
+	defaults = normalizeBillingPreferenceDefaults(defaults)
 	if input.TenantID == "" {
 		return TenantPreferences{}, errors.New("tenant_id is required")
 	}
@@ -87,9 +103,12 @@ func (s *Store) SetDeveloperPreferences(ctx context.Context, input SetDeveloperP
 		ON CONFLICT (tenant_id) DO UPDATE SET default_model=EXCLUDED.default_model,
 			fallback_model=EXCLUDED.fallback_model, updated_at=now()
 		RETURNING tenant_id::text, default_model, fallback_model, low_balance_enabled,
-			low_balance_threshold_micros, updated_at`, input.TenantID, input.DefaultModel, input.FallbackModel).Scan(
+			low_balance_threshold_micros, spend_anomaly_enabled, COALESCE(spend_anomaly_multiplier,$4),
+			COALESCE(spend_anomaly_min_micros,$5), updated_at`, input.TenantID, input.DefaultModel, input.FallbackModel,
+		defaults.SpendAnomalyMultiplier, defaults.SpendAnomalyMinMicros).Scan(
 		&result.TenantID, &defaultModel, &fallbackModel, &result.LowBalanceEnabled,
-		&result.LowBalanceThresholdMicros, &updatedAt); err != nil {
+		&result.LowBalanceThresholdMicros, &result.SpendAnomalyEnabled, &result.SpendAnomalyMultiplier,
+		&result.SpendAnomalyMinMicros, &updatedAt); err != nil {
 		return TenantPreferences{}, fmt.Errorf("save developer preferences: %w", err)
 	}
 	if defaultModel != nil {
@@ -105,17 +124,21 @@ func (s *Store) SetDeveloperPreferences(ctx context.Context, input SetDeveloperP
 	return result, nil
 }
 
-func (s *Store) SetBillingPreferences(ctx context.Context, input SetBillingPreferencesInput, defaultThresholdMicros int64) (TenantPreferences, error) {
+func (s *Store) SetBillingPreferences(ctx context.Context, input SetBillingPreferencesInput, defaults BillingPreferenceDefaults) (TenantPreferences, error) {
 	input.TenantID = strings.TrimSpace(input.TenantID)
 	if input.TenantID == "" {
 		return TenantPreferences{}, errors.New("tenant_id is required")
 	}
-	if defaultThresholdMicros <= 0 {
-		defaultThresholdMicros = 5_000_000
-	}
-	if input.LowBalanceThresholdMicros != nil && (*input.LowBalanceThresholdMicros < 0 || *input.LowBalanceThresholdMicros > maxLowBalanceThresholdMicros) {
+	defaults = normalizeBillingPreferenceDefaults(defaults)
+	if input.LowBalanceThresholdMicros != nil && (*input.LowBalanceThresholdMicros < 0 || *input.LowBalanceThresholdMicros > maxAlertThresholdMicros) {
 		return TenantPreferences{}, errors.New("low balance threshold is outside the supported range")
 	}
+	if input.SpendAnomalyMultiplier != nil && (*input.SpendAnomalyMultiplier < 2 || *input.SpendAnomalyMultiplier > 1000) {
+		return TenantPreferences{}, errors.New("spend anomaly multiplier must be between 2 and 1000")
+	}
+	if input.SpendAnomalyMinMicros != nil && (*input.SpendAnomalyMinMicros < 0 || *input.SpendAnomalyMinMicros > maxAlertThresholdMicros) {
+		return TenantPreferences{}, errors.New("spend anomaly minimum is outside the supported range")
+	}
 	tx, err := s.db.Begin(ctx)
 	if err != nil {
 		return TenantPreferences{}, err
@@ -131,15 +154,25 @@ func (s *Store) SetBillingPreferences(ctx context.Context, input SetBillingPrefe
 	var defaultModel, fallbackModel *string
 	var updatedAt time.Time
 	if err := tx.QueryRow(ctx, `
-		INSERT INTO tenant_preferences (tenant_id, low_balance_enabled, low_balance_threshold_micros)
-		VALUES ($1, COALESCE($2::boolean, TRUE), COALESCE($3::bigint, $4::bigint))
+		INSERT INTO tenant_preferences (tenant_id, low_balance_enabled, low_balance_threshold_micros,
+			spend_anomaly_enabled, spend_anomaly_multiplier, spend_anomaly_min_micros)
+		VALUES ($1, COALESCE($2::boolean, TRUE), COALESCE($3::bigint, $7::bigint),
+			COALESCE($4::boolean, TRUE), $5::bigint, $6::bigint)
 		ON CONFLICT (tenant_id) DO UPDATE SET
 			low_balance_enabled=COALESCE($2::boolean, tenant_preferences.low_balance_enabled),
-			low_balance_threshold_micros=COALESCE($3::bigint, tenant_preferences.low_balance_threshold_micros), updated_at=now()
+			low_balance_threshold_micros=COALESCE($3::bigint, tenant_preferences.low_balance_threshold_micros),
+			spend_anomaly_enabled=COALESCE($4::boolean, tenant_preferences.spend_anomaly_enabled),
+			spend_anomaly_multiplier=COALESCE($5::bigint, tenant_preferences.spend_anomaly_multiplier),
+			spend_anomaly_min_micros=COALESCE($6::bigint, tenant_preferences.spend_anomaly_min_micros), updated_at=now()
 		RETURNING tenant_id::text, default_model, fallback_model, low_balance_enabled,
-			low_balance_threshold_micros, updated_at`, input.TenantID, input.LowBalanceEnabled, input.LowBalanceThresholdMicros, defaultThresholdMicros).Scan(
+			low_balance_threshold_micros, spend_anomaly_enabled,
+			COALESCE(spend_anomaly_multiplier,$8), COALESCE(spend_anomaly_min_micros,$9), updated_at`,
+		input.TenantID, input.LowBalanceEnabled, input.LowBalanceThresholdMicros, input.SpendAnomalyEnabled,
+		input.SpendAnomalyMultiplier, input.SpendAnomalyMinMicros, defaults.LowBalanceThresholdMicros,
+		defaults.SpendAnomalyMultiplier, defaults.SpendAnomalyMinMicros).Scan(
 		&result.TenantID, &defaultModel, &fallbackModel, &result.LowBalanceEnabled,
-		&result.LowBalanceThresholdMicros, &updatedAt); err != nil {
+		&result.LowBalanceThresholdMicros, &result.SpendAnomalyEnabled, &result.SpendAnomalyMultiplier,
+		&result.SpendAnomalyMinMicros, &updatedAt); err != nil {
 		return TenantPreferences{}, fmt.Errorf("save billing preferences: %w", err)
 	}
 	if defaultModel != nil {