feat: complete commercial developer workflowspublish-commercial-control-plane

Add tenant-safe usage observability, prepaid billing controls, API key lifecycle management, Embeddings metering, configurable billing alerts, and resilient provider health propagation. Harden Stripe failure handling, migrations, readiness, and the authenticated control-plane UI with end-to-end verification evidence.

1 files changed, 56 insertions, 6 deletions

diff --git a/scripts/redis-fault-drill.sh b/scripts/redis-fault-drill.sh
index 8be92ff..cc58132 100755
--- a/scripts/redis-fault-drill.sh
+++ b/scripts/redis-fault-drill.sh
@@ -3,16 +3,66 @@ set -euo pipefail
 
 : "${AIGW_READY_URL:?AIGW_READY_URL is required}"
 : "${AIGW_REDIS_CONTAINER:?AIGW_REDIS_CONTAINER is required}"
-cleanup() { docker start "$AIGW_REDIS_CONTAINER" >/dev/null 2>&1 || true; }
+timeout_seconds="${AIGW_REDIS_DRILL_TIMEOUT_SECONDS:-60}"
+redis_stopped=false
+cleanup() {
+  if [ "$redis_stopped" = true ]; then
+    docker start "$AIGW_REDIS_CONTAINER" >/dev/null 2>&1 || true
+  fi
+}
 trap cleanup EXIT
-docker stop "$AIGW_REDIS_CONTAINER" >/dev/null
-for _ in $(seq 1 20); do
+
+readiness_has_redis_status() {
+  expected="$1"
   body="$(curl --silent --show-error --fail "$AIGW_READY_URL" || true)"
-  if printf '%s' "$body" | grep -q '"redis"' && printf '%s' "$body" | grep -q '"required":false'; then
-    printf 'redis degradation confirmed; gateway remained ready\n'
+  printf '%s' "$body" | grep -q '"status":"ok"' &&
+    printf '%s' "$body" | grep -q "\"redis\":{\"required\":false,\"status\":\"$expected\"}"
+}
+
+shared_history_connected() {
+  [ -n "${AIGW_METRICS_URL:-}" ] || return 0
+  curl --silent --show-error --fail "$AIGW_METRICS_URL" |
+    grep -q '^aigw_provider_health_shared_connected 1$'
+}
+
+shared_history_disconnected() {
+  [ -n "${AIGW_METRICS_URL:-}" ] || return 0
+  curl --silent --show-error --fail "$AIGW_METRICS_URL" |
+    grep -q '^aigw_provider_health_shared_connected 0$'
+}
+
+if ! readiness_has_redis_status ok; then
+  printf 'Redis must be healthy before starting the drill\n' >&2
+  exit 1
+fi
+if ! shared_history_connected; then
+  printf 'shared provider-health history must be connected before starting the drill\n' >&2
+  exit 1
+fi
+
+docker stop "$AIGW_REDIS_CONTAINER" >/dev/null
+redis_stopped=true
+degraded=false
+for _ in $(seq 1 "$timeout_seconds"); do
+  if readiness_has_redis_status degraded && shared_history_disconnected; then
+    degraded=true
+    break
+  fi
+  sleep 1
+done
+if [ "$degraded" != true ]; then
+  printf 'gateway did not report recoverable Redis degradation before timeout\n' >&2
+  exit 1
+fi
+
+docker start "$AIGW_REDIS_CONTAINER" >/dev/null
+redis_stopped=false
+for _ in $(seq 1 "$timeout_seconds"); do
+  if readiness_has_redis_status ok && shared_history_connected; then
+    printf 'redis degradation and automatic recovery confirmed; gateway remained ready\n'
     exit 0
   fi
   sleep 1
 done
-printf 'gateway did not report recoverable Redis degradation\n' >&2
+printf 'Redis restarted but gateway did not reconnect before timeout\n' >&2
 exit 1