blob: cc5813209c47db6a3469acf208e3e0a400958900 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
|
#!/usr/bin/env bash
set -euo pipefail
: "${AIGW_READY_URL:?AIGW_READY_URL is required}"
: "${AIGW_REDIS_CONTAINER:?AIGW_REDIS_CONTAINER is required}"
timeout_seconds="${AIGW_REDIS_DRILL_TIMEOUT_SECONDS:-60}"
redis_stopped=false
cleanup() {
if [ "$redis_stopped" = true ]; then
docker start "$AIGW_REDIS_CONTAINER" >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT
readiness_has_redis_status() {
expected="$1"
body="$(curl --silent --show-error --fail "$AIGW_READY_URL" || true)"
printf '%s' "$body" | grep -q '"status":"ok"' &&
printf '%s' "$body" | grep -q "\"redis\":{\"required\":false,\"status\":\"$expected\"}"
}
shared_history_connected() {
[ -n "${AIGW_METRICS_URL:-}" ] || return 0
curl --silent --show-error --fail "$AIGW_METRICS_URL" |
grep -q '^aigw_provider_health_shared_connected 1$'
}
shared_history_disconnected() {
[ -n "${AIGW_METRICS_URL:-}" ] || return 0
curl --silent --show-error --fail "$AIGW_METRICS_URL" |
grep -q '^aigw_provider_health_shared_connected 0$'
}
if ! readiness_has_redis_status ok; then
printf 'Redis must be healthy before starting the drill\n' >&2
exit 1
fi
if ! shared_history_connected; then
printf 'shared provider-health history must be connected before starting the drill\n' >&2
exit 1
fi
docker stop "$AIGW_REDIS_CONTAINER" >/dev/null
redis_stopped=true
degraded=false
for _ in $(seq 1 "$timeout_seconds"); do
if readiness_has_redis_status degraded && shared_history_disconnected; then
degraded=true
break
fi
sleep 1
done
if [ "$degraded" != true ]; then
printf 'gateway did not report recoverable Redis degradation before timeout\n' >&2
exit 1
fi
docker start "$AIGW_REDIS_CONTAINER" >/dev/null
redis_stopped=false
for _ in $(seq 1 "$timeout_seconds"); do
if readiness_has_redis_status ok && shared_history_connected; then
printf 'redis degradation and automatic recovery confirmed; gateway remained ready\n'
exit 0
fi
sleep 1
done
printf 'Redis restarted but gateway did not reconnect before timeout\n' >&2
exit 1
|
