107 lines
3.4 KiB
Bash
107 lines
3.4 KiB
Bash
#!/usr/bin/env bash
|
|
# TST-017: Disaster Recovery Test — CC-047
|
|
# Simulates container failure and measures RTO (Recovery Time Objective).
|
|
# Target RTO: < 60 seconds from crash to healthy.
|
|
# Usage: bash scripts/test-dr.sh [image_tag]
|
|
# Exit codes: 0=PASS (RTO met), 1=FAIL
|
|
|
|
set -euo pipefail
|
|
|
|
IMAGE="${1:-ceo-api-staging:cc046}"
|
|
CONTAINER="dr-test-$(date +%s)"
|
|
TARGET_RTO_SECONDS=60
|
|
HEALTH_URL="http://localhost:13099/health"
|
|
|
|
log() { echo "[$(date -u +%H:%M:%S)] $*"; }
|
|
pass() { log "PASS: $*"; }
|
|
fail() { log "FAIL: $*"; exit 1; }
|
|
|
|
log "=== TST-017: Disaster Recovery Test ==="
|
|
log "Image: $IMAGE"
|
|
log "Container: $CONTAINER"
|
|
log "Target RTO: ${TARGET_RTO_SECONDS}s"
|
|
|
|
cleanup() { docker rm -f "$CONTAINER" &>/dev/null || true; }
|
|
trap cleanup EXIT
|
|
|
|
# 1. Start container (simulate normal operation)
|
|
log "Phase 1: Starting container..."
|
|
docker run -d --name "$CONTAINER" \
|
|
-p 13099:3001 \
|
|
-e NODE_ENV=staging \
|
|
-e PORT=3001 \
|
|
-e DATABASE_URL=postgresql://p:p@localhost/test \
|
|
-e REDIS_URL=redis://localhost:6379 \
|
|
-e SUPABASE_URL=https://placeholder.supabase.co \
|
|
-e SUPABASE_SERVICE_ROLE_KEY=placeholder \
|
|
-e CORS_ORIGINS=http://localhost:3000 \
|
|
-e CLEANUP_ENABLED=false \
|
|
"$IMAGE" >/dev/null
|
|
|
|
# Wait for initial health
|
|
WAIT=0
|
|
until curl -sf "$HEALTH_URL" &>/dev/null; do
|
|
sleep 1; ((WAIT++)) || true
|
|
if [[ "$WAIT" -gt 30 ]]; then fail "Container did not become healthy within 30s"; fi
|
|
done
|
|
log "Container healthy after ${WAIT}s startup"
|
|
|
|
# 2. Verify it's actually serving
|
|
HEALTH_BODY=$(curl -s "$HEALTH_URL")
|
|
if echo "$HEALTH_BODY" | grep -q '"status":"ok"'; then
|
|
pass "Health endpoint returns {status:ok}"
|
|
else
|
|
fail "Health endpoint returned unexpected: $HEALTH_BODY"
|
|
fi
|
|
|
|
# 3. Simulate crash (SIGKILL)
|
|
log "Phase 2: Simulating crash (SIGKILL)..."
|
|
CRASH_TIME=$(date +%s)
|
|
docker kill --signal=SIGKILL "$CONTAINER" >/dev/null
|
|
log "Container killed at $(date -u)"
|
|
|
|
# 4. Start replacement container (simulate restart/orchestration)
|
|
log "Phase 3: Starting replacement container..."
|
|
docker rm -f "$CONTAINER" &>/dev/null || true
|
|
docker run -d --name "$CONTAINER" \
|
|
-p 13099:3001 \
|
|
-e NODE_ENV=staging \
|
|
-e PORT=3001 \
|
|
-e DATABASE_URL=postgresql://p:p@localhost/test \
|
|
-e REDIS_URL=redis://localhost:6379 \
|
|
-e SUPABASE_URL=https://placeholder.supabase.co \
|
|
-e SUPABASE_SERVICE_ROLE_KEY=placeholder \
|
|
-e CORS_ORIGINS=http://localhost:3000 \
|
|
-e CLEANUP_ENABLED=false \
|
|
"$IMAGE" >/dev/null
|
|
|
|
# 5. Measure RTO
|
|
RECOVER_WAIT=0
|
|
until curl -sf "$HEALTH_URL" &>/dev/null; do
|
|
sleep 1; ((RECOVER_WAIT++)) || true
|
|
if [[ "$RECOVER_WAIT" -gt "$TARGET_RTO_SECONDS" ]]; then
|
|
fail "RTO exceeded: container not healthy after ${TARGET_RTO_SECONDS}s"
|
|
fi
|
|
done
|
|
|
|
RECOVER_TIME=$(date +%s)
|
|
ACTUAL_RTO=$((RECOVER_TIME - CRASH_TIME))
|
|
log "Recovery achieved in ${ACTUAL_RTO}s (target: ${TARGET_RTO_SECONDS}s)"
|
|
|
|
if [[ "$ACTUAL_RTO" -le "$TARGET_RTO_SECONDS" ]]; then
|
|
pass "RTO met: ${ACTUAL_RTO}s <= ${TARGET_RTO_SECONDS}s target"
|
|
else
|
|
fail "RTO exceeded: ${ACTUAL_RTO}s > ${TARGET_RTO_SECONDS}s target"
|
|
fi
|
|
|
|
# 6. Verify health after recovery
|
|
RECOVERY_BODY=$(curl -s "$HEALTH_URL")
|
|
if echo "$RECOVERY_BODY" | grep -q '"status":"ok"'; then
|
|
pass "Post-recovery health check: {status:ok}"
|
|
else
|
|
fail "Post-recovery health endpoint returned: $RECOVERY_BODY"
|
|
fi
|
|
|
|
log ""
|
|
log "=== TST-017 RESULT: PASS ==="
|
|
log "RTO: ${ACTUAL_RTO}s | Target: ${TARGET_RTO_SECONDS}s | Margin: $((TARGET_RTO_SECONDS - ACTUAL_RTO))s"
|