#!/usr/bin/env bash # TST-017: Disaster Recovery Test — CC-047 # Simulates container failure and measures RTO (Recovery Time Objective). # Target RTO: < 60 seconds from crash to healthy. # Usage: bash scripts/test-dr.sh [image_tag] # Exit codes: 0=PASS (RTO met), 1=FAIL set -euo pipefail IMAGE="${1:-ceo-api-staging:cc046}" CONTAINER="dr-test-$(date +%s)" TARGET_RTO_SECONDS=60 HEALTH_URL="http://localhost:13099/health" log() { echo "[$(date -u +%H:%M:%S)] $*"; } pass() { log "PASS: $*"; } fail() { log "FAIL: $*"; exit 1; } log "=== TST-017: Disaster Recovery Test ===" log "Image: $IMAGE" log "Container: $CONTAINER" log "Target RTO: ${TARGET_RTO_SECONDS}s" cleanup() { docker rm -f "$CONTAINER" &>/dev/null || true; } trap cleanup EXIT # 1. Start container (simulate normal operation) log "Phase 1: Starting container..." docker run -d --name "$CONTAINER" \ -p 13099:3001 \ -e NODE_ENV=staging \ -e PORT=3001 \ -e DATABASE_URL=postgresql://p:p@localhost/test \ -e REDIS_URL=redis://localhost:6379 \ -e SUPABASE_URL=https://placeholder.supabase.co \ -e SUPABASE_SERVICE_ROLE_KEY=placeholder \ -e CORS_ORIGINS=http://localhost:3000 \ -e CLEANUP_ENABLED=false \ "$IMAGE" >/dev/null # Wait for initial health WAIT=0 until curl -sf "$HEALTH_URL" &>/dev/null; do sleep 1; ((WAIT++)) || true if [[ "$WAIT" -gt 30 ]]; then fail "Container did not become healthy within 30s"; fi done log "Container healthy after ${WAIT}s startup" # 2. Verify it's actually serving HEALTH_BODY=$(curl -s "$HEALTH_URL") if echo "$HEALTH_BODY" | grep -q '"status":"ok"'; then pass "Health endpoint returns {status:ok}" else fail "Health endpoint returned unexpected: $HEALTH_BODY" fi # 3. Simulate crash (SIGKILL) log "Phase 2: Simulating crash (SIGKILL)..." CRASH_TIME=$(date +%s) docker kill --signal=SIGKILL "$CONTAINER" >/dev/null log "Container killed at $(date -u)" # 4. Start replacement container (simulate restart/orchestration) log "Phase 3: Starting replacement container..." docker rm -f "$CONTAINER" &>/dev/null || true docker run -d --name "$CONTAINER" \ -p 13099:3001 \ -e NODE_ENV=staging \ -e PORT=3001 \ -e DATABASE_URL=postgresql://p:p@localhost/test \ -e REDIS_URL=redis://localhost:6379 \ -e SUPABASE_URL=https://placeholder.supabase.co \ -e SUPABASE_SERVICE_ROLE_KEY=placeholder \ -e CORS_ORIGINS=http://localhost:3000 \ -e CLEANUP_ENABLED=false \ "$IMAGE" >/dev/null # 5. Measure RTO RECOVER_WAIT=0 until curl -sf "$HEALTH_URL" &>/dev/null; do sleep 1; ((RECOVER_WAIT++)) || true if [[ "$RECOVER_WAIT" -gt "$TARGET_RTO_SECONDS" ]]; then fail "RTO exceeded: container not healthy after ${TARGET_RTO_SECONDS}s" fi done RECOVER_TIME=$(date +%s) ACTUAL_RTO=$((RECOVER_TIME - CRASH_TIME)) log "Recovery achieved in ${ACTUAL_RTO}s (target: ${TARGET_RTO_SECONDS}s)" if [[ "$ACTUAL_RTO" -le "$TARGET_RTO_SECONDS" ]]; then pass "RTO met: ${ACTUAL_RTO}s <= ${TARGET_RTO_SECONDS}s target" else fail "RTO exceeded: ${ACTUAL_RTO}s > ${TARGET_RTO_SECONDS}s target" fi # 6. Verify health after recovery RECOVERY_BODY=$(curl -s "$HEALTH_URL") if echo "$RECOVERY_BODY" | grep -q '"status":"ok"'; then pass "Post-recovery health check: {status:ok}" else fail "Post-recovery health endpoint returned: $RECOVERY_BODY" fi log "" log "=== TST-017 RESULT: PASS ===" log "RTO: ${ACTUAL_RTO}s | Target: ${TARGET_RTO_SECONDS}s | Margin: $((TARGET_RTO_SECONDS - ACTUAL_RTO))s"