diff --git a/scripts/test-dr.sh b/scripts/test-dr.sh new file mode 100644 index 0000000..5d0c19e --- /dev/null +++ b/scripts/test-dr.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# TST-017: Disaster Recovery Test — CC-047 +# Simulates container failure and measures RTO (Recovery Time Objective). +# Target RTO: < 60 seconds from crash to healthy. +# Usage: bash scripts/test-dr.sh [image_tag] +# Exit codes: 0=PASS (RTO met), 1=FAIL + +set -euo pipefail + +IMAGE="${1:-ceo-api-staging:cc046}" +CONTAINER="dr-test-$(date +%s)" +TARGET_RTO_SECONDS=60 +HEALTH_URL="http://localhost:13099/health" + +log() { echo "[$(date -u +%H:%M:%S)] $*"; } +pass() { log "PASS: $*"; } +fail() { log "FAIL: $*"; exit 1; } + +log "=== TST-017: Disaster Recovery Test ===" +log "Image: $IMAGE" +log "Container: $CONTAINER" +log "Target RTO: ${TARGET_RTO_SECONDS}s" + +cleanup() { docker rm -f "$CONTAINER" &>/dev/null || true; } +trap cleanup EXIT + +# 1. Start container (simulate normal operation) +log "Phase 1: Starting container..." +docker run -d --name "$CONTAINER" \ + -p 13099:3001 \ + -e NODE_ENV=staging \ + -e PORT=3001 \ + -e DATABASE_URL=postgresql://p:p@localhost/test \ + -e REDIS_URL=redis://localhost:6379 \ + -e SUPABASE_URL=https://placeholder.supabase.co \ + -e SUPABASE_SERVICE_ROLE_KEY=placeholder \ + -e CORS_ORIGINS=http://localhost:3000 \ + -e CLEANUP_ENABLED=false \ + "$IMAGE" >/dev/null + +# Wait for initial health +WAIT=0 +until curl -sf "$HEALTH_URL" &>/dev/null; do + sleep 1; ((WAIT++)) || true + if [[ "$WAIT" -gt 30 ]]; then fail "Container did not become healthy within 30s"; fi +done +log "Container healthy after ${WAIT}s startup" + +# 2. Verify it's actually serving +HEALTH_BODY=$(curl -s "$HEALTH_URL") +if echo "$HEALTH_BODY" | grep -q '"status":"ok"'; then + pass "Health endpoint returns {status:ok}" +else + fail "Health endpoint returned unexpected: $HEALTH_BODY" +fi + +# 3. Simulate crash (SIGKILL) +log "Phase 2: Simulating crash (SIGKILL)..." +CRASH_TIME=$(date +%s) +docker kill --signal=SIGKILL "$CONTAINER" >/dev/null +log "Container killed at $(date -u)" + +# 4. Start replacement container (simulate restart/orchestration) +log "Phase 3: Starting replacement container..." +docker rm -f "$CONTAINER" &>/dev/null || true +docker run -d --name "$CONTAINER" \ + -p 13099:3001 \ + -e NODE_ENV=staging \ + -e PORT=3001 \ + -e DATABASE_URL=postgresql://p:p@localhost/test \ + -e REDIS_URL=redis://localhost:6379 \ + -e SUPABASE_URL=https://placeholder.supabase.co \ + -e SUPABASE_SERVICE_ROLE_KEY=placeholder \ + -e CORS_ORIGINS=http://localhost:3000 \ + -e CLEANUP_ENABLED=false \ + "$IMAGE" >/dev/null + +# 5. Measure RTO +RECOVER_WAIT=0 +until curl -sf "$HEALTH_URL" &>/dev/null; do + sleep 1; ((RECOVER_WAIT++)) || true + if [[ "$RECOVER_WAIT" -gt "$TARGET_RTO_SECONDS" ]]; then + fail "RTO exceeded: container not healthy after ${TARGET_RTO_SECONDS}s" + fi +done + +RECOVER_TIME=$(date +%s) +ACTUAL_RTO=$((RECOVER_TIME - CRASH_TIME)) +log "Recovery achieved in ${ACTUAL_RTO}s (target: ${TARGET_RTO_SECONDS}s)" + +if [[ "$ACTUAL_RTO" -le "$TARGET_RTO_SECONDS" ]]; then + pass "RTO met: ${ACTUAL_RTO}s <= ${TARGET_RTO_SECONDS}s target" +else + fail "RTO exceeded: ${ACTUAL_RTO}s > ${TARGET_RTO_SECONDS}s target" +fi + +# 6. Verify health after recovery +RECOVERY_BODY=$(curl -s "$HEALTH_URL") +if echo "$RECOVERY_BODY" | grep -q '"status":"ok"'; then + pass "Post-recovery health check: {status:ok}" +else + fail "Post-recovery health endpoint returned: $RECOVERY_BODY" +fi + +log "" +log "=== TST-017 RESULT: PASS ===" +log "RTO: ${ACTUAL_RTO}s | Target: ${TARGET_RTO_SECONDS}s | Margin: $((TARGET_RTO_SECONDS - ACTUAL_RTO))s"