feat(tst-017): DR test script — container crash + RTO measurement

scripts/test-dr.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# TST-017: Disaster Recovery Test — CC-047
+# Simulates container failure and measures RTO (Recovery Time Objective).
+# Target RTO: < 60 seconds from crash to healthy.
+# Usage: bash scripts/test-dr.sh [image_tag]
+# Exit codes: 0=PASS (RTO met), 1=FAIL
+
+set -euo pipefail
+
+IMAGE="${1:-ceo-api-staging:cc046}"
+CONTAINER="dr-test-$(date +%s)"
+TARGET_RTO_SECONDS=60
+HEALTH_URL="http://localhost:13099/health"
+
+log()  { echo "[$(date -u +%H:%M:%S)] $*"; }
+pass() { log "PASS: $*"; }
+fail() { log "FAIL: $*"; exit 1; }
+
+log "=== TST-017: Disaster Recovery Test ==="
+log "Image: $IMAGE"
+log "Container: $CONTAINER"
+log "Target RTO: ${TARGET_RTO_SECONDS}s"
+
+cleanup() { docker rm -f "$CONTAINER" &>/dev/null || true; }
+trap cleanup EXIT
+
+# 1. Start container (simulate normal operation)
+log "Phase 1: Starting container..."
+docker run -d --name "$CONTAINER" \
+  -p 13099:3001 \
+  -e NODE_ENV=staging \
+  -e PORT=3001 \
+  -e DATABASE_URL=postgresql://p:p@localhost/test \
+  -e REDIS_URL=redis://localhost:6379 \
+  -e SUPABASE_URL=https://placeholder.supabase.co \
+  -e SUPABASE_SERVICE_ROLE_KEY=placeholder \
+  -e CORS_ORIGINS=http://localhost:3000 \
+  -e CLEANUP_ENABLED=false \
+  "$IMAGE" >/dev/null
+
+# Wait for initial health
+WAIT=0
+until curl -sf "$HEALTH_URL" &>/dev/null; do
+  sleep 1; ((WAIT++)) || true
+  if [[ "$WAIT" -gt 30 ]]; then fail "Container did not become healthy within 30s"; fi
+done
+log "Container healthy after ${WAIT}s startup"
+
+# 2. Verify it's actually serving
+HEALTH_BODY=$(curl -s "$HEALTH_URL")
+if echo "$HEALTH_BODY" | grep -q '"status":"ok"'; then
+  pass "Health endpoint returns {status:ok}"
+else
+  fail "Health endpoint returned unexpected: $HEALTH_BODY"
+fi
+
+# 3. Simulate crash (SIGKILL)
+log "Phase 2: Simulating crash (SIGKILL)..."
+CRASH_TIME=$(date +%s)
+docker kill --signal=SIGKILL "$CONTAINER" >/dev/null
+log "Container killed at $(date -u)"
+
+# 4. Start replacement container (simulate restart/orchestration)
+log "Phase 3: Starting replacement container..."
+docker rm -f "$CONTAINER" &>/dev/null || true
+docker run -d --name "$CONTAINER" \
+  -p 13099:3001 \
+  -e NODE_ENV=staging \
+  -e PORT=3001 \
+  -e DATABASE_URL=postgresql://p:p@localhost/test \
+  -e REDIS_URL=redis://localhost:6379 \
+  -e SUPABASE_URL=https://placeholder.supabase.co \
+  -e SUPABASE_SERVICE_ROLE_KEY=placeholder \
+  -e CORS_ORIGINS=http://localhost:3000 \
+  -e CLEANUP_ENABLED=false \
+  "$IMAGE" >/dev/null
+
+# 5. Measure RTO
+RECOVER_WAIT=0
+until curl -sf "$HEALTH_URL" &>/dev/null; do
+  sleep 1; ((RECOVER_WAIT++)) || true
+  if [[ "$RECOVER_WAIT" -gt "$TARGET_RTO_SECONDS" ]]; then
+    fail "RTO exceeded: container not healthy after ${TARGET_RTO_SECONDS}s"
+  fi
+done
+
+RECOVER_TIME=$(date +%s)
+ACTUAL_RTO=$((RECOVER_TIME - CRASH_TIME))
+log "Recovery achieved in ${ACTUAL_RTO}s (target: ${TARGET_RTO_SECONDS}s)"
+
+if [[ "$ACTUAL_RTO" -le "$TARGET_RTO_SECONDS" ]]; then
+  pass "RTO met: ${ACTUAL_RTO}s <= ${TARGET_RTO_SECONDS}s target"
+else
+  fail "RTO exceeded: ${ACTUAL_RTO}s > ${TARGET_RTO_SECONDS}s target"
+fi
+
+# 6. Verify health after recovery
+RECOVERY_BODY=$(curl -s "$HEALTH_URL")
+if echo "$RECOVERY_BODY" | grep -q '"status":"ok"'; then
+  pass "Post-recovery health check: {status:ok}"
+else
+  fail "Post-recovery health endpoint returned: $RECOVERY_BODY"
+fi
+
+log ""
+log "=== TST-017 RESULT: PASS ==="
+log "RTO: ${ACTUAL_RTO}s | Target: ${TARGET_RTO_SECONDS}s | Margin: $((TARGET_RTO_SECONDS - ACTUAL_RTO))s"