diff --git a/deploy.sh b/deploy.sh index b974dc3..8100374 100755 --- a/deploy.sh +++ b/deploy.sh @@ -11,7 +11,7 @@ # vps — triggered by a manual SSH/shell call on the VPS # # What it does: -# 0. Skip deploy if VPS already has the latest commit +# 0. Skip deploy if VPS already has the latest commit (and service is active) # 1. Stop the systemd service (prevents file locks) # 2. git fetch --all + git reset --hard origin/main + git clean # 3. npm ci --omit=dev (install/update production deps) @@ -38,14 +38,16 @@ say "Date: $(date -u '+%Y-%m-%dT%H:%M:%SZ')" cd "$PROJECT_DIR" # --------------------------------------------------------- -# Load .env so BOT_TOKEN and ADMIN_IDS are available for -# Telegram notifications throughout the script. +# Load only the variables needed for Telegram notifications. +# Using set -o allexport would export every line in .env +# (including unrelated or potentially sensitive variables). +# Instead, extract only BOT_TOKEN and ADMIN_IDS explicitly. # --------------------------------------------------------- if [[ -f "$PROJECT_DIR/.env" ]]; then - set -o allexport - # shellcheck source=/dev/null - source "$PROJECT_DIR/.env" - set +o allexport + _env_file="$PROJECT_DIR/.env" + BOT_TOKEN="$(grep -E '^BOT_TOKEN=' "$_env_file" | head -1 | cut -d= -f2- | tr -d '"'"'"' | tr -d $'\r')" || true + ADMIN_IDS="$(grep -E '^ADMIN_IDS=' "$_env_file" | head -1 | cut -d= -f2- | tr -d '"'"'"' | tr -d $'\r')" || true + export BOT_TOKEN ADMIN_IDS fi # --------------------------------------------------------- @@ -72,14 +74,31 @@ send_admin() { } # --------------------------------------------------------- -# Error trap — always notify admins if the script fails +# Track whether the service was stopped so the ERR trap can +# attempt a recovery restart if the deploy fails mid-way. +# --------------------------------------------------------- +_SERVICE_STOPPED=false + +# --------------------------------------------------------- +# Error trap — notify admins and attempt service recovery. +# ${BASH_LINENO[0]} gives the caller's line number when +# evaluated inside an ERR trap (more reliable than $LINENO). # --------------------------------------------------------- _on_error() { - local line="$1" + local line="${BASH_LINENO[0]:-?}" warn "Script failed at line $line" - send_admin "❌ Deploy failed at line $line — check VPS logs. (source: $DEPLOY_SOURCE)" + send_admin "❌ Deploy FAILED at line $line — check VPS logs. (source: $DEPLOY_SOURCE)" + # If the service was already stopped before the failure, try to bring it + # back up on whatever code is currently on disk so the bot isn't left down. + if [[ "$_SERVICE_STOPPED" == "true" ]] && command -v systemctl >/dev/null 2>&1; then + warn "Attempting service recovery restart on existing code…" + systemctl start "${APP_NAME}.service" || true + local _recovery_status + _recovery_status="$(systemctl is-active "${APP_NAME}.service" 2>/dev/null || echo unknown)" + send_admin "🔁 Recovery restart attempted after deploy failure — service is now: $_recovery_status" + fi } -trap '_on_error "$LINENO"' ERR +trap '_on_error' ERR # --------------------------------------------------------- # 0) Skip deploy if VPS already has the latest commit @@ -119,6 +138,7 @@ if command -v systemctl >/dev/null 2>&1; then say "Step 1/4 — Stopping ${APP_NAME} service…" send_admin "🛑 Stopping bot service…" systemctl stop "${APP_NAME}.service" || true + _SERVICE_STOPPED=true fi # --------------------------------------------------------- @@ -147,21 +167,37 @@ else # Restart the service so the bot comes back up on the old code if command -v systemctl >/dev/null 2>&1; then systemctl start "${APP_NAME}.service" || true + _SERVICE_STOPPED=false fi exit 1 fi # --------------------------------------------------------- # 3) Install / update production dependencies +# Wrapped in a conditional identical to the git guard: +# if npm fails the service is left down without this guard. +# On failure we restart on the existing node_modules so +# the bot comes back up on whatever was last working. # --------------------------------------------------------- say "Step 3/4 — Installing production dependencies…" send_admin "📦 Installing dependencies…" -if [[ -f package-lock.json ]]; then - npm ci --omit=dev + +NPM_CMD="npm install --omit=dev" +[[ -f package-lock.json ]] && NPM_CMD="npm ci --omit=dev" + +NPM_OUT="" +if NPM_OUT=$(${NPM_CMD} 2>&1); then + say "Dependencies installed." else - npm install --omit=dev + warn "npm install failed — restarting bot on existing node_modules" + warn "npm output: $NPM_OUT" + send_admin "❌ npm install failed — restarting bot on existing deps. Error: ${NPM_OUT:0:200} (source: $DEPLOY_SOURCE)" + if command -v systemctl >/dev/null 2>&1; then + systemctl start "${APP_NAME}.service" || true + _SERVICE_STOPPED=false + fi + exit 1 fi -say "Dependencies installed." # --------------------------------------------------------- # 4) Start bot via systemctl @@ -170,6 +206,7 @@ say "Step 4/4 — Starting bot via systemctl…" send_admin "🚀 Starting bot service…" if command -v systemctl >/dev/null 2>&1; then systemctl start "${APP_NAME}.service" + _SERVICE_STOPPED=false say "Bot started." else warn "systemctl not found. Start manually: node $PROJECT_DIR/index.js" diff --git a/scripts/disk-protect.sh b/scripts/disk-protect.sh index cadfa19..9f28683 100755 --- a/scripts/disk-protect.sh +++ b/scripts/disk-protect.sh @@ -26,9 +26,28 @@ JOURNAL_VACUUM_MB=300 # ── Logging ──────────────────────────────────────────────────────────────────── -log() { echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] [disk-protect] $*"; } +log() { echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] [disk-protect] $*"; } warn() { echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] [disk-protect] WARN: $*" >&2; } +# ── Helper: delete files found by find and log each one reliably ─────────────── +# Uses process substitution + null-delimited names to handle filenames with +# spaces or special characters, and separates print/delete so logging is never +# suppressed by find implementation differences. +delete_found() { + # Usage: delete_found + # Example: delete_found "$LOG_DIR" -type f -mtime +7 + local _count=0 + while IFS= read -r -d '' f; do + if rm -f "$f"; then + log " Deleted: $f" + _count=$((_count + 1)) + else + warn " Failed to delete: $f" + fi + done < <(find "$@" -print0 2>/dev/null) + log " Total deleted: $_count file(s)" +} + # ── 1. Log rotation (via logrotate if available) ─────────────────────────────── log "Step 1: Log rotation" if command -v logrotate >/dev/null 2>&1; then @@ -45,9 +64,13 @@ fi # ── 2. Compress logs older than 1 day ───────────────────────────────────────── log "Step 2: Compress logs older than 1 day" if [[ -d "$LOG_DIR" ]]; then - find "$LOG_DIR" -type f \( -name "*.log" -o -name "*.out" \) -mtime +1 ! -name "*.gz" | while read -r f; do - gzip -f "$f" && log " Compressed: $f" - done + while IFS= read -r -d '' f; do + if gzip -f "$f" 2>/dev/null; then + log " Compressed: $f" + else + warn " Failed to compress: $f" + fi + done < <(find "$LOG_DIR" -type f \( -name "*.log" -o -name "*.out" \) -mtime +1 ! -name "*.gz" -print0 2>/dev/null) else log " Log directory not found: $LOG_DIR — skipping" fi @@ -55,15 +78,13 @@ fi # ── 3. Delete logs older than LOG_MAX_DAYS days ──────────────────────────────── log "Step 3: Delete logs older than ${LOG_MAX_DAYS} days" if [[ -d "$LOG_DIR" ]]; then - find "$LOG_DIR" -type f -mtime "+${LOG_MAX_DAYS}" -delete -print | while read -r f; do - log " Deleted old log: $f" - done + delete_found "$LOG_DIR" -type f -mtime "+${LOG_MAX_DAYS}" fi # ── 4. journalctl vacuum ─────────────────────────────────────────────────────── log "Step 4: journalctl vacuum (keep up to ${JOURNAL_VACUUM_MB}MB)" if command -v journalctl >/dev/null 2>&1; then - journalctl --vacuum-size="${JOURNAL_VACUUM_MB}M" 2>&1 | grep -v "^$" | while read -r line; do + journalctl --vacuum-size="${JOURNAL_VACUUM_MB}M" 2>&1 | grep -v "^$" | while IFS= read -r line; do log " [journal] $line" done else @@ -73,7 +94,7 @@ fi # ── 5. Clear npm cache ───────────────────────────────────────────────────────── log "Step 5: Clear npm cache" if command -v npm >/dev/null 2>&1; then - npm cache clean --force 2>&1 | tail -1 | while read -r line; do log " [npm] $line"; done + npm cache clean --force 2>&1 | tail -1 | while IFS= read -r line; do log " [npm] $line"; done log " npm cache cleared" else log " npm not found — skipping" @@ -81,47 +102,47 @@ fi # ── 6. Clear temp files ──────────────────────────────────────────────────────── log "Step 6: Clear temp files older than ${TEMP_MAX_DAYS} days" -# Clear any .tmp files left by atomic writes inside the app if [[ -d "$DATA_DIR" ]]; then - find "$DATA_DIR" -type f -name "*.tmp.*" -mtime "+${TEMP_MAX_DAYS}" -delete -print | while read -r f; do - log " Deleted temp file: $f" - done + delete_found "$DATA_DIR" -type f -name "*.tmp.*" -mtime "+${TEMP_MAX_DAYS}" fi -# Clear /tmp files with our app name -find /tmp -type f -name "runewager-*" -mtime "+${TEMP_MAX_DAYS}" -delete 2>/dev/null | true +# Clear /tmp files with our app name (best-effort; ignore errors) +while IFS= read -r -d '' f; do + rm -f "$f" 2>/dev/null && log " Deleted /tmp file: $f" || true +done < <(find /tmp -type f -name "runewager-*" -mtime "+${TEMP_MAX_DAYS}" -print0 2>/dev/null) # ── 7. Delete old runtime state snapshots older than SNAPSHOT_MAX_DAYS ───────── log "Step 7: Delete runtime state snapshots older than ${SNAPSHOT_MAX_DAYS} days" if [[ -d "$BACKUP_DIR" ]]; then - find "$BACKUP_DIR" -type f -name "runtime-state-*.json" -mtime "+${SNAPSHOT_MAX_DAYS}" -delete -print | while read -r f; do - log " Deleted old snapshot: $f" - done + delete_found "$BACKUP_DIR" -type f -name "runtime-state-*.json" -mtime "+${SNAPSHOT_MAX_DAYS}" else log " Backup dir not found: $BACKUP_DIR — skipping" fi # ── 8. NEVER delete critical files — safety check ───────────────────────────── +# Verifies that the cleanup did not accidentally remove key runtime files. +# Sets CRITICAL_OK=false (and emits a warning) for every file that is missing. log "Step 8: Safety check — verifying critical files are intact" CRITICAL_OK=true for f in "${APP_DIR}/.env" "${DATA_DIR}/runtime-state.json"; do if [[ -f "$f" ]]; then log " OK: $f" else - log " (not present, not required): $f" + warn " MISSING: $f — expected file not found after cleanup" + CRITICAL_OK=false fi done if [[ "$CRITICAL_OK" != "true" ]]; then - warn " ⚠️ Critical file check failed — review above output" + warn "⚠️ One or more critical files are missing — review the output above" fi # ── 9. Disk usage summary ────────────────────────────────────────────────────── log "Step 9: Disk usage summary" -df -h / 2>/dev/null | tail -1 | while read -r line; do log " / : $line"; done +df -h / 2>/dev/null | tail -1 | while IFS= read -r line; do log " / : $line"; done if [[ -d "$APP_DIR" ]]; then - du -sh "$APP_DIR" 2>/dev/null | while read -r line; do log " App dir: $line"; done + du -sh "$APP_DIR" 2>/dev/null | while IFS= read -r line; do log " App dir: $line"; done fi if [[ -d "$LOG_DIR" ]]; then - du -sh "$LOG_DIR" 2>/dev/null | while read -r line; do log " Logs: $line"; done + du -sh "$LOG_DIR" 2>/dev/null | while IFS= read -r line; do log " Logs: $line"; done fi log "Disk protection complete."