From 8047b07e4769feae865b77f3bb3e88ee9b984030 Mon Sep 17 00:00:00 2001 From: GambleCodez Affiliates Date: Fri, 20 Feb 2026 19:02:54 -0600 Subject: [PATCH] Improve prod-run health output and service/logrotate resilience --- prod-run.sh | 148 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 119 insertions(+), 29 deletions(-) diff --git a/prod-run.sh b/prod-run.sh index 9cd7f24..6f0f2b7 100755 --- a/prod-run.sh +++ b/prod-run.sh @@ -7,9 +7,13 @@ INDEX_FILE="$PROJECT_DIR/index.js" ENV_FILE="$PROJECT_DIR/.env" LOG_DIR="$PROJECT_DIR/logs" BOT_LOG="$LOG_DIR/runewager.log" +LEGACY_BOT_LOG="$PROJECT_DIR/runewager.log" +CRASH_LOG="$LOG_DIR/crash.log" +SYSTEMD_STDOUT_LOG="$LOG_DIR/runewager-1.log" SERVICE_FILE="/etc/systemd/system/runewager.service" LOGROTATE_FILE="/etc/logrotate.d/runewager" CRON_LINE='0 3 * * * /usr/sbin/logrotate -f /etc/logrotate.d/runewager' +HEALTH_LOG_MAX_AGE_MINUTES=10 say() { printf '[runewager] %s\n' "$*" @@ -31,6 +35,66 @@ ensure_file() { [ -f "$1" ] || : > "$1" } +ensure_log_paths() { + ensure_dir "$LOG_DIR" + ensure_file "$BOT_LOG" + ensure_file "$CRASH_LOG" + ensure_file "$SYSTEMD_STDOUT_LOG" + + if [ ! -f "$LEGACY_BOT_LOG" ]; then + : > "$LEGACY_BOT_LOG" + fi +} + + +get_bot_pid() { + pgrep -f "node .*index\.js" | head -n 1 || true +} + +log_age_seconds() { + local file="$1" + if [ ! -f "$file" ]; then + echo -1 + return + fi + local now mtime + now="$(date +%s)" + mtime="$(stat -c %Y "$file" 2>/dev/null || echo 0)" + echo $((now - mtime)) +} + +print_health_summary() { + local pid + pid="$(get_bot_pid)" + + if [ -n "$pid" ]; then + say "✅ Bot is running (pid: $pid)" + else + err "❌ Bot is not running" + return + fi + + local active_log="$BOT_LOG" + if [ ! -s "$active_log" ] && [ -s "$LEGACY_BOT_LOG" ]; then + active_log="$LEGACY_BOT_LOG" + fi + + local age + age="$(log_age_seconds "$active_log")" + if [ -s "$active_log" ] && [ "$age" -ge 0 ] && [ "$age" -le $((HEALTH_LOG_MAX_AGE_MINUTES * 60)) ]; then + say "✅ Health check passed: recent logs are present (${age}s old) at $active_log" + return + fi + + if [ -s "$active_log" ] && [ "$age" -gt $((HEALTH_LOG_MAX_AGE_MINUTES * 60)) ]; then + warn "Bot process is running but logs are stale (${age}s old) at $active_log" + else + warn "Bot process is running but no recent logs were found" + fi + + warn "If /start is unresponsive, restart with: pkill -f 'node .*index\.js' && bash $PROJECT_DIR/prod-run.sh" +} + fix_working_directory() { if [ "$(pwd)" != "$PROJECT_DIR" ]; then say "Switching working directory to $PROJECT_DIR" @@ -77,9 +141,14 @@ fix_git_remote_and_branch() { } fix_node_modules() { - if [ ! -d "$PROJECT_DIR/node_modules" ] || [ ! -f "$PROJECT_DIR/node_modules/.package-lock.json" ]; then - say "Installing dependencies with npm ci --omit=dev" - npm ci --omit=dev + if [ ! -d "$PROJECT_DIR/node_modules" ]; then + if [ -f "$PROJECT_DIR/package-lock.json" ]; then + say "Installing dependencies with npm ci --omit=dev" + npm ci --omit=dev + else + say "Installing dependencies with npm install --omit=dev (package-lock.json missing)" + npm install --omit=dev + fi fi } @@ -246,9 +315,16 @@ fix_require_and_import_hints() { } ensure_logrotate_config() { + local log_owner="runewager" + local log_group="runewager" + if ! id -u "$log_owner" >/dev/null 2>&1; then + log_owner="$(id -un 2>/dev/null || echo root)" + log_group="$(id -gn 2>/dev/null || echo root)" + fi + local conf - conf=$(cat <<'CONF' -/var/www/html/Runewager/logs/runewager*.log /var/www/html/Runewager/logs/crash.log { + conf=$(cat < "$LOGROTATE_FILE" + fi + if [ -f "$LOGROTATE_FILE" ] && command -v logrotate >/dev/null 2>&1; then - if ! logrotate -d "$LOGROTATE_FILE" >/dev/null 2>&1; then + local logrotate_output='' + if ! logrotate_output="$(logrotate -d "$LOGROTATE_FILE" 2>&1)"; then warn "logrotate dry-run failed for $LOGROTATE_FILE" + warn "$logrotate_output" else say "logrotate config validation passed" fi @@ -295,8 +378,16 @@ ensure_cron_fallback() { } ensure_systemd_service() { + local service_user="runewager" + local service_group="runewager" + if ! id -u "$service_user" >/dev/null 2>&1; then + service_user="$(id -un 2>/dev/null || echo root)" + service_group="$(id -gn 2>/dev/null || echo root)" + warn "runewager user missing; using $service_user:$service_group for systemd service" + fi + local conf - conf=$(cat <<'CONF' + conf=$(cat </dev/null || true)" + if [ -z "$blob" ] && [ -f "$LEGACY_BOT_LOG" ]; then + blob="$(tail -n 200 "$LEGACY_BOT_LOG" 2>/dev/null || true)" + fi say "Human-readable diagnosis" printf '%s\n' "$blob" | grep -qi 'Cannot find module' && say "Root cause: A Node module is missing or a require path is wrong. Run npm ci and verify relative import paths." @@ -382,21 +481,13 @@ print_human_errors() { } is_bot_running() { - pgrep -f "node .*index\.js" >/dev/null 2>&1 + [ -n "$(get_bot_pid)" ] } check_bot_status_twice() { - if is_bot_running; then - say "Bot running" - else - say "Bot NOT running" - fi + print_health_summary sleep 15 - if is_bot_running; then - say "Bot running" - else - say "Bot NOT running" - fi + print_health_summary } start_background_if_needed() { @@ -433,10 +524,9 @@ main() { exit 1 fi - ensure_dir "$LOG_DIR" + ensure_log_paths ensure_dir "$PROJECT_DIR/data" ensure_dir "$PROJECT_DIR/data/backups" - ensure_file "$BOT_LOG" fix_merge_conflicts fix_lockfiles @@ -453,10 +543,10 @@ main() { ensure_cron_fallback ensure_systemd_service - print_human_errors check_bot_status_twice start_background_if_needed check_bot_status_twice + print_human_errors } main "$@"