From 91dd45e856195507403c19bffce7ffc292ac0561 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Mar 2026 11:34:57 +0000 Subject: [PATCH 1/2] fix(cpu-guard): honour MAX_STRIKES in kill ladder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ladder previously hardcoded 3 tiers (strike==1 / strike==2 / else), so MAX_STRIKES had no effect — termination always fired on the 3rd hit regardless of configuration. Fix: replace `strikes == 2` with `strikes < MAX_STRIKES` so the STOP/CONT pause repeats for strikes 2..(MAX_STRIKES-1) and termination only triggers at strike >= MAX_STRIKES. All log and Telegram messages now include the current strike / total (e.g. "STRIKE3/5") for clarity. Reported by CodeAnt AI review of PR #116. https://claude.ai/code/session_01WpEqAiDbpqnBywMjYXJHf4 --- scripts/rw_cpu_guard.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/rw_cpu_guard.sh b/scripts/rw_cpu_guard.sh index dc58d31..a29db8b 100755 --- a/scripts/rw_cpu_guard.sh +++ b/scripts/rw_cpu_guard.sh @@ -390,27 +390,27 @@ _handle_hog() { # ── Strike 1: Graceful throttle ────────────────────────────────────── _do renice +15 -p "$pid" _do ionice -c 3 -p "$pid" - act "STRIKE1 | PID=${pid} CPU=${cpu_pct}% zone=${zone} cmd=${short_cmd}" + act "STRIKE${strikes}/${MAX_STRIKES} | PID=${pid} CPU=${cpu_pct}% zone=${zone} cmd=${short_cmd}" - elif (( strikes == 2 )); then - # ── Strike 2: Hard pause — STOP 3s then CONT ───────────────────────── + elif (( strikes < MAX_STRIKES )); then + # ── Strikes 2..(MAX_STRIKES-1): Hard pause — STOP 3s then CONT ────── _do kill -STOP "$pid" sleep 3 _do kill -CONT "$pid" - act "STRIKE2 | PID=${pid} CPU=${cpu_pct}% zone=${zone} STOP/CONT cmd=${short_cmd}" - tg_alert "⚠️ STOP/CONT throttle — PID=${pid} CPU=${cpu_pct}% zone=${zone} — \`${short_cmd}\`" + act "STRIKE${strikes}/${MAX_STRIKES} | PID=${pid} CPU=${cpu_pct}% zone=${zone} STOP/CONT cmd=${short_cmd}" + tg_alert "⚠️ STOP/CONT throttle (strike ${strikes}/${MAX_STRIKES}) — PID=${pid} CPU=${cpu_pct}% — ${short_cmd}" else - # ── Strike 3+: Termination (SIGTERM then SIGKILL) ──────────────────── + # ── Strike MAX_STRIKES+: Termination (SIGTERM then SIGKILL) ────────── _do kill -TERM "$pid" sleep 2 if _is_alive "$pid"; then _do kill -KILL "$pid" - act "KILLED(SIGKILL) | PID=${pid} CPU=${cpu_pct}% zone=${zone} cmd=${short_cmd}" - tg_alert "💀 SIGKILL — PID=${pid} CPU=${cpu_pct}% zone=${zone} — \`${short_cmd}\`" + act "KILLED(SIGKILL) strike=${strikes}/${MAX_STRIKES} | PID=${pid} CPU=${cpu_pct}% zone=${zone} cmd=${short_cmd}" + tg_alert "💀 SIGKILL (strike ${strikes}/${MAX_STRIKES}) — PID=${pid} CPU=${cpu_pct}% — ${short_cmd}" else - act "KILLED(SIGTERM) | PID=${pid} CPU=${cpu_pct}% zone=${zone} cmd=${short_cmd}" - tg_alert "💀 SIGTERM — PID=${pid} CPU=${cpu_pct}% zone=${zone} — \`${short_cmd}\`" + act "KILLED(SIGTERM) strike=${strikes}/${MAX_STRIKES} | PID=${pid} CPU=${cpu_pct}% zone=${zone} cmd=${short_cmd}" + tg_alert "💀 SIGTERM (strike ${strikes}/${MAX_STRIKES}) — PID=${pid} CPU=${cpu_pct}% — ${short_cmd}" fi fi } From 93d9d009e1642299d18b52e7603a585a1bb9f23f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Mar 2026 11:44:33 +0000 Subject: [PATCH 2/2] feat(ops): add forensics mode to guard + nuclear redeploy script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rw_cpu_guard.sh — v3 upgrade: - --forensics [N] / --forensics=N mode merged from CPU.sh: vmstat (steal time), iostat (disk wait), 1-second ps tracer (catches sub-8s spikes), proc-count fork-storm detector, 10s top snapshots — all collected in /root/rw_cpu_forensics/ - Prints a structured summary after collection: top offenders, peak proc count, steal-time samples, guard ACT log tail - Safe to run alongside the daemon simultaneously (passive only) - FORENSICS_DIR and FORENSICS_DURATION env overrides added - Header updated to v3; all usage docs updated runewager_redeploy.sh — new one-shot master deploy script: 1. Stop runewager systemd service 2. Kill anything holding port 3000 (SIGTERM → SIGKILL) 3. npm ci at low priority (nice+ionice) 4. Install/restart rw_cpu_guard via --install 5. Start runewager systemd service 6. Health-check bot at /health (waits up to 30 s) 7. Launch 5-min background forensics trace (non-blocking) 8. Final status summary (services, port, load, memory, guard ACT log, bot journal tail) - --dry-run and --skip-forensics flags - Colour output; VPS path fallback; no tooltips touched https://claude.ai/code/session_01WpEqAiDbpqnBywMjYXJHf4 --- scripts/runewager_redeploy.sh | 361 ++++++++++++++++++++++++++++++++++ scripts/rw_cpu_guard.sh | 180 +++++++++++++++-- 2 files changed, 524 insertions(+), 17 deletions(-) create mode 100755 scripts/runewager_redeploy.sh diff --git a/scripts/runewager_redeploy.sh b/scripts/runewager_redeploy.sh new file mode 100755 index 0000000..b9877e1 --- /dev/null +++ b/scripts/runewager_redeploy.sh @@ -0,0 +1,361 @@ +#!/usr/bin/env bash +# +# runewager_redeploy.sh — One-Shot Nuclear Redeploy +# Runewager Bot — Hostinger KVM1 +# +# What this does (in order): +# 1. Stop the runewager systemd service +# 2. Kill anything still holding port 3000 +# 3. npm ci — clean dependency rebuild +# 4. Install / restart rw_cpu_guard as a systemd service +# 5. Start the runewager systemd service +# 6. Health-check the bot (waits up to 30 s) +# 7. Launch a background forensics trace (5 min, non-blocking) +# 8. Print a final status summary +# +# Usage: +# sudo ./scripts/runewager_redeploy.sh +# sudo ./scripts/runewager_redeploy.sh --skip-forensics # skip background trace +# sudo ./scripts/runewager_redeploy.sh --dry-run # print steps, no action +# +# Notes: +# - Tooltips are NOT touched — bot-saved tooltips are preserved. +# - Requires root (for systemd + cgroup + port ops). +# + +set -uo pipefail + +# ─────────────────────────────── IDENTITY ──────────────────────────────────── + +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly APP_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +readonly GUARD_SCRIPT="$SCRIPT_DIR/rw_cpu_guard.sh" + +BOT_SERVICE="runewager" +BOT_PORT=3000 +HEALTH_URL="http://127.0.0.1:${BOT_PORT}/health" +HEALTH_TIMEOUT=30 # seconds to wait for bot to respond + +DRY_RUN=0 +SKIP_FORENSICS=0 + +# ─────────────────────────────── COLOUR OUTPUT ─────────────────────────────── + +_bold='\033[1m' +_green='\033[0;32m' +_yellow='\033[1;33m' +_red='\033[0;31m' +_cyan='\033[0;36m' +_reset='\033[0m' + +ts() { date '+%H:%M:%S'; } +hdr() { echo -e "\n${_bold}${_cyan}╔══ $* ══╗${_reset}"; } +ok() { echo -e " ${_green}✔${_reset} $*"; } +warn() { echo -e " ${_yellow}⚠${_reset} $*"; } +fail() { echo -e " ${_red}✘${_reset} $*"; } +step() { echo -e "\n${_bold}[$(ts)] STEP $*${_reset}"; } + +run() { + # run + local desc="$1"; shift + echo -e " ${_cyan}→${_reset} ${desc}" + if (( DRY_RUN )); then + echo -e " ${_yellow}[DRY-RUN]${_reset} $*" + return 0 + fi + if "$@"; then + ok "$desc" + else + warn "$desc returned non-zero (continuing)" + fi +} + +# ─────────────────────────────── CHECKS ────────────────────────────────────── + +_check_root() { + if [[ "$(id -u)" -ne 0 ]]; then + fail "This script must be run as root (sudo $0)" + exit 1 + fi +} + +_check_app_dir() { + if [[ ! -f "$APP_DIR/index.js" ]]; then + # Try VPS default path + local vps_path="/var/www/html/Runewager" + if [[ -f "$vps_path/index.js" ]]; then + # Redefine app-relative paths + warn "index.js not at $APP_DIR — using VPS path $vps_path" + # shellcheck disable=SC2034 + APP_DIR_OVERRIDE="$vps_path" + else + fail "Cannot find index.js — is this the right directory? ($APP_DIR)" + exit 1 + fi + fi + ok "App dir: ${APP_DIR_OVERRIDE:-$APP_DIR}" +} + +# Use APP_DIR_OVERRIDE if set (VPS path fallback) +_app() { echo "${APP_DIR_OVERRIDE:-$APP_DIR}"; } + +# ─────────────────────────────── STEP 1: STOP SERVICE ──────────────────────── + +_stop_bot_service() { + step "1/8 — Stop ${BOT_SERVICE} systemd service" + if systemctl is-active --quiet "$BOT_SERVICE" 2>/dev/null; then + run "Stop ${BOT_SERVICE}" systemctl stop "$BOT_SERVICE" + else + warn "${BOT_SERVICE} service was not running (OK)" + fi +} + +# ─────────────────────────────── STEP 2: KILL PORT ─────────────────────────── + +_kill_port() { + step "2/8 — Kill anything on port ${BOT_PORT}" + local pids + pids=$(lsof -t -i :"${BOT_PORT}" 2>/dev/null) || true + + if [[ -z "$pids" ]]; then + ok "Port ${BOT_PORT} is free" + return 0 + fi + + echo " Found PIDs on :${BOT_PORT}: ${pids}" + if (( DRY_RUN )); then + warn "[DRY-RUN] would kill PIDs: $pids" + return 0 + fi + + # SIGTERM first, then SIGKILL stragglers + echo "$pids" | xargs -r kill -TERM 2>/dev/null || true + sleep 2 + local remaining + remaining=$(lsof -t -i :"${BOT_PORT}" 2>/dev/null) || true + if [[ -n "$remaining" ]]; then + echo "$remaining" | xargs -r kill -KILL 2>/dev/null || true + ok "Port ${BOT_PORT} force-cleared" + else + ok "Port ${BOT_PORT} cleared cleanly" + fi +} + +# ─────────────────────────────── STEP 3: NPM CI ────────────────────────────── + +_npm_rebuild() { + step "3/8 — npm ci (clean dependency rebuild)" + local app; app=$(_app) + + if [[ ! -f "$app/package.json" ]]; then + warn "No package.json at $app — skipping npm ci" + return 0 + fi + + if ! command -v node >/dev/null 2>&1; then + fail "node not found — cannot rebuild" + return 1 + fi + + local node_ver; node_ver=$(node --version 2>/dev/null) + ok "Node version: $node_ver" + + if (( DRY_RUN )); then + warn "[DRY-RUN] would run: cd $app && nice -n 19 npm ci --prefer-offline" + return 0 + fi + + pushd "$app" >/dev/null + # Run at low priority so this install doesn't spike the box + nice -n 19 ionice -c 3 npm ci --prefer-offline 2>&1 \ + | tail -5 \ + | sed 's/^/ /' + ok "npm ci complete" + popd >/dev/null +} + +# ─────────────────────────────── STEP 4: INSTALL GUARD ─────────────────────── + +_install_guard() { + step "4/8 — Install / restart rw_cpu_guard" + + if [[ ! -x "$GUARD_SCRIPT" ]]; then + fail "Guard script not found or not executable: $GUARD_SCRIPT" + return 1 + fi + + if (( DRY_RUN )); then + warn "[DRY-RUN] would run: $GUARD_SCRIPT --install" + return 0 + fi + + "$GUARD_SCRIPT" --install 2>&1 | tail -8 | sed 's/^/ /' + ok "rw_cpu_guard installed and running" +} + +# ─────────────────────────────── STEP 5: START BOT ─────────────────────────── + +_start_bot_service() { + step "5/8 — Start ${BOT_SERVICE} systemd service" + + if ! systemctl list-unit-files --type=service 2>/dev/null | grep -q "^${BOT_SERVICE}.service"; then + warn "No systemd unit for '${BOT_SERVICE}' — attempting prod-run.sh instead" + local app; app=$(_app) + if [[ -x "$app/prod-run.sh" ]]; then + if (( DRY_RUN )); then + warn "[DRY-RUN] would run: $app/prod-run.sh" + else + bash "$app/prod-run.sh" 2>&1 | tail -10 | sed 's/^/ /' + fi + else + fail "prod-run.sh not found either — start the bot manually" + fi + return 0 + fi + + run "Start ${BOT_SERVICE}" systemctl start "$BOT_SERVICE" +} + +# ─────────────────────────────── STEP 6: HEALTH CHECK ──────────────────────── + +_health_check() { + step "6/8 — Health check (waiting up to ${HEALTH_TIMEOUT}s)" + + if (( DRY_RUN )); then + warn "[DRY-RUN] would poll ${HEALTH_URL}" + return 0 + fi + + local elapsed=0 + while (( elapsed < HEALTH_TIMEOUT )); do + local http_code + http_code=$(curl -sS --max-time 3 -o /dev/null -w "%{http_code}" "$HEALTH_URL" 2>/dev/null) || true + if [[ "$http_code" == "200" ]]; then + ok "Bot healthy at ${HEALTH_URL} (${elapsed}s)" + # Print health payload + curl -sS --max-time 3 "$HEALTH_URL" 2>/dev/null | python3 -m json.tool 2>/dev/null \ + | head -20 | sed 's/^/ /' || true + return 0 + fi + sleep 3 + elapsed=$(( elapsed + 3 )) + echo -n " waiting... (${elapsed}s / ${http_code}) " + printf "\r" + done + + echo "" + fail "Bot did not respond at ${HEALTH_URL} within ${HEALTH_TIMEOUT}s" + warn "Check: journalctl -u ${BOT_SERVICE} -n 30 --no-pager" + return 1 +} + +# ─────────────────────────────── STEP 7: BACKGROUND FORENSICS ──────────────── + +_start_forensics() { + step "7/8 — Start background forensics trace (5 min)" + + if (( SKIP_FORENSICS )); then + warn "Forensics skipped (--skip-forensics)" + return 0 + fi + + if (( DRY_RUN )); then + warn "[DRY-RUN] would run: $GUARD_SCRIPT --forensics=300 &" + return 0 + fi + + if [[ ! -x "$GUARD_SCRIPT" ]]; then + warn "Guard script not found — skipping forensics" + return 0 + fi + + "$GUARD_SCRIPT" --forensics=300 >> /var/log/rw_cpu_forensics_deploy.log 2>&1 & + local fpid=$! + ok "Forensics trace running in background (PID ${fpid})" + ok "Log: /var/log/rw_cpu_forensics_deploy.log" + ok "Data: /root/rw_cpu_forensics/" +} + +# ─────────────────────────────── STEP 8: SUMMARY ───────────────────────────── + +_summary() { + step "8/8 — Final status summary" + echo "" + echo -e " ${_bold}Service states:${_reset}" + + for svc in "$BOT_SERVICE" "rw_cpu_guard"; do + local state + state=$(systemctl is-active "$svc" 2>/dev/null) || state="not-found" + if [[ "$state" == "active" ]]; then + ok " ${svc}: ${state}" + else + fail " ${svc}: ${state}" + fi + done + + echo "" + echo -e " ${_bold}Port ${BOT_PORT}:${_reset}" + local port_pids; port_pids=$(lsof -t -i :"$BOT_PORT" 2>/dev/null) || true + if [[ -n "$port_pids" ]]; then + ok " In use by PID(s): $port_pids" + ps -p "$port_pids" -o pid=,cmd= 2>/dev/null | sed 's/^/ /' || true + else + warn " Nothing on port ${BOT_PORT} — bot may still be starting" + fi + + echo "" + echo -e " ${_bold}Load avg:${_reset} $(cat /proc/loadavg)" + echo -e " ${_bold}Memory:${_reset} $(free -m | awk 'NR==2{printf "used=%sMB free=%sMB\n",$3,$4}')" + echo "" + + echo -e " ${_bold}Last guard actions:${_reset}" + grep "\[ACT \]" /var/log/rw_cpu_guard.log 2>/dev/null | tail -5 \ + | sed 's/^/ /' || echo " (no guard log yet)" + + echo "" + echo -e " ${_bold}Bot log tail (last 8 lines):${_reset}" + journalctl -u "$BOT_SERVICE" -n 8 --no-pager 2>/dev/null \ + | sed 's/^/ /' || echo " (no journal entries)" + + echo "" + hdr "REDEPLOY COMPLETE" +} + +# ─────────────────────────────── MAIN ──────────────────────────────────────── + +main() { + for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=1 ;; + --skip-forensics) SKIP_FORENSICS=1 ;; + --help|-h) + grep '^#' "${BASH_SOURCE[0]}" | head -25 | sed 's/^# *//' + exit 0 + ;; + *) + echo "Unknown option: $arg" >&2 + exit 1 + ;; + esac + done + + hdr "Runewager Nuclear Redeploy — $(date)" + echo -e " App dir : $(_app)" + echo -e " Guard : $GUARD_SCRIPT" + echo -e " Dry-run : $( (( DRY_RUN )) && echo YES || echo no )" + echo "" + + _check_root + _check_app_dir + + _stop_bot_service + _kill_port + _npm_rebuild + _install_guard + _start_bot_service + _health_check || true # non-fatal — bot may be slow to start; summary will show state + _start_forensics + _summary +} + +main "$@" diff --git a/scripts/rw_cpu_guard.sh b/scripts/rw_cpu_guard.sh index a29db8b..c0cede7 100755 --- a/scripts/rw_cpu_guard.sh +++ b/scripts/rw_cpu_guard.sh @@ -1,36 +1,43 @@ #!/usr/bin/env bash # -# rw_cpu_guard.sh — NUCLEAR EDITION v2 +# rw_cpu_guard.sh — NUCLEAR EDITION v3 # Hostinger KVM1 hardened CPU governor for Runewager # # Keeps the Runewager bot alive at all costs while ruthlessly # throttling/killing any non-whitelisted CPU hog. +# Includes built-in CPU forensics (merged from CPU.sh). # # Features: # - Load-aware 3-zone response (normal / soft / hard) # - Kill ladder: renice → ionice → cgroup → STOP/CONT → SIGTERM → SIGKILL +# - MAX_STRIKES honoured across the full ladder # - cgroup v1 + v2 auto-detection and CPU quota enforcement # - Fork-bomb detection (process-count ceiling) # - Telegram admin alerts on STOP/CONT and kills (5-min cooldown) # - Duplicate-instance guard via PID file # - Atomic state-file updates (no corruption under concurrent writes) -# - --install: write + enable + start systemd service -# - --status : show live guard state -# - --dry-run: simulate every action without executing it -# - --once : single scan then exit (great for cron) +# - --install : write + enable + start systemd service (requires root) +# - --status : show live guard state + recent log tail +# - --dry-run : simulate every action without executing it +# - --once : single scan then exit (great for cron) +# - --forensics [N]: passive CPU forensics for N seconds (default 300) +# captures vmstat/iostat/ps/proc-count then prints summary +# safe to run alongside the daemon simultaneously # # Usage: -# ./rw_cpu_guard.sh # foreground daemon -# ./rw_cpu_guard.sh --install # install as systemd service (requires root) -# ./rw_cpu_guard.sh --status # print current state -# ./rw_cpu_guard.sh --dry-run # simulate (pair with --once for quick test) -# ./rw_cpu_guard.sh --once # one scan then exit +# ./rw_cpu_guard.sh # foreground daemon +# ./rw_cpu_guard.sh --install # install as systemd service +# ./rw_cpu_guard.sh --status # print current state +# ./rw_cpu_guard.sh --dry-run --once # simulate a single scan +# ./rw_cpu_guard.sh --forensics # 5-min deep forensic trace +# ./rw_cpu_guard.sh --forensics=120 # 2-min forensic trace # # Environment overrides (all optional): # VCPU_COUNT, MAX_LOAD_SOFT, MAX_LOAD_HARD # PROC_CPU_SOFT, PROC_CPU_HARD, PROC_CPU_KILL # SCAN_INTERVAL, MAX_STRIKES, MAX_TOTAL_PROCS # CGROUP_NAME, CGROUP_QUOTA_PCT +# FORENSICS_DIR, FORENSICS_DURATION # TG_BOT_TOKEN, TG_ADMIN_IDS (auto-loaded from .env if absent) # LOG_FILE, STATE_FILE, PID_FILE, DRY_RUN # @@ -79,6 +86,10 @@ PID_FILE="${PID_FILE:-/tmp/${SCRIPT_NAME}.pid}" CGROUP_NAME="${CGROUP_NAME:-rw_cpu_guard}" CGROUP_QUOTA_PCT="${CGROUP_QUOTA_PCT:-70}" # % of one CPU +# Forensics mode +FORENSICS_DIR="${FORENSICS_DIR:-/root/rw_cpu_forensics}" +FORENSICS_DURATION="${FORENSICS_DURATION:-300}" # seconds (default 5 min) + # Telegram alerts — loaded from .env automatically if not set in environment TG_BOT_TOKEN="${TG_BOT_TOKEN:-}" TG_ADMIN_IDS="${TG_ADMIN_IDS:-}" @@ -582,6 +593,138 @@ _show_status() { tail -n 20 "$LOG_FILE" 2>/dev/null || echo " (log not found)" } +# ─────────────────────────────────── FORENSICS ─────────────────────────────── +# +# Passive data collection — never kills anything. +# Safe to run alongside the daemon in a second terminal. +# Merges the capabilities of the standalone CPU.sh script. +# +_run_forensics() { + local duration="${1:-$FORENSICS_DURATION}" + local out="$FORENSICS_DIR" + mkdir -p "$out" + + local trace="$out/cpu_trace.csv" + local vmstat_log="$out/vmstat.log" + local iostat_log="$out/iostat.log" + local proc_log="$out/proc_count.log" + local top_log="$out/top_cpu.log" + local snap="$out/snapshot.txt" + + echo "=== rw_cpu_guard FORENSICS — $(date) | duration=${duration}s ===" + echo " Output dir: $out" + echo "" + + # ── Snapshot ──────────────────────────────────────────────────────────── + { echo "=== UPTIME ==="; uptime + echo "=== LOADAVG ==="; cat /proc/loadavg + echo "=== CPU STAT ==="; head -1 /proc/stat + echo "=== MEMORY ==="; free -m + echo "=== CGROUP V2 ==="; [[ -f /sys/fs/cgroup/cgroup.controllers ]] && cat /sys/fs/cgroup/cgroup.controllers || echo "v1/none" + } > "$snap" 2>/dev/null + + # ── vmstat (1 s — captures steal time) ────────────────────────────────── + { echo "# timestamp r b swpd free buff cache si so bi bo in cs us sy id wa st" + vmstat 1 2>/dev/null | while IFS= read -r l; do echo "$(date '+%F %T') $l"; done + } >> "$vmstat_log" & + local _vmstat_pid=$! + + # ── iostat (1 s — captures disk wait) ─────────────────────────────────── + local _iostat_pid="" + if command -v iostat >/dev/null 2>&1; then + { echo "# timestamp iostat" + iostat -xz 1 2>/dev/null | while IFS= read -r l; do echo "$(date '+%F %T') $l"; done + } >> "$iostat_log" & + _iostat_pid=$! + else + echo "iostat not installed — skipping" > "$iostat_log" + fi + + # ── 1-second CPU tracer (catches sub-8s spikes the guard daemon may miss) ─ + echo "timestamp,pid,ppid,cpu%,mem%,cmd" > "$trace" + ( while true; do + ps -eo pid,ppid,pcpu,pmem,cmd --sort=-pcpu --no-headers 2>/dev/null \ + | awk -v ts="$(date '+%F %T')" '$3+0 > 15 { + gsub(/,/, ";", $5); + printf "%s,%s,%s,%s,%s,%s\n", ts, $1, $2, $3, $4, $5 + }' + sleep 1 + done + ) >> "$trace" & + local _trace_pid=$! + + # ── Top-10 snapshot every 10 s ─────────────────────────────────────────── + ( while true; do + printf "\n===== %s =====\n" "$(date '+%F %T')" >> "$top_log" + ps -eo pid,pcpu,pmem,cmd --sort=-pcpu --no-headers 2>/dev/null \ + | head -15 >> "$top_log" + sleep 10 + done + ) & + local _top_pid=$! + + # ── Process count (fork-storm detector) ───────────────────────────────── + ( while true; do + printf "%s %s\n" "$(date '+%F %T')" \ + "$(ls /proc | grep -cE '^[0-9]+$' 2>/dev/null)" >> "$proc_log" + sleep 5 + done + ) & + local _proc_pid=$! + + # ── Progress ticker ────────────────────────────────────────────────────── + local elapsed=0 + while (( elapsed < duration )); do + sleep 10 + elapsed=$(( elapsed + 10 )) + local load; load=$(awk '{print $1}' /proc/loadavg) + printf " [%3ds/%ds] load=%-5s lines_traced=%s\n" \ + "$elapsed" "$duration" "$load" \ + "$(wc -l < "$trace" 2>/dev/null || echo 0)" + done + + # ── Stop collectors ────────────────────────────────────────────────────── + kill "$_vmstat_pid" "$_trace_pid" "$_top_pid" "$_proc_pid" 2>/dev/null || true + [[ -n "$_iostat_pid" ]] && kill "$_iostat_pid" 2>/dev/null || true + wait 2>/dev/null || true + + # ── Summary ────────────────────────────────────────────────────────────── + echo "" + echo "════════════════════════════════════════════════" + echo " FORENSICS SUMMARY — ${duration}s window" + echo "════════════════════════════════════════════════" + + echo "" + echo "── Top CPU offenders (by peak %CPU) ────────────" + if [[ -s "$trace" ]]; then + tail -n +2 "$trace" | sort -t, -k4 -rn | awk -F, '!seen[$2]++' | head -10 \ + | awk -F, '{printf " PID=%-7s cpu=%-6s mem=%-6s %s\n", $2, $4"%", $5"%", substr($6,1,60)}' + else + echo " (no data — processes may have been below 15% threshold)" + fi + + echo "" + echo "── Peak process count (fork-storm indicator) ───" + sort -k2 -rn "$proc_log" 2>/dev/null | head -5 \ + | awk '{printf " %s procs=%s\n", $1, $2}' || echo " (no data)" + + echo "" + echo "── CPU steal time (last 5 vmstat samples) ──────" + grep -v "^#\|procs\|---\|^$" "$vmstat_log" 2>/dev/null \ + | tail -5 \ + | awk '{printf " %s %s steal=%s%%\n", $1, $2, $NF}' || echo " (no data)" + + echo "" + echo "── Guard kill log (ACT entries) ────────────────" + grep "\[ACT \]" "$LOG_FILE" 2>/dev/null | tail -10 \ + | sed 's/^/ /' || echo " (guard log not found)" + + echo "" + echo "Full data: $out" + echo "Guard log: $LOG_FILE" + echo "════════════════════════════════════════════════" +} + # ─────────────────────────────────── CLEANUP ───────────────────────────────── _cleanup() { @@ -596,12 +739,14 @@ main() { for arg in "$@"; do case "$arg" in - --install) mode="install" ;; - --status) mode="status" ;; - --once) mode="once" ;; - --dry-run) DRY_RUN="1" ;; + --install) mode="install" ;; + --status) mode="status" ;; + --once) mode="once" ;; + --dry-run) DRY_RUN="1" ;; + --forensics) mode="forensics" ;; + --forensics=*) mode="forensics"; FORENSICS_DURATION="${arg#*=}" ;; --help|-h) - grep '^#' "$SCRIPT_PATH" 2>/dev/null | head -30 | sed 's/^# *//' + grep '^#' "$SCRIPT_PATH" 2>/dev/null | head -45 | sed 's/^# *//' exit 0 ;; *) @@ -613,8 +758,9 @@ main() { # Modes that don't need daemon infrastructure case "$mode" in - install) _install_service; exit 0 ;; - status) _show_status; exit 0 ;; + install) _install_service; exit 0 ;; + status) _show_status; exit 0 ;; + forensics) _load_env; _run_forensics "$FORENSICS_DURATION"; exit 0 ;; esac # ── Daemon / once setup ───────────────────────────────────────────────────