diff --git a/index.js b/index.js index f005de4..265215f 100644 --- a/index.js +++ b/index.js @@ -496,7 +496,7 @@ function appendBonusAdminLog(actorId, action, details = '') { fs.mkdirSync(path.dirname(bonusAdminLogFile), { recursive: true }); const line = `[${new Date().toISOString()}] actor=${actorId || 'system'} action=${action}${details ? ` details=${details}` : ''} `; - fs.appendFileSync(bonusAdminLogFile, line, 'utf8'); + fs.appendFileSync(bonusAdminLogFile, line, { encoding: 'utf8', mode: 0o640 }); } catch (_) { /* ignore log write errors */ } } const backupDir = path.join(dataDir, 'backups'); @@ -4538,7 +4538,7 @@ function adminLog(event, payload) { // Also append to persistent NDJSON log file (best-effort) try { ensureDataDir(); - fs.appendFileSync(adminEventsLogFile, JSON.stringify(entry) + '\n'); + fs.appendFileSync(adminEventsLogFile, JSON.stringify(entry) + '\n', { mode: 0o640 }); } catch (_) { /* non-fatal — in-memory log is the source of truth */ } } @@ -4800,7 +4800,7 @@ function writeFileAtomic(filePath, content) { const dirPath = path.dirname(safeFilePath); if (!fs.existsSync(dirPath)) fs.mkdirSync(dirPath, { recursive: true }); const tempFile = `${safeFilePath}.tmp.${process.pid}.${Date.now()}`; - fs.writeFileSync(tempFile, content); + fs.writeFileSync(tempFile, content, { mode: 0o600 }); fs.renameSync(tempFile, safeFilePath); } @@ -4829,16 +4829,25 @@ function saveJson(filePath, data) { writeFileAtomic(filePath, JSON.stringify(data, null, 2)); } -// Expected error codes when the service user cannot write to qa/ (created as root). -const QA_FS_SUPPRESS = new Set(['EACCES', 'EPERM', 'EROFS', 'EEXIST']); +/** + * QA_FS_SUPPRESS — single authoritative set of errno codes that are silently + * swallowed across ALL QA filesystem operations (ensureQaDirs, ensureQaArtifacts, + * persistQaProviderState). Add codes here once and every helper inherits the + * suppression automatically. + * + * EEXIST is intentionally excluded: mkdirSync({ recursive: true }) never throws + * EEXIST for an existing directory, so seeing it here means a *file* sits where a + * directory is expected — a misconfiguration we want surfaced, not silenced. + */ +const QA_FS_SUPPRESS = new Set(['EACCES', 'EPERM', 'EROFS']); function ensureQaDirs() { for (const dir of [qaContextDir, qaStateDir, qaLogsDir]) { try { fs.mkdirSync(dir, { recursive: true }); } catch (e) { - if (!QA_FS_SUPPRESS.has(e.code)) { - logEvent('warn', 'ensureQaDirs: unexpected error creating QA directory', { dir, code: e.code, error: e.message }); + if (!QA_FS_SUPPRESS.has(e?.code)) { + logEvent('warn', 'ensureQaDirs: unexpected error creating QA directory', { dir, code: e?.code, error: e?.message ?? String(e), ...(process.env.LOG_LEVEL === 'debug' && { stack: e?.stack }) }); } } } @@ -4854,7 +4863,7 @@ function getQaLogDirForToday() { function writeQaLog(fileName, payload) { try { const line = `${JSON.stringify({ ts: new Date().toISOString(), ...payload })}\n`; - fs.appendFileSync(path.join(getQaLogDirForToday(), fileName), line, 'utf8'); + fs.appendFileSync(path.join(getQaLogDirForToday(), fileName), line, { encoding: 'utf8', mode: 0o640 }); } catch (_) { /* ignore QA log write errors */ } } @@ -4893,14 +4902,14 @@ function generateQaCapabilities() { function persistQaProviderState() { try { ensureQaDirs(); - fs.writeFileSync(qaProviderStateFile, JSON.stringify(qaRuntime.providerStatus, null, 2)); + fs.writeFileSync(qaProviderStateFile, JSON.stringify(qaRuntime.providerStatus, null, 2), { mode: 0o600 }); } catch (e) { - if (QA_FS_SUPPRESS.has(e.code)) { + if (QA_FS_SUPPRESS.has(e?.code)) { // Expected: service user lacks write permission on qa/ (created as root). Non-fatal. return; } // Unexpected (e.g. JSON serialization failure, bad path) — log so it stays visible. - logEvent('warn', 'persistQaProviderState: unexpected error', { code: e.code, error: e.message }); + logEvent('warn', 'persistQaProviderState: unexpected error', { code: e?.code, error: e?.message ?? String(e), ...(process.env.LOG_LEVEL === 'debug' && { stack: e?.stack }) }); } } @@ -4916,32 +4925,38 @@ function refreshQaProviderCooldowns() { persistQaProviderState(); } -function ensureQaArtifacts() { - // QA artifacts are observability helpers — never fatal. - // The service user may lack write access to qa/ when the directory was - // created as root; that is expected and silently skipped. - // Other errors (bad path, JSON serialization failure) are logged as warnings - // so they remain visible without crashing the bot. +/** + * tryWriteQaJson — write a single QA artifact file, applying QA_FS_SUPPRESS + * suppression semantics. Shared by ensureQaArtifacts so each artifact write + * is guarded independently: one serialization failure cannot block the rest. + */ +function tryWriteQaJson(filePath, payload, label) { try { - ensureQaDirs(); - fs.writeFileSync(qaRepoInfoFile, JSON.stringify({ - repoPath: '/var/www/html/Runewager', - branch: process.env.GIT_BRANCH || 'main', - systemdService: 'runewager.service', - entryFile: 'index.js', - telegramDefault: true, - }, null, 2)); - fs.writeFileSync(qaCapabilitiesFile, JSON.stringify(generateQaCapabilities(), null, 2)); - persistQaProviderState(); + fs.writeFileSync(filePath, JSON.stringify(payload, null, 2), { mode: 0o600 }); } catch (e) { - if (!QA_FS_SUPPRESS.has(e.code)) { - logEvent('warn', 'ensureQaArtifacts: unexpected error writing QA artifacts', { code: e.code, error: e.message }); + if (!QA_FS_SUPPRESS.has(e?.code)) { + logEvent('warn', `${label}: unexpected error`, { code: e?.code, error: e?.message ?? String(e), ...(process.env.LOG_LEVEL === 'debug' && { stack: e?.stack }) }); } // Permission errors (EACCES/EPERM/EROFS) are intentionally silent — // the bot runs fine without QA artifacts. } } +function ensureQaArtifacts() { + // QA artifacts are observability helpers — never fatal. + // Each write is guarded independently so one failure does not block others. + ensureQaDirs(); + tryWriteQaJson(qaRepoInfoFile, { + repoPath: '/var/www/html/Runewager', + branch: process.env.GIT_BRANCH || 'main', + systemdService: 'runewager.service', + entryFile: 'index.js', + telegramDefault: true, + }, 'ensureQaArtifacts: repo_info'); + tryWriteQaJson(qaCapabilitiesFile, generateQaCapabilities(), 'ensureQaArtifacts: capabilities'); + persistQaProviderState(); +} + setInterval(refreshQaProviderCooldowns, qaRuntime.providerStatus.resetIntervalMs).unref(); /** @@ -9270,7 +9285,7 @@ bot.action('sshv_editor_save', async (ctx) => { return; } try { - fs.writeFileSync(session.editorMode.filePath, draft, 'utf8'); + fs.writeFileSync(session.editorMode.filePath, draft, { encoding: 'utf8', mode: 0o600 }); adminLog('sshv_editor_save', { adminId: user.id, filePath: session.editorMode.filePath, size: draft.length }); session.buffer = `Saved ${session.editorMode.filePath}`; session.editorMode = null; diff --git a/prod-run.sh b/prod-run.sh index 23c268b..bb26872 100755 --- a/prod-run.sh +++ b/prod-run.sh @@ -385,13 +385,44 @@ if [[ -f "$DISK_PROTECT_SCRIPT" ]] && command -v crontab >/dev/null 2>&1; then fi # --------------------------------------------------------- -# 9c) Kill anything blocking the bot port before restart +# 9c) Aggressively clear port conflicts BEFORE restart +# Kills any process (including stray backend.js, old node instances) +# holding the bot port so systemd never races against a squatter. PORT_PRESTART="$(read_env_value PORT || echo 3000)" PORT_PRESTART="${PORT_PRESTART:-3000}" + +_kill_port_squatters() { + local port="$1" own_pid="${2:-}" + local pids pid + # Use lsof if available (most reliable), fall back to port_listener_pids + if command -v lsof >/dev/null 2>&1; then + pids="$(lsof -t -iTCP:"${port}" -sTCP:LISTEN 2>/dev/null | sort -u || true)" + else + pids="$(port_listener_pids "$port")" + fi + for pid in $pids; do + [[ -z "$pid" ]] && continue + [[ -n "$own_pid" && "$pid" == "$own_pid" ]] && continue + warn "Pre-start: killing PID $pid squatting port ${port}" + kill -TERM "$pid" 2>/dev/null || true + sleep 1 + kill -0 "$pid" 2>/dev/null && { kill -KILL "$pid" 2>/dev/null || true; } + done +} + if is_port_listening "$PORT_PRESTART"; then - say "Port $PORT_PRESTART in use — freeing before restart..." - free_port_if_conflicted "$PORT_PRESTART" "${PID:-}" || true + say "Port $PORT_PRESTART in use — aggressively clearing before restart..." + _kill_port_squatters "$PORT_PRESTART" "${PID:-}" sleep 1 + # Second pass — if still occupied, SIGKILL everything left + if is_port_listening "$PORT_PRESTART"; then + warn "Port $PORT_PRESTART still occupied — forcing SIGKILL..." + if command -v lsof >/dev/null 2>&1; then + lsof -t -iTCP:"${PORT_PRESTART}" -sTCP:LISTEN 2>/dev/null \ + | xargs -r kill -9 2>/dev/null || true + fi + sleep 1 + fi fi # --------------------------------------------------------- @@ -476,168 +507,54 @@ command -v systemctl >/dev/null 2>&1 && [[ -f "$SERVICE_FILE" ]] && systemctl re } # ========================================================= -# 11) GOD-MODE HEAL LOGIC + TELEGRAM REPORT +# 11) POST-START HEALTH CHECK + TELEGRAM REPORT (single pass) # ========================================================= -say "⚡ Running integrated God-Mode Heal..." - -# Load admin IDs and bot token -ADMIN_IDS=$(grep -E '^ADMIN_IDS=' .env | cut -d= -f2 | tr -d '"') -TELEGRAM_BOT_TOKEN=$(grep -E '^TELEGRAM_BOT_TOKEN=' .env | cut -d= -f2 | tr -d '"') - -PORT=$(grep -E '^PORT=' .env | cut -d= -f2 || true) -PORT=${PORT:-3000} - -BLOCKING_PID=$(lsof -ti :"$PORT" || true) -if [[ -n "$BLOCKING_PID" ]]; then - say "🚨 Port $PORT blocked by PID $BLOCKING_PID — killing..." - kill -9 "$BLOCKING_PID" || true - sleep 1 - say "♻️ Restarting ${APP_NAME} after freeing port..." - systemctl restart "${APP_NAME}" 2>/dev/null || nohup node "$PROJECT_DIR/index.js" >> "$MAIN_LOG" 2>> "$ERROR_LOG" < /dev/null & - sleep 2 -else - say "✅ Port $PORT free — continuing" -fi - -HEALTH_OK=false -if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then - HEALTH_OK=true - say "💓 Health endpoint OK" -else - say "❌ Health endpoint FAILED — capturing logs" - echo "---- Last 50 lines of $MAIN_LOG ----"; tail -n 50 "$MAIN_LOG" - echo "---- Last 50 lines of $ERROR_LOG ----"; tail -n 50 "$ERROR_LOG" -fi - -# Telegram admin notification -if [[ -n "$TELEGRAM_BOT_TOKEN" && -n "$ADMIN_IDS" ]]; then - for ADMIN_ID in ${ADMIN_IDS//,/ }; do - REPORT=$( (tail -n 50 "$MAIN_LOG"; tail -n 50 "$ERROR_LOG") | sed 's/"/\\"/g') - curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ - -d chat_id="$ADMIN_ID" \ - -d parse_mode="Markdown" \ - -d text="🏁 *Runewager Heal Report*\nHealth: $( [[ "$HEALTH_OK" == true ]] && echo '✅ OK' || echo '❌ FAILED')\nPort: $PORT\nPID: ${PID:-none}\n\n_Last 50 log lines:_\n\`\`\`${REPORT}\`\`\`" >/dev/null 2>&1 || true - done -fi - -SYSTEMD_ACTIVE="$(systemctl is-active "${APP_NAME}.service" 2>/dev/null || echo inactive)" -HEALTH_URL="$(resolve_health_url)" -HEALTH_PORT="${HEALTH_URL#*://127.0.0.1:}" -HEALTH_PORT="${HEALTH_PORT%%/*}" - -if wait_for_health "$HEALTH_URL" 20 2; then - HEALTH_STATUS="healthy" -else - HEALTH_STATUS="unhealthy" -fi - -if is_port_listening "$HEALTH_PORT"; then - PORT_STATUS="listening" -else - PORT_STATUS="not-listening" -fi - -SYSTEMD_ACTIVE="$(systemctl is-active "${APP_NAME}.service" 2>/dev/null || echo inactive)" -HEALTH_URL="$(resolve_health_url)" -HEALTH_PORT="${HEALTH_URL#*://127.0.0.1:}" -HEALTH_PORT="${HEALTH_PORT%%/*}" - -if wait_for_health "$HEALTH_URL" 20 2; then - HEALTH_STATUS="healthy" -else - HEALTH_STATUS="unhealthy" -fi +say "Running post-start health check..." -PRECHECK_HEALTH_URL="$(resolve_health_url)" -PRECHECK_PORT="${PRECHECK_HEALTH_URL#*://127.0.0.1:}" -PRECHECK_PORT="${PRECHECK_PORT%%/*}" -if is_port_listening "$PRECHECK_PORT"; then - free_port_if_conflicted "$PRECHECK_PORT" "$PID" || true -fi - -if command -v systemctl >/dev/null 2>&1 && [[ -f "$SERVICE_FILE" ]]; then - if systemctl restart "${APP_NAME}.service" 2>&1; then - sleep 3 - PID="$(get_bot_pid)" - else - warn "systemctl restart failed — falling back to kill+nohup" - [[ -n "$PID" ]] && { kill "$PID" 2>/dev/null || true; sleep 2; } - nohup node "$PROJECT_DIR/index.js" \ - >> "$MAIN_LOG" 2>> "$ERROR_LOG" < /dev/null & - disown || true - sleep 3 - PID="$(get_bot_pid)" - fi -else - PORT_STATUS="not-listening" -fi - -SYSTEMD_ACTIVE="$(systemctl is-active "${APP_NAME}.service" 2>/dev/null || echo inactive)" HEALTH_URL="$(resolve_health_url)" HEALTH_PORT="${HEALTH_URL#*://127.0.0.1:}" HEALTH_PORT="${HEALTH_PORT%%/*}" - -if wait_for_health "$HEALTH_URL" 20 2; then - HEALTH_STATUS="healthy" -else - HEALTH_STATUS="unhealthy" -fi - -if is_port_listening "$HEALTH_PORT"; then - PORT_STATUS="listening" -else - PORT_STATUS="not-listening" -fi - SYSTEMD_ACTIVE="$(systemctl is-active "${APP_NAME}.service" 2>/dev/null || echo inactive)" -HEALTH_URL="$(resolve_health_url)" -HEALTH_PORT="${HEALTH_URL#*://127.0.0.1:}" -HEALTH_PORT="${HEALTH_PORT%%/*}" +HEALTH_STATUS="unhealthy" +PORT_STATUS="not-listening" -if wait_for_health "$HEALTH_URL" 20 2; then +# Wait up to 30s for health (15 x 2s) — single pass +if wait_for_health "$HEALTH_URL" 15 2; then HEALTH_STATUS="healthy" else - HEALTH_STATUS="unhealthy" - - # Auto-recover from port conflicts, then retry health once. + # One auto-recovery: clear any squatter and restart once if is_port_listening "$HEALTH_PORT"; then - free_port_if_conflicted "$HEALTH_PORT" "$PID" || true + warn "Port $HEALTH_PORT still squatted after start — forcing clear + one restart" + _kill_port_squatters "$HEALTH_PORT" "${PID:-}" + sleep 1 if command -v systemctl >/dev/null 2>&1 && [[ -f "$SERVICE_FILE" ]]; then systemctl restart "${APP_NAME}.service" 2>/dev/null || true else - [[ -n "$PID" ]] && { kill "$PID" 2>/dev/null || true; sleep 2; } + [[ -n "${PID:-}" ]] && { kill "$PID" 2>/dev/null || true; sleep 2; } nohup node "$PROJECT_DIR/index.js" >> "$MAIN_LOG" 2>> "$ERROR_LOG" < /dev/null & disown || true fi sleep 3 PID="$(get_bot_pid)" - if wait_for_health "$HEALTH_URL" 10 2; then - HEALTH_STATUS="healthy" - fi fi + # Final health probe (10 x 2s) + wait_for_health "$HEALTH_URL" 10 2 && HEALTH_STATUS="healthy" fi -if is_port_listening "$HEALTH_PORT"; then - PORT_STATUS="listening" -else - PORT_STATUS="not-listening" -fi - -SYSTEMD_ACTIVE="$(systemctl is-active "${APP_NAME}.service" 2>/dev/null || echo inactive)" -HEALTH_URL="$(resolve_health_url)" -HEALTH_PORT="${HEALTH_URL#*://127.0.0.1:}" -HEALTH_PORT="${HEALTH_PORT%%/*}" - -if wait_for_health "$HEALTH_URL" 20 2; then - HEALTH_STATUS="healthy" -else - HEALTH_STATUS="unhealthy" -fi +is_port_listening "$HEALTH_PORT" && PORT_STATUS="listening" -if is_port_listening "$HEALTH_PORT"; then - PORT_STATUS="listening" -else - PORT_STATUS="not-listening" +# Telegram admin notification +_ADMIN_IDS="$(read_env_value ADMIN_IDS || true)" +_BOT_TOKEN="$(read_env_value TELEGRAM_BOT_TOKEN || true)" +if [[ -n "$_BOT_TOKEN" && -n "$_ADMIN_IDS" ]]; then + _REPORT="$(( tail -n 30 "$MAIN_LOG"; tail -n 20 "$ERROR_LOG" ) 2>/dev/null \ + | sed 's/"/\\"/g' | head -c 3500)" + for _AID in ${_ADMIN_IDS//,/ }; do + curl -s -X POST "https://api.telegram.org/bot${_BOT_TOKEN}/sendMessage" \ + -d chat_id="$_AID" \ + -d parse_mode="Markdown" \ + -d text="Deploy complete: Health $( [[ "$HEALTH_STATUS" == healthy ]] && echo OK || echo FAILED) Port: $HEALTH_PORT PID: ${PID:-none}" >/dev/null 2>&1 || true + done fi say "✔ Systemd service: ${SYSTEMD_ACTIVE}" diff --git a/runewager.service b/runewager.service index 9d53522..f03339b 100644 --- a/runewager.service +++ b/runewager.service @@ -51,28 +51,38 @@ StandardError=append:/var/www/html/Runewager/logs/runewager-error.log # (strict would make ALL paths read-only, requiring explicit ReadWritePaths # for every directory Node touches — too brittle for a Node.js app.) # -# ProtectHome=false — the service runs as root, whose home is /root. -# Node.js and npm use /root for caches and internal temp files; blocking -# /root causes subtle startup failures under strict sandboxing. +# ProtectHome=true — blocks blanket /root access; only the specific npm/ +# cache paths listed in ReadWritePaths below are permitted. This prevents +# the process from writing to the rest of /root while still allowing Node.js +# and npm to reach their cache directories. # # PrivateTmp=true — the process gets its own /tmp namespace. # # NoNewPrivileges=true — process and children cannot gain new privileges. # -# UMask=0022 — created files are world-readable (644) and -# directories are world-executable (755). Required so the log files written -# by root are readable by non-root log-tailing utilities. +# UMask=0077 — files default to mode 600 (owner rw only), directories +# to 700. All write calls in index.js pass an explicit mode where broader +# access is intentional: +# 0o640 — append-only log files (adminEventsLogFile, bonusAdminLogFile, +# writeQaLog daily logs) — readable by group log-tailing utilities +# 0o600 — sensitive data files (runtime-state.json via writeFileAtomic, +# qaProviderStateFile, QA artifact files, editor session drafts) +# Note: StandardOutput/StandardError log files (runewager.log, +# runewager-error.log) are opened by systemd, not the process; their +# permissions are set by systemd and are unaffected by this UMask. Grant +# log-reader access via group membership: usermod -aG root , or +# use a dedicated log group with setfacl on the logs/ directory. # ProtectSystem=full -ProtectHome=false +ProtectHome=true PrivateTmp=true NoNewPrivileges=true -UMask=0022 +UMask=0077 -# Explicit write grants under /var (belt-and-suspenders — /var is already -# writable under ProtectSystem=full, but listing these makes intent clear -# and ensures compatibility if ProtectSystem is ever tightened again). -ReadWritePaths=/var/www/html/Runewager/logs /var/www/html/Runewager/data /var/www/html/Runewager/qa +# Explicit write grants: /var paths for bot data, logs, and QA artifacts; +# /root/.npm and /root/.cache for Node.js/npm tooling caches (required with +# ProtectHome=true since /root itself is now restricted). +ReadWritePaths=/var/www/html/Runewager/logs /var/www/html/Runewager/data /var/www/html/Runewager/qa /root/.npm /root/.cache # File descriptor limit (for concurrent connections) LimitNOFILE=65536