Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 45 additions & 30 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -496,7 +496,7 @@ function appendBonusAdminLog(actorId, action, details = '') {
fs.mkdirSync(path.dirname(bonusAdminLogFile), { recursive: true });
const line = `[${new Date().toISOString()}] actor=${actorId || 'system'} action=${action}${details ? ` details=${details}` : ''}
`;
fs.appendFileSync(bonusAdminLogFile, line, 'utf8');
fs.appendFileSync(bonusAdminLogFile, line, { encoding: 'utf8', mode: 0o640 });
} catch (_) { /* ignore log write errors */ }
}
const backupDir = path.join(dataDir, 'backups');
Expand Down Expand Up @@ -4538,7 +4538,7 @@ function adminLog(event, payload) {
// Also append to persistent NDJSON log file (best-effort)
try {
ensureDataDir();
fs.appendFileSync(adminEventsLogFile, JSON.stringify(entry) + '\n');
fs.appendFileSync(adminEventsLogFile, JSON.stringify(entry) + '\n', { mode: 0o640 });
} catch (_) { /* non-fatal — in-memory log is the source of truth */ }
}

Expand Down Expand Up @@ -4800,7 +4800,7 @@ function writeFileAtomic(filePath, content) {
const dirPath = path.dirname(safeFilePath);
if (!fs.existsSync(dirPath)) fs.mkdirSync(dirPath, { recursive: true });
const tempFile = `${safeFilePath}.tmp.${process.pid}.${Date.now()}`;
fs.writeFileSync(tempFile, content);
fs.writeFileSync(tempFile, content, { mode: 0o600 });
fs.renameSync(tempFile, safeFilePath);
}

Expand Down Expand Up @@ -4829,16 +4829,25 @@ function saveJson(filePath, data) {
writeFileAtomic(filePath, JSON.stringify(data, null, 2));
}

// Expected error codes when the service user cannot write to qa/ (created as root).
const QA_FS_SUPPRESS = new Set(['EACCES', 'EPERM', 'EROFS', 'EEXIST']);
/**
* QA_FS_SUPPRESS — single authoritative set of errno codes that are silently
* swallowed across ALL QA filesystem operations (ensureQaDirs, ensureQaArtifacts,
* persistQaProviderState). Add codes here once and every helper inherits the
* suppression automatically.
*
* EEXIST is intentionally excluded: mkdirSync({ recursive: true }) never throws
* EEXIST for an existing directory, so seeing it here means a *file* sits where a
* directory is expected — a misconfiguration we want surfaced, not silenced.
*/
const QA_FS_SUPPRESS = new Set(['EACCES', 'EPERM', 'EROFS']);

function ensureQaDirs() {
for (const dir of [qaContextDir, qaStateDir, qaLogsDir]) {
try {
fs.mkdirSync(dir, { recursive: true });
} catch (e) {
if (!QA_FS_SUPPRESS.has(e.code)) {
logEvent('warn', 'ensureQaDirs: unexpected error creating QA directory', { dir, code: e.code, error: e.message });
if (!QA_FS_SUPPRESS.has(e?.code)) {
logEvent('warn', 'ensureQaDirs: unexpected error creating QA directory', { dir, code: e?.code, error: e?.message ?? String(e), ...(process.env.LOG_LEVEL === 'debug' && { stack: e?.stack }) });
}
}
}
Expand All @@ -4854,7 +4863,7 @@ function getQaLogDirForToday() {
function writeQaLog(fileName, payload) {
try {
const line = `${JSON.stringify({ ts: new Date().toISOString(), ...payload })}\n`;
fs.appendFileSync(path.join(getQaLogDirForToday(), fileName), line, 'utf8');
fs.appendFileSync(path.join(getQaLogDirForToday(), fileName), line, { encoding: 'utf8', mode: 0o640 });
} catch (_) { /* ignore QA log write errors */ }
}

Expand Down Expand Up @@ -4893,14 +4902,14 @@ function generateQaCapabilities() {
function persistQaProviderState() {
try {
ensureQaDirs();
fs.writeFileSync(qaProviderStateFile, JSON.stringify(qaRuntime.providerStatus, null, 2));
fs.writeFileSync(qaProviderStateFile, JSON.stringify(qaRuntime.providerStatus, null, 2), { mode: 0o600 });
} catch (e) {
if (QA_FS_SUPPRESS.has(e.code)) {
if (QA_FS_SUPPRESS.has(e?.code)) {
// Expected: service user lacks write permission on qa/ (created as root). Non-fatal.
return;
}
// Unexpected (e.g. JSON serialization failure, bad path) — log so it stays visible.
logEvent('warn', 'persistQaProviderState: unexpected error', { code: e.code, error: e.message });
logEvent('warn', 'persistQaProviderState: unexpected error', { code: e?.code, error: e?.message ?? String(e), ...(process.env.LOG_LEVEL === 'debug' && { stack: e?.stack }) });
}
}

Expand All @@ -4916,32 +4925,38 @@ function refreshQaProviderCooldowns() {
persistQaProviderState();
}

function ensureQaArtifacts() {
// QA artifacts are observability helpers — never fatal.
// The service user may lack write access to qa/ when the directory was
// created as root; that is expected and silently skipped.
// Other errors (bad path, JSON serialization failure) are logged as warnings
// so they remain visible without crashing the bot.
/**
* tryWriteQaJson — write a single QA artifact file, applying QA_FS_SUPPRESS
* suppression semantics. Shared by ensureQaArtifacts so each artifact write
* is guarded independently: one serialization failure cannot block the rest.
*/
function tryWriteQaJson(filePath, payload, label) {
try {
ensureQaDirs();
fs.writeFileSync(qaRepoInfoFile, JSON.stringify({
repoPath: '/var/www/html/Runewager',
branch: process.env.GIT_BRANCH || 'main',
systemdService: 'runewager.service',
entryFile: 'index.js',
telegramDefault: true,
}, null, 2));
fs.writeFileSync(qaCapabilitiesFile, JSON.stringify(generateQaCapabilities(), null, 2));
persistQaProviderState();
fs.writeFileSync(filePath, JSON.stringify(payload, null, 2), { mode: 0o600 });
} catch (e) {
if (!QA_FS_SUPPRESS.has(e.code)) {
logEvent('warn', 'ensureQaArtifacts: unexpected error writing QA artifacts', { code: e.code, error: e.message });
if (!QA_FS_SUPPRESS.has(e?.code)) {
logEvent('warn', `${label}: unexpected error`, { code: e?.code, error: e?.message ?? String(e), ...(process.env.LOG_LEVEL === 'debug' && { stack: e?.stack }) });
}
// Permission errors (EACCES/EPERM/EROFS) are intentionally silent —
// the bot runs fine without QA artifacts.
}
}

function ensureQaArtifacts() {
// QA artifacts are observability helpers — never fatal.
// Each write is guarded independently so one failure does not block others.
ensureQaDirs();
tryWriteQaJson(qaRepoInfoFile, {
repoPath: '/var/www/html/Runewager',
branch: process.env.GIT_BRANCH || 'main',
systemdService: 'runewager.service',
entryFile: 'index.js',
telegramDefault: true,
}, 'ensureQaArtifacts: repo_info');
tryWriteQaJson(qaCapabilitiesFile, generateQaCapabilities(), 'ensureQaArtifacts: capabilities');
persistQaProviderState();
}

setInterval(refreshQaProviderCooldowns, qaRuntime.providerStatus.resetIntervalMs).unref();

/**
Expand Down Expand Up @@ -9270,7 +9285,7 @@ bot.action('sshv_editor_save', async (ctx) => {
return;
}
try {
fs.writeFileSync(session.editorMode.filePath, draft, 'utf8');
fs.writeFileSync(session.editorMode.filePath, draft, { encoding: 'utf8', mode: 0o600 });
adminLog('sshv_editor_save', { adminId: user.id, filePath: session.editorMode.filePath, size: draft.length });
session.buffer = `Saved ${session.editorMode.filePath}`;
session.editorMode = null;
Expand Down
203 changes: 60 additions & 143 deletions prod-run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -385,13 +385,44 @@ if [[ -f "$DISK_PROTECT_SCRIPT" ]] && command -v crontab >/dev/null 2>&1; then
fi

# ---------------------------------------------------------
# 9c) Kill anything blocking the bot port before restart
# 9c) Aggressively clear port conflicts BEFORE restart
# Kills any process (including stray backend.js, old node instances)
# holding the bot port so systemd never races against a squatter.
PORT_PRESTART="$(read_env_value PORT || echo 3000)"
PORT_PRESTART="${PORT_PRESTART:-3000}"

_kill_port_squatters() {
local port="$1" own_pid="${2:-}"
local pids pid
# Use lsof if available (most reliable), fall back to port_listener_pids
if command -v lsof >/dev/null 2>&1; then
pids="$(lsof -t -iTCP:"${port}" -sTCP:LISTEN 2>/dev/null | sort -u || true)"
else
pids="$(port_listener_pids "$port")"
fi
for pid in $pids; do
[[ -z "$pid" ]] && continue
[[ -n "$own_pid" && "$pid" == "$own_pid" ]] && continue
warn "Pre-start: killing PID $pid squatting port ${port}"
kill -TERM "$pid" 2>/dev/null || true
sleep 1
kill -0 "$pid" 2>/dev/null && { kill -KILL "$pid" 2>/dev/null || true; }
done
}

if is_port_listening "$PORT_PRESTART"; then
say "Port $PORT_PRESTART in use — freeing before restart..."
free_port_if_conflicted "$PORT_PRESTART" "${PID:-}" || true
say "Port $PORT_PRESTART in use — aggressively clearing before restart..."
_kill_port_squatters "$PORT_PRESTART" "${PID:-}"
sleep 1
# Second pass — if still occupied, SIGKILL everything left
if is_port_listening "$PORT_PRESTART"; then
warn "Port $PORT_PRESTART still occupied — forcing SIGKILL..."
if command -v lsof >/dev/null 2>&1; then
lsof -t -iTCP:"${PORT_PRESTART}" -sTCP:LISTEN 2>/dev/null \
| xargs -r kill -9 2>/dev/null || true
fi
sleep 1
fi
fi

# ---------------------------------------------------------
Expand Down Expand Up @@ -476,168 +507,54 @@ command -v systemctl >/dev/null 2>&1 && [[ -f "$SERVICE_FILE" ]] && systemctl re
}

# =========================================================
# 11) GOD-MODE HEAL LOGIC + TELEGRAM REPORT
# 11) POST-START HEALTH CHECK + TELEGRAM REPORT (single pass)
# =========================================================
say "⚡ Running integrated God-Mode Heal..."

# Load admin IDs and bot token
ADMIN_IDS=$(grep -E '^ADMIN_IDS=' .env | cut -d= -f2 | tr -d '"')
TELEGRAM_BOT_TOKEN=$(grep -E '^TELEGRAM_BOT_TOKEN=' .env | cut -d= -f2 | tr -d '"')

PORT=$(grep -E '^PORT=' .env | cut -d= -f2 || true)
PORT=${PORT:-3000}

BLOCKING_PID=$(lsof -ti :"$PORT" || true)
if [[ -n "$BLOCKING_PID" ]]; then
say "🚨 Port $PORT blocked by PID $BLOCKING_PID — killing..."
kill -9 "$BLOCKING_PID" || true
sleep 1
say "♻️ Restarting ${APP_NAME} after freeing port..."
systemctl restart "${APP_NAME}" 2>/dev/null || nohup node "$PROJECT_DIR/index.js" >> "$MAIN_LOG" 2>> "$ERROR_LOG" < /dev/null &
sleep 2
else
say "✅ Port $PORT free — continuing"
fi

HEALTH_OK=false
if curl -sf "http://127.0.0.1:$PORT/health" >/dev/null 2>&1; then
HEALTH_OK=true
say "💓 Health endpoint OK"
else
say "❌ Health endpoint FAILED — capturing logs"
echo "---- Last 50 lines of $MAIN_LOG ----"; tail -n 50 "$MAIN_LOG"
echo "---- Last 50 lines of $ERROR_LOG ----"; tail -n 50 "$ERROR_LOG"
fi

# Telegram admin notification
if [[ -n "$TELEGRAM_BOT_TOKEN" && -n "$ADMIN_IDS" ]]; then
for ADMIN_ID in ${ADMIN_IDS//,/ }; do
REPORT=$( (tail -n 50 "$MAIN_LOG"; tail -n 50 "$ERROR_LOG") | sed 's/"/\\"/g')
curl -s -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
-d chat_id="$ADMIN_ID" \
-d parse_mode="Markdown" \
-d text="🏁 *Runewager Heal Report*\nHealth: $( [[ "$HEALTH_OK" == true ]] && echo '✅ OK' || echo '❌ FAILED')\nPort: $PORT\nPID: ${PID:-none}\n\n_Last 50 log lines:_\n\`\`\`${REPORT}\`\`\`" >/dev/null 2>&1 || true
done
fi

SYSTEMD_ACTIVE="$(systemctl is-active "${APP_NAME}.service" 2>/dev/null || echo inactive)"
HEALTH_URL="$(resolve_health_url)"
HEALTH_PORT="${HEALTH_URL#*://127.0.0.1:}"
HEALTH_PORT="${HEALTH_PORT%%/*}"

if wait_for_health "$HEALTH_URL" 20 2; then
HEALTH_STATUS="healthy"
else
HEALTH_STATUS="unhealthy"
fi

if is_port_listening "$HEALTH_PORT"; then
PORT_STATUS="listening"
else
PORT_STATUS="not-listening"
fi

SYSTEMD_ACTIVE="$(systemctl is-active "${APP_NAME}.service" 2>/dev/null || echo inactive)"
HEALTH_URL="$(resolve_health_url)"
HEALTH_PORT="${HEALTH_URL#*://127.0.0.1:}"
HEALTH_PORT="${HEALTH_PORT%%/*}"

if wait_for_health "$HEALTH_URL" 20 2; then
HEALTH_STATUS="healthy"
else
HEALTH_STATUS="unhealthy"
fi
say "Running post-start health check..."

PRECHECK_HEALTH_URL="$(resolve_health_url)"
PRECHECK_PORT="${PRECHECK_HEALTH_URL#*://127.0.0.1:}"
PRECHECK_PORT="${PRECHECK_PORT%%/*}"
if is_port_listening "$PRECHECK_PORT"; then
free_port_if_conflicted "$PRECHECK_PORT" "$PID" || true
fi

if command -v systemctl >/dev/null 2>&1 && [[ -f "$SERVICE_FILE" ]]; then
if systemctl restart "${APP_NAME}.service" 2>&1; then
sleep 3
PID="$(get_bot_pid)"
else
warn "systemctl restart failed — falling back to kill+nohup"
[[ -n "$PID" ]] && { kill "$PID" 2>/dev/null || true; sleep 2; }
nohup node "$PROJECT_DIR/index.js" \
>> "$MAIN_LOG" 2>> "$ERROR_LOG" < /dev/null &
disown || true
sleep 3
PID="$(get_bot_pid)"
fi
else
PORT_STATUS="not-listening"
fi

SYSTEMD_ACTIVE="$(systemctl is-active "${APP_NAME}.service" 2>/dev/null || echo inactive)"
HEALTH_URL="$(resolve_health_url)"
HEALTH_PORT="${HEALTH_URL#*://127.0.0.1:}"
HEALTH_PORT="${HEALTH_PORT%%/*}"

if wait_for_health "$HEALTH_URL" 20 2; then
HEALTH_STATUS="healthy"
else
HEALTH_STATUS="unhealthy"
fi

if is_port_listening "$HEALTH_PORT"; then
PORT_STATUS="listening"
else
PORT_STATUS="not-listening"
fi

SYSTEMD_ACTIVE="$(systemctl is-active "${APP_NAME}.service" 2>/dev/null || echo inactive)"
HEALTH_URL="$(resolve_health_url)"
HEALTH_PORT="${HEALTH_URL#*://127.0.0.1:}"
HEALTH_PORT="${HEALTH_PORT%%/*}"
HEALTH_STATUS="unhealthy"
PORT_STATUS="not-listening"

if wait_for_health "$HEALTH_URL" 20 2; then
# Wait up to 30s for health (15 x 2s) — single pass
if wait_for_health "$HEALTH_URL" 15 2; then
HEALTH_STATUS="healthy"
else
HEALTH_STATUS="unhealthy"

# Auto-recover from port conflicts, then retry health once.
# One auto-recovery: clear any squatter and restart once
if is_port_listening "$HEALTH_PORT"; then
free_port_if_conflicted "$HEALTH_PORT" "$PID" || true
warn "Port $HEALTH_PORT still squatted after start — forcing clear + one restart"
_kill_port_squatters "$HEALTH_PORT" "${PID:-}"
sleep 1
if command -v systemctl >/dev/null 2>&1 && [[ -f "$SERVICE_FILE" ]]; then
systemctl restart "${APP_NAME}.service" 2>/dev/null || true
Comment on lines 523 to 531

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

issue (bug_risk): SYSTEMD_ACTIVE can become stale after the auto-recovery restart branch.

Because SYSTEMD_ACTIVE is set only once and reused in the final summary, the reported status may not match the actual state after systemctl restart runs in the auto-recovery path. Please either move the systemctl is-active call to just before the final summary, or recompute SYSTEMD_ACTIVE after the restart so the final line reflects the post-heal state.

else
[[ -n "$PID" ]] && { kill "$PID" 2>/dev/null || true; sleep 2; }
[[ -n "${PID:-}" ]] && { kill "$PID" 2>/dev/null || true; sleep 2; }
nohup node "$PROJECT_DIR/index.js" >> "$MAIN_LOG" 2>> "$ERROR_LOG" < /dev/null &
disown || true
fi
sleep 3
PID="$(get_bot_pid)"
if wait_for_health "$HEALTH_URL" 10 2; then
HEALTH_STATUS="healthy"
fi
fi
# Final health probe (10 x 2s)
wait_for_health "$HEALTH_URL" 10 2 && HEALTH_STATUS="healthy"
fi

if is_port_listening "$HEALTH_PORT"; then
PORT_STATUS="listening"
else
PORT_STATUS="not-listening"
fi

SYSTEMD_ACTIVE="$(systemctl is-active "${APP_NAME}.service" 2>/dev/null || echo inactive)"
HEALTH_URL="$(resolve_health_url)"
HEALTH_PORT="${HEALTH_URL#*://127.0.0.1:}"
HEALTH_PORT="${HEALTH_PORT%%/*}"

if wait_for_health "$HEALTH_URL" 20 2; then
HEALTH_STATUS="healthy"
else
HEALTH_STATUS="unhealthy"
fi
is_port_listening "$HEALTH_PORT" && PORT_STATUS="listening"

if is_port_listening "$HEALTH_PORT"; then
PORT_STATUS="listening"
else
PORT_STATUS="not-listening"
# Telegram admin notification
_ADMIN_IDS="$(read_env_value ADMIN_IDS || true)"
_BOT_TOKEN="$(read_env_value TELEGRAM_BOT_TOKEN || true)"
if [[ -n "$_BOT_TOKEN" && -n "$_ADMIN_IDS" ]]; then
_REPORT="$(( tail -n 30 "$MAIN_LOG"; tail -n 20 "$ERROR_LOG" ) 2>/dev/null \
| sed 's/"/\\"/g' | head -c 3500)"
Comment on lines +550 to +551

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

issue: _REPORT is computed but never used in the Telegram notification.

_REPORT is built from the log tails but never used in the curl request. Please either remove this construction or wire it into the Telegram payload (or a separate message) if the intent is to send recent logs to admins.

for _AID in ${_ADMIN_IDS//,/ }; do
curl -s -X POST "https://api.telegram.org/bot${_BOT_TOKEN}/sendMessage" \
-d chat_id="$_AID" \
-d parse_mode="Markdown" \
-d text="Deploy complete: Health $( [[ "$HEALTH_STATUS" == healthy ]] && echo OK || echo FAILED) Port: $HEALTH_PORT PID: ${PID:-none}" >/dev/null 2>&1 || true
Comment on lines +550 to +556

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggestion: The Telegram deploy notification block attempts to build a log snippet using arithmetic expansion ($(( ... ))) instead of command substitution ($( ... )), so the tail commands are never executed as commands, and the resulting _REPORT content is meaningless and never included in the message body; switch to proper command substitution and append _REPORT to the text payload so admins actually receive the intended log excerpt. [logic error]

Severity Level: Critical 🚨
- ❌ prod-run.sh exits when Telegram alerts are configured.
- ⚠️ Telegram deploy messages lack intended log context. 
Suggested change
_REPORT="$(( tail -n 30 "$MAIN_LOG"; tail -n 20 "$ERROR_LOG" ) 2>/dev/null \
| sed 's/"/\\"/g' | head -c 3500)"
for _AID in ${_ADMIN_IDS//,/ }; do
curl -s -X POST "https://api.telegram.org/bot${_BOT_TOKEN}/sendMessage" \
-d chat_id="$_AID" \
-d parse_mode="Markdown" \
-d text="Deploy complete: Health $( [[ "$HEALTH_STATUS" == healthy ]] && echo OK || echo FAILED) Port: $HEALTH_PORT PID: ${PID:-none}" >/dev/null 2>&1 || true
_REPORT="$( ( tail -n 30 "$MAIN_LOG"; tail -n 20 "$ERROR_LOG" ) 2>/dev/null \
| sed 's/"/\\"/g' | head -c 3500)"
for _AID in ${_ADMIN_IDS//,/ }; do
curl -s -X POST "https://api.telegram.org/bot${_BOT_TOKEN}/sendMessage" \
-d chat_id="$_AID" \
-d parse_mode="Markdown" \
-d text="Deploy complete: Health $( [[ "$HEALTH_STATUS" == healthy ]] && echo OK || echo FAILED) Port: $HEALTH_PORT PID: ${PID:-none}\n\n${_REPORT}" >/dev/null 2>&1 || true
done
Steps of Reproduction ✅
1. Ensure Telegram notifications are configured by setting `TELEGRAM_BOT_TOKEN` and
`ADMIN_IDS` in `.env`, which are read by `read_env_value` at `prod-run.sh:547-548`.

2. Deploy using the documented entrypoint, e.g. run `npm run prod` (which invokes
`./prod-run.sh` per `package.json:14`) or execute `bash prod-run.sh` from the project
root.

3. After the bot is restarted and the health check passes/finalizes (logic at
`prod-run.sh:514-542`), execution reaches the Telegram admin notification block at
`prod-run.sh:546-557`.

4. With `set -euo pipefail` enabled at `prod-run.sh:9`, the line `_REPORT="$(( tail -n 30
"$MAIN_LOG"; tail -n 20 "$ERROR_LOG" ) 2>/dev/null \` at `prod-run.sh:550-551` is parsed
as arithmetic expansion: `tail`/`sed` tokens are not valid arithmetic, causing a Bash
"syntax error in expression" and immediate script exit before any `tail` commands run or
any `curl` notification is sent; even if `set -e` were relaxed, `_REPORT` would not
contain the intended log snippet and is never interpolated into the `text` field at
`prod-run.sh:556`, so admins would receive at most a bare status line without logs.
Prompt for AI Agent 🤖
This is a comment left during a code review.

**Path:** prod-run.sh
**Line:** 550:556
**Comment:**
	*Logic Error: The Telegram deploy notification block attempts to build a log snippet using arithmetic expansion (`$(( ... ))`) instead of command substitution (`$( ... )`), so the `tail` commands are never executed as commands, and the resulting `_REPORT` content is meaningless and never included in the message body; switch to proper command substitution and append `_REPORT` to the `text` payload so admins actually receive the intended log excerpt.

Validate the correctness of the flagged issue. If correct, How can I resolve this? If you propose a fix, implement it and please make it concise.
👍 | 👎

done
fi

say "✔ Systemd service: ${SYSTEMD_ACTIVE}"
Expand Down
Loading