diff --git a/keeper/src/index.js b/keeper/src/index.js index 70ba8d4..a05349f 100644 --- a/keeper/src/index.js +++ b/keeper/src/index.js @@ -8,9 +8,20 @@ // accepting a price from here. The worst a hostile keeper can do is withhold // ticks, which is why ticking is open to anyone. +import { createServer } from "node:http"; import { createPublicClient, createWalletClient, http, parseAbi, parseEventLogs, formatEther } from "viem"; import { privateKeyToAccount } from "viem/accounts"; -import { handleFetchResultResponse, determineRelayAction } from "./lib.js"; +import { + handleFetchResultResponse, + determineRelayAction, + calculateBackoff, + incrementFailure, + resetFailure, + isExceeded, + evictExpiredPending, + isRpcError, + handleHealthRequest, +} from "./lib.js"; const RPC_URL = process.env.RPC_URL ?? "https://coston2-api.flare.network/ext/C/rpc"; const WRAITH_ADDRESS = required("WRAITH_ADDRESS"); @@ -20,6 +31,12 @@ const POLL_INTERVAL_MS = Number(process.env.POLL_INTERVAL_MS ?? 15_000); const INSTRUCTION_FEE_WEI = BigInt(process.env.INSTRUCTION_FEE_WEI ?? "0"); const SUBMISSION_TAG = process.env.SUBMISSION_TAG ?? "submit"; +// Configurable environment variables for hardening +const BACKOFF_MAX_MS = Number(process.env.BACKOFF_MAX_MS ?? 300_000); +const ORDER_MAX_RETRIES = Number(process.env.ORDER_MAX_RETRIES ?? 5); +const PENDING_TTL_MS = Number(process.env.PENDING_TTL_MS ?? 3_600_000); +const HEALTH_PORT = Number(process.env.HEALTH_PORT ?? 8080); + const coston2 = { id: 114, name: "Coston2", @@ -39,9 +56,16 @@ const account = privateKeyToAccount(PRIVATE_KEY); const publicClient = createPublicClient({ chain: coston2, transport: http(RPC_URL) }); const walletClient = createWalletClient({ account, chain: coston2, transport: http(RPC_URL) }); -/** instructionId -> orderId, for instructions whose result has not arrived yet. */ +/** instructionId -> { orderId, addedAt } */ const pending = new Map(); +/** orderId -> consecutive failures count */ +const orderFailures = new Map(); + +// Variables for health monitoring +let lastSuccessfulLoopTime = null; +let keeperBalanceString = "0 C2FLR"; + function required(name) { const value = process.env[name]; if (!value) { @@ -66,12 +90,27 @@ async function tickOrders() { const count = await publicClient.readContract({ address: WRAITH_ADDRESS, abi, functionName: "orderCount" }); for (let orderId = 0n; orderId < count; orderId++) { - const tickable = await publicClient.readContract({ - address: WRAITH_ADDRESS, - abi, - functionName: "canTick", - args: [orderId], - }); + if (isExceeded(orderFailures, orderId, ORDER_MAX_RETRIES)) { + continue; + } + + let tickable = false; + try { + tickable = await publicClient.readContract({ + address: WRAITH_ADDRESS, + abi, + functionName: "canTick", + args: [orderId], + }); + } catch (error) { + if (isRpcError(error)) { + throw error; + } + console.error(`canTick failed for order ${orderId}: ${error.message}`); + incrementFailure(orderFailures, orderId); + continue; + } + if (!tickable) continue; try { @@ -84,25 +123,35 @@ async function tickOrders() { }); const receipt = await publicClient.waitForTransactionReceipt({ hash }); + // Reset failure count upon a successful tick + resetFailure(orderFailures, orderId); + // OrderTicked carries the instruction id the proxy will key the result by. const events = parseEventLogs({ abi, logs: receipt.logs, eventName: "OrderTicked" }); for (const event of events) { - pending.set(event.args.instructionId, orderId); + pending.set(event.args.instructionId, { orderId, addedAt: Date.now() }); console.log(`ticked order ${orderId} -> instruction ${event.args.instructionId}`); } } catch (error) { - // One bad order must not stall every other order. + if (isRpcError(error)) { + throw error; + } console.error(`tick failed for order ${orderId}: ${error.shortMessage ?? error.message}`); + incrementFailure(orderFailures, orderId); } } } async function relayResults() { - for (const [instructionId, orderId] of [...pending]) { + for (const [instructionId, entry] of [...pending]) { + const { orderId } = entry; let result; try { result = await fetchResult(instructionId); } catch (error) { + if (isRpcError(error)) { + throw error; + } console.error(`polling ${instructionId}: ${error.message}`); continue; } @@ -112,6 +161,7 @@ async function relayResults() { if (result.status !== 1) { console.error(`order ${orderId}: TEE reported failure (status ${result.status})`); + incrementFailure(orderFailures, orderId); continue; } @@ -129,25 +179,68 @@ async function relayResults() { }); await publicClient.waitForTransactionReceipt({ hash }); console.log(`order ${orderId} executed in ${hash}`); + resetFailure(orderFailures, orderId); } catch (error) { + if (isRpcError(error)) { + throw error; + } console.error(`execute failed for order ${orderId}: ${error.shortMessage ?? error.message}`); + incrementFailure(orderFailures, orderId); } } } +function getHealthData() { + return { + lastSuccessfulLoopTime, + pendingCount: pending.size, + keeperBalance: keeperBalanceString, + }; +} + async function main() { - const balance = await publicClient.getBalance({ address: account.address }); - console.log(`keeper ${account.address} (${formatEther(balance)} C2FLR)`); - console.log(`watching ${WRAITH_ADDRESS} via ${EXT_PROXY_URL}, every ${POLL_INTERVAL_MS}ms`); + // Start health HTTP server + const server = createServer((req, res) => { + handleHealthRequest(req, res, getHealthData); + }); + server.listen(HEALTH_PORT, "0.0.0.0", () => { + console.log(`Health server listening on port ${HEALTH_PORT}`); + }); + + console.log(`keeper ${account.address} watching ${WRAITH_ADDRESS} via ${EXT_PROXY_URL}`); + console.log(`POLL_INTERVAL_MS=${POLL_INTERVAL_MS}, PENDING_TTL_MS=${PENDING_TTL_MS}, ORDER_MAX_RETRIES=${ORDER_MAX_RETRIES}`); + + let consecutiveRpcFailures = 0; for (;;) { + let delayMs = POLL_INTERVAL_MS; + try { + const balance = await publicClient.getBalance({ address: account.address }); + keeperBalanceString = `${formatEther(balance)} C2FLR`; + await tickOrders(); await relayResults(); + + const evictedCount = evictExpiredPending(pending, PENDING_TTL_MS); + if (evictedCount > 0) { + console.log(`evicted ${evictedCount} expired pending instructions from memory`); + } + + // Loop completed successfully + lastSuccessfulLoopTime = new Date().toISOString(); + consecutiveRpcFailures = 0; } catch (error) { - console.error(`loop error: ${error.message}`); + if (isRpcError(error)) { + consecutiveRpcFailures++; + delayMs = calculateBackoff(consecutiveRpcFailures, POLL_INTERVAL_MS, BACKOFF_MAX_MS); + console.error(`RPC failure (consecutive count: ${consecutiveRpcFailures}). backing off for ${delayMs}ms: ${error.message}`); + } else { + console.error(`unexpected loop error: ${error.message}`); + } } - await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + + await new Promise((resolve) => setTimeout(resolve, delayMs)); } } diff --git a/keeper/src/lib.js b/keeper/src/lib.js index deaa49b..8781817 100644 --- a/keeper/src/lib.js +++ b/keeper/src/lib.js @@ -38,3 +38,144 @@ export function determineRelayAction(result, defaultSubmissionTag = "submit") { signature: result.signature, }; } + +/** + * Calculates exponential backoff with full jitter. + * + * @param {number} consecutiveFailures + * @param {number} baseDelay + * @param {number} maxDelay + * @param {function} [random=Math.random] - Optional random number generator for testing. + * @returns {number} + */ +export function calculateBackoff(consecutiveFailures, baseDelay, maxDelay, random = Math.random) { + if (consecutiveFailures <= 0) { + return baseDelay; + } + const temp = Math.min(maxDelay, baseDelay * Math.pow(2, consecutiveFailures)); + return Math.floor(random() * temp); +} + +/** + * Increments the failure count for a given orderId. + * + * @param {Map} failuresMap + * @param {bigint} orderId + * @returns {number} The new failure count. + */ +export function incrementFailure(failuresMap, orderId) { + const count = (failuresMap.get(orderId) ?? 0) + 1; + failuresMap.set(orderId, count); + return count; +} + +/** + * Resets the failure count for a given orderId. + * + * @param {Map} failuresMap + * @param {bigint} orderId + */ +export function resetFailure(failuresMap, orderId) { + failuresMap.delete(orderId); +} + +/** + * Checks if the failure count for a given orderId has exceeded the cap. + * + * @param {Map} failuresMap + * @param {bigint} orderId + * @param {number} maxRetries + * @returns {boolean} + */ +export function isExceeded(failuresMap, orderId, maxRetries) { + const count = failuresMap.get(orderId) ?? 0; + return count >= maxRetries; +} + +/** + * Evicts pending instructions older than the TTL. + * + * @param {Map} pendingMap + * @param {number} ttlMs + * @param {number} now + * @returns {number} The number of evicted instructions. + */ +export function evictExpiredPending(pendingMap, ttlMs, now = Date.now()) { + let count = 0; + for (const [instructionId, entry] of pendingMap.entries()) { + if (now - entry.addedAt > ttlMs) { + pendingMap.delete(instructionId); + count++; + } + } + return count; +} + +/** + * Determines whether an error is a Flare RPC/network failure. + * We want to distinguish transient RPC issues (which should trigger backoff) + * from permanent EVM execution failures/reverts (which should not trigger loop backoff). + * + * @param {any} error + * @returns {boolean} + */ +export function isRpcError(error) { + if (!error) return false; + + const msg = (error.message || "").toLowerCase(); + const shortMsg = (error.shortMessage || "").toLowerCase(); + const name = (error.name || "").toLowerCase(); + + // If the error or its name contains revert, it's an execution revert, not an RPC failure. + if (msg.includes("revert") || shortMsg.includes("revert") || name.includes("revert")) { + return false; + } + + // Common network, RPC, connection, or timeout indicators + if ( + msg.includes("fetch") || + msg.includes("network") || + msg.includes("timeout") || + msg.includes("request") || + msg.includes("socket") || + msg.includes("eaddrnotavail") || + msg.includes("econnrefused") || + msg.includes("rate limit") || + msg.includes("status 429") || + msg.includes("status 503") || + msg.includes("status 502") || + msg.includes("status 504") || + msg.includes("rpc") || + msg.includes("connection") || + name.includes("httprequesterror") || + name.includes("timeouterror") || + name.includes("rpcrequesterror") + ) { + return true; + } + + // Recurse into cause if available + if (error.cause) { + return isRpcError(error.cause); + } + + return false; +} + +/** + * Handles HTTP requests for the health endpoint. + * + * @param {any} req + * @param {any} res + * @param {function} getHealthData + */ +export function handleHealthRequest(req, res, getHealthData) { + const url = new URL(req.url, "http://localhost"); + if (req.method === "GET" && url.pathname === "/health") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify(getHealthData())); + } else { + res.writeHead(404, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: "Not Found" })); + } +} diff --git a/keeper/test/lib.test.js b/keeper/test/lib.test.js index c625cca..0b35096 100644 --- a/keeper/test/lib.test.js +++ b/keeper/test/lib.test.js @@ -1,6 +1,16 @@ import test from "node:test"; import assert from "node:assert"; -import { handleFetchResultResponse, determineRelayAction } from "../src/lib.js"; +import { + handleFetchResultResponse, + determineRelayAction, + calculateBackoff, + incrementFailure, + resetFailure, + isExceeded, + evictExpiredPending, + isRpcError, + handleHealthRequest, +} from "../src/lib.js"; test("handleFetchResultResponse - proxy 404 -> null", async () => { const response = new Response(null, { status: 404 }); @@ -75,3 +85,142 @@ test("determineRelayAction - status 1 with no data or 0x -> skipped (null)", () assert.strictEqual(determineRelayAction({ status: 1, data: "0x" }), null); assert.strictEqual(determineRelayAction(null), null); }); + +test("calculateBackoff - calculates correctly with jitter and bounds", () => { + // consecutiveFailures = 0 should return base delay + assert.strictEqual(calculateBackoff(0, 1000, 15000), 1000); + + // Mock random generator to always return 0.5 (halfway jitter) + const mockRandom = () => 0.5; + + // consecutiveFailures = 1: base = 1000, 2^1 = 2 -> temp = 2000, half is 1000 + assert.strictEqual(calculateBackoff(1, 1000, 15000, mockRandom), 1000); + + // consecutiveFailures = 4: base = 1000, 2^4 = 16 -> temp = min(15000, 16000) = 15000, half is 7500 + assert.strictEqual(calculateBackoff(4, 1000, 15000, mockRandom), 7500); +}); + +test("incrementFailure, resetFailure, and isExceeded", () => { + const failures = new Map(); + const orderId = 12n; + + assert.strictEqual(isExceeded(failures, orderId, 3), false); + + assert.strictEqual(incrementFailure(failures, orderId), 1); + assert.strictEqual(isExceeded(failures, orderId, 3), false); + + assert.strictEqual(incrementFailure(failures, orderId), 2); + assert.strictEqual(isExceeded(failures, orderId, 3), false); + + assert.strictEqual(incrementFailure(failures, orderId), 3); + assert.strictEqual(isExceeded(failures, orderId, 3), true); + + resetFailure(failures, orderId); + assert.strictEqual(failures.has(orderId), false); + assert.strictEqual(isExceeded(failures, orderId, 3), false); +}); + +test("evictExpiredPending - evicts only expired items", () => { + const pending = new Map([ + ["inst-1", { orderId: 1n, addedAt: 1000 }], + ["inst-2", { orderId: 2n, addedAt: 2000 }], + ["inst-3", { orderId: 3n, addedAt: 3000 }], + ]); + + const now = 3500; + const ttlMs = 1000; // items older than 2500 should be evicted + + const evicted = evictExpiredPending(pending, ttlMs, now); + assert.strictEqual(evicted, 2); // inst-1 and inst-2 should be evicted + assert.strictEqual(pending.has("inst-1"), false); + assert.strictEqual(pending.has("inst-2"), false); + assert.strictEqual(pending.has("inst-3"), true); +}); + +test("isRpcError - detects RPC / network errors correctly", () => { + // Standard VM Execution Reverts should not be counted as RPC errors + const revertErr = new Error("Execution reverted: WraithOrders: order not tickable"); + assert.strictEqual(isRpcError(revertErr), false); + + const customRevertErr = { + name: "ContractFunctionExecutionError", + message: "The contract function 'tick' reverted with the following reason:\nWraithOrders: order already ticked", + }; + assert.strictEqual(isRpcError(customRevertErr), false); + + // Network / Fetch errors + const fetchErr = new Error("fetch failed"); + assert.strictEqual(isRpcError(fetchErr), true); + + const timeoutErr = { + name: "TimeoutError", + message: "The request timed out.", + }; + assert.strictEqual(isRpcError(timeoutErr), true); + + const rateLimitErr = new Error("Too Many Requests: Status 429"); + assert.strictEqual(isRpcError(rateLimitErr), true); + + const nestedErr = { + message: "Something failed", + cause: new Error("socket hang up"), + }; + assert.strictEqual(isRpcError(nestedErr), true); +}); + +test("handleHealthRequest - responds with 200 for GET /health", () => { + let responseData = ""; + let writtenStatus = 0; + let headers = {}; + + const req = { + method: "GET", + url: "/health", + }; + + const res = { + writeHead(status, head) { + writtenStatus = status; + headers = head; + }, + end(data) { + responseData = data; + }, + }; + + const mockHealthData = { + lastSuccessfulLoopTime: "2023-10-10T10:10:10.000Z", + pendingCount: 4, + balance: "1.2345 C2FLR", + }; + + handleHealthRequest(req, res, () => mockHealthData); + + assert.strictEqual(writtenStatus, 200); + assert.deepStrictEqual(headers, { "Content-Type": "application/json" }); + assert.deepStrictEqual(JSON.parse(responseData), mockHealthData); +}); + +test("handleHealthRequest - responds with 404 for other endpoints or methods", () => { + let responseData = ""; + let writtenStatus = 0; + + const req = { + method: "POST", + url: "/health", + }; + + const res = { + writeHead(status) { + writtenStatus = status; + }, + end(data) { + responseData = data; + }, + }; + + handleHealthRequest(req, res, () => ({})); + + assert.strictEqual(writtenStatus, 404); + assert.deepStrictEqual(JSON.parse(responseData), { error: "Not Found" }); +});