LSUDOKO · LSUDOKO · Aug 13, 2026
@@ -8,9 +8,20 @@
 // accepting a price from here. The worst a hostile keeper can do is withhold
 // ticks, which is why ticking is open to anyone.
 
+import { createServer } from "node:http";
 import { createPublicClient, createWalletClient, http, parseAbi, parseEventLogs, formatEther } from "viem";
 import { privateKeyToAccount } from "viem/accounts";
-import { handleFetchResultResponse, determineRelayAction } from "./lib.js";
+import {
+  handleFetchResultResponse,
+  determineRelayAction,
+  calculateBackoff,
+  incrementFailure,
+  resetFailure,
+  isExceeded,
+  evictExpiredPending,
+  isRpcError,
+  handleHealthRequest,
+} from "./lib.js";
 
 const RPC_URL = process.env.RPC_URL ?? "https://coston2-api.flare.network/ext/C/rpc";
 const WRAITH_ADDRESS = required("WRAITH_ADDRESS");
@@ -20,6 +31,12 @@ const POLL_INTERVAL_MS = Number(process.env.POLL_INTERVAL_MS ?? 15_000);
 const INSTRUCTION_FEE_WEI = BigInt(process.env.INSTRUCTION_FEE_WEI ?? "0");
 const SUBMISSION_TAG = process.env.SUBMISSION_TAG ?? "submit";
 
+// Configurable environment variables for hardening
+const BACKOFF_MAX_MS = Number(process.env.BACKOFF_MAX_MS ?? 300_000);
+const ORDER_MAX_RETRIES = Number(process.env.ORDER_MAX_RETRIES ?? 5);
+const PENDING_TTL_MS = Number(process.env.PENDING_TTL_MS ?? 3_600_000);
+const HEALTH_PORT = Number(process.env.HEALTH_PORT ?? 8080);
+
 const coston2 = {
   id: 114,
   name: "Coston2",
@@ -39,9 +56,16 @@ const account = privateKeyToAccount(PRIVATE_KEY);
 const publicClient = createPublicClient({ chain: coston2, transport: http(RPC_URL) });
 const walletClient = createWalletClient({ account, chain: coston2, transport: http(RPC_URL) });
 
-/** instructionId -> orderId, for instructions whose result has not arrived yet. */
+/** instructionId -> { orderId, addedAt } */
 const pending = new Map();
 
+/** orderId -> consecutive failures count */
+const orderFailures = new Map();
+
+// Variables for health monitoring
+let lastSuccessfulLoopTime = null;
+let keeperBalanceString = "0 C2FLR";
+
 function required(name) {
   const value = process.env[name];
   if (!value) {
@@ -66,12 +90,27 @@ async function tickOrders() {
   const count = await publicClient.readContract({ address: WRAITH_ADDRESS, abi, functionName: "orderCount" });
 
   for (let orderId = 0n; orderId < count; orderId++) {
-    const tickable = await publicClient.readContract({
-      address: WRAITH_ADDRESS,
-      abi,
-      functionName: "canTick",
-      args: [orderId],
-    });
+    if (isExceeded(orderFailures, orderId, ORDER_MAX_RETRIES)) {
+      continue;
+    }
+
+    let tickable = false;
+    try {
+      tickable = await publicClient.readContract({
+        address: WRAITH_ADDRESS,
+        abi,
+        functionName: "canTick",
+        args: [orderId],
+      });
+    } catch (error) {
+      if (isRpcError(error)) {
+        throw error;
+      }
+      console.error(`canTick failed for order ${orderId}: ${error.message}`);
+      incrementFailure(orderFailures, orderId);
+      continue;
+    }
+
     if (!tickable) continue;
 
     try {
@@ -84,25 +123,35 @@ async function tickOrders() {
       });
       const receipt = await publicClient.waitForTransactionReceipt({ hash });
 
+      // Reset failure count upon a successful tick
+      resetFailure(orderFailures, orderId);
+
       // OrderTicked carries the instruction id the proxy will key the result by.
       const events = parseEventLogs({ abi, logs: receipt.logs, eventName: "OrderTicked" });
       for (const event of events) {
-        pending.set(event.args.instructionId, orderId);
+        pending.set(event.args.instructionId, { orderId, addedAt: Date.now() });
         console.log(`ticked order ${orderId} -> instruction ${event.args.instructionId}`);
       }
     } catch (error) {
-      // One bad order must not stall every other order.
+      if (isRpcError(error)) {
+        throw error;
+      }
       console.error(`tick failed for order ${orderId}: ${error.shortMessage ?? error.message}`);
+      incrementFailure(orderFailures, orderId);
     }
   }
 }
 
 async function relayResults() {
-  for (const [instructionId, orderId] of [...pending]) {
+  for (const [instructionId, entry] of [...pending]) {
+    const { orderId } = entry;
     let result;
     try {
       result = await fetchResult(instructionId);
     } catch (error) {
+      if (isRpcError(error)) {
+        throw error;
+      }
       console.error(`polling ${instructionId}: ${error.message}`);
       continue;
     }
@@ -112,6 +161,7 @@ async function relayResults() {
 
     if (result.status !== 1) {
       console.error(`order ${orderId}: TEE reported failure (status ${result.status})`);
+      incrementFailure(orderFailures, orderId);
       continue;
     }
 
@@ -129,25 +179,68 @@ async function relayResults() {
       });
       await publicClient.waitForTransactionReceipt({ hash });
       console.log(`order ${orderId} executed in ${hash}`);
+      resetFailure(orderFailures, orderId);
     } catch (error) {
+      if (isRpcError(error)) {
+        throw error;
+      }
       console.error(`execute failed for order ${orderId}: ${error.shortMessage ?? error.message}`);
+      incrementFailure(orderFailures, orderId);
     }
   }
 }
 
+function getHealthData() {
+  return {
+    lastSuccessfulLoopTime,
+    pendingCount: pending.size,
+    keeperBalance: keeperBalanceString,
+  };
+}
+
 async function main() {
-  const balance = await publicClient.getBalance({ address: account.address });
-  console.log(`keeper ${account.address} (${formatEther(balance)} C2FLR)`);
-  console.log(`watching ${WRAITH_ADDRESS} via ${EXT_PROXY_URL}, every ${POLL_INTERVAL_MS}ms`);
+  // Start health HTTP server
+  const server = createServer((req, res) => {
+    handleHealthRequest(req, res, getHealthData);
+  });
+  server.listen(HEALTH_PORT, "0.0.0.0", () => {
+    console.log(`Health server listening on port ${HEALTH_PORT}`);
+  });
+
+  console.log(`keeper ${account.address} watching ${WRAITH_ADDRESS} via ${EXT_PROXY_URL}`);
+  console.log(`POLL_INTERVAL_MS=${POLL_INTERVAL_MS}, PENDING_TTL_MS=${PENDING_TTL_MS}, ORDER_MAX_RETRIES=${ORDER_MAX_RETRIES}`);
+
+  let consecutiveRpcFailures = 0;
 
   for (;;) {
+    let delayMs = POLL_INTERVAL_MS;
+
     try {
+      const balance = await publicClient.getBalance({ address: account.address });
+      keeperBalanceString = `${formatEther(balance)} C2FLR`;
+
       await tickOrders();
       await relayResults();
+
+      const evictedCount = evictExpiredPending(pending, PENDING_TTL_MS);
+      if (evictedCount > 0) {
+        console.log(`evicted ${evictedCount} expired pending instructions from memory`);
+      }
+
+      // Loop completed successfully
+      lastSuccessfulLoopTime = new Date().toISOString();
+      consecutiveRpcFailures = 0;
     } catch (error) {
-      console.error(`loop error: ${error.message}`);
+      if (isRpcError(error)) {
+        consecutiveRpcFailures++;
+        delayMs = calculateBackoff(consecutiveRpcFailures, POLL_INTERVAL_MS, BACKOFF_MAX_MS);
+        console.error(`RPC failure (consecutive count: ${consecutiveRpcFailures}). backing off for ${delayMs}ms: ${error.message}`);
+      } else {
+        console.error(`unexpected loop error: ${error.message}`);
+      }
     }
-    await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
+
+    await new Promise((resolve) => setTimeout(resolve, delayMs));
   }
 }
 

@@ -38,3 +38,144 @@ export function determineRelayAction(result, defaultSubmissionTag = "submit") {
     signature: result.signature,
   };
 }
+
+/**
+ * Calculates exponential backoff with full jitter.
+ *
+ * @param {number} consecutiveFailures
+ * @param {number} baseDelay
+ * @param {number} maxDelay
+ * @param {function} [random=Math.random] - Optional random number generator for testing.
+ * @returns {number}
+ */
+export function calculateBackoff(consecutiveFailures, baseDelay, maxDelay, random = Math.random) {
+  if (consecutiveFailures <= 0) {
+    return baseDelay;
+  }
+  const temp = Math.min(maxDelay, baseDelay * Math.pow(2, consecutiveFailures));
+  return Math.floor(random() * temp);
+}
+
+/**
+ * Increments the failure count for a given orderId.
+ *
+ * @param {Map<bigint, number>} failuresMap
+ * @param {bigint} orderId
+ * @returns {number} The new failure count.
+ */
+export function incrementFailure(failuresMap, orderId) {
+  const count = (failuresMap.get(orderId) ?? 0) + 1;
+  failuresMap.set(orderId, count);
+  return count;
+}
+
+/**
+ * Resets the failure count for a given orderId.
+ *
+ * @param {Map<bigint, number>} failuresMap
+ * @param {bigint} orderId
+ */
+export function resetFailure(failuresMap, orderId) {
+  failuresMap.delete(orderId);
+}
+
+/**
+ * Checks if the failure count for a given orderId has exceeded the cap.
+ *
+ * @param {Map<bigint, number>} failuresMap
+ * @param {bigint} orderId
+ * @param {number} maxRetries
+ * @returns {boolean}
+ */
+export function isExceeded(failuresMap, orderId, maxRetries) {
+  const count = failuresMap.get(orderId) ?? 0;
+  return count >= maxRetries;
+}
+
+/**
+ * Evicts pending instructions older than the TTL.
+ *
+ * @param {Map<string, { orderId: bigint, addedAt: number }>} pendingMap
+ * @param {number} ttlMs
+ * @param {number} now
+ * @returns {number} The number of evicted instructions.
+ */
+export function evictExpiredPending(pendingMap, ttlMs, now = Date.now()) {
+  let count = 0;
+  for (const [instructionId, entry] of pendingMap.entries()) {
+    if (now - entry.addedAt > ttlMs) {
+      pendingMap.delete(instructionId);
+      count++;
+    }
+  }
+  return count;
+}
+
+/**
+ * Determines whether an error is a Flare RPC/network failure.
+ * We want to distinguish transient RPC issues (which should trigger backoff)
+ * from permanent EVM execution failures/reverts (which should not trigger loop backoff).
+ *
+ * @param {any} error
+ * @returns {boolean}
+ */
+export function isRpcError(error) {
+  if (!error) return false;
+
+  const msg = (error.message || "").toLowerCase();
+  const shortMsg = (error.shortMessage || "").toLowerCase();
+  const name = (error.name || "").toLowerCase();
+
+  // If the error or its name contains revert, it's an execution revert, not an RPC failure.
+  if (msg.includes("revert") || shortMsg.includes("revert") || name.includes("revert")) {
+    return false;
+  }
+
+  // Common network, RPC, connection, or timeout indicators
+  if (
+    msg.includes("fetch") ||
+    msg.includes("network") ||
+    msg.includes("timeout") ||
+    msg.includes("request") ||
+    msg.includes("socket") ||
+    msg.includes("eaddrnotavail") ||
+    msg.includes("econnrefused") ||
+    msg.includes("rate limit") ||
+    msg.includes("status 429") ||
+    msg.includes("status 503") ||
+    msg.includes("status 502") ||
+    msg.includes("status 504") ||
+    msg.includes("rpc") ||
+    msg.includes("connection") ||
+    name.includes("httprequesterror") ||
+    name.includes("timeouterror") ||
+    name.includes("rpcrequesterror")
+  ) {
+    return true;
+  }
+
+  // Recurse into cause if available
+  if (error.cause) {
+    return isRpcError(error.cause);
+  }
+
+  return false;
+}
+
+/**
+ * Handles HTTP requests for the health endpoint.
+ *
+ * @param {any} req
+ * @param {any} res
+ * @param {function} getHealthData
+ */
+export function handleHealthRequest(req, res, getHealthData) {
+  const url = new URL(req.url, "http://localhost");
+  if (req.method === "GET" && url.pathname === "/health") {
+    res.writeHead(200, { "Content-Type": "application/json" });
+    res.end(JSON.stringify(getHealthData()));
+  } else {
+    res.writeHead(404, { "Content-Type": "application/json" });
+    res.end(JSON.stringify({ error: "Not Found" }));
+  }
+}