diff --git a/pending-transactions.csv b/pending-transactions.csv deleted file mode 100644 index 5fd1016..0000000 --- a/pending-transactions.csv +++ /dev/null @@ -1 +0,0 @@ -nonce,hash,status,transactionId,to,value,data,speed,gasLimit,type,gasDetails,sentAt \ No newline at end of file diff --git a/src/configuration/errorLogger.ts b/src/configuration/errorLogger.ts index 09b685a..d18d907 100644 --- a/src/configuration/errorLogger.ts +++ b/src/configuration/errorLogger.ts @@ -35,6 +35,33 @@ export class ErrorLogger { this.consoleLog = config.consoleLog ?? true; } + /** + * Recursively converts BigInt values to strings for JSON serialization + */ + private serializeBigInts(obj: unknown): unknown { + if (obj === null || obj === undefined) { + return obj; + } + + if (typeof obj === 'bigint') { + return obj.toString(); + } + + if (Array.isArray(obj)) { + return obj.map(item => this.serializeBigInts(item)); + } + + if (typeof obj === 'object') { + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + result[key] = this.serializeBigInts(value); + } + return result; + } + + return obj; + } + /** * Log an error to the database and optionally to the console */ @@ -50,14 +77,14 @@ export class ErrorLogger { // Extract context if available (from BaseError) const context = error instanceof BaseError ? error.context : undefined; - // Create error log object + // Create error log object with BigInt serialization handling const errorLog: ErrorLog = { service_name: this.serviceName, error_message: errorMessage, stack_trace: stackTrace, severity, - meta, - context, + meta: this.serializeBigInts(meta) as Record | undefined, + context: this.serializeBigInts(context) as Record | undefined, }; // Log to console if enabled diff --git a/src/executor/strategies/RelayerExecutor.ts b/src/executor/strategies/RelayerExecutor.ts index 1aed876..18aff83 100644 --- a/src/executor/strategies/RelayerExecutor.ts +++ b/src/executor/strategies/RelayerExecutor.ts @@ -409,26 +409,62 @@ export class RelayerExecutor implements IExecutor { profitability.estimates.expected_profit || BigInt(0); const payoutAmount = profitability.estimates.payout_amount || BigInt(0); - // If expected profit is 0 or negative, it means total rewards < (payout + gas cost) - if (expectedProfit <= BigInt(0)) { + // Use sophisticated validation that matches BaseExecutor logic + // Calculate required profit margin like BaseExecutor does + const gasCost = profitability.estimates.gas_cost || BigInt(0); + const baseAmountForMargin = payoutAmount + (CONFIG.profitability.includeGasCost ? gasCost : BigInt(0)); + const minProfitMarginPercent = CONFIG.profitability.minProfitMargin; + const requiredProfitValue = (baseAmountForMargin * BigInt(Math.round(minProfitMarginPercent * 100))) / 10000n; + const minimumRequiredProfit = baseAmountForMargin + requiredProfitValue; + + if (expectedProfit < minimumRequiredProfit) { + const gasEstimate = profitability.estimates.gas_estimate || BigInt(0); + const totalRewards = profitability.deposit_details?.reduce((sum, d) => sum + (d.rewards || BigInt(0)), BigInt(0)) || BigInt(0); + const error = new TransactionValidationError( - `Transaction not profitable: expected profit is ${expectedProfit}. Skipping expensive simulation.`, + `Transaction does not meet profit requirements: expected profit ${ethers.formatEther(expectedProfit)} ETH < required ${ethers.formatEther(minimumRequiredProfit)} ETH`, { depositIds: depositIds.map(String), expectedProfit: expectedProfit.toString(), + minimumRequiredProfit: minimumRequiredProfit.toString(), payoutAmount: payoutAmount.toString(), + gasEstimate: gasEstimate.toString(), + gasCost: gasCost.toString(), + totalRewards: totalRewards.toString(), + baseAmountForMargin: baseAmountForMargin.toString(), + requiredProfitValue: requiredProfitValue.toString(), + minProfitMarginPercent: `${minProfitMarginPercent * 100}%`, }, ); this.logger.warn( - 'Skipping simulation due to non-profitable transaction', + 'Rejecting transaction - insufficient profit margin', { - expectedProfit: expectedProfit.toString(), - payoutAmount: payoutAmount.toString(), + expectedProfit: ethers.formatEther(expectedProfit), + minimumRequiredProfit: ethers.formatEther(minimumRequiredProfit), + payoutAmount: ethers.formatEther(payoutAmount), + gasEstimate: gasEstimate.toString(), + gasCost: ethers.formatEther(gasCost), + totalRewards: ethers.formatEther(totalRewards), depositCount: depositIds.length, + profitMarginPercent: `${minProfitMarginPercent * 100}%`, }, ); return { isValid: false, error }; } + + // Log marginal but acceptable transactions for monitoring + const profitMargin = expectedProfit - baseAmountForMargin; + if (profitMargin < BigInt(10000000000000000)) { // Less than 0.01 ETH margin + this.logger.info( + 'Proceeding with marginal profit transaction', + { + expectedProfit: ethers.formatEther(expectedProfit), + profitMargin: ethers.formatEther(profitMargin), + payoutAmount: ethers.formatEther(payoutAmount), + depositCount: depositIds.length, + }, + ); + } // Try to get a better gas estimate using simulation if available if (this.simulationService) { diff --git a/src/monitor/StakerMonitor.ts b/src/monitor/StakerMonitor.ts index 880517d..131d6fa 100644 --- a/src/monitor/StakerMonitor.ts +++ b/src/monitor/StakerMonitor.ts @@ -30,6 +30,7 @@ import { EventProcessingError, MonitorError } from '@/configuration/errors'; import { stakerAbi } from '@/configuration/abis'; import { ErrorLogger } from '@/configuration/errorLogger'; import { Deposit } from '@/database/interfaces/types'; +import { DepositCountPredictor } from '@/utils/DepositCountPredictor'; /** * Extended MonitorConfig that includes the error logger @@ -59,6 +60,7 @@ export class StakerMonitor extends EventEmitter { private cachedBlockNumber: number; private lastBlockNumberUpdate: number; private readonly BLOCK_NUMBER_CACHE_TTL = 5000; // 5 seconds cache + private readonly depositPredictor: DepositCountPredictor; constructor(config: ExtendedMonitorConfig) { super(); @@ -87,6 +89,7 @@ export class StakerMonitor extends EventEmitter { this.depositScanInProgress = false; this.cachedBlockNumber = 0; this.lastBlockNumberUpdate = 0; + this.depositPredictor = new DepositCountPredictor(this.contract, this.logger); } /** @@ -273,6 +276,43 @@ export class StakerMonitor extends EventEmitter { } } + /** + * Retry function with exponential backoff for rate limiting + */ + private async retryWithBackoff( + operation: () => Promise, + operationName: string, + maxRetries = 3 + ): Promise { + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + return await operation(); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + + // Check if it's a rate limiting error + if (errorMessage.includes('compute units per second capacity') || + errorMessage.includes('429') || + errorMessage.includes('rate limit')) { + + if (attempt < maxRetries) { + const delay = Math.min(1000 * Math.pow(2, attempt - 1), 5000); // Exponential backoff, max 5s + this.logger.warn( + `Rate limit hit for ${operationName}, retrying in ${delay}ms (attempt ${attempt}/${maxRetries})` + ); + await new Promise(resolve => setTimeout(resolve, delay)); + continue; + } + } + + // If it's not a rate limit error or we've exhausted retries, throw + throw error; + } + } + + throw new Error(`Failed after ${maxRetries} attempts`); + } + /** * Queries contract events in batches for improved efficiency * @param contract - Contract instance to query @@ -288,23 +328,35 @@ export class StakerMonitor extends EventEmitter { toBlock: number, ): Promise { try { - const eventPromises = eventNames.map(async (eventName) => { + // Execute event queries sequentially with rate limiting to avoid CU/second limits + // Each eth_getLogs costs 60 CUs, so we need to be careful with parallel requests + const allEvents: ethers.Log[] = []; + + for (const eventName of eventNames) { try { const filter = contract.filters[eventName]; if (typeof filter === 'function') { - return await contract.queryFilter(filter(), fromBlock, toBlock); + // Retry with exponential backoff for rate limiting + const events = await this.retryWithBackoff(async () => { + return await contract.queryFilter(filter(), fromBlock, toBlock); + }, eventName); + + allEvents.push(...events); + + // Add delay between requests to respect rate limits (60 CUs each) + // Small delay to stay under CU/second capacity + if (eventNames.indexOf(eventName) < eventNames.length - 1) { + await new Promise(resolve => setTimeout(resolve, 150)); // 150ms delay + } } - return []; } catch (error) { this.logger.warn( - `Failed to query ${eventName} events: ${error instanceof Error ? error.message : String(error)}`, + `Failed to query ${eventName} events after retries: ${error instanceof Error ? error.message : String(error)}`, ); - return []; } - }); - - const eventArrays = await Promise.all(eventPromises); - return eventArrays.flat(); + } + + return allEvents; } catch (error) { this.logger.error('Error in batch event querying:', { error: error instanceof Error ? error.message : String(error), @@ -1438,15 +1490,27 @@ export class StakerMonitor extends EventEmitter { this.logger.info(`Found ${existingDepositIds.size} deposits in database`); - // Process sequentially starting from deposit ID 1 + // Get predicted scan limit to reduce API calls + const predictedScanLimit = await this.depositPredictor.getPredictedScanLimit(); + const stats = this.depositPredictor.getStats(); + + this.logger.info(`🔮 Deposit prediction stats`, { + predictedLimit: predictedScanLimit, + lastKnownMax: stats.lastKnownMax, + avgGrowthPerMin: stats.avgGrowthPerMinute.toFixed(2), + sampleCount: stats.samples, + lastSampleMinutesAgo: stats.lastSampleMinutesAgo.toFixed(1) + }); + + // Process sequentially starting from deposit ID 1, but limit scan range let currentId = 1; let emptyCounter = 0; const MAX_EMPTY_TO_STOP = 10; // Very conservative - only stop after many empty deposits - this.logger.info(`Starting sequential deposit scan from ID ${currentId}`); + this.logger.info(`Starting sequential deposit scan from ID ${currentId} to ~${predictedScanLimit}`); - // Keep scanning until we find many consecutive completely empty deposits - while (emptyCounter < MAX_EMPTY_TO_STOP) { + // Keep scanning until we find many consecutive completely empty deposits OR reach predicted limit + while (emptyCounter < MAX_EMPTY_TO_STOP && currentId <= predictedScanLimit) { try { if (!this.contract || typeof this.contract.deposits !== 'function') { throw new Error( diff --git a/src/profitability/strategies/GovLstProfitabilityEngine.ts b/src/profitability/strategies/GovLstProfitabilityEngine.ts index f596267..0679efb 100644 --- a/src/profitability/strategies/GovLstProfitabilityEngine.ts +++ b/src/profitability/strategies/GovLstProfitabilityEngine.ts @@ -17,6 +17,11 @@ import { TokenPrice } from '@/prices/interface'; import { SimulationService } from '@/simulation'; import { estimateGasUsingSimulation } from '@/executor/strategies/helpers/simulation-helpers'; import { MulticallBatcher } from '@/utils/multicall'; +import { BlockNumberCache } from '@/utils/BlockNumberCache'; +import { ContractDataCache } from '@/utils/ContractDataCache'; +import { CUMonitor } from '@/utils/CUMonitor'; +import { SimplifiedLogger } from '@/utils/SimplifiedLogger'; +import { DepositCountPredictor } from '@/utils/DepositCountPredictor'; /** * Updated ProfitabilityConfig to include errorLogger @@ -35,6 +40,11 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { private readonly errorLogger?: ErrorLogger; private readonly priceFeed: CoinMarketCapFeed; private readonly multicallBatcher: MulticallBatcher; + private readonly blockNumberCache: BlockNumberCache; + private readonly contractDataCache: ContractDataCache; + private readonly cuMonitor: CUMonitor; + private readonly simpleLogger: SimplifiedLogger; + private readonly depositPredictor: DepositCountPredictor; private isRunning: boolean; private lastGasPrice: bigint; private lastUpdateTimestamp: number; @@ -46,7 +56,7 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { } | null = null; private readonly PRICE_CACHE_DURATION = 10 * 60 * 1000; // 10 minutes in milliseconds public readonly config: ProfitabilityConfig; - private static readonly BATCH_SIZE = 100; // Number of deposits to fetch in a single batch + private static readonly BATCH_SIZE = 20; // Number of deposits to fetch in a single batch (reduced for rate limiting) private activeBin: GovLstDepositGroup | null = null; // Current active bin being filled private readonly simulationService: SimulationService | null; @@ -86,7 +96,50 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { this.lastUpdateTimestamp = 0; this.config = config; this.simulationService = simulationService; - this.multicallBatcher = new MulticallBatcher(provider, this.logger); + + // Initialize monitoring components first + this.blockNumberCache = new BlockNumberCache(this.logger); + this.contractDataCache = new ContractDataCache(this.logger); + this.cuMonitor = new CUMonitor(this.logger); + this.simpleLogger = new SimplifiedLogger(this.logger, 'Profitability'); + this.depositPredictor = new DepositCountPredictor(this.stakerContract, this.logger); + + // Initialize multicall batcher with CU monitor + this.multicallBatcher = new MulticallBatcher( + provider, + this.logger, + 5, // Max 5 concurrent calls to respect rate limits + 200, // 200ms delay between batches + false, // Disable Multicall3 - it's not working with current setup + this.cuMonitor // Pass CU monitor for tracking + ); + + // Periodic cache cleanup and CU monitoring + setInterval(() => { + this.contractDataCache.cleanup(); + const cuStats = this.cuMonitor.getUsageStats(); + const recommendations = this.cuMonitor.getOptimizationRecommendations(); + + this.simpleLogger.critical('System maintenance', { + caches: { + blocks: this.blockNumberCache.getStats().size, + contracts: this.contractDataCache.getStats().deposits.size + }, + cuUsage: { + daily: cuStats.dailyUsage, + percent: cuStats.percentUsed + '%' + } + }); + + if (recommendations.length > 0) { + this.simpleLogger.warn('CU optimization needed', { + count: recommendations.length, + top: recommendations[0] + }); + } + + this.simpleLogger.maybeSummary(); + }, 600000); // Every 10 minutes // Initialize price feed this.priceFeed = new CoinMarketCapFeed( @@ -494,20 +547,18 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { 'Fetching payout amount', ); - // Calculate realistic gas cost based on actual usage patterns - // Production data shows ~2-2.6M gas for 60-70 deposits - const REALISTIC_GAS_UNITS = BigInt(2500000); // 2.5M gas based on production simulations + // Calculate conservative gas cost for Stage 1 threshold + // Use 3x realistic estimate to avoid Stage 1 shortfalls + const REALISTIC_GAS_UNITS = BigInt(7500000); // 7.5M gas (3x 2.5M) for conservative Stage 1 const gasPrice = await this.getGasPriceWithBuffer(); const gasCostWei = gasPrice * REALISTIC_GAS_UNITS; const realisticGasCost = await this.convertGasCostToRewardTokens(gasCostWei); - this.logger.info('Using realistic gas estimate for Stage 1 threshold', { + this.simpleLogger.debug('Conservative gas estimate for Stage 1', { gasUnits: REALISTIC_GAS_UNITS.toString(), - gasPrice: ethers.formatUnits(gasPrice, 'gwei'), - gasCostWei: gasCostWei.toString(), - gasCostInTokens: realisticGasCost.toString(), - gasCostInTokensFormatted: ethers.formatEther(realisticGasCost), + gasPriceGwei: ethers.formatUnits(gasPrice, 'gwei'), + gasCostETH: ethers.formatEther(realisticGasCost), }); // We'll calculate enhanced gas cost and actualGasEstimate later, after we know total rewards @@ -636,14 +687,7 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { // Stage 1: Initial check - trigger first simulation when rewards > payout + margin if (this.activeBin?.total_rewards >= optimalThreshold) { - this.logger.info( - 'Stage 1: Initial threshold reached, running first simulation', - { - totalRewards: this.activeBin.total_rewards.toString(), - optimalThreshold: optimalThreshold.toString(), - depositCount: this.activeBin.deposit_ids.length, - }, - ); + this.simpleLogger.critical(`Stage 1: Found ${this.activeBin.deposit_ids.length} deposits worth ${ethers.formatEther(this.activeBin.total_rewards)} ETH`); // First simulation to get accurate gas cost let firstSimulationGasCost = realisticGasCost; @@ -701,16 +745,7 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { : BigInt(0)) + marginOnPayout; - this.logger.info('Stage 2: Checking against refined threshold', { - totalRewards: this.activeBin.total_rewards.toString(), - stage2Threshold: stage2Threshold.toString(), - firstSimulationGasCost: firstSimulationGasCost.toString(), - payoutAmount: payoutAmount.toString(), - scaledProfitMarginBasisPoints: - scaledProfitMarginBasisPoints.toString(), - marginOnPayout: marginOnPayout.toString(), - depositCount: this.activeBin.deposit_ids.length, - }); + // Silent threshold check if (this.activeBin.total_rewards >= stage2Threshold) { // Run second simulation for final validation @@ -764,32 +799,11 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { : BigInt(0); this.activeBin.expected_profit = expectedProfit; - this.logger.info('Final profitability calculation', { - totalRewards: this.activeBin.total_rewards.toString(), - finalGasCost: finalGasCost.toString(), - expectedProfit: expectedProfit.toString(), - depositCount: this.activeBin.deposit_ids.length, - }); + this.simpleLogger.critical(`🎯 PROFITABLE: ${this.activeBin.deposit_ids.length} deposits, ${ethers.formatEther(expectedProfit)} ETH profit`); readyBins.push(this.activeBin); } else { - this.logger.info( - 'Stage 2: Insufficient rewards after accurate gas estimation', - { - totalRewards: this.activeBin.total_rewards.toString(), - stage2Threshold: stage2Threshold.toString(), - shortfall: ( - stage2Threshold - this.activeBin.total_rewards - ).toString(), - breakdown: { - payoutAmount: payoutAmount.toString(), - gasCost: firstSimulationGasCost.toString(), - marginBasisPoints: scaledProfitMarginBasisPoints.toString(), - marginAmount: marginOnPayout.toString(), - calculation: `${payoutAmount} + ${firstSimulationGasCost} + ${marginOnPayout} = ${stage2Threshold}`, - }, - }, - ); + // Silent rejection - no logging needed } } @@ -1080,10 +1094,7 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { const rewardsMap = new Map(); const batchSize = GovLstProfitabilityEngine.BATCH_SIZE; - this.logger.info('Starting batch fetch of unclaimed rewards:', { - totalDeposits: depositIds.length, - batchSize, - }); + // Silent start - no logging // Function to process a batch with retries const processBatch = async (batchIds: bigint[]) => { @@ -1101,9 +1112,13 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { reward: batchedRewards[index] || BigInt(0), // Use 0 if batch call failed })); - // Check if all rewards are 0, might indicate we need to wait for chain update - const allZero = results.every(({ reward }) => reward === BigInt(0)); - if (allZero) { + // Check if all rewards are 0 or null, might indicate RPC issues or chain state + const validResults = results.filter(({ reward }) => reward !== null); + const allZero = validResults.length > 0 && validResults.every(({ reward }) => reward === BigInt(0)); + + // Only throw if we have valid results but they're all zero + // If we have no valid results, it's likely an RPC issue, not a chain state issue + if (allZero && validResults.length === results.length) { this.logger.info( 'All rewards are 0, waiting for chain update...', ); @@ -1112,6 +1127,15 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { 'All rewards are zero - waiting for chain update', ); } + + // If we have mixed results (some null, some valid), continue without throwing + if (validResults.length < results.length) { + this.logger.warn('Some reward calls failed, continuing with valid results', { + validResults: validResults.length, + totalResults: results.length, + failedCalls: results.length - validResults.length + }); + } return results; }, @@ -1120,14 +1144,14 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { 2000, ); - // Store results + // Store results (only for non-null rewards) results.forEach(({ id, reward }) => { - rewardsMap.set(id.toString(), reward); - if (reward > BigInt(0)) { - this.logger.info('Fetched non-zero reward:', { - depositId: id.toString(), - reward: ethers.formatEther(reward), - }); + if (reward !== null) { + rewardsMap.set(id.toString(), reward); + // Track non-zero rewards but don't log each one + if (reward > BigInt(0)) { + this.simpleLogger.trackRewards(1, ethers.formatEther(reward)); + } } }); }; @@ -1137,9 +1161,9 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { const batchIds = depositIds.slice(i, i + batchSize); await processBatch(batchIds); - // Small delay between batches + // Longer delay between batches to respect rate limits if (i + batchSize < depositIds.length) { - await new Promise((resolve) => setTimeout(resolve, 100)); + await new Promise((resolve) => setTimeout(resolve, 500)); } } @@ -1151,12 +1175,10 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { (reward) => reward > BigInt(0), ); - this.logger.info('Completed batch fetch of unclaimed rewards:', { - totalDeposits: depositIds.length, - successfulFetches: rewardsMap.size, - nonZeroRewards: nonZeroRewards.length, - totalRewardsInEther: ethers.formatEther(totalRewards), - }); + // Only log if significant rewards found + if (nonZeroRewards.length > 10) { + this.simpleLogger.critical(`Found ${nonZeroRewards.length} profitable deposits (${ethers.formatEther(totalRewards)} ETH)`); + } return rewardsMap; } catch (error) { @@ -1755,8 +1777,23 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { } if (attempt < maxRetries) { + // Exponential backoff with jitter for rate limiting + const isRateLimited = lastError.message.includes('compute units per second') || + lastError.message.includes('429') || + lastError.message.includes('rate limit'); + + const backoffDelay = isRateLimited + ? delayMs * Math.pow(2, attempt) + Math.random() * 1000 // Longer backoff for rate limits + : delayMs * attempt; + + this.logger.info(`Waiting ${backoffDelay}ms before retry ${attempt + 1}/${maxRetries}`, { + context, + isRateLimited, + backoffDelay + }); + await new Promise((resolve) => - setTimeout(resolve, delayMs * attempt), + setTimeout(resolve, backoffDelay), ); } } diff --git a/src/utils/AlchemyCUTracker.ts b/src/utils/AlchemyCUTracker.ts new file mode 100644 index 0000000..cc8a1a1 --- /dev/null +++ b/src/utils/AlchemyCUTracker.ts @@ -0,0 +1,253 @@ +import { Logger } from '@/monitor/logging'; + +/** + * Accurate Alchemy Compute Unit (CU) tracker based on official documentation + * Tracks real-time CU usage and provides detailed analytics + */ +export class AlchemyCUTracker { + private readonly logger: Logger; + private dailyUsage = 0; + private dailyLimit: number; + private lastResetDate = new Date().toDateString(); + private methodUsage: Map = new Map(); + + // Official Alchemy CU costs from documentation + private static readonly OFFICIAL_CU_COSTS = { + // Standard Methods + 'eth_call': 26, + 'eth_blockNumber': 10, + 'eth_getTransactionReceipt': 20, + 'eth_estimateGas': 20, + 'eth_sendRawTransaction': 40, + 'eth_getLogs': 60, + 'eth_getBalance': 20, + 'eth_getCode': 20, + 'eth_chainId': 0, + + // Contract methods (all use eth_call) + 'unclaimedReward': 26, + 'deposits': 26, + 'payoutAmount': 26, + + // Multicall3 - single eth_call regardless of batch size + 'multicall3': 26, + + // Enhanced APIs + 'getNFTMetadata': 100, + 'getFloorPrice': 100, + } as const; + + constructor(logger: Logger, dailyLimitCU = 20_000_000) { + this.logger = logger; + this.dailyLimit = dailyLimitCU; + } + + /** + * Track a single method call + */ + trackMethod(method: string, count = 1): number { + const cuCost = AlchemyCUTracker.OFFICIAL_CU_COSTS[method as keyof typeof AlchemyCUTracker.OFFICIAL_CU_COSTS] || 26; + const totalCUs = cuCost * count; + + this.resetDailyUsageIfNeeded(); + this.dailyUsage += totalCUs; + + // Update method-specific tracking + const existing = this.methodUsage.get(method) || { calls: 0, totalCUs: 0 }; + this.methodUsage.set(method, { + calls: existing.calls + count, + totalCUs: existing.totalCUs + totalCUs + }); + + return totalCUs; + } + + /** + * Track multicall batch - always 26 CUs regardless of batch size + */ + trackMulticall(individualMethodCount: number): { cuUsed: number; cuSaved: number } { + const cuUsed = 26; // Multicall3 always costs 26 CUs + const cuSaved = (individualMethodCount - 1) * 26; // What we saved vs individual calls + + this.trackMethod('multicall3', 1); + + return { cuUsed, cuSaved }; + } + + /** + * Track individual calls when multicall fails + */ + trackIndividualCalls(method: string, count: number): number { + return this.trackMethod(method, count); + } + + /** + * Get current usage statistics + */ + getUsageStats() { + this.resetDailyUsageIfNeeded(); + + return { + dailyUsage: this.dailyUsage, + dailyLimit: this.dailyLimit, + percentUsed: ((this.dailyUsage / this.dailyLimit) * 100).toFixed(2), + remainingCUs: this.dailyLimit - this.dailyUsage, + costInUSD: this.calculateCostUSD(), + resetDate: this.lastResetDate, + methodBreakdown: this.getTopMethods() + }; + } + + /** + * Get top methods by CU usage + */ + private getTopMethods() { + return Array.from(this.methodUsage.entries()) + .sort(([,a], [,b]) => b.totalCUs - a.totalCUs) + .slice(0, 10) + .map(([method, stats]) => ({ + method, + calls: stats.calls, + totalCUs: stats.totalCUs, + avgCUsPerCall: (stats.totalCUs / stats.calls).toFixed(1) + })); + } + + /** + * Calculate estimated USD cost based on Alchemy pricing + */ + private calculateCostUSD(): string { + // Alchemy Growth plan: $199/month for 300M CUs + $1.20 per additional 1M CUs + const includedCUs = 300_000_000; // Monthly included CUs + const additionalCUCost = 1.20; // Cost per 1M additional CUs + + // For daily usage, approximate monthly cost + const estimatedMonthlyCUs = this.dailyUsage * 30; + + if (estimatedMonthlyCUs <= includedCUs) { + return '$199.00'; // Base plan cost + } else { + const additionalCUs = estimatedMonthlyCUs - includedCUs; + const additionalCost = (additionalCUs / 1_000_000) * additionalCUCost; + return `$${(199 + additionalCost).toFixed(2)}`; + } + } + + /** + * Get optimization recommendations + */ + getOptimizationRecommendations(): string[] { + const recommendations: string[] = []; + const stats = this.getUsageStats(); + + // High usage warning + if (parseFloat(stats.percentUsed) > 80) { + recommendations.push(`⚠️ Daily CU usage at ${stats.percentUsed}% of limit`); + } + + // Check method-specific issues + const topMethods = this.getTopMethods(); + + if (topMethods.length > 0) { + const topMethod = topMethods[0]; + + if (topMethod?.method === 'eth_call' && topMethod.totalCUs > 10000) { + recommendations.push(`🔄 High eth_call usage (${topMethod.totalCUs} CUs) - ensure multicall batching is working`); + } + + if (topMethod?.method === 'unclaimedReward' && topMethod.calls > 100) { + recommendations.push(`📦 High unclaimedReward calls (${topMethod.calls}) - batch size too small?`); + } + + if (topMethod?.method === 'eth_blockNumber' && topMethod.totalCUs > 500) { + recommendations.push(`⏰ High block number calls (${topMethod.totalCUs} CUs) - implement caching`); + } + } + + // Cost optimization + const monthlyCost = parseFloat(stats.costInUSD.replace('$', '')); + if (monthlyCost > 400) { + recommendations.push(`💰 High monthly cost estimate (${stats.costInUSD}) - consider provider alternatives`); + } + + return recommendations; + } + + /** + * Export detailed usage report + */ + exportDetailedReport() { + const stats = this.getUsageStats(); + + return { + timestamp: new Date().toISOString(), + usage: { + daily: stats.dailyUsage, + limit: stats.dailyLimit, + percentage: stats.percentUsed, + remaining: stats.remainingCUs + }, + cost: { + estimated: stats.costInUSD, + breakdown: 'Based on Alchemy Growth plan ($199/month + $1.20/M additional CUs)' + }, + methods: stats.methodBreakdown, + recommendations: this.getOptimizationRecommendations(), + cuSavingsFromMulticall: this.calculateMulticallSavings() + }; + } + + /** + * Calculate total CU savings from multicall usage + */ + private calculateMulticallSavings(): number { + const multicallStats = this.methodUsage.get('multicall3'); + if (!multicallStats) return 0; + + // Estimate average batch size based on individual call reductions + // This is approximate since we don't track exact batch sizes + const estimatedBatchSize = 20; // Conservative estimate + return multicallStats.calls * (estimatedBatchSize - 1) * 26; + } + + /** + * Reset daily usage if it's a new day + */ + private resetDailyUsageIfNeeded(): void { + const today = new Date().toDateString(); + if (today !== this.lastResetDate) { + this.logger.info('🔄 Daily CU usage reset', { + previousDate: this.lastResetDate, + previousUsage: this.dailyUsage, + previousCost: this.calculateCostUSD(), + newDate: today + }); + + this.dailyUsage = 0; + this.methodUsage.clear(); + this.lastResetDate = today; + } + } + + /** + * Log periodic summary with actionable insights + */ + logSummary(): void { + const stats = this.getUsageStats(); + const recommendations = this.getOptimizationRecommendations(); + + this.logger.info('📊 CU Usage Summary', { + usage: `${stats.dailyUsage.toLocaleString()} / ${stats.dailyLimit.toLocaleString()} (${stats.percentUsed}%)`, + cost: stats.costInUSD, + topMethod: stats.methodBreakdown[0]?.method || 'none', + savings: `~${this.calculateMulticallSavings().toLocaleString()} CUs saved via multicall` + }); + + if (recommendations.length > 0) { + this.logger.warn('💡 CU Optimization Recommendations', { + count: recommendations.length, + recommendations: recommendations.slice(0, 3) // Top 3 recommendations + }); + } + } +} \ No newline at end of file diff --git a/src/utils/BlockNumberCache.ts b/src/utils/BlockNumberCache.ts new file mode 100644 index 0000000..717768b --- /dev/null +++ b/src/utils/BlockNumberCache.ts @@ -0,0 +1,83 @@ +import { ethers } from 'ethers'; +import { Logger } from '@/monitor/logging'; + +/** + * Caches block numbers to reduce redundant getBlockNumber calls + * Saves 10 CUs per avoided call + */ +export class BlockNumberCache { + private cache: Map = new Map(); + private readonly cacheTimeout: number; + private readonly logger: Logger; + + constructor(logger: Logger, cacheTimeoutMs = 12000) { // ~1 block time + this.logger = logger; + this.cacheTimeout = cacheTimeoutMs; + } + + /** + * Gets block number with caching to reduce CU usage + * @param provider Ethereum provider + * @param cacheKey Optional cache key for different contexts + * @returns Current block number + */ + async getCachedBlockNumber( + provider: ethers.Provider, + cacheKey = 'default' + ): Promise { + const now = Date.now(); + const cached = this.cache.get(cacheKey); + + // Return cached value if still fresh + if (cached && (now - cached.timestamp) < this.cacheTimeout) { + this.logger.debug('Using cached block number', { + blockNumber: cached.blockNumber, + age: now - cached.timestamp, + cacheKey + }); + return cached.blockNumber; + } + + // Fetch fresh block number + try { + const blockNumber = await provider.getBlockNumber(); + this.cache.set(cacheKey, { blockNumber, timestamp: now }); + + this.logger.debug('Fetched fresh block number (10 CUs)', { + blockNumber, + cacheKey, + cacheSize: this.cache.size + }); + + return blockNumber; + } catch (error) { + // Return stale cache if available on error + if (cached) { + this.logger.warn('Using stale cached block number due to error', { + error: error instanceof Error ? error.message : String(error), + blockNumber: cached.blockNumber, + age: now - cached.timestamp + }); + return cached.blockNumber; + } + throw error; + } + } + + /** + * Clears the cache + */ + clear(): void { + this.cache.clear(); + } + + /** + * Gets cache statistics + */ + getStats(): { size: number; keys: string[] } { + return { + size: this.cache.size, + keys: Array.from(this.cache.keys()) + }; + } +} \ No newline at end of file diff --git a/src/utils/CUMonitor.ts b/src/utils/CUMonitor.ts new file mode 100644 index 0000000..7a1db87 --- /dev/null +++ b/src/utils/CUMonitor.ts @@ -0,0 +1,169 @@ +import { Logger } from '@/monitor/logging'; + +/** + * Monitors and tracks Compute Unit (CU) usage to help optimize costs + */ +export class CUMonitor { + private cuUsage: Map = new Map(); + private dailyUsage = 0; + private dailyLimit = 20_000_000; // 20M CUs as mentioned by user + private lastResetDate = new Date().toDateString(); + private readonly logger: Logger; + + // CU costs for common operations (from Alchemy docs) + private static readonly CU_COSTS = { + 'eth_call': 26, + 'eth_blockNumber': 10, + 'eth_getTransactionReceipt': 20, + 'eth_estimateGas': 20, + 'eth_sendRawTransaction': 40, + 'eth_getLogs': 60, + 'multicall3': 26, // Single multicall cost vs N * 26 for individual calls + 'deposits': 26, // Contract call + 'unclaimedReward': 26, // Contract call + }; + + constructor(logger: Logger, dailyLimitCU = 20_000_000) { + this.logger = logger; + this.dailyLimit = dailyLimitCU; + } + + /** + * Track CU usage for an operation + */ + trackUsage(operation: string, count = 1): void { + const cost = CUMonitor.CU_COSTS[operation as keyof typeof CUMonitor.CU_COSTS] || 26; // Default to eth_call cost + const totalCost = cost * count; + + this.resetDailyUsageIfNeeded(); + + // Update operation-specific usage + const currentUsage = this.cuUsage.get(operation) || 0; + this.cuUsage.set(operation, currentUsage + totalCost); + + // Update daily usage + this.dailyUsage += totalCost; + + this.logger.debug('CU usage tracked', { + operation, + count, + costPerOperation: cost, + totalCost, + dailyUsage: this.dailyUsage, + percentOfLimit: (this.dailyUsage / this.dailyLimit * 100).toFixed(2) + }); + + // Warning if approaching limit + if (this.dailyUsage > this.dailyLimit * 0.8) { + this.logger.warn('High CU usage detected', { + dailyUsage: this.dailyUsage, + percentOfLimit: (this.dailyUsage / this.dailyLimit * 100).toFixed(2), + remainingCUs: this.dailyLimit - this.dailyUsage + }); + } + } + + /** + * Track batch operation with potential savings + */ + trackBatchOperation(operation: string, batchSize: number, usedMulticall = false): void { + if (usedMulticall && batchSize > 1) { + // Multicall: 1 call instead of N calls = saves (N-1) * 26 CUs + this.trackUsage('multicall3', 1); + const saved = (batchSize - 1) * 26; + this.logger.info('CU savings from multicall', { + operation, + batchSize, + cuSaved: saved, + cuUsed: 26, + cuWouldHaveUsed: batchSize * 26 + }); + } else { + // Individual calls + this.trackUsage(operation, batchSize); + } + } + + /** + * Get current usage statistics + */ + getUsageStats() { + this.resetDailyUsageIfNeeded(); + + return { + dailyUsage: this.dailyUsage, + dailyLimit: this.dailyLimit, + percentUsed: (this.dailyUsage / this.dailyLimit * 100).toFixed(2), + remainingCUs: this.dailyLimit - this.dailyUsage, + topOperations: Array.from(this.cuUsage.entries()) + .sort(([,a], [,b]) => b - a) + .slice(0, 10) + .map(([op, usage]) => ({ operation: op, cuUsed: usage })), + resetDate: this.lastResetDate + }; + } + + /** + * Get optimization recommendations + */ + getOptimizationRecommendations(): string[] { + const recommendations: string[] = []; + const stats = this.getUsageStats(); + + // High usage warning + if (parseFloat(stats.percentUsed) > 80) { + recommendations.push(`Daily CU usage is at ${stats.percentUsed}% - consider optimization`); + } + + // Check for high individual call usage + const unclaimedRewardUsage = this.cuUsage.get('unclaimedReward') || 0; + const depositsUsage = this.cuUsage.get('deposits') || 0; + + if (unclaimedRewardUsage > 10000) { + recommendations.push(`High unclaimedReward usage (${unclaimedRewardUsage} CUs) - ensure you're using multicall batching`); + } + + if (depositsUsage > 5000) { + recommendations.push(`High deposits usage (${depositsUsage} CUs) - consider caching deposit data`); + } + + // Block number optimization + const blockNumberUsage = this.cuUsage.get('eth_blockNumber') || 0; + if (blockNumberUsage > 1000) { + recommendations.push(`High block number calls (${blockNumberUsage} CUs) - implement block number caching`); + } + + return recommendations; + } + + /** + * Reset daily usage if it's a new day + */ + private resetDailyUsageIfNeeded(): void { + const today = new Date().toDateString(); + if (today !== this.lastResetDate) { + this.logger.info('Daily CU usage reset', { + previousDate: this.lastResetDate, + previousUsage: this.dailyUsage, + newDate: today + }); + + this.dailyUsage = 0; + this.cuUsage.clear(); + this.lastResetDate = today; + } + } + + /** + * Export usage data for analysis + */ + exportUsageData() { + return { + timestamp: new Date().toISOString(), + dailyUsage: this.dailyUsage, + operations: Object.fromEntries(this.cuUsage), + stats: this.getUsageStats(), + recommendations: this.getOptimizationRecommendations() + }; + } +} \ No newline at end of file diff --git a/src/utils/ContractDataCache.ts b/src/utils/ContractDataCache.ts new file mode 100644 index 0000000..8a4965f --- /dev/null +++ b/src/utils/ContractDataCache.ts @@ -0,0 +1,164 @@ +import { ethers } from 'ethers'; +import { Logger } from '@/monitor/logging'; + +/** + * Caches contract data to reduce redundant calls + * Saves 26 CUs per avoided eth_call + */ +export class ContractDataCache { + private depositCache: Map = new Map(); + private rewardCache: Map = new Map(); + private readonly depositCacheTimeout: number; + private readonly rewardCacheTimeout: number; + private readonly logger: Logger; + + constructor( + logger: Logger, + depositCacheTimeoutMs = 300000, // 5 minutes - deposit data changes rarely + rewardCacheTimeoutMs = 30000 // 30 seconds - rewards change frequently + ) { + this.logger = logger; + this.depositCacheTimeout = depositCacheTimeoutMs; + this.rewardCacheTimeout = rewardCacheTimeoutMs; + } + + /** + * Gets deposit data with caching + */ + async getCachedDepositData( + contract: ethers.Contract, + depositId: string + ): Promise { + const now = Date.now(); + const cached = this.depositCache.get(depositId); + + if (cached && (now - cached.timestamp) < this.depositCacheTimeout) { + this.logger.debug('Using cached deposit data', { + depositId, + age: now - cached.timestamp + }); + return cached.data; + } + + try { + const data = await contract.deposits?.(depositId); + this.depositCache.set(depositId, { data, timestamp: now }); + + this.logger.debug('Fetched fresh deposit data', { + depositId, + cacheSize: this.depositCache.size + }); + + return data; + } catch (error) { + if (cached) { + this.logger.warn('Using stale cached deposit data due to error', { + error: error instanceof Error ? error.message : String(error), + depositId, + age: now - cached.timestamp + }); + return cached.data; + } + throw error; + } + } + + /** + * Gets reward data with shorter caching due to frequent changes + */ + async getCachedReward( + contract: ethers.Contract, + depositId: string + ): Promise { + const now = Date.now(); + const cached = this.rewardCache.get(depositId); + + if (cached && (now - cached.timestamp) < this.rewardCacheTimeout) { + this.logger.debug('Using cached reward', { + depositId, + reward: ethers.formatEther(cached.reward), + age: now - cached.timestamp + }); + return cached.reward; + } + + try { + const reward = await contract.unclaimedReward?.(depositId); + this.rewardCache.set(depositId, { reward, timestamp: now }); + + this.logger.debug('Fetched fresh reward', { + depositId, + reward: ethers.formatEther(reward), + cacheSize: this.rewardCache.size + }); + + return reward; + } catch (error) { + if (cached) { + this.logger.warn('Using stale cached reward due to error', { + error: error instanceof Error ? error.message : String(error), + depositId, + reward: ethers.formatEther(cached.reward), + age: now - cached.timestamp + }); + return cached.reward; + } + throw error; + } + } + + /** + * Invalidates cache for specific deposit + */ + invalidateDeposit(depositId: string): void { + this.depositCache.delete(depositId); + this.rewardCache.delete(depositId); + } + + /** + * Clears all caches + */ + clear(): void { + this.depositCache.clear(); + this.rewardCache.clear(); + } + + /** + * Gets cache statistics + */ + getStats() { + return { + deposits: { + size: this.depositCache.size, + keys: Array.from(this.depositCache.keys()).slice(0, 10) // First 10 for brevity + }, + rewards: { + size: this.rewardCache.size, + keys: Array.from(this.rewardCache.keys()).slice(0, 10) + } + }; + } + + /** + * Cleanup expired entries + */ + cleanup(): void { + const now = Date.now(); + + // Cleanup deposit cache + for (const [key, value] of this.depositCache.entries()) { + if (now - value.timestamp > this.depositCacheTimeout) { + this.depositCache.delete(key); + } + } + + // Cleanup reward cache + for (const [key, value] of this.rewardCache.entries()) { + if (now - value.timestamp > this.rewardCacheTimeout) { + this.rewardCache.delete(key); + } + } + + this.logger.debug('Cache cleanup completed', this.getStats()); + } +} \ No newline at end of file diff --git a/src/utils/DepositCountPredictor.ts b/src/utils/DepositCountPredictor.ts new file mode 100644 index 0000000..d89c5f5 --- /dev/null +++ b/src/utils/DepositCountPredictor.ts @@ -0,0 +1,200 @@ +import { ethers } from 'ethers'; +import { Logger } from '@/monitor/logging'; +import { SimplifiedLogger } from './SimplifiedLogger'; + +/** + * Tracks deposit count growth by sampling every 5 minutes and extrapolating + * to reduce unnecessary API calls during profitability analysis + */ +export class DepositCountPredictor { + private readonly contract: ethers.Contract; + private readonly logger: Logger; + private readonly simpleLogger: SimplifiedLogger; + + // Sample history - store recent measurements + private samples: Array<{ timestamp: number; maxDepositId: number }> = []; + + // Prediction settings + private readonly SAMPLE_INTERVAL = 5 * 60 * 1000; // 5 minutes + private readonly MAX_SAMPLES = 12; // Keep last 12 samples (1 hour of history) + private readonly SAFETY_BUFFER = 50; // Extra deposits to scan beyond prediction + + constructor(contract: ethers.Contract, logger: Logger) { + this.contract = contract; + this.logger = logger; + this.simpleLogger = new SimplifiedLogger(logger, 'DepositPredictor'); + } + + /** + * Get the predicted upper bound for deposit IDs to scan + * Returns the estimated max deposit ID plus safety buffer + */ + async getPredictedScanLimit(): Promise { + const now = Date.now(); + + // Check if we need a new sample + await this.maybeTakeSample(now); + + // If we have less than 2 samples, take a full scan + if (this.samples.length < 2) { + const currentMax = await this.findCurrentMaxDepositId(); + this.samples.push({ timestamp: now, maxDepositId: currentMax }); + return currentMax + this.SAFETY_BUFFER; + } + + // Extrapolate based on recent growth + return this.extrapolateMaxDepositId(now) + this.SAFETY_BUFFER; + } + + /** + * Take a sample if enough time has passed since the last one + */ + private async maybeTakeSample(now: number): Promise { + const lastSampleTime = this.samples.length > 0 + ? this.samples[this.samples.length - 1]?.timestamp + : 0; + + if (now && lastSampleTime && now - lastSampleTime >= this.SAMPLE_INTERVAL) { + try { + const currentMax = await this.findCurrentMaxDepositId(); + + // Add new sample + this.samples.push({ timestamp: now, maxDepositId: currentMax }); + + // Keep only recent samples + if (this.samples.length > this.MAX_SAMPLES) { + this.samples = this.samples.slice(-this.MAX_SAMPLES); + } + + // Log growth info if we have previous sample + if (this.samples.length >= 2) { + const prevSample = this.samples[this.samples.length - 2]; + const growth = currentMax - (prevSample?.maxDepositId || 0); + const timeElapsed = (now - (prevSample?.timestamp || 0)) / 1000 / 60; // minutes + const growthPerMinute = growth / timeElapsed; + + this.simpleLogger.critical( + `📊 Deposit growth: +${growth} deposits in ${timeElapsed.toFixed(1)}min (${growthPerMinute.toFixed(2)}/min)` + ); + } + + } catch (error) { + this.simpleLogger.error('Failed to take deposit sample', error); + } + } + } + + /** + * Find the current maximum deposit ID by scanning + */ + private async findCurrentMaxDepositId(): Promise { + // Start from the last known position if we have samples + const startId = this.samples.length > 0 + ? Math.max(1, (this.samples[this.samples.length - 1]?.maxDepositId || 0) - 5) // Go back 5 for safety + : 1; + + let currentId = startId; + let emptyCounter = 0; + const MAX_EMPTY_TO_STOP = 10; + let lastValidId = startId; + + while (emptyCounter < MAX_EMPTY_TO_STOP) { + try { + const deposit = await this.contract.deposits?.(currentId); + + const owner = deposit[0] || ethers.ZeroAddress; + const amount = deposit[1] || BigInt(0); + const earningPower = deposit[2] || BigInt(0); + const delegatee = deposit[3] || ethers.ZeroAddress; + + const isCompletelyEmpty = + owner === ethers.ZeroAddress && + amount === BigInt(0) && + earningPower === BigInt(0) && + delegatee === ethers.ZeroAddress; + + if (isCompletelyEmpty) { + emptyCounter++; + } else { + emptyCounter = 0; // Reset counter when we find a deposit + lastValidId = currentId; + } + + currentId++; + + } catch (error) { + // If we hit an error, assume we've reached the end + break; + } + } + + return lastValidId; + } + + /** + * Extrapolate maximum deposit ID based on recent growth trends + */ + private extrapolateMaxDepositId(currentTime: number): number { + if (this.samples.length < 2) { + return this.samples[0]?.maxDepositId || 1; + } + + // Use the most recent two samples for linear extrapolation + const latestSample = this.samples[this.samples.length - 1]; + const previousSample = this.samples[this.samples.length - 2]; + + const timeDiff = (latestSample?.timestamp || 0) - (previousSample?.timestamp || 0); // milliseconds + const depositDiff = (latestSample?.maxDepositId || 0) - (previousSample?.maxDepositId || 0); + + // Calculate growth rate (deposits per millisecond) + const growthRate = depositDiff / timeDiff; + + // Extrapolate based on time elapsed since last sample + const timeElapsed = currentTime - (latestSample?.timestamp || 0); + const predictedGrowth = growthRate * timeElapsed; + + return Math.ceil((latestSample?.maxDepositId || 0) + predictedGrowth); + } + + /** + * Get deposit count statistics for monitoring + */ + getStats(): { + currentScanLimit: number; + lastKnownMax: number; + samples: number; + avgGrowthPerMinute: number; + lastSampleMinutesAgo: number; + } { + const now = Date.now(); + const currentScanLimit = this.samples.length >= 2 + ? this.extrapolateMaxDepositId(now) + this.SAFETY_BUFFER + : (this.samples[0]?.maxDepositId || 0) + this.SAFETY_BUFFER; + + const lastKnownMax = this.samples.length > 0 + ? this.samples[this.samples.length - 1]?.maxDepositId || 0 + : 0; + + // Calculate average growth rate from all samples + let avgGrowthPerMinute = 0; + if (this.samples.length >= 2) { + const firstSample = this.samples[0]; + const lastSample = this.samples[this.samples.length - 1]; + const totalGrowth = (lastSample?.maxDepositId || 0) - (firstSample?.maxDepositId || 0); + const totalTime = ((lastSample?.timestamp || 0) - (firstSample?.timestamp || 0)) / 1000 / 60; // minutes + avgGrowthPerMinute = totalTime > 0 ? totalGrowth / totalTime : 0; + } + + const lastSampleMinutesAgo = this.samples.length > 0 + ? (now - (this.samples[this.samples.length - 1]?.timestamp || 0)) / 1000 / 60 + : Infinity; + + return { + currentScanLimit, + lastKnownMax, + samples: this.samples.length, + avgGrowthPerMinute, + lastSampleMinutesAgo + }; + } +} \ No newline at end of file diff --git a/src/utils/SimplifiedLogger.ts b/src/utils/SimplifiedLogger.ts new file mode 100644 index 0000000..cf57591 --- /dev/null +++ b/src/utils/SimplifiedLogger.ts @@ -0,0 +1,109 @@ +import { Logger } from '@/monitor/logging'; +import { AlchemyCUTracker } from './AlchemyCUTracker'; + +/** + * Simplified logging wrapper that reduces verbosity and shows only critical information + */ +export class SimplifiedLogger { + private readonly logger: Logger; + private readonly component: string; + private readonly cuTracker: AlchemyCUTracker; + private lastSummaryTime = Date.now(); + private batchCounter = 0; + private rewardCounter = 0; + + constructor(logger: Logger, component: string) { + this.logger = logger; + this.component = component; + this.cuTracker = new AlchemyCUTracker(logger); + } + + /** + * Log critical information only + */ + critical(message: string, data?: any): void { + this.logger.info(`[${this.component}] ${message}`, data); + } + + /** + * Track batch operations silently - only count, don't log + */ + trackBatch(operation: string, count: number, isMulticall = false): void { + this.batchCounter++; + + if (isMulticall) { + this.cuTracker.trackMulticall(count); + } else { + this.cuTracker.trackIndividualCalls(operation, count); + } + + // No logging - just tracking + } + + /** + * Track rewards silently + */ + trackRewards(nonZeroCount: number, totalValue: string): void { + this.rewardCounter += nonZeroCount; + // No logging - just counting + } + + /** + * Silent periodic maintenance + */ + maybeSummary(): void { + const now = Date.now(); + if (now - this.lastSummaryTime > 600000) { // 10 minutes + // Only log if approaching CU limits + const stats = this.cuTracker.getUsageStats(); + if (parseFloat(stats.percentUsed) > 75) { + this.logger.warn(`[${this.component}] ⚠️ High CU usage: ${stats.percentUsed}% (${stats.dailyUsage.toLocaleString()} / ${stats.dailyLimit.toLocaleString()})`); + } + this.lastSummaryTime = now; + } + } + + /** + * Stage analysis results - always log these as they're critical + */ + stageResult(stage: string, success: boolean, data: any): void { + this.logger.info(`[${this.component}] ${stage}`, { success, ...data }); + } + + /** + * Error logging - always show errors + */ + error(message: string, error: any): void { + this.logger.error(`[${this.component}] ${message}`, error); + } + + /** + * Warning logging - always show warnings + */ + warn(message: string, data?: any): void { + this.logger.warn(`[${this.component}] ${message}`, data); + } + + /** + * Debug logging - only in development + */ + debug(message: string, data?: any): void { + if (process.env.NODE_ENV === 'development') { + this.logger.debug(`[${this.component}] ${message}`, data); + } + } + + /** + * Get CU tracker for direct access + */ + getCUTracker(): AlchemyCUTracker { + return this.cuTracker; + } + + /** + * Track a specific method call + */ + trackMethod(method: string, count = 1): void { + this.cuTracker.trackMethod(method, count); + } +} \ No newline at end of file diff --git a/src/utils/multicall.ts b/src/utils/multicall.ts index 87bc4fc..68c7fcb 100644 --- a/src/utils/multicall.ts +++ b/src/utils/multicall.ts @@ -1,5 +1,15 @@ import { ethers } from 'ethers'; import { Logger } from '@/monitor/logging'; +import { CUMonitor } from './CUMonitor'; +import { SimplifiedLogger } from './SimplifiedLogger'; + +// Multicall3 contract address (deployed on all major networks) +const MULTICALL3_ADDRESS = '0xcA11bde05977b3631167028862bE2a173976CA11'; + +// Multicall3 ABI - minimal interface for aggregate3 +const MULTICALL3_ABI = [ + 'function aggregate3(tuple(address target, bool allowFailure, bytes callData)[] calls) payable returns (tuple(bool success, bytes returnData)[] returnData)' +]; /** * Utility class for batch contract calls using multicall pattern @@ -7,14 +17,37 @@ import { Logger } from '@/monitor/logging'; export class MulticallBatcher { private readonly provider: ethers.Provider; private readonly logger: Logger; + private readonly simpleLogger: SimplifiedLogger; + private readonly maxConcurrentCalls: number; + private readonly delayBetweenBatches: number; + private readonly multicall3Contract: ethers.Contract; + private readonly useNativeMulticall: boolean; + private readonly cuMonitor?: CUMonitor; - constructor(provider: ethers.Provider, logger: Logger) { + constructor( + provider: ethers.Provider, + logger: Logger, + maxConcurrentCalls = 10, + delayBetweenBatches = 100, + useNativeMulticall = true, + cuMonitor?: CUMonitor + ) { this.provider = provider; this.logger = logger; + this.simpleLogger = new SimplifiedLogger(logger, 'Multicall'); + this.maxConcurrentCalls = maxConcurrentCalls; + this.delayBetweenBatches = delayBetweenBatches; + this.useNativeMulticall = useNativeMulticall; + this.cuMonitor = cuMonitor; + this.multicall3Contract = new ethers.Contract( + MULTICALL3_ADDRESS, + MULTICALL3_ABI, + provider + ); } /** - * Batches multiple contract calls into a single RPC request using eth_call + * Batches multiple contract calls using Multicall3 for maximum CU efficiency * @param calls Array of contract calls to batch * @returns Array of decoded results */ @@ -28,8 +61,94 @@ export class MulticallBatcher { ): Promise> { if (calls.length === 0) return []; + if (this.useNativeMulticall && calls.length > 1) { + return this.batchCallsWithMulticall3(calls); + } else { + return this.batchCallsIndividually(calls); + } + } + + /** + * Uses Multicall3 contract for true batching - saves significant CUs + * Multiple eth_calls become 1 eth_call = ~26 CUs instead of 26*N CUs + */ + private async batchCallsWithMulticall3( + calls: Array<{ + contract: ethers.Contract; + method: string; + params: unknown[]; + resultDecoder: (data: string) => T; + }>, + ): Promise> { try { - // Group calls by contract for efficiency + // Prepare multicall data + const multicallData = await Promise.all( + calls.map(async (call) => ({ + target: await call.contract.getAddress(), + allowFailure: true, + callData: call.contract.interface.encodeFunctionData( + call.method, + call.params + ) + })) + ); + + // Execute single multicall + + const results = await this.multicall3Contract.aggregate3?.(multicallData); + + // Process results + const decodedResults = results.map((result: any, index: number) => { + if (!result.success) { + this.logger.warn('Individual call failed in multicall batch', { + method: calls[index]?.method, + index + }); + return null; + } + + try { + return calls[index]?.resultDecoder(result.returnData); + } catch (error) { + this.logger.warn('Failed to decode multicall result', { + method: calls[index]?.method, + index, + error: error instanceof Error ? error.message : String(error) + }); + return null; + } + }); + + // Track CU usage with accurate calculation + if (this.cuMonitor) { + this.cuMonitor.trackBatchOperation('multicall_batch', calls.length, true); + } + + const successCount = decodedResults.filter((r: any) => r !== null).length; + this.simpleLogger.trackBatch('multicall3', calls.length, true); + + // No logging for partial failures + + return decodedResults; + } catch (error) { + // Silently fallback to individual calls - no logging needed + return this.batchCallsIndividually(calls); + } + } + + /** + * Fallback method using individual calls with rate limiting + */ + private async batchCallsIndividually( + calls: Array<{ + contract: ethers.Contract; + method: string; + params: unknown[]; + resultDecoder: (data: string) => T; + }>, + ): Promise> { + try { + // Create individual call promises const callPromises = calls.map(async (call) => { try { // Encode the function call @@ -47,7 +166,8 @@ export class MulticallBatcher { // Decode the result return call.resultDecoder(result); } catch (error) { - this.logger.warn('Individual call failed in batch', { + // Only log individual failures in debug mode + this.simpleLogger.debug('Individual call failed', { method: call.method, error: error instanceof Error ? error.message : String(error), }); @@ -55,14 +175,18 @@ export class MulticallBatcher { } }); - // Execute all calls in parallel - const results = await Promise.all(callPromises); + // Execute calls with rate limiting + const results = await this.executeWithRateLimit(callPromises); - this.logger.info('Batch call completed', { - totalCalls: calls.length, - successfulCalls: results.filter((r) => r !== null).length, - failedCalls: results.filter((r) => r === null).length, - }); + // Track CU usage for individual calls + if (this.cuMonitor) { + this.cuMonitor.trackBatchOperation('eth_call', calls.length, false); + } + + const successCount = results.filter((r) => r !== null).length; + this.simpleLogger.trackBatch('eth_call', calls.length, false); + + // No logging for partial failures return results; } catch (error) { @@ -157,4 +281,43 @@ export class MulticallBatcher { return this.batchCalls(calls); } + + /** + * Executes promises with rate limiting to avoid overwhelming the RPC provider + * @param promises Array of promises to execute + * @returns Array of results + */ + private async executeWithRateLimit( + promises: Promise[] + ): Promise { + const results: T[] = []; + + // Process in batches to avoid overwhelming the RPC provider + for (let i = 0; i < promises.length; i += this.maxConcurrentCalls) { + const batch = promises.slice(i, i + this.maxConcurrentCalls); + + try { + const batchResults = await Promise.all(batch); + results.push(...batchResults); + + // Add delay between batches if there are more to process + if (i + this.maxConcurrentCalls < promises.length) { + await new Promise(resolve => setTimeout(resolve, this.delayBetweenBatches)); + } + } catch (error) { + this.logger.warn('Batch execution failed, retrying with exponential backoff', { + batchStart: i, + batchSize: batch.length, + error: error instanceof Error ? error.message : String(error) + }); + + // Retry with exponential backoff + await new Promise(resolve => setTimeout(resolve, this.delayBetweenBatches * 2)); + const batchResults = await Promise.all(batch); + results.push(...batchResults); + } + } + + return results; + } }