From e5e9d78ef7743f77c0372640592276acfa923623 Mon Sep 17 00:00:00 2001 From: seroxdesign Date: Wed, 30 Jul 2025 05:34:10 -0400 Subject: [PATCH 1/3] c --- .gitignore | 2 ++ .../strategies/GovLstProfitabilityEngine.ts | 34 +++++++++++-------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index bcbcfb3..fb81da5 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,5 @@ pnpm-debug.log* .DS_Store Thumbs.db .tasks + +CLAUDE.md \ No newline at end of file diff --git a/src/profitability/strategies/GovLstProfitabilityEngine.ts b/src/profitability/strategies/GovLstProfitabilityEngine.ts index 72a2926..eae2dc1 100644 --- a/src/profitability/strategies/GovLstProfitabilityEngine.ts +++ b/src/profitability/strategies/GovLstProfitabilityEngine.ts @@ -485,29 +485,35 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { 'Fetching payout amount', ); - const gasCost = await this.getContractDataWithRetry( - () => this.estimateGasCostInRewardToken(), - 'Estimating gas cost', - ); + // Calculate realistic gas cost based on actual usage patterns (600k gas) + const REALISTIC_GAS_UNITS = BigInt(600000); // 600k gas based on production data + const gasPrice = await this.getGasPriceWithBuffer(); + const gasCostWei = gasPrice * REALISTIC_GAS_UNITS; + const realisticGasCost = await this.convertGasCostToRewardTokens(gasCostWei); + + this.logger.info('Using realistic gas estimate for Stage 1 threshold', { + gasUnits: REALISTIC_GAS_UNITS.toString(), + gasPrice: ethers.formatUnits(gasPrice, 'gwei'), + gasCostWei: gasCostWei.toString(), + gasCostInTokens: realisticGasCost.toString(), + gasCostInTokensFormatted: ethers.formatEther(realisticGasCost), + }); // We'll calculate enhanced gas cost and actualGasEstimate later, after we know total rewards - let enhancedGasCost = gasCost; - let actualGasEstimate = GAS_CONSTANTS.FALLBACK_GAS_ESTIMATE; + let enhancedGasCost = realisticGasCost; + let actualGasEstimate = REALISTIC_GAS_UNITS; const profitMargin = this.config.minProfitMargin; // Get all deposit IDs const depositIds = normalizedDeposits.map((d) => d.deposit_id); - // Calculate optimal threshold based on payout amount, gas cost and profit margin percentage + // Calculate optimal threshold based on payout amount and realistic gas cost const effectiveGasCost = CONFIG.profitability.includeGasCost - ? enhancedGasCost + ? realisticGasCost : BigInt(0); - // Add gas buffer to account for estimation inaccuracy before simulation - // Buffer of 100 tokens to ensure we have enough rewards for actual gas costs - const gasBuffer = ethers.parseEther('100'); - const baseAmount = payoutAmount + effectiveGasCost + gasBuffer; + const baseAmount = payoutAmount + effectiveGasCost; // Scale profit margin based on deposit count const depositCount = BigInt(depositIds.length); @@ -620,8 +626,8 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { }); // First simulation to get accurate gas cost - let firstSimulationGasCost = gasCost; - let firstSimulationGasUnits = GAS_CONSTANTS.FALLBACK_GAS_ESTIMATE; + let firstSimulationGasCost = realisticGasCost; + let firstSimulationGasUnits = REALISTIC_GAS_UNITS; if (this.simulationService && this.activeBin.total_rewards >= payoutAmount) { try { From 354a6d4f00e911cb0c568db3d6a05f443496ee37 Mon Sep 17 00:00:00 2001 From: seroxdesign Date: Wed, 30 Jul 2025 05:37:34 -0400 Subject: [PATCH 2/3] some gas calcs, format, lint --- DEPLOYMENT.md | 8 +- SYSTEM_ARCHITECTURE.md | 77 +++-- src/INDEX_README.md | 77 +++-- src/configuration/README.md | 30 +- src/configuration/index.ts | 3 +- src/database/README.md | 64 ++-- src/executor/README.md | 65 ++-- src/executor/strategies/RelayerExecutor.ts | 84 +++-- src/executor/strategies/helpers/helpers.ts | 16 +- .../strategies/helpers/simulation-helpers.ts | 30 +- src/monitor/README.md | 84 +++-- src/prices/GasCostEstimator.ts | 4 +- src/prices/README.md | 75 +++-- src/profitability/README.md | 60 ++-- .../strategies/GovLstProfitabilityEngine.ts | 300 +++++++++++------- src/simulation/README.md | 167 +++++----- src/simulation/index.ts | 45 ++- src/tests/README.md | 99 +++--- 18 files changed, 800 insertions(+), 488 deletions(-) diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index f810585..c53e24a 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -91,11 +91,13 @@ GOVLST_MAX_BATCH_SIZE=10 ## Supabase Setup 1. **Create a new Supabase project** + - Go to [supabase.com](https://supabase.com) - Create a new project - Note your project URL and anon key 2. **Run database migrations** + ```bash npm run migrate ``` @@ -190,6 +192,7 @@ mkdir -p logs PM2 is a production process manager that keeps your application running continuously, restarts it if it crashes, and provides monitoring capabilities. **Why PM2 is Essential:** + - **Auto-restart**: Automatically restarts your app if it crashes - **Load balancing**: Can run multiple instances - **Monitoring**: Built-in monitoring and logging @@ -311,15 +314,18 @@ pm2 restart staker-bots ### Common Issues 1. **Environment Variables Not Loading** + - Ensure `.env` file is in the root directory - Check file permissions: `chmod 644 .env` 2. **Database Connection Issues** + - Verify Supabase URL and key - Check network connectivity - Ensure migrations have run successfully 3. **PM2 Not Starting on Boot** + - Run `pm2 startup` and follow instructions - Ensure `pm2 save` was run after starting processes @@ -347,4 +353,4 @@ If you encounter issues: --- -This deployment guide should get your Staker Bots application running in production. The combination of Digital Ocean's reliable infrastructure and PM2's process management provides a robust foundation for your application. \ No newline at end of file +This deployment guide should get your Staker Bots application running in production. The combination of Digital Ocean's reliable infrastructure and PM2's process management provides a robust foundation for your application. diff --git a/SYSTEM_ARCHITECTURE.md b/SYSTEM_ARCHITECTURE.md index 3384d1c..2867334 100644 --- a/SYSTEM_ARCHITECTURE.md +++ b/SYSTEM_ARCHITECTURE.md @@ -15,16 +15,16 @@ graph TB DEFENDER[OpenZeppelin Defender] SUPABASE[Supabase Database] end - + subgraph "Staker-Bots System" ENTRY[Application Entry Point] - + subgraph "Core Modules" MONITOR[Monitor Module] PROFIT[Profitability Module] EXECUTOR[Executor Module] end - + subgraph "Support Modules" CONFIG[Configuration Module] DATABASE[Database Module] @@ -33,16 +33,16 @@ graph TB TESTS[Tests Module] end end - + subgraph "Storage" JSON_DB[JSON Database] CLOUD_DB[Supabase Cloud DB] end - + ENTRY --> MONITOR ENTRY --> PROFIT ENTRY --> EXECUTOR - + MONITOR --> CONFIG MONITOR --> DATABASE PROFIT --> CONFIG @@ -52,7 +52,7 @@ graph TB EXECUTOR --> CONFIG EXECUTOR --> DATABASE EXECUTOR --> SIMULATION - + DATABASE --> JSON_DB DATABASE --> CLOUD_DB PRICES --> CMC @@ -60,7 +60,7 @@ graph TB EXECUTOR --> DEFENDER EXECUTOR --> BLOCKCHAIN MONITOR --> BLOCKCHAIN - + CLOUD_DB --> SUPABASE ``` @@ -75,13 +75,13 @@ sequenceDiagram participant SIM as Simulation participant EXEC as Executor participant USR as Users - + loop Continuous Monitoring BC->>MON: Events (deposits, withdrawals) MON->>DB: Store deposit data MON->>DB: Add to processing queue end - + loop Profitability Analysis PROFIT->>DB: Get pending deposits PROFIT->>BC: Fetch unclaimed rewards @@ -94,7 +94,7 @@ sequenceDiagram PROFIT->>DB: Mark as skipped end end - + loop Transaction Execution EXEC->>SIM: Pre-execution validation EXEC->>BC: Submit transaction @@ -110,12 +110,14 @@ sequenceDiagram **Purpose**: Continuously watches blockchain for staking-related events. **Key Features**: + - Event polling with checkpoint management - Deposit lifecycle tracking - Delegation change monitoring - Reorg-safe block processing **Data Flow**: + ``` Blockchain Events → Event Processing → Database Storage → Processing Queue ``` @@ -125,12 +127,14 @@ Blockchain Events → Event Processing → Database Storage → Processing Queue **Purpose**: Analyzes deposits to determine profitable reward claiming opportunities. **Key Features**: + - Two-stage gas estimation - Batch optimization - Price-aware calculations - Margin scaling **Algorithm**: + ``` 1. Fetch unclaimed rewards 2. Calculate gas costs in reward tokens @@ -144,12 +148,14 @@ Blockchain Events → Event Processing → Database Storage → Processing Queue **Purpose**: Submits and monitors reward claim transactions. **Key Features**: + - Multiple execution strategies (Wallet, Defender) - Transaction queue management - Gas optimization - Retry logic **Execution Flow**: + ``` Validation → Simulation → Submission → Monitoring → Confirmation ``` @@ -159,6 +165,7 @@ Validation → Simulation → Submission → Monitoring → Confirmation **Purpose**: Centralized configuration management with environment-based settings. **Components**: + - Environment variable loading - Type-safe configuration objects - Contract ABIs @@ -169,6 +176,7 @@ Validation → Simulation → Submission → Monitoring → Confirmation **Purpose**: Flexible data persistence with multiple backend support. **Features**: + - Supabase and JSON storage - Automatic fallback - Transaction support @@ -179,6 +187,7 @@ Validation → Simulation → Submission → Monitoring → Confirmation **Purpose**: Real-time price feeds for gas cost calculations. **Components**: + - CoinMarketCap integration - Price caching - Currency conversion @@ -189,6 +198,7 @@ Validation → Simulation → Submission → Monitoring → Confirmation **Purpose**: Transaction simulation using Tenderly for accurate gas estimation. **Features**: + - Pre-execution validation - Gas estimation - Error detection @@ -199,6 +209,7 @@ Validation → Simulation → Submission → Monitoring → Confirmation **Purpose**: Comprehensive testing suite for system validation. **Coverage**: + - Unit tests for components - Integration tests for workflows - Mock providers and contracts @@ -218,7 +229,7 @@ erDiagram timestamp created_at timestamp updated_at } - + PROCESSING_QUEUE { string id PK string deposit_id FK @@ -229,7 +240,7 @@ erDiagram timestamp created_at timestamp updated_at } - + TRANSACTION_QUEUE { string id PK string deposit_id FK @@ -240,7 +251,7 @@ erDiagram timestamp created_at timestamp updated_at } - + TRANSACTION_DETAILS { string id PK string transaction_id UK @@ -251,14 +262,14 @@ erDiagram timestamp created_at timestamp updated_at } - + CHECKPOINTS { string component_type PK int last_block_number string block_hash timestamp last_update } - + ERROR_LOGS { string id PK string service_name @@ -267,7 +278,7 @@ erDiagram json context timestamp created_at } - + DEPOSITS ||--o{ PROCESSING_QUEUE : processes DEPOSITS ||--o{ TRANSACTION_QUEUE : executes TRANSACTION_QUEUE ||--o| TRANSACTION_DETAILS : details @@ -276,6 +287,7 @@ erDiagram ## Technology Stack ### Core Technologies + - **Runtime**: Node.js with TypeScript - **Blockchain**: Ethers.js v6 - **Database**: Supabase (PostgreSQL) with JSON fallback @@ -284,6 +296,7 @@ erDiagram - **Price Feeds**: CoinMarketCap API ### External Services + - **Blockchain RPC**: Ethereum mainnet/testnet - **Gas Simulation**: Tenderly - **Price Oracle**: CoinMarketCap @@ -291,6 +304,7 @@ erDiagram - **Database**: Supabase Cloud ### Development Tools + - **Type Safety**: Full TypeScript coverage - **Code Quality**: ESLint, Prettier - **Testing**: Unit and integration tests @@ -309,37 +323,37 @@ graph TB DB[(Supabase DB)] REDIS[(Redis Cache)] end - + subgraph "External APIs" RPC[Ethereum RPC] TEND[Tenderly] CMC[CoinMarketCap] DEF[Defender] end - + subgraph "Monitoring" LOGS[Centralized Logging] METRICS[Metrics Collection] ALERTS[Alert Manager] end - + LB --> APP1 LB --> APP2 APP1 --> DB APP2 --> DB APP1 --> REDIS APP2 --> REDIS - + APP1 --> RPC APP1 --> TEND APP1 --> CMC APP1 --> DEF - + APP2 --> RPC APP2 --> TEND APP2 --> CMC APP2 --> DEF - + APP1 --> LOGS APP2 --> LOGS APP1 --> METRICS @@ -369,21 +383,25 @@ COMPONENTS=all npm start ## Security Considerations ### Private Key Management + - Environment variable storage - Support for hardware security modules - Rotation capabilities ### API Security + - Rate limiting for external APIs - Error handling for API failures - Credential rotation ### Transaction Security + - Pre-execution simulation - Gas limit enforcement - Slippage protection ### Operational Security + - Error logging without sensitive data - Secure database connections - Network isolation @@ -391,16 +409,19 @@ COMPONENTS=all npm start ## Performance Characteristics ### Throughput + - **Event Processing**: 1000+ events/minute - **Profitability Checks**: 100+ deposits/minute - **Transaction Execution**: 10+ tx/minute (limited by gas) ### Latency + - **Event Detection**: < 30 seconds - **Profitability Analysis**: < 10 seconds - **Transaction Submission**: < 5 seconds ### Resource Usage + - **Memory**: 256MB - 1GB depending on queue size - **CPU**: Low, event-driven architecture - **Network**: Moderate, API calls for prices and simulation @@ -409,24 +430,28 @@ COMPONENTS=all npm start ## Monitoring and Observability ### Health Checks + - Component status monitoring - Database connectivity - External API availability - Blockchain sync status ### Metrics + - Transaction success rates - Profitability accuracy - Processing latency - Error rates by component ### Alerting + - Failed transactions - Component failures - API rate limits - Balance thresholds ### Logging + - Structured JSON logging - Component-specific log levels - Error context preservation @@ -435,26 +460,30 @@ COMPONENTS=all npm start ## Scalability and Reliability ### Horizontal Scaling + - Stateless component design - Database-backed coordination - Load balancing support ### Fault Tolerance + - Graceful error handling - Automatic retry mechanisms - Fallback strategies - Circuit breakers ### Data Consistency + - Checkpoint-based processing - Atomic transactions - Idempotent operations - Reorg handling ### Recovery Mechanisms + - Automatic restart on failures - State restoration from checkpoints - Queue replay capabilities - Manual intervention tools -This architecture provides a robust, scalable, and maintainable system for automated staking reward management while ensuring economic viability and user safety. \ No newline at end of file +This architecture provides a robust, scalable, and maintainable system for automated staking reward management while ensuring economic viability and user safety. diff --git a/src/INDEX_README.md b/src/INDEX_README.md index 32db57d..2923750 100644 --- a/src/INDEX_README.md +++ b/src/INDEX_README.md @@ -10,55 +10,55 @@ The main `index.ts` file serves as the application entry point and orchestrates graph TB subgraph "Application Entry Point" MAIN[index.ts] - + subgraph "Initialization" DB_INIT[Database Init] LOGGER_INIT[Logger Init] CONFIG_LOAD[Config Load] ERROR_INIT[Error Logger Init] end - + subgraph "Component Management" COMP_SELECT[Component Selection] HEALTH_CHECK[Health Monitoring] SHUTDOWN[Graceful Shutdown] end end - + subgraph "System Components" MONITOR[StakerMonitor] EXECUTOR[ExecutorWrapper] PROFIT[ProfitabilityEngine] end - + subgraph "Infrastructure" DATABASE[Database] PROVIDER[Blockchain Provider] CONTRACTS[Smart Contracts] ERROR_LOG[Error Logging] end - + MAIN --> INITIALIZATION MAIN --> COMP_SELECT MAIN --> HEALTH_CHECK MAIN --> SHUTDOWN - + COMP_SELECT --> MONITOR COMP_SELECT --> EXECUTOR COMP_SELECT --> PROFIT - + MONITOR --> DATABASE EXECUTOR --> DATABASE PROFIT --> DATABASE - + MONITOR --> PROVIDER EXECUTOR --> PROVIDER PROFIT --> PROVIDER - + MONITOR --> CONTRACTS EXECUTOR --> CONTRACTS PROFIT --> CONTRACTS - + MONITOR --> ERROR_LOG EXECUTOR --> ERROR_LOG PROFIT --> ERROR_LOG @@ -74,13 +74,13 @@ sequenceDiagram participant MON as Monitor participant EXEC as Executor participant PROFIT as Profitability - + ENV->>MAIN: Load Configuration MAIN->>DB: Initialize Database MAIN->>MAIN: Create Loggers MAIN->>MAIN: Setup Error Handlers MAIN->>DB: Ensure Checkpoints - + rect rgb(200, 220, 240) note right of MAIN: Component Initialization MAIN->>MON: Initialize Monitor @@ -90,7 +90,7 @@ sequenceDiagram MAIN->>PROFIT: Initialize Profitability PROFIT->>PROFIT: Start Analysis Loop end - + MAIN->>MAIN: Setup Health Checks MAIN->>MAIN: Register Shutdown Handlers MAIN->>ENV: Application Ready @@ -114,13 +114,23 @@ const database = new DatabaseWrapper({ ```typescript const loggers = { main: new ConsoleLogger('info'), - monitor: new ConsoleLogger('info', { color: '\x1b[34m', prefix: '[Monitor]' }), - profitability: new ConsoleLogger('info', { color: '\x1b[32m', prefix: '[Profitability]' }), - executor: new ConsoleLogger('info', { color: '\x1b[33m', prefix: '[Executor]' }) + monitor: new ConsoleLogger('info', { + color: '\x1b[34m', + prefix: '[Monitor]', + }), + profitability: new ConsoleLogger('info', { + color: '\x1b[32m', + prefix: '[Profitability]', + }), + executor: new ConsoleLogger('info', { + color: '\x1b[33m', + prefix: '[Executor]', + }), }; ``` **Features**: + - Color-coded output for visual distinction - Component-specific prefixes - Configurable log levels @@ -133,7 +143,7 @@ const errorLoggers = { main: createErrorLogger('main-service', database), monitor: createErrorLogger('monitor-service', database), profitability: createErrorLogger('profitability-service', database), - executor: createErrorLogger('executor-service', database) + executor: createErrorLogger('executor-service', database), }; ``` @@ -144,7 +154,9 @@ const errorLoggers = { ```typescript function createProvider() { if (!CONFIG.monitor.rpcUrl) { - throw new Error('RPC URL is not configured. Please set RPC_URL environment variable.'); + throw new Error( + 'RPC URL is not configured. Please set RPC_URL environment variable.', + ); } return new ethers.JsonRpcProvider(CONFIG.monitor.rpcUrl); } @@ -157,8 +169,10 @@ function createProvider() { ### Environment-Based Configuration ```typescript -const rawComponents = process.env.COMPONENTS?.split(',').map(c => c.trim().toLowerCase()) || ['all']; -const componentsToRun = rawComponents.includes('all') +const rawComponents = process.env.COMPONENTS?.split(',').map((c) => + c.trim().toLowerCase(), +) || ['all']; +const componentsToRun = rawComponents.includes('all') ? ['monitor', 'executor', 'profitability'] : rawComponents; ``` @@ -194,12 +208,13 @@ graph TD C --> P C --> B C --> A - + A -.-> B B -.-> C ``` **Dependency Rules**: + - All components require Database - Executor and Profitability require Provider - Profitability requires Executor @@ -269,7 +284,7 @@ sequenceDiagram participant PROFIT as Profitability participant EXEC as Executor participant MON as Monitor - + SIG->>MAIN: SIGINT/SIGTERM MAIN->>PROFIT: Stop Engine PROFIT->>PROFIT: Finish Current Analysis @@ -283,20 +298,20 @@ sequenceDiagram ```typescript async function shutdown(signal: string) { mainLogger.info(`Received ${signal}. Starting graceful shutdown...`); - + // Stop components in reverse order if (runningComponents.profitabilityEngine) { await runningComponents.profitabilityEngine.stop(); } - + if (runningComponents.executor) { await runningComponents.executor.stop(); } - + if (runningComponents.monitor) { await runningComponents.monitor.stop(); } - + process.exit(0); } ``` @@ -381,10 +396,10 @@ try { async function ensureCheckpointsAtStartBlock(database, logger, errorLogger) { const startBlock = CONFIG.monitor.startBlock; const componentTypes = ['staker-monitor', 'executor', 'profitability-engine']; - + for (const componentType of componentTypes) { const checkpoint = await database.getCheckpoint(componentType); - + if (!checkpoint || checkpoint.last_block_number < startBlock) { await database.updateCheckpoint({ component_type: componentType, @@ -402,6 +417,7 @@ async function ensureCheckpointsAtStartBlock(database, logger, errorLogger) { ## Deployment Modes ### Development Mode + ```bash NODE_ENV=development COMPONENTS=all @@ -410,6 +426,7 @@ DB=json ``` ### Production Mode + ```bash NODE_ENV=production COMPONENTS=all @@ -419,12 +436,14 @@ TENDERLY_USE_SIMULATE=true ``` ### Monitor-Only Mode + ```bash COMPONENTS=monitor DB=supabase ``` ### Executor-Only Mode + ```bash COMPONENTS=executor,profitability EXECUTOR_TYPE=defender @@ -439,4 +458,4 @@ EXECUTOR_TYPE=defender 5. **Configuration Validation**: Fail fast on missing critical config 6. **Structured Logging**: Consistent log format across components -This entry point provides a robust foundation for running the staker-bots system in various deployment scenarios while maintaining reliability and observability. \ No newline at end of file +This entry point provides a robust foundation for running the staker-bots system in various deployment scenarios while maintaining reliability and observability. diff --git a/src/configuration/README.md b/src/configuration/README.md index cf39733..3f7f982 100644 --- a/src/configuration/README.md +++ b/src/configuration/README.md @@ -12,7 +12,7 @@ graph TB ENV[.env file] PROC[Process Environment] end - + subgraph "Configuration Module" INDEX[index.ts] CONST[constants.ts] @@ -21,14 +21,14 @@ graph TB HELPERS[helpers.ts] LOGGER[errorLogger.ts] end - + subgraph "Configuration Objects" CONFIG[CONFIG Object] CONSTANTS[CONSTANTS Object] ABI[Contract ABIs] ERR[Error Classes] end - + subgraph "Consumers" MONITOR[Monitor Module] EXEC[Executor Module] @@ -37,21 +37,21 @@ graph TB PRICE[Prices Module] SIM[Simulation Module] end - + ENV --> INDEX PROC --> INDEX INDEX --> CONFIG CONST --> CONSTANTS ABIS --> ABI ERRORS --> ERR - + CONFIG --> MONITOR CONFIG --> EXEC CONFIG --> PROFIT CONFIG --> DB CONFIG --> PRICE CONFIG --> SIM - + CONSTANTS --> MONITOR CONSTANTS --> EXEC CONSTANTS --> PROFIT @@ -69,14 +69,17 @@ graph TB **Purpose**: Loads environment variables and creates the main CONFIG object. **Input**: + - Environment variables from `.env` file - Process environment variables **Output**: + - `CONFIG` object with all system configuration - `createProvider()` helper function **Key Sections**: + - **supabase**: Database connection settings - **monitor**: Event monitoring configuration - **executor**: Transaction execution settings @@ -91,6 +94,7 @@ graph TB **Purpose**: Defines immutable system constants organized by domain. **Categories**: + - **NETWORK_CONSTANTS**: Chain and network parameters - **TIME_CONSTANTS**: Time durations in seconds - **GAS_CONSTANTS**: Gas-related settings @@ -106,6 +110,7 @@ graph TB **Purpose**: Stores Application Binary Interfaces for smart contracts. **Contracts**: + - Staker contract ABI - GovLst contract ABI - ERC20 token ABI @@ -116,6 +121,7 @@ graph TB **Purpose**: Defines custom error types for better error handling. **Error Types**: + - `ConfigurationError`: Configuration validation failures - `GasEstimationError`: Gas calculation errors - `ProfitabilityError`: Profitability check failures @@ -128,6 +134,7 @@ graph TB **Purpose**: Utility functions for configuration validation and transformation. **Functions**: + - `validateAddress()`: Ethereum address validation - `parseWei()`: Safe BigInt parsing for Wei values - `getRequiredEnv()`: Required environment variable getter @@ -138,6 +145,7 @@ graph TB **Purpose**: Centralized error logging with context. **Features**: + - Structured error logging - Context preservation - Integration with monitoring services @@ -150,7 +158,7 @@ sequenceDiagram participant ENV as Environment participant CFG as Configuration participant MOD as Other Modules - + ENV->>CFG: Load .env variables CFG->>CFG: Validate required vars CFG->>CFG: Parse and transform values @@ -166,7 +174,7 @@ sequenceDiagram ```mermaid graph TD CONFIG[Main CONFIG Object] - + CONFIG --> SUPABASE[Supabase Config] CONFIG --> MONITOR[Monitor Config] CONFIG --> EXECUTOR[Executor Config] @@ -175,7 +183,7 @@ graph TD CONFIG --> PROFIT[Profitability Config] CONFIG --> GOVLST[GovLst Config] CONFIG --> TENDERLY[Tenderly Config] - + EXECUTOR --> SWAP[Swap Config] DEFENDER --> RELAYER[Relayer Config] RELAYER --> GAS_POLICY[Gas Policy] @@ -185,12 +193,14 @@ graph TD ## Environment Variables ### Required Variables + - `RPC_URL`: Ethereum RPC endpoint - `STAKER_CONTRACT_ADDRESS`: Staker contract address - `CHAIN_ID`: Ethereum chain ID - `LST_ADDRESS`: Liquid staking token address ### Optional Variables + - `SUPABASE_URL`: Database URL - `SUPABASE_KEY`: Database authentication key - `PRIVATE_KEY`: Executor wallet private key @@ -249,4 +259,4 @@ if (isNaN(parseInt(value))) { - **Provider Creation**: Use createProvider() helper - **Logging**: Integrate with ErrorLogger -This module forms the foundation of the entire system, ensuring consistent configuration across all components. \ No newline at end of file +This module forms the foundation of the entire system, ensuring consistent configuration across all components. diff --git a/src/configuration/index.ts b/src/configuration/index.ts index 3b02afb..431fd48 100644 --- a/src/configuration/index.ts +++ b/src/configuration/index.ts @@ -147,7 +147,8 @@ export const CONFIG = { }, tenderly: { useSimulation: process.env.TENDERLY_USE_SIMULATE === 'true', - skipSimulationBelowPayout: process.env.TENDERLY_SKIP_SIMULATION_BELOW_PAYOUT !== 'false', + skipSimulationBelowPayout: + process.env.TENDERLY_SKIP_SIMULATION_BELOW_PAYOUT !== 'false', accessKey: process.env.TENDERLY_ACCESS_KEY || '', accountName: process.env.TENDERLY_ACCOUNT_NAME || '', projectName: process.env.TENDERLY_PROJECT_NAME || '', diff --git a/src/database/README.md b/src/database/README.md index 1350aa4..3f3cd68 100644 --- a/src/database/README.md +++ b/src/database/README.md @@ -12,16 +12,16 @@ graph TB SUPABASE[Supabase Cloud] FILESYSTEM[Local Filesystem] end - + subgraph "Database Module" WRAPPER[DatabaseWrapper] IFACE[IDatabase Interface] - + subgraph "Implementations" SUPA_IMPL[Supabase Implementation] JSON_IMPL[JsonDatabase] end - + subgraph "Domain Modules" DEPOSITS[deposits.ts] CHECKPOINTS[checkpoints.ts] @@ -32,17 +32,17 @@ graph TB ERRORS[errors.ts] end end - + subgraph "Consumers" MONITOR[Monitor Module] EXECUTOR[Executor Module] PROFIT[Profitability Module] end - + WRAPPER --> IFACE WRAPPER --> SUPA_IMPL WRAPPER --> JSON_IMPL - + SUPA_IMPL --> DEPOSITS SUPA_IMPL --> CHECKPOINTS SUPA_IMPL --> PROC_QUEUE @@ -50,7 +50,7 @@ graph TB SUPA_IMPL --> TX_DETAILS SUPA_IMPL --> GOVLST SUPA_IMPL --> ERRORS - + DEPOSITS --> SUPABASE CHECKPOINTS --> SUPABASE PROC_QUEUE --> SUPABASE @@ -58,9 +58,9 @@ graph TB TX_DETAILS --> SUPABASE GOVLST --> SUPABASE ERRORS --> SUPABASE - + JSON_IMPL --> FILESYSTEM - + MONITOR --> WRAPPER EXECUTOR --> WRAPPER PROFIT --> WRAPPER @@ -81,7 +81,7 @@ erDiagram timestamp created_at timestamp updated_at } - + PROCESSING_QUEUE { string id PK string deposit_id FK @@ -93,7 +93,7 @@ erDiagram timestamp created_at timestamp updated_at } - + TRANSACTION_QUEUE { string id PK string deposit_id FK @@ -107,7 +107,7 @@ erDiagram timestamp created_at timestamp updated_at } - + TRANSACTION_DETAILS { string id PK string transaction_id UK @@ -122,14 +122,14 @@ erDiagram timestamp created_at timestamp updated_at } - + CHECKPOINTS { string component_type PK int last_block_number string block_hash timestamp last_update } - + GOVLST_CLAIM_HISTORY { string id PK string govlst_address @@ -140,7 +140,7 @@ erDiagram string transaction_hash timestamp created_at } - + ERROR_LOGS { string id PK string service_name @@ -151,7 +151,7 @@ erDiagram json context timestamp created_at } - + DEPOSITS ||--o{ PROCESSING_QUEUE : has DEPOSITS ||--o{ TRANSACTION_QUEUE : has TRANSACTION_QUEUE ||--o| TRANSACTION_DETAILS : details @@ -164,6 +164,7 @@ erDiagram **Purpose**: Defines the contract for all database implementations. **Key Method Categories**: + - **Deposits**: CRUD operations for staking deposits - **Checkpoints**: Block processing state management - **Processing Queue**: Profitability check queue management @@ -177,19 +178,21 @@ erDiagram **Purpose**: Provides automatic fallback between database implementations. **Features**: + - Primary database selection (Supabase/JSON) - Automatic fallback to JSON on Supabase failure - Transparent error handling - Method wrapping for all database operations **Flow**: + ```mermaid sequenceDiagram participant Client participant Wrapper participant Supabase participant JsonDB - + Client->>Wrapper: Database operation Wrapper->>Supabase: Try operation alt Success @@ -208,6 +211,7 @@ sequenceDiagram **Purpose**: Cloud-based database storage using Supabase. **Structure**: + - `client.ts`: Supabase client initialization - `deposits.ts`: Deposit operations - `checkpoints.ts`: Checkpoint operations @@ -223,6 +227,7 @@ sequenceDiagram **Purpose**: Local file-based storage for development and fallback. **Features**: + - In-memory data with file persistence - Thread-safe write operations - Automatic file creation @@ -238,7 +243,7 @@ sequenceDiagram participant Module participant DB as DatabaseWrapper participant Store as Storage - + Module->>DB: Create/Update Entity DB->>DB: Validate Data DB->>Store: Write Operation @@ -254,7 +259,7 @@ sequenceDiagram participant Module participant DB as DatabaseWrapper participant Store as Storage - + Module->>DB: Query Request DB->>Store: Fetch Data Store->>Store: Apply Filters @@ -266,31 +271,37 @@ sequenceDiagram ## Key Tables ### 1. Deposits + - Tracks all staking deposits - Links owners, depositors, and delegatees - Stores amounts and earning power ### 2. Processing Queue + - Manages deposits awaiting profitability checks - Tracks check attempts and results - Maintains processing status ### 3. Transaction Queue + - Queues profitable claims for execution - Tracks transaction submission status - Stores gas prices and tip information ### 4. Transaction Details + - Comprehensive transaction execution records - Links multiple deposits to single transaction - Tracks profitability calculations and actual results ### 5. Checkpoints + - Maintains blockchain synchronization state - Prevents reprocessing of blocks - Handles chain reorganizations ### 6. Error Logs + - Centralized error tracking - Categorized by service and severity - Includes context and metadata @@ -298,6 +309,7 @@ sequenceDiagram ## Usage Examples ### Creating a Deposit + ```typescript await database.createDeposit({ deposit_id: '12345', @@ -305,27 +317,29 @@ await database.createDeposit({ amount: '1000000000000000000', delegatee_address: '0x...', created_at: new Date().toISOString(), - updated_at: new Date().toISOString() + updated_at: new Date().toISOString(), }); ``` ### Processing Queue Management + ```typescript // Add to queue const item = await database.createProcessingQueueItem({ deposit_id: '12345', status: ProcessingQueueStatus.PENDING, - delegatee: '0x...' + delegatee: '0x...', }); // Update status await database.updateProcessingQueueItem(item.id, { status: ProcessingQueueStatus.PROCESSING, - last_profitability_check: JSON.stringify(checkResult) + last_profitability_check: JSON.stringify(checkResult), }); ``` ### Transaction Tracking + ```typescript // Create transaction details await database.createTransactionDetails({ @@ -333,13 +347,14 @@ await database.createTransactionDetails({ deposit_ids: ['12345', '67890'], status: TransactionDetailsStatus.PENDING, min_expected_reward: '1000000000000000000', - gas_cost_estimate: '50000000000000000' + gas_cost_estimate: '50000000000000000', }); ``` ## Migration System SQL migrations are automatically applied on startup: + 1. Core tables (deposits, checkpoints) 2. Monitor tables (processing states) 3. Queue tables (transaction management) @@ -351,6 +366,7 @@ SQL migrations are automatically applied on startup: ## Error Handling The module implements comprehensive error handling: + - Automatic retry logic for transient failures - Fallback to JSON database on persistent errors - Error logging with full context @@ -372,4 +388,4 @@ The module implements comprehensive error handling: - **Query optimization**: Use specific queries over getAllX methods - **Caching**: Implement caching for frequently accessed data -This module ensures reliable data persistence across different deployment scenarios, from local development to production cloud environments. \ No newline at end of file +This module ensures reliable data persistence across different deployment scenarios, from local development to production cloud environments. diff --git a/src/executor/README.md b/src/executor/README.md index f4c1ee4..81e1626 100644 --- a/src/executor/README.md +++ b/src/executor/README.md @@ -13,23 +13,23 @@ graph TB DEFENDER[OpenZeppelin Defender] TENDERLY[Tenderly Simulation] end - + subgraph "Executor Module" WRAPPER[ExecutorWrapper] IFACE[IExecutor Interface] - + subgraph "Implementations" BASE[BaseExecutor] RELAYER[RelayerExecutor] end - + subgraph "Helpers" SIM_HELP[simulation-helpers.ts] DEF_HELP[defender-helpers.ts] GAS_HELP[helpers.ts] SWAP_HELP[token-swap-helpers.ts] end - + subgraph "Core Components" QUEUE[Transaction Queue] GAS_MGR[Gas Manager] @@ -37,30 +37,30 @@ graph TB SWAP_MGR[Swap Strategy] end end - + subgraph "Consumers" PROFIT[Profitability Module] MONITOR[Monitor Module] end - + WRAPPER --> IFACE WRAPPER --> BASE WRAPPER --> RELAYER BASE -.-> RELAYER - + BASE --> QUEUE BASE --> GAS_MGR BASE --> SIM_MGR RELAYER --> DEFENDER - + GAS_MGR --> GAS_HELP SIM_MGR --> SIM_HELP SIM_MGR --> TENDERLY RELAYER --> DEF_HELP - + BASE --> BLOCKCHAIN SWAP_MGR --> SWAP_HELP - + PROFIT --> WRAPPER MONITOR --> WRAPPER ``` @@ -72,22 +72,22 @@ stateDiagram-v2 [*] --> Validation Validation --> Queued: Valid Validation --> Rejected: Invalid - + Queued --> Simulation Simulation --> GasEstimation: Success Simulation --> Failed: Failure - + GasEstimation --> Submission Submission --> Pending: Submitted Submission --> Failed: Error - + Pending --> Monitoring Monitoring --> Confirmed: Mined Monitoring --> Failed: Reverted Monitoring --> Replacement: Stuck - + Replacement --> Pending: Resubmitted - + Confirmed --> [*] Failed --> [*] Rejected --> [*] @@ -100,6 +100,7 @@ stateDiagram-v2 **Purpose**: Factory and manager for executor implementations. **Responsibilities**: + - Creates appropriate executor instance based on configuration - Provides unified interface - Manages executor lifecycle @@ -109,6 +110,7 @@ stateDiagram-v2 **Purpose**: Direct wallet-based transaction execution. **Key Features**: + - In-memory transaction queue - Gas price optimization - Transaction simulation @@ -116,6 +118,7 @@ stateDiagram-v2 - Tip accumulation and transfer **Process Flow**: + ```mermaid sequenceDiagram participant Client @@ -123,11 +126,11 @@ sequenceDiagram participant Simulation participant Blockchain participant Database - + Client->>Executor: queueTransaction() Executor->>Executor: Validate transaction Executor->>Database: Create queue item - + loop Process Queue Executor->>Simulation: Simulate transaction Simulation-->>Executor: Gas estimate @@ -146,6 +149,7 @@ sequenceDiagram **Purpose**: OpenZeppelin Defender-based execution. **Key Features**: + - Managed gas pricing - Private mempool options - MEV protection @@ -153,6 +157,7 @@ sequenceDiagram - Automatic balance management **Defender Integration**: + ```mermaid graph LR EXEC[RelayerExecutor] --> API[Defender API] @@ -167,6 +172,7 @@ graph LR ### 4. Transaction Queue **States**: + - `QUEUED`: Awaiting processing - `PENDING`: Submitted to network - `CONFIRMED`: Successfully mined @@ -174,6 +180,7 @@ graph LR - `REPLACED`: Resubmitted with higher gas **Queue Processing**: + ```typescript // Process every 3 seconds setInterval(() => processQueue(), 3000) @@ -187,6 +194,7 @@ setInterval(() => processQueue(), 3000) ### 5. Gas Management **Gas Estimation Flow**: + ```mermaid flowchart TD A[Start] --> B{Simulation Available?} @@ -202,18 +210,19 @@ flowchart TD ``` **Gas Calculation**: + ```typescript // Base calculation -gasLimit = BASE_GAS + (depositCount * GAS_PER_DEPOSIT) +gasLimit = BASE_GAS + depositCount * GAS_PER_DEPOSIT; // With reentrancy protection (20+ deposits) if (depositCount >= 20) { - gasLimit *= 1.25 // 25% additional buffer + gasLimit *= 1.25; // 25% additional buffer } // Price calculation -gasPrice = currentGasPrice * (1 + buffer) -maxFeePerGas = baseFee * 2 + priorityFee +gasPrice = currentGasPrice * (1 + buffer); +maxFeePerGas = baseFee * 2 + priorityFee; ``` ### 6. Simulation Integration @@ -221,6 +230,7 @@ maxFeePerGas = baseFee * 2 + priorityFee **Purpose**: Pre-execution validation and accurate gas estimation. **Process**: + 1. Encode transaction data 2. Set simulation parameters 3. Run Tenderly simulation @@ -228,6 +238,7 @@ maxFeePerGas = baseFee * 2 + priorityFee 5. Apply safety buffer **Benefits**: + - Prevents failed transactions - Accurate gas estimates - Error detection before submission @@ -244,7 +255,7 @@ graph TD ANALYZE -->|Nonce Issue| RESET[Reset Nonce] ANALYZE -->|Network Error| WAIT[Wait & Retry] ANALYZE -->|Contract Error| FAIL[Mark Failed] - + INCREASE --> RETRY[Retry Transaction] RESET --> RETRY WAIT --> RETRY @@ -264,6 +275,7 @@ graph TD ## Configuration ### Wallet Executor + ```typescript { privateKey: string, @@ -276,6 +288,7 @@ graph TD ``` ### Relayer Executor + ```typescript { apiKey: string, @@ -291,6 +304,7 @@ graph TD ## Transaction Validation ### Pre-submission Checks + 1. **Sufficient Rewards**: Total rewards > payout + gas + margin 2. **Valid Deposits**: All deposits exist and qualify 3. **Gas Estimation**: Successful simulation or fallback @@ -298,6 +312,7 @@ graph TD 5. **Queue Limits**: Not exceeding max queue size ### Post-submission Monitoring + 1. **Receipt Polling**: Check transaction status 2. **Timeout Detection**: Identify stuck transactions 3. **Replacement Logic**: Resubmit with higher gas @@ -314,15 +329,19 @@ graph TD ## Common Issues and Solutions ### Issue: Transaction Stuck + **Solution**: Automatic replacement after timeout with higher gas ### Issue: Simulation Failures + **Solution**: Fallback to conservative gas estimates ### Issue: Insufficient Balance + **Solution**: Alert and pause execution until funded ### Issue: High Failure Rate + **Solution**: Circuit breaker activates after threshold ## Performance Metrics @@ -341,4 +360,4 @@ graph TD - **Price Module**: Gas cost calculations - **Monitor Module**: Triggers execution -This module ensures reliable and cost-effective execution of reward claims while protecting against common blockchain transaction failures. \ No newline at end of file +This module ensures reliable and cost-effective execution of reward claims while protecting against common blockchain transaction failures. diff --git a/src/executor/strategies/RelayerExecutor.ts b/src/executor/strategies/RelayerExecutor.ts index 153d1f9..1aed876 100644 --- a/src/executor/strategies/RelayerExecutor.ts +++ b/src/executor/strategies/RelayerExecutor.ts @@ -405,24 +405,28 @@ export class RelayerExecutor implements IExecutor { ): Promise<{ isValid: boolean; error: TransactionValidationError | null }> { try { // Early check: Skip simulation if expected profit is negative or zero - const expectedProfit = profitability.estimates.expected_profit || BigInt(0); + const expectedProfit = + profitability.estimates.expected_profit || BigInt(0); const payoutAmount = profitability.estimates.payout_amount || BigInt(0); - + // If expected profit is 0 or negative, it means total rewards < (payout + gas cost) if (expectedProfit <= BigInt(0)) { const error = new TransactionValidationError( `Transaction not profitable: expected profit is ${expectedProfit}. Skipping expensive simulation.`, - { + { depositIds: depositIds.map(String), expectedProfit: expectedProfit.toString(), - payoutAmount: payoutAmount.toString() + payoutAmount: payoutAmount.toString(), + }, + ); + this.logger.warn( + 'Skipping simulation due to non-profitable transaction', + { + expectedProfit: expectedProfit.toString(), + payoutAmount: payoutAmount.toString(), + depositCount: depositIds.length, }, ); - this.logger.warn('Skipping simulation due to non-profitable transaction', { - expectedProfit: expectedProfit.toString(), - payoutAmount: payoutAmount.toString(), - depositCount: depositIds.length, - }); return { isValid: false, error }; } @@ -461,20 +465,29 @@ export class RelayerExecutor implements IExecutor { { error: error instanceof Error ? error.message : String(error), depositIds: depositIds.map(String), - errorType: error instanceof Error ? error.constructor.name : typeof error, + errorType: + error instanceof Error ? error.constructor.name : typeof error, }, ); - + // For certain types of simulation errors during low gas periods, add extra buffer - const errorMessage = error instanceof Error ? error.message : String(error); - if (errorMessage.includes('gas') || errorMessage.includes('simulation')) { - this.logger.info('Adding extra gas buffer due to simulation issues', { - originalEstimate: profitability.estimates.gas_estimate.toString(), - bufferMultiplier: 1.5, - }); - + const errorMessage = + error instanceof Error ? error.message : String(error); + if ( + errorMessage.includes('gas') || + errorMessage.includes('simulation') + ) { + this.logger.info( + 'Adding extra gas buffer due to simulation issues', + { + originalEstimate: + profitability.estimates.gas_estimate.toString(), + bufferMultiplier: 1.5, + }, + ); + // Add 50% buffer to original estimate when simulation fails during potential low-gas periods - profitability.estimates.gas_estimate = + profitability.estimates.gas_estimate = (profitability.estimates.gas_estimate * 150n) / 100n; } } @@ -992,7 +1005,8 @@ export class RelayerExecutor implements IExecutor { const signerAddress = await this.relaySigner.getAddress(); // Early check: Skip simulation if expected profit is negative or zero - const expectedProfit = tx.profitability.estimates.expected_profit || BigInt(0); + const expectedProfit = + tx.profitability.estimates.expected_profit || BigInt(0); if (expectedProfit <= BigInt(0)) { this.logger.warn( 'Skipping simulation - not profitable, will retry later', @@ -1250,30 +1264,36 @@ export class RelayerExecutor implements IExecutor { // Apply slight boost to gas values for reliable inclusion if (maxFeePerGas) { maxFeePerGas = (maxFeePerGas * 125n) / 100n; - + // Ensure minimum gas price for Flashbots stability const MIN_GAS_PRICE_WEI = 1000000000n; // 1 gwei if (maxFeePerGas < MIN_GAS_PRICE_WEI) { - this.logger.warn('Max fee per gas is very low, using minimum threshold', { - actualGasPriceGwei: Number(maxFeePerGas) / 1e9, - minGasPriceGwei: 1, - usingMinimum: true, - }); + this.logger.warn( + 'Max fee per gas is very low, using minimum threshold', + { + actualGasPriceGwei: Number(maxFeePerGas) / 1e9, + minGasPriceGwei: 1, + usingMinimum: true, + }, + ); maxFeePerGas = MIN_GAS_PRICE_WEI; } } if (maxPriorityFeePerGas) { maxPriorityFeePerGas = (maxPriorityFeePerGas * 125n) / 100n; - + // Ensure minimum priority fee const MIN_PRIORITY_FEE_WEI = 100000000n; // 0.1 gwei minimum priority fee if (maxPriorityFeePerGas < MIN_PRIORITY_FEE_WEI) { - this.logger.warn('Max priority fee per gas is very low, using minimum threshold', { - actualPriorityFeeGwei: Number(maxPriorityFeePerGas) / 1e9, - minPriorityFeeGwei: 0.1, - usingMinimum: true, - }); + this.logger.warn( + 'Max priority fee per gas is very low, using minimum threshold', + { + actualPriorityFeeGwei: Number(maxPriorityFeePerGas) / 1e9, + minPriorityFeeGwei: 0.1, + usingMinimum: true, + }, + ); maxPriorityFeePerGas = MIN_PRIORITY_FEE_WEI; } } diff --git a/src/executor/strategies/helpers/helpers.ts b/src/executor/strategies/helpers/helpers.ts index b596a77..dc61304 100644 --- a/src/executor/strategies/helpers/helpers.ts +++ b/src/executor/strategies/helpers/helpers.ts @@ -108,11 +108,12 @@ export async function calculateGasParameters( const finalGasLimit = calculateGasLimit(gasEstimate, 1, logger); const feeData = await provider.getFeeData(); const baseGasPrice = feeData.gasPrice || 0n; - + // Ensure minimum gas price for stability const MIN_GAS_PRICE_WEI = 1000000000n; // 1 gwei - const safeGasPrice = baseGasPrice < MIN_GAS_PRICE_WEI ? MIN_GAS_PRICE_WEI : baseGasPrice; - + const safeGasPrice = + baseGasPrice < MIN_GAS_PRICE_WEI ? MIN_GAS_PRICE_WEI : baseGasPrice; + if (baseGasPrice < MIN_GAS_PRICE_WEI) { logger.warn('Base gas price is very low, using minimum threshold', { actualGasPriceGwei: Number(baseGasPrice) / 1e9, @@ -120,7 +121,7 @@ export async function calculateGasParameters( usingMinimum: true, }); } - + const gasBoostMultiplier = BigInt(100 + gasBoostPercentage); const boostedGasPrice = (safeGasPrice * gasBoostMultiplier) / 100n; @@ -448,12 +449,13 @@ export async function validateRelayerTransaction( ), }; } - + // Ensure minimum gas price for validation calculations const MIN_GAS_PRICE_WEI = 1000000000n; // 1 gwei const maxFeePerGas = BigInt(feeData.maxFeePerGas.toString()); - const safeGasPrice = maxFeePerGas < MIN_GAS_PRICE_WEI ? MIN_GAS_PRICE_WEI : maxFeePerGas; - + const safeGasPrice = + maxFeePerGas < MIN_GAS_PRICE_WEI ? MIN_GAS_PRICE_WEI : maxFeePerGas; + const estimatedGasCost = safeGasPrice * profitability.estimates.gas_estimate; // Get payout amount from contract diff --git a/src/executor/strategies/helpers/simulation-helpers.ts b/src/executor/strategies/helpers/simulation-helpers.ts index 7c6317c..3c22c6c 100644 --- a/src/executor/strategies/helpers/simulation-helpers.ts +++ b/src/executor/strategies/helpers/simulation-helpers.ts @@ -79,17 +79,17 @@ export async function simulateTransaction( // Get current gas price for realistic cost calculations let realGasPrice: bigint = BigInt(0); let simulationGasPrice: string; - + try { const provider = lstContract.runner?.provider as ethers.Provider; if (provider) { const feeData = await provider.getFeeData(); realGasPrice = feeData.gasPrice || BigInt(0); - + // For simulation: Apply minimum threshold for Tenderly stability const MIN_SIMULATION_GAS_PRICE = ethers.parseUnits('1', 'gwei'); let adjustedGasPrice = realGasPrice; - + if (realGasPrice < MIN_SIMULATION_GAS_PRICE) { adjustedGasPrice = MIN_SIMULATION_GAS_PRICE; logger.info('Using minimum gas price for simulation stability', { @@ -98,7 +98,7 @@ export async function simulateTransaction( reason: 'sub-1-gwei protection for Tenderly', }); } - + simulationGasPrice = adjustedGasPrice.toString(); logger.debug('Gas price setup for simulation', { realGasPriceGwei: Number(realGasPrice) / 1e9, @@ -128,9 +128,10 @@ export async function simulateTransaction( // Ensure minimum gas limit for simulation to avoid failures with very low estimates const MIN_SIMULATION_GAS = 2000000; // 2m gas minimum for complex operations - const simulationGasLimit = Number(gasLimit) < MIN_SIMULATION_GAS - ? MIN_SIMULATION_GAS - : Number(gasLimit); + const simulationGasLimit = + Number(gasLimit) < MIN_SIMULATION_GAS + ? MIN_SIMULATION_GAS + : Number(gasLimit); // Add debug logging logger.debug('Preparing simulation transaction', { @@ -209,7 +210,7 @@ export async function simulateTransaction( ) { const newGasLimit = Math.max( Math.ceil(simulationResult.gasUsed * 1.52), // 50% buffer - MIN_SIMULATION_GAS // Ensure minimum + MIN_SIMULATION_GAS, // Ensure minimum ); const retrySimulationTx = { @@ -365,17 +366,17 @@ export async function estimateGasUsingSimulation( // Get current gas price for realistic cost calculations let realGasPrice: bigint = BigInt(0); let simulationGasPrice: string; - + try { const provider = lstContract.runner?.provider as ethers.Provider; if (provider) { const feeData = await provider.getFeeData(); realGasPrice = feeData.gasPrice || BigInt(0); - + // For simulation: Apply minimum threshold for Tenderly stability const MIN_SIMULATION_GAS_PRICE = ethers.parseUnits('1', 'gwei'); let adjustedGasPrice = realGasPrice; - + if (realGasPrice < MIN_SIMULATION_GAS_PRICE) { adjustedGasPrice = MIN_SIMULATION_GAS_PRICE; logger.info('Using minimum gas price for simulation stability', { @@ -384,7 +385,7 @@ export async function estimateGasUsingSimulation( reason: 'sub-1-gwei protection for Tenderly', }); } - + simulationGasPrice = adjustedGasPrice.toString(); logger.debug('Gas price setup for simulation', { realGasPriceGwei: Number(realGasPrice) / 1e9, @@ -422,7 +423,10 @@ export async function estimateGasUsingSimulation( // Use a high gas limit for estimation but ensure it's reasonable const MIN_ESTIMATION_GAS = 1000000; // 1M gas minimum for gas estimation const MAX_ESTIMATION_GAS = 10000000; // 10M gas maximum to avoid excessive costs - const estimationGasLimit = Math.min(MAX_ESTIMATION_GAS, Math.max(MIN_ESTIMATION_GAS, 5000000)); + const estimationGasLimit = Math.min( + MAX_ESTIMATION_GAS, + Math.max(MIN_ESTIMATION_GAS, 5000000), + ); // Create simulation transaction with a high gas limit to ensure it doesn't fail due to gas const simulationTx: SimulationTransaction = { diff --git a/src/monitor/README.md b/src/monitor/README.md index 06a942d..a605aa2 100644 --- a/src/monitor/README.md +++ b/src/monitor/README.md @@ -13,11 +13,11 @@ graph TB LST[LST Contract] EVENTS[Contract Events] end - + subgraph "Monitor Module" MONITOR[StakerMonitor] PROCESSOR[EventProcessor] - + subgraph "Event Types" STAKE_DEP[StakeDeposited] STAKE_WITH[StakeWithdrawn] @@ -25,7 +25,7 @@ graph TB DEP_INIT[DepositInitialized] DEP_UPD[DepositUpdated] end - + subgraph "Core Functions" LOOP[Processing Loop] BATCH[Batch Fetcher] @@ -33,37 +33,37 @@ graph TB CHECK[Checkpoint Manager] end end - + subgraph "Storage" DB[Database] QUEUE[Processing Queue] end - + subgraph "Consumers" PROFIT[Profitability Module] EXEC[Executor Module] end - + STAKER --> EVENTS LST --> EVENTS EVENTS --> MONITOR - + MONITOR --> PROCESSOR MONITOR --> LOOP LOOP --> BATCH LOOP --> REORG LOOP --> CHECK - + PROCESSOR --> STAKE_DEP PROCESSOR --> STAKE_WITH PROCESSOR --> DEL_ALTER PROCESSOR --> DEP_INIT PROCESSOR --> DEP_UPD - + PROCESSOR --> DB MONITOR --> DB DB --> QUEUE - + QUEUE --> PROFIT PROFIT --> EXEC ``` @@ -77,19 +77,19 @@ sequenceDiagram participant PROC as EventProcessor participant DB as Database participant QUEUE as Processing Queue - + loop Every Poll Interval MON->>BC: Get latest block MON->>BC: Fetch events (batch) BC-->>MON: Event logs - + loop For each event MON->>PROC: Process event PROC->>DB: Update deposit PROC->>QUEUE: Add to queue PROC-->>MON: Result end - + MON->>DB: Update checkpoint end ``` @@ -101,6 +101,7 @@ sequenceDiagram **Purpose**: Main orchestrator for blockchain event monitoring. **Key Responsibilities**: + - Maintains connection to blockchain - Fetches events in batches - Handles chain reorganizations @@ -110,7 +111,9 @@ sequenceDiagram **Core Methods**: #### `processLoop()` + Continuous loop that: + 1. Calculates block range to process 2. Fetches events from contracts 3. Groups events by transaction @@ -119,6 +122,7 @@ Continuous loop that: 6. Handles errors and retries #### `fetchEventsInBatches()` + ```mermaid flowchart TD A[Calculate Range] --> B{Range > Max?} @@ -138,21 +142,25 @@ flowchart TD **Event Handlers**: #### `processStakeDeposited()` + - Creates new deposit or updates existing - Tracks owner, depositor, delegatee - Accumulates deposit amounts #### `processStakeWithdrawn()` + - Reduces deposit amount - Removes deposit if fully withdrawn - Maintains withdrawal history #### `processDelegateeAltered()` + - Updates deposit delegatee - Tracks delegation changes - Validates deposit exists #### `processDepositUpdated()` + - Updates earning power - Tracks reward eligibility - Syncs with contract state @@ -161,21 +169,21 @@ flowchart TD ```typescript interface StakeDepositedEvent { - depositId: string - ownerAddress: string - depositorAddress: string - delegateeAddress: string - amount: bigint - blockNumber: number - transactionHash: string + depositId: string; + ownerAddress: string; + depositorAddress: string; + delegateeAddress: string; + amount: bigint; + blockNumber: number; + transactionHash: string; } interface DelegateeAlteredEvent { - depositId: string - oldDelegatee: string - newDelegatee: string - blockNumber: number - transactionHash: string + depositId: string; + oldDelegatee: string; + newDelegatee: string; + blockNumber: number; + transactionHash: string; } ``` @@ -194,6 +202,7 @@ graph LR ``` **Configuration**: + - `maxBlockRange`: 2000 blocks per request - `confirmations`: 20 blocks for finality - `reorgDepth`: 64 blocks safety buffer @@ -201,8 +210,9 @@ graph LR ### 2. Transaction Grouping Events are grouped by transaction for atomic processing: + ```typescript -Map // Groups events by transaction +Map; // Groups events by transaction ``` This ensures related events (e.g., deposit + delegation) are processed together. @@ -215,13 +225,14 @@ stateDiagram-v2 LoadCheckpoint --> ProcessBlocks ProcessBlocks --> UpdateCheckpoint UpdateCheckpoint --> ProcessBlocks - + ProcessBlocks --> HandleReorg: Reorg Detected HandleReorg --> Rollback Rollback --> ProcessBlocks ``` Checkpoints track: + - `last_block_number`: Last successfully processed block - `block_hash`: Hash for reorg detection - `last_update`: Timestamp of last update @@ -229,6 +240,7 @@ Checkpoints track: ### 4. Reorg Handling When a reorganization is detected: + 1. Roll back to safe block (current - reorgDepth) 2. Re-process affected blocks 3. Update checkpoint with new chain state @@ -239,8 +251,8 @@ When a reorganization is detected: ```typescript // Exponential backoff with jitter -delay = baseDelay * Math.pow(2, attempt) + jitter -maxRetries = 5 +delay = baseDelay * Math.pow(2, attempt) + jitter; +maxRetries = 5; ``` ### Error Categories @@ -264,16 +276,19 @@ graph TD ## Performance Optimization ### 1. Batch Processing + - Fetch multiple blocks in single request - Process events in transaction groups - Bulk database operations ### 2. Caching + - Contract interfaces cached - Recent blocks kept in memory - Checkpoint updates batched ### 3. Parallel Processing + - Multiple contract queries in parallel - Independent events processed concurrently - Database writes queued @@ -288,7 +303,7 @@ sequenceDiagram participant Database participant Queue participant Profitability - + Monitor->>Database: Create/Update Deposit Monitor->>Queue: Add to Processing Queue Queue->>Profitability: Check Profitability @@ -298,6 +313,7 @@ sequenceDiagram ### Event Emission The monitor emits events for: + - New deposits discovered - Deposit amounts changed - Delegations altered @@ -306,6 +322,7 @@ The monitor emits events for: ## Configuration ### Key Parameters + ```typescript { pollInterval: 15, // Seconds between polls @@ -317,6 +334,7 @@ The monitor emits events for: ``` ### Health Monitoring + - Tracks last successful poll - Monitors error rates - Checks block lag @@ -333,15 +351,19 @@ The monitor emits events for: ## Common Issues and Solutions ### Issue: Processing Lag + **Solution**: Increase batch size or reduce poll interval ### Issue: RPC Rate Limits + **Solution**: Implement request throttling and caching ### Issue: Memory Growth + **Solution**: Limit in-memory event buffer size ### Issue: Missed Events + **Solution**: Use deposit discovery scan periodically ## Metrics @@ -352,4 +374,4 @@ The monitor emits events for: - **Error Rate**: Failures per time period - **Queue Depth**: Pending items for processing -This module ensures complete and accurate tracking of all staking positions, forming the foundation for profitability analysis and reward claiming. \ No newline at end of file +This module ensures complete and accurate tracking of all staking positions, forming the foundation for profitability analysis and reward claiming. diff --git a/src/prices/GasCostEstimator.ts b/src/prices/GasCostEstimator.ts index ae69e2a..a403706 100644 --- a/src/prices/GasCostEstimator.ts +++ b/src/prices/GasCostEstimator.ts @@ -63,11 +63,11 @@ export class GasCostEstimator { if (feeData?.gasPrice) { let currentGasPrice = BigInt(feeData.gasPrice.toString()); - + // Handle very low gas prices (sub 1 gwei) that can cause issues const MIN_GAS_PRICE_GWEI = 1n; // 1 gwei minimum const MIN_GAS_PRICE_WEI = MIN_GAS_PRICE_GWEI * 10n ** 9n; - + if (currentGasPrice < MIN_GAS_PRICE_WEI) { this.logger.warn('Gas price is very low, using minimum threshold', { actualGasPriceGwei: Number(currentGasPrice) / 1e9, diff --git a/src/prices/README.md b/src/prices/README.md index 1f6fafe..27c7a37 100644 --- a/src/prices/README.md +++ b/src/prices/README.md @@ -12,15 +12,15 @@ graph TB CMC[CoinMarketCap API] BLOCKCHAIN[Blockchain RPC] end - + subgraph "Prices Module" INTERFACE[IPriceFeed Interface] - + subgraph "Implementations" CMC_FEED[CoinMarketCapFeed] GAS_EST[GasCostEstimator] end - + subgraph "Core Functions" PRICE_CACHE[Price Cache] ID_CACHE[Token ID Cache] @@ -28,21 +28,21 @@ graph TB GAS_CALC[Gas Calculator] end end - + subgraph "Consumers" PROFIT[Profitability Module] EXECUTOR[Executor Module] end - + CMC_FEED --> CMC GAS_EST --> BLOCKCHAIN GAS_EST --> CMC_FEED - + CMC_FEED --> PRICE_CACHE CMC_FEED --> ID_CACHE CMC_FEED --> CONV GAS_EST --> GAS_CALC - + PROFIT --> CMC_FEED PROFIT --> GAS_EST EXECUTOR --> GAS_EST @@ -55,15 +55,19 @@ graph TB **Purpose**: Defines the contract for price feed implementations. **Methods**: + ```typescript interface IPriceFeed { - getTokenPrice(tokenAddress: string): Promise - getTokenPriceInWei(tokenAddress: string, amount: BigNumberish): Promise + getTokenPrice(tokenAddress: string): Promise; + getTokenPriceInWei( + tokenAddress: string, + amount: BigNumberish, + ): Promise; } interface TokenPrice { - usd: number - lastUpdated: Date + usd: number; + lastUpdated: Date; } ``` @@ -72,6 +76,7 @@ interface TokenPrice { **Purpose**: Fetches real-time token prices from CoinMarketCap API. **Key Features**: + - Token address to CMC ID resolution - Price caching with TTL - Special handling for ETH and known tokens @@ -79,13 +84,14 @@ interface TokenPrice { - Error handling with retries **Price Fetching Flow**: + ```mermaid sequenceDiagram participant Client participant Feed as CoinMarketCapFeed participant Cache participant CMC as CoinMarketCap API - + Client->>Feed: getTokenPrice(address) Feed->>Cache: Check cache alt Cache Hit @@ -102,6 +108,7 @@ sequenceDiagram ``` **Token ID Resolution**: + ```mermaid flowchart TD A[Token Address] --> B{Known Token?} @@ -120,6 +127,7 @@ flowchart TD **Purpose**: Estimates gas costs and converts them to reward token amounts. **Key Features**: + - Real-time gas price fetching - EIP-1559 support with fallbacks - Price conversion with decimal handling @@ -127,6 +135,7 @@ flowchart TD - Caching for performance **Gas Estimation Process**: + ```mermaid graph LR A[Get Gas Price] --> B[Apply Buffer] @@ -137,18 +146,20 @@ graph LR ``` **Conversion Formula**: + ```typescript // Gas cost in ETH -gasCostETH = gasPrice * gasUnits +gasCostETH = gasPrice * gasUnits; // Convert to USD -gasCostUSD = gasCostETH * ethPriceUSD +gasCostUSD = gasCostETH * ethPriceUSD; // Convert to reward tokens -gasCostTokens = gasCostUSD / tokenPriceUSD +gasCostTokens = gasCostUSD / tokenPriceUSD; // With decimal adjustment -gasCostTokens = (gasCostETH * ethPriceScaled) / (tokenPriceScaled * decimalAdjustment) +gasCostTokens = + (gasCostETH * ethPriceScaled) / (tokenPriceScaled * decimalAdjustment); ``` ## Price Caching Strategy @@ -162,13 +173,14 @@ graph TD L2[Token ID Cache - Permanent] L3[Symbol Cache - Permanent] end - + L1 --> API[API Request] L2 --> API L3 --> API ``` ### Cache Management + - **Price Cache**: 10-minute TTL for price data - **ID Cache**: Permanent cache for address→CMC_ID mapping - **Symbol Cache**: Permanent cache for address→symbol mapping @@ -191,16 +203,18 @@ stateDiagram-v2 ``` ### Fallback Values + ```typescript -FALLBACK_ETH_PRICE_USD = 2500 // Conservative ETH price -FALLBACK_TOKEN_PRICE_USD = 0.1 // Conservative token price -MINIMUM_GAS_COST_USD = 30 // Minimum transaction cost -FALLBACK_GAS_PRICE_GWEI = 3 // Safe gas price +FALLBACK_ETH_PRICE_USD = 2500; // Conservative ETH price +FALLBACK_TOKEN_PRICE_USD = 0.1; // Conservative token price +MINIMUM_GAS_COST_USD = 30; // Minimum transaction cost +FALLBACK_GAS_PRICE_GWEI = 3; // Safe gas price ``` ## Configuration ### Environment Variables + ```typescript { COINMARKETCAP_API_KEY: string, // CMC API key @@ -210,6 +224,7 @@ FALLBACK_GAS_PRICE_GWEI = 3 // Safe gas price ``` ### Price Feed Config + ```typescript { apiKey: string, @@ -224,6 +239,7 @@ FALLBACK_GAS_PRICE_GWEI = 3 // Safe gas price ## Usage Examples ### Get Token Price + ```typescript const feed = new CoinMarketCapFeed(config, logger); const price = await feed.getTokenPrice('0x...'); @@ -231,17 +247,19 @@ console.log(`Price: $${price.usd}`); ``` ### Estimate Gas Cost + ```typescript const estimator = new GasCostEstimator(); const gasUnits = 300000n; const gasPrice = await provider.getFeeData(); const costInTokens = await estimator.estimateGasCostInRewardTokens( gasUnits, - gasPrice.gasPrice + gasPrice.gasPrice, ); ``` ### Convert ETH to Tokens + ```typescript const prices = await feed.getTokenPrices(); const ethAmount = ethers.parseEther('0.01'); // 0.01 ETH @@ -251,16 +269,19 @@ const tokenAmount = (ethAmount * ethPriceScaled) / tokenPriceScaled; ## Performance Optimizations ### 1. Caching Strategy + - **Multi-layer caching**: Price, ID, and symbol caches - **TTL management**: Automatic cache expiration - **Batch requests**: Parallel price fetching ### 2. Request Optimization + - **Connection pooling**: Reuse HTTP connections - **Timeout handling**: Prevent hanging requests - **Rate limiting**: Respect API limits ### 3. Fallback Mechanisms + - **Graceful degradation**: Use cached/fallback values - **Error recovery**: Automatic retry with backoff - **Minimum enforcement**: Prevent unrealistic values @@ -268,6 +289,7 @@ const tokenAmount = (ethAmount * ethPriceScaled) / tokenPriceScaled; ## Integration Points ### With Profitability Module + ```typescript // Gas cost conversion const gasCostInTokens = await estimator.convertGasUnitsToRewardToken(gasUnits); @@ -278,6 +300,7 @@ const priceRatio = gasToken.usd / rewardToken.usd; ``` ### With Executor Module + ```typescript // Gas price estimation const gasPrice = await estimator.getCurrentGasPrice(provider); @@ -307,29 +330,35 @@ if (estimatedCostUSD < minCostUSD) { ## Common Issues and Solutions ### Issue: Token Not Found + **Solution**: Verify contract address and add manual ID mapping ### Issue: Stale Prices + **Solution**: Reduce cache TTL or implement push notifications ### Issue: API Rate Limits + **Solution**: Implement exponential backoff and request batching ### Issue: Price Volatility + **Solution**: Use price buffers and frequent updates ## Monitoring ### Key Metrics + - **Cache Hit Rate**: Percentage of requests served from cache - **API Response Time**: Average latency for price requests - **Error Rate**: Failed requests per time period - **Price Variance**: Deviation from expected ranges ### Health Checks + - API connectivity validation - Price data freshness verification - Cache consistency checks - Fallback value testing -This module ensures accurate and reliable price data for profitable transaction execution while handling the inherent volatility and unreliability of external price feeds. \ No newline at end of file +This module ensures accurate and reliable price data for profitable transaction execution while handling the inherent volatility and unreliability of external price feeds. diff --git a/src/profitability/README.md b/src/profitability/README.md index fe05c9a..2f9e546 100644 --- a/src/profitability/README.md +++ b/src/profitability/README.md @@ -13,12 +13,12 @@ graph TB TENDERLY[Tenderly Simulation] PRICE_FEED[Price Oracle] end - + subgraph "Profitability Module" WRAPPER[ProfitabilityEngineWrapper] IFACE[IProfitabilityEngine] ENGINE[GovLstProfitabilityEngine] - + subgraph "Core Components" PRICE_CONV[Price Converter] GAS_EST[Gas Estimator] @@ -26,23 +26,23 @@ graph TB THRESHOLD[Threshold Calculator] end end - + subgraph "Consumers" EXECUTOR[Executor Module] MONITOR[Monitor Module] end - + WRAPPER --> IFACE WRAPPER --> ENGINE ENGINE --> PRICE_CONV ENGINE --> GAS_EST ENGINE --> BATCH_OPT ENGINE --> THRESHOLD - + PRICE_CONV --> PRICE_FEED GAS_EST --> TENDERLY ENGINE --> BLOCKCHAIN - + EXECUTOR --> WRAPPER MONITOR --> WRAPPER ``` @@ -56,19 +56,19 @@ stateDiagram-v2 [*] --> FetchDeposits FetchDeposits --> CalculateRewards CalculateRewards --> Stage1Check - + Stage1Check --> Stage1Pass: Rewards > Threshold1 Stage1Check --> Reject: Rewards < Threshold1 - + Stage1Pass --> RunSimulation1 RunSimulation1 --> Stage2Check - + Stage2Check --> Stage2Pass: Rewards > Threshold2 Stage2Check --> Reject: Rewards < Threshold2 - + Stage2Pass --> RunSimulation2 RunSimulation2 --> Profitable - + Profitable --> [*] Reject --> [*] ``` @@ -76,11 +76,13 @@ stateDiagram-v2 ### Stage Details **Stage 1: Initial Check** + - Uses fallback gas estimates - Calculates: `threshold1 = payoutAmount + initialGasCost + margin` - Quick rejection of clearly unprofitable deposits **Stage 2: Refined Check** + - Uses actual simulated gas costs - Calculates: `threshold2 = payoutAmount + simulatedGasCost + (payoutAmount * scaledMargin)` - Final validation before execution @@ -92,6 +94,7 @@ stateDiagram-v2 **Purpose**: Factory and lifecycle manager for profitability engines. **Responsibilities**: + - Creates appropriate engine instances - Manages engine lifecycle - Provides consistent interface @@ -103,6 +106,7 @@ stateDiagram-v2 **Key Methods**: #### `analyzeAndGroupDeposits(deposits: GovLstDeposit[])` + ```mermaid flowchart LR A[Input Deposits] --> B[Fetch Rewards] @@ -116,6 +120,7 @@ flowchart LR ``` #### `checkGroupProfitability(deposits: GovLstDeposit[])` + - Validates earning power requirements - Calculates total unclaimed rewards - Estimates gas costs in reward tokens @@ -124,6 +129,7 @@ flowchart LR ### 3. Price Conversion **Flow**: + ```mermaid graph LR ETH_GAS[ETH Gas Cost] --> CONV[Converter] @@ -133,6 +139,7 @@ graph LR ``` **Formula**: + ``` tokenCost = (ethGasCost * ethPriceUSD) / tokenPriceUSD ``` @@ -140,11 +147,13 @@ tokenCost = (ethGasCost * ethPriceUSD) / tokenPriceUSD ### 4. Gas Estimation **Sources**: + 1. **Fallback Estimate**: 300,000 gas units 2. **Tenderly Simulation**: Accurate gas prediction -3. **Dynamic Calculation**: Base + (perDeposit * count) +3. **Dynamic Calculation**: Base + (perDeposit \* count) **Buffer Application**: + - Simulation result + 30% buffer - Minimum 300,000 gas units - Maximum 10,000,000 gas units @@ -159,18 +168,18 @@ sequenceDiagram participant PriceFeed participant Tenderly participant Executor - + Monitor->>ProfitEngine: analyzeAndGroupDeposits(deposits) ProfitEngine->>Contracts: getPayoutAmount() ProfitEngine->>Contracts: batchFetchUnclaimedRewards() ProfitEngine->>PriceFeed: getPrices() - + rect rgb(200, 220, 240) note right of ProfitEngine: Stage 1 ProfitEngine->>ProfitEngine: Calculate initial threshold ProfitEngine->>ProfitEngine: Check total rewards end - + rect rgb(220, 200, 240) note right of ProfitEngine: Stage 2 ProfitEngine->>Tenderly: simulateTransaction() @@ -178,7 +187,7 @@ sequenceDiagram ProfitEngine->>ProfitEngine: Calculate refined threshold ProfitEngine->>ProfitEngine: Validate profitability end - + ProfitEngine-->>Monitor: Profitable batches Monitor->>Executor: Execute batch ``` @@ -209,28 +218,32 @@ minExpectedReward = payoutAmount + gasCost + (payoutAmount * scaledMargin) ## Configuration ### Environment Variables + - `PROFITABILITY_MIN_PROFIT_MARGIN_PERCENT`: Base profit margin - `PROFITABILITY_INCLUDE_GAS_COST`: Include gas in calculations - `GOVLST_PAYOUT_AMOUNT`: Fixed payout requirement - `GOVLST_MIN_EARNING_POWER`: Minimum deposit eligibility ### Constants + ```typescript GAS_CONSTANTS = { FALLBACK_GAS_ESTIMATE: 300_000n, GAS_PRICE_UPDATE_INTERVAL: 60_000, - MIN_GAS_PRICE: 1_000_000_000n // 1 gwei -} + MIN_GAS_PRICE: 1_000_000_000n, // 1 gwei +}; ``` ## Error Handling ### Retry Logic + - Contract calls: 3 retries with exponential backoff - Price feeds: Cached results with fallback - Simulations: Graceful degradation to estimates ### Error Types + - `GasEstimationError`: Simulation failures - `BatchFetchError`: Reward fetching issues - `QueueProcessingError`: Processing pipeline errors @@ -238,19 +251,25 @@ GAS_CONSTANTS = { ## Optimization Strategies ### 1. Batch Accumulation + Deposits are accumulated until reaching optimal size: + - Maximizes gas efficiency - Ensures sufficient profit margin - Allows multiple profitable claims ### 2. Reward Sorting + Deposits sorted by reward size: + - Prioritizes highest value deposits - Reaches threshold faster - Minimizes transaction count ### 3. Early Termination + Stops adding deposits when threshold met: + - Prevents oversized transactions - Maintains profitability ratio - Enables remaining deposits for future batches @@ -266,14 +285,17 @@ Stops adding deposits when threshold met: ## Common Issues and Solutions ### Issue: "Insufficient rewards after accurate gas estimation" + **Cause**: Gas costs exceed profit margin after simulation **Solution**: Accumulate more deposits or wait for better gas prices ### Issue: High gas costs for small token values + **Cause**: Low token price relative to ETH **Solution**: Increase minimum batch sizes or profit margins ### Issue: Simulation failures + **Cause**: Network issues or invalid transactions **Solution**: Fallback to conservative estimates @@ -284,4 +306,4 @@ Stops adding deposits when threshold met: - **Price Cache Duration**: 10 minutes - **Gas Price Updates**: Every 60 seconds -This module ensures that every reward claim is profitable, protecting users from losing money on gas costs while maximizing their staking rewards. \ No newline at end of file +This module ensures that every reward claim is profitable, protecting users from losing money on gas costs while maximizing their staking rewards. diff --git a/src/profitability/strategies/GovLstProfitabilityEngine.ts b/src/profitability/strategies/GovLstProfitabilityEngine.ts index eae2dc1..00e52bf 100644 --- a/src/profitability/strategies/GovLstProfitabilityEngine.ts +++ b/src/profitability/strategies/GovLstProfitabilityEngine.ts @@ -10,7 +10,6 @@ import { } from '../interfaces/types'; import { GAS_CONSTANTS, CONTRACT_CONSTANTS, EVENTS } from '../constants'; import { - GasEstimationError, QueueProcessingError, BatchFetchError, } from '@/configuration/errors'; @@ -273,16 +272,19 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { // Early check: Skip simulation if total rewards < payout amount if (totalRewards < payoutAmount) { - this.logger.warn('Skipping gas simulation - rewards insufficient for payout', { - totalRewards: totalRewards.toString(), - payoutAmount: payoutAmount.toString(), - depositCount: depositIds.length, - }); - + this.logger.warn( + 'Skipping gas simulation - rewards insufficient for payout', + { + totalRewards: totalRewards.toString(), + payoutAmount: payoutAmount.toString(), + depositCount: depositIds.length, + }, + ); + // Use fallback gas estimate without simulation const enhancedGasCost = gasCostInRewardToken; const actualGasEstimate = GAS_CONSTANTS.FALLBACK_GAS_ESTIMATE; - + // Return early with non-profitable result return { is_profitable: false, @@ -322,20 +324,27 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { this.simulationService, this.logger, ); - + if (simulatedGasUnits !== null) { actualGasEstimate = simulatedGasUnits; - this.logger.info('Using simulated gas estimate for profitability calculation', { - simulatedGasUnits: simulatedGasUnits.toString(), - fallbackEstimate: GAS_CONSTANTS.FALLBACK_GAS_ESTIMATE.toString(), - depositCount: depositIds.length, - }); + this.logger.info( + 'Using simulated gas estimate for profitability calculation', + { + simulatedGasUnits: simulatedGasUnits.toString(), + fallbackEstimate: + GAS_CONSTANTS.FALLBACK_GAS_ESTIMATE.toString(), + depositCount: depositIds.length, + }, + ); } } catch (error) { - this.logger.warn('Failed to get gas units from simulation for profitability', { - error: error instanceof Error ? error.message : String(error), - depositIds: depositIds.map(String), - }); + this.logger.warn( + 'Failed to get gas units from simulation for profitability', + { + error: error instanceof Error ? error.message : String(error), + depositIds: depositIds.map(String), + }, + ); } } @@ -489,7 +498,8 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { const REALISTIC_GAS_UNITS = BigInt(600000); // 600k gas based on production data const gasPrice = await this.getGasPriceWithBuffer(); const gasCostWei = gasPrice * REALISTIC_GAS_UNITS; - const realisticGasCost = await this.convertGasCostToRewardTokens(gasCostWei); + const realisticGasCost = + await this.convertGasCostToRewardTokens(gasCostWei); this.logger.info('Using realistic gas estimate for Stage 1 threshold', { gasUnits: REALISTIC_GAS_UNITS.toString(), @@ -500,7 +510,6 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { }); // We'll calculate enhanced gas cost and actualGasEstimate later, after we know total rewards - let enhancedGasCost = realisticGasCost; let actualGasEstimate = REALISTIC_GAS_UNITS; const profitMargin = this.config.minProfitMargin; @@ -512,7 +521,7 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { const effectiveGasCost = CONFIG.profitability.includeGasCost ? realisticGasCost : BigInt(0); - + const baseAmount = payoutAmount + effectiveGasCost; // Scale profit margin based on deposit count @@ -565,10 +574,12 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { for (const [, reward] of rewardsMap) { totalAvailableRewards += reward; } - + this.logger.info('Total available rewards across all deposits', { totalAvailableRewards: totalAvailableRewards.toString(), - totalAvailableRewardsFormatted: ethers.formatEther(totalAvailableRewards), + totalAvailableRewardsFormatted: ethers.formatEther( + totalAvailableRewards, + ), depositCount: depositIds.length, }); @@ -579,7 +590,9 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { const rewardB = rewardsMap.get(b.deposit_id.toString()) || BigInt(0); const rewardDiff = Number(rewardB - rewardA); // If rewards are equal, sort by deposit_id for deterministic ordering - return rewardDiff !== 0 ? rewardDiff : Number(a.deposit_id - b.deposit_id); + return rewardDiff !== 0 + ? rewardDiff + : Number(a.deposit_id - b.deposit_id); }); // Track which deposits were added to the bin @@ -603,12 +616,15 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { // Check if we've hit optimal threshold after adding this deposit if (this.activeBin.total_rewards >= optimalThreshold) { - this.logger.info('Optimal threshold reached, stopping deposit addition', { - depositsAdded: this.activeBin.deposit_ids.length, - totalRewardsInBin: this.activeBin.total_rewards.toString(), - optimalThreshold: optimalThreshold.toString(), - remainingDeposits: sortedDeposits.length - addedDeposits.length, - }); + this.logger.info( + 'Optimal threshold reached, stopping deposit addition', + { + depositsAdded: this.activeBin.deposit_ids.length, + totalRewardsInBin: this.activeBin.total_rewards.toString(), + optimalThreshold: optimalThreshold.toString(), + remainingDeposits: sortedDeposits.length - addedDeposits.length, + }, + ); break; } } @@ -619,19 +635,26 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { // Stage 1: Initial check - trigger first simulation when rewards > payout + margin if (this.activeBin?.total_rewards >= optimalThreshold) { - this.logger.info('Stage 1: Initial threshold reached, running first simulation', { - totalRewards: this.activeBin.total_rewards.toString(), - optimalThreshold: optimalThreshold.toString(), - depositCount: this.activeBin.deposit_ids.length, - }); + this.logger.info( + 'Stage 1: Initial threshold reached, running first simulation', + { + totalRewards: this.activeBin.total_rewards.toString(), + optimalThreshold: optimalThreshold.toString(), + depositCount: this.activeBin.deposit_ids.length, + }, + ); // First simulation to get accurate gas cost let firstSimulationGasCost = realisticGasCost; let firstSimulationGasUnits = REALISTIC_GAS_UNITS; - - if (this.simulationService && this.activeBin.total_rewards >= payoutAmount) { + + if ( + this.simulationService && + this.activeBin.total_rewards >= payoutAmount + ) { try { - const mockRecipientAddress = CONFIG.profitability.defaultTipReceiver; + const mockRecipientAddress = + CONFIG.profitability.defaultTipReceiver; const simulatedGasUnits = await estimateGasUsingSimulation( this.activeBin.deposit_ids, mockRecipientAddress, @@ -640,11 +663,12 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { this.simulationService, this.logger, ); - + if (simulatedGasUnits !== null) { firstSimulationGasUnits = simulatedGasUnits; - firstSimulationGasCost = await this.convertGasUnitsToRewardToken(simulatedGasUnits); - + firstSimulationGasCost = + await this.convertGasUnitsToRewardToken(simulatedGasUnits); + this.logger.info('Stage 1 simulation completed', { simulatedGasUnits: simulatedGasUnits.toString(), simulatedGasCost: firstSimulationGasCost.toString(), @@ -652,29 +676,37 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { }); } } catch (error) { - this.logger.warn('Stage 1 simulation failed, using fallback estimates', { - error: error instanceof Error ? error.message : String(error), - depositIds: this.activeBin.deposit_ids.map(String), - }); + this.logger.warn( + 'Stage 1 simulation failed, using fallback estimates', + { + error: error instanceof Error ? error.message : String(error), + depositIds: this.activeBin.deposit_ids.map(String), + }, + ); } } // Stage 2: Check if rewards > payout + margin + gas cost from stage 1 const scaledProfitMarginBasisPoints = this.calculateScaledProfitMargin( - BigInt(this.activeBin.deposit_ids.length), - this.config.minProfitMargin + BigInt(this.activeBin.deposit_ids.length), + this.config.minProfitMargin, ); - const marginOnPayout = (payoutAmount * scaledProfitMarginBasisPoints) / 10000n; - const stage2Threshold = payoutAmount + - (CONFIG.profitability.includeGasCost ? firstSimulationGasCost : BigInt(0)) + + const marginOnPayout = + (payoutAmount * scaledProfitMarginBasisPoints) / 10000n; + const stage2Threshold = + payoutAmount + + (CONFIG.profitability.includeGasCost + ? firstSimulationGasCost + : BigInt(0)) + marginOnPayout; - + this.logger.info('Stage 2: Checking against refined threshold', { totalRewards: this.activeBin.total_rewards.toString(), stage2Threshold: stage2Threshold.toString(), firstSimulationGasCost: firstSimulationGasCost.toString(), payoutAmount: payoutAmount.toString(), - scaledProfitMarginBasisPoints: scaledProfitMarginBasisPoints.toString(), + scaledProfitMarginBasisPoints: + scaledProfitMarginBasisPoints.toString(), marginOnPayout: marginOnPayout.toString(), depositCount: this.activeBin.deposit_ids.length, }); @@ -683,10 +715,11 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { // Run second simulation for final validation let finalGasCost = firstSimulationGasCost; let finalGasUnits = firstSimulationGasUnits; - + if (this.simulationService) { try { - const mockRecipientAddress = CONFIG.profitability.defaultTipReceiver; + const mockRecipientAddress = + CONFIG.profitability.defaultTipReceiver; const secondSimulatedGasUnits = await estimateGasUsingSimulation( this.activeBin.deposit_ids, mockRecipientAddress, @@ -695,11 +728,13 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { this.simulationService, this.logger, ); - + if (secondSimulatedGasUnits !== null) { finalGasUnits = secondSimulatedGasUnits; - finalGasCost = await this.convertGasUnitsToRewardToken(secondSimulatedGasUnits); - + finalGasCost = await this.convertGasUnitsToRewardToken( + secondSimulatedGasUnits, + ); + this.logger.info('Stage 2 simulation completed', { secondSimulatedGasUnits: secondSimulatedGasUnits.toString(), finalGasCost: finalGasCost.toString(), @@ -707,21 +742,25 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { }); } } catch (error) { - this.logger.warn('Stage 2 simulation failed, using stage 1 estimates', { - error: error instanceof Error ? error.message : String(error), - depositIds: this.activeBin.deposit_ids.map(String), - }); + this.logger.warn( + 'Stage 2 simulation failed, using stage 1 estimates', + { + error: error instanceof Error ? error.message : String(error), + depositIds: this.activeBin.deposit_ids.map(String), + }, + ); } } - + // Update gas estimates with final simulation results - enhancedGasCost = finalGasCost; actualGasEstimate = finalGasUnits; this.activeBin.gas_estimate = actualGasEstimate; - + // Calculate expected profit - total rewards minus gas cost - const expectedProfit = this.activeBin.total_rewards > finalGasCost ? - this.activeBin.total_rewards - finalGasCost : BigInt(0); + const expectedProfit = + this.activeBin.total_rewards > finalGasCost + ? this.activeBin.total_rewards - finalGasCost + : BigInt(0); this.activeBin.expected_profit = expectedProfit; this.logger.info('Final profitability calculation', { @@ -733,18 +772,23 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { readyBins.push(this.activeBin); } else { - this.logger.info('Stage 2: Insufficient rewards after accurate gas estimation', { - totalRewards: this.activeBin.total_rewards.toString(), - stage2Threshold: stage2Threshold.toString(), - shortfall: (stage2Threshold - this.activeBin.total_rewards).toString(), - breakdown: { - payoutAmount: payoutAmount.toString(), - gasCost: firstSimulationGasCost.toString(), - marginBasisPoints: scaledProfitMarginBasisPoints.toString(), - marginAmount: marginOnPayout.toString(), - calculation: `${payoutAmount} + ${firstSimulationGasCost} + ${marginOnPayout} = ${stage2Threshold}`, + this.logger.info( + 'Stage 2: Insufficient rewards after accurate gas estimation', + { + totalRewards: this.activeBin.total_rewards.toString(), + stage2Threshold: stage2Threshold.toString(), + shortfall: ( + stage2Threshold - this.activeBin.total_rewards + ).toString(), + breakdown: { + payoutAmount: payoutAmount.toString(), + gasCost: firstSimulationGasCost.toString(), + marginBasisPoints: scaledProfitMarginBasisPoints.toString(), + marginAmount: marginOnPayout.toString(), + calculation: `${payoutAmount} + ${firstSimulationGasCost} + ${marginOnPayout} = ${stage2Threshold}`, + }, }, - }); + ); } } @@ -840,12 +884,15 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { // Early check: Skip simulation if total rewards < payout amount if (this.activeBin.total_rewards < payoutAmount) { - this.logger.warn('Skipping gas simulation for active bin - rewards insufficient for payout', { - totalRewards: this.activeBin.total_rewards.toString(), - payoutAmount: payoutAmount.toString(), - depositCount: this.activeBin.deposit_ids.length, - }); - + this.logger.warn( + 'Skipping gas simulation for active bin - rewards insufficient for payout', + { + totalRewards: this.activeBin.total_rewards.toString(), + payoutAmount: payoutAmount.toString(), + depositCount: this.activeBin.deposit_ids.length, + }, + ); + // Return early with non-profitable result return { is_profitable: false, @@ -885,26 +932,35 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { this.simulationService, this.logger, ); - + if (simulatedGasUnits !== null) { actualGasEstimate = simulatedGasUnits; - this.logger.info('Using simulated gas estimate for active bin profitability', { - simulatedGasUnits: simulatedGasUnits.toString(), - fallbackEstimate: GAS_CONSTANTS.FALLBACK_GAS_ESTIMATE.toString(), - depositCount: this.activeBin.deposit_ids.length, - }); + this.logger.info( + 'Using simulated gas estimate for active bin profitability', + { + simulatedGasUnits: simulatedGasUnits.toString(), + fallbackEstimate: + GAS_CONSTANTS.FALLBACK_GAS_ESTIMATE.toString(), + depositCount: this.activeBin.deposit_ids.length, + }, + ); } } catch (error) { - this.logger.warn('Failed to get gas units from simulation for active bin', { - error: error instanceof Error ? error.message : String(error), - depositIds: this.activeBin.deposit_ids.map(String), - }); + this.logger.warn( + 'Failed to get gas units from simulation for active bin', + { + error: error instanceof Error ? error.message : String(error), + depositIds: this.activeBin.deposit_ids.map(String), + }, + ); } } // Expected profit should be total rewards minus gas cost - const expectedProfit = this.activeBin.total_rewards > enhancedGasCost ? - this.activeBin.total_rewards - enhancedGasCost : BigInt(0); + const expectedProfit = + this.activeBin.total_rewards > enhancedGasCost + ? this.activeBin.total_rewards - enhancedGasCost + : BigInt(0); this.activeBin.expected_profit = expectedProfit; this.activeBin.gas_estimate = actualGasEstimate; // Use actual simulation gas estimate @@ -1130,11 +1186,11 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { const feeData = await this.provider.getFeeData(); let gasPrice = feeData.gasPrice ?? BigInt(0); - + // Handle very low gas prices (sub 1 gwei) that can cause simulation issues const MIN_GAS_PRICE_GWEI = 1n; // 1 gwei minimum const MIN_GAS_PRICE_WEI = MIN_GAS_PRICE_GWEI * 10n ** 9n; - + if (gasPrice < MIN_GAS_PRICE_WEI) { this.logger.warn('Gas price is very low, using minimum threshold', { actualGasPriceGwei: Number(gasPrice) / 1e9, @@ -1180,10 +1236,13 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { method: 'getGasPriceWithBuffer', }; - this.logger.warn('Failed to get gas price from provider, using fallback', { - error: error instanceof Error ? error.message : String(error), - lastGasPrice: this.lastGasPrice.toString(), - }); + this.logger.warn( + 'Failed to get gas price from provider, using fallback', + { + error: error instanceof Error ? error.message : String(error), + lastGasPrice: this.lastGasPrice.toString(), + }, + ); // Use cached value if available, otherwise use fallback if (this.lastGasPrice > 0n) { @@ -1284,7 +1343,8 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { // Calculate cost in tokens with decimal adjustment // Formula: (gasCostWei * ethPriceUSD) / tokenPriceUSD // Note: We need to adjust for the difference in decimals between ETH and the reward token - const decimalAdjustment = 10n ** BigInt(Math.abs(ETH_DECIMALS - TOKEN_DECIMALS)); + const decimalAdjustment = + 10n ** BigInt(Math.abs(ETH_DECIMALS - TOKEN_DECIMALS)); let costInRewardTokens: bigint; this.logger.info('Gas cost conversion details', { @@ -1297,27 +1357,37 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { decimalAdjustment: decimalAdjustment.toString(), ethPriceUsd: Number(ethPriceScaled) / 10 ** ETH_DECIMALS, tokenPriceUsd: Number(tokenPriceScaled) / 10 ** TOKEN_DECIMALS, - priceRatio: (Number(ethPriceScaled) / 10 ** ETH_DECIMALS) / (Number(tokenPriceScaled) / 10 ** TOKEN_DECIMALS), + priceRatio: + Number(ethPriceScaled) / + 10 ** ETH_DECIMALS / + (Number(tokenPriceScaled) / 10 ** TOKEN_DECIMALS), }); if (ETH_DECIMALS >= TOKEN_DECIMALS) { - costInRewardTokens = (gasCost * ethPriceScaled) / (tokenPriceScaled * decimalAdjustment); + costInRewardTokens = + (gasCost * ethPriceScaled) / (tokenPriceScaled * decimalAdjustment); } else { - costInRewardTokens = (gasCost * ethPriceScaled * decimalAdjustment) / tokenPriceScaled; + costInRewardTokens = + (gasCost * ethPriceScaled * decimalAdjustment) / tokenPriceScaled; } // Add check for zero result and apply minimum cost if needed if (costInRewardTokens === 0n) { const minimumCostUSD = 30; // $30 minimum like in GasCostEstimator - const minimumCost = (BigInt(minimumCostUSD) * 10n ** BigInt(TOKEN_DECIMALS)) / tokenPriceScaled; - - this.logger.warn('Gas cost in reward token calculated as zero, using minimum cost', { - gasCost: gasCost.toString(), - ethPrice: ethPriceScaled.toString(), - tokenPrice: tokenPriceScaled.toString(), - minimumCost: minimumCost.toString(), - }); - + const minimumCost = + (BigInt(minimumCostUSD) * 10n ** BigInt(TOKEN_DECIMALS)) / + tokenPriceScaled; + + this.logger.warn( + 'Gas cost in reward token calculated as zero, using minimum cost', + { + gasCost: gasCost.toString(), + ethPrice: ethPriceScaled.toString(), + tokenPrice: tokenPriceScaled.toString(), + minimumCost: minimumCost.toString(), + }, + ); + return minimumCost; } @@ -1325,7 +1395,7 @@ export class GovLstProfitabilityEngine implements IGovLstProfitabilityEngine { costInRewardTokens: costInRewardTokens.toString(), formattedCost: ethers.formatUnits(costInRewardTokens, TOKEN_DECIMALS), gasCostEth: ethers.formatEther(gasCost), - conversionRate: `1 ETH = ${(Number(costInRewardTokens) / Number(gasCost) * 1e18).toFixed(2)} reward tokens`, + conversionRate: `1 ETH = ${((Number(costInRewardTokens) / Number(gasCost)) * 1e18).toFixed(2)} reward tokens`, }); return costInRewardTokens; diff --git a/src/simulation/README.md b/src/simulation/README.md index 0dc5bc4..37244e4 100644 --- a/src/simulation/README.md +++ b/src/simulation/README.md @@ -12,17 +12,17 @@ graph TB TENDERLY[Tenderly API] BLOCKCHAIN[Blockchain State] end - + subgraph "Simulation Module" SERVICE[SimulationService] - + subgraph "Core Functions" SIMULATE[Transaction Simulation] BUNDLE[Bundle Simulation] GAS_EST[Gas Estimation] ERROR_PARSE[Error Parser] end - + subgraph "Interfaces" SIM_TX[SimulationTransaction] SIM_RES[SimulationResult] @@ -30,25 +30,25 @@ graph TB GAS_COST[GasCostEstimate] end end - + subgraph "Consumers" PROFIT[Profitability Module] EXECUTOR[Executor Module] end - + SERVICE --> TENDERLY TENDERLY --> BLOCKCHAIN - + SERVICE --> SIMULATE SERVICE --> BUNDLE SERVICE --> GAS_EST SERVICE --> ERROR_PARSE - + SIMULATE --> SIM_TX SIMULATE --> SIM_RES GAS_EST --> GAS_COST ERROR_PARSE --> SIM_ERR - + PROFIT --> SERVICE EXECUTOR --> SERVICE ``` @@ -62,31 +62,40 @@ graph TB **Key Methods**: #### `simulateTransaction(transaction, options)` + Simulates a single transaction with full trace and logs: + ```typescript -const result = await simulationService.simulateTransaction({ - from: '0x...', - to: '0x...', - data: '0x...', - gas: 300000, - gasPrice: '20000000000' -}, { - networkId: '1', - save: false -}); +const result = await simulationService.simulateTransaction( + { + from: '0x...', + to: '0x...', + data: '0x...', + gas: 300000, + gasPrice: '20000000000', + }, + { + networkId: '1', + save: false, + }, +); ``` #### `simulateBundle(transactions, options)` + Simulates multiple transactions as a bundle: + ```typescript -const results = await simulationService.simulateBundle([ - transaction1, - transaction2 -], options); +const results = await simulationService.simulateBundle( + [transaction1, transaction2], + options, +); ``` #### `estimateGasCosts(transaction, options)` + Fast gas estimation without full simulation: + ```typescript const estimate = await simulationService.estimateGasCosts(transaction, options); // Returns: { gasUnits, gasPrice, gasPriceDetails, timestamp } @@ -95,39 +104,46 @@ const estimate = await simulationService.estimateGasCosts(transaction, options); ### 2. Data Structures #### SimulationTransaction + ```typescript interface SimulationTransaction { - from: string // Sender address - to: string // Contract address - data?: string // Encoded function call - value?: string | number // ETH value to send - gas: number // Gas limit - gasPrice?: string // Gas price in wei - maxFeePerGas?: number // EIP-1559 max fee - maxPriorityFeePerGas?: number // EIP-1559 priority fee + from: string; // Sender address + to: string; // Contract address + data?: string; // Encoded function call + value?: string | number; // ETH value to send + gas: number; // Gas limit + gasPrice?: string; // Gas price in wei + maxFeePerGas?: number; // EIP-1559 max fee + maxPriorityFeePerGas?: number; // EIP-1559 priority fee } ``` #### SimulationResult + ```typescript interface SimulationResult { - success: boolean - gasUsed: number - error?: SimulationError - trace?: TransactionTrace - logs?: Array - returnValue?: string - status?: boolean + success: boolean; + gasUsed: number; + error?: SimulationError; + trace?: TransactionTrace; + logs?: Array; + returnValue?: string; + status?: boolean; } ``` #### SimulationError + ```typescript interface SimulationError { - code: 'INSUFFICIENT_FUNDS' | 'EXECUTION_REVERTED' | - 'GAS_LIMIT_EXCEEDED' | 'SIMULATION_FAILED' | 'UNKNOWN_ERROR' - message: string - details?: string + code: + | 'INSUFFICIENT_FUNDS' + | 'EXECUTION_REVERTED' + | 'GAS_LIMIT_EXCEEDED' + | 'SIMULATION_FAILED' + | 'UNKNOWN_ERROR'; + message: string; + details?: string; } ``` @@ -141,7 +157,7 @@ sequenceDiagram participant Service as SimulationService participant Tenderly as Tenderly API participant Blockchain - + Client->>Service: simulateTransaction(tx, options) Service->>Service: Process gas price Service->>Service: Build request payload @@ -173,14 +189,14 @@ flowchart TD ### Minimum Gas Price Protection ```typescript -const MIN_SIMULATION_GAS_PRICE_GWEI = 1 // 1 gwei minimum -const DEFAULT_SIMULATION_GAS_PRICE_GWEI = 20 // 20 gwei default +const MIN_SIMULATION_GAS_PRICE_GWEI = 1; // 1 gwei minimum +const DEFAULT_SIMULATION_GAS_PRICE_GWEI = 20; // 20 gwei default // Process gas price if (!gasPrice || gasPrice === '0') { - gasPrice = ethers.parseUnits('20', 'gwei').toString() + gasPrice = ethers.parseUnits('20', 'gwei').toString(); } else if (gasPriceWei < minGasPriceWei) { - gasPrice = minGasPriceWei.toString() + gasPrice = minGasPriceWei.toString(); } ``` @@ -189,6 +205,7 @@ if (!gasPrice || gasPrice === '0') { ### EIP-1559 Support The service supports both legacy and EIP-1559 transactions: + - Legacy: Uses `gasPrice` - EIP-1559: Uses `maxFeePerGas` and `maxPriorityFeePerGas` @@ -203,7 +220,7 @@ graph TD PARSE --> REVERT{Execution Reverted?} PARSE --> GAS{Gas Limit Exceeded?} PARSE --> UNKNOWN[Unknown Error] - + FUNDS --> FUNDS_CODE[INSUFFICIENT_FUNDS] REVERT --> REVERT_CODE[EXECUTION_REVERTED] GAS --> GAS_CODE[GAS_LIMIT_EXCEEDED] @@ -214,20 +231,21 @@ graph TD ```typescript try { - return await simulateTransaction(tx, options) + return await simulateTransaction(tx, options); } catch (error) { - const simulationError = parseSimulationError(error) + const simulationError = parseSimulationError(error); return { success: false, gasUsed: 0, - error: simulationError - } + error: simulationError, + }; } ``` ## Configuration ### Environment Variables + ```typescript { TENDERLY_ACCESS_KEY: string, // 32-char API key @@ -239,6 +257,7 @@ try { ``` ### API Endpoints + - **Single Transaction**: `/api/v1/account/{account}/project/{project}/simulate` - **Bundle**: `/api/v1/account/{account}/project/{project}/simulate-bundle` - **Gas Estimation**: Same endpoint with `estimate_gas: true` @@ -253,13 +272,13 @@ const options = { '0x...': { balance: '1000000000000000000000', // 1000 ETH nonce: 0, - code: '0x...', // Custom contract code + code: '0x...', // Custom contract code state: { - '0x0': '0x1' // Storage slot overrides - } - } - } -} + '0x0': '0x1', // Storage slot overrides + }, + }, + }, +}; ``` ## Performance Optimization @@ -268,12 +287,12 @@ const options = { ```typescript // Quick estimation (faster) -simulation_type: 'quick' -estimate_gas: true +simulation_type: 'quick'; +estimate_gas: true; // Full simulation (detailed) -simulation_type: 'full' -generate_access_list: true +simulation_type: 'full'; +generate_access_list: true; ``` ### 2. Caching Strategy @@ -294,29 +313,29 @@ generate_access_list: true ```typescript // Pre-execution validation -const simulation = await simulationService.simulateTransaction(tx, options) +const simulation = await simulationService.simulateTransaction(tx, options); if (!simulation.success) { - throw new Error(`Transaction would fail: ${simulation.error?.message}`) + throw new Error(`Transaction would fail: ${simulation.error?.message}`); } // Accurate gas estimation -const gasEstimate = await simulationService.estimateGasCosts(tx, options) -const gasCostInTokens = await convertGasToTokens(gasEstimate.gasUnits) +const gasEstimate = await simulationService.estimateGasCosts(tx, options); +const gasCostInTokens = await convertGasToTokens(gasEstimate.gasUnits); ``` ### Executor Module Integration ```typescript // Validate before submission -const validation = await simulateTransaction(queuedTx, options) +const validation = await simulateTransaction(queuedTx, options); if (!validation.success) { - markTransactionFailed(queuedTx.id, validation.error) - return + markTransactionFailed(queuedTx.id, validation.error); + return; } // Optimize gas parameters -const estimate = await estimateGasCosts(queuedTx, options) -queuedTx.gasLimit = estimate.gasUnits * 1.2 // 20% buffer +const estimate = await estimateGasCosts(queuedTx, options); +queuedTx.gasLimit = estimate.gasUnits * 1.2; // 20% buffer ``` ## Best Practices @@ -330,26 +349,32 @@ queuedTx.gasLimit = estimate.gasUnits * 1.2 // 20% buffer ## Common Issues and Solutions ### Issue: "Insufficient Funds" Errors + **Solution**: Use state overrides to provide sufficient balance ### Issue: Very Low Gas Price Failures + **Solution**: Enforce minimum 1 gwei gas price for simulations ### Issue: Contract Not Found + **Solution**: Verify contract address and network ID ### Issue: Rate Limiting + **Solution**: Implement request throttling and exponential backoff ## Monitoring and Metrics ### Key Metrics + - **Simulation Success Rate**: Percentage of successful simulations - **Gas Estimation Accuracy**: Actual vs simulated gas usage - **Response Time**: API latency for simulations - **Error Distribution**: Breakdown of error types ### Health Checks + - API connectivity validation - Authentication verification - Network state consistency @@ -363,4 +388,4 @@ queuedTx.gasLimit = estimate.gasUnits * 1.2 // 20% buffer 4. **Rate Limiting**: Respect API usage limits 5. **Data Sanitization**: Validate all input parameters -This module ensures transaction reliability and cost optimization by providing comprehensive pre-execution validation and accurate gas estimation capabilities. \ No newline at end of file +This module ensures transaction reliability and cost optimization by providing comprehensive pre-execution validation and accurate gas estimation capabilities. diff --git a/src/simulation/index.ts b/src/simulation/index.ts index c5f3eac..40f0ff8 100644 --- a/src/simulation/index.ts +++ b/src/simulation/index.ts @@ -142,18 +142,23 @@ export class SimulationService { // Ensure minimum gas price for simulation to prevent Tenderly failures const MIN_SIMULATION_GAS_PRICE_GWEI = 1; // 1 gwei minimum const DEFAULT_SIMULATION_GAS_PRICE_GWEI = 20; // 20 gwei default - + let gasPrice = transaction.gasPrice; - + // Handle missing or very low gas price if (!gasPrice || gasPrice === '0') { // Use default gas price for simulation - gasPrice = ethers.parseUnits(DEFAULT_SIMULATION_GAS_PRICE_GWEI.toString(), 'gwei').toString(); + gasPrice = ethers + .parseUnits(DEFAULT_SIMULATION_GAS_PRICE_GWEI.toString(), 'gwei') + .toString(); } else { // Parse the provided gas price and ensure it meets minimum const gasPriceWei = BigInt(gasPrice); - const minGasPriceWei = ethers.parseUnits(MIN_SIMULATION_GAS_PRICE_GWEI.toString(), 'gwei'); - + const minGasPriceWei = ethers.parseUnits( + MIN_SIMULATION_GAS_PRICE_GWEI.toString(), + 'gwei', + ); + if (gasPriceWei < minGasPriceWei) { gasPrice = minGasPriceWei.toString(); } @@ -217,20 +222,25 @@ export class SimulationService { // Ensure minimum gas price for simulation to prevent Tenderly failures const MIN_SIMULATION_GAS_PRICE_GWEI = 1; // 1 gwei minimum const DEFAULT_SIMULATION_GAS_PRICE_GWEI = 20; // 20 gwei default - + // Process gas prices for all transactions const processedTransactions = transactions.map((tx) => { let gasPrice = tx.gasPrice; - + // Handle missing or very low gas price if (!gasPrice || gasPrice === '0') { // Use default gas price for simulation - gasPrice = ethers.parseUnits(DEFAULT_SIMULATION_GAS_PRICE_GWEI.toString(), 'gwei').toString(); + gasPrice = ethers + .parseUnits(DEFAULT_SIMULATION_GAS_PRICE_GWEI.toString(), 'gwei') + .toString(); } else { // Parse the provided gas price and ensure it meets minimum const gasPriceWei = BigInt(gasPrice); - const minGasPriceWei = ethers.parseUnits(MIN_SIMULATION_GAS_PRICE_GWEI.toString(), 'gwei'); - + const minGasPriceWei = ethers.parseUnits( + MIN_SIMULATION_GAS_PRICE_GWEI.toString(), + 'gwei', + ); + if (gasPriceWei < minGasPriceWei) { gasPrice = minGasPriceWei.toString(); } @@ -302,18 +312,23 @@ export class SimulationService { // Ensure minimum gas price for simulation to prevent Tenderly failures const MIN_SIMULATION_GAS_PRICE_GWEI = 1; // 1 gwei minimum const DEFAULT_SIMULATION_GAS_PRICE_GWEI = 20; // 20 gwei default - + let gasPrice = transaction.gasPrice; - + // Handle missing or very low gas price if (!gasPrice || gasPrice === '0') { // Use default gas price for simulation - gasPrice = ethers.parseUnits(DEFAULT_SIMULATION_GAS_PRICE_GWEI.toString(), 'gwei').toString(); + gasPrice = ethers + .parseUnits(DEFAULT_SIMULATION_GAS_PRICE_GWEI.toString(), 'gwei') + .toString(); } else { // Parse the provided gas price and ensure it meets minimum const gasPriceWei = BigInt(gasPrice); - const minGasPriceWei = ethers.parseUnits(MIN_SIMULATION_GAS_PRICE_GWEI.toString(), 'gwei'); - + const minGasPriceWei = ethers.parseUnits( + MIN_SIMULATION_GAS_PRICE_GWEI.toString(), + 'gwei', + ); + if (gasPriceWei < minGasPriceWei) { gasPrice = minGasPriceWei.toString(); } diff --git a/src/tests/README.md b/src/tests/README.md index 6b9a82e..aa327a2 100644 --- a/src/tests/README.md +++ b/src/tests/README.md @@ -13,38 +13,38 @@ graph TB SETUP[Test Setup & Helpers] MOCKS[Mock Providers & Services] end - + subgraph "Unit Tests" GAS_TEST[GasCostEstimator.test.ts] SIM_TEST[SimulationService.test.ts] UNIT_UTILS[Unit Test Utilities] end - + subgraph "Integration Tests" PROFIT_EXEC[GovLstProfitabilityExecutor.test.ts] RELAYER_CLEAN[RelayerCleanup.test.ts] SWAP_SIM[UniswapSwapSimulation.test.ts] INT_UTILS[Integration Test Utilities] end - + subgraph "Test Targets" MODULES[All System Modules] WORKFLOWS[End-to-End Workflows] APIS[External API Integrations] end - + JEST --> UNIT_TESTS JEST --> INTEGRATION_TESTS - + UNIT_TESTS --> GAS_TEST UNIT_TESTS --> SIM_TEST UNIT_TESTS --> UNIT_UTILS - + INTEGRATION_TESTS --> PROFIT_EXEC INTEGRATION_TESTS --> RELAYER_CLEAN INTEGRATION_TESTS --> SWAP_SIM INTEGRATION_TESTS --> INT_UTILS - + GAS_TEST --> MODULES SIM_TEST --> MODULES PROFIT_EXEC --> WORKFLOWS @@ -59,6 +59,7 @@ graph TB **Purpose**: Test individual components in isolation with mocked dependencies. #### GasCostEstimator.test.ts + - **Scope**: Price conversion and gas cost calculations - **Key Tests**: - Gas cost calculation with known prices @@ -76,19 +77,20 @@ describe('GasCostEstimator', () => { maxPriorityFeePerGas: ethers.parseUnits('2', 'gwei'), }), }; - + const estimator = new GasCostEstimator(); const gasCostInTokens = await estimator.estimateGasCostInRewardToken( mockProvider, - 300000n + 300000n, ); - + expect(gasCostInTokens).toBeGreaterThan(0n); }); }); ``` #### SimulationService.test.ts + - **Scope**: Tenderly API integration and response parsing - **Key Tests**: - Transaction simulation success/failure @@ -105,9 +107,9 @@ describe('SimulationService', () => { to: '0x...', data: '0x...', gas: 300000, - gasPrice: '20000000000' + gasPrice: '20000000000', }); - + expect(result.success).toBe(true); expect(result.gasUsed).toBeGreaterThan(0); }); @@ -119,6 +121,7 @@ describe('SimulationService', () => { **Purpose**: Test complete workflows with real or realistic dependencies. #### GovLstProfitabilityExecutor.test.ts + - **Scope**: End-to-end profitability analysis and execution - **Key Tests**: - Complete profit analysis workflow @@ -131,20 +134,20 @@ describe('GovLst Profitability Executor Integration', () => { it('should process profitable deposits end-to-end', async () => { // Setup test data const deposits = TEST_DEPOSITS; - + // Initialize components const database = new DatabaseWrapper(); const profitEngine = new GovLstProfitabilityEngineWrapper(); const executor = new ExecutorWrapper(); - + // Run profitability analysis const analysis = await profitEngine.analyzeAndGroupDeposits(deposits); - + // Execute profitable batches for (const group of analysis.deposit_groups) { const tx = await executor.queueTransaction( group.deposit_ids, - profitabilityCheck + profitabilityCheck, ); expect(tx.status).toBe(TransactionStatus.QUEUED); } @@ -153,6 +156,7 @@ describe('GovLst Profitability Executor Integration', () => { ``` #### RelayerCleanup.test.ts + - **Scope**: OpenZeppelin Defender integration and cleanup - **Key Tests**: - Relayer transaction management @@ -161,6 +165,7 @@ describe('GovLst Profitability Executor Integration', () => { - Error recovery #### UniswapSwapSimulation.test.ts + - **Scope**: Token swap functionality and simulation - **Key Tests**: - Swap simulation accuracy @@ -180,24 +185,25 @@ const TEST_DEPOSITS = [ owner_address: '0x6Fbb31f8c459d773A8d0f67C8C055a70d943C1F1', amount: '19200000000000000000000', earning_power: '15000000000000000000000', - rewards: '500000000000000000000' + rewards: '500000000000000000000', }, // ... more test data ]; // Mock provider responses const mockProvider = { - getFeeData: () => Promise.resolve({ - gasPrice: ethers.parseUnits('20', 'gwei'), - maxFeePerGas: ethers.parseUnits('25', 'gwei') - }), - getTransactionReceipt: (hash) => Promise.resolve(mockReceipt) + getFeeData: () => + Promise.resolve({ + gasPrice: ethers.parseUnits('20', 'gwei'), + maxFeePerGas: ethers.parseUnits('25', 'gwei'), + }), + getTransactionReceipt: (hash) => Promise.resolve(mockReceipt), }; // Mock contract responses const mockContract = { payoutAmount: () => Promise.resolve(ethers.parseEther('4400')), - unclaimedReward: (id) => Promise.resolve(rewardMap.get(id)) + unclaimedReward: (id) => Promise.resolve(rewardMap.get(id)), }; ``` @@ -207,14 +213,14 @@ const mockContract = { beforeEach(async () => { // Initialize test database database = new DatabaseWrapper({ type: 'json', jsonDbPath: ':memory:' }); - + // Setup mock provider provider = new MockProvider(); - + // Initialize contracts with test ABIs stakerContract = new ethers.Contract(STAKER_ADDRESS, stakerAbi, provider); govLstContract = new ethers.Contract(GOVLST_ADDRESS, govlstAbi, provider); - + // Seed test data await seedTestData(); }); @@ -234,9 +240,10 @@ afterEach(async () => { export class MockProviderFactory { static createWithGasPrice(gasPriceGwei: number): ethers.Provider { return { - getFeeData: () => Promise.resolve({ - gasPrice: ethers.parseUnits(gasPriceGwei.toString(), 'gwei') - }) + getFeeData: () => + Promise.resolve({ + gasPrice: ethers.parseUnits(gasPriceGwei.toString(), 'gwei'), + }), } as ethers.Provider; } } @@ -244,10 +251,10 @@ export class MockProviderFactory { export class MockContractFactory { static createStakerContract(rewardMap: Map) { return { - unclaimedReward: (depositId: bigint) => + unclaimedReward: (depositId: bigint) => Promise.resolve(rewardMap.get(depositId.toString()) || 0n), - deposits: (depositId: bigint) => - Promise.resolve(mockDepositData.get(depositId.toString())) + deposits: (depositId: bigint) => + Promise.resolve(mockDepositData.get(depositId.toString())), }; } } @@ -258,7 +265,7 @@ export class MockContractFactory { ```typescript export async function createTestDeposit( database: DatabaseWrapper, - overrides: Partial = {} + overrides: Partial = {}, ): Promise { const deposit = { deposit_id: '12345', @@ -266,16 +273,16 @@ export async function createTestDeposit( amount: '1000000000000000000000', created_at: new Date().toISOString(), updated_at: new Date().toISOString(), - ...overrides + ...overrides, }; - + await database.createDeposit(deposit); return deposit; } export function expectProfitabilityCheck( check: GovLstProfitabilityCheck, - expectedProfitable: boolean + expectedProfitable: boolean, ): void { expect(check.is_profitable).toBe(expectedProfitable); expect(check.estimates.total_shares).toBeGreaterThan(0n); @@ -322,22 +329,22 @@ describe('Error Scenarios', () => { it('should handle network failures gracefully', async () => { // Mock network failure mockProvider.getFeeData.mockRejectedValue(new Error('Network error')); - + const result = await estimator.estimateGasCostInRewardToken( mockProvider, - 300000n + 300000n, ); - + // Should use fallback values expect(result).toBeGreaterThan(0n); }); - + it('should handle insufficient balance', async () => { // Mock insufficient balance scenario mockContract.balanceOf.mockResolvedValue(0n); - + await expect( - executor.validateTransaction(depositIds, profitabilityCheck) + executor.validateTransaction(depositIds, profitabilityCheck), ).rejects.toThrow('Insufficient balance'); }); }); @@ -370,11 +377,7 @@ module.exports = { transform: { '^.+\\.ts$': 'ts-jest', }, - collectCoverageFrom: [ - 'src/**/*.ts', - '!src/**/*.d.ts', - '!src/tests/**', - ], + collectCoverageFrom: ['src/**/*.ts', '!src/**/*.d.ts', '!src/tests/**'], coverageThreshold: { global: { branches: 80, @@ -426,4 +429,4 @@ npm run test:unit - **Error Handling**: Test all error conditions - **Edge Cases**: Test boundary conditions and unusual inputs -This comprehensive test suite ensures system reliability and provides confidence for production deployments and ongoing development. \ No newline at end of file +This comprehensive test suite ensures system reliability and provides confidence for production deployments and ongoing development. From adf09bd198042b2636c90357fc715157cd843cde Mon Sep 17 00:00:00 2001 From: seroxdesign Date: Wed, 30 Jul 2025 05:41:27 -0400 Subject: [PATCH 3/3] some gas calcs, format, lint --- src/profitability/strategies/GovLstProfitabilityEngine.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/profitability/strategies/GovLstProfitabilityEngine.ts b/src/profitability/strategies/GovLstProfitabilityEngine.ts index 00e52bf..a390973 100644 --- a/src/profitability/strategies/GovLstProfitabilityEngine.ts +++ b/src/profitability/strategies/GovLstProfitabilityEngine.ts @@ -9,10 +9,7 @@ import { ProfitabilityConfig, } from '../interfaces/types'; import { GAS_CONSTANTS, CONTRACT_CONSTANTS, EVENTS } from '../constants'; -import { - QueueProcessingError, - BatchFetchError, -} from '@/configuration/errors'; +import { QueueProcessingError, BatchFetchError } from '@/configuration/errors'; import { CONFIG } from '@/configuration'; import { ErrorLogger } from '@/configuration/errorLogger'; import { CoinMarketCapFeed } from '@/prices/CoinmarketcapFeed';