diff --git a/.github/workflows/commit-check.yml b/.github/workflows/commit-check.yml new file mode 100644 index 0000000..f90fe2e --- /dev/null +++ b/.github/workflows/commit-check.yml @@ -0,0 +1,40 @@ +name: Commit Conventions, Signing, and Sign-off + +on: + pull_request: + types: [opened, synchronize, reopened] + push: + branches: + - main + +jobs: + dco_check: + runs-on: ubuntu-22.04 + name: DCO + if: ${{ github.actor != 'dependabot[bot]' }} + steps: + - name: Get PR Commits + id: 'get-pr-commits' + uses: tim-actions/get-pr-commits@198af03565609bb4ed924d1260247b4881f09e7d + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: DCO Check + uses: tim-actions/dco@f2279e6e62d5a7d9115b0cb8e837b777b1b02e21 + with: + commits: ${{ steps.get-pr-commits.outputs.commits }} + + commitlint: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4.1.7 + with: + fetch-depth: 0 + - uses: wagoid/commitlint-github-action@v5.4.1 + + main: + name: Validate PR title + runs-on: ubuntu-22.04 + steps: + - uses: amannn/action-semantic-pull-request@01d5fd8a8ebb9aafe902c40c53f0f4744f7381eb #v5.0.2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..3003cae --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,49 @@ +name: Python Tests + +on: + pull_request: + branches: + - main + paths: + - "**.py" + - "tests/**" + +jobs: + test: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:14 + env: + POSTGRES_USER: testuser + POSTGRES_PASSWORD: testpass + POSTGRES_DB: testdb + ports: + - 5432:5432 + options: >- + --health-cmd="pg_isready -U testuser" + --health-interval=10s + --health-timeout=5s + --health-retries=5 + + env: + DATABASE_URL: postgresql://testuser:testpass@localhost:5432/testdb + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run tests with pytest + run: | + python -m pytest tests/** -vvv -s --tb=short --maxfail=1 diff --git a/cli/cli.py b/cli/cli.py index 1d435d8..899fc17 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -47,7 +47,7 @@ def extract_data(args): "Loading contracts and ABIs...", ) ) - extractor = Extractor(bridge, blockchain) + extractor = Extractor(bridge, blockchain, blockchains) except Exception as e: log_to_cli( build_log_message_2( diff --git a/config/rpcs_base_config.yaml b/config/rpcs_base_config.yaml index fe59e4d..14df14a 100644 --- a/config/rpcs_base_config.yaml +++ b/config/rpcs_base_config.yaml @@ -53,10 +53,12 @@ blockchains: rpcs: - "https://rpc.gnosischain.com" - "https://rpc.gnosis.gateway.fm" - - "https://rpc.ankr.com/gnosis" - - "https://gnosis.blockpi.network/v1/rpc/public" - "https://gnosis.oat.farm" - "https://gnosis-rpc.publicnode.com" + - "https://gnosis.drpc.org" + - "https://gnosis-mainnet.public.blastapi.io" + - "https://rpc.ap-southeast-1.gateway.fm/v4/gnosis/non-archival/mainnet" + - name: optimism contract: "0x4200000000000000000000000000000000000006" @@ -78,16 +80,14 @@ blockchains: start_block: "0x30B2ADB" end_block: "0x30B32AB" rpcs: - - "https://polygon.llamarpc.com" - - - "https://polygon-rpc.com/" - "https://rpc-mainnet.matic.quiknode.pro" - "https://polygon.gateway.tenderly.co" - - "https://polygon.drpc.org" - - "https://rpc.ankr.com/polygon" - - "https://polygon.api.onfinality.io/public" - - "https://polygon.rpc.subquery.network/public" - - "https://polygon.meowrpc.com" + # - "https://polygon-rpc.com/" # This RPC is unreliable and often down / returns inconsistent results + # - "https://polygon.drpc.org" # This RPC is unreliable and often down / returns inconsistent results + # - "https://rpc.ankr.com/polygon" # This RPC is unreliable and often down / returns inconsistent results + # - "https://polygon.api.onfinality.io/public" # This RPC is unreliable and often down / returns inconsistent results + # - "https://polygon.rpc.subquery.network/public" # This RPC is unreliable and often down / returns inconsistent results + # - "https://polygon.meowrpc.com" # This RPC is unreliable and often down / returns inconsistent results - name: base contract: "0xfde4C96c8593536E31F229EA8f37b2ADa2699bb2" @@ -106,7 +106,7 @@ blockchains: contract: "0xBA5Fe23f8a3a24BEd3236F05F2FcF35fd0BF0B5C" topics: - "0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925" - start_block: "0x2AA96E0" + start_block: "0x2AA9AC8" end_block: "0x2AA9EB0" rpcs: - "https://bsc.rpc.blxrbdn.com" @@ -117,13 +117,11 @@ blockchains: - "https://bsc.meowrpc.com" - "https://api.zan.top/bsc-mainnet" - "https://bnb.api.onfinality.io/public" - - "https://rpc.owlracle.info/bsc/70d38ce1826c4a60bb2a8e05a6c8b20f" - "https://1rpc.io/bnb" - - "https://bsc-mainnet.4everland.org/v1/37fa9972c1b1cd5fab542c7bdd4cde2f" - "https://binance.nodereal.io" - - "https://bsc-mainnet.nodereal.io/v1/64a9df0874fb4a93b9d0a3849de012d3" - "https://bsc-rpc.publicnode.com" - "https://bsc.drpc.org" + - "https://binance.llamarpc.com" - name: avalanche contract: "0xB97EF9Ef8734C71904D8002F8b6Bc66Dd9c48a6E" diff --git a/extractor/across/handler.py b/extractor/across/handler.py index bc103b2..85c9588 100644 --- a/extractor/across/handler.py +++ b/extractor/across/handler.py @@ -4,7 +4,7 @@ from extractor.across.constants import BRIDGE_CONFIG from extractor.base_handler import BaseHandler from repository.across.repository import * -from repository.database import DBSession, engine +from repository.database import DBSession from utils.rpc_utils import RPCClient from utils.utils import CustomException, convert_bin_to_hex, log_error @@ -12,8 +12,8 @@ class AcrossHandler(BaseHandler): CLASS_NAME = "AcrossHandler" - def __init__(self, rpc_client: RPCClient) -> None: - super().__init__(rpc_client) + def __init__(self, rpc_client: RPCClient, blockchains: list) -> None: + super().__init__(rpc_client, blockchains) self.bridge = Bridge.ACROSS def get_bridge_contracts_and_topics( @@ -124,7 +124,6 @@ def handle_v3_funds_deposited(self, blockchain, event): if self.across_v3_funds_deposited_repo.event_exists(event["depositId"]): return None - print(f"Creating record for {event['transaction_hash']} in {func_name}") self.across_v3_funds_deposited_repo.create( { "blockchain": blockchain, @@ -237,17 +236,3 @@ def handle_relayer_refund(self, blockchain, event): ) return event - - def convert_id_to_blockchain_name(self, id: str) -> str: - func_name = "convert_id_to_blockchain_name" - - id = str(id) - - if id in BLOCKCHAIN_IDS: - return BLOCKCHAIN_IDS[id]["name"] - else: - e = CustomException( - self.CLASS_NAME, func_name, f"Blockchain not found for EID: {id}" - ) - # log_to_file(e, "data/out_of_scope_blockchains.log") - return None diff --git a/extractor/base_handler.py b/extractor/base_handler.py index 32c196a..3ab6a10 100644 --- a/extractor/base_handler.py +++ b/extractor/base_handler.py @@ -2,18 +2,20 @@ from typing import Any, Dict, List from annotated_types import T -from sqlalchemy.orm import scoped_session, sessionmaker - -from repository.database import engine +from config.constants import BLOCKCHAIN_IDS from utils.rpc_utils import RPCClient from utils.utils import CustomException, convert_bin_to_hex class BaseHandler(ABC): + CLASS_NAME = "BaseHandler" - def __init__(self, rpc_client: RPCClient): + def __init__(self, rpc_client: RPCClient, blockchains: List[str]): self.rpc_client = rpc_client self.bind_db_to_repos() + + # Map of blockchains that are involved in the analysis, used to filter events. + self.counterPartyBlockchainsMap = {b: True for b in blockchains} @abstractmethod def handle_events( @@ -85,3 +87,22 @@ def flatten_object(obj): else: flattened[key] = value return flattened + + + def convert_id_to_blockchain_name(self, id: str) -> str | None: + func_name = "convert_id_to_blockchain_name" + + id = str(id) + + if id in BLOCKCHAIN_IDS: + blockchain_name = BLOCKCHAIN_IDS[id]["name"] + # If the blockchain name is not in the list of blockchains specified by the user, return None + if self.counterPartyBlockchainsMap.get(blockchain_name): + return BLOCKCHAIN_IDS[id]["name"] + + # If the blockchain ID is not found, log an error and return None + e = CustomException( + self.CLASS_NAME, func_name, f"Blockchain with ID {id} not included" + ) + # log_to_file(e, "data/out_of_scope_blockchains.log") + return None diff --git a/extractor/ccip/handler.py b/extractor/ccip/handler.py index f96cf81..2a52085 100644 --- a/extractor/ccip/handler.py +++ b/extractor/ccip/handler.py @@ -4,7 +4,7 @@ from extractor.base_handler import BaseHandler from extractor.ccip.constants import BRIDGE_CONFIG from repository.ccip.repository import * -from repository.database import DBSession, engine +from repository.database import DBSession from utils.rpc_utils import RPCClient from utils.utils import CustomException, log_error, unpad_address @@ -12,8 +12,8 @@ class CcipHandler(BaseHandler): CLASS_NAME = "CcipHandler" - def __init__(self, rpc_client: RPCClient) -> None: - super().__init__(rpc_client) + def __init__(self, rpc_client: RPCClient, blockchains: list) -> None: + super().__init__(rpc_client, blockchains) self.bridge = Bridge.CCIP def get_bridge_contracts_and_topics( diff --git a/extractor/cctp/handler.py b/extractor/cctp/handler.py index 06bb172..202508e 100644 --- a/extractor/cctp/handler.py +++ b/extractor/cctp/handler.py @@ -15,8 +15,8 @@ class CctpHandler(BaseHandler): CLASS_NAME = "CctpHandler" - def __init__(self, rpc_client: RPCClient) -> None: - super().__init__(rpc_client) + def __init__(self, rpc_client: RPCClient, blockchains: list) -> None: + super().__init__(rpc_client, blockchains) self.bridge = Bridge.CCTP def get_bridge_contracts_and_topics( @@ -191,7 +191,10 @@ def convert_doman_id_to_blockchain_name(self, id: str) -> str: id = str(id) if id in BLOCKCHAIN_IDS: - return BLOCKCHAIN_IDS[id]["name"] + blockchain_name = BLOCKCHAIN_IDS[id]["name"] + + if self.counterPartyBlockchainsMap.get(blockchain_name): + return blockchain_name else: e = CustomException( self.CLASS_NAME, func_name, f"Blockchain not found for Domain ID: {id}" diff --git a/extractor/debridge/handler.py b/extractor/debridge/handler.py index 3267fbc..ab6269d 100644 --- a/extractor/debridge/handler.py +++ b/extractor/debridge/handler.py @@ -14,8 +14,8 @@ class DebridgeHandler(BaseHandler): CLASS_NAME = "DebridgeHandler" - def __init__(self, rpc_client: RPCClient) -> None: - super().__init__(rpc_client) + def __init__(self, rpc_client: RPCClient, blockchains: list) -> None: + super().__init__(rpc_client, blockchains) self.bridge = Bridge.DEBRIDGE def get_bridge_contracts_and_topics( @@ -254,8 +254,6 @@ def handle_claimed_unlock(self, blockchain, event): ) def convert_id_to_blockchain_name(self, id: str) -> str: - func_name = "convert_id_to_blockchain_name" - id = str(id) # Discard smaller blockchains as we focus on the ones with more transferred value @@ -263,11 +261,4 @@ def convert_id_to_blockchain_name(self, id: str) -> str: if id.startswith("1000000"): return None - if id in BLOCKCHAIN_IDS: - return BLOCKCHAIN_IDS[id]["name"] - else: - e = CustomException( - self.CLASS_NAME, func_name, f"Blockchain not found for ID: {id}" - ) - # log_to_file(e, "data/out_of_scope_blockchains.log") - return None + return super().convert_id_to_blockchain_name(id) \ No newline at end of file diff --git a/extractor/extractor.py b/extractor/extractor.py index 688eb62..7dd4009 100644 --- a/extractor/extractor.py +++ b/extractor/extractor.py @@ -11,9 +11,41 @@ class Extractor: + """ + Extractor is a class responsible for orchestrating the extraction of blockchain logs and transactions for a specified bridge and blockchain. It manages the division of block ranges, multi-threaded processing, dynamic handler loading, and the decoding and handling of logs and transactions. + + Attributes: + CLASS_NAME (str): The name of the class. + task_queue (Queue): Queue to manage block range tasks for worker threads. + threads (list): List of active worker threads. + blockchain (str): The blockchain network to extract data from. + bridge (Bridge): The bridge instance specifying the protocol/bridge. + rpc_client (RPCClient): Client for interacting with blockchain RPC endpoints. + decoder (BridgeDecoder): Decoder for parsing logs specific to the bridge. + handler (BaseHandler): Handler for processing and storing extracted data. + + Methods: + __init__(self, bridge: Bridge, blockchain: str): + Initializes the Extractor with the specified bridge and blockchain, sets up the RPC client, decoder, and handler. + + load_handler(self) -> BaseHandler: + Dynamically loads and returns the handler for the specified bridge. + + divide_block_ranges(start_block: int, end_block: int, chunk_size: int = 1000): + Divides a block range into smaller chunks for parallel processing. + + work(self, contract: str, topics: list, start_block: int, end_block: int): + Processes logs and transactions for a given contract and block range, decodes logs, and invokes the bridge handler. + + worker(self): + Worker function for threads to process block ranges from the task queue. + + extract_data(self, start_block: int, end_block: int, blockchains: list): + Main extraction logic that validates contracts, divides block ranges, launches worker threads, and coordinates the extraction process. + """ CLASS_NAME = "Extractor" - def __init__(self, bridge: Bridge, blockchain: str): + def __init__(self, bridge: Bridge, blockchain: str, blockchains: list): self.task_queue = Queue() self.threads = [] self.blockchain = blockchain @@ -25,10 +57,10 @@ def __init__(self, bridge: Bridge, blockchain: str): self.decoder = BridgeDecoder(bridge, self.rpc_client.get_random_rpc(blockchain)) # load the bridge handler and initiate a DB session - self.handler = self.load_handler() + self.handler = self.load_handler(blockchains) - def load_handler(self) -> BaseHandler: + def load_handler(self, blockchains: list) -> BaseHandler: """Dynamically loads the handler for the specified bridge.""" func_name = "load_handler" bridge_name = self.bridge.value @@ -38,14 +70,14 @@ def load_handler(self) -> BaseHandler: handler_class_name = f"{bridge_name.capitalize()}Handler" handler_class = getattr(module, handler_class_name) - return handler_class(self.rpc_client) + return handler_class(self.rpc_client, blockchains) except Exception as e: raise CustomException( self.CLASS_NAME, func_name, f"Bridge {bridge_name} not supported. {e}" ) @staticmethod - def divide_block_ranges(start_block: int, end_block: int, chunk_size: int = 2000): + def divide_block_ranges(start_block: int, end_block: int, chunk_size: int = 1000): """Divide block range into smaller chunks.""" ranges = [] for block in range(start_block, end_block + 1, chunk_size): @@ -139,7 +171,7 @@ def worker(self): def extract_data(self, start_block: int, end_block: int): """Main extraction logic.""" - # validate t and blockchain mapping + # load the bridge contract addresses and topics from the configuration file bridge_blockchain_pairs = self.handler.get_bridge_contracts_and_topics( self.bridge, self.blockchain ) @@ -153,7 +185,7 @@ def extract_data(self, start_block: int, end_block: int): self.rpc_client.max_threads_per_blockchain(self.blockchain) * 2 ) - chunk_size = min((end_block - start_block) // num_threads, 2000) + chunk_size = min((end_block - start_block) // num_threads, 1000) if chunk_size < 1: block_ranges = self.divide_block_ranges(start_block, end_block) diff --git a/extractor/omnibridge/handler.py b/extractor/omnibridge/handler.py index 89a17d0..f6a7d41 100644 --- a/extractor/omnibridge/handler.py +++ b/extractor/omnibridge/handler.py @@ -14,8 +14,8 @@ class OmnibridgeHandler(BaseHandler): CLASS_NAME = "OmnibridgeHandler" - def __init__(self, rpc_client: RPCClient) -> None: - super().__init__(rpc_client) + def __init__(self, rpc_client: RPCClient, blockchains: list) -> None: + super().__init__(rpc_client, blockchains) self.bridge = Bridge.OMNIBRIDGE def get_bridge_contracts_and_topics( diff --git a/extractor/polygon/handler.py b/extractor/polygon/handler.py index aa62289..9013f9a 100644 --- a/extractor/polygon/handler.py +++ b/extractor/polygon/handler.py @@ -12,8 +12,8 @@ class PolygonHandler(BaseHandler): CLASS_NAME = "PolygonHandler" - def __init__(self, rpc_client: RPCClient) -> None: - super().__init__(rpc_client) + def __init__(self, rpc_client: RPCClient, blockchains: list) -> None: + super().__init__(rpc_client, blockchains) self.bridge = Bridge.POLYGON diff --git a/extractor/ronin/handler.py b/extractor/ronin/handler.py index 11ce185..8d4ec2e 100644 --- a/extractor/ronin/handler.py +++ b/extractor/ronin/handler.py @@ -14,8 +14,8 @@ class RoninHandler(BaseHandler): CLASS_NAME = "RoninHandler" - def __init__(self, rpc_client: RPCClient) -> None: - super().__init__(rpc_client) + def __init__(self, rpc_client: RPCClient, blockchains: list) -> None: + super().__init__(rpc_client, blockchains) self.bridge = Bridge.RONIN def get_bridge_contracts_and_topics( diff --git a/extractor/stargate/handler.py b/extractor/stargate/handler.py index c8d5891..d96d97b 100644 --- a/extractor/stargate/handler.py +++ b/extractor/stargate/handler.py @@ -15,8 +15,8 @@ class StargateHandler(BaseHandler): CLASS_NAME = "StargateHandler" - def __init__(self, rpc_client: RPCClient) -> None: - super().__init__(rpc_client) + def __init__(self, rpc_client: RPCClient, blockchains: list) -> None: + super().__init__(rpc_client, blockchains) self.bridge = Bridge.STARGATE def get_bridge_contracts_and_topics( @@ -949,7 +949,10 @@ def convert_eid_to_blockchain_name(self, eid: str) -> str: eid = str(eid) if eid in BLOCKCHAIN_IDS: - return BLOCKCHAIN_IDS[eid]["name"] + blockchain_name = BLOCKCHAIN_IDS[eid]["name"] + + if self.counterPartyBlockchainsMap.get(blockchain_name): + return blockchain_name else: e = CustomException( self.CLASS_NAME, func_name, f"Blockchain not found for EID: {eid}" diff --git a/generator/omnibridge/generator.py b/generator/omnibridge/generator.py index 642b25d..a05c59b 100644 --- a/generator/omnibridge/generator.py +++ b/generator/omnibridge/generator.py @@ -222,8 +222,6 @@ def match_omnibridge_cctxs(self): build_log_message_generator(self.bridge, "Matching Omnibridge token transfers...") ) - self.xdai_cross_chain_transactions.empty_table() - query_gnosis_to_ethereum = text( """ INSERT INTO omnibridge_cross_chain_transactions ( @@ -267,8 +265,8 @@ def match_omnibridge_cctxs(self): deposit.message_id, deposit2.sender, fill.recipient, - deposit2.token, -- we hardcode the xDAI address in Ethereum - fill.token, -- we hardcode the xDAI address in Gnosis + deposit2.token, + fill.token, fill.value, NULL as amount_usd FROM omnibridge_user_request_for_signature deposit @@ -323,8 +321,8 @@ def match_omnibridge_cctxs(self): deposit.message_id, deposit2.sender, fill.recipient, - deposit2.token, -- we hardcode the xDAI address in Ethereum - fill.token, -- we hardcode the xDAI address in Gnosis + deposit2.token, + fill.token, fill.value, NULL as amount_usd FROM omnibridge_user_request_for_affirmation deposit diff --git a/repository/across/repository.py b/repository/across/repository.py index 33f5339..1fbc261 100644 --- a/repository/across/repository.py +++ b/repository/across/repository.py @@ -46,7 +46,7 @@ def __init__(self, session_factory): def get_transaction_by_hash(self, transaction_hash: str): with self.get_session() as session: - return session.query(AcrossBlockchainTransaction).get(transaction_hash) + return session.get(AcrossBlockchainTransaction, transaction_hash) def get_min_timestamp(self): with self.get_session() as session: diff --git a/repository/base.py b/repository/base.py index 98f3cc5..5023f7d 100644 --- a/repository/base.py +++ b/repository/base.py @@ -28,6 +28,17 @@ def get_session(self): else: session.close() + def get_all(self): + """ + Fetch all records from the model provided. + Returns a list of objects. + """ + if self.model is None: + raise ValueError("Model is not defined for this repository.") + + with self.get_session() as session: + return session.query(self.model).all() + def create(self, obj_data: dict): """ Create a new record using the model provided. diff --git a/repository/ccip/repository.py b/repository/ccip/repository.py index 45ec8c8..96d8271 100644 --- a/repository/ccip/repository.py +++ b/repository/ccip/repository.py @@ -33,7 +33,7 @@ def __init__(self, session_factory): def get_transaction_by_hash(self, transaction_hash: str): with self.get_session() as session: - return session.query(CCIPBlockchainTransaction).get(transaction_hash) + return session.get(CCIPBlockchainTransaction, transaction_hash) def get_min_timestamp(self): with self.get_session() as session: diff --git a/repository/cctp/repository.py b/repository/cctp/repository.py index 5cdbb0a..a5f671c 100644 --- a/repository/cctp/repository.py +++ b/repository/cctp/repository.py @@ -35,7 +35,7 @@ def __init__(self, session_factory): def get_transaction_by_hash(self, transaction_hash: str): with self.get_session() as session: - return session.query(CCTPBlockchainTransaction).get(transaction_hash) + return session.get(CCTPBlockchainTransaction, transaction_hash) def get_min_timestamp(self): with self.get_session() as session: diff --git a/repository/database.py b/repository/database.py index 4ea371d..13eea46 100644 --- a/repository/database.py +++ b/repository/database.py @@ -1,7 +1,7 @@ import os from sqlalchemy import create_engine -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import declarative_base from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy_utils import create_database, database_exists diff --git a/repository/debridge/repository.py b/repository/debridge/repository.py index cc7e71e..b20a033 100644 --- a/repository/debridge/repository.py +++ b/repository/debridge/repository.py @@ -11,7 +11,7 @@ def __init__(self, session_factory): def get_transaction_by_hash(self, transaction_hash: str): with self.get_session() as session: - return session.query(DeBridgeBlockchainTransaction).get(transaction_hash) + return session.get(DeBridgeBlockchainTransaction, transaction_hash) def get_min_timestamp(self): with self.get_session() as session: diff --git a/repository/omnibridge/models.py b/repository/omnibridge/models.py index e60f16f..4a43cbe 100644 --- a/repository/omnibridge/models.py +++ b/repository/omnibridge/models.py @@ -124,7 +124,7 @@ class OmnibridgeUserRequestForSignature(Base): blockchain = Column(String(10), nullable=False) transaction_hash = Column(String(66), nullable=False) message_id = Column(String(66), nullable=True) - encoded_data = Column(String(66), nullable=True) + encoded_data = Column(String(1000), nullable=True) encoded_data_hash = Column(String(66), nullable=True) recipient = Column(String(42), nullable=True) value = Column(Numeric(30,0), nullable=True) @@ -173,7 +173,7 @@ class OmnibridgeUserRequestForAffirmation(Base): blockchain = Column(String(10), nullable=False) transaction_hash = Column(String(66), nullable=False) message_id = Column(String(66), nullable=True) - encoded_data = Column(String(66), nullable=True) + encoded_data = Column(String(1000), nullable=True) recipient = Column(String(42), nullable=True) value = Column(Numeric(30,0), nullable=True) diff --git a/repository/omnibridge/repository.py b/repository/omnibridge/repository.py index 7e7e4e0..7a2121b 100644 --- a/repository/omnibridge/repository.py +++ b/repository/omnibridge/repository.py @@ -11,7 +11,7 @@ def __init__(self, session_factory): def get_transaction_by_hash(self, transaction_hash: str): with self.get_session() as session: - return session.query(OmnibridgeBlockchainTransaction).get(transaction_hash) + return session.get(OmnibridgeBlockchainTransaction, transaction_hash) def get_min_timestamp(self): with self.get_session() as session: diff --git a/repository/polygon/repository.py b/repository/polygon/repository.py index 3af4578..265f1ea 100644 --- a/repository/polygon/repository.py +++ b/repository/polygon/repository.py @@ -110,7 +110,7 @@ def __init__(self, session_factory): def get_transaction_by_hash(self, transaction_hash: str): with self.get_session() as session: - return session.query(PolygonBlockchainTransaction).get(transaction_hash) + return session.get(PolygonBlockchainTransaction, transaction_hash) def get_min_timestamp(self): with self.get_session() as session: diff --git a/repository/ronin/repository.py b/repository/ronin/repository.py index fd3e246..5dce26b 100644 --- a/repository/ronin/repository.py +++ b/repository/ronin/repository.py @@ -52,7 +52,7 @@ def __init__(self, session_factory): def get_transaction_by_hash(self, transaction_hash: str): with self.get_session() as session: - return session.query(RoninBlockchainTransaction).get(transaction_hash) + return session.get(RoninBlockchainTransaction, transaction_hash) def get_min_timestamp(self): with self.get_session() as session: diff --git a/repository/stargate/repository.py b/repository/stargate/repository.py index b3dc5a8..a66aa00 100644 --- a/repository/stargate/repository.py +++ b/repository/stargate/repository.py @@ -257,7 +257,7 @@ def __init__(self, session_factory): def get_transaction_by_hash(self, transaction_hash: str): with self.get_session() as session: - return session.query(StargateBlockchainTransaction).get(transaction_hash) + return session.get(StargateBlockchainTransaction, transaction_hash) def get_min_timestamp(self): with self.get_session() as session: diff --git a/requirements.txt b/requirements.txt index 381a34e..654e8e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -32,7 +32,7 @@ fastjsonschema==2.21.1 frozenlist==1.5.0 hexbytes==1.2.1 idna==3.10 -ipython==8.10.0 +ipython==8.12.3 isort==5.13.2 jedi==0.19.2 Jinja2==3.1.6 @@ -70,6 +70,7 @@ pydantic_core==2.27.1 Pygments==2.18.0 python-dateutil==2.9.0.post0 pyunormalize==16.0.0 +pytest==8.4.0 PyYAML==6.0.2 pyzmq==26.2.0 referencing==0.35.1 diff --git a/rpcs/generate_rpc_configs.py b/rpcs/generate_rpc_configs.py index d67b853..da43260 100644 --- a/rpcs/generate_rpc_configs.py +++ b/rpcs/generate_rpc_configs.py @@ -58,9 +58,15 @@ def test_rpcs(configs, blockchains): ], ) + if "result" not in response: + raise Exception(f"Invalid RPC response: {response}") + + if "error" in response: + raise Exception(f"Error in RPC response: {response['error']}") + if len(response["result"]) == 0: raise Exception("Something is going on here... no logs found...") - + rpcs.append(rpc) except Exception as e: diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/across/__init__.py b/tests/across/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/across/test_data_extraction.py b/tests/across/test_data_extraction.py new file mode 100644 index 0000000..3438ea0 --- /dev/null +++ b/tests/across/test_data_extraction.py @@ -0,0 +1,39 @@ +import argparse +from cli import Cli + +def test_extract_data(): + args = argparse.Namespace( + blockchains=["ethereum", "arbitrum", "polygon", "optimism", "base", "scroll", "linea"], + bridge="across", + start_ts=1733011200, # 1st Dec 2024 00:00 + end_ts=1733014800, # 1st Dec 2024 01:00 + ) + + Cli.extract_data(args) + + # now we can check if the data was extracted correctly + # For example, we can check if the AcrossV3FundsDeposited table has been populated + from repository.across.repository import AcrossV3FundsDepositedRepository + from repository.database import DBSession + + across_v3_funds_deposited = AcrossV3FundsDepositedRepository(DBSession) + + events = across_v3_funds_deposited.get_all() + + print(f"Number of events in AcrossV3FundsDeposited: {len(events)}") + assert len(events) == 911, "Expected 911 events in AcrossV3FundsDepositedRepository table after extraction." + + args = argparse.Namespace( + bridge="across", + ) + Cli.generate_data(args) + + # Here we can check if the data was generated correctly + from repository.across.repository import AcrossCrossChainTransactionRepository + from repository.database import DBSession + + across_cross_chain_transactions_repo = AcrossCrossChainTransactionRepository(DBSession) + transactions = across_cross_chain_transactions_repo.get_all() + + print(f"Number of transactions in AcrossCrossChainTransaction: {len(transactions)}") + assert len(transactions) == 900, "Expected 900 events in AcrossCrossChainTransaction table after extraction." diff --git a/tests/ccip/__init__.py b/tests/ccip/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/ccip/test_data_extraction.py b/tests/ccip/test_data_extraction.py new file mode 100644 index 0000000..8220d9f --- /dev/null +++ b/tests/ccip/test_data_extraction.py @@ -0,0 +1,39 @@ +import argparse +from cli import Cli + +def test_extract_data(): + args = argparse.Namespace( + blockchains=["ethereum", "arbitrum", "avalanche", "polygon", "optimism", "bnb", "base", "linea", "scroll"], + bridge="ccip", + start_ts=1735689600, # 1st Jan 2025 + end_ts=1735776000, # 2nd Jan 2025 + ) + + Cli.extract_data(args) + + # now we can check if the data was extracted correctly + # For example, we can check if the CCIPSendRequested table has been populated + from repository.ccip.repository import CCIPSendRequestedRepository + from repository.database import DBSession + + ccip_send_requested_repo = CCIPSendRequestedRepository(DBSession) + + events = ccip_send_requested_repo.get_all() + + print(f"Number of events in CCIPSendRequested: {len(events)}") + assert len(events) == 107, "Expected 107 events in CCIPSendRequested table after extraction." + + args = argparse.Namespace( + bridge="ccip", + ) + Cli.generate_data(args) + + # Here we can check if the data was generated correctly + from repository.ccip.repository import CCIPCrossChainTransactionsRepository + from repository.database import DBSession + + ccip_cross_chain_transactions_repo = CCIPCrossChainTransactionsRepository(DBSession) + transactions = ccip_cross_chain_transactions_repo.get_all() + + print(f"Number of transactions in CCIPCrossChainTransactions: {len(transactions)}") + assert len(transactions) == 107, "Expected 107 events in CCIPCrossChainTransactions table after extraction." diff --git a/tests/cctp/__init__.py b/tests/cctp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/cctp/test_data_extraction.py b/tests/cctp/test_data_extraction.py new file mode 100644 index 0000000..55c3e2b --- /dev/null +++ b/tests/cctp/test_data_extraction.py @@ -0,0 +1,37 @@ +import argparse +from cli import Cli + +def test_extract_data(): + args = argparse.Namespace( + blockchains=["ethereum", "arbitrum", "avalanche", "polygon", "optimism", "base"], + bridge="cctp", + start_ts=1735689600, # 1st Dec 2024 00:00 + end_ts=1735776000, # 2nd Jan 2024 00:00 + ) + + Cli.extract_data(args) + + from repository.cctp.repository import CCTPDepositForBurnRepository + from repository.database import DBSession + + cctp_send_requested_repo = CCTPDepositForBurnRepository(DBSession) + + events = cctp_send_requested_repo.get_all() + + print(f"Number of events in CCTPDepositForBurn: {len(events)}") + assert len(events) == 2675, "Expected 2675 events in CCTPDepositForBurn table after extraction." + + args = argparse.Namespace( + bridge="cctp", + ) + Cli.generate_data(args) + + # Here we can check if the data was generated correctly + from repository.cctp.repository import CctpCrossChainTransactionsRepository + from repository.database import DBSession + + cctp_cross_chain_transactions_repo = CctpCrossChainTransactionsRepository(DBSession) + transactions = cctp_cross_chain_transactions_repo.get_all() + + print(f"Number of transactions in CCTPCrossChainTransactions: {len(transactions)}") + assert len(transactions) == 2619, "Expected 2619 events in CCTPCrossChainTransactions table after extraction." diff --git a/tests/debridge/__init__.py b/tests/debridge/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/debridge/test_data_extraction.py b/tests/debridge/test_data_extraction.py new file mode 100644 index 0000000..ed5b98e --- /dev/null +++ b/tests/debridge/test_data_extraction.py @@ -0,0 +1,49 @@ +import argparse +from cli import Cli + +def test_extract_data(): + args = argparse.Namespace( + blockchains=["ethereum", "arbitrum", "bnb", "base"], + bridge="debridge", + start_ts=1733011200, # 1st Dec 2024 00:00 + end_ts=1733014800, # 1st Dec 2024 01:00 + ) + + Cli.extract_data(args) + + from repository.debridge.repository import DeBridgeCreatedOrderRepository + from repository.debridge.repository import DeBridgeFulfilledOrderRepository + from repository.debridge.repository import DeBridgeSentOrderUnlockRepository + from repository.debridge.repository import DeBridgeClaimedUnlockRepository + from repository.database import DBSession + + debridge_created_order_repo = DeBridgeCreatedOrderRepository(DBSession) + events = debridge_created_order_repo.get_all() + print(f"Number of events in DeBridgeCreatedOrder: {len(events)}") + assert len(events) == 72, "Expected 72 events in DeBridgeCreatedOrder table after extraction." + debridge_fulfilled_order_repo = DeBridgeFulfilledOrderRepository(DBSession) + events = debridge_fulfilled_order_repo.get_all() + print(f"Number of events in DeBridgeFulfilledOrder: {len(events)}") + assert len(events) == 71, "Expected 71 events in DeBridgeFulfilledOrder table after extraction." + debridge_sent_order_unlock_repo = DeBridgeSentOrderUnlockRepository(DBSession) + events = debridge_sent_order_unlock_repo.get_all() + print(f"Number of events in DeBridgeSentOrderUnlock: {len(events)}") + assert len(events) == 147, "Expected 147 events in DeBridgeSentOrderUnlock table after extraction." + debridge_claimed_unlock_repo = DeBridgeClaimedUnlockRepository(DBSession) + events = debridge_claimed_unlock_repo.get_all() + print(f"Number of events in DeBridgeClaimedUnlock: {len(events)}") + assert len(events) == 108, "Expected 108 events in DeBridgeClaimedUnlock table after extraction." + + args = argparse.Namespace( + bridge="debridge", + ) + Cli.generate_data(args) + + # Here we can check if the data was generated correctly + from repository.debridge.repository import DeBridgeCrossChainTransactionsRepository + from repository.database import DBSession + + debridge_cross_chain_transactions_repo = DeBridgeCrossChainTransactionsRepository(DBSession) + events = debridge_cross_chain_transactions_repo.get_all() + print(f"Number of events in DeBridgeCrossChainTransactions: {len(events)}") + assert len(events) == 71, "Expected 71 events in DeBridgeCrossChainTransactions table after extraction." diff --git a/tests/omnibridge/__init__.py b/tests/omnibridge/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/omnibridge/test_data_extraction.py b/tests/omnibridge/test_data_extraction.py new file mode 100644 index 0000000..2406843 --- /dev/null +++ b/tests/omnibridge/test_data_extraction.py @@ -0,0 +1,71 @@ +import argparse +from cli import Cli + +def test_extract_data(): + args = argparse.Namespace( + blockchains=["ethereum", "gnosis"], + bridge="omnibridge", + start_ts=1735689600, # 1st Jan 2025 + end_ts=1735776000, # 2nd Jan 2025 + ) + + Cli.extract_data(args) + + from repository.omnibridge.repository import OmnibridgeUserRequestForSignatureRepository + from repository.omnibridge.repository import OmnibridgeRelayedMessageRepository + from repository.omnibridge.repository import OmnibridgeUserRequestForAffirmationRepository + from repository.omnibridge.repository import OmnibridgeAffirmationCompletedRepository + from repository.omnibridge.repository import OmnibridgeTokensBridgedRepository + from repository.omnibridge.repository import OmnibridgeTokensBridgingInitiatedRepository + from repository.database import DBSession + + omnibridge_user_req_for_sign = OmnibridgeUserRequestForSignatureRepository(DBSession) + events = omnibridge_user_req_for_sign.get_all() + print(f"Number of events in OmnibridgeUserRequestForSignature: {len(events)}") + assert len(events) == 51, "Expected 51 events in OmnibridgeUserRequestForSignature table after extraction." + + omnibridge_relayed_message = OmnibridgeRelayedMessageRepository(DBSession) + events = omnibridge_relayed_message.get_all() + print(f"Number of events in OmnibridgeRelayedMessage: {len(events)}") + assert len(events) == 13, "Expected 13 events in OmnibridgeRelayedMessage table after extraction." + + omnibridge_user_req_for_aff = OmnibridgeUserRequestForAffirmationRepository(DBSession) + events = omnibridge_user_req_for_aff.get_all() + print(f"Number of events in OmnibridgeUserRequestForAffirmation: {len(events)}") + assert len(events) == 53, "Expected 53 events in OmnibridgeUserRequestForAffirmation table after extraction." + + omnibridge_affirmation_completed = OmnibridgeAffirmationCompletedRepository(DBSession) + events = omnibridge_affirmation_completed.get_all() + print(f"Number of events in OmnibridgeAffirmationCompleted: {len(events)}") + assert len(events) == 12, "Expected 12 events in OmnibridgeAffirmationCompleted table after extraction." + + omnibridge_tokens_bridged = OmnibridgeTokensBridgedRepository(DBSession) + events = omnibridge_tokens_bridged.get_all() + print(f"Number of events in OmnibridgeTokensBridged: {len(events)}") + assert len(events) == 75, "Expected 75 events in OmnibridgeTokensBridged table after extraction." + + omnibridge_tokens_bridging_initiated = OmnibridgeTokensBridgingInitiatedRepository(DBSession) + events = omnibridge_tokens_bridging_initiated.get_all() + print(f"Number of events in OmnibridgeTokensBridgingInitiated: {len(events)}") + assert len(events) == 79, "Expected 79 events in OmnibridgeTokensBridgingInitiated table after extraction." + + args = argparse.Namespace( + bridge="omnibridge", + ) + Cli.generate_data(args) + + # Here we can check if the data was generated correctly + from repository.omnibridge.repository import OmnibridgeCrossChainTransactionsRepository + from repository.omnibridge.repository import OmnibridgeOperatorTransactionsRepository + from repository.database import DBSession + + omnibridge_cross_chain_transactions_repo = OmnibridgeCrossChainTransactionsRepository(DBSession) + transactions = omnibridge_cross_chain_transactions_repo.get_all() + + print(f"Number of transactions in OmnibridgeCrossChainTransactions: {len(transactions)}") + assert len(transactions) == 97, "Expected 97 events in OmnibridgeCrossChainTransactions table after extraction." + + omnibridge_operator_transactions_repo = OmnibridgeOperatorTransactionsRepository(DBSession) + operator_transactions = omnibridge_operator_transactions_repo.get_all() + print(f"Number of transactions in OmnibridgeOperatorTransactions: {len(operator_transactions)}") + assert len(operator_transactions) == 156, "Expected 156 events in OmnibridgeOperatorTransactions table after extraction." diff --git a/tests/polygon/__init__.py b/tests/polygon/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/polygon/test_data_extraction.py b/tests/polygon/test_data_extraction.py new file mode 100644 index 0000000..52ae98b --- /dev/null +++ b/tests/polygon/test_data_extraction.py @@ -0,0 +1,58 @@ +import argparse +from cli import Cli + +def test_extract_data(): + args = argparse.Namespace( + blockchains=["ethereum", "polygon"], + bridge="polygon", + start_ts=1735689600, # 1st Jan 2025 + end_ts=1735776000, # 2nd Jan 2025 + ) + + Cli.extract_data(args) + + from repository.polygon.repository import PolygonLockedTokenRepository + from repository.polygon.repository import PolygonNewDepositBlockRepository + from repository.polygon.repository import PolygonStateSyncedRepository + from repository.polygon.repository import PolygonStateCommittedRepository + from repository.database import DBSession + + polygon_deposit_requested = PolygonLockedTokenRepository(DBSession) + events = polygon_deposit_requested.get_all() + print(f"Number of events in PolygonLockedToken: {len(events)}") + assert len(events) == 143, "Expected 143 events in PolygonLockedToken table after extraction." + + polygon_state_synced = PolygonStateSyncedRepository(DBSession) + events = polygon_state_synced.get_all() + print(f"Number of events in PolygonStateSynced: {len(events)}") + assert len(events) == 245, "Expected 245 events in PolygonStateSynced table after extraction." + + polygon_state_committed = PolygonStateCommittedRepository(DBSession) + events = polygon_state_committed.get_all() + print(f"Number of events in PolygonStateCommitted: {len(events)}") + assert len(events) == 253, "Expected 253 events in PolygonStateCommitted table after extraction." + + polygon_new_deposit_block = PolygonNewDepositBlockRepository(DBSession) + events = polygon_new_deposit_block.get_all() + print(f"Number of events in PolygonNewDepositBlock: {len(events)}") + assert len(events) == 16, "Expected 16 events in PolygonNewDepositBlock table after extraction." + + args = argparse.Namespace( + bridge="polygon", + ) + Cli.generate_data(args) + + # # Here we can check if the data was generated correctly + from repository.polygon.repository import PolygonCrossChainTransactionsRepository + from repository.polygon.repository import PolygonPlasmaCrossChainTransactionsRepository + from repository.database import DBSession + + polygon_cross_chain_transactions_repo = PolygonCrossChainTransactionsRepository(DBSession) + transactions = polygon_cross_chain_transactions_repo.get_all() + print(f"Number of transactions in PolygonCrossChainTransactions: {len(transactions)}") + assert len(transactions) == 152, "Expected 152 events in PolygonCrossChainTransactions table after extraction." + + polygon_plasma_cross_chain_transactions_repo = PolygonPlasmaCrossChainTransactionsRepository(DBSession) + plasma_transactions = polygon_plasma_cross_chain_transactions_repo.get_all() + print(f"Number of transactions in PolygonPlasmaCrossChainTransactions: {len(plasma_transactions)}") + assert len(plasma_transactions) == 19, "Expected 19 events in PolygonPlasmaCrossChainTransactions table after extraction." diff --git a/tests/ronin/__init__.py b/tests/ronin/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/ronin/test_data_extraction.py b/tests/ronin/test_data_extraction.py new file mode 100644 index 0000000..62cea33 --- /dev/null +++ b/tests/ronin/test_data_extraction.py @@ -0,0 +1,45 @@ +import argparse +from cli import Cli + +def test_extract_data(): + args = argparse.Namespace( + blockchains=["ethereum", "ronin"], + bridge="ronin", + start_ts=1735689600, # 1st Jan 2025 + end_ts=1735776000, # 2nd Jan 2025 + ) + + Cli.extract_data(args) + + from repository.ronin.repository import RoninDepositRequestedRepository + from repository.ronin.repository import RoninWithdrawalRequestedRepository + from repository.database import DBSession + + ronin_deposit_requested = RoninDepositRequestedRepository(DBSession) + + events = ronin_deposit_requested.get_all() + + print(f"Number of events in RoninDepositRequested: {len(events)}") + assert len(events) == 30, "Expected 30 events in RoninDepositRequested table after extraction." + + ronin_withdrawal_requested = RoninWithdrawalRequestedRepository(DBSession) + + events = ronin_withdrawal_requested.get_all() + + print(f"Number of events in RoninWithdrawalRequested: {len(events)}") + assert len(events) == 49, "Expected 49 events in RoninWithdrawalRequested table after extraction." + + args = argparse.Namespace( + bridge="ronin", + ) + Cli.generate_data(args) + + # Here we can check if the data was generated correctly + from repository.ronin.repository import RoninCrossChainTransactionRepository + from repository.database import DBSession + + ronin_cross_chain_transactions_repo = RoninCrossChainTransactionRepository(DBSession) + transactions = ronin_cross_chain_transactions_repo.get_all() + + print(f"Number of transactions in RoninCrossChainTransactions: {len(transactions)}") + assert len(transactions) == 70, "Expected 70 events in RoninCrossChainTransactions table after extraction."