test(global): created extraction and generation tests, and created ci

.github/workflows/commit-check.yml

@@ -0,0 +1,40 @@
+name: Commit Conventions, Signing, and Sign-off
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  push:
+    branches:
+      - main
+
+jobs:
+  dco_check:
+    runs-on: ubuntu-22.04
+    name: DCO
+    if: ${{ github.actor != 'dependabot[bot]' }}
+    steps:
+    - name: Get PR Commits
+      id: 'get-pr-commits'
+      uses: tim-actions/get-pr-commits@198af03565609bb4ed924d1260247b4881f09e7d
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+    - name: DCO Check
+      uses: tim-actions/dco@f2279e6e62d5a7d9115b0cb8e837b777b1b02e21
+      with:
+        commits: ${{ steps.get-pr-commits.outputs.commits }}
+
+  commitlint:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4.1.7
+        with:
+          fetch-depth: 0
+      - uses: wagoid/commitlint-github-action@v5.4.1
+
+  main:
+    name: Validate PR title
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: amannn/action-semantic-pull-request@01d5fd8a8ebb9aafe902c40c53f0f4744f7381eb #v5.0.2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/tests.yml

@@ -0,0 +1,49 @@
+name: Python Tests
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "**.py"
+      - "tests/**"
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:14
+        env:
+          POSTGRES_USER: testuser
+          POSTGRES_PASSWORD: testpass
+          POSTGRES_DB: testdb
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd="pg_isready -U testuser"
+          --health-interval=10s
+          --health-timeout=5s
+          --health-retries=5
+
+    env:
+      DATABASE_URL: postgresql://testuser:testpass@localhost:5432/testdb
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Run tests with pytest
+        run: |
+          python -m pytest tests/** -vvv -s --tb=short --maxfail=1

cli/cli.py

@@ -47,7 +47,7 @@ def extract_data(args):
                         "Loading contracts and ABIs...",
                     )
                 )
-                extractor = Extractor(bridge, blockchain)
+                extractor = Extractor(bridge, blockchain, blockchains)
             except Exception as e:
                 log_to_cli(
                     build_log_message_2(

config/rpcs_base_config.yaml

@@ -53,10 +53,12 @@ blockchains:
   rpcs:
   - "https://rpc.gnosischain.com"
   - "https://rpc.gnosis.gateway.fm"
-  - "https://rpc.ankr.com/gnosis"
-  - "https://gnosis.blockpi.network/v1/rpc/public"
   - "https://gnosis.oat.farm"
   - "https://gnosis-rpc.publicnode.com"
+  - "https://gnosis.drpc.org"
+  - "https://gnosis-mainnet.public.blastapi.io"
+  - "https://rpc.ap-southeast-1.gateway.fm/v4/gnosis/non-archival/mainnet"
+
 
 - name: optimism
   contract: "0x4200000000000000000000000000000000000006"
@@ -78,16 +80,14 @@ blockchains:
   start_block: "0x30B2ADB"
   end_block: "0x30B32AB"
   rpcs:
-  - "https://polygon.llamarpc.com"
-
-  - "https://polygon-rpc.com/"
   - "https://rpc-mainnet.matic.quiknode.pro"
   - "https://polygon.gateway.tenderly.co"
-  - "https://polygon.drpc.org"
-  - "https://rpc.ankr.com/polygon"
-  - "https://polygon.api.onfinality.io/public"
-  - "https://polygon.rpc.subquery.network/public"
-  - "https://polygon.meowrpc.com"
+  # - "https://polygon-rpc.com/" # This RPC is unreliable and often down / returns inconsistent results
+  # - "https://polygon.drpc.org" # This RPC is unreliable and often down / returns inconsistent results
+  # - "https://rpc.ankr.com/polygon" # This RPC is unreliable and often down / returns inconsistent results
+  # - "https://polygon.api.onfinality.io/public" # This RPC is unreliable and often down / returns inconsistent results
+  # - "https://polygon.rpc.subquery.network/public" # This RPC is unreliable and often down / returns inconsistent results
+  # - "https://polygon.meowrpc.com" # This RPC is unreliable and often down / returns inconsistent results
 
 - name: base
   contract: "0xfde4C96c8593536E31F229EA8f37b2ADa2699bb2"
@@ -106,7 +106,7 @@ blockchains:
   contract: "0xBA5Fe23f8a3a24BEd3236F05F2FcF35fd0BF0B5C"
   topics:
   - "0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925"
-  start_block: "0x2AA96E0"
+  start_block: "0x2AA9AC8"
   end_block: "0x2AA9EB0"
   rpcs:
   - "https://bsc.rpc.blxrbdn.com"
@@ -117,13 +117,11 @@ blockchains:
   - "https://bsc.meowrpc.com"
   - "https://api.zan.top/bsc-mainnet"
   - "https://bnb.api.onfinality.io/public"
-  - "https://rpc.owlracle.info/bsc/70d38ce1826c4a60bb2a8e05a6c8b20f"
   - "https://1rpc.io/bnb"
-  - "https://bsc-mainnet.4everland.org/v1/37fa9972c1b1cd5fab542c7bdd4cde2f"
   - "https://binance.nodereal.io"
-  - "https://bsc-mainnet.nodereal.io/v1/64a9df0874fb4a93b9d0a3849de012d3"
   - "https://bsc-rpc.publicnode.com"
   - "https://bsc.drpc.org"
+  - "https://binance.llamarpc.com"
 
 - name: avalanche
   contract: "0xB97EF9Ef8734C71904D8002F8b6Bc66Dd9c48a6E"

extractor/across/handler.py

@@ -4,16 +4,16 @@
 from extractor.across.constants import BRIDGE_CONFIG
 from extractor.base_handler import BaseHandler
 from repository.across.repository import *
-from repository.database import DBSession, engine
+from repository.database import DBSession
 from utils.rpc_utils import RPCClient
 from utils.utils import CustomException, convert_bin_to_hex, log_error
 
 
 class AcrossHandler(BaseHandler):
     CLASS_NAME = "AcrossHandler"
 
-    def __init__(self, rpc_client: RPCClient) -> None:
-        super().__init__(rpc_client)
+    def __init__(self, rpc_client: RPCClient, blockchains: list) -> None:
+        super().__init__(rpc_client, blockchains)
         self.bridge = Bridge.ACROSS
 
     def get_bridge_contracts_and_topics(
@@ -124,7 +124,6 @@ def handle_v3_funds_deposited(self, blockchain, event):
             if self.across_v3_funds_deposited_repo.event_exists(event["depositId"]):
                 return None
 
-            print(f"Creating record for {event['transaction_hash']} in {func_name}")
             self.across_v3_funds_deposited_repo.create(
                 {
                     "blockchain": blockchain,
@@ -237,17 +236,3 @@ def handle_relayer_refund(self, blockchain, event):
                 )
 
         return event
-
-    def convert_id_to_blockchain_name(self, id: str) -> str:
-        func_name = "convert_id_to_blockchain_name"
-
-        id = str(id)
-
-        if id in BLOCKCHAIN_IDS:
-            return BLOCKCHAIN_IDS[id]["name"]
-        else:
-            e = CustomException(
-                self.CLASS_NAME, func_name, f"Blockchain not found for EID: {id}"
-            )
-            # log_to_file(e, "data/out_of_scope_blockchains.log")
-            return None

extractor/base_handler.py

@@ -2,18 +2,20 @@
 from typing import Any, Dict, List
 
 from annotated_types import T
-from sqlalchemy.orm import scoped_session, sessionmaker
-
-from repository.database import engine
+from config.constants import BLOCKCHAIN_IDS
 from utils.rpc_utils import RPCClient
 from utils.utils import CustomException, convert_bin_to_hex
 
 
 class BaseHandler(ABC):
+    CLASS_NAME = "BaseHandler"
 
-    def __init__(self, rpc_client: RPCClient):
+    def __init__(self, rpc_client: RPCClient, blockchains: List[str]):
         self.rpc_client = rpc_client
         self.bind_db_to_repos()
+
+        # Map of blockchains that are involved in the analysis, used to filter events.
+        self.counterPartyBlockchainsMap = {b: True for b in blockchains}
 
     @abstractmethod
     def handle_events(
@@ -85,3 +87,22 @@ def flatten_object(obj):
             else:
                 flattened[key] = value
         return flattened
+
+
+    def convert_id_to_blockchain_name(self, id: str) -> str | None:
+        func_name = "convert_id_to_blockchain_name"
+
+        id = str(id)
+
+        if id in BLOCKCHAIN_IDS:
+            blockchain_name = BLOCKCHAIN_IDS[id]["name"]
+            # If the blockchain name is not in the list of blockchains specified by the user, return None
+            if self.counterPartyBlockchainsMap.get(blockchain_name):
+                return BLOCKCHAIN_IDS[id]["name"]
+
+        # If the blockchain ID is not found, log an error and return None
+        e = CustomException(
+            self.CLASS_NAME, func_name, f"Blockchain with ID {id} not included"
+        )
+        # log_to_file(e, "data/out_of_scope_blockchains.log")
+        return None

extractor/ccip/handler.py

@@ -4,16 +4,16 @@
 from extractor.base_handler import BaseHandler
 from extractor.ccip.constants import BRIDGE_CONFIG
 from repository.ccip.repository import *
-from repository.database import DBSession, engine
+from repository.database import DBSession
 from utils.rpc_utils import RPCClient
 from utils.utils import CustomException, log_error, unpad_address
 
 
 class CcipHandler(BaseHandler):
     CLASS_NAME = "CcipHandler"
 
-    def __init__(self, rpc_client: RPCClient) -> None:
-        super().__init__(rpc_client)
+    def __init__(self, rpc_client: RPCClient, blockchains: list) -> None:
+        super().__init__(rpc_client, blockchains)
         self.bridge = Bridge.CCIP
 
     def get_bridge_contracts_and_topics(

extractor/cctp/handler.py

@@ -15,8 +15,8 @@
 class CctpHandler(BaseHandler):
     CLASS_NAME = "CctpHandler"
 
-    def __init__(self, rpc_client: RPCClient) -> None:
-        super().__init__(rpc_client)
+    def __init__(self, rpc_client: RPCClient, blockchains: list) -> None:
+        super().__init__(rpc_client, blockchains)
         self.bridge = Bridge.CCTP
 
     def get_bridge_contracts_and_topics(
@@ -191,7 +191,10 @@ def convert_doman_id_to_blockchain_name(self, id: str) -> str:
         id = str(id)
 
         if id in BLOCKCHAIN_IDS:
-            return BLOCKCHAIN_IDS[id]["name"]
+            blockchain_name = BLOCKCHAIN_IDS[id]["name"]
+
+            if self.counterPartyBlockchainsMap.get(blockchain_name):
+                return blockchain_name
         else:
             e = CustomException(
                 self.CLASS_NAME, func_name, f"Blockchain not found for Domain ID: {id}"

extractor/debridge/handler.py

@@ -14,8 +14,8 @@
 class DebridgeHandler(BaseHandler):
     CLASS_NAME = "DebridgeHandler"
 
-    def __init__(self, rpc_client: RPCClient) -> None:
-        super().__init__(rpc_client)
+    def __init__(self, rpc_client: RPCClient, blockchains: list) -> None:
+        super().__init__(rpc_client, blockchains)
         self.bridge = Bridge.DEBRIDGE
 
     def get_bridge_contracts_and_topics(
@@ -254,20 +254,11 @@ def handle_claimed_unlock(self, blockchain, event):
             )
 
     def convert_id_to_blockchain_name(self, id: str) -> str:
-        func_name = "convert_id_to_blockchain_name"
-
         id = str(id)
 
         # Discard smaller blockchains as we focus on the ones with more transferred value
         # see list here https://docs.debridge.finance/the-debridge-messaging-protocol/fees-and-supported-chains
         if id.startswith("1000000"):
             return None
 
-        if id in BLOCKCHAIN_IDS:
-            return BLOCKCHAIN_IDS[id]["name"]
-        else:
-            e = CustomException(
-                self.CLASS_NAME, func_name, f"Blockchain not found for ID: {id}"
-            )
-            # log_to_file(e, "data/out_of_scope_blockchains.log")
-            return None
+        return super().convert_id_to_blockchain_name(id)