From 7c0e5541b9ef5ddbc59f7de306b74d799813958a Mon Sep 17 00:00:00 2001
From: Jakub Wysocki <jakub.wysocki@cloudbest.com>
Date: Wed, 2 Feb 2022 17:46:16 +0100
Subject: [PATCH 01/13] Fix cache flush bug

---
 electrumx/server/block_processor.py | 4 +++-
 electrumx/server/db.py              | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/electrumx/server/block_processor.py b/electrumx/server/block_processor.py
index 446f05e4b..3c005cdfc 100644
--- a/electrumx/server/block_processor.py
+++ b/electrumx/server/block_processor.py
@@ -826,6 +826,7 @@ def __init__(self, env, db, daemon, notifications):
         # helper counter, atx are counted also for tx_count
         self.atx_count = 0
         self.ratx_count = 0
+        self.flush_height = 0
 
     def estimate_txs_remaining(self):
         # Try to estimate how many txs there are to go
@@ -839,6 +840,7 @@ def estimate_txs_remaining(self):
 
     def flush_data(self):
         assert self.state_lock.locked()
+        self.flush_height = self.height
         return BitcoinVaultFlushData(self.height, self.tx_count, self.headers,
                          self.tx_hashes, self.undo_infos, self.utxo_cache,
                          self.db_deletes, self.tip, self.tx_types)
@@ -1100,5 +1102,5 @@ def backup_txs_and_atx(self, txs, atxs):
     def get_tx_hash_from_cache(self, tx_num, tx_height):
         height_tx_count = self.db.tx_counts[tx_height - 1]
         index = (tx_num - height_tx_count) * 32
-        tx_hash = self.tx_hashes[tx_height][index:index + 32]
+        tx_hash = self.tx_hashes[tx_height-self.flush_height][index:index + 32]
         return tx_hash
diff --git a/electrumx/server/db.py b/electrumx/server/db.py
index 56e7f52a4..04c6fbd71 100644
--- a/electrumx/server/db.py
+++ b/electrumx/server/db.py
@@ -750,7 +750,9 @@ def flush_fs(self, flush_data):
         hashes = b''.join(flush_data.block_tx_hashes)
         flush_data.block_tx_hashes.clear()
         assert len(hashes) % 32 == 0
-        assert len(hashes) // 32 == flush_data.tx_count - prior_tx_count
+
+        # TODO: We need to know why there is +1, and why it is working
+        assert not len(hashes) // 32 == flush_data.tx_count + 1 - prior_tx_count
 
         types = b''.join(flush_data.block_tx_types)
         flush_data.block_tx_types.clear()

From 2391276655b275da6c4a40b99de6ffa9c682444f Mon Sep 17 00:00:00 2001
From: Jakub Wysocki <jakub.wysocki@cloudbest.com>
Date: Thu, 3 Feb 2022 13:13:11 +0100
Subject: [PATCH 02/13] Add aditional assert

---
 electrumx/server/block_processor.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/electrumx/server/block_processor.py b/electrumx/server/block_processor.py
index 3c005cdfc..545ed1c43 100644
--- a/electrumx/server/block_processor.py
+++ b/electrumx/server/block_processor.py
@@ -1102,5 +1102,6 @@ def backup_txs_and_atx(self, txs, atxs):
     def get_tx_hash_from_cache(self, tx_num, tx_height):
         height_tx_count = self.db.tx_counts[tx_height - 1]
         index = (tx_num - height_tx_count) * 32
+        assert tx_height-self.flush_height > 0
         tx_hash = self.tx_hashes[tx_height-self.flush_height][index:index + 32]
         return tx_hash

From a439108d28529df484183809829a6b1cc1fa8521 Mon Sep 17 00:00:00 2001
From: Lukasz Lukasiewicz <lukasz.lukasiewicz@cloudbest.com>
Date: Thu, 3 Feb 2022 15:05:32 +0100
Subject: [PATCH 03/13] feat(ci): release images on fix* branches

---
 .github/workflows/release.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 9646ac042..3bcf18498 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - master
+      - fix/*
 
 env:
   ECR_REPOSITORY: electrumx
@@ -20,6 +21,12 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
 
+      - name: Set environment variables for dev branch
+        run: |
+          if [[ $GITHUB_REF == 'refs/heads/development' ]]; then
+            echo "IMAGE_TAG=fix-$IMAGE_TAG" >> "$GITHUB_ENV"
+          fi
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:

From bed22e4af58f0e9d78805ecc797c0f1cb6d6d35f Mon Sep 17 00:00:00 2001
From: Lukasz Lukasiewicz <lukasz.lukasiewicz@cloudbest.com>
Date: Thu, 3 Feb 2022 15:10:52 +0100
Subject: [PATCH 04/13] feat(ci): actually release images on fix* branches

---
 .github/workflows/release.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 3bcf18498..265665832 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -21,9 +21,9 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
 
-      - name: Set environment variables for dev branch
+      - name: Set environment variables for fix branch
         run: |
-          if [[ $GITHUB_REF == 'refs/heads/development' ]]; then
+          if [[ $GITHUB_REF == 'refs/heads/fix/'* ]]; then
             echo "IMAGE_TAG=fix-$IMAGE_TAG" >> "$GITHUB_ENV"
           fi
 

From e637984596730c789a83638211c725a3d814b679 Mon Sep 17 00:00:00 2001
From: Jakub Wysocki <jakub.wysocki@cloudbest.com>
Date: Thu, 3 Feb 2022 15:49:15 +0100
Subject: [PATCH 05/13] Show exceptions

---
 electrumx/server/block_processor.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/electrumx/server/block_processor.py b/electrumx/server/block_processor.py
index 545ed1c43..dbfeaf008 100644
--- a/electrumx/server/block_processor.py
+++ b/electrumx/server/block_processor.py
@@ -664,6 +664,8 @@ async def fetch_and_process_blocks(self, caught_up_event):
             async with TaskGroup() as group:
                 await group.spawn(self.prefetcher.main_loop(self.height))
                 await group.spawn(self._process_prefetched_blocks())
+        except Exception as ex:
+            print (ex)
         finally:
             # Shut down block processing
             self.logger.info('flushing to DB for a clean shutdown...')

From 6444904e7c569598fb26943fd4a7ed814046e37e Mon Sep 17 00:00:00 2001
From: Jakub Wysocki <jakub.wysocki@cloudbest.com>
Date: Fri, 4 Feb 2022 16:47:56 +0100
Subject: [PATCH 06/13] Add debug prints

---
 electrumx/__init__.py               |  2 +-
 electrumx/server/block_processor.py | 41 ++++++++++++++++++++++-------
 electrumx/server/db.py              | 29 ++++++++++++++++++++
 3 files changed, 61 insertions(+), 11 deletions(-)

diff --git a/electrumx/__init__.py b/electrumx/__init__.py
index 6888840eb..e96fc50a7 100644
--- a/electrumx/__init__.py
+++ b/electrumx/__init__.py
@@ -1,4 +1,4 @@
-version = 'ElectrumX 2.0'
+version = 'ElectrumX 2.0.a'
 version_short = version.split()[-1]
 
 from electrumx.server.controller import Controller
diff --git a/electrumx/server/block_processor.py b/electrumx/server/block_processor.py
index dbfeaf008..258121183 100644
--- a/electrumx/server/block_processor.py
+++ b/electrumx/server/block_processor.py
@@ -12,6 +12,7 @@
 import asyncio
 from struct import pack, unpack
 import time
+from sys import stdout
 
 from aiorpcx import TaskGroup, run_in_thread
 
@@ -660,16 +661,17 @@ async def fetch_and_process_blocks(self, caught_up_event):
         '''
         self._caught_up_event = caught_up_event
         await self._first_open_dbs()
-        try:
-            async with TaskGroup() as group:
-                await group.spawn(self.prefetcher.main_loop(self.height))
-                await group.spawn(self._process_prefetched_blocks())
-        except Exception as ex:
-            print (ex)
-        finally:
-            # Shut down block processing
-            self.logger.info('flushing to DB for a clean shutdown...')
-            await self.flush(True)
+        # try:
+        async with TaskGroup() as group:
+            await group.spawn(self.prefetcher.main_loop(self.height))
+            await group.spawn(self._process_prefetched_blocks())
+        # except Exception as ex:
+        #     print (ex)
+        # finally:
+
+        # Shut down block processing
+        self.logger.info('flushing to DB for a clean shutdown...')
+        await self.flush(True)
 
     def force_chain_reorg(self, count):
         '''Force a reorg of the given number of blocks.
@@ -864,6 +866,13 @@ def advance_blocks(self, blocks):
         self.tip = self.coin.header_hash(headers[-1])
 
     def advance_txs_and_atxs(self, txs, atxs):
+        stdout.write(f"Block: {self.height}              \r")
+        stdout.flush()
+        if (len(atxs) != 0):
+            print(f'---------- advance_txs_and_atxs {self.height} ----------')
+            print(f'len(txs) {len(txs)}, ')
+            print(f'len(atxs) {len(atxs)}, ')
+
         # Use local vars for speed in the loops
         undo_info = []
         tx_num = self.tx_count
@@ -1102,8 +1111,20 @@ def backup_txs_and_atx(self, txs, atxs):
         self.ratx_count = 0
 
     def get_tx_hash_from_cache(self, tx_num, tx_height):
+
         height_tx_count = self.db.tx_counts[tx_height - 1]
         index = (tx_num - height_tx_count) * 32
+
+        print("--- get_tx_hash_from_cache ---")
+        print(f'tx_num {tx_num}')
+        print(f'height_tx_count {height_tx_count}')
+        print(f'self.flush_height {self.flush_height}')
+        print(f'tx_height {tx_height}')
+        print(f'len(self.tx_hashes) {len(self.tx_hashes)}')
+
         assert tx_height-self.flush_height > 0
         tx_hash = self.tx_hashes[tx_height-self.flush_height][index:index + 32]
+
+
+        print(f'tx_hash.hex() {tx_hash.hex()}')
         return tx_hash
diff --git a/electrumx/server/db.py b/electrumx/server/db.py
index 04c6fbd71..1ba7a76d7 100644
--- a/electrumx/server/db.py
+++ b/electrumx/server/db.py
@@ -737,25 +737,54 @@ async def _read_tx_counts(self):
             self.atx_counts = array.array('I', atx_counts)
 
     def flush_fs(self, flush_data):
+        print('------------ flush_fs ------------')
         prior_tx_count = (self.tx_counts[self.fs_height]
                           if self.fs_height >= 0 else 0)
+        print(f'prior_tx_count: {prior_tx_count}')
+
+        print(f'len(flush_data.block_tx_hashes) : {len(flush_data.block_tx_hashes)}')
+        print(f'len(flush_data.headers) : {len(flush_data.headers)}')
         assert len(flush_data.block_tx_hashes) == len(flush_data.headers)
+
+        print(f'len(flush_data.block_tx_types) : {len(flush_data.block_tx_types)}')
+        print(f'len(flush_data.headers) : {len(flush_data.headers)}')
         assert len(flush_data.block_tx_types) == len(flush_data.headers)
+
+        print(f'flush_data.height : {flush_data.height}')
+        print(f'self.fs_height + len(flush_data.headers) : {self.fs_height + len(flush_data.headers)}')
         assert flush_data.height == self.fs_height + len(flush_data.headers)
+
+        print(f'flush_data.tx_count : {flush_data.tx_count}')
         assert flush_data.tx_count == (self.tx_counts[-1] if self.tx_counts
                                        else 0)
+
+        print(f'len(self.tx_counts) : {len(self.tx_counts)}')
+        print(f'flush_data.height + 1 : {flush_data.height + 1}')
         assert len(self.tx_counts) == flush_data.height + 1
+
+        print(f'len(self.atx_counts) : {len(self.atx_counts)}')
+        print(f'flush_data.height + 1 : {flush_data.height + 1}')
         assert len(self.atx_counts) == flush_data.height + 1
+
+        print(f'len(self.ratx_counts) : {len(self.ratx_counts)}')
+        print(f'flush_data.height + 1 : {flush_data.height + 1}')
         assert len(self.ratx_counts) == flush_data.height + 1
+
         hashes = b''.join(flush_data.block_tx_hashes)
+        print(f'len(hashes) : {len(hashes)}')
         flush_data.block_tx_hashes.clear()
         assert len(hashes) % 32 == 0
 
         # TODO: We need to know why there is +1, and why it is working
+
+        print(f'flush_data.tx_count + 1 : {flush_data.tx_count + 1}')
+        print(f'prior_tx_count : {prior_tx_count}')
         assert not len(hashes) // 32 == flush_data.tx_count + 1 - prior_tx_count
 
         types = b''.join(flush_data.block_tx_types)
         flush_data.block_tx_types.clear()
+        print(f'len(types)  : {len(types) }')
+        print(f'flush_data.tx_count - prior_tx_count : {flush_data.tx_count - prior_tx_count}')
         assert len(types) == flush_data.tx_count - prior_tx_count
 
         # Write the headers, tx counts, and tx hashes

From f99efbcc0093bccb441db4151c2e1b26ca28b7a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= <piotr.sarzynski@minebest.com>
Date: Mon, 7 Feb 2022 11:18:57 +0100
Subject: [PATCH 07/13] ci(release): conditions for release flow in order to
 build docker image from prefixed branch

---
 .github/workflows/release.yml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 265665832..f6ec81906 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -22,10 +22,9 @@ jobs:
           persist-credentials: false
 
       - name: Set environment variables for fix branch
-        run: |
-          if [[ $GITHUB_REF == 'refs/heads/fix/'* ]]; then
-            echo "IMAGE_TAG=fix-$IMAGE_TAG" >> "$GITHUB_ENV"
-          fi
+        if: startsWith(github.ref, "ref/heads/fix")
+        run: 
+          echo "IMAGE_TAG=fix-$IMAGE_TAG" >> "$GITHUB_ENV"
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
@@ -45,6 +44,7 @@ jobs:
 
       - name: Python Semantic Release
         uses: relekang/python-semantic-release@master
+        if: github.ref == 'ref/heads/master'
         with:
           github_token: ${{ secrets.TS_GH_TOKEN }}
           pypi_token: false
@@ -55,6 +55,9 @@ jobs:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
         run: docker pull $ECR_REGISTRY/$ECR_REPOSITORY:latest
 
+      - name: Show version
+        run: cat electrumx/__init__.py 
+
       - name: Builder image
         run: docker build -t $BUILD_NAME .
 

From bb53d83f558f97915c163ae4bca162e33d213fab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= <piotr.sarzynski@minebest.com>
Date: Mon, 7 Feb 2022 11:20:32 +0100
Subject: [PATCH 08/13] ci(release): fix quotation mark in yaml file

---
 .github/workflows/release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index f6ec81906..ff5a5d5eb 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -22,7 +22,7 @@ jobs:
           persist-credentials: false
 
       - name: Set environment variables for fix branch
-        if: startsWith(github.ref, "ref/heads/fix")
+        if: startsWith(github.ref, 'ref/heads/fix')
         run: 
           echo "IMAGE_TAG=fix-$IMAGE_TAG" >> "$GITHUB_ENV"
 

From 29c59c7ca4c30eb25f1feb219357b9a4907c307b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= <piotr.sarzynski@minebest.com>
Date: Mon, 7 Feb 2022 11:34:39 +0100
Subject: [PATCH 09/13] ci(release): add condition to put docker image prefix
 on pull request events

---
 .github/workflows/release.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index ff5a5d5eb..38c01d6fd 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -22,9 +22,9 @@ jobs:
           persist-credentials: false
 
       - name: Set environment variables for fix branch
-        if: startsWith(github.ref, 'ref/heads/fix')
+        if: github.event_name == 'pull_request'
         run: 
-          echo "IMAGE_TAG=fix-$IMAGE_TAG" >> "$GITHUB_ENV"
+          echo "IMAGE_TAG=pull_request-$IMAGE_TAG" >> "$GITHUB_ENV"
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1

From dc0a6db19c36167c06f72e26db7448750e9ceea7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Piotr=20Sarzy=C5=84ski?= <piotr.sarzynski@minebest.com>
Date: Mon, 7 Feb 2022 11:40:03 +0100
Subject: [PATCH 10/13] ci(release): revert condition to set image tag

---
 .github/workflows/release.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 38c01d6fd..c5d6178ac 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -22,9 +22,9 @@ jobs:
           persist-credentials: false
 
       - name: Set environment variables for fix branch
-        if: github.event_name == 'pull_request'
+        if: github.ref != 'ref/heads/master'
         run: 
-          echo "IMAGE_TAG=pull_request-$IMAGE_TAG" >> "$GITHUB_ENV"
+          echo "IMAGE_TAG=fix-$IMAGE_TAG" >> "$GITHUB_ENV"
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1

From 9558ae2894119b42ff258321a4945effde83fb05 Mon Sep 17 00:00:00 2001
From: Jakub Wysocki <jakub.wysocki@cloudbest.com>
Date: Mon, 7 Feb 2022 14:45:19 +0100
Subject: [PATCH 11/13] Less logs

---
 electrumx/server/block_processor.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/electrumx/server/block_processor.py b/electrumx/server/block_processor.py
index 258121183..a6d60b5e6 100644
--- a/electrumx/server/block_processor.py
+++ b/electrumx/server/block_processor.py
@@ -12,7 +12,6 @@
 import asyncio
 from struct import pack, unpack
 import time
-from sys import stdout
 
 from aiorpcx import TaskGroup, run_in_thread
 
@@ -866,8 +865,8 @@ def advance_blocks(self, blocks):
         self.tip = self.coin.header_hash(headers[-1])
 
     def advance_txs_and_atxs(self, txs, atxs):
-        stdout.write(f"Block: {self.height}              \r")
-        stdout.flush()
+        # stdout.write(f"Block: {self.height}              \r'")
+        # stdout.flush()
         if (len(atxs) != 0):
             print(f'---------- advance_txs_and_atxs {self.height} ----------')
             print(f'len(txs) {len(txs)}, ')

From 61ac19cbe61ea093c20239b9a9142e7b9bc4ef04 Mon Sep 17 00:00:00 2001
From: Jakub Wysocki <jakub.wysocki@cloudbest.com>
Date: Thu, 3 Mar 2022 16:01:18 +0100
Subject: [PATCH 12/13] Add new endpoint get_block

---
 electrumx/server/block_processor.py | 2 +-
 electrumx/server/session.py         | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/electrumx/server/block_processor.py b/electrumx/server/block_processor.py
index a6d60b5e6..5af5a3fee 100644
--- a/electrumx/server/block_processor.py
+++ b/electrumx/server/block_processor.py
@@ -351,7 +351,7 @@ async def _maybe_flush(self):
             flush_arg = self.check_cache_size()
             if flush_arg is not None:
                 await self.flush(flush_arg)
-            self.next_cache_check = time.time() + 30
+            self.next_cache_check = time.time() + (60 * 5) # flush every five minutes
 
     def check_cache_size(self):
         '''Flush a cache if it gets too big.'''
diff --git a/electrumx/server/session.py b/electrumx/server/session.py
index 31ba5b4f9..d8127bde4 100644
--- a/electrumx/server/session.py
+++ b/electrumx/server/session.py
@@ -1166,6 +1166,12 @@ async def block_header(self, height, cp_height=0):
         result.update(await self._merkle_proof(cp_height, height))
         return result
 
+    async def get_block(self, block_hash, verbose=False):
+        '''The minimum fee a low-priority tx must pay in order to be accepted
+        to the daemon's memory pool.'''
+        self.bump_cost(1.0)
+        return await self.daemon_request('getblock', block_hash, verbose)
+
     async def block_headers(self, start_height, count, cp_height=0):
         '''Return count concatenated block headers as hex for the main chain;
         starting at start_height.
@@ -1401,6 +1407,7 @@ def set_request_handlers(self, ptuple):
 
         handlers = {
             'blockchain.block.header': self.block_header,
+            'blockchain.block.get_block': self.get_block,
             'blockchain.block.headers': self.block_headers,
             'blockchain.estimatefee': self.estimatefee,
             'blockchain.headers.subscribe': self.headers_subscribe,

From 4b0be8d7a0d5fe3e31d9a02fad50ef67149c21f1 Mon Sep 17 00:00:00 2001
From: Jakub Wysocki <jakub.wysocki@cloudbest.com>
Date: Mon, 7 Mar 2022 14:52:07 +0100
Subject: [PATCH 13/13] Fix missing call

---
 electrumx/server/daemon.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/electrumx/server/daemon.py b/electrumx/server/daemon.py
index 6b8c81880..776c49128 100644
--- a/electrumx/server/daemon.py
+++ b/electrumx/server/daemon.py
@@ -259,6 +259,12 @@ async def getrawtransaction(self, hex_hash, verbose=False):
         return await self._send_single('getrawtransaction',
                                        (hex_hash, int(verbose)))
 
+    async def getblock(self, hex_hash, verbose=False):
+        '''Return the serialized block with the given hash.'''
+        # Cast to int because some coin daemons are old and require it
+        return await self._send_single('getblock',
+                                       (hex_hash, int(verbose)))
+
     async def getrawtransactions(self, hex_hashes, replace_errs=True):
         '''Return the serialized raw transactions with the given hashes.