From 6de1faf0f86d2f79768bfaf3116190b8704ab693 Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Wed, 12 May 2021 12:54:11 -0700
Subject: [PATCH 01/78] lmdb

---
 contracting/db/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 61efd63a..d888d58b 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -379,7 +379,7 @@ def get(self, item: str):
 
 
 class CacheDriver:
-    def __init__(self, driver: Driver=Driver()):
+    def __init__(self, driver: Driver=LMDBDriver()):
         self.driver = driver
         self.cache = {}
 

From 5bb69cbc27180593f31b43716653e1282850930c Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Thu, 10 Jun 2021 15:16:23 -0700
Subject: [PATCH 02/78] new cache

---
 contracting/db/driver.py | 49 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index d888d58b..2a8e49af 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -386,6 +386,19 @@ def __init__(self, driver: Driver=LMDBDriver()):
         self.reads = set()
         self.pending_writes = {}
 
+        self.pending_deltas = {}
+
+    def soft_apply(self, hcl: str, state_changes: dict):
+        deltas = {}
+
+        for k, v in state_changes.items():
+            current = self.get(k)
+            deltas[k] = (current, v)
+
+            self.set(k, v)
+
+        self.pending_deltas[hcl] = deltas
+
     def get(self, key: str, mark=True):
         # Try to get from cache
         v = self.cache.get(key)
@@ -425,6 +438,42 @@ def commit(self):
             else:
                 self.driver.set(k, v)
 
+    def hard_apply(self, hlc):
+        # see if the HCL even exists
+        if self.pending_deltas.get(hlc) is None:
+            return
+
+        # Run through the sorted HCLs from oldest to newest applying each one until the hcl committed is
+
+        to_delete = []
+        for _hlc, _deltas in sorted(self.pending_deltas.items()):
+
+            # Run through all state changes, taking the second value, which is the post delta
+            for key, delta in _deltas.items():
+                self.driver.set(key, delta[1])
+
+                try:
+                    self.cache.pop(key)
+                except KeyError:
+                    pass
+
+            # Add the key (
+            to_delete.append(_hlc)
+            if _hlc == hlc:
+                break
+
+        # Remove the deltas from the set
+        [self.pending_deltas.pop(key) for key in to_delete]
+
+    def rollback(self):
+        # Run through the state changes in reverse, reversing the newest to the oldest
+        for _hlc, _deltas in reversed(sorted(self.pending_deltas.items())):
+            # Run through all state changes, taking the first value, which is the pre delta
+            for key, delta in _deltas.items():
+                self.set(key, delta[0])
+
+        self.pending_deltas.clear()
+
     def clear_pending_state(self):
         self.cache.clear()
         self.reads.clear()

From e950bbde8eb5b1527e5dcd338e3dc60cde8b3d4d Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Mon, 14 Jun 2021 15:04:40 -0700
Subject: [PATCH 03/78] lamdb instead of lmdb

---
 contracting/db/driver.py | 9 ++++++---
 setup.py                 | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 2a8e49af..3b3086bd 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -12,7 +12,7 @@
 from pathlib import Path
 import shutil
 import hashlib
-import lmdb
+import lamdb
 
 FILE_EXT = '.d'
 HASH_EXT = '.x'
@@ -286,8 +286,8 @@ def __init__(self, filename=STORAGE_HOME.joinpath('state')):
         self.filename = filename
         self.filename.mkdir(exist_ok=True, parents=True)
 
-        self.db_writer = lmdb.open(path=str(self.filename), map_size=int(1e12), readonly=False)
-        self.db_reader = lmdb.open(path=str(self.filename), map_size=int(1e12), readonly=True, lock=False)
+        self.db_writer = lamdb.open(path=str(self.filename), map_size=int(1e12), readonly=False)
+        self.db_reader = lamdb.open(path=str(self.filename), map_size=int(1e12), readonly=True, lock=False)
 
     def get(self, item: str):
         with self.db_reader.begin() as tx:
@@ -479,6 +479,9 @@ def clear_pending_state(self):
         self.reads.clear()
         self.pending_writes.clear()
 
+    def snapshot(self):
+        pass
+
 
 class ContractDriver(CacheDriver):
     def __init__(self, *args, **kwargs):
diff --git a/setup.py b/setup.py
index 77b23362..9d1f9737 100644
--- a/setup.py
+++ b/setup.py
@@ -8,7 +8,7 @@
 
 __version__ = '1.0.5.2'
 
-requirements = ['astor', 'pymongo', 'autopep8', 'stdlib_list']
+requirements = ['astor', 'pymongo', 'autopep8', 'stdlib_list', 'lamdb']
 
 ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
 

From b5ba23b9788d5be26b1066b60a420b90f9405a98 Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Wed, 23 Jun 2021 09:19:59 -0700
Subject: [PATCH 04/78] driver get

---
 contracting/db/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 3b3086bd..be4b4574 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -392,7 +392,7 @@ def soft_apply(self, hcl: str, state_changes: dict):
         deltas = {}
 
         for k, v in state_changes.items():
-            current = self.get(k)
+            current = self.driver.get(k)
             deltas[k] = (current, v)
 
             self.set(k, v)

From 97ea541b9690494a4ee3bd88c1219e0a10d3baaa Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Wed, 23 Jun 2021 09:35:36 -0700
Subject: [PATCH 05/78] add tests

---
 tests/unit/test_new_cache_driver.py | 175 +++++++++++++++++++++++++++-
 1 file changed, 174 insertions(+), 1 deletion(-)

diff --git a/tests/unit/test_new_cache_driver.py b/tests/unit/test_new_cache_driver.py
index 6cbcb363..36b9c47c 100644
--- a/tests/unit/test_new_cache_driver.py
+++ b/tests/unit/test_new_cache_driver.py
@@ -66,4 +66,177 @@ def test_clear_pending_state_resets_all_variables(self):
 
         self.assertFalse(len(self.c.cache) > 0)
         self.assertFalse(len(self.c.reads) > 0)
-        self.assertFalse(len(self.c.pending_writes) > 0)
\ No newline at end of file
+        self.assertFalse(len(self.c.pending_writes) > 0)
+
+    def test_soft_apply_adds_changes_to_pending_deltas(self):
+        self.c.set('thing1', 9999)
+
+        state_changes = {
+            'thing1': 8888
+        }
+
+        self.c.soft_apply('0', state_changes)
+
+        expected_deltas = {
+            '0': {
+                'thing1': (9999, 8888)
+            }
+        }
+
+        self.assertDictEqual(self.c.pending_deltas, expected_deltas)
+
+    def test_soft_apply_applies_the_changes_to_the_driver_but_not_hard_driver(self):
+        self.c.set('thing1', 9999)
+        self.c.commit()
+
+        state_changes = {
+            'thing1': 8888
+        }
+
+        self.c.soft_apply('0', state_changes)
+
+        res = self.c.get('thing1')
+
+        self.assertEqual(res, 8888)
+        self.assertEqual(self.c.driver.get('thing1'), 9999)
+
+    def test_hard_apply_applies_hcl_if_exists(self):
+        self.c.set('thing1', 9999)
+        self.c.commit()
+
+        state_changes = {
+            'thing1': 8888
+        }
+
+        self.c.soft_apply('0', state_changes)
+        self.c.hard_apply('0')
+
+        res = self.c.get('thing1')
+
+        self.assertEqual(res, 8888)
+
+        self.assertEqual(self.c.driver.get('thing1'), 8888)
+
+    def test_hard_apply_only_applies_changes_up_to_delta(self):
+        self.c.set('thing1', 9999)
+        self.c.commit()
+
+        state_changes = {
+            'thing1': 8888
+        }
+
+        self.c.soft_apply('0', state_changes)
+
+        state_changes = {
+            'thing1': 7777
+        }
+
+        self.c.soft_apply('1', state_changes)
+
+        state_changes = {
+            'thing1': 6666
+        }
+
+        self.c.soft_apply('2', state_changes)
+
+        self.c.hard_apply('1')
+
+        res = self.c.get('thing1')
+
+        self.assertEqual(res, 7777)
+
+        self.assertEqual(self.c.driver.get('thing1'), 7777)
+
+    def test_hard_apply_removes_hcls(self):
+        self.c.set('thing1', 9999)
+        self.c.commit()
+
+        state_changes = {
+            'thing1': 8888
+        }
+
+        self.c.soft_apply('0', state_changes)
+
+        state_changes = {
+            'thing1': 7777
+        }
+
+        self.c.soft_apply('1', state_changes)
+
+        state_changes = {
+            'thing1': 6666
+        }
+
+        self.c.soft_apply('2', state_changes)
+
+        self.c.hard_apply('0')
+
+        hcls = {
+            '1': {
+                'thing1': (8888, 7777)
+            },
+            '2': {
+                'thing1': (7777, 6666)
+            }
+        }
+
+        self.assertDictEqual(self.c.pending_deltas, hcls)
+
+    def test_rollback_returns_to_initial_state(self):
+        self.c.set('thing1', 9999)
+        self.c.commit()
+
+        state_changes = {
+            'thing1': 8888
+        }
+
+        self.c.soft_apply('0', state_changes)
+        self.assertEqual(self.c.get('thing1'), 8888)
+
+        state_changes = {
+            'thing1': 7777
+        }
+
+        self.c.soft_apply('1', state_changes)
+        self.assertEqual(self.c.get('thing1'), 7777)
+
+        state_changes = {
+            'thing1': 6666
+        }
+
+        self.c.soft_apply('2', state_changes)
+        self.assertEqual(self.c.get('thing1'), 6666)
+
+        self.c.rollback()
+
+        self.assertEqual(self.c.get('thing1'), 9999)
+        self.assertEqual(self.c.driver.get('thing1'), 9999)
+
+    def test_rollback_removes_hlcs(self):
+        self.c.set('thing1', 9999)
+        self.c.commit()
+
+        state_changes = {
+            'thing1': 8888
+        }
+
+        self.c.soft_apply('0', state_changes)
+        self.assertEqual(self.c.get('thing1'), 8888)
+
+        state_changes = {
+            'thing1': 7777
+        }
+
+        self.c.soft_apply('1', state_changes)
+        self.assertEqual(self.c.get('thing1'), 7777)
+
+        state_changes = {
+            'thing1': 6666
+        }
+
+        self.c.soft_apply('2', state_changes)
+        self.assertEqual(self.c.get('thing1'), 6666)
+
+        self.c.rollback()
+
+        self.assertDictEqual(self.c.pending_deltas, {})

From e8054f45c4650d542d96aee63e64f5d5740946d3 Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Wed, 23 Jun 2021 14:16:40 -0700
Subject: [PATCH 06/78] new cache

---
 contracting/db/driver.py            | 122 ++++++++++++++++++---
 contracting/execution/executor.py   |   2 +-
 tests/unit/test_new_cache_driver.py | 158 ++++++++++++----------------
 3 files changed, 176 insertions(+), 106 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index be4b4574..57671758 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -569,15 +569,113 @@ def flush(self):
     def get_contract_keys(self, name):
         return self.keys(name)
 
-    # Set cache to None
-    # Set pending writes to none
-    # def delete(self, key):
-    #     # if self.cache.get(key) is not None:
-    #     #     del self.cache[key]
-    #     #
-    #     # if self.pending_writes.get(key) is not None:
-    #     #     del self.pending_writes[key]
-    #     #
-    #     # self.driver.delete(key)
-    #     self.cache[key] = None
-    #     self.pending_writes[key] = None
\ No newline at end of file
+
+class CacheDriver:
+    def __init__(self, driver: Driver=LMDBDriver()):
+        self.pending_writes = {}    # L2 cache
+        self.cache = {}             # L1 cache
+        self.driver = driver        # L0 cache
+
+        self.pending_reads = {}
+
+        self.pending_deltas = {}
+
+    def find(self, key: str):
+        value = self.pending_writes.get(key)
+        if value is not None:
+            return value
+
+        value = self.cache.get(key)
+        if value is not None:
+            return value
+
+        value = self.driver.get(key)
+        if value is not None:
+            return value
+
+        return None
+
+    def get(self, key: str):
+
+        value = self.find(key)
+
+        if self.pending_reads.get(key) is None:
+            self.pending_reads[key] = value
+
+        if value is not None:
+            rt.deduct_read(*encode_kv(key, value))
+
+        return value
+
+    def set(self, key, value):
+        rt.deduct_write(*encode_kv(key, value))
+
+        if self.pending_reads.get(key) is None:
+            self.get(key)
+
+        if type(value) == decimal.Decimal or type(value) == float:
+            value = ContractingDecimal(str(value))
+
+        self.pending_writes[key] = value
+
+    def soft_apply(self, hcl: str):
+        deltas = {}
+
+        for k, v in self.pending_writes.items():
+            current = self.pending_reads.get(k)
+            deltas[k] = (current, v)
+
+            self.cache[k] = v
+
+        self.pending_deltas[hcl] = deltas
+
+        # Clear the top cache
+        self.pending_reads.clear()
+        self.pending_writes.clear()
+
+    def hard_apply(self, hlc):
+        # see if the HCL even exists
+        if self.pending_deltas.get(hlc) is None:
+            return
+
+        # Run through the sorted HCLs from oldest to newest applying each one until the hcl committed is
+
+        to_delete = []
+        for _hlc, _deltas in sorted(self.pending_deltas.items()):
+
+            # Run through all state changes, taking the second value, which is the post delta
+            for key, delta in _deltas.items():
+                self.driver.set(key, delta[1])
+                self.cache[key] = delta[1]
+
+            # Add the key (
+            to_delete.append(_hlc)
+            if _hlc == hlc:
+                break
+
+        # Remove the deltas from the set
+        [self.pending_deltas.pop(key) for key in to_delete]
+
+    # Same as hard apply but for only the most recent changes and the cache
+    def commit(self):
+        self.cache.update(self.pending_writes)
+
+        for k, v in self.cache.items():
+            if v is None:
+                self.driver.delete(k)
+            else:
+                self.driver.set(k, v)
+
+        self.cache.clear()
+        self.pending_writes.clear()
+        self.pending_reads.clear()
+
+    def rollback(self):
+        # Returns to disk state which should be whatever it was prior to any write sessions
+        self.cache.clear()
+        self.pending_reads.clear()
+        self.pending_writes.clear()
+        self.pending_deltas.clear()
+
+    def clear_pending_state(self):
+        self.rollback()
diff --git a/contracting/execution/executor.py b/contracting/execution/executor.py
index 19b791df..bdcd1e92 100644
--- a/contracting/execution/executor.py
+++ b/contracting/execution/executor.py
@@ -155,7 +155,7 @@ def execute(self, sender, contract_name, function_name, kwargs,
             'result': result,
             'stamps_used': stamps_used,
             'writes': deepcopy(driver.pending_writes),
-            'reads': driver.reads
+            'reads': driver.pending_reads
         }
 
         disable_restricted_imports()
diff --git a/tests/unit/test_new_cache_driver.py b/tests/unit/test_new_cache_driver.py
index 36b9c47c..8fe86264 100644
--- a/tests/unit/test_new_cache_driver.py
+++ b/tests/unit/test_new_cache_driver.py
@@ -1,5 +1,5 @@
 from unittest import TestCase
-from contracting.db.driver import CacheDriver, Driver
+from contracting.db.driver import CacheDriver, Driver, NewCacheDriver
 
 
 class TestCacheDriver(TestCase):
@@ -7,26 +7,16 @@ def setUp(self):
         self.d = Driver()
         self.d.flush()
 
-        self.c = CacheDriver(self.d)
+        self.c = NewCacheDriver(self.d)
 
     def test_get_adds_to_read(self):
         self.c.get('thing')
-        self.assertTrue('thing' in self.c.reads)
+        self.assertTrue('thing' in self.c.pending_reads)
 
     def test_set_adds_to_cache_and_pending_writes(self):
         self.c.set('thing', 1234)
-        self.assertEqual(self.c.cache['thing'], 1234)
         self.assertEqual(self.c.pending_writes['thing'], 1234)
 
-    def test_object_added_to_cache_if_read_from_db(self):
-        self.assertIsNone(self.c.cache.get('thing'))
-
-        self.d.set('thing', 8999)
-
-        self.c.get('thing')
-
-        self.assertEqual(self.c.cache['thing'], 8999)
-
     def test_object_in_cache_returns_from_cache(self):
         self.d.set('thing', 8999)
         self.c.get('thing')
@@ -58,24 +48,19 @@ def test_clear_pending_state_resets_all_variables(self):
         self.c.set('thing2', 1235)
         self.c.get('something')
 
-        self.assertTrue(len(self.c.cache) > 0)
-        self.assertTrue(len(self.c.reads) > 0)
+        self.assertTrue(len(self.c.pending_reads) > 0)
         self.assertTrue(len(self.c.pending_writes) > 0)
 
-        self.c.clear_pending_state()
+        self.c.rollback()
 
-        self.assertFalse(len(self.c.cache) > 0)
-        self.assertFalse(len(self.c.reads) > 0)
+        self.assertFalse(len(self.c.pending_reads) > 0)
         self.assertFalse(len(self.c.pending_writes) > 0)
 
     def test_soft_apply_adds_changes_to_pending_deltas(self):
-        self.c.set('thing1', 9999)
-
-        state_changes = {
-            'thing1': 8888
-        }
+        self.c.driver.set('thing1', 9999)
 
-        self.c.soft_apply('0', state_changes)
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
 
         expected_deltas = {
             '0': {
@@ -89,11 +74,8 @@ def test_soft_apply_applies_the_changes_to_the_driver_but_not_hard_driver(self):
         self.c.set('thing1', 9999)
         self.c.commit()
 
-        state_changes = {
-            'thing1': 8888
-        }
-
-        self.c.soft_apply('0', state_changes)
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
 
         res = self.c.get('thing1')
 
@@ -104,11 +86,9 @@ def test_hard_apply_applies_hcl_if_exists(self):
         self.c.set('thing1', 9999)
         self.c.commit()
 
-        state_changes = {
-            'thing1': 8888
-        }
+        self.c.set('thing1', 8888)
 
-        self.c.soft_apply('0', state_changes)
+        self.c.soft_apply('0')
         self.c.hard_apply('0')
 
         res = self.c.get('thing1')
@@ -121,23 +101,14 @@ def test_hard_apply_only_applies_changes_up_to_delta(self):
         self.c.set('thing1', 9999)
         self.c.commit()
 
-        state_changes = {
-            'thing1': 8888
-        }
-
-        self.c.soft_apply('0', state_changes)
-
-        state_changes = {
-            'thing1': 7777
-        }
-
-        self.c.soft_apply('1', state_changes)
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
 
-        state_changes = {
-            'thing1': 6666
-        }
+        self.c.set('thing1', 7777)
+        self.c.soft_apply('1')
 
-        self.c.soft_apply('2', state_changes)
+        self.c.set('thing1', 6666)
+        self.c.soft_apply('2')
 
         self.c.hard_apply('1')
 
@@ -151,23 +122,14 @@ def test_hard_apply_removes_hcls(self):
         self.c.set('thing1', 9999)
         self.c.commit()
 
-        state_changes = {
-            'thing1': 8888
-        }
-
-        self.c.soft_apply('0', state_changes)
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
 
-        state_changes = {
-            'thing1': 7777
-        }
-
-        self.c.soft_apply('1', state_changes)
+        self.c.set('thing1', 7777)
+        self.c.soft_apply('1')
 
-        state_changes = {
-            'thing1': 6666
-        }
-
-        self.c.soft_apply('2', state_changes)
+        self.c.set('thing1', 6666)
+        self.c.soft_apply('2')
 
         self.c.hard_apply('0')
 
@@ -186,25 +148,16 @@ def test_rollback_returns_to_initial_state(self):
         self.c.set('thing1', 9999)
         self.c.commit()
 
-        state_changes = {
-            'thing1': 8888
-        }
-
-        self.c.soft_apply('0', state_changes)
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
         self.assertEqual(self.c.get('thing1'), 8888)
 
-        state_changes = {
-            'thing1': 7777
-        }
-
-        self.c.soft_apply('1', state_changes)
+        self.c.set('thing1', 7777)
+        self.c.soft_apply('1')
         self.assertEqual(self.c.get('thing1'), 7777)
 
-        state_changes = {
-            'thing1': 6666
-        }
-
-        self.c.soft_apply('2', state_changes)
+        self.c.set('thing1', 6666)
+        self.c.soft_apply('2')
         self.assertEqual(self.c.get('thing1'), 6666)
 
         self.c.rollback()
@@ -216,27 +169,46 @@ def test_rollback_removes_hlcs(self):
         self.c.set('thing1', 9999)
         self.c.commit()
 
-        state_changes = {
-            'thing1': 8888
-        }
-
-        self.c.soft_apply('0', state_changes)
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
         self.assertEqual(self.c.get('thing1'), 8888)
 
-        state_changes = {
-            'thing1': 7777
-        }
-
-        self.c.soft_apply('1', state_changes)
+        self.c.set('thing1', 7777)
+        self.c.soft_apply('1')
         self.assertEqual(self.c.get('thing1'), 7777)
 
-        state_changes = {
-            'thing1': 6666
-        }
-
-        self.c.soft_apply('2', state_changes)
+        self.c.set('thing1', 6666)
+        self.c.soft_apply('2')
         self.assertEqual(self.c.get('thing1'), 6666)
 
         self.c.rollback()
 
         self.assertDictEqual(self.c.pending_deltas, {})
+
+    def test_find_returns_none(self):
+        x = self.c.find('none')
+        self.assertIsNone(x)
+
+    def test_find_returns_driver(self):
+        self.c.driver.set('none', 123)
+
+        x = self.c.find('none')
+
+        self.assertEqual(x, 123)
+
+    def test_find_returns_cache(self):
+        self.c.driver.set('none', 123)
+        self.c.cache['none'] = 999
+
+        x = self.c.find('none')
+
+        self.assertEqual(x, 999)
+
+    def test_find_returns_pending_writes(self):
+        self.c.driver.set('none', 123)
+        self.c.cache['none'] = 999
+        self.c.pending_writes['none'] = 5555
+
+        x = self.c.find('none')
+
+        self.assertEqual(x, 5555)

From 2329351c97f6f95881ebb6f268fb45f7d4c1002a Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Wed, 23 Jun 2021 14:27:02 -0700
Subject: [PATCH 07/78] new cache integrated

---
 contracting/db/driver.py               | 336 +++++++++++++------------
 tests/unit/test_new_cache_driver.py    |   4 +-
 tests/unit/test_new_contract_driver.py |   3 +
 3 files changed, 178 insertions(+), 165 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 57671758..b58165e8 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -378,65 +378,175 @@ def get(self, item: str):
         return decode(r.json()['value'])
 
 
+# class CacheDriver:
+#     def __init__(self, driver: Driver=LMDBDriver()):
+#         self.driver = driver
+#         self.cache = {}
+#
+#         self.reads = set()
+#         self.pending_writes = {}
+#
+#         self.pending_deltas = {}
+#
+#     def soft_apply(self, hcl: str, state_changes: dict):
+#         deltas = {}
+#
+#         for k, v in state_changes.items():
+#             current = self.driver.get(k)
+#             deltas[k] = (current, v)
+#
+#             self.set(k, v)
+#
+#         self.pending_deltas[hcl] = deltas
+#
+#     def get(self, key: str, mark=True):
+#         # Try to get from cache
+#         v = self.cache.get(key)
+#         if v is not None:
+#             rt.deduct_read(*encode_kv(key, v))
+#             return v
+#
+#         # If it doesn't exist, get from db, add to cache
+#         dv = self.driver.get(key)
+#         rt.deduct_read(*encode_kv(key, dv))
+#
+#         self.cache[key] = dv
+#
+#         # Add key to reads
+#         if mark:
+#             self.reads.add(key)
+#
+#         return dv
+#
+#     def set(self, key, value, mark=True):
+#         rt.deduct_write(*encode_kv(key, value))
+#
+#         if type(value) == decimal.Decimal or type(value) == float:
+#             value = ContractingDecimal(str(value))
+#
+#         self.cache[key] = value
+#         if mark:
+#             self.pending_writes[key] = value
+#
+#     def delete(self, key, mark=True):
+#         self.set(key, None, mark=mark)
+#
+#     def commit(self):
+#         for k, v in self.pending_writes.items():
+#             if v is None:
+#                 self.driver.delete(k)
+#             else:
+#                 self.driver.set(k, v)
+#
+#     def hard_apply(self, hlc):
+#         # see if the HCL even exists
+#         if self.pending_deltas.get(hlc) is None:
+#             return
+#
+#         # Run through the sorted HCLs from oldest to newest applying each one until the hcl committed is
+#
+#         to_delete = []
+#         for _hlc, _deltas in sorted(self.pending_deltas.items()):
+#
+#             # Run through all state changes, taking the second value, which is the post delta
+#             for key, delta in _deltas.items():
+#                 self.driver.set(key, delta[1])
+#
+#                 try:
+#                     self.cache.pop(key)
+#                 except KeyError:
+#                     pass
+#
+#             # Add the key (
+#             to_delete.append(_hlc)
+#             if _hlc == hlc:
+#                 break
+#
+#         # Remove the deltas from the set
+#         [self.pending_deltas.pop(key) for key in to_delete]
+#
+#     def rollback(self):
+#         # Run through the state changes in reverse, reversing the newest to the oldest
+#         for _hlc, _deltas in reversed(sorted(self.pending_deltas.items())):
+#             # Run through all state changes, taking the first value, which is the pre delta
+#             for key, delta in _deltas.items():
+#                 self.set(key, delta[0])
+#
+#         self.pending_deltas.clear()
+#
+#     def clear_pending_state(self):
+#         self.cache.clear()
+#         self.reads.clear()
+#         self.pending_writes.clear()
+#
+#     def snapshot(self):
+#         pass
+
 class CacheDriver:
     def __init__(self, driver: Driver=LMDBDriver()):
-        self.driver = driver
-        self.cache = {}
+        self.pending_writes = {}    # L2 cache
+        self.cache = {}             # L1 cache
+        self.driver = driver        # L0 cache
 
-        self.reads = set()
-        self.pending_writes = {}
+        self.pending_reads = {}
 
         self.pending_deltas = {}
 
-    def soft_apply(self, hcl: str, state_changes: dict):
-        deltas = {}
+    def find(self, key: str):
+        value = self.pending_writes.get(key)
+        if value is not None:
+            return value
 
-        for k, v in state_changes.items():
-            current = self.driver.get(k)
-            deltas[k] = (current, v)
+        value = self.cache.get(key)
+        if value is not None:
+            return value
 
-            self.set(k, v)
+        value = self.driver.get(key)
+        if value is not None:
+            return value
 
-        self.pending_deltas[hcl] = deltas
+        return None
 
-    def get(self, key: str, mark=True):
-        # Try to get from cache
-        v = self.cache.get(key)
-        if v is not None:
-            rt.deduct_read(*encode_kv(key, v))
-            return v
+    def get(self, key: str):
 
-        # If it doesn't exist, get from db, add to cache
-        dv = self.driver.get(key)
-        rt.deduct_read(*encode_kv(key, dv))
+        value = self.find(key)
 
-        self.cache[key] = dv
+        if self.pending_reads.get(key) is None:
+            self.pending_reads[key] = value
 
-        # Add key to reads
-        if mark:
-            self.reads.add(key)
+        if value is not None:
+            rt.deduct_read(*encode_kv(key, value))
 
-        return dv
+        return value
 
-    def set(self, key, value, mark=True):
+    def set(self, key, value):
         rt.deduct_write(*encode_kv(key, value))
 
+        if self.pending_reads.get(key) is None:
+            self.get(key)
+
         if type(value) == decimal.Decimal or type(value) == float:
             value = ContractingDecimal(str(value))
 
-        self.cache[key] = value
-        if mark:
-            self.pending_writes[key] = value
+        self.pending_writes[key] = value
 
-    def delete(self, key, mark=True):
-        self.set(key, None, mark=mark)
+    def delete(self, key):
+        self.set(key, None)
+
+    def soft_apply(self, hcl: str):
+        deltas = {}
 
-    def commit(self):
         for k, v in self.pending_writes.items():
-            if v is None:
-                self.driver.delete(k)
-            else:
-                self.driver.set(k, v)
+            current = self.pending_reads.get(k)
+            deltas[k] = (current, v)
+
+            self.cache[k] = v
+
+        self.pending_deltas[hcl] = deltas
+
+        # Clear the top cache
+        self.pending_reads.clear()
+        self.pending_writes.clear()
 
     def hard_apply(self, hlc):
         # see if the HCL even exists
@@ -451,11 +561,7 @@ def hard_apply(self, hlc):
             # Run through all state changes, taking the second value, which is the post delta
             for key, delta in _deltas.items():
                 self.driver.set(key, delta[1])
-
-                try:
-                    self.cache.pop(key)
-                except KeyError:
-                    pass
+                self.cache[key] = delta[1]
 
             # Add the key (
             to_delete.append(_hlc)
@@ -465,22 +571,29 @@ def hard_apply(self, hlc):
         # Remove the deltas from the set
         [self.pending_deltas.pop(key) for key in to_delete]
 
-    def rollback(self):
-        # Run through the state changes in reverse, reversing the newest to the oldest
-        for _hlc, _deltas in reversed(sorted(self.pending_deltas.items())):
-            # Run through all state changes, taking the first value, which is the pre delta
-            for key, delta in _deltas.items():
-                self.set(key, delta[0])
+    # Same as hard apply but for only the most recent changes and the cache
+    def commit(self):
+        self.cache.update(self.pending_writes)
 
-        self.pending_deltas.clear()
+        for k, v in self.cache.items():
+            if v is None:
+                self.driver.delete(k)
+            else:
+                self.driver.set(k, v)
 
-    def clear_pending_state(self):
         self.cache.clear()
-        self.reads.clear()
         self.pending_writes.clear()
+        self.pending_reads.clear()
 
-    def snapshot(self):
-        pass
+    def rollback(self):
+        # Returns to disk state which should be whatever it was prior to any write sessions
+        self.cache.clear()
+        self.pending_reads.clear()
+        self.pending_writes.clear()
+        self.pending_deltas.clear()
+
+    def clear_pending_state(self):
+        self.rollback()
 
 
 class ContractDriver(CacheDriver):
@@ -492,6 +605,12 @@ def items(self, prefix=''):
         # Get all of the items in the cache currently
         _items = {}
         keys = set()
+
+        for k, v in self.pending_writes.items():
+            if k.startswith(prefix) and v is not None:
+                _items[k] = v
+                keys.add(k)
+
         for k, v in self.cache.items():
             if k.startswith(prefix) and v is not None:
                 _items[k] = v
@@ -520,11 +639,11 @@ def make_key(self, contract, variable, args=[]):
 
     def get_var(self, contract, variable, arguments=[], mark=True):
         key = self.make_key(contract, variable, arguments)
-        return self.get(key, mark=mark)
+        return self.get(key)
 
     def set_var(self, contract, variable, arguments=[], value=None, mark=True):
         key = self.make_key(contract, variable, arguments)
-        self.set(key, value, mark=mark)
+        self.set(key, value)
 
     def get_contract(self, name):
         return self.get_var(name, CODE_KEY)
@@ -570,112 +689,3 @@ def get_contract_keys(self, name):
         return self.keys(name)
 
 
-class CacheDriver:
-    def __init__(self, driver: Driver=LMDBDriver()):
-        self.pending_writes = {}    # L2 cache
-        self.cache = {}             # L1 cache
-        self.driver = driver        # L0 cache
-
-        self.pending_reads = {}
-
-        self.pending_deltas = {}
-
-    def find(self, key: str):
-        value = self.pending_writes.get(key)
-        if value is not None:
-            return value
-
-        value = self.cache.get(key)
-        if value is not None:
-            return value
-
-        value = self.driver.get(key)
-        if value is not None:
-            return value
-
-        return None
-
-    def get(self, key: str):
-
-        value = self.find(key)
-
-        if self.pending_reads.get(key) is None:
-            self.pending_reads[key] = value
-
-        if value is not None:
-            rt.deduct_read(*encode_kv(key, value))
-
-        return value
-
-    def set(self, key, value):
-        rt.deduct_write(*encode_kv(key, value))
-
-        if self.pending_reads.get(key) is None:
-            self.get(key)
-
-        if type(value) == decimal.Decimal or type(value) == float:
-            value = ContractingDecimal(str(value))
-
-        self.pending_writes[key] = value
-
-    def soft_apply(self, hcl: str):
-        deltas = {}
-
-        for k, v in self.pending_writes.items():
-            current = self.pending_reads.get(k)
-            deltas[k] = (current, v)
-
-            self.cache[k] = v
-
-        self.pending_deltas[hcl] = deltas
-
-        # Clear the top cache
-        self.pending_reads.clear()
-        self.pending_writes.clear()
-
-    def hard_apply(self, hlc):
-        # see if the HCL even exists
-        if self.pending_deltas.get(hlc) is None:
-            return
-
-        # Run through the sorted HCLs from oldest to newest applying each one until the hcl committed is
-
-        to_delete = []
-        for _hlc, _deltas in sorted(self.pending_deltas.items()):
-
-            # Run through all state changes, taking the second value, which is the post delta
-            for key, delta in _deltas.items():
-                self.driver.set(key, delta[1])
-                self.cache[key] = delta[1]
-
-            # Add the key (
-            to_delete.append(_hlc)
-            if _hlc == hlc:
-                break
-
-        # Remove the deltas from the set
-        [self.pending_deltas.pop(key) for key in to_delete]
-
-    # Same as hard apply but for only the most recent changes and the cache
-    def commit(self):
-        self.cache.update(self.pending_writes)
-
-        for k, v in self.cache.items():
-            if v is None:
-                self.driver.delete(k)
-            else:
-                self.driver.set(k, v)
-
-        self.cache.clear()
-        self.pending_writes.clear()
-        self.pending_reads.clear()
-
-    def rollback(self):
-        # Returns to disk state which should be whatever it was prior to any write sessions
-        self.cache.clear()
-        self.pending_reads.clear()
-        self.pending_writes.clear()
-        self.pending_deltas.clear()
-
-    def clear_pending_state(self):
-        self.rollback()
diff --git a/tests/unit/test_new_cache_driver.py b/tests/unit/test_new_cache_driver.py
index 8fe86264..f24ca3db 100644
--- a/tests/unit/test_new_cache_driver.py
+++ b/tests/unit/test_new_cache_driver.py
@@ -1,5 +1,5 @@
 from unittest import TestCase
-from contracting.db.driver import CacheDriver, Driver, NewCacheDriver
+from contracting.db.driver import CacheDriver, Driver
 
 
 class TestCacheDriver(TestCase):
@@ -7,7 +7,7 @@ def setUp(self):
         self.d = Driver()
         self.d.flush()
 
-        self.c = NewCacheDriver(self.d)
+        self.c = CacheDriver(self.d)
 
     def test_get_adds_to_read(self):
         self.c.get('thing')
diff --git a/tests/unit/test_new_contract_driver.py b/tests/unit/test_new_contract_driver.py
index 04a8180a..7be80e76 100644
--- a/tests/unit/test_new_contract_driver.py
+++ b/tests/unit/test_new_contract_driver.py
@@ -5,6 +5,7 @@
 import marshal
 from datetime import datetime
 
+
 class TestContractDriver(TestCase):
     def setUp(self):
         self.d = Driver()
@@ -129,6 +130,8 @@ def test_items_in_both_cache_and_db_works(self):
         for k, v in kvs_1.items():
             self.c.set(k, v)
 
+        self.c.commit()
+
         for k, v in kvs_2.items():
             self.c.driver.set(k, v)
 

From 598611061b968211ab957d63576aca57a585b3ae Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Wed, 4 Aug 2021 10:59:22 -0700
Subject: [PATCH 08/78] no cache hard apply

---
 contracting/db/driver.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index b58165e8..5e5cf2e2 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -533,6 +533,7 @@ def set(self, key, value):
     def delete(self, key):
         self.set(key, None)
 
+    #TODO: Fix bug where rolling back on a key written to twice rolls back to the initial state instead of the immediate previous value
     def soft_apply(self, hcl: str):
         deltas = {}
 
@@ -561,7 +562,7 @@ def hard_apply(self, hlc):
             # Run through all state changes, taking the second value, which is the post delta
             for key, delta in _deltas.items():
                 self.driver.set(key, delta[1])
-                self.cache[key] = delta[1]
+                # self.cache[key] = delta[1]
 
             # Add the key (
             to_delete.append(_hlc)

From a550df995895be46662e044d5341170e01be35c5 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Mon, 9 Aug 2021 11:25:10 -0400
Subject: [PATCH 09/78] handle OSError in the FS_DRIVER flush method for
 testing.  SHould probably figure out why this is happening

---
 contracting/db/driver.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 5e5cf2e2..b8086af5 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -178,6 +178,10 @@ def flush(self):
             shutil.rmtree(self.root)
         except FileNotFoundError:
             pass
+        except OSError as os_err:
+            # TO DO This was causing a "Directory not empty" error so I'm accepting it.
+            # I don't know why the error is happening
+            print(os_err)
 
     def delete(self, key: str):
         self.__delitem__(key)

From 3953fe0253425a8fdb6ad15a0c085a4e55beae92 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Thu, 12 Aug 2021 14:56:37 -0400
Subject: [PATCH 10/78] Encode balance in executor

---
 contracting/execution/executor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contracting/execution/executor.py b/contracting/execution/executor.py
index bdcd1e92..2f62f1b1 100644
--- a/contracting/execution/executor.py
+++ b/contracting/execution/executor.py
@@ -69,9 +69,9 @@ def execute(self, sender, contract_name, function_name, kwargs,
 
                 balance = driver.get(balances_key)
                 if balance is None:
-                    balance = 0
+                    balance = ContractingDecimal(0)
 
-                assert balance * stamp_cost >= stamps, 'Sender does not have enough stamps for the transaction. \
+                assert balance * ContractingDecimal(stamp_cost) >= stamps, 'Sender does not have enough stamps for the transaction. \
                                                                Balance at key {} is {}'.format(balances_key,
                                                                                                balance)
 

From 23a6cbd7da7cdba372ca805a23e1b984f7a842ee Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Thu, 12 Aug 2021 15:02:13 -0400
Subject: [PATCH 11/78] log out values

---
 contracting/execution/executor.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/contracting/execution/executor.py b/contracting/execution/executor.py
index 2f62f1b1..efe773ea 100644
--- a/contracting/execution/executor.py
+++ b/contracting/execution/executor.py
@@ -68,10 +68,17 @@ def execute(self, sender, contract_name, function_name, kwargs,
                                                    sender)
 
                 balance = driver.get(balances_key)
+
                 if balance is None:
-                    balance = ContractingDecimal(0)
+                    balance = 0
+
+                log.debug({
+                    'balance':balance,
+                    'stamp_cost': stamp_cost,
+                    'stamps': stamps
+                })
 
-                assert balance * ContractingDecimal(stamp_cost) >= stamps, 'Sender does not have enough stamps for the transaction. \
+                assert balance * stamp_cost >= stamps, 'Sender does not have enough stamps for the transaction. \
                                                                Balance at key {} is {}'.format(balances_key,
                                                                                                balance)
 

From 6f11b181d4fd90cc576aca9a652785f99ebef700 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Thu, 12 Aug 2021 15:07:30 -0400
Subject: [PATCH 12/78] convert balance to ContractingDecimal

---
 contracting/execution/executor.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/contracting/execution/executor.py b/contracting/execution/executor.py
index efe773ea..3b915843 100644
--- a/contracting/execution/executor.py
+++ b/contracting/execution/executor.py
@@ -69,6 +69,9 @@ def execute(self, sender, contract_name, function_name, kwargs,
 
                 balance = driver.get(balances_key)
 
+                if balance.get('__fixed__') is not None:
+                    balance = ContractingDecimal(balance.get('__fixed__'))
+
                 if balance is None:
                     balance = 0
 

From 7eadbcdba756d0d7c4f81955467199a248dd9673 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Thu, 12 Aug 2021 15:17:08 -0400
Subject: [PATCH 13/78] troubleshooting statements

---
 contracting/execution/executor.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/contracting/execution/executor.py b/contracting/execution/executor.py
index 3b915843..76049e4c 100644
--- a/contracting/execution/executor.py
+++ b/contracting/execution/executor.py
@@ -76,7 +76,7 @@ def execute(self, sender, contract_name, function_name, kwargs,
                     balance = 0
 
                 log.debug({
-                    'balance':balance,
+                    'balance': balance,
                     'stamp_cost': stamp_cost,
                     'stamps': stamps
                 })
@@ -85,6 +85,8 @@ def execute(self, sender, contract_name, function_name, kwargs,
                                                                Balance at key {} is {}'.format(balances_key,
                                                                                                balance)
 
+                log.info("GOT HERE 1")
+
             runtime.rt.env.update(environment)
             status_code = 0
             runtime.rt.set_up(stmps=stamps * 1000, meter=metering) # Multiply stamps by 1000 because we divide by it later
@@ -109,7 +111,9 @@ def execute(self, sender, contract_name, function_name, kwargs,
                     kwargs[k] = ContractingDecimal(str(v))
 
             enable_restricted_imports()
+            log.info("GOT HERE 2")
             result = func(**kwargs)
+            log.info("GOT HERE 3")
             disable_restricted_imports()
 
             if auto_commit:

From 55d8c07ffdd68509c698ff446c69099c1e933ed1 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Thu, 12 Aug 2021 15:19:59 -0400
Subject: [PATCH 14/78] if balance is dict then make ContractingDecimal

---
 contracting/execution/executor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/execution/executor.py b/contracting/execution/executor.py
index 76049e4c..5f33349b 100644
--- a/contracting/execution/executor.py
+++ b/contracting/execution/executor.py
@@ -69,7 +69,7 @@ def execute(self, sender, contract_name, function_name, kwargs,
 
                 balance = driver.get(balances_key)
 
-                if balance.get('__fixed__') is not None:
+                if type(balance) == dict:
                     balance = ContractingDecimal(balance.get('__fixed__'))
 
                 if balance is None:

From 2233e083b16cfc6fdfa7c72b67cee50604a80be6 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Thu, 12 Aug 2021 16:57:57 -0400
Subject: [PATCH 15/78] remove troubleshooting

---
 contracting/execution/executor.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/contracting/execution/executor.py b/contracting/execution/executor.py
index 5f33349b..01ac8e4a 100644
--- a/contracting/execution/executor.py
+++ b/contracting/execution/executor.py
@@ -85,8 +85,6 @@ def execute(self, sender, contract_name, function_name, kwargs,
                                                                Balance at key {} is {}'.format(balances_key,
                                                                                                balance)
 
-                log.info("GOT HERE 1")
-
             runtime.rt.env.update(environment)
             status_code = 0
             runtime.rt.set_up(stmps=stamps * 1000, meter=metering) # Multiply stamps by 1000 because we divide by it later
@@ -111,9 +109,7 @@ def execute(self, sender, contract_name, function_name, kwargs,
                     kwargs[k] = ContractingDecimal(str(v))
 
             enable_restricted_imports()
-            log.info("GOT HERE 2")
             result = func(**kwargs)
-            log.info("GOT HERE 3")
             disable_restricted_imports()
 
             if auto_commit:

From be4711e376adfef48dfbd799e71015bbe3ee34c4 Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Sat, 4 Sep 2021 17:42:24 -0700
Subject: [PATCH 16/78] new rollback

---
 contracting/db/driver.py            | 38 ++++++++++++++++---
 tests/unit/test_new_cache_driver.py | 59 ++++++++++++++++++++++++++++-
 2 files changed, 90 insertions(+), 7 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 5e5cf2e2..25a6f274 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -586,12 +586,38 @@ def commit(self):
         self.pending_writes.clear()
         self.pending_reads.clear()
 
-    def rollback(self):
-        # Returns to disk state which should be whatever it was prior to any write sessions
-        self.cache.clear()
-        self.pending_reads.clear()
-        self.pending_writes.clear()
-        self.pending_deltas.clear()
+    def rollback(self, hlc=None):
+        if hlc is None:
+            # Returns to disk state which should be whatever it was prior to any write sessions
+            self.cache.clear()
+            self.pending_reads.clear()
+            self.pending_writes.clear()
+            self.pending_deltas.clear()
+        else:
+            if self.pending_deltas.get(hlc) is None:
+                return
+
+                # Run through the sorted HCLs from oldest to newest applying each one until the hcl committed is
+
+            to_delete = []
+            for _hlc, _deltas in sorted(self.pending_deltas.items())[::-1]:
+                # Clears the current reads/writes, and the reads/writes that get made when rolling back from the
+                # last HLC
+                self.pending_reads.clear()
+                self.pending_writes.clear()
+
+                # Run through all state changes, taking the second value, which is the post delta
+                for key, delta in _deltas.items():
+                    self.set(key, delta[0])
+                    # self.cache[key] = delta[1]
+
+                # Add the key (
+                to_delete.append(_hlc)
+                if _hlc == hlc:
+                    break
+
+            # Remove the deltas from the set
+            [self.pending_deltas.pop(key) for key in to_delete]
 
     def clear_pending_state(self):
         self.rollback()
diff --git a/tests/unit/test_new_cache_driver.py b/tests/unit/test_new_cache_driver.py
index f24ca3db..e79b9012 100644
--- a/tests/unit/test_new_cache_driver.py
+++ b/tests/unit/test_new_cache_driver.py
@@ -97,6 +97,60 @@ def test_hard_apply_applies_hcl_if_exists(self):
 
         self.assertEqual(self.c.driver.get('thing1'), 8888)
 
+    def test_rollback_applies_hcl_if_exists(self):
+        self.c.set('thing1', 9999)
+        self.c.commit()
+
+        self.c.set('thing1', 8888)
+
+        self.c.soft_apply('0')
+        self.c.rollback('0')
+
+        res = self.c.get('thing1')
+
+        self.assertEqual(res, 9999)
+
+        self.assertEqual(self.c.driver.get('thing1'), 9999)
+
+    def test_rollback_twice_returns(self):
+        self.c.set('thing1', 9999)
+        self.c.commit()
+
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
+
+        self.c.set('thing1', 7777)
+        self.c.soft_apply('1')
+
+        self.c.set('thing1', 6666)
+        self.c.soft_apply('2')
+
+        self.c.rollback('1')
+
+        res = self.c.get('thing1')
+
+        self.assertEqual(res, 8888)
+
+        self.assertEqual(self.c.driver.get('thing1'), 9999)
+
+    def test_rollback_removes_hlcs(self):
+        self.c.set('thing1', 9999)
+        self.c.commit()
+
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
+
+        self.c.set('thing1', 7777)
+        self.c.soft_apply('1')
+
+        self.c.set('thing1', 6666)
+        self.c.soft_apply('2')
+
+        self.c.rollback('1')
+
+        self.assertIsNone(self.c.pending_deltas.get('2'))
+        self.assertIsNone(self.c.pending_deltas.get('1'))
+
     def test_hard_apply_only_applies_changes_up_to_delta(self):
         self.c.set('thing1', 9999)
         self.c.commit()
@@ -110,11 +164,14 @@ def test_hard_apply_only_applies_changes_up_to_delta(self):
         self.c.set('thing1', 6666)
         self.c.soft_apply('2')
 
+        self.c.set('thing1', 5555)
+        self.c.soft_apply('3')
+
         self.c.hard_apply('1')
 
         res = self.c.get('thing1')
 
-        self.assertEqual(res, 7777)
+        self.assertEqual(res, 5555)
 
         self.assertEqual(self.c.driver.get('thing1'), 7777)
 

From 6183a5b6b8c1d4902e7476ca7b04cdd7114ce09d Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Sat, 4 Sep 2021 17:51:41 -0700
Subject: [PATCH 17/78] reads

---
 contracting/db/driver.py            | 16 +++++++++++++---
 tests/unit/test_new_cache_driver.py | 18 ++++++++----------
 2 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 0f0d4fdc..1a5a7210 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -486,6 +486,13 @@ def get(self, item: str):
 #     def snapshot(self):
 #         pass
 
+
+class Delta:
+    def __init__(self, writes, reads):
+        self.writes = writes
+        self.reads = reads
+
+
 class CacheDriver:
     def __init__(self, driver: Driver=LMDBDriver()):
         self.pending_writes = {}    # L2 cache
@@ -547,7 +554,10 @@ def soft_apply(self, hcl: str):
 
             self.cache[k] = v
 
-        self.pending_deltas[hcl] = deltas
+        self.pending_deltas[hcl] = {
+            'writes': deltas,
+            'reads': self.pending_reads
+        }
 
         # Clear the top cache
         self.pending_reads.clear()
@@ -564,7 +574,7 @@ def hard_apply(self, hlc):
         for _hlc, _deltas in sorted(self.pending_deltas.items()):
 
             # Run through all state changes, taking the second value, which is the post delta
-            for key, delta in _deltas.items():
+            for key, delta in _deltas['writes'].items():
                 self.driver.set(key, delta[1])
                 # self.cache[key] = delta[1]
 
@@ -611,7 +621,7 @@ def rollback(self, hlc=None):
                 self.pending_writes.clear()
 
                 # Run through all state changes, taking the second value, which is the post delta
-                for key, delta in _deltas.items():
+                for key, delta in _deltas['writes'].items():
                     self.set(key, delta[0])
                     # self.cache[key] = delta[1]
 
diff --git a/tests/unit/test_new_cache_driver.py b/tests/unit/test_new_cache_driver.py
index e79b9012..17848c85 100644
--- a/tests/unit/test_new_cache_driver.py
+++ b/tests/unit/test_new_cache_driver.py
@@ -64,7 +64,8 @@ def test_soft_apply_adds_changes_to_pending_deltas(self):
 
         expected_deltas = {
             '0': {
-                'thing1': (9999, 8888)
+                'writes': {'thing1': (9999, 8888)},
+                'reads': {}
             }
         }
 
@@ -190,16 +191,13 @@ def test_hard_apply_removes_hcls(self):
 
         self.c.hard_apply('0')
 
-        hcls = {
-            '1': {
-                'thing1': (8888, 7777)
-            },
-            '2': {
-                'thing1': (7777, 6666)
-            }
-        }
+        hlcs = {'1':
+                    {'writes': {'thing1': (8888, 7777)}, 'reads': {}},
+                '2':
+                    {'writes': {'thing1': (7777, 6666)}, 'reads': {}}
+                }
 
-        self.assertDictEqual(self.c.pending_deltas, hcls)
+        self.assertDictEqual(self.c.pending_deltas, hlcs)
 
     def test_rollback_returns_to_initial_state(self):
         self.c.set('thing1', 9999)

From 627d2664ad5e689f8ec93d7086d61acfd2643c41 Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Sat, 4 Sep 2021 17:57:01 -0700
Subject: [PATCH 18/78] reads get passed

---
 contracting/db/driver.py            | 8 ++++----
 tests/unit/test_new_cache_driver.py | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 1a5a7210..a610c3df 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -560,7 +560,7 @@ def soft_apply(self, hcl: str):
         }
 
         # Clear the top cache
-        self.pending_reads.clear()
+        self.pending_reads = {}
         self.pending_writes.clear()
 
     def hard_apply(self, hlc):
@@ -598,13 +598,13 @@ def commit(self):
 
         self.cache.clear()
         self.pending_writes.clear()
-        self.pending_reads.clear()
+        self.pending_reads = {}
 
     def rollback(self, hlc=None):
         if hlc is None:
             # Returns to disk state which should be whatever it was prior to any write sessions
             self.cache.clear()
-            self.pending_reads.clear()
+            self.pending_reads = {}
             self.pending_writes.clear()
             self.pending_deltas.clear()
         else:
@@ -617,7 +617,7 @@ def rollback(self, hlc=None):
             for _hlc, _deltas in sorted(self.pending_deltas.items())[::-1]:
                 # Clears the current reads/writes, and the reads/writes that get made when rolling back from the
                 # last HLC
-                self.pending_reads.clear()
+                self.pending_reads = {}
                 self.pending_writes.clear()
 
                 # Run through all state changes, taking the second value, which is the post delta
diff --git a/tests/unit/test_new_cache_driver.py b/tests/unit/test_new_cache_driver.py
index 17848c85..a148c91e 100644
--- a/tests/unit/test_new_cache_driver.py
+++ b/tests/unit/test_new_cache_driver.py
@@ -65,7 +65,7 @@ def test_soft_apply_adds_changes_to_pending_deltas(self):
         expected_deltas = {
             '0': {
                 'writes': {'thing1': (9999, 8888)},
-                'reads': {}
+                'reads': {'thing1': 9999}
             }
         }
 
@@ -192,9 +192,9 @@ def test_hard_apply_removes_hcls(self):
         self.c.hard_apply('0')
 
         hlcs = {'1':
-                    {'writes': {'thing1': (8888, 7777)}, 'reads': {}},
+                    {'writes': {'thing1': (8888, 7777)}, 'reads': {'thing1': 8888}},
                 '2':
-                    {'writes': {'thing1': (7777, 6666)}, 'reads': {}}
+                    {'writes': {'thing1': (7777, 6666)}, 'reads': {'thing1': 7777}}
                 }
 
         self.assertDictEqual(self.c.pending_deltas, hlcs)

From 8e00d51f1069ec8f88ceec53480e5621a615faeb Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Tue, 7 Sep 2021 10:57:32 -0400
Subject: [PATCH 19/78] fixed rollback method to rollback to a point in HCL
 time. All test cases are passing

---
 contracting/db/driver.py | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index a610c3df..3fe27ab8 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -608,11 +608,6 @@ def rollback(self, hlc=None):
             self.pending_writes.clear()
             self.pending_deltas.clear()
         else:
-            if self.pending_deltas.get(hlc) is None:
-                return
-
-                # Run through the sorted HCLs from oldest to newest applying each one until the hcl committed is
-
             to_delete = []
             for _hlc, _deltas in sorted(self.pending_deltas.items())[::-1]:
                 # Clears the current reads/writes, and the reads/writes that get made when rolling back from the
@@ -620,15 +615,17 @@ def rollback(self, hlc=None):
                 self.pending_reads = {}
                 self.pending_writes.clear()
 
-                # Run through all state changes, taking the second value, which is the post delta
-                for key, delta in _deltas['writes'].items():
-                    self.set(key, delta[0])
-                    # self.cache[key] = delta[1]
 
-                # Add the key (
-                to_delete.append(_hlc)
-                if _hlc == hlc:
+                if _hlc < hlc:
+                    # if we are less than the HLC then top processing anymore, this is our rollback point
                     break
+                else:
+                    # if we are still greater than or equal to then mark this as delete and rollback its changes
+                    to_delete.append(_hlc)
+                    # Run through all state changes, taking the second value, which is the post delta
+                    for key, delta in _deltas['writes'].items():
+                        # self.set(key, delta[0])
+                        self.cache[key] = delta[0]
 
             # Remove the deltas from the set
             [self.pending_deltas.pop(key) for key in to_delete]

From 53616bc06895433580c0336e70304c073638f9cb Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 23 Feb 2022 12:41:10 -0700
Subject: [PATCH 20/78] hdf5 poc base

---
 contracting/db/driver.py | 150 ++++++++-------------------------------
 1 file changed, 28 insertions(+), 122 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index b3ec3b0c..b435599c 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -15,6 +15,8 @@
 import lmdb
 import motor.motor_asyncio
 import asyncio
+import h5py
+import json
 
 FILE_EXT = '.d'
 HASH_EXT = '.x'
@@ -205,35 +207,34 @@ def __delitem__(self, key: str):
         except KeyError:
             pass
 
-
 class FSDriver:
-    OS_KEY_LIMIT = (256 - 1) - len(FILE_EXT)
+    def __init__(self, root=Path.home().joinpath('fs')):
+        self.root = root
+        self.root.mkdir(exist_ok=True, parents=True)
 
-    def __init__(self, root='fs'):
-        self.root = os.path.join(Path.home(), root)
+    def __contract_name_to_path(self, contract_name: str) -> str:
+        return self.root.joinpath(contract_name)
 
-    def get(self, item: str):
-        try:
-            filename = self._key_to_file(item)
-            with open(filename, 'r') as f:
-                v = f.read()
+    def __parse_key(self, key: str) -> tuple:
+        contract_name, variable = key.split('.', 1)
+        group_name = variable.replace(':', '/')
 
-        except FileNotFoundError:
-            return None
+        return contract_name, group_name
 
-        return decode(v)
+    def get(self, key: str) -> str:
+        contract_name, group_name = self.__parse_key(key)
+        try:
+            with h5py.File(self.__contract_name_to_path(contract_name), 'r') as f:
+                return decode(f[group_name].attrs.get('value'))
+        except:
+            return None
 
     def set(self, key, value):
-        if value is None:
-            self.__delitem__(key)
-        else:
-            v = encode(value)
-            filename = self._key_to_file(key)
-
-            os.makedirs(filename.parents[0], exist_ok=True)
-
-            with open(filename, 'w') as f:
-                f.write(v)
+        contract_name, group_name = self.__parse_key(key)
+        with h5py.File(self.__contract_name_to_path(contract_name), 'a') as f:
+            if group_name not in f:
+                f.create_group(group_name)
+            f[group_name].attrs.create('value', encode(value))
 
     def flush(self):
         try:
@@ -242,106 +243,11 @@ def flush(self):
             pass
 
     def delete(self, key: str):
-        self.__delitem__(key)
-
-    def iter(self, prefix: str='', length=0):
-        keys = []
-
-        for (r, dirs, files) in sorted(os.walk(self.root, topdown=True)):
-            files.sort()
-            base = r[len(self.root):]
-
-            for f in files:
-                if f.endswith(FILE_EXT) and f.startswith(prefix):
-                    keys.append(self.path_to_key(os.path.join(base, f)))
-
-                if 0 < length <= len(keys):
-                    break
-
-            if 0 < length <= len(keys):
-                break
-
-        keys.sort()
-
-        return keys
-
-    def _iter(self, prefix: str='', length=0):
-        keys = []
-
-        for (r, dirs, files) in os.walk(os.path.join(self.root, prefix), topdown=True):
-            base = r[len(self.root):]
-
-            for f in files:
-                if f.endswith(FILE_EXT):
-                    keys.append(self.path_to_key(os.path.join(base, f)))
-
-                if 0 < length <= len(keys):
-                    break
-
-            if 0 < length <= len(keys):
-                break
-
-        keys.sort()
-
-        return keys
-
-    def keys(self):
-        return self.iter()
-
-    def __getitem__(self, item: str):
-        value = self.get(item)
-        if value is None:
-            raise KeyError
-        return value
-
-    def __setitem__(self, key: str, value):
-        self.set(key, value)
-
-    def __delitem__(self, key: str):
-        filename = self._key_to_file(key)
-        try:
-            os.remove(filename)
-        except FileNotFoundError:
-            pass
-
-    @staticmethod
-    def hash_key(key: str):
-        h = hashlib.sha3_256()
-        h.update(key.encode())
-
-        return h.hexdigest()[:-2] + HASH_EXT
-
-    def _key_to_file(self, key: str):
-        filename = key.replace(':', '/')
-        filename = filename.replace('.', '/')
-        filename += FILE_EXT
-        filename = os.path.join(self.root, filename)
-        return Path(filename)
-
-    def path_to_key(self, path):
-        if path.endswith(FILE_EXT):
-            path = path[:-len(FILE_EXT)]
-
-        pth = path.split('/')
-
-        pth = [p for p in pth if p != '']
-
-        contract = pth.pop(0)
-
-        if len(pth) == 0:
-            return contract
-
-        variable = pth.pop(0)
-
-        key = '.'.join((contract, variable))
-
-        if len(pth) == 0:
-            return key
-
-        key = ':'.join((key, *pth))
-
-        return key
-
+        contract_name, group_name = self.__parse_key(key)
+        with h5py.File(self.__contract_name_to_path(contract_name), 'a') as f:
+            if group_name in f:
+                if f[group_name].attrs.__contains__('value'):
+                    f[group_name].attrs.__delitem__('value')
 
 class LMDBDriver:
     def __init__(self, filename=STORAGE_HOME.joinpath('state')):

From 2f598dd74cf53881dc6d87cfb745628265c1674b Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 23 Feb 2022 14:05:14 -0700
Subject: [PATCH 21/78] fix some test cases

---
 contracting/db/driver.py      | 22 +++++++++++++++++-----
 tests/unit/test_new_driver.py |  1 -
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index b435599c..0e41a3b2 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -212,22 +212,25 @@ def __init__(self, root=Path.home().joinpath('fs')):
         self.root = root
         self.root.mkdir(exist_ok=True, parents=True)
 
-    def __contract_name_to_path(self, contract_name: str) -> str:
+    def __contract_name_to_path(self, contract_name: str):
         return self.root.joinpath(contract_name)
 
-    def __parse_key(self, key: str) -> tuple:
-        contract_name, variable = key.split('.', 1)
+    def __parse_key(self, key: str):
+        try:
+            contract_name, variable = key.split('.', 1)
+        except ValueError: # NOTE: not enough values to unpack case
+            contract_name, variable = key, key
         group_name = variable.replace(':', '/')
 
         return contract_name, group_name
 
-    def get(self, key: str) -> str:
+    def get(self, key: str):
         contract_name, group_name = self.__parse_key(key)
         try:
             with h5py.File(self.__contract_name_to_path(contract_name), 'r') as f:
                 return decode(f[group_name].attrs.get('value'))
         except:
-            return None
+            raise KeyError
 
     def set(self, key, value):
         contract_name, group_name = self.__parse_key(key)
@@ -249,6 +252,15 @@ def delete(self, key: str):
                 if f[group_name].attrs.__contains__('value'):
                     f[group_name].attrs.__delitem__('value')
 
+    def __getitem__(self, key):
+        return self.get(key)
+
+    def __setitem__(self, key, value):
+        self.set(key, value)
+
+    def __delitem__(self, key):
+        self.delete(key)
+
 class LMDBDriver:
     def __init__(self, filename=STORAGE_HOME.joinpath('state')):
         self.filename = filename
diff --git a/tests/unit/test_new_driver.py b/tests/unit/test_new_driver.py
index 6a8a8974..9b01d60d 100644
--- a/tests/unit/test_new_driver.py
+++ b/tests/unit/test_new_driver.py
@@ -530,7 +530,6 @@ class TestFSDriver(TestCase):
     # Flush this sucker every test
     def setUp(self):
         self.d = FSDriver()
-        self.d.flush()
 
     def tearDown(self):
         self.d.flush()

From 997012d253b150410c78de002aaa9988c4bef901 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 23 Feb 2022 14:48:40 -0700
Subject: [PATCH 22/78] revert raise KeyError

---
 contracting/db/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 0e41a3b2..ad0efc53 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -230,7 +230,7 @@ def get(self, key: str):
             with h5py.File(self.__contract_name_to_path(contract_name), 'r') as f:
                 return decode(f[group_name].attrs.get('value'))
         except:
-            raise KeyError
+            return None
 
     def set(self, key, value):
         contract_name, group_name = self.__parse_key(key)

From 443ac0d9099fa84d920811d843805107db70f059 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Fri, 25 Feb 2022 16:31:41 -0700
Subject: [PATCH 23/78] fix file size constantly increases

---
 contracting/db/driver.py      | 19 +++++++++----------
 tests/unit/test_new_driver.py |  5 ++---
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index ad0efc53..d494f999 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -16,7 +16,6 @@
 import motor.motor_asyncio
 import asyncio
 import h5py
-import json
 
 FILE_EXT = '.d'
 HASH_EXT = '.x'
@@ -212,10 +211,7 @@ def __init__(self, root=Path.home().joinpath('fs')):
         self.root = root
         self.root.mkdir(exist_ok=True, parents=True)
 
-    def __contract_name_to_path(self, contract_name: str):
-        return self.root.joinpath(contract_name)
-
-    def __parse_key(self, key: str):
+    def __parse_key(self, key):
         try:
             contract_name, variable = key.split('.', 1)
         except ValueError: # NOTE: not enough values to unpack case
@@ -224,7 +220,10 @@ def __parse_key(self, key: str):
 
         return contract_name, group_name
 
-    def get(self, key: str):
+    def __contract_name_to_path(self, contract_name):
+        return self.root.joinpath(contract_name)
+
+    def get(self, key):
         contract_name, group_name = self.__parse_key(key)
         try:
             with h5py.File(self.__contract_name_to_path(contract_name), 'r') as f:
@@ -237,7 +236,8 @@ def set(self, key, value):
         with h5py.File(self.__contract_name_to_path(contract_name), 'a') as f:
             if group_name not in f:
                 f.create_group(group_name)
-            f[group_name].attrs.create('value', encode(value))
+            ev = encode(value)
+            f[group_name].attrs.create('value', ev, dtype='S'+str(len(ev)))
 
     def flush(self):
         try:
@@ -248,9 +248,8 @@ def flush(self):
     def delete(self, key: str):
         contract_name, group_name = self.__parse_key(key)
         with h5py.File(self.__contract_name_to_path(contract_name), 'a') as f:
-            if group_name in f:
-                if f[group_name].attrs.__contains__('value'):
-                    f[group_name].attrs.__delitem__('value')
+            if group_name in f and 'value' in f[group_name].attrs:
+                del f[group_name].attrs['value']
 
     def __getitem__(self, key):
         return self.get(key)
diff --git a/tests/unit/test_new_driver.py b/tests/unit/test_new_driver.py
index 9b01d60d..16823282 100644
--- a/tests/unit/test_new_driver.py
+++ b/tests/unit/test_new_driver.py
@@ -729,9 +729,8 @@ def test_iter_with_length_returns_list_of_size_l(self):
 
         self.assertListEqual(prefix_2_keys[:5], p2)
 
-    def test_key_error_if_getitem_doesnt_exist(self):
-        with self.assertRaises(KeyError):
-            print(self.d['thing'])
+    def test_key_none_if_getitem_doesnt_exist(self):
+        self.assertIsNone(self.d['thing'])
 
     def test_keys_returns_all_keys(self):
         prefix_1_keys = [

From e293a7b1e408e46f5b22843915923a7418d3523c Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Sun, 27 Feb 2022 23:07:29 -0700
Subject: [PATCH 24/78] fetch keys api

---
 contracting/db/driver.py      |  37 +++++++-
 tests/unit/test_new_driver.py | 170 +++++++++++++++++-----------------
 2 files changed, 117 insertions(+), 90 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index d494f999..ff7c3a16 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -210,12 +210,10 @@ class FSDriver:
     def __init__(self, root=Path.home().joinpath('fs')):
         self.root = root
         self.root.mkdir(exist_ok=True, parents=True)
+        self._groups_with_values = []
 
     def __parse_key(self, key):
-        try:
-            contract_name, variable = key.split('.', 1)
-        except ValueError: # NOTE: not enough values to unpack case
-            contract_name, variable = key, key
+        contract_name, variable = key.split('.', 1)
         group_name = variable.replace(':', '/')
 
         return contract_name, group_name
@@ -223,6 +221,21 @@ def __parse_key(self, key):
     def __contract_name_to_path(self, contract_name):
         return self.root.joinpath(contract_name)
 
+    def __store_group_if_has_value_cb(self, name, obj):
+        if 'value' in obj.attrs:
+            self._groups_with_values.append(name)
+
+    def __get_contracts(self):
+        return sorted([contract_name for contract_name in os.listdir(self.root)])
+
+    def __get_keys_from_contract(self, contract):
+        self._groups_with_values = []
+        with h5py.File(self.__contract_name_to_path(contract), 'r') as f:
+            f.visititems(self.__store_group_if_has_value_cb)
+        keys = [contract + '.' + group.replace('/', ':') for group in self._groups_with_values]
+        
+        return keys
+
     def get(self, key):
         contract_name, group_name = self.__parse_key(key)
         try:
@@ -245,12 +258,26 @@ def flush(self):
         except FileNotFoundError:
             pass
 
-    def delete(self, key: str):
+    def delete(self, key):
         contract_name, group_name = self.__parse_key(key)
         with h5py.File(self.__contract_name_to_path(contract_name), 'a') as f:
             if group_name in f and 'value' in f[group_name].attrs:
                 del f[group_name].attrs['value']
 
+    def iter(self, prefix='', length=0):
+        contracts = self.__get_contracts()
+        keys = []
+        for contract in contracts:
+            if contract.startswith(prefix):
+                keys.extend(self.__get_keys_from_contract(contract))
+            if length > 0 and len(keys) >= length:
+                break
+
+        return keys if length == 0 else keys[:length]
+
+    def keys(self):
+        return self.iter()
+
     def __getitem__(self, key):
         return self.get(key)
 
diff --git a/tests/unit/test_new_driver.py b/tests/unit/test_new_driver.py
index 16823282..1f7d03d7 100644
--- a/tests/unit/test_new_driver.py
+++ b/tests/unit/test_new_driver.py
@@ -536,49 +536,49 @@ def tearDown(self):
 
     def test_get_set(self):
         for v in TEST_DATA:
-            self.d.set('b', v)
+            self.d.set('b.b', v)
 
-            b = self.d.get('b')
+            b = self.d.get('b.b')
             self.assertEqual(v, b) if not isinstance(v, dict) else self.assertDictEqual(v, b)
 
     def test_delete(self):
         for v in TEST_DATA:
-            self.d.set('b', v)
+            self.d.set('b.b', v)
 
-            b = self.d.get('b')
+            b = self.d.get('b.b')
             self.assertEqual(v, b) if not isinstance(v, dict) else self.assertDictEqual(v, b)
 
-            self.d.delete('b')
+            self.d.delete('b.b')
 
-            b = self.d.get('b')
+            b = self.d.get('b.b')
             self.assertIsNone(b)
 
     def test_iter(self):
 
         prefix_1_keys = [
-            'b77aa343e339bed781c7c2be1267cd597',
-            'bc22ede6e6fb4046d78bf2f9d1f8afdb6',
-            'b93dbb37d993846d70b8a92779cbfbfe9',
-            'be1a2783019de6ea7ef169cc55e48a3ae',
-            'b1fe8db32b9185d628f4c346f0455023e',
-            'bef918f83b6a0d1e4f980013342807cf8',
-            'b004cb9235acb5f689d20904692bc026e',
-            'b869ff9519d67354816af90867f4a5425',
-            'bcd8e9100dcb601f65e849c147e3e972e',
-            'b0111919a698f9816862b4ae662a6ed06',
+            'b77aa343e339bed78.1c7c2be1267cd597',
+            'bc22ede6e6fb4046d.78bf2f9d1f8afdb6',
+            'b93dbb37d993846d7.0b8a92779cbfbfe9',
+            'be1a2783019de6ea7.ef169cc55e48a3ae',
+            'b1fe8db32b9185d62.8f4c346f0455023e',
+            'bef918f83b6a0d1e4.f980013342807cf8',
+            'b004cb9235acb5f68.9d20904692bc026e',
+            'b869ff9519d673548.16af90867f4a5425',
+            'bcd8e9100dcb601f6.5e849c147e3e972e',
+            'b0111919a698f9816.862b4ae662a6ed06',
         ]
 
         prefix_2_keys = [
-            'x37fbab0bd2e60563c79469e5be41e515',
-            'x30c6eb2ad176773b5ce6d590d2472dfe',
-            'x3d4fc9480f0a07b28aa7646d5066b54d',
-            'x387c3d4ab7f0c1c6ef549198fc14b525',
-            'x5c74dc83e132e435e8512599e1075bc0',
-            'x1472425d0d9bb5ff511e132896d54b13',
-            'x2cedb5c52163c22a0b5f179001959dd2',
-            'x6223f65e553280cd25cadeac6657555c',
-            'xeae18af37c223dde92a71fef55e64afe',
-            'xb8810784ffb360cd3ffc57b1d088e537',
+            'x37fbab0bd2e60563.c79469e5be41e515',
+            'x30c6eb2ad176773b.5ce6d590d2472dfe',
+            'x3d4fc9480f0a07b2.8aa7646d5066b54d',
+            'x387c3d4ab7f0c1c6.ef549198fc14b525',
+            'x5c74dc83e132e435.e8512599e1075bc0',
+            'x1472425d0d9bb5ff.511e132896d54b13',
+            'x2cedb5c52163c22a.0b5f179001959dd2',
+            'x6223f65e553280cd.25cadeac6657555c',
+            'xeae18af37c223dde.92a71fef55e64afe',
+            'xb8810784ffb360cd.3ffc57b1d088e537',
         ]
 
         keys = prefix_1_keys + prefix_2_keys
@@ -614,32 +614,32 @@ def test_set_object_returns_properly(self):
             'x': None
         }
 
-        self.d.set('thing', thing)
+        self.d.set('thing.thing', thing)
 
-        t = self.d.get('thing')
+        t = self.d.get('thing.thing')
         self.assertDictEqual(thing, t)
 
     def test_set_none_deletes(self):
         t = 123
 
-        self.d.set('t', t)
+        self.d.set('t.t', t)
 
-        self.assertEqual(self.d.get('t'), 123)
+        self.assertEqual(self.d.get('t.t'), 123)
 
-        self.d.set('t', None)
+        self.d.set('t.t', None)
 
-        self.assertEqual(self.d.get('t'), None)
+        self.assertEqual(self.d.get('t.t'), None)
 
     def test_delete_sets_to_none(self):
         t = 123
 
-        self.d.set('t', t)
+        self.d.set('t.t', t)
 
-        self.assertEqual(self.d.get('t'), 123)
+        self.assertEqual(self.d.get('t.t'), 123)
 
-        self.d.delete('t')
+        self.d.delete('t.t')
 
-        self.assertEqual(self.d.get('t'), None)
+        self.assertEqual(self.d.get('t.t'), None)
 
     def test_getitem_works_like_get(self):
         thing = {
@@ -648,9 +648,9 @@ def test_getitem_works_like_get(self):
             'x': None
         }
 
-        self.d.set('thing', thing)
+        self.d.set('thing.thing', thing)
 
-        t = self.d['thing']
+        t = self.d['thing.thing']
         self.assertDictEqual(thing, t)
 
     def test_setitem_works_like_set(self):
@@ -660,47 +660,47 @@ def test_setitem_works_like_set(self):
             'x': None
         }
 
-        self.d['thing'] = thing
+        self.d['thing.thing'] = thing
 
-        t = self.d['thing']
+        t = self.d['thing.thing']
         self.assertDictEqual(thing, t)
 
     def test_delitem_works_like_del(self):
         t = 123
 
-        self.d.set('t', t)
+        self.d.set('t.t', t)
 
-        self.assertEqual(self.d.get('t'), 123)
+        self.assertEqual(self.d.get('t.t'), 123)
 
-        del self.d['t']
+        del self.d['t.t']
 
-        self.assertEqual(self.d.get('t'), None)
+        self.assertEqual(self.d.get('t.t'), None)
 
     def test_iter_with_length_returns_list_of_size_l(self):
         prefix_1_keys = [
-            'b77aa343e339bed781c7c2be1267cd597',
-            'bc22ede6e6fb4046d78bf2f9d1f8afdb6',
-            'b93dbb37d993846d70b8a92779cbfbfe9',
-            'be1a2783019de6ea7ef169cc55e48a3ae',
-            'b1fe8db32b9185d628f4c346f0455023e',
-            'bef918f83b6a0d1e4f980013342807cf8',
-            'b004cb9235acb5f689d20904692bc026e',
-            'b869ff9519d67354816af90867f4a5425',
-            'bcd8e9100dcb601f65e849c147e3e972e',
-            'b0111919a698f9816862b4ae662a6ed06',
+            'b77aa343e339bed7.81c7c2be1267cd597',
+            'bc22ede6e6fb4046.d78bf2f9d1f8afdb6',
+            'b93dbb37d993846d.70b8a92779cbfbfe9',
+            'be1a2783019de6ea.7ef169cc55e48a3ae',
+            'b1fe8db32b9185d6.28f4c346f0455023e',
+            'bef918f83b6a0d1e.4f980013342807cf8',
+            'b004cb9235acb5f6.89d20904692bc026e',
+            'b869ff9519d67354.816af90867f4a5425',
+            'bcd8e9100dcb601f.65e849c147e3e972e',
+            'b0111919a698f981.6862b4ae662a6ed06',
         ]
 
         prefix_2_keys = [
-            'x37fbab0bd2e60563c79469e5be41e515',
-            'x30c6eb2ad176773b5ce6d590d2472dfe',
-            'x3d4fc9480f0a07b28aa7646d5066b54d',
-            'x387c3d4ab7f0c1c6ef549198fc14b525',
-            'x5c74dc83e132e435e8512599e1075bc0',
-            'x1472425d0d9bb5ff511e132896d54b13',
-            'x2cedb5c52163c22a0b5f179001959dd2',
-            'x6223f65e553280cd25cadeac6657555c',
-            'xeae18af37c223dde92a71fef55e64afe',
-            'xb8810784ffb360cd3ffc57b1d088e537',
+            'x37fbab0bd2e6056.3c79469e5be41e515',
+            'x30c6eb2ad176773.b5ce6d590d2472dfe',
+            'x3d4fc9480f0a07b.28aa7646d5066b54d',
+            'x387c3d4ab7f0c1c.6ef549198fc14b525',
+            'x5c74dc83e132e43.5e8512599e1075bc0',
+            'x1472425d0d9bb5f.f511e132896d54b13',
+            'x2cedb5c52163c22.a0b5f179001959dd2',
+            'x6223f65e553280c.d25cadeac6657555c',
+            'xeae18af37c223dd.e92a71fef55e64afe',
+            'xb8810784ffb360c.d3ffc57b1d088e537',
         ]
 
         keys = prefix_1_keys + prefix_2_keys
@@ -730,33 +730,33 @@ def test_iter_with_length_returns_list_of_size_l(self):
         self.assertListEqual(prefix_2_keys[:5], p2)
 
     def test_key_none_if_getitem_doesnt_exist(self):
-        self.assertIsNone(self.d['thing'])
+        self.assertIsNone(self.d['thing.thing'])
 
     def test_keys_returns_all_keys(self):
         prefix_1_keys = [
-            'b77aa343e339bed781c7c2be1267cd597',
-            'bc22ede6e6fb4046d78bf2f9d1f8afdb6',
-            'b93dbb37d993846d70b8a92779cbfbfe9',
-            'be1a2783019de6ea7ef169cc55e48a3ae',
-            'b1fe8db32b9185d628f4c346f0455023e',
-            'bef918f83b6a0d1e4f980013342807cf8',
-            'b004cb9235acb5f689d20904692bc026e',
-            'b869ff9519d67354816af90867f4a5425',
-            'bcd8e9100dcb601f65e849c147e3e972e',
-            'b0111919a698f9816862b4ae662a6ed06',
+            'b77aa343e339bed7.81c7c2be1267cd597',
+            'bc22ede6e6fb4046.d78bf2f9d1f8afdb6',
+            'b93dbb37d993846d.70b8a92779cbfbfe9',
+            'be1a2783019de6ea.7ef169cc55e48a3ae',
+            'b1fe8db32b9185d6.28f4c346f0455023e',
+            'bef918f83b6a0d1e.4f980013342807cf8',
+            'b004cb9235acb5f6.89d20904692bc026e',
+            'b869ff9519d67354.816af90867f4a5425',
+            'bcd8e9100dcb601f.65e849c147e3e972e',
+            'b0111919a698f981.6862b4ae662a6ed06',
         ]
 
         prefix_2_keys = [
-            'x37fbab0bd2e60563c79469e5be41e515',
-            'x30c6eb2ad176773b5ce6d590d2472dfe',
-            'x3d4fc9480f0a07b28aa7646d5066b54d',
-            'x387c3d4ab7f0c1c6ef549198fc14b525',
-            'x5c74dc83e132e435e8512599e1075bc0',
-            'x1472425d0d9bb5ff511e132896d54b13',
-            'x2cedb5c52163c22a0b5f179001959dd2',
-            'x6223f65e553280cd25cadeac6657555c',
-            'xeae18af37c223dde92a71fef55e64afe',
-            'xb8810784ffb360cd3ffc57b1d088e537',
+            'x37fbab0bd2e6056.3c79469e5be41e515',
+            'x30c6eb2ad176773.b5ce6d590d2472dfe',
+            'x3d4fc9480f0a07b.28aa7646d5066b54d',
+            'x387c3d4ab7f0c1c.6ef549198fc14b525',
+            'x5c74dc83e132e43.5e8512599e1075bc0',
+            'x1472425d0d9bb5f.f511e132896d54b13',
+            'x2cedb5c52163c22.a0b5f179001959dd2',
+            'x6223f65e553280c.d25cadeac6657555c',
+            'xeae18af37c223dd.e92a71fef55e64afe',
+            'xb8810784ffb360c.d3ffc57b1d088e537',
         ]
 
         keys = prefix_1_keys + prefix_2_keys

From 873ceae56da969080cfbe70ca3b0c17187451829 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Mon, 28 Feb 2022 12:10:50 -0700
Subject: [PATCH 25/78] get rid of iter method

---
 contracting/db/driver.py      |  9 +++------
 tests/unit/test_new_driver.py | 12 ++++++------
 2 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index ff7c3a16..a716aa27 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -264,19 +264,16 @@ def delete(self, key):
             if group_name in f and 'value' in f[group_name].attrs:
                 del f[group_name].attrs['value']
 
-    def iter(self, prefix='', length=0):
+    def keys(self, prefix='', num_keys=0):
         contracts = self.__get_contracts()
         keys = []
         for contract in contracts:
             if contract.startswith(prefix):
                 keys.extend(self.__get_keys_from_contract(contract))
-            if length > 0 and len(keys) >= length:
+            if num_keys > 0 and len(keys) >= num_keys:
                 break
 
-        return keys if length == 0 else keys[:length]
-
-    def keys(self):
-        return self.iter()
+        return keys if num_keys == 0 else keys[:num_keys]
 
     def __getitem__(self, key):
         return self.get(key)
diff --git a/tests/unit/test_new_driver.py b/tests/unit/test_new_driver.py
index 1f7d03d7..76a27ada 100644
--- a/tests/unit/test_new_driver.py
+++ b/tests/unit/test_new_driver.py
@@ -553,7 +553,7 @@ def test_delete(self):
             b = self.d.get('b.b')
             self.assertIsNone(b)
 
-    def test_iter(self):
+    def test_keys_with_prefix(self):
 
         prefix_1_keys = [
             'b77aa343e339bed78.1c7c2be1267cd597',
@@ -589,7 +589,7 @@ def test_iter(self):
 
         p1 = []
 
-        for k in self.d.iter(prefix='b'):
+        for k in self.d.keys(prefix='b'):
             p1.append(k)
 
         prefix_1_keys.sort()
@@ -599,7 +599,7 @@ def test_iter(self):
 
         p2 = []
 
-        for k in self.d.iter(prefix='x'):
+        for k in self.d.keys(prefix='x'):
             p2.append(k)
 
         prefix_2_keys.sort()
@@ -676,7 +676,7 @@ def test_delitem_works_like_del(self):
 
         self.assertEqual(self.d.get('t.t'), None)
 
-    def test_iter_with_length_returns_list_of_size_l(self):
+    def test_keys_with_length_returns_list_of_size_l(self):
         prefix_1_keys = [
             'b77aa343e339bed7.81c7c2be1267cd597',
             'bc22ede6e6fb4046.d78bf2f9d1f8afdb6',
@@ -711,7 +711,7 @@ def test_iter_with_length_returns_list_of_size_l(self):
 
         p1 = []
 
-        for k in self.d.iter(prefix='b', length=3):
+        for k in self.d.keys(prefix='b', num_keys=3):
             p1.append(k)
 
         prefix_1_keys.sort()
@@ -721,7 +721,7 @@ def test_iter_with_length_returns_list_of_size_l(self):
 
         p2 = []
 
-        for k in self.d.iter(prefix='x', length=5):
+        for k in self.d.keys(prefix='x', num_keys=5):
             p2.append(k)
 
         prefix_2_keys.sort()

From e02e089f556a8c53e8f16265bc926fe1750b8624 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Mon, 28 Feb 2022 12:45:30 -0700
Subject: [PATCH 26/78] remove redundant code

---
 contracting/db/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index a716aa27..57680bc3 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -226,7 +226,7 @@ def __store_group_if_has_value_cb(self, name, obj):
             self._groups_with_values.append(name)
 
     def __get_contracts(self):
-        return sorted([contract_name for contract_name in os.listdir(self.root)])
+        return sorted(os.listdir(self.root))
 
     def __get_keys_from_contract(self, contract):
         self._groups_with_values = []

From 47a42086cf4d78971240cde479cb836f88bad236 Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Mon, 28 Feb 2022 21:34:21 +0000
Subject: [PATCH 27/78] default

---
 contracting/db/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 57680bc3..37128987 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -382,7 +382,7 @@ def get(self, item: str):
 
 
 class CacheDriver:
-    def __init__(self, driver: Driver=Driver()):
+    def __init__(self, driver: Driver=FSDriver()):
         self.driver = driver
         self.cache = {}
 

From 965dcf154b5391d7918394db493611d7cd1fb808 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Mon, 28 Feb 2022 15:59:16 -0700
Subject: [PATCH 28/78] fix flush adhering integration tests

---
 contracting/db/driver.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 37128987..fc10f897 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -253,10 +253,8 @@ def set(self, key, value):
             f[group_name].attrs.create('value', ev, dtype='S'+str(len(ev)))
 
     def flush(self):
-        try:
-            shutil.rmtree(self.root)
-        except FileNotFoundError:
-            pass
+        for f in os.listdir(self.root):
+            os.remove(self.root.joinpath(f))
 
     def delete(self, key):
         contract_name, group_name = self.__parse_key(key)

From 441bf6d2ae0b2f56b34d911cc3e9700ad2bfd32c Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Mon, 28 Feb 2022 16:01:59 -0700
Subject: [PATCH 29/78] fix integration tests failures

---
 contracting/db/driver.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index fc10f897..ae867f71 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -262,7 +262,7 @@ def delete(self, key):
             if group_name in f and 'value' in f[group_name].attrs:
                 del f[group_name].attrs['value']
 
-    def keys(self, prefix='', num_keys=0):
+    def iter(self, prefix='', num_keys=0):
         contracts = self.__get_contracts()
         keys = []
         for contract in contracts:
@@ -272,6 +272,9 @@ def keys(self, prefix='', num_keys=0):
                 break
 
         return keys if num_keys == 0 else keys[:num_keys]
+    
+    def keys(self):
+        return self.iter()
 
     def __getitem__(self, key):
         return self.get(key)

From eafd78ed81d0a23fefab2d44b3b049c6df866505 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Mon, 28 Feb 2022 16:13:24 -0700
Subject: [PATCH 30/78] adhere old api

---
 contracting/db/driver.py      | 6 +++---
 tests/unit/test_new_driver.py | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index ae867f71..cf8944de 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -262,16 +262,16 @@ def delete(self, key):
             if group_name in f and 'value' in f[group_name].attrs:
                 del f[group_name].attrs['value']
 
-    def iter(self, prefix='', num_keys=0):
+    def iter(self, prefix='', length=0):
         contracts = self.__get_contracts()
         keys = []
         for contract in contracts:
             if contract.startswith(prefix):
                 keys.extend(self.__get_keys_from_contract(contract))
-            if num_keys > 0 and len(keys) >= num_keys:
+            if length > 0 and len(keys) >= length:
                 break
 
-        return keys if num_keys == 0 else keys[:num_keys]
+        return keys if length == 0 else keys[:length]
     
     def keys(self):
         return self.iter()
diff --git a/tests/unit/test_new_driver.py b/tests/unit/test_new_driver.py
index 76a27ada..b2b750af 100644
--- a/tests/unit/test_new_driver.py
+++ b/tests/unit/test_new_driver.py
@@ -589,7 +589,7 @@ def test_keys_with_prefix(self):
 
         p1 = []
 
-        for k in self.d.keys(prefix='b'):
+        for k in self.d.iter(prefix='b'):
             p1.append(k)
 
         prefix_1_keys.sort()
@@ -599,7 +599,7 @@ def test_keys_with_prefix(self):
 
         p2 = []
 
-        for k in self.d.keys(prefix='x'):
+        for k in self.d.iter(prefix='x'):
             p2.append(k)
 
         prefix_2_keys.sort()
@@ -711,7 +711,7 @@ def test_keys_with_length_returns_list_of_size_l(self):
 
         p1 = []
 
-        for k in self.d.keys(prefix='b', num_keys=3):
+        for k in self.d.iter(prefix='b', length=3):
             p1.append(k)
 
         prefix_1_keys.sort()
@@ -721,7 +721,7 @@ def test_keys_with_length_returns_list_of_size_l(self):
 
         p2 = []
 
-        for k in self.d.keys(prefix='x', num_keys=5):
+        for k in self.d.iter(prefix='x', length=5):
             p2.append(k)
 
         prefix_2_keys.sort()

From e35e4a4c6a5bbb072212f191060b0650cd3904ed Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Tue, 1 Mar 2022 13:33:12 -0700
Subject: [PATCH 31/78] refactoring

---
 contracting/db/driver.py | 83 ++++++++++++++++------------------------
 contracting/db/hdf5.py   | 36 +++++++++++++++++
 2 files changed, 69 insertions(+), 50 deletions(-)
 create mode 100644 contracting/db/hdf5.py

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index cf8944de..6ff658e3 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -15,7 +15,7 @@
 import lmdb
 import motor.motor_asyncio
 import asyncio
-import h5py
+import contracting.db.hdf5 as hdf5
 
 FILE_EXT = '.d'
 HASH_EXT = '.x'
@@ -210,64 +210,56 @@ class FSDriver:
     def __init__(self, root=Path.home().joinpath('fs')):
         self.root = root
         self.root.mkdir(exist_ok=True, parents=True)
-        self._groups_with_values = []
 
     def __parse_key(self, key):
-        contract_name, variable = key.split('.', 1)
+        filename, variable = key.split('.', 1)
         group_name = variable.replace(':', '/')
 
-        return contract_name, group_name
+        return filename, group_name
 
-    def __contract_name_to_path(self, contract_name):
-        return self.root.joinpath(contract_name)
+    def __filename_to_path(self, filename):
+        return self.root.joinpath(filename)
 
-    def __store_group_if_has_value_cb(self, name, obj):
-        if 'value' in obj.attrs:
-            self._groups_with_values.append(name)
-
-    def __get_contracts(self):
+    def __get_files(self):
         return sorted(os.listdir(self.root))
 
-    def __get_keys_from_contract(self, contract):
-        self._groups_with_values = []
-        with h5py.File(self.__contract_name_to_path(contract), 'r') as f:
-            f.visititems(self.__store_group_if_has_value_cb)
-        keys = [contract + '.' + group.replace('/', ':') for group in self._groups_with_values]
-        
-        return keys
+    def __get_keys_from_file(self, filename):
+        return [filename + '.' + group.replace('/', ':') for group in hdf5.get_groups(self.__filename_to_path(filename))]
+
+    def __getitem__(self, key):
+        return self.get(key)
+
+    def __setitem__(self, key, value):
+        self.set(key, value)
+
+    def __delitem__(self, key):
+        self.delete(key)
 
     def get(self, key):
-        contract_name, group_name = self.__parse_key(key)
-        try:
-            with h5py.File(self.__contract_name_to_path(contract_name), 'r') as f:
-                return decode(f[group_name].attrs.get('value'))
-        except:
-            return None
+        filename, group_name = self.__parse_key(key)
+
+        return hdf5.get_value(self.__filename_to_path(filename), group_name)
 
     def set(self, key, value):
-        contract_name, group_name = self.__parse_key(key)
-        with h5py.File(self.__contract_name_to_path(contract_name), 'a') as f:
-            if group_name not in f:
-                f.create_group(group_name)
-            ev = encode(value)
-            f[group_name].attrs.create('value', ev, dtype='S'+str(len(ev)))
+        filename, group_name = self.__parse_key(key)
+        hdf5.set_value(self.__filename_to_path(filename), group_name, value)
 
     def flush(self):
-        for f in os.listdir(self.root):
-            os.remove(self.root.joinpath(f))
+        try:
+            shutil.rmtree(self.root)
+            self.root.mkdir(exist_ok=True, parents=True)
+        except FileNotFoundError:
+            pass
 
     def delete(self, key):
-        contract_name, group_name = self.__parse_key(key)
-        with h5py.File(self.__contract_name_to_path(contract_name), 'a') as f:
-            if group_name in f and 'value' in f[group_name].attrs:
-                del f[group_name].attrs['value']
+        filename, group_name = self.__parse_key(key)
+        hdf5.del_value(self.__filename_to_path(filename), group_name)
 
     def iter(self, prefix='', length=0):
-        contracts = self.__get_contracts()
         keys = []
-        for contract in contracts:
-            if contract.startswith(prefix):
-                keys.extend(self.__get_keys_from_contract(contract))
+        for filename in self.__get_files():
+            if filename.startswith(prefix):
+                keys.extend(self.__get_keys_from_file(filename))
             if length > 0 and len(keys) >= length:
                 break
 
@@ -276,15 +268,6 @@ def iter(self, prefix='', length=0):
     def keys(self):
         return self.iter()
 
-    def __getitem__(self, key):
-        return self.get(key)
-
-    def __setitem__(self, key, value):
-        self.set(key, value)
-
-    def __delitem__(self, key):
-        self.delete(key)
-
 class LMDBDriver:
     def __init__(self, filename=STORAGE_HOME.joinpath('state')):
         self.filename = filename
@@ -383,7 +366,7 @@ def get(self, item: str):
 
 
 class CacheDriver:
-    def __init__(self, driver: Driver=FSDriver()):
+    def __init__(self, driver: FSDriver=FSDriver()):
         self.driver = driver
         self.cache = {}
 
diff --git a/contracting/db/hdf5.py b/contracting/db/hdf5.py
new file mode 100644
index 00000000..0ea42330
--- /dev/null
+++ b/contracting/db/hdf5.py
@@ -0,0 +1,36 @@
+import h5py
+from contracting.db.encoder import encode, decode
+
+_groups_with_values = []
+
+def _store_group_if_has_value_cb(name, obj):
+    global _groups_with_values
+    if 'value' in obj.attrs:
+        _groups_with_values.append(name)
+
+def set_value(filename, group_name, value):
+    with h5py.File(filename, 'a') as f:
+        if group_name not in f:
+            f.create_group(group_name)
+        ev = encode(value)
+        f[group_name].attrs.create('value', ev, dtype='S'+str(len(ev)))
+
+def get_value(filename, group_name):
+    try:
+        with h5py.File(filename, 'r') as f:
+            return decode(f[group_name].attrs['value'])
+    except:
+        return None
+
+def del_value(filename, group_name):
+    with h5py.File(filename, 'a') as f:
+        if group_name in f and 'value' in f[group_name].attrs:
+            del f[group_name].attrs['value']
+
+def get_groups(filename):
+    global _groups_with_values
+    _groups_with_values = []
+    with h5py.File(filename, 'r') as f:
+        f.visititems(_store_group_if_has_value_cb)
+
+    return _groups_with_values

From fe9dd587404c4ea52b91726c1fa4b5af2f146b60 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Tue, 1 Mar 2022 14:54:07 -0700
Subject: [PATCH 32/78] refactoring

---
 contracting/db/driver.py | 18 +++++++++---------
 contracting/db/hdf5.py   | 36 ++++++++++++++++++------------------
 2 files changed, 27 insertions(+), 27 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 6ff658e3..5441e1d9 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -213,9 +213,9 @@ def __init__(self, root=Path.home().joinpath('fs')):
 
     def __parse_key(self, key):
         filename, variable = key.split('.', 1)
-        group_name = variable.replace(':', '/')
+        variable = variable.replace(':', '/')
 
-        return filename, group_name
+        return filename, variable
 
     def __filename_to_path(self, filename):
         return self.root.joinpath(filename)
@@ -224,7 +224,7 @@ def __get_files(self):
         return sorted(os.listdir(self.root))
 
     def __get_keys_from_file(self, filename):
-        return [filename + '.' + group.replace('/', ':') for group in hdf5.get_groups(self.__filename_to_path(filename))]
+        return [filename + '.' + var.replace('/', ':') for var in hdf5.get_vars(self.__filename_to_path(filename))]
 
     def __getitem__(self, key):
         return self.get(key)
@@ -236,13 +236,13 @@ def __delitem__(self, key):
         self.delete(key)
 
     def get(self, key):
-        filename, group_name = self.__parse_key(key)
+        filename, variable = self.__parse_key(key)
 
-        return hdf5.get_value(self.__filename_to_path(filename), group_name)
+        return hdf5.get_value(self.__filename_to_path(filename), variable)
 
     def set(self, key, value):
-        filename, group_name = self.__parse_key(key)
-        hdf5.set_value(self.__filename_to_path(filename), group_name, value)
+        filename, variable = self.__parse_key(key)
+        hdf5.set_value(self.__filename_to_path(filename), variable, value)
 
     def flush(self):
         try:
@@ -252,8 +252,8 @@ def flush(self):
             pass
 
     def delete(self, key):
-        filename, group_name = self.__parse_key(key)
-        hdf5.del_value(self.__filename_to_path(filename), group_name)
+        filename, variable = self.__parse_key(key)
+        hdf5.del_value(self.__filename_to_path(filename), variable)
 
     def iter(self, prefix='', length=0):
         keys = []
diff --git a/contracting/db/hdf5.py b/contracting/db/hdf5.py
index 0ea42330..c956dcd0 100644
--- a/contracting/db/hdf5.py
+++ b/contracting/db/hdf5.py
@@ -1,36 +1,36 @@
 import h5py
 from contracting.db.encoder import encode, decode
 
-_groups_with_values = []
+_vars_with_values = []
 
-def _store_group_if_has_value_cb(name, obj):
-    global _groups_with_values
+def _store_var_if_has_value_cb(name, obj):
+    global _vars_with_values
     if 'value' in obj.attrs:
-        _groups_with_values.append(name)
+        _vars_with_values.append(name)
 
-def set_value(filename, group_name, value):
+def set_value(filename, variable, value):
     with h5py.File(filename, 'a') as f:
-        if group_name not in f:
-            f.create_group(group_name)
+        if variable not in f:
+            f.create_group(variable)
         ev = encode(value)
-        f[group_name].attrs.create('value', ev, dtype='S'+str(len(ev)))
+        f[variable].attrs.create('value', ev, dtype='S'+str(len(ev)))
 
-def get_value(filename, group_name):
+def get_value(filename, variable):
     try:
         with h5py.File(filename, 'r') as f:
-            return decode(f[group_name].attrs['value'])
+            return decode(f[variable].attrs['value'])
     except:
         return None
 
-def del_value(filename, group_name):
+def del_value(filename, variable):
     with h5py.File(filename, 'a') as f:
-        if group_name in f and 'value' in f[group_name].attrs:
-            del f[group_name].attrs['value']
+        if variable in f and 'value' in f[variable].attrs:
+            del f[variable].attrs['value']
 
-def get_groups(filename):
-    global _groups_with_values
-    _groups_with_values = []
+def get_vars(filename):
+    global _vars_with_values
+    _vars_with_values = []
     with h5py.File(filename, 'r') as f:
-        f.visititems(_store_group_if_has_value_cb)
+        f.visititems(_store_var_if_has_value_cb)
 
-    return _groups_with_values
+    return _vars_with_values

From 60d2246625a23ac809058c58155986290fa6922c Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Thu, 3 Mar 2022 17:13:35 -0700
Subject: [PATCH 33/78] refactoring

---
 contracting/db/driver.py |  9 +++++----
 contracting/db/hdf5.py   | 37 +++++++++++++++++++------------------
 2 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 5441e1d9..ddf72804 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -21,6 +21,7 @@
 HASH_EXT = '.x'
 
 STORAGE_HOME = Path().home().joinpath('.lamden')
+FSDRIVER_HOME = Path.home().joinpath('fs')
 
 # DB maps bytes to bytes
 # Driver maps string to python object
@@ -207,13 +208,13 @@ def __delitem__(self, key: str):
             pass
 
 class FSDriver:
-    def __init__(self, root=Path.home().joinpath('fs')):
+    def __init__(self, root=FSDRIVER_HOME):
         self.root = root
         self.root.mkdir(exist_ok=True, parents=True)
 
     def __parse_key(self, key):
-        filename, variable = key.split('.', 1)
-        variable = variable.replace(':', '/')
+        filename, variable = key.split(config.INDEX_SEPARATOR, 1)
+        variable = variable.replace(config.DELIMITER, hdf5.GROUP_SEPARATOR)
 
         return filename, variable
 
@@ -224,7 +225,7 @@ def __get_files(self):
         return sorted(os.listdir(self.root))
 
     def __get_keys_from_file(self, filename):
-        return [filename + '.' + var.replace('/', ':') for var in hdf5.get_vars(self.__filename_to_path(filename))]
+        return [filename + config.INDEX_SEPARATOR + g.replace(hdf5.GROUP_SEPARATOR, config.DELIMITER) for g in hdf5.get_groups(self.__filename_to_path(filename))]
 
     def __getitem__(self, key):
         return self.get(key)
diff --git a/contracting/db/hdf5.py b/contracting/db/hdf5.py
index c956dcd0..f42c026b 100644
--- a/contracting/db/hdf5.py
+++ b/contracting/db/hdf5.py
@@ -1,36 +1,37 @@
 import h5py
 from contracting.db.encoder import encode, decode
 
-_vars_with_values = []
+GROUP_SEPARATOR = '/'
+_groups = []
 
-def _store_var_if_has_value_cb(name, obj):
-    global _vars_with_values
+def _store_group_if_has_value_cb(name, obj):
+    global _groups
     if 'value' in obj.attrs:
-        _vars_with_values.append(name)
+        _groups.append(name)
 
-def set_value(filename, variable, value):
+def set_value(filename, group, value):
     with h5py.File(filename, 'a') as f:
-        if variable not in f:
-            f.create_group(variable)
+        if group not in f:
+            f.create_group(group)
         ev = encode(value)
-        f[variable].attrs.create('value', ev, dtype='S'+str(len(ev)))
+        f[group].attrs.create('value', ev, dtype='S'+str(len(ev)))
 
-def get_value(filename, variable):
+def get_value(filename, group):
     try:
         with h5py.File(filename, 'r') as f:
-            return decode(f[variable].attrs['value'])
+            return decode(f[group].attrs['value'])
     except:
         return None
 
-def del_value(filename, variable):
+def del_value(filename, group):
     with h5py.File(filename, 'a') as f:
-        if variable in f and 'value' in f[variable].attrs:
-            del f[variable].attrs['value']
+        if group in f and 'value' in f[group].attrs:
+            del f[group].attrs['value']
 
-def get_vars(filename):
-    global _vars_with_values
-    _vars_with_values = []
+def get_groups(filename):
+    global _groups
+    _groups = []
     with h5py.File(filename, 'r') as f:
-        f.visititems(_store_var_if_has_value_cb)
+        f.visititems(_store_group_if_has_value_cb)
 
-    return _vars_with_values
+    return _groups

From a8d0e5c428aa9af6982c5d9ce2c45224d8bb2812 Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Fri, 11 Mar 2022 16:23:35 +0000
Subject: [PATCH 34/78] merge

---
 contracting/execution/executor.py      | 10 ++++++++++
 setup.py                               |  2 +-
 tests/unit/test_new_contract_driver.py |  3 +++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/contracting/execution/executor.py b/contracting/execution/executor.py
index 19b791df..ef0ed9d4 100644
--- a/contracting/execution/executor.py
+++ b/contracting/execution/executor.py
@@ -68,9 +68,19 @@ def execute(self, sender, contract_name, function_name, kwargs,
                                                    sender)
 
                 balance = driver.get(balances_key)
+
+                if type(balance) == dict:
+                    balance = ContractingDecimal(balance.get('__fixed__'))
+
                 if balance is None:
                     balance = 0
 
+                log.debug({
+                    'balance': balance,
+                    'stamp_cost': stamp_cost,
+                    'stamps': stamps
+                })
+
                 assert balance * stamp_cost >= stamps, 'Sender does not have enough stamps for the transaction. \
                                                                Balance at key {} is {}'.format(balances_key,
                                                                                                balance)
diff --git a/setup.py b/setup.py
index 77b23362..9d1f9737 100644
--- a/setup.py
+++ b/setup.py
@@ -8,7 +8,7 @@
 
 __version__ = '1.0.5.2'
 
-requirements = ['astor', 'pymongo', 'autopep8', 'stdlib_list']
+requirements = ['astor', 'pymongo', 'autopep8', 'stdlib_list', 'lamdb']
 
 ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
 
diff --git a/tests/unit/test_new_contract_driver.py b/tests/unit/test_new_contract_driver.py
index 04a8180a..7be80e76 100644
--- a/tests/unit/test_new_contract_driver.py
+++ b/tests/unit/test_new_contract_driver.py
@@ -5,6 +5,7 @@
 import marshal
 from datetime import datetime
 
+
 class TestContractDriver(TestCase):
     def setUp(self):
         self.d = Driver()
@@ -129,6 +130,8 @@ def test_items_in_both_cache_and_db_works(self):
         for k, v in kvs_1.items():
             self.c.set(k, v)
 
+        self.c.commit()
+
         for k, v in kvs_2.items():
             self.c.driver.set(k, v)
 

From 00a53ee8d8df8330ec88da581d86e18f1bdbc931 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Fri, 11 Mar 2022 12:46:01 -0500
Subject: [PATCH 35/78] Updated CacheDriver and tests from lmdb-jeff

---
 contracting/db/driver.py            | 148 +++++++++++-------
 tests/unit/test_new_cache_driver.py | 227 ++++++++++++++++------------
 2 files changed, 223 insertions(+), 152 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index ddf72804..60b5a6f9 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -366,65 +366,80 @@ def get(self, item: str):
         return decode(r.json()['value'])
 
 
+class Delta:
+    def __init__(self, writes, reads):
+        self.writes = writes
+        self.reads = reads
+
 class CacheDriver:
-    def __init__(self, driver: FSDriver=FSDriver()):
-        self.driver = driver
-        self.cache = {}
+    def __init__(self, driver: Driver=LMDBDriver()):
+        self.pending_writes = {}    # L2 cache
+        self.cache = {}             # L1 cache
+        self.driver = driver        # L0 cache
 
-        self.reads = set()
-        self.pending_writes = {}
+        self.pending_reads = {}
 
         self.pending_deltas = {}
 
-    def soft_apply(self, hcl: str, state_changes: dict):
-        deltas = {}
+    def find(self, key: str):
+        value = self.pending_writes.get(key)
+        if value is not None:
+            return value
 
-        for k, v in state_changes.items():
-            current = self.get(k)
-            deltas[k] = (current, v)
+        value = self.cache.get(key)
+        if value is not None:
+            return value
 
-            self.set(k, v)
+        value = self.driver.get(key)
+        if value is not None:
+            return value
 
-        self.pending_deltas[hcl] = deltas
+        return None
 
-    def get(self, key: str, mark=True):
-        # Try to get from cache
-        v = self.cache.get(key)
-        if v is not None:
-            rt.deduct_read(*encode_kv(key, v))
-            return v
+    def get(self, key: str):
 
-        # If it doesn't exist, get from db, add to cache
-        dv = self.driver.get(key)
-        rt.deduct_read(*encode_kv(key, dv))
+        value = self.find(key)
 
-        self.cache[key] = dv
+        if self.pending_reads.get(key) is None:
+            self.pending_reads[key] = value
 
-        # Add key to reads
-        if mark:
-            self.reads.add(key)
+        if value is not None:
+            rt.deduct_read(*encode_kv(key, value))
 
-        return dv
+        return value
 
-    def set(self, key, value, mark=True):
+    def set(self, key, value):
         rt.deduct_write(*encode_kv(key, value))
 
+        if self.pending_reads.get(key) is None:
+            self.get(key)
+
         if type(value) == decimal.Decimal or type(value) == float:
             value = ContractingDecimal(str(value))
 
-        self.cache[key] = value
-        if mark:
-            self.pending_writes[key] = value
+        self.pending_writes[key] = value
+
+    def delete(self, key):
+        self.set(key, None)
 
-    def delete(self, key, mark=True):
-        self.set(key, None, mark=mark)
+    #TODO: Fix bug where rolling back on a key written to twice rolls back to the initial state instead of the immediate previous value
+    def soft_apply(self, hcl: str):
+        deltas = {}
 
-    def commit(self):
         for k, v in self.pending_writes.items():
-            if v is None:
-                self.driver.delete(k)
-            else:
-                self.driver.set(k, v)
+            current = self.pending_reads.get(k)
+            deltas[k] = (current, v)
+
+            self.cache[k] = v
+
+        self.pending_deltas[hcl] = {
+            'writes': deltas,
+            'reads': self.pending_reads
+        }
+
+        # Clear the top cache
+        self.pending_reads = {}
+        self.pending_writes.clear()
 
     def hard_apply(self, hlc):
         # see if the HCL even exists
@@ -437,13 +452,9 @@ def hard_apply(self, hlc):
         for _hlc, _deltas in sorted(self.pending_deltas.items()):
 
             # Run through all state changes, taking the second value, which is the post delta
-            for key, delta in _deltas.items():
+            for key, delta in _deltas['writes'].items():
                 self.driver.set(key, delta[1])
-
-                try:
-                    self.cache.pop(key)
-                except KeyError:
-                    pass
+                # self.cache[key] = delta[1]
 
             # Add the key (
             to_delete.append(_hlc)
@@ -453,19 +464,52 @@ def hard_apply(self, hlc):
         # Remove the deltas from the set
         [self.pending_deltas.pop(key) for key in to_delete]
 
-    def rollback(self):
-        # Run through the state changes in reverse, reversing the newest to the oldest
-        for _hlc, _deltas in reversed(sorted(self.pending_deltas.items())):
-            # Run through all state changes, taking the first value, which is the pre delta
-            for key, delta in _deltas.items():
-                self.set(key, delta[0])
+    # Same as hard apply but for only the most recent changes and the cache
+    def commit(self):
+        self.cache.update(self.pending_writes)
 
-        self.pending_deltas.clear()
+        for k, v in self.cache.items():
+            if v is None:
+                self.driver.delete(k)
+            else:
+                self.driver.set(k, v)
 
-    def clear_pending_state(self):
         self.cache.clear()
-        self.reads.clear()
         self.pending_writes.clear()
+        self.pending_reads = {}
+
+    def rollback(self, hlc=None):
+        if hlc is None:
+            # Returns to disk state which should be whatever it was prior to any write sessions
+            self.cache.clear()
+            self.pending_reads = {}
+            self.pending_writes.clear()
+            self.pending_deltas.clear()
+        else:
+            to_delete = []
+            for _hlc, _deltas in sorted(self.pending_deltas.items())[::-1]:
+                # Clears the current reads/writes, and the reads/writes that get made when rolling back from the
+                # last HLC
+                self.pending_reads = {}
+                self.pending_writes.clear()
+
+
+                if _hlc < hlc:
+                    # if we are less than the HLC then top processing anymore, this is our rollback point
+                    break
+                else:
+                    # if we are still greater than or equal to then mark this as delete and rollback its changes
+                    to_delete.append(_hlc)
+                    # Run through all state changes, taking the second value, which is the post delta
+                    for key, delta in _deltas['writes'].items():
+                        # self.set(key, delta[0])
+                        self.cache[key] = delta[0]
+
+            # Remove the deltas from the set
+            [self.pending_deltas.pop(key) for key in to_delete]
+
+    def clear_pending_state(self):
+        self.rollback()
 
 
 class ContractDriver(CacheDriver):
diff --git a/tests/unit/test_new_cache_driver.py b/tests/unit/test_new_cache_driver.py
index 36b9c47c..21762b2d 100644
--- a/tests/unit/test_new_cache_driver.py
+++ b/tests/unit/test_new_cache_driver.py
@@ -11,22 +11,12 @@ def setUp(self):
 
     def test_get_adds_to_read(self):
         self.c.get('thing')
-        self.assertTrue('thing' in self.c.reads)
+        self.assertTrue('thing' in self.c.pending_reads)
 
     def test_set_adds_to_cache_and_pending_writes(self):
         self.c.set('thing', 1234)
-        self.assertEqual(self.c.cache['thing'], 1234)
         self.assertEqual(self.c.pending_writes['thing'], 1234)
 
-    def test_object_added_to_cache_if_read_from_db(self):
-        self.assertIsNone(self.c.cache.get('thing'))
-
-        self.d.set('thing', 8999)
-
-        self.c.get('thing')
-
-        self.assertEqual(self.c.cache['thing'], 8999)
-
     def test_object_in_cache_returns_from_cache(self):
         self.d.set('thing', 8999)
         self.c.get('thing')
@@ -58,28 +48,24 @@ def test_clear_pending_state_resets_all_variables(self):
         self.c.set('thing2', 1235)
         self.c.get('something')
 
-        self.assertTrue(len(self.c.cache) > 0)
-        self.assertTrue(len(self.c.reads) > 0)
+        self.assertTrue(len(self.c.pending_reads) > 0)
         self.assertTrue(len(self.c.pending_writes) > 0)
 
-        self.c.clear_pending_state()
+        self.c.rollback()
 
-        self.assertFalse(len(self.c.cache) > 0)
-        self.assertFalse(len(self.c.reads) > 0)
+        self.assertFalse(len(self.c.pending_reads) > 0)
         self.assertFalse(len(self.c.pending_writes) > 0)
 
     def test_soft_apply_adds_changes_to_pending_deltas(self):
-        self.c.set('thing1', 9999)
-
-        state_changes = {
-            'thing1': 8888
-        }
+        self.c.driver.set('thing1', 9999)
 
-        self.c.soft_apply('0', state_changes)
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
 
         expected_deltas = {
             '0': {
-                'thing1': (9999, 8888)
+                'writes': {'thing1': (9999, 8888)},
+                'reads': {'thing1': 9999}
             }
         }
 
@@ -89,11 +75,8 @@ def test_soft_apply_applies_the_changes_to_the_driver_but_not_hard_driver(self):
         self.c.set('thing1', 9999)
         self.c.commit()
 
-        state_changes = {
-            'thing1': 8888
-        }
-
-        self.c.soft_apply('0', state_changes)
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
 
         res = self.c.get('thing1')
 
@@ -104,11 +87,9 @@ def test_hard_apply_applies_hcl_if_exists(self):
         self.c.set('thing1', 9999)
         self.c.commit()
 
-        state_changes = {
-            'thing1': 8888
-        }
+        self.c.set('thing1', 8888)
 
-        self.c.soft_apply('0', state_changes)
+        self.c.soft_apply('0')
         self.c.hard_apply('0')
 
         res = self.c.get('thing1')
@@ -117,94 +98,121 @@ def test_hard_apply_applies_hcl_if_exists(self):
 
         self.assertEqual(self.c.driver.get('thing1'), 8888)
 
-    def test_hard_apply_only_applies_changes_up_to_delta(self):
+    def test_rollback_applies_hcl_if_exists(self):
         self.c.set('thing1', 9999)
         self.c.commit()
 
-        state_changes = {
-            'thing1': 8888
-        }
+        self.c.set('thing1', 8888)
 
-        self.c.soft_apply('0', state_changes)
+        self.c.soft_apply('0')
+        self.c.rollback('0')
 
-        state_changes = {
-            'thing1': 7777
-        }
+        res = self.c.get('thing1')
 
-        self.c.soft_apply('1', state_changes)
+        self.assertEqual(res, 9999)
 
-        state_changes = {
-            'thing1': 6666
-        }
+        self.assertEqual(self.c.driver.get('thing1'), 9999)
 
-        self.c.soft_apply('2', state_changes)
+    def test_rollback_twice_returns(self):
+        self.c.set('thing1', 9999)
+        self.c.commit()
 
-        self.c.hard_apply('1')
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
+
+        self.c.set('thing1', 7777)
+        self.c.soft_apply('1')
+
+        self.c.set('thing1', 6666)
+        self.c.soft_apply('2')
+
+        self.c.rollback('1')
 
         res = self.c.get('thing1')
 
-        self.assertEqual(res, 7777)
+        self.assertEqual(res, 8888)
 
-        self.assertEqual(self.c.driver.get('thing1'), 7777)
+        self.assertEqual(self.c.driver.get('thing1'), 9999)
 
-    def test_hard_apply_removes_hcls(self):
+    def test_rollback_removes_hlcs(self):
         self.c.set('thing1', 9999)
         self.c.commit()
 
-        state_changes = {
-            'thing1': 8888
-        }
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
 
-        self.c.soft_apply('0', state_changes)
+        self.c.set('thing1', 7777)
+        self.c.soft_apply('1')
 
-        state_changes = {
-            'thing1': 7777
-        }
+        self.c.set('thing1', 6666)
+        self.c.soft_apply('2')
 
-        self.c.soft_apply('1', state_changes)
+        self.c.rollback('1')
 
-        state_changes = {
-            'thing1': 6666
-        }
+        self.assertIsNone(self.c.pending_deltas.get('2'))
+        self.assertIsNone(self.c.pending_deltas.get('1'))
+
+    def test_hard_apply_only_applies_changes_up_to_delta(self):
+        self.c.set('thing1', 9999)
+        self.c.commit()
+
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
+
+        self.c.set('thing1', 7777)
+        self.c.soft_apply('1')
 
-        self.c.soft_apply('2', state_changes)
+        self.c.set('thing1', 6666)
+        self.c.soft_apply('2')
+
+        self.c.set('thing1', 5555)
+        self.c.soft_apply('3')
+
+        self.c.hard_apply('1')
+
+        res = self.c.get('thing1')
+
+        self.assertEqual(res, 5555)
+
+        self.assertEqual(self.c.driver.get('thing1'), 7777)
+
+    def test_hard_apply_removes_hcls(self):
+        self.c.set('thing1', 9999)
+        self.c.commit()
+
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
+
+        self.c.set('thing1', 7777)
+        self.c.soft_apply('1')
+
+        self.c.set('thing1', 6666)
+        self.c.soft_apply('2')
 
         self.c.hard_apply('0')
 
-        hcls = {
-            '1': {
-                'thing1': (8888, 7777)
-            },
-            '2': {
-                'thing1': (7777, 6666)
-            }
-        }
+        hlcs = {'1':
+                    {'writes': {'thing1': (8888, 7777)}, 'reads': {'thing1': 8888}},
+                '2':
+                    {'writes': {'thing1': (7777, 6666)}, 'reads': {'thing1': 7777}}
+                }
 
-        self.assertDictEqual(self.c.pending_deltas, hcls)
+        self.assertDictEqual(self.c.pending_deltas, hlcs)
 
     def test_rollback_returns_to_initial_state(self):
         self.c.set('thing1', 9999)
         self.c.commit()
 
-        state_changes = {
-            'thing1': 8888
-        }
-
-        self.c.soft_apply('0', state_changes)
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
         self.assertEqual(self.c.get('thing1'), 8888)
 
-        state_changes = {
-            'thing1': 7777
-        }
-
-        self.c.soft_apply('1', state_changes)
+        self.c.set('thing1', 7777)
+        self.c.soft_apply('1')
         self.assertEqual(self.c.get('thing1'), 7777)
 
-        state_changes = {
-            'thing1': 6666
-        }
-
-        self.c.soft_apply('2', state_changes)
+        self.c.set('thing1', 6666)
+        self.c.soft_apply('2')
         self.assertEqual(self.c.get('thing1'), 6666)
 
         self.c.rollback()
@@ -216,27 +224,46 @@ def test_rollback_removes_hlcs(self):
         self.c.set('thing1', 9999)
         self.c.commit()
 
-        state_changes = {
-            'thing1': 8888
-        }
-
-        self.c.soft_apply('0', state_changes)
+        self.c.set('thing1', 8888)
+        self.c.soft_apply('0')
         self.assertEqual(self.c.get('thing1'), 8888)
 
-        state_changes = {
-            'thing1': 7777
-        }
-
-        self.c.soft_apply('1', state_changes)
+        self.c.set('thing1', 7777)
+        self.c.soft_apply('1')
         self.assertEqual(self.c.get('thing1'), 7777)
 
-        state_changes = {
-            'thing1': 6666
-        }
-
-        self.c.soft_apply('2', state_changes)
+        self.c.set('thing1', 6666)
+        self.c.soft_apply('2')
         self.assertEqual(self.c.get('thing1'), 6666)
 
         self.c.rollback()
 
         self.assertDictEqual(self.c.pending_deltas, {})
+
+    def test_find_returns_none(self):
+        x = self.c.find('none')
+        self.assertIsNone(x)
+
+    def test_find_returns_driver(self):
+        self.c.driver.set('none', 123)
+
+        x = self.c.find('none')
+
+        self.assertEqual(x, 123)
+
+    def test_find_returns_cache(self):
+        self.c.driver.set('none', 123)
+        self.c.cache['none'] = 999
+
+        x = self.c.find('none')
+
+        self.assertEqual(x, 999)
+
+    def test_find_returns_pending_writes(self):
+        self.c.driver.set('none', 123)
+        self.c.cache['none'] = 999
+        self.c.pending_writes['none'] = 5555
+
+        x = self.c.find('none')
+
+        self.assertEqual(x, 5555)
\ No newline at end of file

From 9d94baaf9e422c885a8215a1316ad5ac1117db55 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Fri, 11 Mar 2022 12:55:39 -0500
Subject: [PATCH 36/78] remove mark argument being passed to CacheDriver get
 method

---
 contracting/db/driver.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 60b5a6f9..8b08d4d7 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -549,11 +549,11 @@ def make_key(self, contract, variable, args=[]):
 
     def get_var(self, contract, variable, arguments=[], mark=True):
         key = self.make_key(contract, variable, arguments)
-        return self.get(key, mark=mark)
+        return self.get(key)
 
     def set_var(self, contract, variable, arguments=[], value=None, mark=True):
         key = self.make_key(contract, variable, arguments)
-        self.set(key, value, mark=mark)
+        self.set(key, value)
 
     def get_contract(self, name):
         return self.get_var(name, CODE_KEY)

From 8819494d79c98c4f620b53063b1a231627ed5574 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Fri, 11 Mar 2022 12:59:29 -0500
Subject: [PATCH 37/78] executor to use pending_reads instead of reads

---
 contracting/execution/executor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/execution/executor.py b/contracting/execution/executor.py
index ef0ed9d4..01ac8e4a 100644
--- a/contracting/execution/executor.py
+++ b/contracting/execution/executor.py
@@ -165,7 +165,7 @@ def execute(self, sender, contract_name, function_name, kwargs,
             'result': result,
             'stamps_used': stamps_used,
             'writes': deepcopy(driver.pending_writes),
-            'reads': driver.reads
+            'reads': driver.pending_reads
         }
 
         disable_restricted_imports()

From 7d305a458051923f895b45f5bc18eb113574e242 Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Fri, 11 Mar 2022 18:02:19 +0000
Subject: [PATCH 38/78] ref

---
 contracting/db/driver.py | 42 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index ddf72804..70c854c9 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -16,6 +16,7 @@
 import motor.motor_asyncio
 import asyncio
 import contracting.db.hdf5 as hdf5
+import logging
 
 FILE_EXT = '.d'
 HASH_EXT = '.x'
@@ -472,6 +473,7 @@ class ContractDriver(CacheDriver):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.delimiter = '.'
+        self.log = logging.getLogger('Driver')
 
     def items(self, prefix=''):
         # Get all of the items in the cache currently
@@ -554,6 +556,46 @@ def flush(self):
     def get_contract_keys(self, name):
         return self.keys(name)
 
+    def rollback_drivers(self, hlc_timestamp):
+        # Roll back the current state to the point of the last block consensus
+        self.log.debug(f"Length of Pending Deltas BEFORE {len(self.driver.pending_deltas.keys())}")
+        self.log.debug(f"rollback to hlc_timestamp: {hlc_timestamp}")
+
+        if hlc_timestamp is None:
+            # Returns to disk state which should be whatever it was prior to any write sessions
+            self.cache.clear()
+            self.reads = set()
+            self.pending_writes.clear()
+            self.pending_deltas.clear()
+        else:
+            to_delete = []
+            for _hlc, _deltas in sorted(self.pending_deltas.items())[::-1]:
+                # Clears the current reads/writes, and the reads/writes that get made when rolling back from the
+                # last HLC
+                self.reads = set()
+                self.pending_writes.clear()
+
+
+                if _hlc < hlc_timestamp:
+                    self.log.debug(f"{_hlc} is less than {hlc_timestamp}, breaking!")
+                    # if we are less than the HLC then top processing anymore, this is our rollback point
+                    break
+                else:
+                    # if we are still greater than or equal to then mark this as delete and rollback its changes
+                    to_delete.append(_hlc)
+                    # Run through all state changes, taking the second value, which is the post delta
+                    for key, delta in _deltas['writes'].items():
+                        # self.set(key, delta[0])
+                        self.cache[key] = delta[0]
+
+            # Remove the deltas from the set
+            self.log.debug(to_delete)
+            [self.pending_deltas.pop(key) for key in to_delete]
+
+        #self.driver.rollback(hlc=hlc_timestamp)
+
+        self.log.debug(f"Length of Pending Deltas AFTER {len(self.driver.pending_deltas.keys())}")
+
     # Set cache to None
     # Set pending writes to none
     # def delete(self, key):

From 675177d5e6f9f304009d38ddfabc5f12ba577d75 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Mon, 14 Mar 2022 16:34:28 -0600
Subject: [PATCH 39/78] add h5py to dependencies

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 77b23362..eddc44ff 100644
--- a/setup.py
+++ b/setup.py
@@ -8,7 +8,7 @@
 
 __version__ = '1.0.5.2'
 
-requirements = ['astor', 'pymongo', 'autopep8', 'stdlib_list']
+requirements = ['astor', 'pymongo', 'autopep8', 'stdlib_list', 'h5py==3.1.0']
 
 ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
 

From c749cf1b1601f5328e349f8dddfbec961c398375 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 16 Mar 2022 12:36:43 -0600
Subject: [PATCH 40/78] change hdf5 driver home

---
 contracting/db/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index ddf72804..de09c26c 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -21,7 +21,7 @@
 HASH_EXT = '.x'
 
 STORAGE_HOME = Path().home().joinpath('.lamden')
-FSDRIVER_HOME = Path.home().joinpath('fs')
+FSDRIVER_HOME = STORAGE_HOME.joinpath('state')
 
 # DB maps bytes to bytes
 # Driver maps string to python object

From 9f8b994da965faae73e27803206f64c6bf753262 Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Thu, 17 Mar 2022 18:13:35 +0000
Subject: [PATCH 41/78] filename fix

---
 contracting/db/driver.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 70c854c9..ac1d7073 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -214,8 +214,13 @@ def __init__(self, root=FSDRIVER_HOME):
         self.root.mkdir(exist_ok=True, parents=True)
 
     def __parse_key(self, key):
-        filename, variable = key.split(config.INDEX_SEPARATOR, 1)
-        variable = variable.replace(config.DELIMITER, hdf5.GROUP_SEPARATOR)
+        try:
+            filename, variable = key.split(config.INDEX_SEPARATOR, 1)
+            variable = variable.replace(config.DELIMITER, hdf5.GROUP_SEPARATOR)
+        except ValueError:
+            filename = '__misc'
+            variable = key
+            variable = variable.replace(config.DELIMITER, hdf5.GROUP_SEPARATOR)
 
         return filename, variable
 

From 42e9afdf559d1df0523dfadd3d10050cb0d6406f Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Tue, 29 Mar 2022 11:53:42 -0600
Subject: [PATCH 42/78] remove redundant checks

---
 contracting/db/hdf5.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/contracting/db/hdf5.py b/contracting/db/hdf5.py
index f42c026b..db681855 100644
--- a/contracting/db/hdf5.py
+++ b/contracting/db/hdf5.py
@@ -11,8 +11,10 @@ def _store_group_if_has_value_cb(name, obj):
 
 def set_value(filename, group, value):
     with h5py.File(filename, 'a') as f:
-        if group not in f:
+        try:
             f.create_group(group)
+        except ValueError:
+            pass
         ev = encode(value)
         f[group].attrs.create('value', ev, dtype='S'+str(len(ev)))
 
@@ -25,8 +27,10 @@ def get_value(filename, group):
 
 def del_value(filename, group):
     with h5py.File(filename, 'a') as f:
-        if group in f and 'value' in f[group].attrs:
+        try:
             del f[group].attrs['value']
+        except KeyError:
+            pass
 
 def get_groups(filename):
     global _groups

From d86a3789f7601862407ef3a0f79b839e9180c64d Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 30 Mar 2022 13:53:52 -0600
Subject: [PATCH 43/78] c extension for hdf5

---
 contracting/db/driver.py        |  20 +++---
 contracting/db/hdf5.py          |  41 -----------
 contracting/db/hdf5/__init__.py |   0
 contracting/db/hdf5/h5.py       |  28 ++++++++
 contracting/db/hdf5/h5c.c       | 116 ++++++++++++++++++++++++++++++++
 setup.py                        |   3 +-
 6 files changed, 158 insertions(+), 50 deletions(-)
 delete mode 100644 contracting/db/hdf5.py
 create mode 100644 contracting/db/hdf5/__init__.py
 create mode 100644 contracting/db/hdf5/h5.py
 create mode 100644 contracting/db/hdf5/h5c.c

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index de09c26c..859c5bae 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -15,7 +15,7 @@
 import lmdb
 import motor.motor_asyncio
 import asyncio
-import contracting.db.hdf5 as hdf5
+from contracting.db.hdf5 import h5
 
 FILE_EXT = '.d'
 HASH_EXT = '.x'
@@ -213,19 +213,23 @@ def __init__(self, root=FSDRIVER_HOME):
         self.root.mkdir(exist_ok=True, parents=True)
 
     def __parse_key(self, key):
-        filename, variable = key.split(config.INDEX_SEPARATOR, 1)
-        variable = variable.replace(config.DELIMITER, hdf5.GROUP_SEPARATOR)
+        try:
+            filename, variable = key.split(config.INDEX_SEPARATOR, 1)
+            variable = variable.replace(config.DELIMITER, h5.GROUP_SEPARATOR)
+        except:
+            filename = '__misc'
+            variable = key.replace(config.DELIMITER, h5.GROUP_SEPARATOR)
 
         return filename, variable
 
     def __filename_to_path(self, filename):
-        return self.root.joinpath(filename)
+        return str(self.root.joinpath(filename))
 
     def __get_files(self):
         return sorted(os.listdir(self.root))
 
     def __get_keys_from_file(self, filename):
-        return [filename + config.INDEX_SEPARATOR + g.replace(hdf5.GROUP_SEPARATOR, config.DELIMITER) for g in hdf5.get_groups(self.__filename_to_path(filename))]
+        return [filename + config.INDEX_SEPARATOR + g.replace(h5.GROUP_SEPARATOR, config.DELIMITER) for g in h5.get_groups(self.__filename_to_path(filename))]
 
     def __getitem__(self, key):
         return self.get(key)
@@ -239,11 +243,11 @@ def __delitem__(self, key):
     def get(self, key):
         filename, variable = self.__parse_key(key)
 
-        return hdf5.get_value(self.__filename_to_path(filename), variable)
+        return h5.get_value(self.__filename_to_path(filename), variable)
 
     def set(self, key, value):
         filename, variable = self.__parse_key(key)
-        hdf5.set_value(self.__filename_to_path(filename), variable, value)
+        h5.set_value(self.__filename_to_path(filename), variable, value)
 
     def flush(self):
         try:
@@ -254,7 +258,7 @@ def flush(self):
 
     def delete(self, key):
         filename, variable = self.__parse_key(key)
-        hdf5.del_value(self.__filename_to_path(filename), variable)
+        h5.del_value(self.__filename_to_path(filename), variable)
 
     def iter(self, prefix='', length=0):
         keys = []
diff --git a/contracting/db/hdf5.py b/contracting/db/hdf5.py
deleted file mode 100644
index db681855..00000000
--- a/contracting/db/hdf5.py
+++ /dev/null
@@ -1,41 +0,0 @@
-import h5py
-from contracting.db.encoder import encode, decode
-
-GROUP_SEPARATOR = '/'
-_groups = []
-
-def _store_group_if_has_value_cb(name, obj):
-    global _groups
-    if 'value' in obj.attrs:
-        _groups.append(name)
-
-def set_value(filename, group, value):
-    with h5py.File(filename, 'a') as f:
-        try:
-            f.create_group(group)
-        except ValueError:
-            pass
-        ev = encode(value)
-        f[group].attrs.create('value', ev, dtype='S'+str(len(ev)))
-
-def get_value(filename, group):
-    try:
-        with h5py.File(filename, 'r') as f:
-            return decode(f[group].attrs['value'])
-    except:
-        return None
-
-def del_value(filename, group):
-    with h5py.File(filename, 'a') as f:
-        try:
-            del f[group].attrs['value']
-        except KeyError:
-            pass
-
-def get_groups(filename):
-    global _groups
-    _groups = []
-    with h5py.File(filename, 'r') as f:
-        f.visititems(_store_group_if_has_value_cb)
-
-    return _groups
diff --git a/contracting/db/hdf5/__init__.py b/contracting/db/hdf5/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/contracting/db/hdf5/h5.py b/contracting/db/hdf5/h5.py
new file mode 100644
index 00000000..8febbab6
--- /dev/null
+++ b/contracting/db/hdf5/h5.py
@@ -0,0 +1,28 @@
+import h5py
+from contracting.db.encoder import encode, decode
+import contracting.db.hdf5.h5c as h5c
+
+GROUP_SEPARATOR = '/'
+_groups = []
+
+def _store_group_if_has_value_cb(name, obj):
+    global _groups
+    if 'value' in obj.attrs:
+        _groups.append(name)
+
+def set_value(filepath, group, value):
+    h5c.set(filepath, group, encode(value))
+
+def get_value(filepath, group):
+    return decode(h5c.get(filepath, group))
+
+def del_value(filepath, group):
+    h5c.delete(filepath, group)
+
+def get_groups(filepath):
+    global _groups
+    _groups = []
+    with h5py.File(filepath, 'r') as f:
+        f.visititems(_store_group_if_has_value_cb)
+
+    return _groups
diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
new file mode 100644
index 00000000..6d9a6839
--- /dev/null
+++ b/contracting/db/hdf5/h5c.c
@@ -0,0 +1,116 @@
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <hdf5.h>
+
+#define BUFSIZE 64000
+#define ATT_NAME "value"
+
+static PyObject *
+set(PyObject *self, PyObject *args)
+{
+    H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+
+    char *filepath, *group, *value;
+    if(!PyArg_ParseTuple(args, "ssz", &filepath, &group, &value))
+        return NULL;
+
+    hid_t fid = H5Fopen(filepath, H5F_ACC_RDWR, H5P_DEFAULT);
+    if(fid < 0)
+        fid = H5Fcreate(filepath, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+
+    hid_t gid = H5Gopen(fid, group, H5P_DEFAULT);
+    if(gid < 0)
+    {
+        hid_t lcpl = H5Pcreate(H5P_LINK_CREATE);
+        H5Pset_create_intermediate_group(lcpl, 1);
+        gid = H5Gcreate(fid, group, lcpl, H5P_DEFAULT, H5P_DEFAULT);
+        H5Pclose(lcpl);
+    }
+
+    hid_t atype = H5Tcopy(H5T_C_S1);
+    H5Tset_size(atype, strlen(value));
+    H5Adelete(gid, ATT_NAME);
+    hid_t aid = H5Acreate(gid, ATT_NAME, atype, H5Screate(H5S_SCALAR), H5P_DEFAULT, H5P_DEFAULT);
+    H5Awrite(aid, atype, value);
+
+    H5Tclose(atype);
+    H5Aclose(aid);
+    H5Gclose(gid);
+    H5Fclose(fid);
+
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+get(PyObject *self, PyObject *args)
+{
+    static char buf[BUFSIZE];
+
+    H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+
+    char *filepath, *group;
+    if(!PyArg_ParseTuple(args, "ss", &filepath, &group))
+        return NULL;
+
+    hid_t fid = H5Fopen(filepath, H5F_ACC_RDONLY, H5P_DEFAULT);
+    if(fid < 0)
+        Py_RETURN_NONE;
+
+    hid_t aid = H5Aopen_by_name(fid, group, ATT_NAME, H5P_DEFAULT, H5P_DEFAULT);
+    if(aid < 0)
+    {
+        H5Fclose(fid);
+        Py_RETURN_NONE;
+    }
+
+    hid_t atype = H5Tcopy(H5T_C_S1);
+    H5Tset_size(atype, BUFSIZE);
+    memset(buf, 0, BUFSIZE);
+    if(H5Aread(aid, atype, buf) < 0)
+        Py_RETURN_NONE;
+
+    H5Tclose(atype);
+    H5Aclose(aid);
+    H5Fclose(fid);
+
+    return PyUnicode_FromString(buf);
+}
+
+static PyObject *
+delete(PyObject *self, PyObject *args)
+{
+    H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+
+    char *filepath, *group;
+    if(!PyArg_ParseTuple(args, "ss", &filepath, &group))
+        return NULL;
+
+    hid_t fid = H5Fopen(filepath, H5F_ACC_RDWR, H5P_DEFAULT);
+    hid_t gid = H5Gopen(fid, group, H5P_DEFAULT);
+    H5Adelete(gid, ATT_NAME);
+
+    H5Gclose(gid);
+    H5Fclose(fid);
+
+    Py_RETURN_NONE;
+}
+
+static PyMethodDef methods[] = {
+    {"set", set, METH_VARARGS, "Set value"},
+    {"get", get, METH_VARARGS, "Get value"},
+    {"delete", delete, METH_VARARGS, "Delete value"}
+};
+
+static struct PyModuleDef h5cmodule = {
+    PyModuleDef_HEAD_INIT,
+    "h5c",
+    NULL,
+    -1,
+    methods
+};
+
+PyMODINIT_FUNC
+PyInit_h5c(void)
+{
+    return PyModule_Create(&h5cmodule);
+}
diff --git a/setup.py b/setup.py
index eddc44ff..4345a512 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 from setuptools.extension import Extension
 from distutils.command.build_ext import build_ext, CCompilerError, DistutilsExecError, DistutilsPlatformError
 from distutils import errors
-import sys
+import sys, os
 
 major = 0
 
@@ -58,6 +58,7 @@ def build_extension(self, ext):
     include_package_data=True,
     ext_modules=[
         Extension('contracting.execution.metering.tracer', sources=['contracting/execution/metering/tracer.c']),
+        Extension('contracting.db.hdf5.h5c', sources=['contracting/db/hdf5/h5c.c'], include_dirs=[os.system('pkg-config --cflags hdf5')], libraries=[os.system('pkg-config --libs hdf5')])
     ],
     cmdclass={
         'build_ext': ve_build_ext,

From 313176a11010c8695a8aea9e7e93cf0e21d0449e Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 30 Mar 2022 15:25:04 -0600
Subject: [PATCH 44/78] utilize pkgconfig for extension dependencies

---
 setup.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 4345a512..ec247f2c 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 from setuptools.extension import Extension
 from distutils.command.build_ext import build_ext, CCompilerError, DistutilsExecError, DistutilsPlatformError
 from distutils import errors
-import sys, os
+import sys, subprocess
 
 major = 0
 
@@ -42,6 +42,15 @@ def build_extension(self, ext):
                 raise BuildFailed()
             raise
 
+def pkgconfig(package):
+    flag_map = {'-I': 'include_dirs', '-L': 'library_dirs', '-l': 'libraries'}
+    res = {}
+    output = subprocess.getoutput('pkg-config --cflags --libs {}'.format(package))
+    for token in output.strip().split():
+        res.setdefault(flag_map.get(token[:2]), []).append(token[2:])
+
+    return res
+
 setup(
     name='contracting',
     version=__version__,
@@ -58,7 +67,7 @@ def build_extension(self, ext):
     include_package_data=True,
     ext_modules=[
         Extension('contracting.execution.metering.tracer', sources=['contracting/execution/metering/tracer.c']),
-        Extension('contracting.db.hdf5.h5c', sources=['contracting/db/hdf5/h5c.c'], include_dirs=[os.system('pkg-config --cflags hdf5')], libraries=[os.system('pkg-config --libs hdf5')])
+        Extension('contracting.db.hdf5.h5c', sources=['contracting/db/hdf5/h5c.c'], **pkgconfig('hdf5'))
     ],
     cmdclass={
         'build_ext': ve_build_ext,

From 143913777f36729728a68773684534add92ab891 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Mon, 2 May 2022 17:09:26 -0600
Subject: [PATCH 45/78] Rollback default driver to MongoDB driver

---
 contracting/db/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 859c5bae..8e770c4b 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -371,7 +371,7 @@ def get(self, item: str):
 
 
 class CacheDriver:
-    def __init__(self, driver: FSDriver=FSDriver()):
+    def __init__(self, driver: Driver=Driver()):
         self.driver = driver
         self.cache = {}
 

From 0c2d93461460e10401d5c49cb0b93e83057bc9b7 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 4 May 2022 12:46:06 -0600
Subject: [PATCH 46/78] Use HDF5 driver

---
 contracting/db/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 8e770c4b..859c5bae 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -371,7 +371,7 @@ def get(self, item: str):
 
 
 class CacheDriver:
-    def __init__(self, driver: Driver=Driver()):
+    def __init__(self, driver: FSDriver=FSDriver()):
         self.driver = driver
         self.cache = {}
 

From 0022de2ca13f1db5a661856c4a271518c6b3c8f4 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Tue, 17 May 2022 12:10:04 -0400
Subject: [PATCH 47/78] change default to FSDRIVER

---
 contracting/db/driver.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index d141e9a2..e0b30f41 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -281,8 +281,8 @@ def __init__(self, filename=STORAGE_HOME.joinpath('state')):
         self.filename = filename
         self.filename.mkdir(exist_ok=True, parents=True)
 
-        self.db_writer = lamdb.open(path=str(self.filename), map_size=int(1e12), readonly=False)
-        self.db_reader = lamdb.open(path=str(self.filename), map_size=int(1e12), readonly=True, lock=False)
+        self.db_writer = lmdb.open(path=str(self.filename), map_size=int(1e12), readonly=False)
+        self.db_reader = lmdb.open(path=str(self.filename), map_size=int(1e12), readonly=True, lock=False)
 
     def get(self, item: str):
         with self.db_reader.begin() as tx:
@@ -373,7 +373,7 @@ def get(self, item: str):
         return decode(r.json()['value'])
 
 class CacheDriver:
-    def __init__(self, driver: Driver=LMDBDriver()):
+    def __init__(self, driver: Driver = FSDriver()):
         self.pending_writes = {}    # L2 cache
         self.cache = {}             # L1 cache
         self.driver = driver        # L0 cache

From e84306383fb470794c510ac3c12892ca7d470420 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Wed, 15 Jun 2022 11:03:05 -0400
Subject: [PATCH 48/78] driver tweaks

---
 contracting/db/driver.py | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index e0b30f41..e90bf7d0 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -159,8 +159,8 @@ def __init__(self):
         super().__init__()
         self.db = {}
 
-    def get(self, item):
-        key = item.encode()
+    def get(self, key):
+        key = key.encode()
         value = self.db.get(key)
         return decode(value)
 
@@ -397,15 +397,16 @@ def find(self, key: str):
 
         return None
 
-    def get(self, key: str):
+    def get(self, key: str, save: bool = True):
 
         value = self.find(key)
 
-        if self.pending_reads.get(key) is None:
-            self.pending_reads[key] = value
+        if save:
+            if self.pending_reads.get(key) is None:
+                self.pending_reads[key] = value
 
-        if value is not None:
-            rt.deduct_read(*encode_kv(key, value))
+            if value is not None:
+                rt.deduct_read(*encode_kv(key, value))
 
         return value
 
@@ -442,6 +443,21 @@ def soft_apply(self, hcl: str):
         self.pending_reads = {}
         self.pending_writes.clear()
 
+    def soft_apply_rewards(self, hcl: str):
+        deltas = {}
+
+        for k, v in self.pending_writes.items():
+            current = self.pending_reads.get(k)
+            deltas[k] = (current, v)
+
+            self.cache[k] = v
+
+        self.pending_deltas[hcl]['rewards'] = deltas
+
+        # Clear the top cache
+        self.pending_reads = {}
+        self.pending_writes.clear()
+
     def hard_apply(self, hlc):
         # see if the HCL even exists
         if self.pending_deltas.get(hlc) is None:

From 8af8d5b06630b1c4927cdc369fde3730364a09b9 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Thu, 23 Jun 2022 13:30:18 -0400
Subject: [PATCH 49/78] merge lmdb-jeff cache driver changes

---
 contracting/db/driver.py                 |  8 ++++----
 setup.py                                 | 26 ++++++++++++++++++++----
 tests/integration/test_misc_contracts.py |  2 +-
 tests/unit/test_linter.py                |  2 +-
 4 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index fde246fb..6dadc6c6 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -159,8 +159,8 @@ def __init__(self):
         super().__init__()
         self.db = {}
 
-    def get(self, key):
-        key = key.encode()
+    def get(self, item: str):
+        key = item.encode()
         value = self.db.get(key)
         return decode(value)
 
@@ -242,8 +242,8 @@ def __setitem__(self, key, value):
     def __delitem__(self, key):
         self.delete(key)
 
-    def get(self, key):
-        filename, variable = self.__parse_key(key)
+    def get(self, item: str):
+        filename, variable = self.__parse_key(item)
 
         return h5.get_value(self.__filename_to_path(filename), variable)
 
diff --git a/setup.py b/setup.py
index 1974bf62..8e7bf69d 100644
--- a/setup.py
+++ b/setup.py
@@ -3,12 +3,13 @@
 from distutils.command.build_ext import build_ext, CCompilerError, DistutilsExecError, DistutilsPlatformError
 from distutils import errors
 import sys, subprocess
+from sys import platform
 
 major = 0
 
 __version__ = '1.0.5.2'
 
-requirements = ['astor', 'pymongo', 'autopep8', 'stdlib_list', 'h5py==3.1.0', 'lamdb']
+requirements = ['astor', 'pymongo', 'autopep8', 'stdlib_list', 'h5py==3.1.0', 'lmdb']
 
 ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
 
@@ -45,9 +46,26 @@ def build_extension(self, ext):
 def pkgconfig(package):
     flag_map = {'-I': 'include_dirs', '-L': 'library_dirs', '-l': 'libraries'}
     res = {}
-    output = subprocess.getoutput('pkg-config --cflags --libs {}'.format(package))
-    for token in output.strip().split():
-        res.setdefault(flag_map.get(token[:2]), []).append(token[2:])
+
+    if platform == "linux" or platform == "linux2":
+        output = subprocess.getoutput('pkg-config --cflags --libs {}'.format(package))
+        for token in output.strip().split():
+            res.setdefault(flag_map.get(token[:2]), []).append(token[2:])
+
+    elif platform == "darwin":
+        res['libraries'] = ['hdf5']
+        output = subprocess.getoutput('brew list hdf5')
+        print(output)
+        if 'Error: No sach keg:' in output:
+            raise ModuleNotFoundError('Install HDF5 package using brew. "brew install hdf5"')
+        for token in output.strip().split():
+            if "/include/" in token:
+                res['include_dirs'] = [token.split('/include/')[0] + '/include/']
+            if "/lib/" in token:
+                res['library_dirs'] = [token.split('/lib/')[0] + '/lib/']
+
+    elif platform == "win32":
+        raise NotImplemented("Cannot install on Windows")
 
     return res
 
diff --git a/tests/integration/test_misc_contracts.py b/tests/integration/test_misc_contracts.py
index ae8a675e..25f5d158 100644
--- a/tests/integration/test_misc_contracts.py
+++ b/tests/integration/test_misc_contracts.py
@@ -8,7 +8,7 @@ def too_many_writes():
 
     @export
     def single():
-        v.set('a' * (32 * 1024 + 1))
+        v.set('a' * (64 * 1024 + 1))
 
     @export
     def multiple():
diff --git a/tests/unit/test_linter.py b/tests/unit/test_linter.py
index 66459c0c..a3567556 100644
--- a/tests/unit/test_linter.py
+++ b/tests/unit/test_linter.py
@@ -439,7 +439,7 @@ def greeting(name: str) -> str:
         c = ast.parse(code)
         chk = self.l.check(c)
 
-        self.assertEqual(chk, ['Line 2 : S18- Illegal use of return annotation : str'])
+        self.assertEqual(['Line 2 : S18- Illegal use of return annotation : str'], chk)
 
 
     def test_contract_annotation(self):

From 7bfa0ee0c37aff47ff41be26d84f836f0e79d9d7 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Thu, 23 Jun 2022 18:07:26 -0600
Subject: [PATCH 50/78] generic pkgconfig for darwin

---
 setup.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/setup.py b/setup.py
index 8e7bf69d..d005dd79 100644
--- a/setup.py
+++ b/setup.py
@@ -1,9 +1,9 @@
+from distutils import errors
+from distutils.command.build_ext import build_ext, CCompilerError, DistutilsExecError, DistutilsPlatformError
 from setuptools import setup, find_packages
 from setuptools.extension import Extension
-from distutils.command.build_ext import build_ext, CCompilerError, DistutilsExecError, DistutilsPlatformError
-from distutils import errors
-import sys, subprocess
 from sys import platform
+import sys, subprocess, pathlib
 
 major = 0
 
@@ -53,16 +53,12 @@ def pkgconfig(package):
             res.setdefault(flag_map.get(token[:2]), []).append(token[2:])
 
     elif platform == "darwin":
-        res['libraries'] = ['hdf5']
-        output = subprocess.getoutput('brew list hdf5')
-        print(output)
-        if 'Error: No sach keg:' in output:
-            raise ModuleNotFoundError('Install HDF5 package using brew. "brew install hdf5"')
-        for token in output.strip().split():
-            if "/include/" in token:
-                res['include_dirs'] = [token.split('/include/')[0] + '/include/']
-            if "/lib/" in token:
-                res['library_dirs'] = [token.split('/lib/')[0] + '/lib/']
+        res['libraries'] = [f'{package}']
+        output = subprocess.getoutput(f'brew --prefix {package}')
+        if 'Error:' in output or not pathlib.Path(output).is_dir():
+            raise ModuleNotFoundError(f'{output}\nInstall "{package}"" package using brew. "brew install {package}"')
+        res['include_dirs'] = [output + '/include/']
+        res['library_dirs'] = [output + '/lib/']
 
     elif platform == "win32":
         raise NotImplemented("Cannot install on Windows")

From e12d22856359243d72f3c4ae3d041ac329e657b0 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Fri, 24 Jun 2022 13:02:01 -0600
Subject: [PATCH 51/78] fix default argument constructor isn't called

---
 contracting/db/driver.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 6dadc6c6..6733eb72 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -372,10 +372,10 @@ def get(self, item: str):
         return decode(r.json()['value'])
 
 class CacheDriver:
-    def __init__(self, driver: Driver = FSDriver()):
-        self.pending_writes = {}    # L2 cache
-        self.cache = {}             # L1 cache
-        self.driver = driver        # L0 cache
+    def __init__(self, driver=None):
+        self.pending_writes = {}             # L2 cache
+        self.cache = {}                      # L1 cache
+        self.driver = driver or FSDriver()   # L0 cache
 
         self.pending_reads = {}
 

From d468900a2b4e003a5a2d822cea055231c8337034 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Fri, 24 Jun 2022 13:02:59 -0600
Subject: [PATCH 52/78] fix flush

---
 contracting/db/driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 6733eb72..f6363622 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -254,9 +254,9 @@ def set(self, key, value):
     def flush(self):
         try:
             shutil.rmtree(self.root)
-            self.root.mkdir(exist_ok=True, parents=True)
         except FileNotFoundError:
             pass
+        self.root.mkdir(exist_ok=True, parents=True)
 
     def delete(self, key):
         filename, variable = self.__parse_key(key)

From f4e80e24c37da873062e630fbdb4b3206803e44c Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 13 Jul 2022 11:38:30 -0600
Subject: [PATCH 53/78] fix convert_dict

---
 contracting/db/encoder.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/contracting/db/encoder.py b/contracting/db/encoder.py
index 0bdbe37f..40661bcf 100644
--- a/contracting/db/encoder.py
+++ b/contracting/db/encoder.py
@@ -170,6 +170,9 @@ def convert(k, v):
 
 
 def convert_dict(d):
+    if not isinstance(d, dict):
+        return d
+
     d2 = dict()
     for k, v in d.items():
         if k in TYPES:

From 7bd7241bc085ae286080f2d0d896611764ab88b3 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 20 Jul 2022 18:57:06 -0600
Subject: [PATCH 54/78] h5 driver: enable debug in set_value

---
 contracting/db/hdf5/h5c.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index 6d9a6839..4f5f507c 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -8,7 +8,7 @@
 static PyObject *
 set(PyObject *self, PyObject *args)
 {
-    H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+    //H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
 
     char *filepath, *group, *value;
     if(!PyArg_ParseTuple(args, "ssz", &filepath, &group, &value))

From 282bf34fece8ae67a24d4f9977366e3386a20411 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 20 Jul 2022 19:05:55 -0600
Subject: [PATCH 55/78] h5 driver: enable logging everywhere

---
 contracting/db/hdf5/h5c.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index 4f5f507c..20210591 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -46,7 +46,7 @@ get(PyObject *self, PyObject *args)
 {
     static char buf[BUFSIZE];
 
-    H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+    //H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
 
     char *filepath, *group;
     if(!PyArg_ParseTuple(args, "ss", &filepath, &group))
@@ -79,7 +79,7 @@ get(PyObject *self, PyObject *args)
 static PyObject *
 delete(PyObject *self, PyObject *args)
 {
-    H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+    //H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
 
     char *filepath, *group;
     if(!PyArg_ParseTuple(args, "ss", &filepath, &group))

From 21d18392b8aa23241dbe90545626f12b825472c8 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 20 Jul 2022 19:21:13 -0600
Subject: [PATCH 56/78] h5 driver: raise error if we try to create a file which
 already exists instead of truncating the existing file

---
 contracting/db/hdf5/h5c.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index 20210591..0e6b7537 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -16,7 +16,7 @@ set(PyObject *self, PyObject *args)
 
     hid_t fid = H5Fopen(filepath, H5F_ACC_RDWR, H5P_DEFAULT);
     if(fid < 0)
-        fid = H5Fcreate(filepath, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+        fid = H5Fcreate(filepath, H5F_ACC_EXCL, H5P_DEFAULT, H5P_DEFAULT);
 
     hid_t gid = H5Gopen(fid, group, H5P_DEFAULT);
     if(gid < 0)

From a191500fe2c0bb5bc461b39aba5301dc5e14310d Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 20 Jul 2022 20:14:43 -0600
Subject: [PATCH 57/78] h5 driver: if file already exists but H5Fopen fails for
 some reason (most likely file is already opened by someone else) - then loop
 trying to open it until we manage to do it

---
 contracting/db/hdf5/h5c.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index 0e6b7537..3a0eee51 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -16,7 +16,12 @@ set(PyObject *self, PyObject *args)
 
     hid_t fid = H5Fopen(filepath, H5F_ACC_RDWR, H5P_DEFAULT);
     if(fid < 0)
+    {
         fid = H5Fcreate(filepath, H5F_ACC_EXCL, H5P_DEFAULT, H5P_DEFAULT);
+        if(fid < 0)
+            while((fid = H5Fopen(filepath, H5F_ACC_RDWR, H5P_DEFAULT)) < 0)
+                ;
+    }
 
     hid_t gid = H5Gopen(fid, group, H5P_DEFAULT);
     if(gid < 0)

From 9cb0de1c6e1ccb8d12a0fbfbaf00347cce984ae6 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 20 Jul 2022 22:27:16 -0600
Subject: [PATCH 58/78] h5 driver: add lock

---
 contracting/db/hdf5/h5c.c | 44 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index 3a0eee51..1b4401cd 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -1,9 +1,34 @@
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <hdf5.h>
+#include <libgen.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
 
 #define BUFSIZE 64000
 #define ATT_NAME "value"
+#define LOCK_DIR "/tmp/"
+#define PATHBUF_SIZE 4096
+
+static char lock_path[PATHBUF_SIZE];
+
+void lock_acquire(char *filepath)
+{
+    strcat(lock_path, LOCK_DIR);
+    strcat(lock_path, basename(filepath));
+    while(mkdir(lock_path, S_IRWXU) != 0)
+        ;
+    memset(lock_path, 0, PATHBUF_SIZE);
+}
+
+void lock_release(char *filepath)
+{
+    strcat(lock_path, LOCK_DIR);
+    strcat(lock_path, basename(filepath));
+    rmdir(lock_path);
+    memset(lock_path, 0, PATHBUF_SIZE);
+}
 
 static PyObject *
 set(PyObject *self, PyObject *args)
@@ -14,6 +39,8 @@ set(PyObject *self, PyObject *args)
     if(!PyArg_ParseTuple(args, "ssz", &filepath, &group, &value))
         return NULL;
 
+    lock_acquire(filepath);
+
     hid_t fid = H5Fopen(filepath, H5F_ACC_RDWR, H5P_DEFAULT);
     if(fid < 0)
     {
@@ -43,6 +70,8 @@ set(PyObject *self, PyObject *args)
     H5Gclose(gid);
     H5Fclose(fid);
 
+    lock_release(filepath);
+
     Py_RETURN_NONE;
 }
 
@@ -57,14 +86,20 @@ get(PyObject *self, PyObject *args)
     if(!PyArg_ParseTuple(args, "ss", &filepath, &group))
         return NULL;
 
+    lock_acquire(filepath);
+
     hid_t fid = H5Fopen(filepath, H5F_ACC_RDONLY, H5P_DEFAULT);
     if(fid < 0)
+    {
+        lock_release(filepath);
         Py_RETURN_NONE;
+    }
 
     hid_t aid = H5Aopen_by_name(fid, group, ATT_NAME, H5P_DEFAULT, H5P_DEFAULT);
     if(aid < 0)
     {
         H5Fclose(fid);
+        lock_release(filepath);
         Py_RETURN_NONE;
     }
 
@@ -72,12 +107,17 @@ get(PyObject *self, PyObject *args)
     H5Tset_size(atype, BUFSIZE);
     memset(buf, 0, BUFSIZE);
     if(H5Aread(aid, atype, buf) < 0)
+    {
+        lock_release(filepath);
         Py_RETURN_NONE;
+    }
 
     H5Tclose(atype);
     H5Aclose(aid);
     H5Fclose(fid);
 
+    lock_release(filepath);
+
     return PyUnicode_FromString(buf);
 }
 
@@ -90,6 +130,8 @@ delete(PyObject *self, PyObject *args)
     if(!PyArg_ParseTuple(args, "ss", &filepath, &group))
         return NULL;
 
+    lock_acquire(filepath);
+
     hid_t fid = H5Fopen(filepath, H5F_ACC_RDWR, H5P_DEFAULT);
     hid_t gid = H5Gopen(fid, group, H5P_DEFAULT);
     H5Adelete(gid, ATT_NAME);
@@ -97,6 +139,8 @@ delete(PyObject *self, PyObject *args)
     H5Gclose(gid);
     H5Fclose(fid);
 
+    lock_release(filepath);
+
     Py_RETURN_NONE;
 }
 

From 646de053cfa7090503339e3c9f9a88db1ee8e18e Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 20 Jul 2022 22:41:45 -0600
Subject: [PATCH 59/78] h5 driver: simplify lock

---
 contracting/db/hdf5/h5c.c | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index 1b4401cd..6aec16d5 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -1,33 +1,21 @@
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <hdf5.h>
-#include <libgen.h>
-#include <string.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
 #define BUFSIZE 64000
 #define ATT_NAME "value"
-#define LOCK_DIR "/tmp/"
-#define PATHBUF_SIZE 4096
-
-static char lock_path[PATHBUF_SIZE];
 
 void lock_acquire(char *filepath)
 {
-    strcat(lock_path, LOCK_DIR);
-    strcat(lock_path, basename(filepath));
-    while(mkdir(lock_path, S_IRWXU) != 0)
+    while(mkdir(filepath, S_IRWXU) != 0)
         ;
-    memset(lock_path, 0, PATHBUF_SIZE);
 }
 
 void lock_release(char *filepath)
 {
-    strcat(lock_path, LOCK_DIR);
-    strcat(lock_path, basename(filepath));
-    rmdir(lock_path);
-    memset(lock_path, 0, PATHBUF_SIZE);
+    rmdir(filepath);
 }
 
 static PyObject *

From 8da72b1304e4b8569ed777f5dfb341fa20b54048 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 20 Jul 2022 22:49:41 -0600
Subject: [PATCH 60/78] h5 driver: fix lock

---
 contracting/db/hdf5/h5c.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index 6aec16d5..eb3b0b83 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -3,19 +3,30 @@
 #include <hdf5.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <string.h>
 
 #define BUFSIZE 64000
 #define ATT_NAME "value"
+#define DIRBUF_SIZE 4096
+#define LOCK_SUFFIX "-lock"
+
+static char dirbuf[DIRBUF_SIZE];
 
 void lock_acquire(char *filepath)
 {
+    strcat(dirbuf, filepath);
+    strcat(dirbuf, LOCK_SUFFIX);
     while(mkdir(filepath, S_IRWXU) != 0)
         ;
+    memset(dirbuf, 0, DIRBUF_SIZE);
 }
 
 void lock_release(char *filepath)
 {
+    strcat(dirbuf, filepath);
+    strcat(dirbuf, LOCK_SUFFIX);
     rmdir(filepath);
+    memset(dirbuf, 0, DIRBUF_SIZE);
 }
 
 static PyObject *

From 6ffc9480cfa1c8ade1f35baca532a78eb6a2628f Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 20 Jul 2022 23:00:01 -0600
Subject: [PATCH 61/78] h5 driver: fix lock

---
 contracting/db/hdf5/h5c.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index eb3b0b83..7a976c45 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -16,7 +16,7 @@ void lock_acquire(char *filepath)
 {
     strcat(dirbuf, filepath);
     strcat(dirbuf, LOCK_SUFFIX);
-    while(mkdir(filepath, S_IRWXU) != 0)
+    while(mkdir(dirbuf, S_IRWXU) != 0)
         ;
     memset(dirbuf, 0, DIRBUF_SIZE);
 }
@@ -25,7 +25,7 @@ void lock_release(char *filepath)
 {
     strcat(dirbuf, filepath);
     strcat(dirbuf, LOCK_SUFFIX);
-    rmdir(filepath);
+    rmdir(dirbuf);
     memset(dirbuf, 0, DIRBUF_SIZE);
 }
 

From dc74f1ec029bf477c8492154299594a41575624e Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 20 Jul 2022 23:17:14 -0600
Subject: [PATCH 62/78] h5 driver: disable debug logs

---
 contracting/db/hdf5/h5c.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index 7a976c45..afb920d4 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -32,7 +32,7 @@ void lock_release(char *filepath)
 static PyObject *
 set(PyObject *self, PyObject *args)
 {
-    //H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+    H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
 
     char *filepath, *group, *value;
     if(!PyArg_ParseTuple(args, "ssz", &filepath, &group, &value))
@@ -79,7 +79,7 @@ get(PyObject *self, PyObject *args)
 {
     static char buf[BUFSIZE];
 
-    //H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+    H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
 
     char *filepath, *group;
     if(!PyArg_ParseTuple(args, "ss", &filepath, &group))
@@ -123,7 +123,7 @@ get(PyObject *self, PyObject *args)
 static PyObject *
 delete(PyObject *self, PyObject *args)
 {
-    //H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+    H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
 
     char *filepath, *group;
     if(!PyArg_ParseTuple(args, "ss", &filepath, &group))

From df5a9c545d08d6f2b741b2760630d8bd9ddcf553 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Sat, 23 Jul 2022 16:17:24 -0600
Subject: [PATCH 63/78] h5 driver: fetch keys from contract with C HDF5 api;
 remove h5py completely

---
 .gitignore                    |  4 ++-
 contracting/config.py         |  1 +
 contracting/db/driver.py      | 14 +++++-----
 contracting/db/hdf5/h5.py     | 28 --------------------
 contracting/db/hdf5/h5c.c     | 48 +++++++++++++++++++++++++++++++++--
 setup.py                      |  2 +-
 tests/unit/test_new_driver.py |  1 +
 7 files changed, 59 insertions(+), 39 deletions(-)
 delete mode 100644 contracting/db/hdf5/h5.py

diff --git a/.gitignore b/.gitignore
index 8546f0d1..8d5fa805 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,4 +4,6 @@ contracting.egg-info
 build
 .github
 .idea
-logs
\ No newline at end of file
+logs
+*.so
+.ccls
diff --git a/contracting/config.py b/contracting/config.py
index ceb13090..0013b46b 100644
--- a/contracting/config.py
+++ b/contracting/config.py
@@ -5,6 +5,7 @@
 
 DELIMITER = ':'
 INDEX_SEPARATOR = '.'
+HDF5_GROUP_SEPARATOR = '/'
 
 DECIMAL_PRECISION = 64
 
diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index f6363622..3d193f6e 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -16,8 +16,8 @@
 import lmdb
 import motor.motor_asyncio
 import asyncio
-from contracting.db.hdf5 import h5
 import logging
+from contracting.db.hdf5 import h5c
 
 FILE_EXT = '.d'
 HASH_EXT = '.x'
@@ -217,10 +217,10 @@ def __init__(self, root=FSDRIVER_HOME):
     def __parse_key(self, key):
         try:
             filename, variable = key.split(config.INDEX_SEPARATOR, 1)
-            variable = variable.replace(config.DELIMITER, h5.GROUP_SEPARATOR)
+            variable = variable.replace(config.DELIMITER, config.HDF5_GROUP_SEPARATOR)
         except:
             filename = '__misc'
-            variable = key.replace(config.DELIMITER, h5.GROUP_SEPARATOR)
+            variable = key.replace(config.DELIMITER, config.HDF5_GROUP_SEPARATOR)
 
         return filename, variable
 
@@ -231,7 +231,7 @@ def __get_files(self):
         return sorted(os.listdir(self.root))
 
     def __get_keys_from_file(self, filename):
-        return [filename + config.INDEX_SEPARATOR + g.replace(h5.GROUP_SEPARATOR, config.DELIMITER) for g in h5.get_groups(self.__filename_to_path(filename))]
+        return [filename + config.INDEX_SEPARATOR + g.replace(config.HDF5_GROUP_SEPARATOR, config.DELIMITER) for g in h5c.get_groups(self.__filename_to_path(filename))]
 
     def __getitem__(self, key):
         return self.get(key)
@@ -245,11 +245,11 @@ def __delitem__(self, key):
     def get(self, item: str):
         filename, variable = self.__parse_key(item)
 
-        return h5.get_value(self.__filename_to_path(filename), variable)
+        return decode(h5c.get(self.__filename_to_path(filename), variable))
 
     def set(self, key, value):
         filename, variable = self.__parse_key(key)
-        h5.set_value(self.__filename_to_path(filename), variable, value)
+        h5c.set(self.__filename_to_path(filename), variable, encode(value))
 
     def flush(self):
         try:
@@ -260,7 +260,7 @@ def flush(self):
 
     def delete(self, key):
         filename, variable = self.__parse_key(key)
-        h5.del_value(self.__filename_to_path(filename), variable)
+        h5c.delete(self.__filename_to_path(filename), variable)
 
     def iter(self, prefix='', length=0):
         keys = []
diff --git a/contracting/db/hdf5/h5.py b/contracting/db/hdf5/h5.py
deleted file mode 100644
index 8febbab6..00000000
--- a/contracting/db/hdf5/h5.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import h5py
-from contracting.db.encoder import encode, decode
-import contracting.db.hdf5.h5c as h5c
-
-GROUP_SEPARATOR = '/'
-_groups = []
-
-def _store_group_if_has_value_cb(name, obj):
-    global _groups
-    if 'value' in obj.attrs:
-        _groups.append(name)
-
-def set_value(filepath, group, value):
-    h5c.set(filepath, group, encode(value))
-
-def get_value(filepath, group):
-    return decode(h5c.get(filepath, group))
-
-def del_value(filepath, group):
-    h5c.delete(filepath, group)
-
-def get_groups(filepath):
-    global _groups
-    _groups = []
-    with h5py.File(filepath, 'r') as f:
-        f.visititems(_store_group_if_has_value_cb)
-
-    return _groups
diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index afb920d4..b3f9abfc 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -1,9 +1,9 @@
 #define PY_SSIZE_T_CLEAN
 #include <Python.h>
 #include <hdf5.h>
+#include <string.h>
 #include <sys/stat.h>
 #include <unistd.h>
-#include <string.h>
 
 #define BUFSIZE 64000
 #define ATT_NAME "value"
@@ -107,6 +107,9 @@ get(PyObject *self, PyObject *args)
     memset(buf, 0, BUFSIZE);
     if(H5Aread(aid, atype, buf) < 0)
     {
+        H5Tclose(atype);
+        H5Aclose(aid);
+        H5Fclose(fid);
         lock_release(filepath);
         Py_RETURN_NONE;
     }
@@ -143,10 +146,51 @@ delete(PyObject *self, PyObject *args)
     Py_RETURN_NONE;
 }
 
+static herr_t
+store_group_name(hid_t id, const char *name, const H5O_info1_t *object_info, void *op_data)
+{
+    if(strcmp(name, ".") == 0)
+        return 0;
+    return PyList_Append((PyObject *) op_data, PyUnicode_FromString(name));
+}
+
+static PyObject *
+get_groups(PyObject *self, PyObject *args)
+{
+    H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+
+    char *filepath;
+    if(!PyArg_ParseTuple(args, "s", &filepath))
+        return NULL;
+
+    lock_acquire(filepath);
+    hid_t fid = H5Fopen(filepath, H5F_ACC_RDONLY, H5P_DEFAULT);
+    if(fid < 0)
+    {
+        lock_release(filepath);
+        Py_RETURN_NONE;
+    }
+
+    PyObject *group_names = PyList_New(0);
+    if(H5Ovisit1(fid, H5_INDEX_NAME, H5_ITER_NATIVE, store_group_name, group_names) < 0)
+    {
+        H5Fclose(fid);
+        lock_release(filepath);
+        Py_RETURN_NONE;
+    }
+
+    H5Fclose(fid);
+
+    lock_release(filepath);
+
+    return group_names;
+}
+
 static PyMethodDef methods[] = {
     {"set", set, METH_VARARGS, "Set value"},
     {"get", get, METH_VARARGS, "Get value"},
-    {"delete", delete, METH_VARARGS, "Delete value"}
+    {"delete", delete, METH_VARARGS, "Delete value"},
+    {"get_groups", get_groups, METH_VARARGS, "Get groups"}
 };
 
 static struct PyModuleDef h5cmodule = {
diff --git a/setup.py b/setup.py
index d005dd79..19c6fdf5 100644
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 __version__ = '1.0.5.2'
 
-requirements = ['astor', 'pymongo', 'autopep8', 'stdlib_list', 'h5py==3.1.0', 'lmdb']
+requirements = ['astor', 'pymongo', 'autopep8', 'stdlib_list', 'lmdb']
 
 ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
 
diff --git a/tests/unit/test_new_driver.py b/tests/unit/test_new_driver.py
index b2b750af..37687d7d 100644
--- a/tests/unit/test_new_driver.py
+++ b/tests/unit/test_new_driver.py
@@ -570,6 +570,7 @@ def test_keys_with_prefix(self):
 
         prefix_2_keys = [
             'x37fbab0bd2e60563.c79469e5be41e515',
+            'x37fbab0bd2e60563.c79469e5be41e515:something',
             'x30c6eb2ad176773b.5ce6d590d2472dfe',
             'x3d4fc9480f0a07b2.8aa7646d5066b54d',
             'x387c3d4ab7f0c1c6.ef549198fc14b525',

From d57784183c1b3fd6e9a066cf0b77006797c4be9d Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Sat, 23 Jul 2022 16:25:10 -0600
Subject: [PATCH 64/78] fix for h5 driver: use H5OVisit as per documentation

---
 contracting/db/hdf5/h5c.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index b3f9abfc..65b70b25 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -147,7 +147,7 @@ delete(PyObject *self, PyObject *args)
 }
 
 static herr_t
-store_group_name(hid_t id, const char *name, const H5O_info1_t *object_info, void *op_data)
+store_group_name(hid_t id, const char *name, const H5O_info_t *object_info, void *op_data)
 {
     if(strcmp(name, ".") == 0)
         return 0;
@@ -172,7 +172,7 @@ get_groups(PyObject *self, PyObject *args)
     }
 
     PyObject *group_names = PyList_New(0);
-    if(H5Ovisit1(fid, H5_INDEX_NAME, H5_ITER_NATIVE, store_group_name, group_names) < 0)
+    if(H5Ovisit(fid, H5_INDEX_NAME, H5_ITER_NATIVE, store_group_name, group_names) < 0)
     {
         H5Fclose(fid);
         lock_release(filepath);

From 67f6e7ad249acd479cd62ac0081a511024f26e30 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Mon, 25 Jul 2022 10:16:55 -0600
Subject: [PATCH 65/78] h5 driver: fixed too few arguments to H5Ovisit function
 call by calling it with different parameters depending on lib version; raise
 exception in set() if neither could open nor create the file; minor naming
 changes

---
 contracting/db/hdf5/h5c.c | 53 ++++++++++++++++++++++++++-------------
 1 file changed, 35 insertions(+), 18 deletions(-)

diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index 65b70b25..8c07a8be 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -5,34 +5,37 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
-#define BUFSIZE 64000
+#define ATT_LEN_MAX 64000 // http://davis.lbl.gov/Manuals/HDF5-1.8.7/UG/13_Attributes.html#SpecIssues
 #define ATT_NAME "value"
-#define DIRBUF_SIZE 4096
 #define LOCK_SUFFIX "-lock"
 
-static char dirbuf[DIRBUF_SIZE];
+static char dirname_buf[PATH_MAX + 1];
 
-void lock_acquire(char *filepath)
+static void
+lock_acquire(char *filepath)
 {
-    strcat(dirbuf, filepath);
-    strcat(dirbuf, LOCK_SUFFIX);
-    while(mkdir(dirbuf, S_IRWXU) != 0)
+    strcat(dirname_buf, filepath);
+    strcat(dirname_buf, LOCK_SUFFIX);
+    while(mkdir(dirname_buf, S_IRWXU) != 0)
         ;
-    memset(dirbuf, 0, DIRBUF_SIZE);
+    memset(dirname_buf, 0, sizeof(dirname_buf));
 }
 
-void lock_release(char *filepath)
+static void
+lock_release(char *filepath)
 {
-    strcat(dirbuf, filepath);
-    strcat(dirbuf, LOCK_SUFFIX);
-    rmdir(dirbuf);
-    memset(dirbuf, 0, DIRBUF_SIZE);
+    strcat(dirname_buf, filepath);
+    strcat(dirname_buf, LOCK_SUFFIX);
+    rmdir(dirname_buf);
+    memset(dirname_buf, 0, sizeof(dirname_buf));
 }
 
 static PyObject *
 set(PyObject *self, PyObject *args)
 {
+#ifndef DEBUG
     H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+#endif
 
     char *filepath, *group, *value;
     if(!PyArg_ParseTuple(args, "ssz", &filepath, &group, &value))
@@ -45,8 +48,10 @@ set(PyObject *self, PyObject *args)
     {
         fid = H5Fcreate(filepath, H5F_ACC_EXCL, H5P_DEFAULT, H5P_DEFAULT);
         if(fid < 0)
-            while((fid = H5Fopen(filepath, H5F_ACC_RDWR, H5P_DEFAULT)) < 0)
-                ;
+        {
+            lock_release(filepath);
+            return PyErr_Format(PyExc_OSError, "failed to open/create file \"%s\"", filepath);
+        }
     }
 
     hid_t gid = H5Gopen(fid, group, H5P_DEFAULT);
@@ -77,9 +82,11 @@ set(PyObject *self, PyObject *args)
 static PyObject *
 get(PyObject *self, PyObject *args)
 {
-    static char buf[BUFSIZE];
+    static char buf[ATT_LEN_MAX + 1];
 
+#ifndef DEBUG
     H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+#endif
 
     char *filepath, *group;
     if(!PyArg_ParseTuple(args, "ss", &filepath, &group))
@@ -103,8 +110,8 @@ get(PyObject *self, PyObject *args)
     }
 
     hid_t atype = H5Tcopy(H5T_C_S1);
-    H5Tset_size(atype, BUFSIZE);
-    memset(buf, 0, BUFSIZE);
+    H5Tset_size(atype, sizeof(buf));
+    memset(buf, 0, sizeof(buf));
     if(H5Aread(aid, atype, buf) < 0)
     {
         H5Tclose(atype);
@@ -126,7 +133,9 @@ get(PyObject *self, PyObject *args)
 static PyObject *
 delete(PyObject *self, PyObject *args)
 {
+#ifndef DEBUG
     H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+#endif
 
     char *filepath, *group;
     if(!PyArg_ParseTuple(args, "ss", &filepath, &group))
@@ -157,13 +166,16 @@ store_group_name(hid_t id, const char *name, const H5O_info_t *object_info, void
 static PyObject *
 get_groups(PyObject *self, PyObject *args)
 {
+#ifndef DEBUG
     H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+#endif
 
     char *filepath;
     if(!PyArg_ParseTuple(args, "s", &filepath))
         return NULL;
 
     lock_acquire(filepath);
+
     hid_t fid = H5Fopen(filepath, H5F_ACC_RDONLY, H5P_DEFAULT);
     if(fid < 0)
     {
@@ -172,7 +184,12 @@ get_groups(PyObject *self, PyObject *args)
     }
 
     PyObject *group_names = PyList_New(0);
+    // https://docs.hdfgroup.org/hdf5/develop/group___h5_o.html#ga5ce86255fcc34ceaf84a62551cd24233
+#if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 10 && H5_VERS_RELEASE >= 3) || H5_VERS_MINOR > 10)) || H5_VERS_MAJOR > 1
+    if(H5Ovisit(fid, H5_INDEX_NAME, H5_ITER_NATIVE, store_group_name, group_names, H5O_INFO_BASIC) < 0)
+#else
     if(H5Ovisit(fid, H5_INDEX_NAME, H5_ITER_NATIVE, store_group_name, group_names) < 0)
+#endif
     {
         H5Fclose(fid);
         lock_release(filepath);

From ab99d7ba1ed4339579c345f2a986de43c6036a6c Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Thu, 28 Jul 2022 09:24:24 -0600
Subject: [PATCH 66/78] h5 driver: store group only if has value

---
 contracting/db/driver.py  | 1 +
 contracting/db/hdf5/h5c.c | 7 ++++---
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 3d193f6e..42c1410b 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -269,6 +269,7 @@ def iter(self, prefix='', length=0):
                 keys.extend(self.__get_keys_from_file(filename))
             if length > 0 and len(keys) >= length:
                 break
+        keys.sort()
 
         return keys if length == 0 else keys[:length]
 
diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index 8c07a8be..d3f8cfb4 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -5,6 +5,8 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+// HDF5 Reference Manual: https://support.hdfgroup.org/HDF5/doc/RM/RM_H5Front.html
+
 #define ATT_LEN_MAX 64000 // http://davis.lbl.gov/Manuals/HDF5-1.8.7/UG/13_Attributes.html#SpecIssues
 #define ATT_NAME "value"
 #define LOCK_SUFFIX "-lock"
@@ -158,7 +160,7 @@ delete(PyObject *self, PyObject *args)
 static herr_t
 store_group_name(hid_t id, const char *name, const H5O_info_t *object_info, void *op_data)
 {
-    if(strcmp(name, ".") == 0)
+    if(strcmp(name, ".") == 0 || object_info->num_attrs == 0)
         return 0;
     return PyList_Append((PyObject *) op_data, PyUnicode_FromString(name));
 }
@@ -184,9 +186,8 @@ get_groups(PyObject *self, PyObject *args)
     }
 
     PyObject *group_names = PyList_New(0);
-    // https://docs.hdfgroup.org/hdf5/develop/group___h5_o.html#ga5ce86255fcc34ceaf84a62551cd24233
 #if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 10 && H5_VERS_RELEASE >= 3) || H5_VERS_MINOR > 10)) || H5_VERS_MAJOR > 1
-    if(H5Ovisit(fid, H5_INDEX_NAME, H5_ITER_NATIVE, store_group_name, group_names, H5O_INFO_BASIC) < 0)
+    if(H5Ovisit(fid, H5_INDEX_NAME, H5_ITER_NATIVE, store_group_name, group_names, H5O_INFO_ALL) < 0)
 #else
     if(H5Ovisit(fid, H5_INDEX_NAME, H5_ITER_NATIVE, store_group_name, group_names) < 0)
 #endif

From f6958047133001dd3b8bcbc9c6f8a5f44ab35822 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 3 Aug 2022 10:40:51 -0600
Subject: [PATCH 67/78] create genesis block: add flag in order to not commit
 state changes made by genesis contracts setup for testing

---
 contracting/client.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/contracting/client.py b/contracting/client.py
index b89c606e..46637308 100644
--- a/contracting/client.py
+++ b/contracting/client.py
@@ -186,7 +186,7 @@ def __init__(self, signer='sys',
 
         self.submission_contract = self.get_contract('submission')
 
-    def set_submission_contract(self, filename=None):
+    def set_submission_contract(self, filename=None, commit=True):
         if filename is None:
             filename = self.submission_filename
 
@@ -197,9 +197,9 @@ def set_submission_contract(self, filename=None):
         self.raw_driver.set_contract(name='submission',
                                      code=contract)
 
-        self.raw_driver.commit()
-
-        self.submission_contract = self.get_contract('submission')
+        if commit:
+            self.raw_driver.commit()
+            self.submission_contract = self.get_contract('submission')
 
     def flush(self):
         # flushes db and resubmits genesis contracts

From d01a00f8c19fc677bdf96a17ad6420f3d25ded66 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Wed, 3 Aug 2022 12:04:41 -0600
Subject: [PATCH 68/78] h5 driver: modify version conditional macro statement
 to use h5ovisit3 if >= 1.12

---
 contracting/db/hdf5/h5c.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index d3f8cfb4..9d7fa38b 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -186,7 +186,7 @@ get_groups(PyObject *self, PyObject *args)
     }
 
     PyObject *group_names = PyList_New(0);
-#if (H5_VERS_MAJOR == 1 && ((H5_VERS_MINOR == 10 && H5_VERS_RELEASE >= 3) || H5_VERS_MINOR > 10)) || H5_VERS_MAJOR > 1
+#if (H5_VERS_MAJOR == 1 && H5_VERS_MINOR >= 12) || H5_VERS_MAJOR > 1
     if(H5Ovisit(fid, H5_INDEX_NAME, H5_ITER_NATIVE, store_group_name, group_names, H5O_INFO_ALL) < 0)
 #else
     if(H5Ovisit(fid, H5_INDEX_NAME, H5_ITER_NATIVE, store_group_name, group_names) < 0)

From 8649d8dfe9c5b7d8190f72ca268e40aa051a8832 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Wed, 3 Aug 2022 14:36:26 -0400
Subject: [PATCH 69/78] remove lmdb driver and requirements

---
 contracting/db/driver.py      |  80 -----------
 setup.py                      |   2 +-
 tests/unit/test_new_driver.py | 250 +---------------------------------
 3 files changed, 2 insertions(+), 330 deletions(-)

diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index 3d193f6e..f6281235 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -13,7 +13,6 @@
 import shutil
 import hashlib
 
-import lmdb
 import motor.motor_asyncio
 import asyncio
 import logging
@@ -275,85 +274,6 @@ def iter(self, prefix='', length=0):
     def keys(self):
         return self.iter()
 
-class LMDBDriver:
-    def __init__(self, filename=STORAGE_HOME.joinpath('state')):
-        self.filename = filename
-        self.filename.mkdir(exist_ok=True, parents=True)
-
-        self.db_writer = lmdb.open(path=str(self.filename), map_size=int(1e12), readonly=False)
-        self.db_reader = lmdb.open(path=str(self.filename), map_size=int(1e12), readonly=True, lock=False)
-
-    def get(self, item: str):
-        with self.db_reader.begin() as tx:
-            v = tx.get(item.encode())
-
-        if v is None:
-            return None
-
-        return decode(v)
-
-    def set(self, key, value):
-        if value is None:
-            self.__delitem__(key)
-        else:
-            v = encode(value)
-            with self.db_writer.begin(write=True) as tx:
-                tx.put(key.encode(), v.encode())
-
-    def flush(self):
-        with self.db_writer.begin(write=True) as tx:
-            cursor = tx.cursor()
-            for key, _ in cursor:
-                tx.delete(key)
-
-    def delete(self, key: str):
-        self.__delitem__(key)
-
-    def iter(self, prefix: str, length=0):
-        keys = []
-
-        with self.db_reader.begin() as tx:
-            cursor = tx.cursor()
-
-            if not cursor.set_range(prefix.encode()):
-                return []
-
-            else:
-                for key, _ in cursor:
-                    if not key.startswith(prefix.encode()):
-                        break
-
-                    keys.append(key.decode())
-
-                    if len(keys) >= length > 0:
-                        break
-
-        return keys
-
-    def keys(self):
-        keys = []
-
-        with self.db_reader.begin() as tx:
-            cursor = tx.cursor()
-            for key, _ in cursor:
-                keys.append(key.decode())
-
-        return keys
-
-    def __getitem__(self, item: str):
-        value = self.get(item)
-        if value is None:
-            raise KeyError
-        return value
-
-    def __setitem__(self, key: str, value):
-        self.set(key, value)
-
-    def __delitem__(self, key: str):
-        with self.db_writer.begin(write=True) as tx:
-            tx.delete(key.encode())
-
-
 class WebDriver(InMemDriver):
     def __init__(self, masternode='http://masternode-01.lamden.io'):
         super().__init__()
diff --git a/setup.py b/setup.py
index 19c6fdf5..468d1bbd 100644
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 __version__ = '1.0.5.2'
 
-requirements = ['astor', 'pymongo', 'autopep8', 'stdlib_list', 'lmdb']
+requirements = ['astor', 'pymongo==4.1.1', 'autopep8', 'stdlib_list']
 
 ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
 
diff --git a/tests/unit/test_new_driver.py b/tests/unit/test_new_driver.py
index 37687d7d..4d86b490 100644
--- a/tests/unit/test_new_driver.py
+++ b/tests/unit/test_new_driver.py
@@ -1,5 +1,5 @@
 from unittest import TestCase
-from contracting.db.driver import Driver, InMemDriver, FSDriver, LMDBDriver
+from contracting.db.driver import Driver, InMemDriver, FSDriver
 from contracting.db.encoder import MONGO_MAX_INT
 from contracting.stdlib.bridge.time import Datetime, Timedelta
 from contracting.stdlib.bridge.decimal import ContractingDecimal
@@ -771,251 +771,3 @@ def test_keys_returns_all_keys(self):
         got_keys = self.d.keys()
 
         self.assertListEqual(keys, got_keys)
-
-
-class TestLMDBDriver(TestCase):
-    # Flush this sucker every test
-    def setUp(self):
-        self.d = LMDBDriver()
-        self.d.flush()
-
-    def tearDown(self):
-        self.d.flush()
-
-    def test_get_set(self):
-        for v in TEST_DATA:
-            self.d.set('b', v)
-
-            b = self.d.get('b')
-            self.assertEqual(v, b) if not isinstance(v, dict) else self.assertDictEqual(v, b)
-
-    def test_delete(self):
-        for v in TEST_DATA:
-            self.d.set('b', v)
-
-            b = self.d.get('b')
-            self.assertEqual(v, b) if not isinstance(v, dict) else self.assertDictEqual(v, b)
-
-            self.d.delete('b')
-
-            b = self.d.get('b')
-            self.assertIsNone(b)
-
-    def test_iter(self):
-
-        prefix_1_keys = [
-            'b77aa343e339bed781c7c2be1267cd597',
-            'bc22ede6e6fb4046d78bf2f9d1f8afdb6',
-            'b93dbb37d993846d70b8a92779cbfbfe9',
-            'be1a2783019de6ea7ef169cc55e48a3ae',
-            'b1fe8db32b9185d628f4c346f0455023e',
-            'bef918f83b6a0d1e4f980013342807cf8',
-            'b004cb9235acb5f689d20904692bc026e',
-            'b869ff9519d67354816af90867f4a5425',
-            'bcd8e9100dcb601f65e849c147e3e972e',
-            'b0111919a698f9816862b4ae662a6ed06',
-        ]
-
-        prefix_2_keys = [
-            'x37fbab0bd2e60563c79469e5be41e515',
-            'x30c6eb2ad176773b5ce6d590d2472dfe',
-            'x3d4fc9480f0a07b28aa7646d5066b54d',
-            'x387c3d4ab7f0c1c6ef549198fc14b525',
-            'x5c74dc83e132e435e8512599e1075bc0',
-            'x1472425d0d9bb5ff511e132896d54b13',
-            'x2cedb5c52163c22a0b5f179001959dd2',
-            'x6223f65e553280cd25cadeac6657555c',
-            'xeae18af37c223dde92a71fef55e64afe',
-            'xb8810784ffb360cd3ffc57b1d088e537',
-        ]
-
-        keys = prefix_1_keys + prefix_2_keys
-        random.shuffle(keys)
-
-        for k in keys:
-            self.d.set(k, k)
-
-        p1 = []
-
-        for k in self.d.iter(prefix='b'):
-            p1.append(k)
-
-        prefix_1_keys.sort()
-        p1.sort()
-
-        self.assertListEqual(prefix_1_keys, p1)
-
-        p2 = []
-
-        for k in self.d.iter(prefix='x'):
-            p2.append(k)
-
-        prefix_2_keys.sort()
-        p2.sort()
-
-        self.assertListEqual(prefix_2_keys, p2)
-
-    def test_set_object_returns_properly(self):
-        thing = {
-            'a': 123,
-            'b': False,
-            'x': None
-        }
-
-        self.d.set('thing', thing)
-
-        t = self.d.get('thing')
-        self.assertDictEqual(thing, t)
-
-    def test_set_none_deletes(self):
-        t = 123
-
-        self.d.set('t', t)
-
-        self.assertEqual(self.d.get('t'), 123)
-
-        self.d.set('t', None)
-
-        self.assertEqual(self.d.get('t'), None)
-
-    def test_delete_sets_to_none(self):
-        t = 123
-
-        self.d.set('t', t)
-
-        self.assertEqual(self.d.get('t'), 123)
-
-        self.d.delete('t')
-
-        self.assertEqual(self.d.get('t'), None)
-
-    def test_getitem_works_like_get(self):
-        thing = {
-            'a': 123,
-            'b': False,
-            'x': None
-        }
-
-        self.d.set('thing', thing)
-
-        t = self.d['thing']
-        self.assertDictEqual(thing, t)
-
-    def test_setitem_works_like_set(self):
-        thing = {
-            'a': 123,
-            'b': False,
-            'x': None
-        }
-
-        self.d['thing'] = thing
-
-        t = self.d['thing']
-        self.assertDictEqual(thing, t)
-
-    def test_delitem_works_like_del(self):
-        t = 123
-
-        self.d.set('t', t)
-
-        self.assertEqual(self.d.get('t'), 123)
-
-        del self.d['t']
-
-        self.assertEqual(self.d.get('t'), None)
-
-    def test_iter_with_length_returns_list_of_size_l(self):
-        prefix_1_keys = [
-            'b77aa343e339bed781c7c2be1267cd597',
-            'bc22ede6e6fb4046d78bf2f9d1f8afdb6',
-            'b93dbb37d993846d70b8a92779cbfbfe9',
-            'be1a2783019de6ea7ef169cc55e48a3ae',
-            'b1fe8db32b9185d628f4c346f0455023e',
-            'bef918f83b6a0d1e4f980013342807cf8',
-            'b004cb9235acb5f689d20904692bc026e',
-            'b869ff9519d67354816af90867f4a5425',
-            'bcd8e9100dcb601f65e849c147e3e972e',
-            'b0111919a698f9816862b4ae662a6ed06',
-        ]
-
-        prefix_2_keys = [
-            'x37fbab0bd2e60563c79469e5be41e515',
-            'x30c6eb2ad176773b5ce6d590d2472dfe',
-            'x3d4fc9480f0a07b28aa7646d5066b54d',
-            'x387c3d4ab7f0c1c6ef549198fc14b525',
-            'x5c74dc83e132e435e8512599e1075bc0',
-            'x1472425d0d9bb5ff511e132896d54b13',
-            'x2cedb5c52163c22a0b5f179001959dd2',
-            'x6223f65e553280cd25cadeac6657555c',
-            'xeae18af37c223dde92a71fef55e64afe',
-            'xb8810784ffb360cd3ffc57b1d088e537',
-        ]
-
-        keys = prefix_1_keys + prefix_2_keys
-        random.shuffle(keys)
-
-        for k in keys:
-            self.d.set(k, k)
-
-        p1 = []
-
-        for k in self.d.iter(prefix='b', length=3):
-            p1.append(k)
-
-        prefix_1_keys.sort()
-        p1.sort()
-
-        self.assertListEqual(prefix_1_keys[:3], p1)
-
-        p2 = []
-
-        for k in self.d.iter(prefix='x', length=5):
-            p2.append(k)
-
-        prefix_2_keys.sort()
-        p2.sort()
-
-        self.assertListEqual(prefix_2_keys[:5], p2)
-
-    def test_key_error_if_getitem_doesnt_exist(self):
-        with self.assertRaises(KeyError):
-            print(self.d['thing'])
-
-    def test_keys_returns_all_keys(self):
-        prefix_1_keys = [
-            'b77aa343e339bed781c7c2be1267cd597',
-            'bc22ede6e6fb4046d78bf2f9d1f8afdb6',
-            'b93dbb37d993846d70b8a92779cbfbfe9',
-            'be1a2783019de6ea7ef169cc55e48a3ae',
-            'b1fe8db32b9185d628f4c346f0455023e',
-            'bef918f83b6a0d1e4f980013342807cf8',
-            'b004cb9235acb5f689d20904692bc026e',
-            'b869ff9519d67354816af90867f4a5425',
-            'bcd8e9100dcb601f65e849c147e3e972e',
-            'b0111919a698f9816862b4ae662a6ed06',
-        ]
-
-        prefix_2_keys = [
-            'x37fbab0bd2e60563c79469e5be41e515',
-            'x30c6eb2ad176773b5ce6d590d2472dfe',
-            'x3d4fc9480f0a07b28aa7646d5066b54d',
-            'x387c3d4ab7f0c1c6ef549198fc14b525',
-            'x5c74dc83e132e435e8512599e1075bc0',
-            'x1472425d0d9bb5ff511e132896d54b13',
-            'x2cedb5c52163c22a0b5f179001959dd2',
-            'x6223f65e553280cd25cadeac6657555c',
-            'xeae18af37c223dde92a71fef55e64afe',
-            'xb8810784ffb360cd3ffc57b1d088e537',
-        ]
-
-        keys = prefix_1_keys + prefix_2_keys
-        random.shuffle(keys)
-
-        for k in keys:
-            self.d.set(k, k)
-
-        keys.sort()
-
-        got_keys = self.d.keys()
-
-        self.assertListEqual(keys, got_keys)

From 1cd5807a0526b93c96b6c68dfc7641ece7393195 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Thu, 4 Aug 2022 11:21:44 -0400
Subject: [PATCH 70/78] Add motor package to setup.py

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 468d1bbd..35deead1 100644
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 __version__ = '1.0.5.2'
 
-requirements = ['astor', 'pymongo==4.1.1', 'autopep8', 'stdlib_list']
+requirements = ['astor==0.8.1', 'pymongo==4.1.1', 'autopep8==1.5.7', "stdlib_list==0.8.0", 'motor==2.5.1']
 
 ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
 

From 243662edff01f0e94ab27e40bcebccbf4abd2802 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Thu, 4 Aug 2022 11:25:00 -0400
Subject: [PATCH 71/78] version to 1.1.6

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 35deead1..5eb1b19b 100644
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@
 
 major = 0
 
-__version__ = '1.0.5.2'
+__version__ = '1.1.6'
 
 requirements = ['astor==0.8.1', 'pymongo==4.1.1', 'autopep8==1.5.7', "stdlib_list==0.8.0", 'motor==2.5.1']
 

From 451c73233296d7f60869220b000594f8de8c3763 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Thu, 4 Aug 2022 11:25:27 -0400
Subject: [PATCH 72/78] downgrade pymongo driver version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 5eb1b19b..f5f531f4 100644
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 
 __version__ = '1.1.6'
 
-requirements = ['astor==0.8.1', 'pymongo==4.1.1', 'autopep8==1.5.7', "stdlib_list==0.8.0", 'motor==2.5.1']
+requirements = ['astor==0.8.1', 'pymongo==3.12.3', 'autopep8==1.5.7', "stdlib_list==0.8.0", 'motor==2.5.1']
 
 ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError)
 

From 3a6005de5a9d47883e65e098f0b96a2a30c3a4e8 Mon Sep 17 00:00:00 2001
From: Nikita Tokariev <mykyta.tokariev@gmail.com>
Date: Thu, 4 Aug 2022 16:58:44 -0600
Subject: [PATCH 73/78] h5 driver: store block numbers alongside values so we
 know at which block certain state key was modified

---
 contracting/config.py         |  2 +
 contracting/db/driver.py      | 18 ++++++--
 contracting/db/hdf5/h5c.c     | 87 ++++++++++++++++++++++++-----------
 tests/unit/test_new_driver.py | 17 ++++++-
 4 files changed, 91 insertions(+), 33 deletions(-)

diff --git a/contracting/config.py b/contracting/config.py
index 0013b46b..595dcccf 100644
--- a/contracting/config.py
+++ b/contracting/config.py
@@ -25,3 +25,5 @@
 WRITE_COST_PER_BYTE = 25
 
 STAMPS_PER_TAU = 20
+
+BLOCK_NUM_DEFAULT = -1
diff --git a/contracting/db/driver.py b/contracting/db/driver.py
index e60a9679..c9e100ba 100644
--- a/contracting/db/driver.py
+++ b/contracting/db/driver.py
@@ -34,7 +34,6 @@
 COMPILED_KEY = '__compiled__'
 DEVELOPER_KEY = '__developer__'
 
-
 class Driver:
     def __init__(self, db='lamden', collection='state'):
         self.client = pymongo.MongoClient()
@@ -244,11 +243,22 @@ def __delitem__(self, key):
     def get(self, item: str):
         filename, variable = self.__parse_key(item)
 
-        return decode(h5c.get(self.__filename_to_path(filename), variable))
+        return decode(h5c.get_value(self.__filename_to_path(filename), variable))
 
-    def set(self, key, value):
+    def get_block(self, item: str):
+        filename, variable = self.__parse_key(item)
+        block_num = h5c.get_block(self.__filename_to_path(filename), variable)
+
+        return config.BLOCK_NUM_DEFAULT if block_num is None else int(block_num)
+
+    def set(self, key, value, block_num=None):
         filename, variable = self.__parse_key(key)
-        h5c.set(self.__filename_to_path(filename), variable, encode(value))
+        h5c.set(
+            self.__filename_to_path(filename),
+            variable,
+            encode(value) if value is not None else None,
+            str(block_num) if block_num is not None else None
+        )
 
     def flush(self):
         try:
diff --git a/contracting/db/hdf5/h5c.c b/contracting/db/hdf5/h5c.c
index 9d7fa38b..9c8d1d8c 100644
--- a/contracting/db/hdf5/h5c.c
+++ b/contracting/db/hdf5/h5c.c
@@ -7,8 +7,9 @@
 
 // HDF5 Reference Manual: https://support.hdfgroup.org/HDF5/doc/RM/RM_H5Front.html
 
-#define ATT_LEN_MAX 64000 // http://davis.lbl.gov/Manuals/HDF5-1.8.7/UG/13_Attributes.html#SpecIssues
-#define ATT_NAME "value"
+#define ATTR_LEN_MAX 64000 // http://davis.lbl.gov/Manuals/HDF5-1.8.7/UG/13_Attributes.html#SpecIssues
+#define ATTR_VALUE "value"
+#define ATTR_BLOCK "block"
 #define LOCK_SUFFIX "-lock"
 
 static char dirname_buf[PATH_MAX + 1];
@@ -32,6 +33,21 @@ lock_release(char *filepath)
     memset(dirname_buf, 0, sizeof(dirname_buf));
 }
 
+static void
+write_attr(hid_t gid, char *name, char *value)
+{
+    H5Adelete(gid, name);
+    if(value)
+    {
+        hid_t atype = H5Tcopy(H5T_C_S1);
+        H5Tset_size(atype, strlen(value));
+        hid_t aid = H5Acreate(gid, name, atype, H5Screate(H5S_SCALAR), H5P_DEFAULT, H5P_DEFAULT);
+        H5Awrite(aid, atype, value);
+        H5Aclose(aid);
+        H5Tclose(atype);
+    }
+}
+
 static PyObject *
 set(PyObject *self, PyObject *args)
 {
@@ -39,8 +55,8 @@ set(PyObject *self, PyObject *args)
     H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
 #endif
 
-    char *filepath, *group, *value;
-    if(!PyArg_ParseTuple(args, "ssz", &filepath, &group, &value))
+    char *filepath, *group, *value, *blocknum;
+    if(!PyArg_ParseTuple(args, "sszz", &filepath, &group, &value, &blocknum))
         return NULL;
 
     lock_acquire(filepath);
@@ -65,14 +81,9 @@ set(PyObject *self, PyObject *args)
         H5Pclose(lcpl);
     }
 
-    hid_t atype = H5Tcopy(H5T_C_S1);
-    H5Tset_size(atype, strlen(value));
-    H5Adelete(gid, ATT_NAME);
-    hid_t aid = H5Acreate(gid, ATT_NAME, atype, H5Screate(H5S_SCALAR), H5P_DEFAULT, H5P_DEFAULT);
-    H5Awrite(aid, atype, value);
+    write_attr(gid, ATTR_VALUE, value);
+    write_attr(gid, ATTR_BLOCK, blocknum);
 
-    H5Tclose(atype);
-    H5Aclose(aid);
     H5Gclose(gid);
     H5Fclose(fid);
 
@@ -82,17 +93,9 @@ set(PyObject *self, PyObject *args)
 }
 
 static PyObject *
-get(PyObject *self, PyObject *args)
+get_attr(char *filepath, char *group, char *name)
 {
-    static char buf[ATT_LEN_MAX + 1];
-
-#ifndef DEBUG
-    H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
-#endif
-
-    char *filepath, *group;
-    if(!PyArg_ParseTuple(args, "ss", &filepath, &group))
-        return NULL;
+    static char buf[ATTR_LEN_MAX + 1];
 
     lock_acquire(filepath);
 
@@ -103,7 +106,7 @@ get(PyObject *self, PyObject *args)
         Py_RETURN_NONE;
     }
 
-    hid_t aid = H5Aopen_by_name(fid, group, ATT_NAME, H5P_DEFAULT, H5P_DEFAULT);
+    hid_t aid = H5Aopen_by_name(fid, group, name, H5P_DEFAULT, H5P_DEFAULT);
     if(aid < 0)
     {
         H5Fclose(fid);
@@ -132,6 +135,34 @@ get(PyObject *self, PyObject *args)
     return PyUnicode_FromString(buf);
 }
 
+static PyObject *
+get_value(PyObject *self, PyObject *args)
+{
+#ifndef DEBUG
+    H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+#endif
+
+    char *filepath, *group;
+    if(!PyArg_ParseTuple(args, "ss", &filepath, &group))
+        return NULL;
+
+    return get_attr(filepath, group, ATTR_VALUE);
+}
+
+static PyObject *
+get_block(PyObject *self, PyObject *args)
+{
+#ifndef DEBUG
+    H5Eset_auto2(H5P_DEFAULT, NULL, NULL);
+#endif
+
+    char *filepath, *group;
+    if(!PyArg_ParseTuple(args, "ss", &filepath, &group))
+        return NULL;
+
+    return get_attr(filepath, group, ATTR_BLOCK);
+}
+
 static PyObject *
 delete(PyObject *self, PyObject *args)
 {
@@ -147,7 +178,8 @@ delete(PyObject *self, PyObject *args)
 
     hid_t fid = H5Fopen(filepath, H5F_ACC_RDWR, H5P_DEFAULT);
     hid_t gid = H5Gopen(fid, group, H5P_DEFAULT);
-    H5Adelete(gid, ATT_NAME);
+    H5Adelete(gid, ATTR_VALUE);
+    H5Adelete(gid, ATTR_BLOCK);
 
     H5Gclose(gid);
     H5Fclose(fid);
@@ -205,10 +237,11 @@ get_groups(PyObject *self, PyObject *args)
 }
 
 static PyMethodDef methods[] = {
-    {"set", set, METH_VARARGS, "Set value"},
-    {"get", get, METH_VARARGS, "Get value"},
-    {"delete", delete, METH_VARARGS, "Delete value"},
-    {"get_groups", get_groups, METH_VARARGS, "Get groups"}
+    {"set",         set,        METH_VARARGS, "Set value"},
+    {"get_value",   get_value,  METH_VARARGS, "Get value"},
+    {"get_block",   get_block,  METH_VARARGS, "Get block"},
+    {"delete",      delete,     METH_VARARGS, "Delete value & block"},
+    {"get_groups",  get_groups, METH_VARARGS, "Get groups"}
 };
 
 static struct PyModuleDef h5cmodule = {
diff --git a/tests/unit/test_new_driver.py b/tests/unit/test_new_driver.py
index 4d86b490..819ba3b4 100644
--- a/tests/unit/test_new_driver.py
+++ b/tests/unit/test_new_driver.py
@@ -1,9 +1,10 @@
-from unittest import TestCase
+from contracting import config
 from contracting.db.driver import Driver, InMemDriver, FSDriver
 from contracting.db.encoder import MONGO_MAX_INT
-from contracting.stdlib.bridge.time import Datetime, Timedelta
 from contracting.stdlib.bridge.decimal import ContractingDecimal
+from contracting.stdlib.bridge.time import Datetime, Timedelta
 from decimal import Decimal
+from unittest import TestCase
 import random
 
 SAMPLE_STRING = 'beef'
@@ -530,6 +531,7 @@ class TestFSDriver(TestCase):
     # Flush this sucker every test
     def setUp(self):
         self.d = FSDriver()
+        self.block_num = 33
 
     def tearDown(self):
         self.d.flush()
@@ -540,6 +542,15 @@ def test_get_set(self):
 
             b = self.d.get('b.b')
             self.assertEqual(v, b) if not isinstance(v, dict) else self.assertDictEqual(v, b)
+            self.assertEqual(self.d.get_block('b.b'), config.BLOCK_NUM_DEFAULT)
+
+    def test_get_set_with_block_num(self):
+        for v in TEST_DATA:
+            self.d.set('b.b', v, block_num=self.block_num)
+
+            b = self.d.get('b.b')
+            self.assertEqual(v, b) if not isinstance(v, dict) else self.assertDictEqual(v, b)
+            self.assertEqual(self.d.get_block('b.b'), self.block_num)
 
     def test_delete(self):
         for v in TEST_DATA:
@@ -547,11 +558,13 @@ def test_delete(self):
 
             b = self.d.get('b.b')
             self.assertEqual(v, b) if not isinstance(v, dict) else self.assertDictEqual(v, b)
+            self.assertEqual(self.d.get_block('b.b'), config.BLOCK_NUM_DEFAULT)
 
             self.d.delete('b.b')
 
             b = self.d.get('b.b')
             self.assertIsNone(b)
+            self.assertEqual(self.d.get_block('b.b'), config.BLOCK_NUM_DEFAULT)
 
     def test_keys_with_prefix(self):
 

From 0412b32743befd5c1cde0bf3d2ec3bf84fa9bdb5 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Tue, 9 Aug 2022 10:38:28 -0400
Subject: [PATCH 74/78] allow client.py to accept no submission file and
 instead get the contract from state.

---
 contracting/client.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/contracting/client.py b/contracting/client.py
index 46637308..fa233924 100644
--- a/contracting/client.py
+++ b/contracting/client.py
@@ -175,17 +175,23 @@ def __init__(self, signer='sys',
         self.submission_filename = submission_filename
         self.environment = environment
 
-        # Seed the genesis contracts into the instance
-        with open(self.submission_filename) as f:
-            contract = f.read()
+        # Get submission contract from file
+        if submission_filename is not None:
+            # Seed the genesis contracts into the instance
+            with open(self.submission_filename) as f:
+                contract = f.read()
 
-        self.raw_driver.set_contract(name='submission',
-                                     code=contract)
+            self.raw_driver.set_contract(name='submission',
+                                         code=contract)
 
-        self.raw_driver.commit()
+            self.raw_driver.commit()
 
+        # Get submission contract from state
         self.submission_contract = self.get_contract('submission')
 
+        # Asset submission contract exists
+        assert self.submission_contract, "No submission contract provided or found in state."
+
     def set_submission_contract(self, filename=None, commit=True):
         if filename is None:
             filename = self.submission_filename

From 24a7a140233610e6dfafe670f8dce00d5e404fad Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Tue, 9 Aug 2022 13:47:00 -0400
Subject: [PATCH 75/78] allow client.py to accept no submission file and
 instead get the contract from state.

---
 contracting/client.py     | 32 ++++++++------
 tests/unit/test_client.py | 88 +++++++++++++++++++++++++++++++++++++--
 2 files changed, 104 insertions(+), 16 deletions(-)

diff --git a/contracting/client.py b/contracting/client.py
index fa233924..d896ec66 100644
--- a/contracting/client.py
+++ b/contracting/client.py
@@ -189,29 +189,35 @@ def __init__(self, signer='sys',
         # Get submission contract from state
         self.submission_contract = self.get_contract('submission')
 
-        # Asset submission contract exists
-        assert self.submission_contract, "No submission contract provided or found in state."
-
     def set_submission_contract(self, filename=None, commit=True):
+        state_contract = self.get_contract('submission')
+
         if filename is None:
             filename = self.submission_filename
 
-        with open(filename) as f:
-            contract = f.read()
+        if filename is None and state_contract is None:
+            raise AssertionError("No submission contract provided or found in state.")
 
-        self.raw_driver.delete_contract(name='submission')
-        self.raw_driver.set_contract(name='submission',
-                                     code=contract)
+        if filename is not None:
+            with open(filename) as f:
+                contract = f.read()
+
+            self.raw_driver.delete_contract(name='submission')
+            self.raw_driver.set_contract(name='submission',
+                                         code=contract)
+            if commit:
+                self.raw_driver.commit()
+
+        self.submission_contract = self.get_contract('submission')
 
-        if commit:
-            self.raw_driver.commit()
-            self.submission_contract = self.get_contract('submission')
 
     def flush(self):
         # flushes db and resubmits genesis contracts
         self.raw_driver.flush()
         self.raw_driver.clear_pending_state()
-        self.set_submission_contract()
+
+        if self.submission_filename is not None:
+            self.set_submission_contract()
 
     # Returns abstract contract which has partial methods mapped to each exported function.
     def get_contract(self, name):
@@ -279,6 +285,8 @@ def compile(self, f):
 
     def submit(self, f, name=None, metering=None, owner=None, constructor_args={}, signer=None):
 
+        assert self.submission_contract is not None, "No submission contract set. Try set_submission_contract first."
+
         if isinstance(f, FunctionType):
             f, n = self.closure_to_code_string(f)
             if name is None:
diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py
index 7140660c..781c7b73 100644
--- a/tests/unit/test_client.py
+++ b/tests/unit/test_client.py
@@ -1,16 +1,27 @@
 from unittest import TestCase
 from contracting.client import ContractingClient
-
+from contracting.db.driver import InMemDriver, ContractDriver
+import os
+from pathlib import Path
 
 class TestClient(TestCase):
     def setUp(self):
-        self.client = ContractingClient()
-        self.client.flush()
+        self.client = None
+        self.raw_driver = InMemDriver()
+        self.contract_driver = ContractDriver(driver=self.raw_driver)
+
+        submission_file_path = f'{Path.cwd().parent.parent}/contracting/contracts/submission.s.py'
+        with open(submission_file_path) as f:
+            self.submission_contract_file = f.read()
 
     def tearDown(self):
-        self.client.flush()
+        if self.client:
+            self.client.flush()
 
     def test_set_submission_updates_contract_file(self):
+        self.client = ContractingClient(driver=self.contract_driver)
+        self.client.flush()
+
         submission_1_code = self.client.raw_driver.get('submission.__code__')
 
         self.client.flush()
@@ -19,3 +30,72 @@ def test_set_submission_updates_contract_file(self):
         submission_2_code = self.client.raw_driver.get('submission.__code__')
 
         self.assertNotEqual(submission_1_code, submission_2_code)
+
+    def test_can_create_instance_without_submission_contract(self):
+        self.client = ContractingClient(submission_filename=None, driver=self.contract_driver)
+
+        self.assertIsNotNone(self.client)
+
+
+    def test_gets_submission_contract_from_state_if_no_filename_provided(self):
+        self.contract_driver.set_contract(name='submission', code=self.submission_contract_file)
+        self.contract_driver.commit()
+
+        self.client = ContractingClient(submission_filename=None, driver=self.contract_driver)
+
+        self.assertIsNotNone(self.client.submission_contract)
+
+    def test_set_submission_contract__sets_from_submission_filename_property(self):
+        self.client = ContractingClient(driver=self.contract_driver)
+
+        self.client.raw_driver.flush()
+        self.client.raw_driver.clear_pending_state()
+        self.client.submission_contract = None
+
+        contract = self.client.raw_driver.get_contract('submission')
+        self.assertIsNone(contract)
+        self.assertIsNone(self.client.submission_contract)
+
+        self.client.set_submission_contract()
+
+        contract = self.client.raw_driver.get_contract('submission')
+        self.assertIsNotNone(contract)
+        self.assertIsNotNone(self.client.submission_contract)
+
+    def test_set_submission_contract__sets_from_submission_from_state(self):
+        self.client = ContractingClient(driver=self.contract_driver)
+
+        self.client.raw_driver.flush()
+        self.client.raw_driver.clear_pending_state()
+        self.client.submission_contract = None
+
+        contract = self.client.raw_driver.get_contract('submission')
+        self.assertIsNone(contract)
+        self.assertIsNone(self.client.submission_contract)
+
+        self.contract_driver.set_contract(name='submission', code=self.submission_contract_file)
+        self.contract_driver.commit()
+
+        self.client.set_submission_contract()
+
+        contract = self.client.raw_driver.get_contract('submission')
+        self.assertIsNotNone(contract)
+        self.assertIsNotNone(self.client.submission_contract)
+
+    def test_set_submission_contract__no_contract_provided_or_found_raises_AssertionError(self):
+        self.client = ContractingClient(driver=self.contract_driver)
+
+        self.client.raw_driver.flush()
+        self.client.raw_driver.clear_pending_state()
+        self.client.submission_filename = None
+
+        with self.assertRaises(AssertionError):
+            self.client.set_submission_contract()
+
+    def test_submit__raises_AssertionError_if_no_submission_contract_set(self):
+        self.client = ContractingClient(submission_filename=None, driver=self.contract_driver)
+
+        with self.assertRaises(AssertionError):
+            self.client.submit(f="")
+
+

From 212d6ae5cd9fab49646a7bf156b1770b22ef7abd Mon Sep 17 00:00:00 2001
From: Stuart Farmer <stuart@lamden.io>
Date: Thu, 22 Sep 2022 10:48:44 -0400
Subject: [PATCH 76/78] rt

---
 contracting/compilation/linter.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/contracting/compilation/linter.py b/contracting/compilation/linter.py
index 4be77745..35d33d00 100644
--- a/contracting/compilation/linter.py
+++ b/contracting/compilation/linter.py
@@ -47,6 +47,11 @@ def no_nested_imports(self, node):
     def visit_Name(self, node):
         self.not_system_variable(node.id, node.lineno)
 
+        if node.id == 'rt':
+            self._is_success = False
+            str = "Line {}: ".format(node.lineno) + VIOLATION_TRIGGERS[13]
+            self._violations.append(str)
+
         if node.id in ILLEGAL_BUILTINS and node.id != 'float':
             self._is_success = False
             str = "Line {}: ".format(node.lineno) + VIOLATION_TRIGGERS[13]

From 1db6b61614154ed32af2c6273c91944a71a9ef67 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Wed, 25 Jan 2023 15:32:48 -0500
Subject: [PATCH 77/78] Add AUXILIARY_SALT to random seed from environment.

---
 contracting/stdlib/bridge/random.py             | 11 +++++++----
 tests/integration/test_seneca_client_randoms.py | 15 ++++++++++++++-
 tests/integration/test_stamp_deduction.py       |  4 +---
 3 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/contracting/stdlib/bridge/random.py b/contracting/stdlib/bridge/random.py
index 27f38673..5b8fe428 100644
--- a/contracting/stdlib/bridge/random.py
+++ b/contracting/stdlib/bridge/random.py
@@ -28,12 +28,15 @@ def seed(aux_salt=None):
     block_hash = rt.env.get('block_hash') or '0'
     __input_hash = rt.env.get('__input_hash') or '0'
 
-    # Auxillary salt is used to create completely unique random seeds based on some other properties (optional)
-    auxillary_salt = ''
+    # Auxiliary salt is used to create completely unique random seeds based on some other properties (optional)
+    auxiliary_salt = ''
     if aux_salt is not None and rt.env.get(aux_salt):
-        auxillary_salt = str(rt.env.get(aux_salt))
+        auxiliary_salt = str(rt.env.get(aux_salt))
+    else:
+        if rt.env.get("AUXILIARY_SALT"):
+            auxiliary_salt = str(rt.env.get("AUXILIARY_SALT"))
 
-    s = block_height + block_hash + __input_hash + auxillary_salt
+    s = block_height + block_hash + __input_hash + auxiliary_salt
 
     random.seed(s)
     Seeded.s = True
diff --git a/tests/integration/test_seneca_client_randoms.py b/tests/integration/test_seneca_client_randoms.py
index cc51b672..b9ebc0ad 100644
--- a/tests/integration/test_seneca_client_randoms.py
+++ b/tests/integration/test_seneca_client_randoms.py
@@ -100,10 +100,11 @@ def test_random_num_one_vs_two(self):
 
         self.assertEqual(k2, random.randrange(1000))
 
+    ''' TEST CASE IS IRRELEVANT as getrandbits will never sync with system random.
     def test_random_getrandbits(self):
         b = self.random_contract.random_bits(k=20)
 
-        random.seed('000')
+        random.seed('000'z)
 
         cards = [1, 2, 3, 4, 5, 6, 7, 8]
         random.shuffle(cards)
@@ -111,6 +112,7 @@ def test_random_getrandbits(self):
         random.shuffle(cards)
 
         self.assertEqual(b, random.getrandbits(20))
+    '''
 
     def test_random_range_int(self):
         a = self.random_contract.int_in_range(a=100, b=50000)
@@ -131,3 +133,14 @@ def test_random_choice(self):
         cc = random.choices(c, k=2)
 
         self.assertListEqual(cities, cc)
+
+    def test_auxilary_salt(self):
+        cards_1 = self.random_contract.shuffle_cards(environment={
+            'AUXILIARY_SALT': 'ffd8ded9ced929a41dae83b1f22a6a31b52f79bbf4cdabe6a27d9646dd2bd725fc29c8bc122cb9e37a2904da00e34df499ee7a897505d1de3f0511f9f9c1150c'})
+        cards_2 = self.random_contract.shuffle_cards(environment={
+            'AUXILIARY_SALT': 'ffd8ded9ced929a41dae83b1f22a6a31b52f79bbf4cdabe6a27d9646dd2bd725fc29c8bc122cb9e37a2904da00e34df499ee7a897505d1de3f0511f9f9c1150c'})
+        cards_3 = self.random_contract.shuffle_cards(environment={
+            'AUXILIARY_SALT': 'f79bbded9ced929a41dae83b1f22a6a31b52f79bbf4cdabe6a27d9646dd2bd725fc29c8bc122cb9e37a2904da00e34df499ee7a897505d1de3f0511f9f9c1150c'})
+
+        self.assertEqual(cards_1, cards_2)
+        self.assertNotEqual(cards_1, cards_3)
diff --git a/tests/integration/test_stamp_deduction.py b/tests/integration/test_stamp_deduction.py
index 61188f81..9736bb1d 100644
--- a/tests/integration/test_stamp_deduction.py
+++ b/tests/integration/test_stamp_deduction.py
@@ -65,10 +65,8 @@ def test_too_few_stamps_fails_and_deducts_properly(self):
 
         print(prior_balance)
 
-        small_amount_of_stamps = 1 * STAMPS_PER_TAU
-
         output = self.e.execute('stu', 'currency', 'transfer', kwargs={'amount': 100, 'to': 'colin'},
-                                                stamps=small_amount_of_stamps, auto_commit=True)
+                                                stamps=1, auto_commit=True)
 
         print(output)
 

From cddadb12a706282b21f64803dc769c49858b3370 Mon Sep 17 00:00:00 2001
From: JeffWScott <jeffscott@outlook.com>
Date: Wed, 25 Jan 2023 16:07:44 -0500
Subject: [PATCH 78/78] add new context items

---
 contracting/config.py                      |  1 +
 contracting/execution/executor.py          | 12 +++-
 contracting/execution/runtime.py           | 12 +++-
 contracting/stdlib/bridge/access.py        |  4 +-
 tests/integration/test_rich_ctx_calling.py | 71 ++++++++++++++++++----
 5 files changed, 83 insertions(+), 17 deletions(-)

diff --git a/contracting/config.py b/contracting/config.py
index 595dcccf..6f821ac3 100644
--- a/contracting/config.py
+++ b/contracting/config.py
@@ -9,6 +9,7 @@
 
 DECIMAL_PRECISION = 64
 
+SUBMISSION_CONTRACT_NAME = 'submission'
 PRIVATE_METHOD_PREFIX = '__'
 EXPORT_DECORATOR_STRING = 'export'
 INIT_DECORATOR_STRING = 'construct'
diff --git a/contracting/execution/executor.py b/contracting/execution/executor.py
index 01ac8e4a..2210cf1a 100644
--- a/contracting/execution/executor.py
+++ b/contracting/execution/executor.py
@@ -93,16 +93,22 @@ def execute(self, sender, contract_name, function_name, kwargs,
                 'signer': sender,
                 'caller': sender,
                 'this': contract_name,
-                'owner': driver.get_owner(contract_name)
+                'entry': (contract_name, function_name),
+                'owner': driver.get_owner(contract_name),
+                'submission_name': None
             }
 
+            module = importlib.import_module(contract_name)
+            func = getattr(module, function_name)
+
             if runtime.rt.context.owner is not None and runtime.rt.context.owner != runtime.rt.context.caller:
                 raise Exception(f'Caller {runtime.rt.context.caller} is not the owner {runtime.rt.context.owner}!')
 
             decimal.setcontext(CONTEXT)
 
-            module = importlib.import_module(contract_name)
-            func = getattr(module, function_name)
+            ## add the contract name to the context on a submission call
+            if contract_name == config.SUBMISSION_CONTRACT_NAME:
+                runtime.rt.context._base_state['submission_name'] = kwargs.get('name')
 
             for k, v in kwargs.items():
                 if type(v) == float:
diff --git a/contracting/execution/runtime.py b/contracting/execution/runtime.py
index f19a4f3a..94c2a039 100644
--- a/contracting/execution/runtime.py
+++ b/contracting/execution/runtime.py
@@ -48,12 +48,22 @@ def signer(self):
     def owner(self):
         return self._get_state()['owner']
 
+    @property
+    def entry(self):
+        return self._get_state()['entry']
+
+    @property
+    def submission_name(self):
+        return self._get_state()['submission_name']
+
 
 _context = Context({
         'this': None,
         'caller': None,
         'owner': None,
-        'signer': None
+        'signer': None,
+        'entry': None,
+        'submission_name': None
     })
 
 WRITE_MAX = 1024 * 64
diff --git a/contracting/stdlib/bridge/access.py b/contracting/stdlib/bridge/access.py
index 2d1d24f1..b21aa4ed 100644
--- a/contracting/stdlib/bridge/access.py
+++ b/contracting/stdlib/bridge/access.py
@@ -17,7 +17,9 @@ def __enter__(self, *args, **kwargs):
                 'owner': driver.get_owner(self.contract),
                 'caller': current_state['this'],
                 'signer': current_state['signer'],
-                'this': self.contract
+                'this': self.contract,
+                'entry': current_state['entry'],
+                'submission_name': current_state['submission_name'],
             }
 
             rt.context._add_state(state)
diff --git a/tests/integration/test_rich_ctx_calling.py b/tests/integration/test_rich_ctx_calling.py
index be5838f6..e0211eef 100644
--- a/tests/integration/test_rich_ctx_calling.py
+++ b/tests/integration/test_rich_ctx_calling.py
@@ -1,7 +1,6 @@
 from unittest import TestCase
 from contracting.client import ContractingClient
 
-
 def con_module1():
     @export
     def get_context2():
@@ -10,7 +9,9 @@ def get_context2():
             'owner': ctx.owner,
             'this': ctx.this,
             'signer': ctx.signer,
-            'caller': ctx.caller
+            'caller': ctx.caller,
+            'entry': ctx.entry,
+            'submission_name': ctx.submission_name
         }
 
 
@@ -30,7 +31,9 @@ def call_me_again_again():
             'owner': ctx.owner,
             'this': ctx.this,
             'signer': ctx.signer,
-            'caller': ctx.caller
+            'caller': ctx.caller,
+            'entry': ctx.entry,
+            'submission_name': ctx.submission_name
         })
 
 
@@ -45,7 +48,9 @@ def called_from_a_far():
             'owner': ctx.owner,
             'this': ctx.this,
             'signer': ctx.signer,
-            'caller': ctx.caller
+            'caller': ctx.caller,
+            'entry': ctx.entry,
+            'submission_name': ctx.submission_name
         }]
 
     @export
@@ -53,16 +58,37 @@ def con_called_from_a_far_stacked():
         m = importlib.import_module('con_all_in_one')
         return m.call()
 
+def con_submission_name_test():
+    submission_name = Variable()
+
+    @construct
+    def seed():
+        submission_name.set(ctx.submission_name)
+
+    @export
+    def get_submission_context():
+        return submission_name.get()
+
+    @export
+    def get_entry_context():
+        con_name, func_name = ctx.entry
+
+        return {
+            'entry_contract': con_name,
+            'entry_function': func_name
+        }
+
+
 
 class TestRandomsContract(TestCase):
     def setUp(self):
         self.c = ContractingClient(signer='stu')
         self.c.flush()
 
-        self.c.submit(con_module1)
-
-        self.c.submit(con_all_in_one)
-        self.c.submit(con_dynamic_import)
+        self.c.submit(f=con_module1)
+        self.c.submit(f=con_all_in_one)
+        self.c.submit(f=con_dynamic_import)
+        self.c.submit(f=con_submission_name_test)
 
     def tearDown(self):
         self.c.flush()
@@ -72,10 +98,12 @@ def test_ctx2(self):
         res = module.get_context2()
         expected = {
             'name': 'get_context2',
+            'entry': ('con_module1', 'get_context2'),
             'owner': None,
             'this': 'con_module1',
             'signer': 'stu',
-            'caller': 'stu'
+            'caller': 'stu',
+            'submission_name': None
         }
         self.assertDictEqual(res, expected)
 
@@ -85,10 +113,12 @@ def test_multi_call_doesnt_affect_parameters(self):
 
         expected = {
             'name': 'call_me_again_again',
+            'entry': ('con_all_in_one', 'call_me'),
             'owner': None,
             'this': 'con_all_in_one',
             'signer': 'stu',
-            'caller': 'stu'
+            'caller': 'stu',
+            'submission_name': None
         }
 
         self.assertDictEqual(res, expected)
@@ -99,19 +129,36 @@ def test_dynamic_call(self):
 
         expected1 = {
             'name': 'call_me_again_again',
+            'entry': ('con_dynamic_import', 'called_from_a_far'),
             'owner': None,
             'this': 'con_all_in_one',
             'signer': 'stu',
-            'caller': 'con_dynamic_import'
+            'caller': 'con_dynamic_import',
+            'submission_name': None
         }
 
         expected2 = {
             'name': 'called_from_a_far',
+            'entry': ('con_dynamic_import', 'called_from_a_far'),
             'owner': None,
             'this': 'con_dynamic_import',
             'signer': 'stu',
-            'caller': 'stu'
+            'caller': 'stu',
+            'submission_name': None
         }
 
         self.assertDictEqual(res1, expected1)
         self.assertDictEqual(res2, expected2)
+
+    def test_submission_name_in_construct_function(self):
+        contract = self.c.get_contract('con_submission_name_test')
+        submission_name = contract.get_submission_context()
+
+        self.assertEqual("con_submission_name_test", submission_name)
+
+    def test_entry_context(self):
+        contract = self.c.get_contract('con_submission_name_test')
+        details = contract.get_entry_context()
+
+        self.assertEqual("con_submission_name_test", details.get('entry_contract'))
+        self.assertEqual("get_entry_context", details.get('entry_function'))