Bugfix/assert 1

dbzero/dbzero/dbzero.py

@@ -10,7 +10,7 @@ def load_dynamic(name, path):
 
 def __bootstrap__():
     global __bootstrap__, __loader__, __file__
-    paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/release", "/usr/local/lib/python3/dist-packages/dbzero/"]
+    paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/debug", "/usr/local/lib/python3/dist-packages/dbzero/"]
     __file__ = None
     for path in paths:
         if os.path.isdir(path):

python_tests/test_list.py

@@ -4,6 +4,7 @@
 from .memo_test_types import MemoTestClass, MemoTestSingleton
 from .conftest import DB0_DIR
 from .memo_test_types import MemoTestClass
+import random
 
 
 def make_python_list():
@@ -465,12 +466,14 @@ def test_list_extend_with_none(db0_fixture):
     for i in range(1024):
         assert cut[i] is None
 
+
 def test_db0_list_str_same_as_python_list(db0_fixture):
     db0_list = db0.list([1, "two", 3.0, None])
     py_list = [1, "two", 3.0, None]
     assert str(db0_list) == str(py_list)
     assert repr(db0_list) == repr(py_list)
 
+
 def test_db0_list_str_with_nested_objects(db0_fixture):
     inner_list = db0.list([1, 2, 3])
     db0_list = db0.list([inner_list, "test", None])
@@ -479,6 +482,7 @@ def test_db0_list_str_with_nested_objects(db0_fixture):
     assert str(db0_list) == str(py_list)
     assert repr(db0_list) == repr(py_list)
 
+
 def test_db0_list_str_with_nested_memo_objects(db0_fixture):
     inner_memo = MemoTestClass("inner")
     db0_list = db0.list([inner_memo, "test", None])
@@ -487,15 +491,50 @@ def test_db0_list_str_with_nested_memo_objects(db0_fixture):
     assert str(db0_list) == str(py_list)
     assert repr(db0_list) == repr(py_list)
 
+
 def test_db0_list_islice_iteration(db0_fixture):
     db0_list = db0.list(range(30))
     expected_values = [10, 12, 14, 16, 18]
     for index, value in enumerate(itertools.islice(db0_list, 10, 20, 2)):    
         assert value == expected_values[index]
 
+
 def test_db0_list_compare_with_other_typse(db0_fixture):
     db0_list = db0.list([1, 2, 3])
     python_tuple = (1, 2, 3)
     python_set = {1, 2, 3}
     assert db0_list != python_tuple
-    assert db0_list != python_set
+    assert db0_list != python_set
+
+
+@pytest.mark.stress_test
+@pytest.mark.parametrize("db0_autocommit_fixture", [50], indirect=True)
+def test_append_to_random_lists(db0_autocommit_fixture):
+    print("Creating multiple lists")
+    db0.set_cache_size(8 << 30)
+    lists = db0.dict()
+    for k in range(100000):
+        lists[k] = db0.index()
+
+    RANDOM_BYTES = b'DB0'*22000
+    count = 0
+    db0.commit()
+    print(f"Appending objects to random {len(lists)} lists")
+    for _ in range(200000):
+        item = lists[random.randint(0, len(lists) - 1)]
+        if random.randint(0, 100) < 10:
+            # 20% chance to create a large object
+            data_size = random.randint(8000, 56000)
+        else:
+            # mostly create small objects
+            data_size = random.randint(1, 1500)
+
+        item.add(count, MemoTestClass(value = RANDOM_BYTES[0:data_size]))
+        count += 1
+        if count % 5000 == 0:
+            db0.commit()
+        if count % 10000 == 0:
+            print(f"Appended {count} objects")
+            print(f"Prefix size = {db0.get_storage_stats()['prefix_size']} bytes")
+
+    db0.commit()

src/dbzero/bindings/python/Memo.cpp

@@ -198,7 +198,7 @@ namespace db0::python
     }
 
     template <typename MemoImplT>
-    void MemoObject_del(MemoImplT *memo_obj)
+    void PyAPI_MemoObject_del(MemoImplT *memo_obj)
     {
         PY_API_FUNC
         // destroy associated db0 Object instance
@@ -340,21 +340,7 @@ namespace db0::python
         if (member.get()) {
             return member.steal();
         }
-
-        /* FIXME: log
-        // Use type's tp_getattro to avoid instance dict access issues in Python 3.10
-        // Since we disable Py_TPFLAGS_MANAGED_DICT, PyObject_GenericGetAttr can crash
-        // when it tries to access the instance dictionary. Instead, we use the base type's
-        // getattro or fall back to PyType_Type's implementation.
-        PyTypeObject *type = Py_TYPE(memo_obj);
-        PyTypeObject *base = type->tp_base;
-
-        // Use base class tp_getattro if it's not the memo wrapper itself
-        if (base && base->tp_getattro && base->tp_getattro != (getattrofunc)PyAPI_MemoObject_getattro<MemoImplT>) {
-            return base->tp_getattro(reinterpret_cast<PyObject*>(memo_obj), attr);
-        }
-        */
-
+
         // Fallback to type-level attribute lookup only (no instance dict)
         return PyObject_GenericGetAttr(reinterpret_cast<PyObject*>(memo_obj), attr);
     }
@@ -507,7 +493,7 @@ namespace db0::python
     // Regular memo slots
     static PyType_Slot MemoObject_common_slots[] = {
         {Py_tp_new, (void *)PyAPI_MemoObject_new<MemoObject>},
-        {Py_tp_dealloc, (void *)(MemoObject_del<MemoObject>)},
+        {Py_tp_dealloc, (void *)(PyAPI_MemoObject_del<MemoObject>)},
         {Py_tp_init, (void *)PyAPI_MemoObject_init<MemoObject>},
         {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro<MemoObject>},
         {Py_tp_setattro, (void *)PyAPI_MemoObject_setattro<MemoObject>},
@@ -521,7 +507,7 @@ namespace db0::python
     // Immutable memo slots
     static PyType_Slot MemoImmutableObject_common_slots[] = {
         {Py_tp_new, (void *)PyAPI_MemoObject_new<MemoImmutableObject>},
-        {Py_tp_dealloc, (void *)(MemoObject_del<MemoImmutableObject>)},
+        {Py_tp_dealloc, (void *)(PyAPI_MemoObject_del<MemoImmutableObject>)},
         {Py_tp_init, (void *)PyAPI_MemoObject_init<MemoImmutableObject>},
         {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro<MemoImmutableObject>},
         // set available only on pre-initialized objects
@@ -884,8 +870,6 @@ namespace db0::python
             return member.steal();
         }
 
-        // FIXME: log
-        // return _PyObject_GenericGetAttrWithDict(reinterpret_cast<PyObject*>(memo_obj), attr, NULL, 0);
         return PyObject_GenericGetAttr(reinterpret_cast<PyObject*>(memo_obj), attr);
     }
 
@@ -1000,7 +984,7 @@ namespace db0::python
         }
         Py_RETURN_FALSE;
     }
-        
+
     PyObject *tryGetSchema(PyTypeObject *py_type)
     {
         using SchemaTypeId = db0::object_model::SchemaTypeId;
@@ -1053,8 +1037,8 @@ namespace db0::python
     bool PyAnyMemoType_Check(PyTypeObject *type)
     {
         assert(type);
-        return type->tp_dealloc == reinterpret_cast<destructor>((void(*)(MemoObject*))MemoObject_del<MemoObject>) ||
-               type->tp_dealloc == reinterpret_cast<destructor>((void(*)(MemoImmutableObject*))MemoObject_del<MemoImmutableObject>);
+        return type->tp_dealloc == reinterpret_cast<destructor>((void(*)(MemoObject*))PyAPI_MemoObject_del<MemoObject>) ||
+               type->tp_dealloc == reinterpret_cast<destructor>((void(*)(MemoImmutableObject*))PyAPI_MemoObject_del<MemoImmutableObject>);
     }
 
     template <typename MemoImplT>
@@ -1063,7 +1047,7 @@ namespace db0::python
         assert(obj);
         // needs to stay as 2 lines to proper compile on window
         auto expected = reinterpret_cast<destructor>(
-            static_cast<void(*)(MemoImplT*)>(&MemoObject_del<MemoImplT>)
+            static_cast<void(*)(MemoImplT*)>(&PyAPI_MemoObject_del<MemoImplT>)
         );
         return obj->ob_type->tp_dealloc == expected;        
     }
@@ -1072,9 +1056,9 @@ namespace db0::python
     bool PyMemoType_Check(PyTypeObject *type)
     {
         assert(type);
-        // needs to stay as 2 lines to proper compile on window
+        // needs to stay as 2 lines to proper compile on windows
         auto expected = reinterpret_cast<destructor>(
-            static_cast<void(*)(MemoImplT*)>(&MemoObject_del<MemoImplT>)
+            static_cast<void(*)(MemoImplT*)>(&PyAPI_MemoObject_del<MemoImplT>)
         );
         return type->tp_dealloc == expected;
     }

src/dbzero/bindings/python/collections/PyByteArray.cpp

@@ -295,11 +295,11 @@ namespace db0::python
         lock->getLangCache().add(bytearray_object.get()->ext().getAddress(), bytearray_object.get());
         return bytearray_object.steal();
     }
-
-     ByteArrayObject *PyAPI_makeByteArray(PyObject *self, PyObject *const *args, Py_ssize_t nargs){
+    
+    ByteArrayObject *PyAPI_makeByteArray(PyObject *self, PyObject *const *args, Py_ssize_t nargs) {
         PY_API_FUNC
         return runSafe(tryPyAPI_makeByteArray, self, args, nargs);
-     }
+    }
 
     bool ByteArrayObject_Check(PyObject *object) {
         return Py_TYPE(object) == &ByteArrayObjectType;        

src/dbzero/core/collections/SGB_Tree/SGB_CompressedLookupTree.hpp

@@ -8,6 +8,7 @@
 #include "SGB_LookupTree.hpp"
 #include <dbzero/core/memory/AccessOptions.hpp>
 #include <dbzero/core/compiler_attributes.hpp>
+#include <dbzero/core/metaprog/misc_utils.hpp>
 
 namespace db0
 
@@ -286,6 +287,7 @@ DB0_PACKED_END
             if (this->m_access_type == AccessType::READ_WRITE) {
                 this->onNodeLookup(node);
             }        
+
             /* FIXME: causing segfault in some cases, need to investigate
             if (!node->header().canFit(key)) {
                 return { nullptr, sg_tree_const_iterator() };
@@ -309,6 +311,7 @@ DB0_PACKED_END
             if (this->m_access_type == AccessType::READ_WRITE) {
                 this->onNodeLookup(node);
             }
+
             // within the node look up by compressed key
             // NOTE: if unable to fit key then the item cannot be present in the node
             /* FIXME: causing segfault in some cases, need to investigate
@@ -320,7 +323,7 @@ DB0_PACKED_END
                 }
             }
             */
-
+            
             auto item_ptr = node->lower_equal_bound(node->header().compress(key), this->m_heap_comp);
             if (item_ptr) {
                 // return uncompressed
@@ -384,14 +387,17 @@ DB0_PACKED_END
             if (this->m_access_type == AccessType::READ_WRITE) {
                 this->onNodeLookup(node);
             }
+
+            /* FIXME: log
             if (!node->header().canFit(key)) {
                 return nullptr;
             }
+            */
             // within the node look up by compressed key
             // NOTE: if unable to fit key then the item cannot be present in the node
             return node->lower_equal_bound(node->header().compress(key), this->m_heap_comp);
         }
-
+        
         const TreeHeaderT &treeHeader() const {
             return base_t::getData()->treeHeader();
         }

src/dbzero/core/collections/SGB_Tree/SGB_LookupTree.hpp

@@ -488,5 +488,5 @@ DB0_PACKED_END
         {
         }
     };
-
+    
 }

src/dbzero/core/memory/PrefixImpl.cpp

@@ -313,8 +313,6 @@ namespace db0
         if (timer_ptr) {
             timer = std::make_unique<ProcessTimer>("Prefix::close", timer_ptr);
         }
-        // FIXME: log
-        // m_cache.release();
         m_storage_ptr->close();
     }
 

src/dbzero/core/metaprog/misc_utils.hpp

@@ -1,6 +1,8 @@
 #pragma once
 
 #include <type_traits>
+#include <ostream>
+#include <utility>
 
 namespace db0 
 
@@ -26,3 +28,15 @@ namespace db0
     {};
 
 } 
+
+namespace std 
+
+{
+
+    template<typename T1, typename T2>
+    ostream& operator<<(ostream& os, const std::pair<T1, T2>& p) {
+        return os << "(" << p.first << ", " << p.second << ")";
+    }
+
+}
+

src/dbzero/core/storage/BDevStorage.cpp

@@ -151,12 +151,14 @@ namespace db0
     bool BDevStorage::tryFindMutation(std::uint64_t page_num, StateNumType state_num,
         StateNumType &mutation_id) const
     {
+        std::shared_lock<std::shared_mutex> lock(m_mutex);
         return db0::tryFindMutation(m_sparse_index, m_diff_index, page_num, state_num, mutation_id);
     }
 
     StateNumType BDevStorage::findMutation(std::uint64_t page_num, StateNumType state_num) const
     {
         StateNumType result;
+        std::shared_lock<std::shared_mutex> lock(m_mutex);
         if (!db0::tryFindMutation(m_sparse_index, m_diff_index, page_num, state_num, result)) {
             assert(false && "BDevStorage::findMutation: page not found");
             THROWF(db0::IOException) 
@@ -168,6 +170,7 @@ namespace db0
     void BDevStorage::read(std::uint64_t address, StateNumType state_num, std::size_t size, void *buffer,
         FlagSet<AccessOptions> flags) const
     {
+        std::shared_lock<std::shared_mutex> lock(m_mutex);
         _read(address, state_num, size, buffer, flags);
     }
 
@@ -231,11 +234,13 @@ namespace db0
         assert(state_num > 0 && "BDevStorage::write: state number must be > 0");
         assert((address % m_config.m_page_size == 0) && "BDevStorage::write: address must be page-aligned");
         assert((size % m_config.m_page_size == 0) && "BDevStorage::write: size must be page-aligned");
-
+                
         auto begin_page = address / m_config.m_page_size;
         auto end_page = begin_page + size / m_config.m_page_size;
 
         std::byte *write_buf = reinterpret_cast<std::byte *>(buffer);
+
+        std::unique_lock<std::shared_mutex> lock(m_mutex);
         // write as physical pages and register with the sparse index
         for (auto page_num = begin_page; page_num != end_page; ++page_num, write_buf += m_config.m_page_size) {
             // look up if page has already been added in current transaction
@@ -265,6 +270,7 @@ namespace db0
 
         auto page_num = address / m_config.m_page_size;
 
+        std::unique_lock<std::shared_mutex> lock(m_mutex);
         // Use SparseIndexQuery to determine the current sequence length & check limits
         SparseIndexQuery query(m_sparse_index, m_diff_index, page_num, state_num);
         // if a page has already been written as full-DP in the current transaction then
@@ -300,6 +306,7 @@ namespace db0
 
     bool BDevStorage::flush(ProcessTimer *parent_timer)
     {
+        std::unique_lock<std::shared_mutex> lock(m_mutex);
         std::unique_ptr<ProcessTimer> timer;
         if (parent_timer) {
             timer = std::make_unique<ProcessTimer>("BDevStorage::flush", parent_timer);
@@ -522,6 +529,7 @@ namespace db0
 
     void BDevStorage::getStats(std::function<void(const std::string &, std::uint64_t)> callback) const
     {
+        std::unique_lock<std::shared_mutex> lock(m_mutex);
         callback("dram_io_rand_ops", m_dram_io.getRandOpsCount());
         callback("dram_prefix_size", m_dram_io.getDRAMPrefix().size());
         auto file_rand_ops = m_file.getRandOps();
@@ -541,7 +549,9 @@ namespace db0
         #endif
     }
 
-    std::pair<std::size_t, std::size_t> BDevStorage::getDiff_IOStats() const {
+    std::pair<std::size_t, std::size_t> BDevStorage::getDiff_IOStats() const 
+    {
+        std::unique_lock<std::shared_mutex> lock(m_mutex);
         return m_page_io.getStats();
     }
 
@@ -572,6 +582,7 @@ namespace db0
     void BDevStorage::fetchChangeLogs(StateNumType begin_state, std::optional<StateNumType> end_state,
         std::function<void(StateNumType state_num, const o_change_log &)> f) const
     {
+        std::unique_lock<std::shared_mutex> lock(m_mutex);
         if (m_dp_changelog_io.modified()) {
             THROWF(db0::IOException) << "BDevStorage::fetchChangeLogs: dp-changelog is modified and needs to be flushed first";
         }
@@ -624,7 +635,7 @@ namespace db0
 #endif        
     }
 
-    void BDevStorage::endCommit() 
+    void BDevStorage::endCommit()
     {
 #ifndef NDEBUG        
         m_commit_pending = false;