Issues/610

python_tests/conftest.py

@@ -24,8 +24,9 @@ def __extract_param(request, key, default):
 @pytest.fixture()
 def db0_fixture(request):
     if 'D' in db0.build_flags():
-        db0.enable_storage_validation(__extract_param(request, "storage_validation", False))
-
+        db0.reset_test_params() # reset to defaults
+        db0.enable_storage_validation(__extract_param(request, "storage_validation", False))        
+
     if os.path.exists(DB0_DIR):
         shutil.rmtree(DB0_DIR)    
     os.mkdir(DB0_DIR)
@@ -41,14 +42,15 @@ def db0_fixture(request):
     yield db0
     gc.collect()
     db0.close()
-    if 'D' in db0.build_flags():
-        db0.enable_storage_validation(False)
     if os.path.exists(DB0_DIR):
         shutil.rmtree(DB0_DIR)
 
 
 @pytest.fixture()
-def db0_no_default_fixture():        
+def db0_no_default_fixture():  
+    if 'D' in db0.build_flags():
+        db0.reset_test_params() # reset to defaults
+
     if os.path.exists(DB0_DIR):
         shutil.rmtree(DB0_DIR)
     # create empty directory
@@ -63,9 +65,9 @@ def db0_no_default_fixture():
 
 @pytest.fixture
 def db0_slab_size(request):
-    """
-    DB0 scope with a very short autocommit interval
-    """    
+    if 'D' in db0.build_flags():
+        db0.reset_test_params() # reset to defaults
+
     if os.path.exists(DB0_DIR):
         shutil.rmtree(DB0_DIR)
     # create empty directory
@@ -86,8 +88,11 @@ def db0_slab_size(request):
 @pytest.fixture
 def db0_autocommit_fixture(request):
     """
-    DB0 scope with a very short autocommit interval
+    dbzero scope with a very short autocommit interval
     """    
+    if 'D' in db0.build_flags():
+        db0.reset_test_params() # reset to defaults
+
     if os.path.exists(DB0_DIR):
         shutil.rmtree(DB0_DIR)
     # create empty directory
@@ -103,9 +108,9 @@ def db0_autocommit_fixture(request):
 
 @pytest.fixture()
 def db0_no_autocommit():
-    """
-    DB0 scope with a very short autocommit interval
-    """
+    if 'D' in db0.build_flags():
+        db0.reset_test_params() # reset to defaults
+
     if os.path.exists(DB0_DIR):
         shutil.rmtree(DB0_DIR)
     # create empty directory
@@ -174,8 +179,8 @@ def memo_scoped_enum_tags():
 @pytest.fixture()
 def db0_metaio_fixture():
     """
-    DB0 scope for testing metaio (very small step size)
-    """    
+    dbzero scope for testing metaio (very small step size)
+    """ 
     if os.path.exists(DB0_DIR):
         shutil.rmtree(DB0_DIR)
     # create empty directory
@@ -192,8 +197,11 @@ def db0_metaio_fixture():
 @pytest.fixture()
 def db0_large_lang_cache_no_autocommit():
     """
-    DB0 scope for testing large language cache (no autocommit)
-    """    
+    dbzero scope for testing large language cache (no autocommit)
+    """
+    if 'D' in db0.build_flags():
+        db0.reset_test_params() # reset to defaults
+
     if os.path.exists(DB0_DIR):
         shutil.rmtree(DB0_DIR)
     # create empty directory

python_tests/test_copy_prefix.py

@@ -220,8 +220,8 @@ def validate_copy(copy_id, expected_len = None, expected_min_len = None):
     db0.close()
 
     # in each 'epoch' we modify prefix while making copies
-    # then drop the original prefix and restore if from the last copy
-    epoch_count = 2
+    # then drop the original prefix and restore if from the last copy    
+    epoch_count = 1
     total_len = 0
     for epoch in range(epoch_count):
         print(f"=== Epoch {epoch} ===", flush=True)
@@ -264,10 +264,6 @@ def validate_copy(copy_id, expected_len = None, expected_min_len = None):
         p.join()
         total_len += obj_count * commit_count
 
-        # validate original prefix (no copy yet)
-        print("Validating final prefix ...", flush=True)
-        validate_current_prefix(expected_len = total_len)
-
         # make final stale copy (i.e. without active modifications)
         final_copy = f"./test-copy-final.db0"
         if os.path.exists(final_copy):
@@ -382,4 +378,75 @@ def validate(expected_len):
     # open prefix from recovered and modified copy of a copy
     db0.init(DB0_DIR, prefix=px_name, read_write=False)
     validate(total_len)
+
+
+def test_slow_copy(db0_fixture):
+    """
+    Test simulating fast writer and slow copy/reader process (debug mode only)
+    """
+    if 'D' in db0.build_flags():        
+        px_name = db0.get_current_prefix().name
+        px_path = os.path.join(DB0_DIR, px_name + ".db0")
+
+        def validate_current_prefix(expected_len = None, expected_min_len = None):
+            root = db0.fetch(MemoTestSingleton)
+            assert not expected_min_len or len(root.value) >= expected_min_len
+            assert not expected_len or len(root.value) == expected_len        
+            for item in root.value:
+                assert item.value == "b" * 1024        
+            return len(root.value)
+
+        def validate_copy(copy_id, expected_len = None, expected_min_len = None):
+            file_name = f"./test-copy-{copy_id}.db0"
+            os.remove(px_path)
+            # restore the copy
+            os.rename(file_name, px_path)                        
+            db0.init(DB0_DIR, prefix=px_name, read_write=False)
+            result = validate_current_prefix(expected_len, expected_min_len)
+            db0.close()
+            return result
+
+        db0.close()
+
+        obj_count = 250
+        commit_count = 15
+        # start the writer process for a long run
+        p = multiprocessing.Process(target=writer_process, args=(px_name, obj_count, commit_count, True))
+        p.start()
+
+        db0.init(DB0_DIR)
+        db0.open(px_name, "r")
+        last_len = 0
+        while True:
+            try:
+                if not db0.exists(MemoTestSingleton):
+                    time.sleep(0.1)
+                    continue
+                root = db0.fetch(MemoTestSingleton)
+                if len(root.value) > 1:
+                    last_len = len(root.value)
+                    break
+            except Exception:
+                pass
+            time.sleep(0.1)
+
+        copy_id = 0
+        # copy the prefix multiple times while it is being modified 
+        db0.set_test_params(sleep_interval = 50)
+        while True:
+            if not p.is_alive():
+                break
+            file_name = f"./test-copy-{copy_id}.db0"
+            if os.path.exists(file_name):
+                os.remove(file_name)
+            db0.copy_prefix(file_name, prefix=px_name)
+            copy_id += 1
+            if not p.is_alive():
+                break            
+
+        p.join()
+        db0.close()
+
+        for i in range(copy_id):
+            last_len = validate_copy(i, expected_min_len = last_len)            
 

src/dbzero/bindings/python/PyAPI.cpp

@@ -1473,6 +1473,18 @@ namespace db0::python
         db0::Settings::__storage_validation = enable;
         Py_RETURN_NONE;
     }
+
+    PyObject *PyAPI_setTestParams(PyObject *, PyObject *, PyObject *kwargs)
+    {
+        PY_API_FUNC
+        return runSafe(trySetTestParams, kwargs);        
+    }
+
+    PyObject *PyAPI_resetTestParams(PyObject *, PyObject *) 
+    {
+        PY_API_FUNC
+        return runSafe(tryResetTestParams);        
+    }
 #endif
 
     PyObject *PyAPI_assign(PyObject *, PyObject *args, PyObject *kwargs)

src/dbzero/bindings/python/PyAPI.hpp

@@ -185,9 +185,11 @@ namespace db0::python
     PyObject *getDRAM_IOMap(PyObject *, PyObject *args, PyObject *kwargs);
 
     PyObject *PyAPI_crashFromCommit(PyObject *self, PyObject *const *args, Py_ssize_t nargs);    
-
+    
     PyObject *PyAPI_breakpoint(PyObject *self, PyObject *const *args, Py_ssize_t nargs);
     PyObject *PyAPI_enableStorageValidation(PyObject *, PyObject *args, PyObject *kwargs);
+    PyObject *PyAPI_setTestParams(PyObject *, PyObject *args, PyObject *kwargs);
+    PyObject *PyAPI_resetTestParams(PyObject *, PyObject *);
 #endif
 
     template <typename T> db0::object_model::StorageClass getStorageClass();

src/dbzero/bindings/python/PyInternalAPI.cpp

@@ -24,6 +24,7 @@
 #include <dbzero/core/serialization/Serializable.hpp>
 #include <dbzero/core/memory/SlabAllocator.hpp>
 #include <dbzero/core/storage/BDevStorage.hpp>
+#include <dbzero/workspace/Config.hpp>
 #include <dbzero/bindings/python/collections/PyTuple.hpp>
 #include <dbzero/bindings/python/collections/PyList.hpp>
 #include <dbzero/bindings/python/collections/PyDict.hpp>
@@ -231,11 +232,11 @@ namespace db0::python
     {        
         auto &class_factory = fixture->get<db0::object_model::ClassFactory>();
         // find type associated class with the ClassFactory
-        auto type = class_factory.getExistingType(py_type);
-        if (!type->isSingleton()) {
+        auto type = class_factory.tryGetExistingType(py_type);
+        if (!type || !type->isSingleton()) {
             return false;
         }
-
+        
         return type->isExistingSingleton();
     }
 
@@ -1048,6 +1049,21 @@ namespace db0::python
         }
     }
 
+#ifndef NDEBUG
+    PyObject *trySetTestParams(PyObject *py_dict)
+    {
+        db0::Config config(py_dict);
+        db0::Settings::__sleep_interval = config.get<unsigned long long>("sleep_interval", 0);
+        Py_RETURN_NONE;
+    }
+
+    PyObject *tryResetTestParams()
+    {        
+        db0::Settings::reset();
+        Py_RETURN_NONE;
+    }
+#endif
+
     template PyObject *getMaterializedMemoObject(MemoObject *);
     template PyObject *getMaterializedMemoObject(MemoImmutableObject *);
 

src/dbzero/bindings/python/PyInternalAPI.hpp

@@ -235,6 +235,8 @@ namespace db0::python
     PyObject *tryGetDRAM_IOMap(const Fixture &);
     // opens BDevStorage and reads DRAM_IO map directly, without opening the prefix
     PyObject *tryGetDRAM_IOMapFromFile(const char *file_name);
+    PyObject *trySetTestParams(PyObject *py_dict);
+    PyObject *tryResetTestParams();
 #endif    
 
     PyObject *tryAssign(PyObject *targets, PyObject *key_values);

src/dbzero/bindings/python/dbzero.cpp

@@ -120,6 +120,8 @@ static PyMethodDef dbzero_methods[] =
     {"get_dram_io_map", (PyCFunction)&py::getDRAM_IOMap, METH_VARARGS | METH_KEYWORDS, "Get page_num -> state_num mapping related with a specific DRAM_Prefix"},    
     {"breakpoint", (PyCFunction)&py::PyAPI_breakpoint, METH_FASTCALL, "Testing & debugging function "},
     {"enable_storage_validation", (PyCFunction)&py::PyAPI_enableStorageValidation, METH_VARARGS | METH_KEYWORDS, "Enable full storage validation for testing"},
+    {"set_test_params", (PyCFunction)&py::PyAPI_setTestParams, METH_VARARGS | METH_KEYWORDS, "Test keyword parameters"},
+    {"reset_test_params", (PyCFunction)&py::PyAPI_resetTestParams, METH_NOARGS, "Restore default test parameters"},
 #endif
     {NULL} // Sentinel
 };

src/dbzero/core/memory/config.cpp

@@ -11,10 +11,20 @@ namespace db0
 #ifndef NDEBUG
     bool Settings::__dbg_logs = false;
     bool Settings::__storage_validation = false;
+    unsigned long long Settings::__sleep_interval = 0;
 #endif 
 
     std::function<void()> Settings::m_decode_error = []() {
         THROWF(db0::IOException) << "Data decoding error: corrupt data detected";
     };
+
+    void Settings::reset()
+    {
+#ifndef NDEBUG        
+        __dbg_logs = false;
+        __storage_validation = false;
+        __sleep_interval = 0;
+#endif
+    }
 
 }

src/dbzero/core/memory/config.hpp

@@ -27,9 +27,14 @@ namespace db0
         static bool __dbg_logs;
         // performs storage full read / write validation (with in-memory mirroring)
         static bool __storage_validation;
+        // sleep interval for time-sensitive tests (e.g. copy_prefix) in milliseconds
+        static unsigned long long __sleep_interval;
 #endif
         // Function to throw the data decoding error (i.e. corrupt data detected)
         static std::function<void()> m_decode_error;
+
+        // reset all settings to default values
+        static void reset();
     };
 
 }

src/dbzero/core/serialization/bounded_buf_t.hpp

@@ -32,11 +32,7 @@ namespace db0
 
 		void init(const std::vector<std::byte> &);
 
-		inline const std::byte *get() const
-		{
-			if (begin >= end) {
-				m_throw_func();
-			}
+		inline const std::byte *get() const {
 			return begin;
 		}
 

src/dbzero/core/serialization/packed_int.hpp

@@ -7,12 +7,14 @@
 #include "Types.hpp"
 #include <dbzero/core/exception/Exceptions.hpp>
 #include <dbzero/core/compiler_attributes.hpp>
+#include <dbzero/core/serialization/bounded_buf_t.hpp>
+#include <dbzero/core/memory/config.hpp>
 
 namespace db0
 
 {
-DB0_PACKED_BEGIN
 
+DB0_PACKED_BEGIN
     /**
      * @tparam IntT - underlying encoded unsigned integer type
      * @tparam is_nullable flag indicating if this type can be null-ed
@@ -64,6 +66,21 @@ DB0_PACKED_BEGIN
             }
         }
 
+        // read with bounds validation
+        static IntT read(const std::byte *&at, const std::byte *end)
+        {
+            const_bounded_buf_t safe_buf(Settings::m_decode_error, at, end);
+            if constexpr (is_nullable) {
+                auto result = decodeNullable(safe_buf);
+                at = safe_buf;
+                return result;
+            } else {
+                auto result = decode(safe_buf);
+                at = safe_buf;
+                return result;
+            }
+        }
+
         static void write(std::byte *&at, IntT value)
         {
             auto size_of = measure(value);
@@ -191,7 +208,7 @@ DB0_PACKED_BEGIN
             return value;
         }
 
-        template <class buf_t> static IntT decodeNullable(buf_t &buf) 
+        template <class buf_t> static IntT decodeNullable(buf_t &buf)
         {
             // test for null value
             if (static_cast<std::uint8_t>(*buf) == 0x7f) {
@@ -201,11 +218,11 @@ DB0_PACKED_BEGIN
             return decode(buf);
         }
     };
-
+DB0_PACKED_END
+
     using packed_int32 = o_packed_int<std::uint32_t>;
     using packed_int64 = o_packed_int<std::uint64_t>;
     using nullable_packed_int32 = o_packed_int<std::uint32_t, true>;
     using nullable_packed_int64 = o_packed_int<std::uint64_t, true>;
-
-DB0_PACKED_END
+
 }