Enabling overlay versioning

dbzero/dbzero/dbzero.pyi

@@ -11,48 +11,6 @@ from .interfaces import (
 
 # Core workspace management functions
 
-def init(path: str, config: Optional[Dict[str, Any]] = None, lock_flags: Optional[Dict[str, Any]] = None) -> None:
-    """Initialize the dbzero environment in a specified directory and apply global configurations.
-
-    This function sets up the underlying state management engine.
-    It must be called once per process before interacting with any data.
-    If you need to switch to a different working directory, you should first call 
-    dbzero.close() and then dbzero.init() again with the new path.
-
-    Parameters
-    ----------
-    path : str
-        The path to the data files directory. If the directory doesn't exist, 
-        it will be created.
-    config : dict, optional
-        Dictionary to override default configuration settings:
-
-        * autocommit (bool, default True) to enable automatic commits
-        * autocommit_interval (int, default 250) for commit interval in milliseconds  
-        * cache_size (int, default 2 GiB) for main object cache size in bytes
-        * lang_cache_size (int, optional) for language model data cache size
-    lock_flags : dict, optional
-        Dictionary with configuration of locking behavior when opening the prefix in read-write mode.
-
-        * blocking (bool, default False) to wait when trying to acquire the lock
-        * timeout (int) to set maximum waiting time in seconds when blocking wait is enabled
-        * force_unlock (bool, default False) to force unlocking of existing lock
-
-
-    Examples
-    --------
-    Basic initialization:
-
-    >>> import dbzero
-    >>> dbzero.init("/tmp/my-app-db")
-
-    With custom configuration:
-
-    >>> config = {"autocommit": False, "cache_size": 1 << 30}
-    >>> dbzero.init("/tmp/my-app-db", config=config)
-    """
-    ...
-
 def open(prefix_name: str, open_mode: str = "rw", **kwargs: Any) -> None:
     """Open a data prefix and set it as the current working context.
 
@@ -951,7 +909,7 @@ def find(*query_criteria: Union[Tag, List[Tag], Tuple[Tag], QueryObject, TagSet]
         * List of tags (OR): Objects with at least one of the specified tags
         * Tuple of tags (AND): Objects with all of the specified tags
         * Query: Result of another query
-        * TagSet: Logical set operation.
+        * TagSet: Set logical operation.
     prefix : str, optional
         Optional data prefix to run the query on.
         If omitted, the prefix to run the query is resolved from query criteria.

src/dbzero/core/collections/b_index/bindex_types.hpp

@@ -2,6 +2,7 @@
 
 #include <cstdlib>
 #include <functional>
+#include <dbzero/core/serialization/FixedVersioned.hpp>
 #include <dbzero/core/collections/sgtree/v_sgtree.hpp>
 #include <dbzero/core/collections/sgtree/sgtree_node.hpp>
 #include <dbzero/core/collections/sgtree/intrusive_node.hpp>
@@ -124,7 +125,7 @@ DB0_PACKED_END
         using node_stack = typename bindex_tree_t::join_stack;
 
 DB0_PACKED_BEGIN
-        class DB0_PACKED_ATTR bindex_container : public o_fixed<bindex_container> 
+        class DB0_PACKED_ATTR bindex_container : public o_fixed_versioned<bindex_container> 
         {
         public :
             // common dbzero object header (not copied)

src/dbzero/core/collections/pools/RC_LimitedPool.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "LimitedPool.hpp"
+#include <dbzero/core/serialization/FixedVersioned.hpp>
 #include <dbzero/core/vspace/v_object.hpp>
 #include <dbzero/core/collections/map/v_map.hpp>
 #include <dbzero/core/serialization/compose.hpp>
@@ -11,7 +12,7 @@ namespace db0::pools
 {
 
 DB0_PACKED_BEGIN
-    struct DB0_PACKED_ATTR o_rc_limited_pool: public o_fixed<o_rc_limited_pool>
+    struct DB0_PACKED_ATTR o_rc_limited_pool: public o_fixed_versioned<o_rc_limited_pool>
     {
         Address m_pool_map_address = {};
 

src/dbzero/core/collections/range_tree/IndexBase.hpp

@@ -2,6 +2,7 @@
 
 #include <cstdint>
 #include <array>
+#include <dbzero/core/serialization/FixedVersioned.hpp>
 #include <dbzero/core/vspace/v_object.hpp>
 #include <dbzero/core/serialization/Fixed.hpp>
 #include <dbzero/object_model/object_header.hpp>
@@ -29,15 +30,14 @@ namespace db0
     };
 
 DB0_PACKED_BEGIN
-    struct DB0_PACKED_ATTR o_index: public o_fixed<o_index>
+    struct DB0_PACKED_ATTR o_index: public o_fixed_versioned<o_index>
     {
         // common object header
         o_unique_header m_header;
         IndexType m_type;
         IndexDataType m_data_type = IndexDataType::Auto;
         // address of the actual index instance
         Address m_index_addr = {};
-        std::array<std::uint64_t, 2> m_reserved = {0, 0};
 
         o_index(IndexType, IndexDataType);
         // header not copied

src/dbzero/core/collections/range_tree/RangeTree.hpp

@@ -5,6 +5,7 @@
 #include <optional>
 #include "RangeTreeBlock.hpp"
 #include "RT_NullBlock.hpp"
+#include <dbzero/core/serialization/FixedVersioned.hpp>
 #include <dbzero/core/collections/b_index/v_bindex.hpp>
 #include <dbzero/core/serialization/Types.hpp>
 #include <dbzero/core/compiler_attributes.hpp>
@@ -51,7 +52,7 @@ DB0_PACKED_BEGIN
 DB0_PACKED_END
 
 DB0_PACKED_BEGIN
-    struct DB0_PACKED_ATTR o_range_tree: public o_fixed<o_range_tree>
+    struct DB0_PACKED_ATTR o_range_tree: public o_fixed_versioned<o_range_tree>
     {
         std::uint32_t m_max_block_size;
         // address of the underlying v_bindex

src/dbzero/core/collections/vector/VLimitedMatrix.hpp

@@ -2,6 +2,7 @@
 
 #include "v_bvector.hpp"
 #include "v_sorted_vector.hpp"
+#include <dbzero/core/serialization/FixedVersioned.hpp>
 #include <dbzero/core/memory/Address.hpp>
 #include <dbzero/core/vspace/v_object.hpp>
 #include <dbzero/core/memory/Memspace.hpp>
@@ -14,7 +15,7 @@ namespace db0
 
     template <typename PtrT>
 DB0_PACKED_BEGIN
-    struct DB0_PACKED_ATTR o_limited_matrix: public o_fixed<o_limited_matrix<PtrT> >
+    struct DB0_PACKED_ATTR o_limited_matrix: public o_fixed_versioned<o_limited_matrix<PtrT> >
     {
         // Points to v_bvector representing the entire Dimension 1
         PtrT m_dim1_ptr = {};

src/dbzero/core/collections/vector/v_bvector.hpp

@@ -6,6 +6,7 @@
 #include <cstring>
 #include <optional>
 #include "v_bdata_block.hpp"
+#include <dbzero/core/serialization/FixedVersioned.hpp>
 #include <dbzero/core/serialization/Types.hpp>
 #include <dbzero/core/threading/ProgressiveMutex.hpp>
 #include <dbzero/core/utils/uuid.hpp>
@@ -18,7 +19,7 @@ namespace db0
 
 DB0_PACKED_BEGIN
     template <typename PtrT>
-    struct DB0_PACKED_ATTR o_bvector: public o_fixed<o_bvector<PtrT> >
+    struct DB0_PACKED_ATTR o_bvector: public o_fixed_versioned<o_bvector<PtrT> >
     {
         // common dbzero object header
         db0::o_unique_header m_header;

src/dbzero/core/memory/MetaAllocator.hpp

@@ -8,6 +8,7 @@
 #include <unordered_map>
 #include <unordered_set>
 #include <functional>
+#include <dbzero/core/serialization/FixedVersioned.hpp>
 #include <dbzero/core/serialization/Fixed.hpp>
 #include <dbzero/core/memory/Address.hpp>
 #include <dbzero/core/compiler_attributes.hpp>
@@ -20,7 +21,7 @@ namespace db0
     class SlabManager;
 
 DB0_PACKED_BEGIN
-    struct DB0_PACKED_ATTR o_realm: public o_fixed<o_realm>
+    struct DB0_PACKED_ATTR o_realm: public o_fixed_versioned<o_realm>
     {
         Address m_slab_defs_ptr;
         Address m_capacity_items_ptr;
@@ -31,7 +32,7 @@ DB0_PACKED_BEGIN
 DB0_PACKED_END
 
 DB0_PACKED_BEGIN
-    struct DB0_PACKED_ATTR o_meta_header: public o_fixed<o_meta_header>
+    struct DB0_PACKED_ATTR o_meta_header: public o_fixed_versioned<o_meta_header>
     {
         // NOTE: when needed, this values can be changed to 4 (or 8?) or 1 (no realms)
         static constexpr std::size_t NUM_REALMS = 2;

src/dbzero/core/serialization/Base.hpp

@@ -33,6 +33,9 @@ DB0_PACKED_BEGIN
         std::uint16_t objVer() const {
             return 0;
         }
+        template <class buf_t> static std::uint16_t objVer(buf_t at) {
+            return 0;
+        }
     };
 
     template <std::uint16_t VER>
@@ -52,6 +55,9 @@ DB0_PACKED_BEGIN
         std::uint16_t objVer() const {
             return storedVersion;
         }
+        template <class buf_t> static std::uint16_t objVer(buf_t at) {
+            return reinterpret_cast<const version_base*>((const std::byte*)(at))->objVer();
+        }
     };
 
     template <class T, std::uint16_t VER=0, bool STORE_VER=true>
@@ -135,9 +141,7 @@ DB0_PACKED_BEGIN
         }
 
         template <class buf_t> static Foundation::SafeSize<buf_t> sizeOfMembers(buf_t at) {
-            return Foundation::sizeOfMembers<T>(
-                    baseSize(), at, reinterpret_cast<const ver_type*>((const std::byte*)(at))->objVer()
-            );
+            return Foundation::sizeOfMembers<T>(baseSize(), at, getObjVer(at));
         }
 
         Foundation::SafeSize<const std::byte*> sizeOfMembers() const
@@ -258,11 +262,18 @@ DB0_PACKED_BEGIN
             return ver_type::implVer();
         }
 
+        template <class buf_t> static std::uint16_t getObjVer(buf_t at) {
+            return ver_type::objVer(at);
+        }
+
         std::uint16_t getObjVer() const {
             return ver_type::objVer();
         }
 
         static inline T &__ref(void *buf) {
+#ifndef NDEBUG
+            reinterpret_cast<T*>(buf)->assertImplVersion();
+#endif
             return *reinterpret_cast<T*>(buf);
         }
 

src/dbzero/core/serialization/Ext.hpp

@@ -65,7 +65,7 @@ DB0_PACKED_BEGIN
     template <
         typename T,
         typename super_t,
-        std::uint16_t VER=0, bool STORE_VER=false
+        std::uint16_t VER=0, bool STORE_VER=true
     >
     class DB0_PACKED_ATTR o_ext : public super_t{
         using this_type = o_ext<T, super_t, VER, STORE_VER>;
@@ -236,7 +236,7 @@ DB0_PACKED_BEGIN
 
         // measures space requirement of the base overlaid type
         // plus size of all fixed size members of derived type
-        template <typename... Args> static Meter measureBase(Args&& ...args) 
+        template <typename... Args> static std::size_t measureBase(Args&& ...args) 
         {
             std::size_t result = super_t::measure(std::forward<Args>(args)...);
             // adjust for fixed size members in derived class

src/dbzero/core/serialization/FixedVersioned.hpp

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <dbzero/core/serialization/Base.hpp>
+#include <dbzero/core/serialization/Fixed.hpp>
+
+namespace db0
+{
+
+DB0_PACKED_BEGIN
+
+template<typename T, std::uint16_t VER=0>
+class DB0_PACKED_ATTR o_fixed_versioned : private version_base<VER, true>, public o_fixed<T> {
+    typedef version_base<VER, true> ver_type;
+
+public:
+    using o_fixed<T>::o_fixed;
+
+    static constexpr bool getIsVerStored() {
+        return ver_type::isVerStored();
+    }
+
+    static constexpr std::uint16_t getImplVer() {
+        return ver_type::implVer();
+    }
+
+    template <class buf_t> static std::uint16_t getObjVer(buf_t at) {
+        return ver_type::objVer(at);
+    }
+
+    std::uint16_t getObjVer() const {
+        return ver_type::objVer();
+    }
+};
+
+DB0_PACKED_END
+
+}

src/dbzero/core/serialization/micro_array.hpp

@@ -97,7 +97,7 @@ DB0_PACKED_BEGIN
         typename std::enable_if<B, std::pair<std::size_t, unsigned int> >::type getSizeAndOffset() const 
         {
             std::pair<std::size_t, unsigned int> result;
-            const std::byte *buf = (const std::byte*)this;
+            const std::byte *buf = this->beginOfDynamicArea();
             result.first = packed_int32::read(buf);
             result.second = packed_int32::read(buf);
             return result;
@@ -119,19 +119,19 @@ DB0_PACKED_BEGIN
         static std::size_t measure(const T *begin = nullptr, const T *end = nullptr, unsigned int offset = 0) {
             return measure(end - begin, offset);
         }
-
-        std::size_t sizeOf() const;
 
         template <typename BufT> static std::size_t safeSizeOf(BufT buf)
         {
-            auto start = buf;
-            auto size = packed_int32::__const_ref(buf).value();
-            buf += packed_int32::safeSizeOf(buf);
+            std::size_t size;
             if constexpr (has_offset) {
-                buf += packed_int32::safeSizeOf(buf);
+                size = super_t::sizeOfMembers(buf)
+                    (packed_int32::type())
+                    (packed_int32::type());
+            } else {
+                size = super_t::sizeOfMembers(buf)
+                    (packed_int32::type());
             }
-            buf += size * sizeof(T);
-            return buf - start;
+            return size + super_t::__const_ref(buf).size() * sizeof(T);
         }
 
         inline T *begin()
@@ -219,17 +219,6 @@ DB0_PACKED_BEGIN
         return measure(size, offset);
     }
 
-    template <typename T, bool has_offset> 
-    std::size_t o_micro_array<T, has_offset>::sizeOf() const
-    { 
-        auto result = packed_size().sizeOf();
-        if constexpr (has_offset) {
-            result += packed_offset().sizeOf();
-        }
-        result += packed_size().value() * sizeof(T);
-        return result;
-    }
-
     template <typename T, bool has_offset> 
     std::size_t o_micro_array<T, has_offset>::size() const {
         return packed_size().value();

src/dbzero/core/storage/BDevStorage.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <dbzero/core/serialization/FixedVersioned.hpp>
 #include "SparseIndex.hpp"
 #include "SparsePair.hpp"
 #include "CFile.hpp"
@@ -22,7 +23,7 @@ namespace db0
 {
 
 DB0_PACKED_BEGIN
-    struct DB0_PACKED_ATTR o_prefix_config: public o_fixed<o_prefix_config>
+    struct DB0_PACKED_ATTR o_prefix_config: public o_fixed_versioned<o_prefix_config>
     {
         // magic number for the .db0 file        
         static constexpr std::uint64_t DB0_MAGIC = 0x0DB0DB0DB0DB0DB0;

src/dbzero/core/storage/MetaIOStream.hpp

@@ -22,10 +22,10 @@ DB0_PACKED_BEGIN
     };
 
     // The single log item, possibly associated with multiple managed streams
-    class DB0_PACKED_ATTR o_meta_log: public o_base<o_meta_log, 0, false>
+    class DB0_PACKED_ATTR o_meta_log: public o_base<o_meta_log, 0, true>
     {
     protected:
-        friend class o_base<o_meta_log, 0, false>;
+        friend class o_base<o_meta_log, 0, true>;
 
         o_meta_log(StateNumType state_num, const std::vector<o_meta_item> &);
 
@@ -39,10 +39,8 @@ DB0_PACKED_BEGIN
 
         template <typename T> static std::size_t safeSizeOf(T buf)
         {
-            auto _buf = buf;
-            buf += o_simple<StateNumType>::__const_ref(buf).sizeOf();
-            buf += o_list<o_meta_item>::safeSizeOf(buf);
-            return buf - _buf;
+            return sizeOfMembers(buf)
+                (o_list<o_meta_item>::type());
         }
     };