Feature/cache vars

src/dbzero/bindings/python/Memo.cpp

@@ -610,8 +610,12 @@ namespace db0::python
         (*tp_result)->tp_dict = copyDict(base_class->tp_dict);
         // disable weak-refs (important for Python 3.11.x)
         (*tp_result)->tp_weaklistoffset = 0;
-        // explicitly disable instance dict to prevent segfault in Python 3.10
+#if PY_VERSION_HEX < 0x030B0000  // Python < 3.11
+        (*tp_result)->tp_dictoffset = MemoImplT::getDictOffset();        
+#else
+        // will use managed dict for Python 3.11+
         (*tp_result)->tp_dictoffset = 0;
+#endif
 
         // replace default __str__ and __repr__ implementations
         if (base_class->tp_str == PyType_Type.tp_str) {

src/dbzero/bindings/python/Memo.hpp

@@ -12,26 +12,16 @@
 #include <dbzero/core/memory/AccessOptions.hpp>
 #include "Migration.hpp"
 #include "MemoTypeDecoration.hpp"
+#include "MemoObject.hpp"
 #include <dbzero/object_model/object/Object.hpp>
 #include <dbzero/object_model/object/ObjectImmutableImpl.hpp>
 #include <dbzero/object_model/object/ObjectAnyImpl.hpp>
 
-namespace db0::object_model
-
-{
-
-    class Object;
-
-}
-
 namespace db0::python
 
 {
 
     using AccessType = db0::AccessType;
-    using MemoObject = PyWrapper<db0::object_model::Object>;
-    using MemoImmutableObject = PyWrapper<db0::object_model::ObjectImmutableImpl>;
-    using MemoAnyObject = PyWrapper<db0::object_model::ObjectAnyImpl>;
 
     PyObject *PyAPI_wrapPyClass(PyObject *self, PyObject *, PyObject *kwargs);
     // create a memo object stub

src/dbzero/bindings/python/MemoObject.hpp

@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Copyright (c) 2025 DBZero Software sp. z o.o.
+
+#pragma once
+
+#include <cstddef>
+#include <Python.h>
+#include "PyWrapper.hpp"
+
+namespace db0::object_model
+
+{
+
+    class Object;
+    class ObjectImmutableImpl;
+    class ObjectAnyImpl;
+
+}
+
+namespace db0::python
+
+{
+
+#if PY_VERSION_HEX < 0x030B0000  // Python < 3.11
+    // NOTE: since managed dicts were introduced in Python 3.11, we need to use PyWrapperWithDict
+    using MemoObject = PyWrapperWithDict<db0::object_model::Object>;
+    using MemoImmutableObject = PyWrapperWithDict<db0::object_model::ObjectImmutableImpl>;
+    using MemoAnyObject = PyWrapperWithDict<db0::object_model::ObjectAnyImpl>;
+#else
+    using MemoObject = PyWrapper<db0::object_model::Object>;
+    using MemoImmutableObject = PyWrapper<db0::object_model::ObjectImmutableImpl>;
+    using MemoAnyObject = PyWrapper<db0::object_model::ObjectAnyImpl>;
+#endif
+
+}

src/dbzero/bindings/python/PyInternalAPI.cpp

@@ -417,7 +417,7 @@ namespace db0::python
 
     // MEMO_OBJECT specialization
     template <> void dropInstance<TypeId::MEMO_OBJECT>(PyObject *py_wrapper) {
-        MemoObject_drop(reinterpret_cast<MemoObject*>(py_wrapper));                
+        MemoObject_drop(reinterpret_cast<MemoObject*>(py_wrapper));
     }
 
     // DB0_INDEX specialization

src/dbzero/bindings/python/PyToolkit.cpp

@@ -29,7 +29,6 @@
 #include <dbzero/bindings/python/types/PyEnum.hpp>
 #include <dbzero/bindings/python/types/PyTag.hpp>
 #include <dbzero/bindings/python/PySafeAPI.hpp>
-#include <dbzero/bindings/python/PyCommonBase.hpp>
 
 namespace db0::python
 
@@ -801,11 +800,7 @@ namespace db0::python
     bool PyToolkit::isValid() {
         return Py_IsInitialized();
     }
-
-    bool PyToolkit::hasRefs(ObjectPtr obj_ptr) {
-        return reinterpret_cast<PyCommonBase*>(obj_ptr)->ext().hasRefs();
-    }
-
+
     bool PyToolkit::hasTagRefs(ObjectPtr obj_ptr)
     {
         assert(PyAnyMemo_Check(obj_ptr));

src/dbzero/bindings/python/PyToolkit.hpp

@@ -11,6 +11,7 @@
 #include "PyWorkspace.hpp"
 #include "PyTypes.hpp"
 #include "PyLocks.hpp"
+#include "MemoObject.hpp"
 #include <dbzero/core/collections/pools/StringPools.hpp>
 #include <dbzero/core/memory/swine_ptr.hpp>
 #include <dbzero/core/threading/SafeRMutex.hpp>
@@ -72,7 +73,7 @@ namespace db0::python
         template <typename T> inline static PyWrapper<T> *getWrapperTypeOf(ObjectPtr ptr) {
             return static_cast<PyWrapper<T> *>(ptr);
         }
-
+        
         /**
          * Construct shared type from raw pointer (shared ownership)         
         */
@@ -218,9 +219,6 @@ namespace db0::python
         // indicate failed operation with a specific value/code
         static void setError(ObjectPtr err_obj, std::uint64_t err_value);
 
-        // Check if the object has reference from other dbzero objects or tags
-        // NOTE!!! this only works for CommonBase/PyWrapper objects (e.g. all LangCache objects)
-        static bool hasRefs(ObjectPtr);
         // Check if the object has references from other language objects (other than LangCache)
         static bool hasLangRefs(ObjectPtr);
         // Check if there exist any references except specific number of external references
@@ -266,5 +264,5 @@ namespace db0::python
         static PyWorkspace m_py_workspace;
         static SafeRMutex m_api_mutex;
     };
-
+        
 }

src/dbzero/bindings/python/PyTypeManager.hpp

@@ -13,6 +13,7 @@
 #include "PyTypes.hpp"
 #include <dbzero/bindings/python/types/PyEnumType.hpp>
 #include <dbzero/bindings/python/MemoTypeDecoration.hpp>
+#include "MemoObject.hpp"
 
 namespace db0
 
@@ -53,8 +54,6 @@ namespace db0::python
 {
 
     class MemoTypeDecoration;
-    using MemoObject = PyWrapper<db0::object_model::Object>;
-    using MemoImmutableObject = PyWrapper<db0::object_model::ObjectImmutableImpl>;
 
     /**
      * The class dedicated to recognition of Python types
@@ -69,6 +68,7 @@ namespace db0::python
         using TypeObjectSharedPtr = typename PyTypes::TypeObjectSharedPtr;
         using MemoObject = db0::python::MemoObject;
         using MemoImmutableObject = db0::python::MemoImmutableObject;
+        using MemoAnyObject = db0::python::MemoAnyObject;
         using Object = db0::object_model::Object;
         using ObjectImmutableImpl = db0::object_model::ObjectImmutableImpl;
         using ObjectAnyImpl = db0::object_model::ObjectAnyImpl;

src/dbzero/bindings/python/PyWorkspace.hpp

@@ -12,6 +12,7 @@
 #include "PyWrapper.hpp"
 #include <dbzero/core/memory/AccessOptions.hpp>
 #include <dbzero/core/memory/swine_ptr.hpp>
+#include "MemoObject.hpp"
 
 namespace db0 {
 
@@ -35,8 +36,6 @@ namespace db0::python
 
 {
 
-    using MemoObject = PyWrapper<db0::object_model::Object>;
-
     /**
      * The class to track python module / fixture associations
     */

src/dbzero/bindings/python/PyWrapper.hpp

@@ -12,27 +12,26 @@
 namespace db0::python 
 
 {
-
-    struct __PyTypeMould {
-        PyObject_HEAD
-    };
-
+
     /**
      * Adds a mixed-in (but dynamically initialized)
      * member of type T into the PyObject struct.
      **/
-    template <typename T, bool is_object_base=true> struct PyWrapper: public PyObject
+    template <typename T, bool is_object_base=true, typename BaseT = PyObject>
+    struct PyWrapper: public BaseT
     {
+        // placeholder for the actual instance (since we're unable to calculate sizeof at compile time)
+        std::array<char, 1u> m_ext_storage;
         using ExtT = T;
 
         inline const T &ext() const {
-            return *reinterpret_cast<const T*>((char*)this + sizeof(__PyTypeMould));
+            return *reinterpret_cast<const T*>(&m_ext_storage);
         }
 
         inline T &modifyExt()
         {
             // calculate instance offset
-            auto &result = *reinterpret_cast<T*>((char*)this + sizeof(__PyTypeMould));
+            auto &result = *reinterpret_cast<T*>(&m_ext_storage);
             // only for ObjectBase derived classes
             if constexpr (is_object_base) {
                 // the implementation registers the underlying object for detach (on rollback)
@@ -43,7 +42,8 @@ namespace db0::python
         }
 
         static constexpr std::size_t sizeOf() {
-            return sizeof(__PyTypeMould) + sizeof(T);
+            // adjust size to include actual T size 
+            return sizeof(PyWrapper<T, is_object_base, BaseT>) + sizeof(T) - sizeof(m_ext_storage);
         }
 
         void destroy() {
@@ -68,6 +68,24 @@ namespace db0::python
         }
     };
 
+    struct PyObjectWithDict: public PyObject
+    {        
+        PyObject *m_py_dict = nullptr;
+    };
+
+    // This is a wrapper with additional __dict__ slot
+    // which is required for Python versions before 3.11 (before managed dicts were introduced)
+    template <typename T, bool is_object_base=true>    
+    struct PyWrapperWithDict: public PyWrapper<T, is_object_base, PyObjectWithDict>    
+    {
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Winvalid-offsetof"
+        static constexpr std::size_t getDictOffset() {
+            return offsetof(PyObjectWithDict, m_py_dict);
+        }
+#pragma GCC diagnostic pop
+    };
+
     template <typename T> struct Shared
     {
         std::shared_ptr<T> m_ptr;

src/dbzero/bindings/python/types/PyObjectId.hpp

@@ -9,6 +9,7 @@
 #include <dbzero/bindings/python/WhichType.hpp>
 #include <dbzero/bindings/python/PyWrapper.hpp>
 #include <dbzero/object_model/value/ObjectId.hpp>
+#include <dbzero/bindings/python/MemoObject.hpp>
 
 namespace db0::object_model
 
@@ -23,8 +24,7 @@ namespace db0::object_model
 namespace db0::python
 
 {
-
-    using MemoObject = PyWrapper<db0::object_model::Object>;
+
     using ListObject = PyWrapper<db0::object_model::List>;
     using IndexObject = PyWrapper<db0::object_model::Index>;
     using ObjectId = db0::object_model::ObjectId;

src/dbzero/object_model/object/ObjectAnyBase.hpp

@@ -65,7 +65,7 @@ namespace db0::object_model
         using TypeManager = typename LangToolkit::TypeManager;
         using ObjectStem = ObjectVType<T>;
         using TypeInitializer = ObjectInitializer::TypeInitializer;
-
+        
         db0::swine_ptr<Fixture> tryGetFixture() const;
         db0::swine_ptr<Fixture> getFixture() const;
 

src/dbzero/object_model/value/Member.cpp

@@ -648,6 +648,34 @@ namespace db0::object_model
         functions[static_cast<int>(StorageClass::PACK_2)] = unloadMember<StorageClass::PACK_2, PyToolkit>;
     }
 
+    template <typename T, typename MemoImplT, typename LangToolkit>
+    void unrefMemoObject(db0::swine_ptr<Fixture> &fixture, Address address)
+    {
+        auto obj_ptr = fixture->getLangCache().get(address);
+        if (obj_ptr.get()) {
+            db0::FixtureLock lock(fixture);
+            // decref cached instance via language specific wrapper type
+            auto lang_wrapper = reinterpret_cast<MemoImplT*>(obj_ptr.get());
+            auto &object = lang_wrapper->modifyExt();
+            object.decRef(false);
+            if (!object.hasRefs()) {
+                // NOTE: we'll drop the object immediately on condition it has no language references
+                if (!LangToolkit::hasLangRefs(*obj_ptr)) {
+                    auto unique_addr = object.getUniqueAddress();                    
+                    // drop dbzero instance, replacing it with a "null" placeholder
+                    object.dropInstance(lock);
+                    // might also be removed from lang cache                    
+                    fixture->getLangCache().erase(unique_addr);                    
+                }
+            }
+        } else {
+            T object(fixture, address);
+            object.decRef(false);
+            // member will be deleted by GC0 if its ref-count = 0
+        }
+    }
+
+    // Unreference any ObjectBase-derived type (except Memo types)
     template <typename T, typename LangToolkit>
     void unrefObjectBase(db0::swine_ptr<Fixture> &fixture, Address address)
     {
@@ -675,11 +703,12 @@ namespace db0::object_model
         }
     }
 
-    // OBJECT_REF specialization
+    // OBJECT_REF specialization (MemoAnyImpl)
     template <> void unrefMember<StorageClass::OBJECT_REF, PyToolkit>(
         db0::swine_ptr<Fixture> &fixture, Value value)
     {
-        unrefObjectBase<Object, PyToolkit>(fixture, value.asAddress());
+        using MemoObject = PyToolkit::TypeManager::MemoObject;
+        unrefMemoObject<Object, MemoObject, PyToolkit>(fixture, value.asAddress());
     }
 
     template <> void unrefMember<StorageClass::DB0_LIST, PyToolkit>(