From d367df43a2fa4a58094900af4f9fe349832edecf Mon Sep 17 00:00:00 2001 From: Wojtek Date: Sat, 1 Nov 2025 15:45:21 +0100 Subject: [PATCH 01/19] no-cache handling from fetch / member access --- python_tests/test_memo_no_cache.py | 33 +++++++- src/dbzero/bindings/python/PyAPI.cpp | 2 +- src/dbzero/bindings/python/PyInternalAPI.cpp | 21 +++-- src/dbzero/bindings/python/PyToolkit.cpp | 81 ++++++++++++------- src/dbzero/bindings/python/PyToolkit.hpp | 22 ++--- src/dbzero/core/vspace/v_ptr.cpp | 8 +- src/dbzero/core/vspace/v_ptr.hpp | 1 + src/dbzero/object_model/bytes/ByteArray.cpp | 16 ++-- src/dbzero/object_model/bytes/ByteArray.hpp | 4 +- src/dbzero/object_model/class/Class.hpp | 5 ++ src/dbzero/object_model/dict/Dict.cpp | 14 ++-- src/dbzero/object_model/dict/DictIterator.cpp | 12 +-- src/dbzero/object_model/index/Index.cpp | 10 +-- src/dbzero/object_model/index/Index.hpp | 6 +- .../object_model/iterators/BaseIterator.hpp | 3 + src/dbzero/object_model/list/List.cpp | 12 +-- src/dbzero/object_model/list/ListIterator.cpp | 2 +- src/dbzero/object_model/object/Object.cpp | 34 ++++---- src/dbzero/object_model/object/Object.hpp | 13 +-- src/dbzero/object_model/set/Set.cpp | 22 ++--- src/dbzero/object_model/set/Set.hpp | 12 ++- src/dbzero/object_model/set/SetIterator.cpp | 2 +- src/dbzero/object_model/tuple/Tuple.cpp | 6 +- .../object_model/tuple/TupleIterator.cpp | 2 +- src/dbzero/object_model/value/Member.cpp | 76 ++++++++--------- src/dbzero/object_model/value/Member.hpp | 16 ++-- 26 files changed, 260 insertions(+), 175 deletions(-) diff --git a/python_tests/test_memo_no_cache.py b/python_tests/test_memo_no_cache.py index 9fd60ccb..d7b9ed19 100644 --- a/python_tests/test_memo_no_cache.py +++ b/python_tests/test_memo_no_cache.py @@ -1,8 +1,9 @@ from random import random import pytest import dbzero as db0 -from .memo_test_types import MemoTestClass, TriColor, MemoAnyAttrs +from .memo_test_types import MemoTestClass from dataclasses import dataclass +from .conftest import DB0_DIR import random import string import gc @@ -61,4 +62,32 @@ def test_excluding_no_cache_instances_from_dbzero_cache(db0_fixture): final_cache_size = db0.get_cache_stats()["size"] # make sure cache utilization is low assert abs(final_cache_size - initial_cache_size) < (300 << 10) - \ No newline at end of file + + +def test_fetching_no_cache_objects(db0_fixture): + px_name = db0.get_current_prefix().name + buf = db0.list() + uuid_list = [] + for _ in range(100): + obj = MemoNoCacheClass() + buf.append(obj) + uuid_list.append(db0.uuid(obj)) + + db0.close() + db0.init(DB0_DIR) + db0.open(px_name, "r") + + # now fetch objects by uuid + initial_cache_size = db0.get_cache_stats()["size"] + print(db0.get_cache_stats()) + total_len = 0 + for id in uuid_list: + # NOTE: must fetch with type, otherwise no_cache flag may not be honored + obj = db0.fetch(MemoNoCacheClass, id) + # this forces data retrieval + total_len += len(obj.data) + + final_cache_size = db0.get_cache_stats()["size"] + print(db0.get_cache_stats()) + # make sure cache utilization is low + assert abs(final_cache_size - initial_cache_size) < (300 << 10) diff --git a/src/dbzero/bindings/python/PyAPI.cpp b/src/dbzero/bindings/python/PyAPI.cpp index 170493ec..8b65fde1 100644 --- a/src/dbzero/bindings/python/PyAPI.cpp +++ b/src/dbzero/bindings/python/PyAPI.cpp @@ -144,7 +144,7 @@ namespace db0::python return NULL; } - PY_API_FUNC + PY_API_FUNC return runSafe(tryFetch, py_id, reinterpret_cast(py_type), prefix_name).steal(); } diff --git a/src/dbzero/bindings/python/PyInternalAPI.cpp b/src/dbzero/bindings/python/PyInternalAPI.cpp index 5ee3cc5b..cde42726 100644 --- a/src/dbzero/bindings/python/PyInternalAPI.cpp +++ b/src/dbzero/bindings/python/PyInternalAPI.cpp @@ -170,27 +170,32 @@ namespace db0::python using ClassFactory = db0::object_model::ClassFactory; using Class = db0::object_model::Class; - // validate pre-storage class first + // Validate pre-storage class first if (py_expected_type && !checkObjectIdType(object_id, py_expected_type)) { THROWF(db0::InputException) << "Object ID type mismatch"; } auto storage_class = object_id.m_storage_class; auto addr = object_id.m_address; - if (storage_class == db0::object_model::StorageClass::OBJECT_REF) { + if (storage_class == db0::object_model::StorageClass::OBJECT_REF) { auto &class_factory = db0::object_model::getClassFactory(*fixture); - auto result = PyToolkit::unloadObject(fixture, addr, class_factory, nullptr, addr.getInstanceId()); - auto &memo = reinterpret_cast(result.get())->ext(); - // validate type if requested (no validation for MemoBase) - if (py_expected_type && !PyToolkit::getTypeManager().isMemoBase(py_expected_type)) { + if (py_expected_type && !PyToolkit::getTypeManager().isMemoBase(py_expected_type)) { // in other cases the type must match the actual object type - auto expected_class = class_factory.getExistingType(py_expected_type); + auto expected_class = class_factory.getExistingType(py_expected_type); + // honor class-specific access flags (e.g. type-level no_cache) + auto result = PyToolkit::unloadObject(fixture, addr, class_factory, nullptr, addr.getInstanceId(), + expected_class->getInstanceFlags() + ); + auto &memo = reinterpret_cast(result.get())->ext(); if (memo.getType() != *expected_class) { THROWF(db0::InputException) << "Object type mismatch"; } + return result; + } else { + // unload without type validation + return PyToolkit::unloadObject(fixture, addr, class_factory, nullptr, addr.getInstanceId()); } - return result; } else if (storage_class == db0::object_model::StorageClass::DB0_CLASS) { auto &class_factory = db0::object_model::getClassFactory(*fixture); auto class_ptr = class_factory.getTypeByAddr(addr).m_class; diff --git a/src/dbzero/bindings/python/PyToolkit.cpp b/src/dbzero/bindings/python/PyToolkit.cpp index ae728407..3acb14e7 100644 --- a/src/dbzero/bindings/python/PyToolkit.cpp +++ b/src/dbzero/bindings/python/PyToolkit.cpp @@ -91,10 +91,10 @@ namespace db0::python } PyToolkit::ObjectSharedPtr PyToolkit::unloadObject(db0::swine_ptr &fixture, Address address, - TypeObjectPtr lang_class, std::uint16_t instance_id) + TypeObjectPtr lang_class, std::uint16_t instance_id, AccessFlags access_mode) { auto &class_factory = fixture->get(); - return unloadObject(fixture, address, class_factory, lang_class, instance_id); + return unloadObject(fixture, address, class_factory, lang_class, instance_id, access_mode); } bool PyToolkit::isExistingObject(db0::swine_ptr &fixture, Address address, std::uint16_t instance_id) @@ -124,8 +124,9 @@ namespace db0::python return db0::object_model::Object::checkUnload(fixture, address, instance_id, true); } - PyToolkit::ObjectSharedPtr PyToolkit::tryUnloadObject(db0::swine_ptr &fixture, Address address, - const ClassFactory &class_factory, TypeObjectPtr lang_type_ptr, std::uint16_t instance_id) + PyToolkit::ObjectSharedPtr PyToolkit::tryUnloadObject( + db0::swine_ptr &fixture, Address address, const ClassFactory &class_factory, + TypeObjectPtr lang_type_ptr, std::uint16_t instance_id, AccessFlags access_mode) { // try unloading from cache first auto &lang_cache = fixture->getLangCache(); @@ -148,7 +149,9 @@ namespace db0::python } // Unload from backend otherwise - auto stem = db0::object_model::Object::tryUnloadStem(fixture, address, instance_id); + auto stem = db0::object_model::Object::tryUnloadStem( + fixture, address, instance_id, access_mode + ); if (!stem) { // object not found return {}; @@ -180,17 +183,19 @@ namespace db0::python } PyToolkit::ObjectSharedPtr PyToolkit::unloadObject(db0::swine_ptr &fixture, Address address, - const ClassFactory &class_factory, TypeObjectPtr lang_type_ptr, std::uint16_t instance_id) + const ClassFactory &class_factory, TypeObjectPtr lang_type_ptr, std::uint16_t instance_id, AccessFlags access_mode) { - auto result = tryUnloadObject(fixture, address, class_factory, lang_type_ptr, instance_id); + auto result = tryUnloadObject( + fixture, address, class_factory, lang_type_ptr, instance_id, access_mode + ); if (!result) { THROWF(db0::InputException) << "Invalid UUID or object has been deleted"; } return result; } - + PyToolkit::ObjectSharedPtr PyToolkit::unloadObject(db0::swine_ptr &fixture, Address address, - std::shared_ptr type, TypeObjectPtr lang_class) + std::shared_ptr type, TypeObjectPtr lang_class, AccessFlags access_mode) { assert(lang_class); // try unloading from cache first @@ -204,7 +209,7 @@ namespace db0::python // NOTE: lang_class may be of a base type (e.g. MemoBase) auto *memo_ptr = MemoObjectStub_new(lang_class); // unload with type hint - memo_ptr->unload(fixture, address, type, Object::with_type_hint{}); + memo_ptr->unload(fixture, address, type, Object::with_type_hint{}, access_mode); // NOTE: Py_OWN only possible with a proper object obj_ptr = Py_OWN((PyObject*)memo_ptr); if (!memo_ptr->ext().isNoCache()) { @@ -233,7 +238,8 @@ namespace db0::python return unloadExpiredRef(fixture, weak_ref.getAddress(), weak_ref->m_fixture_uuid, weak_ref->m_address); } - PyToolkit::ObjectSharedPtr PyToolkit::unloadList(db0::swine_ptr fixture, Address address, std::uint16_t) + PyToolkit::ObjectSharedPtr PyToolkit::unloadList(db0::swine_ptr fixture, Address address, + std::uint16_t, AccessFlags access_mode) { using List = db0::object_model::List; @@ -247,13 +253,16 @@ namespace db0::python auto list_object = ListDefaultObject_new(); // retrieve actual dbzero instance - list_object->unload(fixture, address); + list_object->unload(fixture, address, access_mode); // add list object to cache - lang_cache.add(address, list_object.get()); + if (!list_object->ext().isNoCache()) { + lang_cache.add(address, list_object.get()); + } return shared_py_cast(std::move(list_object)); } - PyToolkit::ObjectSharedPtr PyToolkit::unloadByteArray(db0::swine_ptr fixture, Address address) + PyToolkit::ObjectSharedPtr PyToolkit::unloadByteArray(db0::swine_ptr fixture, + Address address, AccessFlags access_mode) { // try pulling from cache first auto &lang_cache = fixture->getLangCache(); @@ -265,13 +274,16 @@ namespace db0::python auto byte_array_object = ByteArrayDefaultObject_new(); // retrieve actual dbzero instance - byte_array_object->unload(fixture, address); + byte_array_object->unload(fixture, address, access_mode); // add byte_array object to cache - lang_cache.add(address, byte_array_object.get()); + if (!byte_array_object->ext().isNoCache()) { + lang_cache.add(address, byte_array_object.get()); + } return shared_py_cast(std::move(byte_array_object)); } - PyToolkit::ObjectSharedPtr PyToolkit::unloadIndex(db0::swine_ptr fixture, Address address, std::uint16_t) + PyToolkit::ObjectSharedPtr PyToolkit::unloadIndex(db0::swine_ptr fixture, + Address address, std::uint16_t, AccessFlags access_mode) { // try pulling from cache first auto &lang_cache = fixture->getLangCache(); @@ -283,14 +295,17 @@ namespace db0::python auto index_object = IndexDefaultObject_new(); // retrieve actual dbzero instance - index_object->unload(fixture, address); + index_object->unload(fixture, address, access_mode); // add list object to cache - lang_cache.add(address, index_object.get()); + if (!index_object->ext().isNoCache()) { + lang_cache.add(address, index_object.get()); + } return shared_py_cast(std::move(index_object)); } - PyToolkit::ObjectSharedPtr PyToolkit::unloadSet(db0::swine_ptr fixture, Address address, std::uint16_t) + PyToolkit::ObjectSharedPtr PyToolkit::unloadSet(db0::swine_ptr fixture, + Address address, std::uint16_t, AccessFlags access_mode) { // try pulling from cache first auto &lang_cache = fixture->getLangCache(); @@ -302,14 +317,17 @@ namespace db0::python auto set_object = SetDefaultObject_new(); // retrieve actual dbzero instance - set_object->unload(fixture, address); + set_object->unload(fixture, address, access_mode); // add list object to cache - lang_cache.add(address, set_object.get()); + if (!set_object->ext().isNoCache()) { + lang_cache.add(address, set_object.get()); + } return shared_py_cast(std::move(set_object)); } - PyToolkit::ObjectSharedPtr PyToolkit::unloadDict(db0::swine_ptr fixture, Address address, std::uint16_t) + PyToolkit::ObjectSharedPtr PyToolkit::unloadDict(db0::swine_ptr fixture, + Address address, std::uint16_t, AccessFlags access_mode) { // try pulling from cache first auto &lang_cache = fixture->getLangCache(); @@ -321,14 +339,17 @@ namespace db0::python auto dict_object = DictDefaultObject_new(); // retrieve actual dbzero instance - dict_object->unload(fixture, address); + dict_object->unload(fixture, address, access_mode); // add list object to cache - lang_cache.add(address, *dict_object); + if (!dict_object->ext().isNoCache()) { + lang_cache.add(address, *dict_object); + } return shared_py_cast(std::move(dict_object)); } - PyToolkit::ObjectSharedPtr PyToolkit::unloadTuple(db0::swine_ptr fixture, Address address, std::uint16_t) + PyToolkit::ObjectSharedPtr PyToolkit::unloadTuple(db0::swine_ptr fixture, + Address address, std::uint16_t, AccessFlags access_mode) { // try pulling from cache first auto &lang_cache = fixture->getLangCache(); @@ -340,13 +361,15 @@ namespace db0::python auto tuple_object = TupleDefaultObject_new(); // retrieve actual dbzero instance - tuple_object->unload(fixture, address); + tuple_object->unload(fixture, address, access_mode); // add list object to cache - lang_cache.add(address, *tuple_object); + if (!tuple_object->ext().isNoCache()) { + lang_cache.add(address, *tuple_object); + } return shared_py_cast(std::move(tuple_object)); } - + PyToolkit::ObjectSharedPtr PyToolkit::deserializeObjectIterable(db0::swine_ptr fixture, std::vector::const_iterator &iter, std::vector::const_iterator end) diff --git a/src/dbzero/bindings/python/PyToolkit.hpp b/src/dbzero/bindings/python/PyToolkit.hpp index 644a6b0e..72cf99bb 100644 --- a/src/dbzero/bindings/python/PyToolkit.hpp +++ b/src/dbzero/bindings/python/PyToolkit.hpp @@ -99,11 +99,11 @@ namespace db0::python // Unload with type resolution // optionally may use specific lang class (e.g. MemoBase) static ObjectSharedPtr unloadObject(db0::swine_ptr &, Address, const ClassFactory &, - TypeObjectPtr lang_class = nullptr, std::uint16_t instance_id = 0); + TypeObjectPtr lang_class = nullptr, std::uint16_t instance_id = 0, AccessFlags = {}); static ObjectSharedPtr tryUnloadObject(db0::swine_ptr &, Address, const ClassFactory &, - TypeObjectPtr lang_class = nullptr, std::uint16_t instance_id = 0); + TypeObjectPtr lang_class = nullptr, std::uint16_t instance_id = 0, AccessFlags = {}); static ObjectSharedPtr unloadObject(db0::swine_ptr &, Address, TypeObjectPtr lang_class = nullptr, - std::uint16_t instance_id = 0); + std::uint16_t instance_id = 0, AccessFlags = {}); static bool isExistingObject(db0::swine_ptr &, Address, std::uint16_t instance_id = 0); @@ -121,15 +121,15 @@ namespace db0::python // Unload with known type & lang class // note that lang_class may be a base of the actual type (e.g. MemoBase) static ObjectSharedPtr unloadObject(db0::swine_ptr &, Address address, - std::shared_ptr, TypeObjectPtr lang_class); + std::shared_ptr, TypeObjectPtr lang_class, AccessFlags = {}); - static ObjectSharedPtr unloadList(db0::swine_ptr, Address, std::uint16_t instance_id = 0); - static ObjectSharedPtr unloadIndex(db0::swine_ptr, Address, std::uint16_t instance_id = 0); - static ObjectSharedPtr unloadSet(db0::swine_ptr, Address, std::uint16_t instance_id = 0); - static ObjectSharedPtr unloadDict(db0::swine_ptr, Address, std::uint16_t instance_id = 0); - static ObjectSharedPtr unloadTuple(db0::swine_ptr, Address, std::uint16_t instance_id = 0); + static ObjectSharedPtr unloadList(db0::swine_ptr, Address, std::uint16_t instance_id = 0, AccessFlags = {}); + static ObjectSharedPtr unloadIndex(db0::swine_ptr, Address, std::uint16_t instance_id = 0, AccessFlags = {}); + static ObjectSharedPtr unloadSet(db0::swine_ptr, Address, std::uint16_t instance_id = 0, AccessFlags = {}); + static ObjectSharedPtr unloadDict(db0::swine_ptr, Address, std::uint16_t instance_id = 0, AccessFlags = {}); + static ObjectSharedPtr unloadTuple(db0::swine_ptr, Address, std::uint16_t instance_id = 0, AccessFlags = {}); // Unload dbzero block instance - static ObjectSharedPtr unloadBlock(db0::swine_ptr, Address, std::uint16_t instance_id = 0); + static ObjectSharedPtr unloadBlock(db0::swine_ptr, Address, std::uint16_t instance_id = 0, AccessFlags = {}); // Unload from serialized bytes static ObjectSharedPtr deserializeObjectIterable(db0::swine_ptr, std::vector::const_iterator &iter, @@ -139,7 +139,7 @@ namespace db0::python static ObjectSharedPtr deserializeEnumValueRepr(db0::swine_ptr, std::vector::const_iterator &iter, std::vector::const_iterator end); - static ObjectSharedPtr unloadByteArray(db0::swine_ptr, Address); + static ObjectSharedPtr unloadByteArray(db0::swine_ptr, Address, AccessFlags = {}); // Creates a new Python instance of EnumValue static ObjectSharedPtr makeEnumValue(const EnumValue &); diff --git a/src/dbzero/core/vspace/v_ptr.cpp b/src/dbzero/core/vspace/v_ptr.cpp index 82932b37..4e184ccc 100644 --- a/src/dbzero/core/vspace/v_ptr.cpp +++ b/src/dbzero/core/vspace/v_ptr.cpp @@ -12,13 +12,19 @@ namespace db0 assertFlags(); assert(!(m_resource_flags.load() & RESOURCE_LOCK)); } - + vtypeless::vtypeless(const vtypeless &other) : m_memspace_ptr(other.m_memspace_ptr) { *this = other; } + vtypeless::vtypeless(vtypeless &&other) + : m_memspace_ptr(other.m_memspace_ptr) + { + *this = std::move(other); + } + vtypeless::vtypeless(Memspace &memspace, Address address, MemLock &&mem_lock, std::uint16_t resource_flags, FlagSet access_mode) : m_address(address) diff --git a/src/dbzero/core/vspace/v_ptr.hpp b/src/dbzero/core/vspace/v_ptr.hpp index 756e0b50..5d5d87ab 100644 --- a/src/dbzero/core/vspace/v_ptr.hpp +++ b/src/dbzero/core/vspace/v_ptr.hpp @@ -68,6 +68,7 @@ namespace db0 FlagSet); vtypeless(const vtypeless& other); + vtypeless(vtypeless&&); /** * @param access_mode additional flags / modes to use diff --git a/src/dbzero/object_model/bytes/ByteArray.cpp b/src/dbzero/object_model/bytes/ByteArray.cpp index d7b43324..897450fd 100644 --- a/src/dbzero/object_model/bytes/ByteArray.cpp +++ b/src/dbzero/object_model/bytes/ByteArray.cpp @@ -8,19 +8,19 @@ namespace db0::object_model GC0_Define(ByteArray) - ByteArray::ByteArray(db0::swine_ptr &fixture, std::byte *bytes, std::size_t size) - : super_t(fixture) + ByteArray::ByteArray(db0::swine_ptr &fixture, std::byte *bytes, std::size_t size, AccessFlags access_mode) + : super_t(fixture, access_mode) { for (auto *bytes_ptr = bytes, *end = bytes + size; bytes_ptr != end; ++bytes_ptr) { v_bvector::push_back(*bytes_ptr); } } - ByteArray::ByteArray(db0::swine_ptr &fixture, Address address) - : super_t(super_t::tag_from_address(), fixture, address) + ByteArray::ByteArray(db0::swine_ptr &fixture, Address address, AccessFlags access_mode) + : super_t(super_t::tag_from_address(), fixture, address, access_mode) { } - + ByteArray::ByteArray(tag_no_gc, db0::swine_ptr &fixture, const ByteArray &byte_array) : super_t(tag_no_gc(), fixture, byte_array) { @@ -38,9 +38,11 @@ namespace db0::object_model THROWF(db0::InputException) << "Index out of range: " << i; } auto fixture = this->getFixture(); - return unloadMember(fixture, StorageClass::INT64, (long)(*this)[i]); + return unloadMember( + fixture, StorageClass::INT64, (long)(*this)[i], 0, this->getMemberFlags() + ); } - + std::byte ByteArray::getByte(std::size_t i) const { return (*this)[i]; } diff --git a/src/dbzero/object_model/bytes/ByteArray.hpp b/src/dbzero/object_model/bytes/ByteArray.hpp index 0e2186cc..69c99a22 100644 --- a/src/dbzero/object_model/bytes/ByteArray.hpp +++ b/src/dbzero/object_model/bytes/ByteArray.hpp @@ -29,9 +29,9 @@ namespace db0::object_model // as null placeholder ByteArray() = default; - ByteArray(db0::swine_ptr &, std::byte *, std::size_t); + ByteArray(db0::swine_ptr &, std::byte *, std::size_t, AccessFlags = {}); ByteArray(tag_no_gc, db0::swine_ptr &, const ByteArray &); - ByteArray(db0::swine_ptr &, Address); + ByteArray(db0::swine_ptr &, Address, AccessFlags = {}); ~ByteArray(); ObjectSharedPtr getItem(std::size_t i) const; diff --git a/src/dbzero/object_model/class/Class.hpp b/src/dbzero/object_model/class/Class.hpp index 2adfb4ef..c8ddad71 100644 --- a/src/dbzero/object_model/class/Class.hpp +++ b/src/dbzero/object_model/class/Class.hpp @@ -253,6 +253,11 @@ DB0_PACKED_END return m_no_cache; } + // Get instance flags to be applied to objects of this class + inline AccessFlags getInstanceFlags() const { + return m_no_cache ? AccessFlags { AccessOptions::no_cache } : AccessFlags {}; + } + protected: friend class ClassFactory; friend ClassPtr; diff --git a/src/dbzero/object_model/dict/Dict.cpp b/src/dbzero/object_model/dict/Dict.cpp index 5b4392c9..311e90be 100644 --- a/src/dbzero/object_model/dict/Dict.cpp +++ b/src/dbzero/object_model/dict/Dict.cpp @@ -120,7 +120,7 @@ namespace db0::object_model auto it_join = bindex.beginJoin(1); while (!it_join.is_end()) { auto [storage_class, value] = (*it_join).m_first; - auto member = unloadMember(*fixture, storage_class, value); + auto member = unloadMember(*fixture, storage_class, value, 0, member_flags); if (LangToolkit::compare(key, member.get())) { bindex.erase(*it_join); unrefMember(*fixture, storage_class, value); @@ -150,10 +150,14 @@ namespace db0::object_model auto it = bindex.beginJoin(1); while (!it.is_end()) { auto [storage_class, value] = (*it).m_first; - auto member = unloadMember(fixture, storage_class, value); + auto member = unloadMember( + fixture, storage_class, value, 0, this->getMemberFlags() + ); if (LangToolkit::compare(key_value, member.get())) { auto [storage_class, value] = (*it).m_second; - return unloadMember(fixture, storage_class, value); + return unloadMember( + fixture, storage_class, value, 0, this->getMemberFlags() + ); } ++it; } @@ -171,7 +175,7 @@ namespace db0::object_model auto it = bindex.beginJoin(1); while (!it.is_end()) { auto [storage_class, value] = (*it).m_first; - auto member = unloadMember(fixture, storage_class, value); + auto member = unloadMember(fixture, storage_class, value, 0, this->getMemberFlags()); if (LangToolkit::compare(key_value, member.get())) { // a matching key was found return true; @@ -200,7 +204,7 @@ namespace db0::object_model auto [storage_class, value] = (*it).m_second; auto fixture = this->getFixture(); - auto member = unloadMember(fixture, storage_class, value); + auto member = unloadMember(fixture, storage_class, value, 0, this->getMemberFlags()); unrefMember(fixture, key_storage_class, key_value); unrefMember(fixture, storage_class, value); diff --git a/src/dbzero/object_model/dict/DictIterator.cpp b/src/dbzero/object_model/dict/DictIterator.cpp index ddb6d826..e13ac1fe 100644 --- a/src/dbzero/object_model/dict/DictIterator.cpp +++ b/src/dbzero/object_model/dict/DictIterator.cpp @@ -13,7 +13,7 @@ namespace db0::object_model DictIterator::DictIterator( Dict::const_iterator iterator, const Dict * ptr, ObjectPtr lang_dict, IteratorType type) : BaseIterator(iterator, ptr, lang_dict) - , m_type(type) + , m_type(type) { setJoinIterator(); } @@ -51,8 +51,8 @@ namespace db0::object_model iterNext(); return { - unloadMember(fixture, key), - unloadMember(fixture, value) + unloadMember(fixture, key, 0, m_member_flags), + unloadMember(fixture, value, 0, m_member_flags) }; } @@ -61,15 +61,15 @@ namespace db0::object_model auto value = (*m_join_iterator).m_second; iterNext(); auto fixture = m_collection->getFixture(); - return unloadMember(fixture, value); + return unloadMember(fixture, value, 0, m_member_flags); } - DictIterator::ObjectSharedPtr DictIterator::nextKey() + DictIterator::ObjectSharedPtr DictIterator::nextKey() { auto key = (*m_join_iterator).m_first; iterNext(); auto fixture = m_collection->getFixture(); - return unloadMember(fixture, key); + return unloadMember(fixture, key, 0, m_member_flags); } DictIterator::ObjectSharedPtr DictIterator::next() diff --git a/src/dbzero/object_model/index/Index.cpp b/src/dbzero/object_model/index/Index.cpp index 6487cb07..f2a6f5c9 100644 --- a/src/dbzero/object_model/index/Index.cpp +++ b/src/dbzero/object_model/index/Index.cpp @@ -18,16 +18,16 @@ namespace db0::object_model { } - Index::Index(db0::swine_ptr &fixture) - : super_t(fixture, IndexType::RangeTree, IndexDataType::Auto) + Index::Index(db0::swine_ptr &fixture, AccessFlags access_mode) + : super_t(fixture, IndexType::RangeTree, IndexDataType::Auto, access_mode) , m_builder(*this) // NOTE: register the mutation handler for supporting "locked" sections , m_mutation_log(fixture->addMutationHandler()) { } - - Index::Index(db0::swine_ptr &fixture, Address address) - : super_t(super_t::tag_from_address(), fixture, address) + + Index::Index(db0::swine_ptr &fixture, Address address, AccessFlags access_mode) + : super_t(super_t::tag_from_address(), fixture, address, access_mode) , m_builder(*this) , m_mutation_log(fixture->addMutationHandler()) { diff --git a/src/dbzero/object_model/index/Index.hpp b/src/dbzero/object_model/index/Index.hpp index e574466b..b158991b 100644 --- a/src/dbzero/object_model/index/Index.hpp +++ b/src/dbzero/object_model/index/Index.hpp @@ -37,11 +37,11 @@ namespace db0::object_model // null instance constructor Index(); - Index(db0::swine_ptr &); - Index(db0::swine_ptr &, Address); + Index(db0::swine_ptr &, AccessFlags = {}); + Index(db0::swine_ptr &, Address, AccessFlags = {}); Index(const Index &) = delete; ~Index(); - + std::size_t size() const; void add(ObjectPtr key, ObjectPtr value); void remove(ObjectPtr key, ObjectPtr value); diff --git a/src/dbzero/object_model/iterators/BaseIterator.hpp b/src/dbzero/object_model/iterators/BaseIterator.hpp index 14046db5..4e8dce8f 100644 --- a/src/dbzero/object_model/iterators/BaseIterator.hpp +++ b/src/dbzero/object_model/iterators/BaseIterator.hpp @@ -27,6 +27,7 @@ namespace db0::object_model : m_iterator(iterator) , m_collection(ptr) , m_lang_collection_shared_ptr(lang_collection_ptr) + , m_member_flags(ptr->getMemberFlags()) { } @@ -47,6 +48,8 @@ namespace db0::object_model const CollectionT *m_collection; // reference to persist collections' related language specific object ObjectSharedPtr m_lang_collection_shared_ptr; + // member access flags (e.g. no_cache) + const AccessFlags m_member_flags; }; } \ No newline at end of file diff --git a/src/dbzero/object_model/list/List.cpp b/src/dbzero/object_model/list/List.cpp index 1ad099f5..b036ab69 100644 --- a/src/dbzero/object_model/list/List.cpp +++ b/src/dbzero/object_model/list/List.cpp @@ -72,7 +72,7 @@ namespace db0::object_model } auto [storage_class, value] = (*this)[i]; auto fixture = this->getFixture(); - return unloadMember(fixture, storage_class, value); + return unloadMember(fixture, storage_class, value, 0, this->getMemberFlags()); } List::ObjectSharedPtr List::pop(FixtureLock &fixture, std::size_t i) @@ -84,7 +84,7 @@ namespace db0::object_model THROWF(db0::InputException) << "Index out of range: " << i; } auto [storage_class, value] = (*this)[i]; - auto member = unloadMember(*fixture, storage_class, value); + auto member = unloadMember(*fixture, storage_class, value, 0, this->getMemberFlags()); this->swapAndPop(fixture, {i}); restoreIterators(); return member; @@ -122,8 +122,8 @@ namespace db0::object_model auto fixture = this->getFixture(); for (auto &elem: (*this)) { auto [elem_storage_class, elem_value] = elem; - if (unloadMember(fixture, elem_storage_class, elem_value) == lang_value) { - count += 1; + if (unloadMember(fixture, elem_storage_class, elem_value, 0, this->getMemberFlags()) == lang_value) { + ++count; } } return count; @@ -135,7 +135,7 @@ namespace db0::object_model auto fixture = this->getFixture(); for (auto &elem: (*this)) { auto [elem_storage_class, elem_value] = elem; - if (unloadMember(fixture, elem_storage_class, elem_value) == lang_value) { + if (unloadMember(fixture, elem_storage_class, elem_value, 0, this->getMemberFlags()) == lang_value) { return index; } ++index; @@ -143,7 +143,7 @@ namespace db0::object_model THROWF(db0::InputException) << "Item is not in a list "; return -1; } - + bool List::operator==(const List &list) const { if (size() != list.size()) { diff --git a/src/dbzero/object_model/list/ListIterator.cpp b/src/dbzero/object_model/list/ListIterator.cpp index 8eb6e479..e4fb2e6a 100644 --- a/src/dbzero/object_model/list/ListIterator.cpp +++ b/src/dbzero/object_model/list/ListIterator.cpp @@ -16,7 +16,7 @@ namespace db0::object_model auto [storage_class, value] = *m_iterator; ++m_iterator; ++m_index; - return unloadMember(fixture, storage_class, value); + return unloadMember(fixture, storage_class, value, 0, m_member_flags); } void ListIterator::restore() diff --git a/src/dbzero/object_model/object/Object.cpp b/src/dbzero/object_model/object/Object.cpp index 9e813634..59025bd9 100644 --- a/src/dbzero/object_model/object/Object.cpp +++ b/src/dbzero/object_model/object/Object.cpp @@ -134,21 +134,21 @@ namespace db0::object_model , m_type(type) { } - - Object::Object(db0::swine_ptr &fixture, Address address) - : super_t(super_t::tag_from_address(), fixture, address) + + Object::Object(db0::swine_ptr &fixture, Address address, AccessFlags access_mode) + : super_t(super_t::tag_from_address(), fixture, address, access_mode) { } Object::Object(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type) : super_t(super_t::tag_from_stem(), fixture, std::move(stem)) - , m_type(type) + , m_type(type) { assert(hasValidClassRef()); } - Object::Object(db0::swine_ptr &fixture, Address address, std::shared_ptr type_hint, with_type_hint) - : Object(fixture, address) + Object::Object(db0::swine_ptr &fixture, Address address, std::shared_ptr type_hint, with_type_hint, AccessFlags access_mode) + : Object(fixture, address, access_mode) { assert(*fixture == *type_hint->getFixture()); setTypeWithHint(type_hint); @@ -158,7 +158,7 @@ namespace db0::object_model : Object(fixture, std::move(stem), getTypeWithHint(*fixture, stem->getClassRef(), type_hint)) { } - + Object::~Object() { // unregister needs to be called before destruction of members @@ -178,14 +178,15 @@ namespace db0::object_model new ((void*)this) Object(unique_addr, ext_refs); } - Object::ObjectStem Object::tryUnloadStem(db0::swine_ptr &fixture, Address address, std::uint16_t instance_id) + Object::ObjectStem Object::tryUnloadStem(db0::swine_ptr &fixture, Address address, + std::uint16_t instance_id, AccessFlags access_mode) { std::size_t size_of; if (!fixture->isAddressValid(address, REALM_ID, &size_of)) { return {}; } // Unload from a verified address - ObjectVType stem(db0::tag_verified(), fixture->myPtr(address), size_of); + ObjectVType stem(db0::tag_verified(), fixture->myPtr(address), size_of, access_mode); if (instance_id && stem->m_header.m_instance_id != instance_id) { // instance ID validation failed return {}; @@ -193,9 +194,10 @@ namespace db0::object_model return stem; } - Object::ObjectStem Object::unloadStem(db0::swine_ptr &fixture, Address address, std::uint16_t instance_id) + Object::ObjectStem Object::unloadStem(db0::swine_ptr &fixture, Address address, + std::uint16_t instance_id, AccessFlags access_mode) { - auto result = tryUnloadStem(fixture, address, instance_id); + auto result = tryUnloadStem(fixture, address, instance_id, access_mode); if (!result) { THROWF(db0::InputException) << "Invalid UUID or object has been deleted"; } @@ -916,7 +918,7 @@ namespace db0::object_model assert(member.first != StorageClass::DELETED && member.first != StorageClass::UNDEFINED); // NOTE: offset is required for lo-fi members return unloadMember( - fixture, member.first, member.second, field_id.maybeOffset() + fixture, member.first, member.second, field_id.maybeOffset(), this->getMemberFlags() ); } @@ -939,7 +941,7 @@ namespace db0::object_model // NOTE: offset is required for lo-fi members return unloadMember( - fixture, member.first, member.second, field_id.getOffset() + fixture, member.first, member.second, field_id.getOffset(), this->getMemberFlags() ); } @@ -1391,11 +1393,13 @@ namespace db0::object_model auto fixture = this->getFixture(); forAll([&](const std::string &name, const XValue &xvalue, unsigned int offset) -> bool { // all references convert to UUID - auto py_member = unloadMember(fixture, xvalue.m_type, xvalue.m_value, offset); + auto py_member = unloadMember( + fixture, xvalue.m_type, xvalue.m_value, offset, this->getMemberFlags() + ); return f(name, py_member); }); } - + bool Object::forAll(XValue xvalue, std::function f) const { assert(xvalue.m_type == StorageClass::PACK_2); diff --git a/src/dbzero/object_model/object/Object.hpp b/src/dbzero/object_model/object/Object.hpp index 4b426c32..e19d2dc4 100644 --- a/src/dbzero/object_model/object/Object.hpp +++ b/src/dbzero/object_model/object/Object.hpp @@ -127,13 +127,14 @@ DB0_PACKED_END // Unload from address with a known type (possibly a base type) // NOTE: unload works faster if type_hint is the exact object's type struct with_type_hint {}; - Object(db0::swine_ptr &, Address, std::shared_ptr type_hint, with_type_hint); - + Object(db0::swine_ptr &, Address, std::shared_ptr type_hint, + with_type_hint, AccessFlags = {}); + // Unload from stem with a known type (possibly a base type) // NOTE: unload works faster if type_hint is the exact object's type Object(db0::swine_ptr &, ObjectStem &&, std::shared_ptr type_hint, with_type_hint); - Object(db0::swine_ptr &, Address); + Object(db0::swine_ptr &, Address, AccessFlags = {}); Object(db0::swine_ptr &, std::shared_ptr, std::pair ref_counts, const PosVT::Data &, unsigned int pos_vt_offset); Object(db0::swine_ptr &, ObjectStem &&, std::shared_ptr); @@ -147,8 +148,10 @@ DB0_PACKED_END void dropInstance(FixtureLock &); // Unload the object stem, to retrieve its type - static ObjectStem tryUnloadStem(db0::swine_ptr &, Address, std::uint16_t instance_id = 0); - static ObjectStem unloadStem(db0::swine_ptr &, Address, std::uint16_t instance_id = 0); + static ObjectStem tryUnloadStem(db0::swine_ptr &, Address, + std::uint16_t instance_id = 0, AccessFlags = {}); + static ObjectStem unloadStem(db0::swine_ptr &, Address, + std::uint16_t instance_id = 0, AccessFlags = {}); // Called to finalize adding members void endInit(); diff --git a/src/dbzero/object_model/set/Set.cpp b/src/dbzero/object_model/set/Set.cpp index ac57e576..b9543d2e 100644 --- a/src/dbzero/object_model/set/Set.cpp +++ b/src/dbzero/object_model/set/Set.cpp @@ -30,9 +30,9 @@ namespace db0::object_model SetIndex bindex(*fixture, item); return { key, bindex }; } - - Set::Set(db0::swine_ptr &fixture) - : super_t(fixture) + + Set::Set(db0::swine_ptr &fixture, AccessFlags access_mode) + : super_t(fixture, access_mode) , m_index(*fixture) { modify().m_index_ptr = m_index.getAddress(); @@ -51,9 +51,9 @@ namespace db0::object_model } modify().m_size = set.size(); } - - Set::Set(db0::swine_ptr &fixture, Address address) - : super_t(super_t::tag_from_address(), fixture, address) + + Set::Set(db0::swine_ptr &fixture, Address address, AccessFlags access_mode) + : super_t(super_t::tag_from_address(), fixture, address, access_mode) , m_index(myPtr((*this)->m_index_ptr)) { } @@ -81,7 +81,7 @@ namespace db0::object_model auto it = bindex.beginJoin(1); while (!it.is_end()) { auto [storage_class, value] = (*it); - auto member = unloadMember(fixture, storage_class, value); + auto member = unloadMember(fixture, storage_class, value, 0, getMemberFlags()); append(fixture, key, member.get()); ++it; } @@ -155,7 +155,7 @@ namespace db0::object_model auto fixture = this->getFixture(); while (!it.is_end()) { auto [storage_class, value] = *it; - auto member = unloadMember(fixture, storage_class, value); + auto member = unloadMember(fixture, storage_class, value, 0, getMemberFlags()); if (LangToolkit::compare(key_value, member.get())) { if (bindex.size() == 1) { m_index.erase(iter); @@ -186,7 +186,7 @@ namespace db0::object_model auto it = bindex.beginJoin(1); while (!it.is_end()) { auto [storage_class, value] = *it; - auto member = unloadMember(fixture, storage_class, value); + auto member = unloadMember(fixture, storage_class, value, 0, getMemberFlags()); if (LangToolkit::compare(key_value, member.get())) { return member; } @@ -214,7 +214,7 @@ namespace db0::object_model auto it = bindex.beginJoin(1); auto [storage_class, value] = *it; auto fixture = this->getFixture(); - auto member = unloadMember(fixture, storage_class, value); + auto member = unloadMember(fixture, storage_class, value, 0, getMemberFlags()); if (bindex.size() == 1) { m_index.erase(iter); bindex.destroy(); @@ -239,7 +239,7 @@ namespace db0::object_model auto it = bindex.beginJoin(1); while (!it.is_end()) { auto [storage_class, value] = *it; - auto member = unloadMember(fixture, storage_class, value); + auto member = unloadMember(fixture, storage_class, value, 0, getMemberFlags()); if (LangToolkit::compare(key_value, member.get())) { return true; } diff --git a/src/dbzero/object_model/set/Set.hpp b/src/dbzero/object_model/set/Set.hpp index 7eb249c3..1aa3c291 100644 --- a/src/dbzero/object_model/set/Set.hpp +++ b/src/dbzero/object_model/set/Set.hpp @@ -22,14 +22,13 @@ namespace db0::object_model { -DB0_PACKED_BEGIN - using Fixture = db0::Fixture; using TypedItem_Address = ValueT_Address; using SetIndex = CollectionIndex; using set_item = db0::key_value>; class SetIterator; +DB0_PACKED_BEGIN struct DB0_PACKED_ATTR o_set: public db0::o_fixed { // common object header @@ -42,6 +41,7 @@ DB0_PACKED_BEGIN return m_header.hasRefs(); } }; +DB0_PACKED_END class Set: public db0::ObjectBase, StorageClass::DB0_SET> { @@ -56,11 +56,11 @@ DB0_PACKED_BEGIN // as null placeholder Set() = default; - explicit Set(db0::swine_ptr &); + explicit Set(db0::swine_ptr &, AccessFlags = {}); explicit Set(tag_no_gc, db0::swine_ptr &, const Set &); - explicit Set(db0::swine_ptr &, Address); + explicit Set(db0::swine_ptr &, Address, AccessFlags = {}); ~Set(); - + void operator=(Set &&); void append(FixtureLock &, std::size_t key, ObjectSharedPtr lang_value); @@ -103,6 +103,4 @@ DB0_PACKED_BEGIN void restoreIterators(); }; -DB0_PACKED_END - } \ No newline at end of file diff --git a/src/dbzero/object_model/set/SetIterator.cpp b/src/dbzero/object_model/set/SetIterator.cpp index 744df581..87a1aee2 100644 --- a/src/dbzero/object_model/set/SetIterator.cpp +++ b/src/dbzero/object_model/set/SetIterator.cpp @@ -33,7 +33,7 @@ namespace db0::object_model auto item = *m_join_iterator; auto [storage_class, value] = item; - auto member = unloadMember(fixture, storage_class, value); + auto member = unloadMember(fixture, storage_class, value, 0, m_member_flags); ++m_join_iterator; if (m_join_iterator.is_end()) { ++m_iterator; diff --git a/src/dbzero/object_model/tuple/Tuple.cpp b/src/dbzero/object_model/tuple/Tuple.cpp index 2810746e..9dee6b31 100644 --- a/src/dbzero/object_model/tuple/Tuple.cpp +++ b/src/dbzero/object_model/tuple/Tuple.cpp @@ -72,7 +72,7 @@ namespace db0::object_model } auto [storage_class, value] = getData()->items()[i]; auto fixture = this->getFixture(); - return unloadMember(fixture, storage_class, value); + return unloadMember(fixture, storage_class, value, 0, this->getMemberFlags()); } void Tuple::setItem(FixtureLock &fixture, std::size_t i, ObjectSharedPtr lang_value) @@ -103,7 +103,7 @@ namespace db0::object_model auto fixture = this->getFixture(); for (auto &elem: this->const_ref().items()) { auto [elem_storage_class, elem_value] = elem; - if (unloadMember(fixture, elem_storage_class, elem_value) == lang_value) { + if (unloadMember(fixture, elem_storage_class, elem_value, 0, this->getMemberFlags()) == lang_value) { count += 1; } } @@ -116,7 +116,7 @@ namespace db0::object_model auto fixture = this->getFixture(); for (auto &elem: this->const_ref().items()){ auto [elem_storage_class, elem_value] = elem; - if (unloadMember(fixture, elem_storage_class, elem_value) == lang_value) { + if (unloadMember(fixture, elem_storage_class, elem_value, 0, this->getMemberFlags()) == lang_value) { return index; } index += 1; diff --git a/src/dbzero/object_model/tuple/TupleIterator.cpp b/src/dbzero/object_model/tuple/TupleIterator.cpp index 9e3b7056..58e48217 100644 --- a/src/dbzero/object_model/tuple/TupleIterator.cpp +++ b/src/dbzero/object_model/tuple/TupleIterator.cpp @@ -15,7 +15,7 @@ namespace db0::object_model auto [storage_class, value] = *m_iterator; ++m_iterator; auto fixture = m_collection->getFixture(); - return unloadMember(fixture, storage_class, value); + return unloadMember(fixture, storage_class, value, 0, m_member_flags); } bool TupleIterator::is_end() const { diff --git a/src/dbzero/object_model/value/Member.cpp b/src/dbzero/object_model/value/Member.cpp index 4ff265f1..fa632694 100644 --- a/src/dbzero/object_model/value/Member.cpp +++ b/src/dbzero/object_model/value/Member.cpp @@ -361,9 +361,9 @@ namespace db0::object_model // STRING_REF specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags access_mode) { - db0::v_object string_ref(fixture->myPtr(value.asAddress())); + db0::v_object string_ref(fixture->myPtr(value.asAddress()), access_mode); auto str_ptr = string_ref->get(); auto result = Py_OWN(PyUnicode_FromStringAndSize(str_ptr.get_raw(), str_ptr.size())); if (!result) { @@ -374,59 +374,59 @@ namespace db0::object_model // INT64 specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { return Py_OWN(PyLong_FromLongLong(value.cast())); } // FLOAT specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { return Py_OWN(PyFloat_FromDouble(value.cast())); } // OBJECT_REF specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { auto &class_factory = fixture->template get(); return PyToolkit::unloadObject(fixture, value.asAddress(), class_factory); } - + // DB0_LIST specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags access_mode) { - return PyToolkit::unloadList(fixture, value.asAddress()); + return PyToolkit::unloadList(fixture, value.asAddress(), 0, access_mode); } // DB0_INDEX specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags access_mode) { - return PyToolkit::unloadIndex(fixture, value.asAddress()); + return PyToolkit::unloadIndex(fixture, value.asAddress(), 0, access_mode); } // DB0_SET specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags access_mode) { - return PyToolkit::unloadSet(fixture, value.asAddress()); + return PyToolkit::unloadSet(fixture, value.asAddress(), 0, access_mode); } // DB0_DICT specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags access_mode) { - return PyToolkit::unloadDict(fixture, value.asAddress()); + return PyToolkit::unloadDict(fixture, value.asAddress(), 0, access_mode); } // BYTES specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags access_mode) { - db0::v_object bytes = fixture->myPtr(value.asAddress()); + db0::v_object bytes(fixture->myPtr(value.asAddress()), access_mode); auto bytes_ptr = bytes->getBuffer(); auto result = Py_OWN(PyBytes_FromStringAndSize(reinterpret_cast(bytes_ptr), bytes->size())); if (!result) { @@ -437,7 +437,7 @@ namespace db0::object_model // DATETIME specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { auto result = Py_OWN(db0::python::uint64ToPyDatetime(value.cast())); if (!result) { @@ -448,7 +448,7 @@ namespace db0::object_model // DATETIME with TZ specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { auto result = Py_OWN(db0::python::uint64ToPyDatetimeWithTZ(value.cast())); if (!result) { @@ -459,7 +459,7 @@ namespace db0::object_model // DATE specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { auto result = Py_OWN(db0::python::uint64ToPyDate(value.cast())); if (!result) { @@ -470,7 +470,7 @@ namespace db0::object_model // Time specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { auto result = Py_OWN(db0::python::uint64ToPyTime(value.cast())); if (!result) { @@ -481,7 +481,7 @@ namespace db0::object_model // Time with Timezone specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { auto result = Py_OWN(db0::python::uint64ToPyTimeWithTz(value.cast())); if (!result) { @@ -492,7 +492,7 @@ namespace db0::object_model // DECIMAL specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { auto result = Py_OWN(db0::python::uint64ToPyDecimal(value.cast())); if (!result) { @@ -503,23 +503,23 @@ namespace db0::object_model // NONE specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { Py_RETURN_NONE; } // DB0_TUPLE specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) - { - return PyToolkit::unloadTuple(fixture, value.asAddress()); + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags access_mode) + { + return PyToolkit::unloadTuple(fixture, value.asAddress(), 0, access_mode); } // DB0_SERIALIZED specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags access_mode) { - db0::v_object bytes = fixture->myPtr(value.asAddress()); + db0::v_object bytes(fixture->myPtr(value.asAddress()), access_mode); std::vector buffer; buffer.resize(bytes->size()); std::copy(bytes->getBuffer(), bytes->getBuffer() + bytes->size(), buffer.begin()); @@ -535,7 +535,7 @@ namespace db0::object_model // ENUM value specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { auto &enum_factory = fixture->get(); auto enum_value_uid = EnumValue_UID(value.cast()); @@ -544,7 +544,7 @@ namespace db0::object_model // ENUM value specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { // NOTE: we use common constant encoding (0 = None, 1 = False, 2 = True) return Py_OWN(db0::python::PyBool_fromBool(value.cast() == 2)); @@ -552,14 +552,14 @@ namespace db0::object_model // DB0_BYTES_ARRAY specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags access_mode) { - return PyToolkit::unloadByteArray(fixture, value.asAddress()); + return PyToolkit::unloadByteArray(fixture, value.asAddress(), access_mode); } // OBJECT_WEAK_REF template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { auto address = value.asUniqueAddress(); // NOTE: instance_id not validated since it's a trusted reference @@ -573,7 +573,7 @@ namespace db0::object_model // OBJECT_LONG_WEAK_REF template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { LongWeakRef weak_ref(fixture, value.asAddress()); auto other_fixture = fixture->getWorkspace().getFixture(weak_ref->m_fixture_uuid); @@ -589,7 +589,7 @@ namespace db0::object_model // CLASS specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int) + db0::swine_ptr &fixture, Value value, unsigned int, AccessFlags) { auto &class_factory = fixture->get(); auto class_item = class_factory.getTypeByAddr(value.asUniqueAddress().getAddress()); @@ -599,14 +599,14 @@ namespace db0::object_model // PACK-2 specialization template <> typename PyToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int offset) + db0::swine_ptr &fixture, Value value, unsigned int offset, AccessFlags) { auto val_code = lofi_store<2>::fromValue(value).get(offset); return PyToolkit::getTypeManager().getLangConstant(val_code); } - + template <> void registerUnloadMemberFunctions( - std::vector &, Value, unsigned int)> &functions) + std::vector &, Value, unsigned int, AccessFlags)> &functions) { functions.resize(static_cast(StorageClass::COUNT)); std::fill(functions.begin(), functions.end(), nullptr); diff --git a/src/dbzero/object_model/value/Member.hpp b/src/dbzero/object_model/value/Member.hpp index e7230086..10cc0cfa 100644 --- a/src/dbzero/object_model/value/Member.hpp +++ b/src/dbzero/object_model/value/Member.hpp @@ -52,27 +52,29 @@ namespace db0::object_model } template typename LangToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, o_typed_item typed_item, unsigned int offset = 0) + db0::swine_ptr &fixture, o_typed_item typed_item, unsigned int offset = 0, AccessFlags access_mode = {}) { - return unloadMember(fixture, typed_item.m_storage_class, typed_item.m_value, offset); + return unloadMember( + fixture, typed_item.m_storage_class, typed_item.m_value, offset, access_mode + ); } template typename LangToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, Value value, unsigned int offset = 0); + db0::swine_ptr &fixture, Value value, unsigned int offset = 0, AccessFlags access_mode = {}); // register StorageClass specializations template void registerUnloadMemberFunctions( - std::vector &, Value, unsigned int)> &functions); + std::vector &, Value, unsigned int, AccessFlags)> &functions); /** * @param name optional name (for error reporting only) * @param offset optional offset for lo-fi members (default = 0) */ template typename LangToolkit::ObjectSharedPtr unloadMember( - db0::swine_ptr &fixture, StorageClass storage_class, Value value, unsigned int offset = 0) + db0::swine_ptr &fixture, StorageClass storage_class, Value value, unsigned int offset = 0, AccessFlags access_mode = {}) { // create member function pointer - using UnloadMemberFunc = typename LangToolkit::ObjectSharedPtr (*)(db0::swine_ptr &, Value, unsigned int); + using UnloadMemberFunc = typename LangToolkit::ObjectSharedPtr (*)(db0::swine_ptr &, Value, unsigned int, AccessFlags); static std::vector unload_member_functions; if (unload_member_functions.empty()) { registerUnloadMemberFunctions(unload_member_functions); @@ -80,7 +82,7 @@ namespace db0::object_model assert(static_cast(storage_class) < unload_member_functions.size()); assert(unload_member_functions[static_cast(storage_class)]); - return unload_member_functions[static_cast(storage_class)](fixture, value, offset); + return unload_member_functions[static_cast(storage_class)](fixture, value, offset, access_mode); } // unreference a member (decref / destroy where applicable) From 8f99a1097e552c16ef6dff7f64717b9fa3fa0a21 Mon Sep 17 00:00:00 2001 From: Wojtek Date: Sat, 1 Nov 2025 16:13:26 +0100 Subject: [PATCH 02/19] no-cache flag handling in db0.find --- python_tests/test_memo_no_cache.py | 26 +++++++++++++++++-- .../object_model/tags/ObjectIterable.cpp | 26 ++++++++++++++++--- .../object_model/tags/ObjectIterable.hpp | 7 ++++- .../object_model/tags/ObjectIterator.cpp | 4 +-- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/python_tests/test_memo_no_cache.py b/python_tests/test_memo_no_cache.py index d7b9ed19..056bd096 100644 --- a/python_tests/test_memo_no_cache.py +++ b/python_tests/test_memo_no_cache.py @@ -79,7 +79,6 @@ def test_fetching_no_cache_objects(db0_fixture): # now fetch objects by uuid initial_cache_size = db0.get_cache_stats()["size"] - print(db0.get_cache_stats()) total_len = 0 for id in uuid_list: # NOTE: must fetch with type, otherwise no_cache flag may not be honored @@ -87,7 +86,30 @@ def test_fetching_no_cache_objects(db0_fixture): # this forces data retrieval total_len += len(obj.data) + final_cache_size = db0.get_cache_stats()["size"] + # make sure cache utilization is low + assert abs(final_cache_size - initial_cache_size) < (300 << 10) + + +def test_find_no_cache_objects(db0_fixture): + px_name = db0.get_current_prefix().name + buf = db0.list() + for _ in range(100): + obj = MemoNoCacheClass() + buf.append(obj) + + db0.close() + db0.init(DB0_DIR) + db0.open(px_name, "r") + + # now retrieve objects using db0.find + initial_cache_size = db0.get_cache_stats()["size"] + total_len = 0 + for obj in db0.find(MemoNoCacheClass): + # this forces data retrieval + total_len += len(obj.data) + + assert total_len > 0 final_cache_size = db0.get_cache_stats()["size"] - print(db0.get_cache_stats()) # make sure cache utilization is low assert abs(final_cache_size - initial_cache_size) < (300 << 10) diff --git a/src/dbzero/object_model/tags/ObjectIterable.cpp b/src/dbzero/object_model/tags/ObjectIterable.cpp index 90b940fd..591b582b 100644 --- a/src/dbzero/object_model/tags/ObjectIterable.cpp +++ b/src/dbzero/object_model/tags/ObjectIterable.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -39,7 +40,8 @@ namespace db0::object_model , m_query_observers(std::move(query_observers)) , m_filters(filters) , m_type(type) - , m_lang_type(lang_type) + , m_lang_type(lang_type) + , m_access_mode(getAccessMode(type)) { } @@ -53,6 +55,7 @@ namespace db0::object_model , m_filters(filters) , m_type(type) , m_lang_type(lang_type) + , m_access_mode(getAccessMode(type)) { } @@ -66,13 +69,15 @@ namespace db0::object_model , m_filters(filters) , m_type(type) , m_lang_type(lang_type) + , m_access_mode(getAccessMode(type)) { } ObjectIterable::ObjectIterable(db0::swine_ptr fixture, const ClassFactory &class_factory, - std::unique_ptr &&ft_query_iterator, std::unique_ptr &&sorted_iterator, + std::unique_ptr &&ft_query_iterator, std::unique_ptr &&sorted_iterator, std::shared_ptr factory, std::vector > &&query_observers, - std::vector &&filters, std::shared_ptr type, TypeObjectPtr lang_type, const SliceDef &slice_def) + std::vector &&filters, std::shared_ptr type, TypeObjectPtr lang_type, + const SliceDef &slice_def, AccessFlags access_mode) : m_fixture(fixture) , m_class_factory(class_factory) , m_query_iterator(std::move(ft_query_iterator)) @@ -83,6 +88,7 @@ namespace db0::object_model , m_type(type) , m_lang_type(lang_type) , m_slice_def(slice_def) + , m_access_mode(access_mode) { } @@ -94,6 +100,7 @@ namespace db0::object_model , m_type(other.m_type) , m_lang_type(other.m_lang_type) , m_slice_def(other.m_slice_def) + , m_access_mode(other.m_access_mode) { m_filters.insert(m_filters.end(), filters.begin(), filters.end()); @@ -115,6 +122,7 @@ namespace db0::object_model , m_type(other.m_type) , m_lang_type(other.m_lang_type) , m_slice_def(other.m_slice_def.combineWith(slice_def)) + , m_access_mode(other.m_access_mode) { std::unique_ptr query_iterator; std::unique_ptr sorted_iterator; @@ -138,6 +146,7 @@ namespace db0::object_model , m_type(other.m_type) , m_lang_type(other.m_lang_type) , m_slice_def(other.m_slice_def) + , m_access_mode(other.m_access_mode) { m_filters.insert(m_filters.end(), filters.begin(), filters.end()); } @@ -153,6 +162,7 @@ namespace db0::object_model , m_type(other.m_type) , m_lang_type(other.m_lang_type) , m_slice_def(other.m_slice_def) + , m_access_mode(other.m_access_mode) { m_filters.insert(m_filters.end(), filters.begin(), filters.end()); } @@ -292,7 +302,7 @@ namespace db0::object_model auto &class_factory = fixture_->get(); return std::unique_ptr(new ObjectIterable(fixture_, class_factory, std::move(query_iterator), - std::move(sorted_iterator), factory, {}, {}, nullptr, nullptr, is_sliced ? SliceDef{start, stop, step} : SliceDef{})); + std::move(sorted_iterator), factory, {}, {}, nullptr, nullptr, is_sliced ? SliceDef{start, stop, step} : SliceDef{}, {})); } double ObjectIterable::compareTo(const ObjectIterable &other) const @@ -419,4 +429,12 @@ namespace db0::object_model return true; } + AccessFlags ObjectIterable::getAccessMode(std::shared_ptr type) const + { + if (type) { + return type->isNoCache() ? AccessFlags { AccessOptions::no_cache } : AccessFlags {}; + } + return {}; + } + } diff --git a/src/dbzero/object_model/tags/ObjectIterable.hpp b/src/dbzero/object_model/tags/ObjectIterable.hpp index 9c1af7d2..7d0b82a6 100644 --- a/src/dbzero/object_model/tags/ObjectIterable.hpp +++ b/src/dbzero/object_model/tags/ObjectIterable.hpp @@ -140,16 +140,21 @@ namespace db0::object_model TypeObjectSharedPtr m_lang_type = nullptr; const SliceDef m_slice_def = {}; mutable ObjectSharedPtr m_lang_context; + // object access mode (e.g. no_cache) + const AccessFlags m_access_mode; // iter constructor ObjectIterable(db0::swine_ptr, const ClassFactory &, std::unique_ptr &&, std::unique_ptr &&, std::shared_ptr, std::vector > &&, - std::vector &&filters, std::shared_ptr, TypeObjectPtr lang_type, const SliceDef & = {}); + std::vector &&filters, std::shared_ptr, TypeObjectPtr lang_type, const SliceDef & = {}, + AccessFlags access_mode = {}); // get the base iterator, possibly initialized from the factory const BaseIterator &getBaseIterator(std::unique_ptr &) const; // retrieve pointer to the already initialized iterator BaseIterator *getIteratorPtr() const; + + AccessFlags getAccessMode(std::shared_ptr) const; }; } \ No newline at end of file diff --git a/src/dbzero/object_model/tags/ObjectIterator.cpp b/src/dbzero/object_model/tags/ObjectIterator.cpp index b252e521..dd841477 100644 --- a/src/dbzero/object_model/tags/ObjectIterator.cpp +++ b/src/dbzero/object_model/tags/ObjectIterator.cpp @@ -81,9 +81,9 @@ namespace db0::object_model { // unload as typed if class is known if (m_type) { - return LangToolkit::unloadObject(fixture, address, m_type, m_lang_type.get()); + return LangToolkit::unloadObject(fixture, address, m_type, m_lang_type.get(), m_access_mode); } else { - return LangToolkit::unloadObject(fixture, address, m_class_factory, m_lang_type.get()); + return LangToolkit::unloadObject(fixture, address, m_class_factory, m_lang_type.get(), 0, m_access_mode); } } From d102987c6c39b68a6598cb085036b1898650dc07 Mon Sep 17 00:00:00 2001 From: Wojtek Date: Sat, 1 Nov 2025 20:28:36 +0100 Subject: [PATCH 03/19] db0.find implemented support for by-index / tuple access --- python_tests/test_find.py | 23 +++++- python_tests/test_memo_no_cache.py | 6 +- src/dbzero/bindings/python/PyAPI.cpp | 2 +- src/dbzero/bindings/python/PyInternalAPI.hpp | 3 + src/dbzero/bindings/python/PyTagsAPI.cpp | 27 +++--- src/dbzero/bindings/python/PyTagsAPI.hpp | 2 +- src/dbzero/bindings/python/PyTypeManager.cpp | 4 +- src/dbzero/bindings/python/PyTypeManager.hpp | 4 +- src/dbzero/bindings/python/dbzero.cpp | 2 +- .../bindings/python/iter/PyObjectIterable.cpp | 82 +++++++++++++------ src/dbzero/core/exception/Exceptions.cpp | 5 ++ src/dbzero/core/exception/Exceptions.hpp | 8 ++ src/dbzero/object_model/list/List.cpp | 6 +- .../object_model/tags/ObjectIterable.cpp | 48 ++++++++++- .../object_model/tags/ObjectIterable.hpp | 11 ++- .../object_model/tags/ObjectIterator.cpp | 18 +++- .../object_model/tags/ObjectIterator.hpp | 9 +- 17 files changed, 198 insertions(+), 62 deletions(-) diff --git a/python_tests/test_find.py b/python_tests/test_find.py index 42308bbf..5ca8291c 100644 --- a/python_tests/test_find.py +++ b/python_tests/test_find.py @@ -4,6 +4,7 @@ from .conftest import DB0_DIR, DATA_PX import itertools from datetime import datetime +import operator @db0.memo() @@ -354,4 +355,24 @@ def test_find_with_scope_defined(db0_fixture): # use scoped find to find instances from different prefixes assert [x.value for x in db0.find(MemoScopedClass, "tag1", prefix = px_name)] == [123] assert [x.value for x in db0.find(MemoScopedClass, "tag1", prefix = other_px_name)] == [456] - \ No newline at end of file + + +def test_find_extract_multiple_indices(db0_no_autocommit, memo_tags): + # pick elements by specific indexes + result = db0.find("tag1")[3, 6, 7] + assert type(result) is tuple + assert len(result) == 3 + + +def test_find_extract_duplicate_indices(db0_no_autocommit, memo_tags): + # pick elements by specific indexes + result = db0.find("tag1")[3, 6, 7, 3, 6] + assert result[0] is result[3] + assert result[1] is result[4] + + +def test_find_extract_invalid_indices(db0_no_autocommit, memo_tags): + query = db0.find("tag1") + assert len(query) == 10 + with pytest.raises(IndexError): + _ = db0.find("tag1")[3, 6, 7, 3, 10] diff --git a/python_tests/test_memo_no_cache.py b/python_tests/test_memo_no_cache.py index 056bd096..91312ccc 100644 --- a/python_tests/test_memo_no_cache.py +++ b/python_tests/test_memo_no_cache.py @@ -18,8 +18,10 @@ def rand_string(max_len): @dataclass class MemoNoCacheClass: data: str - + value: int = 0 + def __init__(self): + self.value = random.randint(0, 1000000) self.data = rand_string(12 << 10) @@ -106,7 +108,7 @@ def test_find_no_cache_objects(db0_fixture): initial_cache_size = db0.get_cache_stats()["size"] total_len = 0 for obj in db0.find(MemoNoCacheClass): - # this forces data retrieval + # this forces data retrieval (but not caching) total_len += len(obj.data) assert total_len > 0 diff --git a/src/dbzero/bindings/python/PyAPI.cpp b/src/dbzero/bindings/python/PyAPI.cpp index 8b65fde1..1f0d67bc 100644 --- a/src/dbzero/bindings/python/PyAPI.cpp +++ b/src/dbzero/bindings/python/PyAPI.cpp @@ -75,7 +75,7 @@ namespace db0::python PY_API_FUNC return runSafe(tryGetCacheStats); } - + PyObject *tryGetLangCacheStats() { auto lang_cache = PyToolkit::getPyWorkspace().getWorkspace().getLangCache(); diff --git a/src/dbzero/bindings/python/PyInternalAPI.hpp b/src/dbzero/bindings/python/PyInternalAPI.hpp index b338ec84..ddebb5a1 100644 --- a/src/dbzero/bindings/python/PyInternalAPI.hpp +++ b/src/dbzero/bindings/python/PyInternalAPI.hpp @@ -117,6 +117,9 @@ namespace db0::python } catch (const db0::ClassNotFoundException &e) { PyErr_SetString(PyToolkit::getTypeManager().getClassNotFoundError(), e.what()); return returnError(); + } catch (const db0::IndexException &e) { + PyErr_SetString(PyExc_IndexError, e.what()); + return returnError(); } #if ENABLE_DEBUG_EXCEPTIONS catch (const db0::AbstractException &e) { diff --git a/src/dbzero/bindings/python/PyTagsAPI.cpp b/src/dbzero/bindings/python/PyTagsAPI.cpp index 6f00ba94..d7c290c1 100644 --- a/src/dbzero/bindings/python/PyTagsAPI.cpp +++ b/src/dbzero/bindings/python/PyTagsAPI.cpp @@ -58,19 +58,8 @@ namespace db0::python return { std::move(result.first), std::move(query_observers) }; } - PyObject *trySplitBy(PyObject *args, PyObject *kwargs) + PyObject *trySplitBy(PyObject *py_tags, PyObject *py_query, bool exclusive) { - // extract 2 object arguments - PyObject *py_tags = nullptr; - PyObject *py_query = nullptr; - int exclusive = true; - // tags, query, exclusive (bool) - static const char *kwlist[] = {"tags", "query", "exclusive", NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|p", const_cast(kwlist), &py_tags, &py_query, &exclusive)) { - PyErr_SetString(PyExc_TypeError, "Invalid argument type"); - return NULL; - } - if (!PyObjectIterable_Check(py_query)) { THROWF(db0::InputException) << "Invalid argument type"; } @@ -84,9 +73,19 @@ namespace db0::python PyObject *PyAPI_splitBy(PyObject *, PyObject *args, PyObject *kwargs) { + // extract 2 object arguments + PyObject *py_tags = nullptr; + PyObject *py_query = nullptr; + int exclusive = true; + // tags, query, exclusive (bool) + static const char *kwlist[] = {"tags", "query", "exclusive", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|p", const_cast(kwlist), &py_tags, &py_query, &exclusive)) { + return NULL; + } + PY_API_FUNC - return runSafe(trySplitBy, args, kwargs); - } + return runSafe(trySplitBy, py_tags, py_query, exclusive); + } PyObject *trySelectModCandidates(const ObjectIterable &iterable, StateNumType from_state, std::optional to_state) diff --git a/src/dbzero/bindings/python/PyTagsAPI.hpp b/src/dbzero/bindings/python/PyTagsAPI.hpp index 357a43b5..2d1b80ac 100644 --- a/src/dbzero/bindings/python/PyTagsAPI.hpp +++ b/src/dbzero/bindings/python/PyTagsAPI.hpp @@ -43,7 +43,7 @@ namespace db0::python const char *prefix_name = nullptr); PyObject *PyAPI_splitBy(PyObject *, PyObject *args, PyObject *kwargs); - + PyObject *PyAPI_selectModCandidates(PyObject *, PyObject *args, PyObject *kwargs); PyObject *PyAPI_splitBySnapshots(PyObject *, PyObject *const *args, Py_ssize_t nargs); diff --git a/src/dbzero/bindings/python/PyTypeManager.cpp b/src/dbzero/bindings/python/PyTypeManager.cpp index b21a7318..703828cc 100644 --- a/src/dbzero/bindings/python/PyTypeManager.cpp +++ b/src/dbzero/bindings/python/PyTypeManager.cpp @@ -108,7 +108,7 @@ namespace db0::python } m_py_bad_prefix_error.steal(); m_py_class_not_found_error.steal(); - m_py_reference_error.steal(); + m_py_reference_error.steal(); } } @@ -487,7 +487,7 @@ namespace db0::python PyTypeManager::ObjectPtr PyTypeManager::getReferenceError() const { return m_py_reference_error.get(); } - + std::shared_ptr PyTypeManager::extractConstClass(ObjectPtr py_class) const { if (!PyClassObject_Check(py_class)) { diff --git a/src/dbzero/bindings/python/PyTypeManager.hpp b/src/dbzero/bindings/python/PyTypeManager.hpp index a2b5a7cf..ba8473c8 100644 --- a/src/dbzero/bindings/python/PyTypeManager.hpp +++ b/src/dbzero/bindings/python/PyTypeManager.hpp @@ -124,7 +124,7 @@ namespace db0::python ObjectPtr getBadPrefixError() const; ObjectPtr getClassNotFoundError() const; - ObjectPtr getReferenceError() const; + ObjectPtr getReferenceError() const; /** * Called with each new memo type @@ -186,7 +186,7 @@ namespace db0::python // error associated with missing / invalid type accessed (e.g. missing import) mutable ObjectSharedPtr m_py_class_not_found_error; // invalid reference error - e.g. UUID or weak proxy expired - mutable ObjectSharedPtr m_py_reference_error; + mutable ObjectSharedPtr m_py_reference_error; // identified reference to a MemoBase type TypeObjectPtr m_memo_base_type = nullptr; std::unordered_set m_dbzero_type_ids; diff --git a/src/dbzero/bindings/python/dbzero.cpp b/src/dbzero/bindings/python/dbzero.cpp index 1bd22f4f..410b2377 100644 --- a/src/dbzero/bindings/python/dbzero.cpp +++ b/src/dbzero/bindings/python/dbzero.cpp @@ -67,7 +67,7 @@ static PyMethodDef dbzero_methods[] = {"serialize", (PyCFunction)&py::PyAPI_serialize, METH_FASTCALL, "Serialize dbzero serializable instance"}, {"deserialize", (PyCFunction)&py::PyAPI_deserialize, METH_FASTCALL, "Serialize dbzero serializable instance"}, {"is_enum_value", (PyCFunction)&py::PyAPI_isEnumValue, METH_FASTCALL, "Check if parameter represents a dbzero enum value"}, - {"split_by", (PyCFunction)&py::PyAPI_splitBy, METH_VARARGS | METH_KEYWORDS, "Split query iterator by a given criteria"}, + {"split_by", (PyCFunction)&py::PyAPI_splitBy, METH_VARARGS | METH_KEYWORDS, "Split query iterator by a given criteria"}, {"filter", (PyCFunction)&py::filter, METH_VARARGS | METH_KEYWORDS, "Filter with a Python callable"}, {"set_prefix", (PyCFunction)&py::PyAPI_setPrefix, METH_VARARGS | METH_KEYWORDS, "Allows dynamically specifying object's prefix during initialization"}, {"get_slab_metrics", (PyCFunction)&py::getSlabMetrics, METH_NOARGS, "Retrieve slab metrics of the current prefix"}, diff --git a/src/dbzero/bindings/python/iter/PyObjectIterable.cpp b/src/dbzero/bindings/python/iter/PyObjectIterable.cpp index e69ff4a6..91a2e88f 100644 --- a/src/dbzero/bindings/python/iter/PyObjectIterable.cpp +++ b/src/dbzero/bindings/python/iter/PyObjectIterable.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace db0::python @@ -127,33 +128,66 @@ namespace db0::python } } - PyObject *tryPyObjectIterable_GetItemSlice(PyObjectIterable *py_iterable, PyObject *py_slice) + std::vector unpackTuple(PyObject *py_tuple) + { + if (!PyTuple_Check(py_tuple)) { + THROWF(db0::InputException) << "Expected a tuple of indexes"; + } + Py_ssize_t num_items = PyTuple_Size(py_tuple); + std::vector result; + result.reserve(num_items); + for (Py_ssize_t i = 0; i < num_items; ++i) { + PyObject *py_item = PyTuple_GetItem(py_tuple, i); + if (!PyLong_Check(py_item)) { + THROWF(db0::InputException) << "Expected integer indexes in the tuple"; + } + result.push_back(PyLong_AsUnsignedLongLong(py_item)); + } + return result; + } + + PyObject *tryPyObjectIterable_GetItemSlice(PyObjectIterable *py_iterable, PyObject *py_key) { using SliceDef = db0::object_model::SliceDef; - Py_ssize_t start, stop, step; - auto size_func = [&](std::optional &size) { - if (!size) { - size = py_iterable->ext().getSize(); + using ObjectSharedPtr = PyToolkit::ObjectSharedPtr; + + if (PyTuple_Check(py_key)) { + // itemgetter's key (item indexes) + auto indices = unpackTuple(py_key); + auto py_result = Py_OWN(PyTuple_New(indices.size())); + db0::object_model::getItemsByIndices(py_iterable->ext(), indices, + [&](unsigned int ord, ObjectSharedPtr obj_ptr) { + PySafeTuple_SetItem(*py_result, ord, obj_ptr); + }); + return py_result.steal(); + } else if (PySlice_Check(py_key)) { + Py_ssize_t start, stop, step; + auto size_func = [&](std::optional &size) { + if (!size) { + size = py_iterable->ext().getSize(); + } + return *size; + }; + + PySlice_GetUnboundIndices(py_key, size_func, start, stop, step); + std::size_t _stop = (stop == PY_SSIZE_T_MAX) ? SliceDef::MAX_STOP() : (std::size_t)stop; + auto slice_def = SliceDef { (std::size_t)start, _stop, (int)step }; + // the default slice just returns itself + if (slice_def.isDefault()) { + Py_INCREF(py_iterable); + return py_iterable; + } + + if (py_iterable->ext().isSliced()) { + THROWF(db0::InputException) << "Cannot slice an already sliced iterable (Operation not supported)"; } - return *size; - }; - - PySlice_GetUnboundIndices(py_slice, size_func, start, stop, step); - std::size_t _stop = (stop == PY_SSIZE_T_MAX) ? SliceDef::MAX_STOP() : (std::size_t)stop; - auto slice_def = SliceDef { (std::size_t)start, _stop, (int)step }; - // the default slice just returns itself - if (slice_def.isDefault()) { - Py_INCREF(py_iterable); - return py_iterable; - } - if (py_iterable->ext().isSliced()) { - THROWF(db0::InputException) << "Cannot slice an already sliced iterable (Operation not supported)"; - } - - auto py_result = PyObjectIterableDefault_new(); - py_result->makeNew(py_iterable->ext(), slice_def); - return py_result.steal(); + auto py_result = PyObjectIterableDefault_new(); + py_result->makeNew(py_iterable->ext(), slice_def); + return py_result.steal(); + } + THROWF(db0::InputException) + << "Invalid subscript type for ObjectIterable: " << Py_TYPE(py_key)->tp_name << THROWF_END; } PyObject *PyAPI_PyObjectIterable_GetItemSlice(PyObjectIterable *py_iterable, PyObject *py_elem) @@ -180,7 +214,7 @@ namespace db0::python static PyMethodDef PyObjectIterable_methods[] = { {"compare", (PyCFunction)PyAPI_PyObjectIterable_compare, METH_FASTCALL, "Compare two iterables"}, - {"signature", (PyCFunction)PyAPI_PyObjectIterable_signature, METH_NOARGS, "Get the signature of the query"}, + {"signature", (PyCFunction)PyAPI_PyObjectIterable_signature, METH_NOARGS, "Get the signature of the query"}, {NULL} }; diff --git a/src/dbzero/core/exception/Exceptions.cpp b/src/dbzero/core/exception/Exceptions.cpp index c587e645..9f5b33ec 100644 --- a/src/dbzero/core/exception/Exceptions.cpp +++ b/src/dbzero/core/exception/Exceptions.cpp @@ -58,5 +58,10 @@ namespace db0 : CriticalException(exception_id) { } + + IndexException::IndexException() + : CriticalException(exception_id) + { + } } \ No newline at end of file diff --git a/src/dbzero/core/exception/Exceptions.hpp b/src/dbzero/core/exception/Exceptions.hpp index 37e17f22..250f5c43 100644 --- a/src/dbzero/core/exception/Exceptions.hpp +++ b/src/dbzero/core/exception/Exceptions.hpp @@ -92,6 +92,14 @@ namespace db0 ClassNotFoundException(); }; + + class IndexException: public CriticalException + { + public: + static constexpr int exception_id = EXCEPTION_ID_PREFIX::BASIC | 0x0e; + + IndexException(); + }; class AccessTypeException: public CriticalException { diff --git a/src/dbzero/object_model/list/List.cpp b/src/dbzero/object_model/list/List.cpp index b036ab69..06764cad 100644 --- a/src/dbzero/object_model/list/List.cpp +++ b/src/dbzero/object_model/list/List.cpp @@ -68,7 +68,7 @@ namespace db0::object_model List::ObjectSharedPtr List::getItem(std::size_t i) const { if (i >= size()) { - THROWF(db0::InputException) << "Index out of range: " << i; + THROWF(db0::IndexException) << "Index out of range: " << i; } auto [storage_class, value] = (*this)[i]; auto fixture = this->getFixture(); @@ -81,7 +81,7 @@ namespace db0::object_model THROWF(db0::InputException) << "Cannot pop from empty container "; } if (i >= size()) { - THROWF(db0::InputException) << "Index out of range: " << i; + THROWF(db0::IndexException) << "Index out of range: " << i; } auto [storage_class, value] = (*this)[i]; auto member = unloadMember(*fixture, storage_class, value, 0, this->getMemberFlags()); @@ -93,7 +93,7 @@ namespace db0::object_model void List::setItem(FixtureLock &fixture, std::size_t i, ObjectPtr lang_value) { if (i >= size()) { - THROWF(db0::InputException) << "Index out of range: " << i; + THROWF(db0::IndexException) << "Index out of range: " << i; } // recognize type ID from language specific object diff --git a/src/dbzero/object_model/tags/ObjectIterable.cpp b/src/dbzero/object_model/tags/ObjectIterable.cpp index 591b582b..d03b31bd 100644 --- a/src/dbzero/object_model/tags/ObjectIterable.cpp +++ b/src/dbzero/object_model/tags/ObjectIterable.cpp @@ -30,7 +30,7 @@ namespace db0::object_model } return std::move(query_iterator); } - + ObjectIterable::ObjectIterable(db0::swine_ptr fixture, std::unique_ptr &&ft_query_iterator, std::shared_ptr type, TypeObjectPtr lang_type, std::vector > &&query_observers, const std::vector &filters) @@ -133,8 +133,8 @@ namespace db0::object_model m_sorted_iterator = other.m_sorted_iterator->beginSorted(); } } - - ObjectIterable::ObjectIterable(const ObjectIterable &other, std::unique_ptr &&sorted_iterator, + + ObjectIterable::ObjectIterable(const ObjectIterable &other, std::unique_ptr &&sorted_iterator, std::vector > &&query_observers, const std::vector &filters) : m_fixture(other.m_fixture) , m_class_factory(other.m_class_factory) @@ -436,5 +436,47 @@ namespace db0::object_model } return {}; } + + void getItemsByIndices(const ObjectIterable &iterable, const std::vector &indices, + std::function callback) + { + std::vector > sorted_indices; + sorted_indices.reserve(indices.size()); + for (unsigned int ord = 0; ord < indices.size(); ++ord) { + sorted_indices.emplace_back(indices[ord], ord); + } + std::sort(sorted_indices.begin(), sorted_indices.end(), + [](const auto &lhs, const auto &rhs) { + return lhs.first < rhs.first; + }); + + // access items in sorted order, populating the callback + auto iter = iterable.iter(); + std::uint64_t current_index = 0; + ObjectIterable::ObjectSharedPtr last_item; + for (const auto &[index, ord] : sorted_indices) { + if (current_index > index) { + // duplicate item + assert(last_item.get()); + callback(ord, last_item); + continue; + } + + if (current_index < index) { + auto to_skip = index - current_index; + if (iter->skip(to_skip) < to_skip) { + THROWF(db0::IndexException) << "Index " << index << " out of range"; + } + current_index = index; + } + last_item = iter->next(); + if (!last_item) { + THROWF(db0::IndexException) << "Index " << index << " out of range"; + } + ++current_index; + callback(ord, last_item); + } + } + } diff --git a/src/dbzero/object_model/tags/ObjectIterable.hpp b/src/dbzero/object_model/tags/ObjectIterable.hpp index 7d0b82a6..64e20d1e 100644 --- a/src/dbzero/object_model/tags/ObjectIterable.hpp +++ b/src/dbzero/object_model/tags/ObjectIterable.hpp @@ -1,6 +1,9 @@ #pragma once #include +#include +#include +#include #include #include #include @@ -64,7 +67,7 @@ namespace db0::object_model // Construct sliced ObjectIterable(const ObjectIterable &, const SliceDef &); - + // Construct sorted ObjectIterable(const ObjectIterable &, std::unique_ptr &&, std::vector > && = {}, const std::vector & = {}); @@ -138,7 +141,7 @@ namespace db0::object_model std::vector m_filters; std::shared_ptr m_type = nullptr; TypeObjectSharedPtr m_lang_type = nullptr; - const SliceDef m_slice_def = {}; + const SliceDef m_slice_def = {}; mutable ObjectSharedPtr m_lang_context; // object access mode (e.g. no_cache) const AccessFlags m_access_mode; @@ -157,4 +160,8 @@ namespace db0::object_model AccessFlags getAccessMode(std::shared_ptr) const; }; + // Retrieve items by specific indices from the iterable + void getItemsByIndices(const ObjectIterable &, const std::vector &, + std::function); + } \ No newline at end of file diff --git a/src/dbzero/object_model/tags/ObjectIterator.cpp b/src/dbzero/object_model/tags/ObjectIterator.cpp index dd841477..4ed452a9 100644 --- a/src/dbzero/object_model/tags/ObjectIterator.cpp +++ b/src/dbzero/object_model/tags/ObjectIterator.cpp @@ -21,14 +21,14 @@ namespace db0::object_model , m_slice(m_iterator_ptr, slice_def) { } - + ObjectIterator::ObjectIterator(db0::swine_ptr fixture, std::unique_ptr &&sorted_iterator, std::shared_ptr type, TypeObjectPtr lang_type, std::vector > &&query_observers, const std::vector &filters, const SliceDef &slice_def) : ObjectIterable(fixture, std::move(sorted_iterator), type, lang_type, {}, filters) , m_iterator_ptr(m_sorted_iterator.get()) , m_decoration(std::move(query_observers)) - , m_slice(m_iterator_ptr, slice_def) + , m_slice(m_iterator_ptr, slice_def) { } @@ -36,7 +36,7 @@ namespace db0::object_model : ObjectIterable(other, filters) , m_iterator_ptr(getIteratorPtr()) , m_decoration(std::move(m_query_observers)) - , m_slice(m_iterator_ptr, m_slice_def) + , m_slice(m_iterator_ptr, m_slice_def) { } @@ -50,7 +50,7 @@ namespace db0::object_model { for (;;) { if (!m_slice.isEnd()) { - // collect decorators if any exist + // Collect decorators if any exist if (!m_decoration.empty()) { auto it = m_decoration.m_query_observers.begin(); for (auto &decor: m_decoration.m_decorators) { @@ -77,6 +77,16 @@ namespace db0::object_model } } + std::size_t ObjectIterator::skip(std::size_t count) + { + std::size_t skipped = 0; + while (skipped < count && !m_slice.isEnd()) { + m_slice.next(); + ++skipped; + } + return skipped; + } + ObjectIterator::ObjectSharedPtr ObjectIterator::unload(db0::swine_ptr &fixture, Address address) const { // unload as typed if class is known diff --git a/src/dbzero/object_model/tags/ObjectIterator.hpp b/src/dbzero/object_model/tags/ObjectIterator.hpp index b99dc89e..99de257d 100644 --- a/src/dbzero/object_model/tags/ObjectIterator.hpp +++ b/src/dbzero/object_model/tags/ObjectIterator.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -41,7 +42,7 @@ namespace db0::object_model // Construct iterator with additional filters ObjectIterator(const ObjectIterable &, const std::vector & = {}); - + virtual ~ObjectIterator() = default; /** @@ -58,6 +59,10 @@ namespace db0::object_model return m_decoration.m_decorators; } + // Try to skip specified number of items + // @return number of actually skipped items + std::size_t skip(std::size_t count); + protected: friend class ObjectIterable; // iterator_ptr valid both in case of m_query_iterator and m_sorted_iterator @@ -81,7 +86,7 @@ namespace db0::object_model }; Decoration m_decoration; - Slice m_slice; + Slice m_slice; // Unload object by address (must be from this iterator) skipping instance ID validation virtual ObjectSharedPtr unload(Address) const; From 4341ac6bf4a1e9843f729014e17e41afc07f28ae Mon Sep 17 00:00:00 2001 From: Wojtek Date: Mon, 3 Nov 2025 20:24:26 +0100 Subject: [PATCH 04/19] WIP: refactor --- src/dbzero/object_model/object/Object.cpp | 1619 +--------------- src/dbzero/object_model/object/Object.hpp | 372 +--- .../object_model/object/ObjectImplBase.cpp | 1653 +++++++++++++++++ .../object_model/object/ObjectImplBase.hpp | 343 ++++ src/dbzero/object_model/object/o_object.cpp | 76 + src/dbzero/object_model/object/o_object.hpp | 75 + 6 files changed, 2156 insertions(+), 1982 deletions(-) create mode 100644 src/dbzero/object_model/object/ObjectImplBase.cpp create mode 100644 src/dbzero/object_model/object/ObjectImplBase.hpp create mode 100644 src/dbzero/object_model/object/o_object.cpp create mode 100644 src/dbzero/object_model/object/o_object.hpp diff --git a/src/dbzero/object_model/object/Object.cpp b/src/dbzero/object_model/object/Object.cpp index 59025bd9..dadaf2f2 100644 --- a/src/dbzero/object_model/object/Object.cpp +++ b/src/dbzero/object_model/object/Object.cpp @@ -1,1626 +1,9 @@ #include "Object.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -DEFINE_ENUM_VALUES(db0::object_model::ObjectOptions, "DROPPED", "DEFUNCT") namespace db0::object_model { - GC0_Define(Object) - ObjectInitializerManager Object::m_init_manager; - - FlagSet getAccessOptions(const Class &type) { - return type.isNoCache() ? FlagSet { AccessOptions::no_cache } : FlagSet {}; - } - - bool isEqual(const KV_Index *kv_ptr_1, const KV_Index *kv_ptr_2) - { - if (!kv_ptr_1) { - return !kv_ptr_2; - } - - if (!kv_ptr_2) { - return false; - } - - // item-wise comparison - return *kv_ptr_1 == *kv_ptr_2; - } - - template IntT safeCast(unsigned int value, const char *err_msg) - { - if (value > std::numeric_limits::max()) { - THROWF(db0::InputException) << err_msg; - } - return static_cast(value); - } - - o_object::o_object(std::uint32_t class_ref, std::pair ref_counts, - std::uint8_t num_type_tags, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, - const XValue *index_vt_begin, const XValue *index_vt_end) - : m_header(ref_counts) - , m_num_type_tags(num_type_tags) - { - arrangeMembers() - (PosVT::type(), pos_vt_data, pos_vt_offset) - (packed_int32::type(), class_ref) - (IndexVT::type(), index_vt_begin, index_vt_end); - } - - std::size_t o_object::measure(std::uint32_t class_ref, std::pair, std::uint8_t, - const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, - const XValue *index_vt_begin, const XValue *index_vt_end) - { - return super_t::measureMembers() - (PosVT::type(), pos_vt_data, pos_vt_offset) - (packed_int32::type(), class_ref) - (IndexVT::type(), index_vt_begin, index_vt_end); - } - - const PosVT &o_object::pos_vt() const { - return getDynFirst(PosVT::type()); - } - - PosVT &o_object::pos_vt() { - return getDynFirst(PosVT::type()); - } - - const packed_int32 &o_object::classRef() const { - return getDynAfter(pos_vt(), packed_int32::type()); - } - - std::uint32_t o_object::getClassRef() const { - return classRef().value(); - } - - const IndexVT &o_object::index_vt() const { - return getDynAfter(classRef(), IndexVT::type()); - } - - IndexVT &o_object::index_vt() { - return getDynAfter(classRef(), IndexVT::type()); - } - - void o_object::incRef(bool is_tag) { - m_header.incRef(is_tag); - } - - bool o_object::hasRefs() const - { - // NOTE: type tags are not counted as "proper" references - if (m_header.m_ref_counter.getFirst() > this->m_num_type_tags) { - return true; - } - return m_header.m_ref_counter.getSecond() > 0; - } - - bool o_object::hasAnyRefs() const { - return m_header.hasRefs(); - } - - Object::Object(UniqueAddress addr, unsigned int ext_refs) - : m_flags { ObjectOptions::DROPPED } - , m_ext_refs(ext_refs) - , m_unique_address(addr) - { - } - - Object::Object(std::shared_ptr db0_class) - { - // prepare for initialization - m_init_manager.addInitializer(*this, db0_class); - } - - Object::Object(TypeInitializer &&type_initializer) - { - // prepare for initialization - m_init_manager.addInitializer(*this, std::move(type_initializer)); - } - - Object::Object(db0::swine_ptr &fixture, std::shared_ptr type, - std::pair ref_counts, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset) - : super_t(fixture, type->getClassRef(), ref_counts, - safeCast(type->getNumBases() + 1, "Too many base classes"), pos_vt_data, pos_vt_offset, nullptr, nullptr, - getAccessOptions(*type)) - , m_type(type) - { - } - - Object::Object(db0::swine_ptr &fixture, Address address, AccessFlags access_mode) - : super_t(super_t::tag_from_address(), fixture, address, access_mode) - { - } - - Object::Object(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type) - : super_t(super_t::tag_from_stem(), fixture, std::move(stem)) - , m_type(type) - { - assert(hasValidClassRef()); - } - - Object::Object(db0::swine_ptr &fixture, Address address, std::shared_ptr type_hint, with_type_hint, AccessFlags access_mode) - : Object(fixture, address, access_mode) - { - assert(*fixture == *type_hint->getFixture()); - setTypeWithHint(type_hint); - } - - Object::Object(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type_hint, with_type_hint) - : Object(fixture, std::move(stem), getTypeWithHint(*fixture, stem->getClassRef(), type_hint)) - { - } - - Object::~Object() - { - // unregister needs to be called before destruction of members - unregister(); - if (!hasInstance()) { - // release initializer if it exists, object not created - m_init_manager.tryCloseInitializer(*this); - } - } - - void Object::dropInstance(FixtureLock &) - { - auto unique_addr = this->getUniqueAddress(); - auto ext_refs = this->getExtRefs(); - this->~Object(); - // construct a null placeholder - new ((void*)this) Object(unique_addr, ext_refs); - } - - Object::ObjectStem Object::tryUnloadStem(db0::swine_ptr &fixture, Address address, - std::uint16_t instance_id, AccessFlags access_mode) - { - std::size_t size_of; - if (!fixture->isAddressValid(address, REALM_ID, &size_of)) { - return {}; - } - // Unload from a verified address - ObjectVType stem(db0::tag_verified(), fixture->myPtr(address), size_of, access_mode); - if (instance_id && stem->m_header.m_instance_id != instance_id) { - // instance ID validation failed - return {}; - } - return stem; - } - - Object::ObjectStem Object::unloadStem(db0::swine_ptr &fixture, Address address, - std::uint16_t instance_id, AccessFlags access_mode) - { - auto result = tryUnloadStem(fixture, address, instance_id, access_mode); - if (!result) { - THROWF(db0::InputException) << "Invalid UUID or object has been deleted"; - } - return result; - } - - void Object::postInit(FixtureLock &fixture) - { - if (!hasInstance()) { - auto &initializer = m_init_manager.getInitializer(*this); - PosVT::Data pos_vt_data; - unsigned int pos_vt_offset = 0; - auto index_vt_data = initializer.getData(pos_vt_data, pos_vt_offset); - - // place object in the same fixture as its class - // construct the dbzero instance & assign to self - m_type = initializer.getClassPtr(); - assert(m_type); - super_t::init(*fixture, m_type->getClassRef(), initializer.getRefCounts(), - safeCast(m_type->getNumBases() + 1, "Too many base classes"), - pos_vt_data, pos_vt_offset, index_vt_data.first, index_vt_data.second, - getAccessOptions(*m_type) - ); - - // reference associated class - m_type->incRef(false); - m_type->updateSchema(pos_vt_offset, pos_vt_data.m_types, pos_vt_data.m_values); - m_type->updateSchema(index_vt_data.first, index_vt_data.second); - - // bind singleton address (now that instance exists) - if (m_type->isSingleton()) { - m_type->setSingletonAddress(*this); - } - initializer.close(); - } - - assert(hasInstance()); - } - - std::pair Object::recognizeType(Fixture &fixture, ObjectPtr lang_value) const - { - auto type_id = LangToolkit::getTypeManager().getTypeId(lang_value); - // NOTE: allow storage as PACK_2 - auto pre_storage_class = TypeUtils::m_storage_class_mapper.getPreStorageClass(type_id, true); - if (type_id == TypeId::MEMO_OBJECT) { - // object reference must be from the same fixture - auto &obj = LangToolkit::getTypeManager().extractObject(lang_value); - if (fixture.getUUID() != obj.getFixture()->getUUID()) { - THROWF(db0::InputException) << "Referencing objects from foreign prefixes is not allowed. Use db0.weak_proxy instead"; - } - } - - // may need to refine the storage class (i.e. long weak ref might be needed instead) - StorageClass storage_class; - if (pre_storage_class == PreStorageClass::OBJECT_WEAK_REF) { - storage_class = db0::getStorageClass(pre_storage_class, fixture, lang_value); - } else { - storage_class = db0::getStorageClass(pre_storage_class); - } - - return { type_id, storage_class }; - } - - void Object::removePreInit(const char *field_name) const - { - auto &initializer = m_init_manager.getInitializer(*this); - auto &type = initializer.getClass(); - - // Find an already existing field index - auto member_id = std::get<0>(type.findField(field_name)); - if (!member_id) { - THROWF(db0::InputException) << "Attribute not found: " << field_name; - } - - for (const auto &field_info: member_id) { - assert(field_info.first); - auto loc = field_info.first.getIndexAndOffset(); - // mark as deleted - if (field_info.second == 0) { - initializer.set(loc, StorageClass::DELETED, {}); - } else { - assert(field_info.second == 2 && "Only fidelity == 2 is supported"); - if (member_id.hasFidelity(0)) { - // remove any existing regular initialization - auto loc0 = member_id.get(0).getIndexAndOffset(); - initializer.remove(loc0); - } - initializer.set(loc, StorageClass::PACK_2, Value::DELETED, - lofi_store<2>::mask(loc.second)); - } - } - } - - void Object::setPreInit(const char *field_name, ObjectPtr obj_ptr) const - { - assert(!hasInstance()); - if (!LangToolkit::isValid(obj_ptr)) { - removePreInit(field_name); - return; - } - - auto &initializer = m_init_manager.getInitializer(*this); - auto fixture = initializer.getFixture(); - auto &type = initializer.getClass(); - auto [type_id, storage_class] = recognizeType(*fixture, obj_ptr); - auto storage_fidelity = getStorageFidelity(storage_class); - - // Find an already existing field index - auto [member_id, is_init_var] = type.findField(field_name); - // NOTE: even if a field already exists we might need to extend its supported fidelities - if (!member_id || !member_id.hasFidelity(storage_fidelity)) { - // update class definition - // use the default fidelity for the storage class - member_id = type.addField(field_name, storage_fidelity); - } + GC0_Define(Object) - if (storage_fidelity == 0) { - if (member_id.hasFidelity(2)) { - // remove any existing lo-fi initialization - auto loc = member_id.get(2).getIndexAndOffset(); - initializer.remove(loc, lofi_store<2>::mask(loc.second)); - } - // register a regular member with the initializer - // NOTE: a new member receives the no-cache flag if set (at the type level) - auto member_flags = type.isNoCache() ? AccessFlags { AccessOptions::no_cache } : AccessFlags(); - initializer.set(member_id.get(0).getIndexAndOffset(), storage_class, - createMember(fixture, type_id, storage_class, obj_ptr, member_flags) - ); - } else { - if (member_id.hasFidelity(0)) { - // remove any existing regular initialization - auto loc = member_id.get(0).getIndexAndOffset(); - initializer.remove(loc); - } - // For now only fidelity == 2 is supported (lo-fi storage) - assert(storage_fidelity == 2); - auto loc = member_id.get(storage_fidelity).getIndexAndOffset(); - // no access flags for lo-fi members - auto value = lofi_store<2>::create(loc.second, - createMember(fixture, type_id, storage_class, obj_ptr, {}).m_store); - // register a lo-fi member with the initializer (using mask) - initializer.set(loc, storage_class, value, lofi_store<2>::mask(loc.second)); - } - } - - void Object::remove(FixtureLock &fixture, const char *field_name) - { - assert(hasInstance()); - if (this->span() > 1) { - // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag - // this is because the actual change may be missed if performed on a different-then the 1st DP - _touch(); - } - - assert(m_type); - // Find an already existing field index - auto [member_id, is_init_var] = m_type->findField(field_name); - if (!member_id) { - THROWF(db0::InputException) << "Attribute not found: " << field_name; - } - - unsigned int pos = 0; - auto [field_info, loc_ptr] = tryGetMemberSlot(member_id, pos); - if (!field_info.first) { - THROWF(db0::InputException) << "Attribute not found: " << field_name; - } - - // NOTE: unreference as DELETED - unrefWithLoc(fixture, field_info.first, loc_ptr, pos, StorageClass::DELETED, - field_info.second); - - // the KV-index erase operation must be registered as the potential silent mutation - // but the operation can be avoided if the object is already marked as modified - if (!loc_ptr && !super_t::isModified()) { - this->_touch(); - } - } - - void Object::unrefPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, StorageClass storage_class, - unsigned int fidelity) - { - auto &pos_vt = modify().pos_vt(); - auto old_storage_class = pos_vt.types()[pos]; - if (fidelity == 0) { - unrefMember(*fixture, old_storage_class, pos_vt.values()[pos]); - // mark member as unreferenced by assigning storage class - pos_vt.set(pos, storage_class, {}); - m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(old_storage_class)); - } else { - assert(fidelity == 2); - auto value = pos_vt.values()[pos]; - auto offset = field_id.getOffset(); - if (storage_class != StorageClass::DELETED && !lofi_store<2>::fromValue(value).isSet(offset)) { - // value is already unset - return; - } - - auto old_type_id = getSchemaTypeId(old_storage_class, lofi_store<2>::fromValue(value).get(offset)); - // either reset or mark as deleted - if (storage_class == StorageClass::DELETED) { - lofi_store<2>::fromValue(value).set(offset, Value::DELETED); - } else { - lofi_store<2>::fromValue(value).reset(offset); - } - pos_vt.set(pos, old_storage_class, value); - m_type->removeFromSchema(field_id, fidelity, old_type_id); - } - } - - void Object::unrefIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, - StorageClass storage_class, unsigned int fidelity) - { - auto &index_vt = modify().index_vt(); - auto old_storage_class = index_vt.xvalues()[index_vt_pos].m_type; - if (fidelity == 0) { - unrefMember(*fixture, index_vt.xvalues()[index_vt_pos]); - // mark member as unreferenced by assigning storage class - index_vt.set(index_vt_pos, storage_class, {}); - m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(old_storage_class)); - } else { - assert(fidelity == 2); - auto value = index_vt.xvalues()[index_vt_pos].m_value; - auto offset = field_id.getOffset(); - if (storage_class != StorageClass::DELETED && !lofi_store<2>::fromValue(value).isSet(offset)) { - // value is already unset - return; - } - auto old_type_id = getSchemaTypeId(old_storage_class, lofi_store<2>::fromValue(value).get(offset)); - if (storage_class == StorageClass::DELETED) { - lofi_store<2>::fromValue(value).set(offset, Value::DELETED); - } else { - lofi_store<2>::fromValue(value).reset(offset); - } - index_vt.set(index_vt_pos, old_storage_class, value); - m_type->removeFromSchema(field_id, fidelity, old_type_id); - } - } - - void Object::unrefKVIndexValue(FixtureLock &fixture, FieldID field_id, StorageClass storage_class, - unsigned int fidelity) - { - auto kv_index_ptr = tryGetKV_Index(); - if (!kv_index_ptr) { - THROWF(db0::InputException) << "Attribute not found"; - } - XValue xvalue(field_id.getIndex()); - if (!kv_index_ptr->findOne(xvalue)) { - THROWF(db0::InputException) << "Attribute not found"; - } - - if (fidelity == 0) { - unrefMember(*fixture, xvalue); - if (storage_class == StorageClass::DELETED) { - // mark as deleted in kv-index - xvalue.m_type = StorageClass::DELETED; - kv_index_ptr->updateExisting(xvalue); - // in case of the IttyIndex updating an element changes the address - // which needs to be updated in the object - if (kv_index_ptr->getIndexType() == bindex::type::itty) { - modify().m_kv_address = kv_index_ptr->getAddress(); - } - } else { - auto old_addr = kv_index_ptr->getAddress(); - kv_index_ptr->erase(xvalue); - auto new_addr = kv_index_ptr->getAddress(); - if (new_addr != old_addr) { - // type or address of the kv-index has changed which needs to be reflected - modify().m_kv_address = new_addr; - modify().m_kv_type = kv_index_ptr->getIndexType(); - } - } - m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(xvalue.m_type)); - } else { - assert(fidelity == 2); - auto value = xvalue.m_value; - auto offset = field_id.getOffset(); - if (storage_class != StorageClass::DELETED && !lofi_store<2>::fromValue(value).isSet(offset)) { - // value is already unset - return; - } - auto old_type_id = getSchemaTypeId(xvalue.m_type, lofi_store<2>::fromValue(value).get(offset)); - if (storage_class == StorageClass::DELETED) { - lofi_store<2>::fromValue(value).set(offset, Value::DELETED); - } else { - lofi_store<2>::fromValue(value).reset(offset); - } - xvalue.m_value = value; - kv_index_ptr->updateExisting(xvalue); - // in case of the IttyIndex updating an element changes the address - // which needs to be updated in the object - if (kv_index_ptr->getIndexType() == bindex::type::itty) { - modify().m_kv_address = kv_index_ptr->getAddress(); - } - - m_type->removeFromSchema(field_id, fidelity, old_type_id); - } - } - - void Object::unrefWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, - StorageClass storage_class, unsigned int fidelity) - { - if (loc_ptr == &(*this)->pos_vt()) { - unrefPosVT(fixture, field_id, pos, storage_class, fidelity); - } else if (loc_ptr == &(*this)->index_vt()) { - unrefIndexVT(fixture, field_id, pos, storage_class, fidelity); - } else { - unrefKVIndexValue(fixture, field_id, storage_class, fidelity); - } - } - - void Object::setPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, - StorageClass storage_class, Value value) - { - auto &pos_vt = modify().pos_vt(); - auto pos_value = pos_vt.values()[pos]; - if (fidelity == 0) { - auto old_storage_class = pos_vt.types()[pos]; - unrefMember(*fixture, old_storage_class, pos_value); - // update attribute stored in the positional value-table - pos_vt.set(pos, storage_class, value); - m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_storage_class), getSchemaTypeId(storage_class)); - } else { - auto offset = field_id.getOffset(); - auto old_type_id = getSchemaTypeId(storage_class, lofi_store<2>::fromValue(pos_value).get(offset)); - lofi_store<2>::fromValue(pos_value).set(offset, value.m_store); - pos_vt.set(pos, storage_class, pos_value); - auto new_type_id = getSchemaTypeId(storage_class, value); - m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); - } - } - - void Object::addToPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, - StorageClass storage_class, Value value) - { - auto &pos_vt = modify().pos_vt(); - auto pos_value = pos_vt.values()[pos]; - if (fidelity == 0) { - // update attribute stored in the positional value-table - pos_vt.set(pos, storage_class, value); - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class)); - } else { - unsigned int offset = field_id.getOffset(); - lofi_store<2>::fromValue(pos_value).set(offset, value.m_store); - pos_vt.set(pos, storage_class, pos_value); - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); - } - } - - void Object::setIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, - unsigned int fidelity, StorageClass storage_class, Value value) - { - auto &index_vt = modify().index_vt(); - if (fidelity == 0) { - auto old_storage_class = index_vt.xvalues()[index_vt_pos].m_type; - unrefMember(*fixture, index_vt.xvalues()[index_vt_pos]); - index_vt.set(index_vt_pos, storage_class, value); - m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_storage_class), getSchemaTypeId(storage_class)); - } else { - auto index_vt_value = index_vt.xvalues()[index_vt_pos].m_value; - auto offset = field_id.getOffset(); - auto old_type_id = getSchemaTypeId(storage_class, lofi_store<2>::fromValue(index_vt_value).get(offset)); - lofi_store<2>::fromValue(index_vt_value).set(offset, value.m_store); - index_vt.set(index_vt_pos, storage_class, index_vt_value); - auto new_type_id = getSchemaTypeId(storage_class, value); - m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); - } - } - - void Object::addToIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, - unsigned int fidelity, StorageClass storage_class, Value value) - { - auto &index_vt = modify().index_vt(); - if (fidelity == 0) { - index_vt.set(index_vt_pos, storage_class, value); - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class)); - } else { - auto index_vt_value = index_vt.xvalues()[index_vt_pos].m_value; - lofi_store<2>::fromValue(index_vt_value).set(field_id.getOffset(), value.m_store); - index_vt.set(index_vt_pos, storage_class, index_vt_value); - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); - } - } - - void Object::setKVIndexValue(FixtureLock &fixture, FieldID field_id, unsigned int fidelity, - StorageClass storage_class, Value value) - { - assert(m_type); - XValue xvalue(field_id.getIndex(), storage_class, value); - // encode for lo-fi storage if needed - if (fidelity != 0) { - xvalue.m_value = lofi_store<2>::create(field_id.getOffset(), value.m_store); - } - - auto kv_index_ptr = addKV_First(xvalue); - if (kv_index_ptr) { - // try updating an existing element first - XValue old_value; - if (kv_index_ptr->updateExisting(xvalue, &old_value)) { - if (fidelity == 0) { - unrefMember(*fixture, old_value); - m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_value.m_type), - getSchemaTypeId(storage_class) - ); - } else { - auto kv_value = old_value.m_value; - auto offset = field_id.getOffset(); - auto old_type_id = getSchemaTypeId(old_value.m_type, lofi_store<2>::fromValue(kv_value).get(offset)); - lofi_store<2>::fromValue(kv_value).set(offset, value.m_store); - xvalue.m_value = kv_value; - kv_index_ptr->updateExisting(xvalue); - auto new_type_id = getSchemaTypeId(storage_class, value); - m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); - } - // in case of the IttyIndex updating an element changes the address - // which needs to be updated in the object - if (kv_index_ptr->getIndexType() == bindex::type::itty) { - modify().m_kv_address = kv_index_ptr->getAddress(); - } - } else { - if (kv_index_ptr->insert(xvalue)) { - // type or address of the kv-index has changed which needs to be reflected - modify().m_kv_address = kv_index_ptr->getAddress(); - modify().m_kv_type = kv_index_ptr->getIndexType(); - } - - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); - } - } else { - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); - } - } - - void Object::addToKVIndex(FixtureLock &fixture, FieldID field_id, unsigned int fidelity, - StorageClass storage_class, Value value) - { - assert(m_type); - XValue xvalue(field_id.getIndex(), storage_class, value); - // encode for lo-fi storage if needed - if (fidelity != 0) { - xvalue.m_value = lofi_store<2>::create(field_id.getOffset(), value.m_store); - } - auto kv_index_ptr = addKV_First(xvalue); - if (kv_index_ptr) { - // NOTE: for fidelity > 0 the element might already exist - XValue old_value; - if (fidelity > 0 && kv_index_ptr->updateExisting(xvalue, &old_value)) { - auto kv_value = old_value.m_value; - lofi_store<2>::fromValue(kv_value).set(field_id.getOffset(), value.m_store); - xvalue.m_value = kv_value; - kv_index_ptr->updateExisting(xvalue); - // in case of the IttyIndex updating an element changes the address - // which needs to be updated in the object - if (kv_index_ptr->getIndexType() == bindex::type::itty) { - modify().m_kv_address = kv_index_ptr->getAddress(); - } - } else { - if (kv_index_ptr->insert(xvalue)) { - // type or address of the kv-index has changed which needs to be reflected - modify().m_kv_address = kv_index_ptr->getAddress(); - modify().m_kv_type = kv_index_ptr->getIndexType(); - } - } - } - - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); - } - - std::pair Object::tryGetMemberSlot(const MemberID &member_id, - unsigned int &pos) const - { - for (auto &field_info: member_id) { - auto [index, offset] = field_info.first.getIndexAndOffset(); - // pos-vt lookup - if ((*this)->pos_vt().find(index, pos)) { - if (field_info.second == 0 || slotExists((*this)->pos_vt().values()[pos], field_info.second, offset)) { - return { field_info, &(*this)->pos_vt() }; - } else { - continue; - } - } - - // index-vt lookup - if ((*this)->index_vt().find(index, pos)) { - if (field_info.second == 0 || slotExists((*this)->index_vt().xvalues()[pos].m_value, field_info.second, offset)) { - return { field_info, &(*this)->index_vt() }; - } else { - continue; - } - } - - // kv-index lookup - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - XValue value(index); - if (kv_index_ptr->findOne(value)) { - if (field_info.second == 0 || slotExists(value.m_value, field_info.second, offset)) { - return { field_info, nullptr }; - } - } - } - } - - // not found or deleted - return { {}, nullptr }; - } - - std::pair Object::tryGetLoc(FieldID field_id) const - { - auto index = field_id.getIndex(); - unsigned int pos = 0; - // pos-vt lookup - if ((*this)->pos_vt().find(index, pos)) { - return { &(*this)->pos_vt(), pos }; - } - // index-vt lookup - if ((*this)->index_vt().find(index, pos)) { - return { &(*this)->index_vt(), pos }; - } - // not found or located in the kv-index - return { nullptr, 0 }; - } - - void Object::setWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, - unsigned int fidelity, StorageClass storage_class, Value value) - { - if (loc_ptr == &(*this)->pos_vt()) { - setPosVT(fixture, field_id, pos, fidelity, storage_class, value); - return; - } - - if (loc_ptr == &(*this)->index_vt()) { - setIndexVT(fixture, field_id, pos, fidelity, storage_class, value); - return; - } - - // must be in the kv-index - assert(!loc_ptr); - setKVIndexValue(fixture, field_id, fidelity, storage_class, value); - } - - void Object::addWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, - unsigned int fidelity, StorageClass storage_class, Value value) - { - if (loc_ptr == &(*this)->pos_vt()) { - addToPosVT(fixture, field_id, pos, fidelity, storage_class, value); - return; - } - - if (loc_ptr == &(*this)->index_vt()) { - addToIndexVT(fixture, field_id, pos, fidelity, storage_class, value); - return; - } - - assert(!loc_ptr); - addToKVIndex(fixture, field_id, fidelity, storage_class, value); - } - - void Object::set(FixtureLock &fixture, const char *field_name, ObjectPtr lang_value) - { - assert(hasInstance()); - // attribute delete operation - if (!PyToolkit::isValid(lang_value)) { - remove(fixture, field_name); - return; - } - - auto [type_id, storage_class] = recognizeType(**fixture, lang_value); - - if (this->span() > 1) { - // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag - // this is because the actual change may be missed if performed on a different-then the 1st DP - _touch(); - } - - assert(m_type); - // find already existing field index - auto [member_id, is_init_var] = m_type->findField(field_name); - auto storage_fidelity = getStorageFidelity(storage_class); - // get field ID matching the required storage fidelity - FieldID field_id; - FieldInfo old_field_info; - unsigned int old_pos = 0; - const void *old_loc_ptr = nullptr; - if (member_id) { - std::tie(old_field_info, old_loc_ptr) = tryGetMemberSlot(member_id, old_pos); - } - - if (!member_id || !(field_id = member_id.tryGet(storage_fidelity))) { - // try mutating the class first - member_id = m_type->addField(field_name, storage_fidelity); - field_id = member_id.get(storage_fidelity); - } - - assert(field_id && member_id); - // NOTE: a new member inherits the parent's no-cache flag - // FIXME: value should be destroyed on exception - auto value = createMember(*fixture, type_id, storage_class, lang_value, getMemberFlags()); - // make sure object address is not null - assert(!(storage_class == StorageClass::OBJECT_REF && value.cast() == 0)); - - if (field_id == old_field_info.first) { - // Set / update value at the existing location - setWithLoc(fixture, field_id, old_loc_ptr, old_pos, storage_fidelity, storage_class, value); - } else { - // must reset / unreference the old value (stored elsewhere) - if (old_field_info.first) { - unrefWithLoc(fixture, old_field_info.first, old_loc_ptr, old_pos, StorageClass::UNDEFINED, - old_field_info.second); - } - - const void *loc_ptr = nullptr; - unsigned int pos = 0; - // NOTE: slot may already exist (pos-vt or index-vt) either for regular or lo-fi storage - std::tie(loc_ptr, pos) = tryGetLoc(field_id); - // Either use existing slot or create a new (kv-index) - addWithLoc(fixture, field_id, loc_ptr, pos, storage_fidelity, storage_class, value); - } - - // the KV-index insert operation must be registered as the potential silent mutation - // but the operation can be avoided if the object is already marked as modified - if (!super_t::isModified()) { - this->_touch(); - } - } - - std::pair Object::findField(const char *name) const - { - if (isDropped()) { - // defunct objects should not be accessed - assert(!isDefunct()); - THROWF(db0::InputException) << "Object does not exist"; - } - - auto class_ptr = m_type.get(); - if (!class_ptr) { - // retrieve class from the initializer - class_ptr = &m_init_manager.getInitializer(*this).getClass(); - } - - assert(class_ptr); - return class_ptr->findField(name); - } - - FieldID Object::tryGetMember(const char *field_name, std::pair &member, - bool &is_init_var) const - { - MemberID member_id; - std::tie(member_id, is_init_var) = this->findField(field_name); - bool exists, deleted = false; - if (member_id) { - std::tie(exists, deleted) = tryGetMemberAt(member_id.primary(), member); - if (exists) { - assert(!deleted); - return member_id.primary().first; - } - - // the primary slot was not occupied, try with the secondary - bool secondary_deleted = false; - std::tie(exists, secondary_deleted) = tryGetMemberAt(member_id.secondary(), member); - if (exists) { - assert(!secondary_deleted); - return member_id.secondary().first; - } - - deleted |= secondary_deleted; - } - - if (is_init_var) { - // unless explicitly deleted, - // report as None even if the field_id has not been assigned yet - member = { deleted ? StorageClass::DELETED : StorageClass::NONE, Value() }; - } - - // member not found - return {}; - } - - std::optional Object::tryGetX(const char *field_name) const - { - auto [member_id, is_init_var] = this->findField(field_name); - bool exists, deleted = false; - if (member_id) { - assert(member_id.primary().first); - std::pair member; - std::tie(exists, deleted) = tryGetMemberAt(member_id.primary(), member); - if (exists) { - assert(!deleted); - return XValue(member_id.primary().first.getIndex(), member.first, member.second); - } - // the primary slot was not occupied, try with the secondary - bool secondary_deleted = false; - std::tie(exists, secondary_deleted) = tryGetMemberAt(member_id.secondary(), member); - if (exists) { - assert(!secondary_deleted); - return XValue(member_id.secondary().first.getIndex(), member.first, member.second); - } - deleted |= secondary_deleted; - } - - if (!deleted && is_init_var) { - // unless explicitly deleted, - // report as None even if the field_id has not been assigned yet - return XValue(member_id.primary().first.getIndex(), StorageClass::NONE, Value()); - } - - return std::nullopt; - } - - Object::ObjectSharedPtr Object::tryGet(const char *field_name) const - { - std::pair member; - bool is_init_var = false; - auto field_id = tryGetMember(field_name, member, is_init_var); - // NOTE: init vars are always reported as None if not explicitly set nor explicitly deleted - if (field_id || (is_init_var && member.first != StorageClass::DELETED)) { - auto fixture = this->getFixture(); - // prevent accessing a deleted or undefined member - assert(member.first != StorageClass::DELETED && member.first != StorageClass::UNDEFINED); - // NOTE: offset is required for lo-fi members - return unloadMember( - fixture, member.first, member.second, field_id.maybeOffset(), this->getMemberFlags() - ); - } - - return nullptr; - } - - Object::ObjectSharedPtr Object::tryGetAs(const char *field_name, TypeObjectPtr lang_type) const - { - std::pair member; - bool is_init_var = false; - auto field_id = tryGetMember(field_name, member, is_init_var); - if (field_id || (is_init_var && member.first != StorageClass::DELETED)) { - // prevent accessing a deleted member - assert(member.first != StorageClass::DELETED && member.first != StorageClass::UNDEFINED); - auto fixture = this->getFixture(); - if (member.first == StorageClass::OBJECT_REF) { - auto &class_factory = getClassFactory(*fixture); - return PyToolkit::unloadObject(fixture, member.second.asAddress(), class_factory, lang_type); - } - - // NOTE: offset is required for lo-fi members - return unloadMember( - fixture, member.first, member.second, field_id.getOffset(), this->getMemberFlags() - ); - } - - return nullptr; - } - - Object::ObjectSharedPtr Object::get(const char *field_name) const - { - auto obj = tryGet(field_name); - if (!obj) { - if (isDropped()) { - THROWF(db0::InputException) << "Object is no longer accessible"; - } - THROWF(db0::InputException) << "Attribute not found: " << field_name; - } - return obj; - } - - bool Object::slotExists(Value value, unsigned int fidelity, unsigned int at) const - { - assert(fidelity != 0 && "Operation only available for lo-fi values"); - // lo-fi value - assert(fidelity == 2); - return lofi_store<2>::fromValue(value).isSet(at); - } - - std::pair Object::hasValueAt(Value value, unsigned int fidelity, unsigned int at) const - { - assert(fidelity != 0 && "Operation only available for lo-fi values"); - // lo-fi value - assert(fidelity == 2); - if (lofi_store<2>::fromValue(value).isSet(at)) { - // might be deleted - bool deleted = (lofi_store<2>::fromValue(value).get(at) == Value::DELETED); - return { !deleted, deleted }; - } else { - // NOTE: unset value is assumed as empty / undefined - return { false, false }; - } - } - - std::pair Object::tryGetMemberAt(std::pair field_info, - std::pair &result) const - { - if (!field_info.first) { - return { false, false }; - } - - auto loc = field_info.first.getIndexAndOffset(); - if (!hasInstance()) { - // try retrieving from initializer - auto initializer_ptr = m_init_manager.findInitializer(*this); - if (!initializer_ptr) { - return { false, false }; - } - return { initializer_ptr->tryGetAt(loc, result), false }; - } - - // retrieve from positionally encoded values - if ((*this)->pos_vt().find(loc.first, result)) { - // NOTE: removed field slots might be marked as DELETED - if (result.first == StorageClass::DELETED) { - // report as deleted - return { false, true }; - } - - if (field_info.second == 0) { - return { result.first != StorageClass::UNDEFINED, false }; - } else { - return hasValueAt(result.second, field_info.second, loc.second); - } - } - - if ((*this)->index_vt().find(loc.first, result)) { - if (result.first == StorageClass::DELETED) { - // report as deleted - return { false, true }; - } - - if (field_info.second == 0) { - return { result.first != StorageClass::UNDEFINED, false }; - } else { - return hasValueAt(result.second, field_info.second, loc.second); - } - } - - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - XValue xvalue(loc.first); - if (kv_index_ptr->findOne(xvalue)) { - assert(xvalue.getIndex() == loc.first); - if (xvalue.m_type == StorageClass::DELETED) { - // report as deleted - return { false, true }; - } - - // member fetched from the kv_index - result.first = xvalue.m_type; - result.second = xvalue.m_value; - - if (field_info.second == 0) { - return { result.first != StorageClass::UNDEFINED, false }; - } else { - return hasValueAt(result.second, field_info.second, loc.second); - } - } - } - - // Does not exist, not explicitly removed - return { false, false }; - } - - db0::swine_ptr Object::tryGetFixture() const - { - if (!hasInstance()) { - if (isDropped()) { - return {}; - } - // retrieve from the initializer - return m_init_manager.getInitializer(*this).tryGetFixture(); - } - return super_t::tryGetFixture(); - } - - db0::swine_ptr Object::getFixture() const - { - auto fixture = this->tryGetFixture(); - if (!fixture) { - THROWF(db0::InternalException) << "Object is no longer accessible"; - } - return fixture; - } - - Memspace &Object::getMemspace() const { - return *getFixture(); - } - - void Object::setType(std::shared_ptr type) - { - assert(!m_type); - m_type = type; - assert(hasValidClassRef()); - } - - void Object::setTypeWithHint(std::shared_ptr type_hint) - { - assert(!m_type); - assert(type_hint); - assert(hasInstance()); - if (type_hint->getClassRef() == (*this)->getClassRef()) { - m_type = type_hint; - } else { - m_type = unloadType(); - } - } - - bool Object::isSingleton() const { - return getType().isSingleton(); - } - - void Object::dropTags(Class &type) const - { - // only drop if any type tags are assigned - if ((*this)->m_header.m_ref_counter.getFirst() > 0) { - auto fixture = this->getFixture(); - assert(fixture); - auto &tag_index = fixture->get(); - const Class *type_ptr = &type; - auto unique_address = this->getUniqueAddress(); - while (type_ptr) { - // remove auto-assigned type (or its base) tag - tag_index.removeTypeTag(unique_address, type_ptr->getAddress()); - // NOTE: no need to decRef since object is being destroyed - type_ptr = type_ptr->getBaseClassPtr(); - } - } - } - - void Object::dropMembers(Class &class_ref) const - { - auto fixture = this->getFixture(); - assert(fixture); - // drop pos-vt members first - { - auto &types = (*this)->pos_vt().types(); - auto &values = (*this)->pos_vt().values(); - auto value = values.begin(); - unsigned int index = types.offset(); - for (auto type = types.begin(); type != types.end(); ++type, ++value, ++index) { - if (*type == StorageClass::DELETED || *type == StorageClass::UNDEFINED) { - // skip undefined or deleted members - continue; - } - unrefMember(fixture, *type, *value); - class_ref.removeFromSchema(index, *type, *value); - } - } - // drop index-vt members next - { - auto &xvalues = (*this)->index_vt().xvalues(); - for (auto &xvalue: xvalues) { - if (xvalue.m_type == StorageClass::DELETED || xvalue.m_type == StorageClass::UNDEFINED) { - // skip undefined or deleted members - continue; - } - unrefMember(fixture, xvalue); - class_ref.removeFromSchema(xvalue); - } - } - // finally drop kv-index members - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto it = kv_index_ptr->beginJoin(1); - for (;!it.is_end(); ++it) { - if ((*it).m_type == StorageClass::DELETED || (*it).m_type == StorageClass::UNDEFINED) { - // skip undefined or deleted members - continue; - } - unrefMember(fixture, *it); - class_ref.removeFromSchema(*it); - } - } - } - - void Object::unSingleton(FixtureLock &) - { - auto &type = getType(); - // drop reference from the class - if (type.isSingleton()) { - // clear singleton address - type.unlinkSingleton(); - modify().m_header.decRef(false); - } - } - - void Object::destroy() const - { - if (hasInstance()) { - // associated class type (may require unloading) - auto type = m_type; - if (!type) { - // retrieve type from the initializer - type = std::const_pointer_cast(unloadType()); - } - - dropTags(*type); - dropMembers(*type); - // dereference associated class - type->decRef(false); - } - super_t::destroy(); - } - - FieldLayout Object::getFieldLayout() const - { - FieldLayout layout; - // collect pos-vt information - for (auto type: (*this)->pos_vt().types()) { - layout.m_pos_vt_fields.push_back(type); - } - - // collect index-vt information - for (auto &xvalue: (*this)->index_vt().xvalues()) { - layout.m_index_vt_fields.emplace_back(xvalue.getIndex(), xvalue.m_type); - } - - // collect kv-index information - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto it = kv_index_ptr->beginJoin(1); - for (;!it.is_end(); ++it) { - layout.m_kv_index_fields.emplace_back((*it).getIndex(), (*it).m_type); - } - } - - return layout; - } - - KV_Index *Object::addKV_First(const XValue &value) - { - if (!m_kv_index) { - if ((*this)->m_kv_address) { - m_kv_index = std::make_unique( - std::make_pair(&getMemspace(), (*this)->m_kv_address), (*this)->m_kv_type - ); - } else { - // create new kv-index intiialized with the first value - m_kv_index = std::make_unique(getMemspace(), value); - modify().m_kv_address = m_kv_index->getAddress(); - modify().m_kv_type = m_kv_index->getIndexType(); - // return nullptr to indicate that the value has been inserted - return nullptr; - } - } - // return reference without inserting - return m_kv_index.get(); - } - - bool Object::hasKV_Index() const { - return m_kv_index || (*this)->m_kv_address; - } - - KV_Index *Object::tryGetKV_Index() const - { - // if KV index address has changed, update the cached instance - if (!m_kv_index || m_kv_index->getAddress() != (*this)->m_kv_address) { - if ((*this)->m_kv_address) { - m_kv_index = std::make_unique( - std::make_pair(&getMemspace(), (*this)->m_kv_address), (*this)->m_kv_type - ); - } - } - - return m_kv_index.get(); - } - - Class &Object::getType() { - return m_type ? *m_type : m_init_manager.getInitializer(*this).getClass(); - } - - void Object::getMembersFrom(const Class &this_type, unsigned int index, StorageClass storage_class, - Value value, std::unordered_set &result) const - { - if (storage_class == StorageClass::DELETED || storage_class == StorageClass::UNDEFINED) { - // skip undefined or deleted members - return; - } - - if (storage_class == StorageClass::PACK_2) { - auto it = lofi_store<2>::fromValue(value).begin(), end = lofi_store<2>::fromValue(value).end(); - for (; it != end; ++it) { - result.insert(this_type.getMember({ index, it.getOffset() }).m_name); - } - } else { - result.insert(this_type.getMember(FieldID::fromIndex(index)).m_name); - } - } - - std::unordered_set Object::getMembers() const - { - std::unordered_set result; - // Visit pos-vt members first - auto &obj_type = this->getType(); - { - auto &types = (*this)->pos_vt().types(); - auto &values = (*this)->pos_vt().values(); - unsigned int index = types.offset(); - auto size = types.size(); - for (unsigned int pos = 0;pos < size; ++index, ++pos) { - getMembersFrom(obj_type, index, types[pos], values[pos], result); - } - } - - // Visit index-vt members next - { - auto &xvalues = (*this)->index_vt().xvalues(); - for (auto &xvalue: xvalues) { - auto index = xvalue.getIndex(); - getMembersFrom(obj_type, index, xvalue.m_type, xvalue.m_value, result); - } - } - - // Finally, visit kv-index members - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto it = kv_index_ptr->beginJoin(1); - for (;!it.is_end(); ++it) { - auto index = (*it).getIndex(); - getMembersFrom(obj_type, index, (*it).m_type, (*it).m_value, result); - } - } - return result; - } - - void Object::forAll(std::function f) const - { - // Visit pos-vt members first - auto &obj_type = this->getType(); - { - auto &types = (*this)->pos_vt().types(); - auto &values = (*this)->pos_vt().values(); - auto value = values.begin(); - unsigned int index = types.offset(); - for (auto type = types.begin(); type != types.end(); ++type, ++value, ++index) { - if (*type == StorageClass::DELETED || *type == StorageClass::UNDEFINED) { - // skip deleted or undefined members - continue; - } - if (*type == StorageClass::PACK_2) { - // iterate individual lo-fi members - if (!forAll({index, *type, *value}, f)) { - return; - } - } else { - if (!f(obj_type.getMember(FieldID::fromIndex(index)).m_name, { index, *type, *value }, 0)) { - return; - } - } - } - } - - // Visit index-vt members next - { - auto &xvalues = (*this)->index_vt().xvalues(); - for (auto &xvalue: xvalues) { - if (xvalue.m_type == StorageClass::DELETED || xvalue.m_type == StorageClass::UNDEFINED) { - // skip deleted or undefined members - continue; - } - if (xvalue.m_type == StorageClass::PACK_2) { - // iterate individual lo-fi members - if (!forAll(xvalue, f)) { - return; - } - } else { - // regular member - if (!f(obj_type.getMember(FieldID::fromIndex(xvalue.getIndex())).m_name, xvalue, 0)) { - return; - } - } - } - } - - // Finally, visit kv-index members - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto it = kv_index_ptr->beginJoin(1); - for (;!it.is_end(); ++it) { - if ((*it).m_type == StorageClass::DELETED || (*it).m_type == StorageClass::UNDEFINED) { - // skip deleted or undefined members - continue; - } - if ((*it).m_type == StorageClass::PACK_2) { - // iterate individual lo-fi members - if (!forAll(*it, f)) { - return; - } - } else { - if (!f(obj_type.getMember(FieldID::fromIndex((*it).getIndex())).m_name, *it, 0)) { - return; - } - } - } - } - } - - void Object::forAll(std::function f) const - { - auto fixture = this->getFixture(); - forAll([&](const std::string &name, const XValue &xvalue, unsigned int offset) -> bool { - // all references convert to UUID - auto py_member = unloadMember( - fixture, xvalue.m_type, xvalue.m_value, offset, this->getMemberFlags() - ); - return f(name, py_member); - }); - } - - bool Object::forAll(XValue xvalue, std::function f) const - { - assert(xvalue.m_type == StorageClass::PACK_2); - unsigned int index = xvalue.getIndex(); - auto _value = xvalue.m_value; - auto it = lofi_store<2>::fromValue(_value).begin(), end = lofi_store<2>::fromValue(_value).end(); - auto &obj_type = this->getType(); - for (; it != end; ++it) { - if (!f(obj_type.getMember(FieldID::fromIndex(index, it.getOffset())).m_name, - xvalue, it.getOffset())) - { - return false; - } - } - return true; - } - - void Object::incRef(bool is_tag) - { - if (hasInstance()) { - super_t::incRef(is_tag); - } else { - // incRef with the initializer - m_init_manager.getInitializer(*this).incRef(is_tag); - } - } - - bool Object::decRef(bool is_tag) - { - // this operation is a potentially silent mutation - _touch(); - super_t::decRef(is_tag); - return !hasRefs(); - } - - bool Object::hasRefs() const - { - assert(hasInstance()); - return (*this)->hasRefs(); - } - - bool Object::hasAnyRefs() const { - return (*this)->hasAnyRefs(); - } - - bool Object::hasTagRefs() const { - return this->hasInstance() && (*this)->m_header.m_ref_counter.getFirst() > 0; - } - - bool Object::equalTo(const Object &other) const - { - if (!hasInstance() || !other.hasInstance()) { - THROWF(db0::InputException) << "Object not initialized"; - } - - if (this->isDefunct() || other.isDefunct()) { - // defunct objects should not be compared - assert(!isDefunct()); - THROWF(db0::InputException) << "Object does not exist"; - } - - if ((*this)->getClassRef() != other->getClassRef()) { - // different types - return false; - } - - if (this->getFixture()->getUUID() == other.getFixture()->getUUID() - && this->getUniqueAddress() == other.getUniqueAddress()) - { - // comparing 2 versions of the same object (fastest) - if (!((*this)->pos_vt() == other->pos_vt())) { - return false; - } - if (!((*this)->index_vt() == other->index_vt())) { - return false; - } - if (!hasKV_Index() && !other.hasKV_Index()) { - return true; - } - return isEqual(this->tryGetKV_Index(), other.tryGetKV_Index()); - } - - // field-wise compare otherwise (slower) - bool result = true; - this->forAll([&](const std::string &name, const XValue &xvalue, unsigned int offset) -> bool { - auto maybe_other_value = other.tryGetX(name.c_str()); - if (!maybe_other_value) { - result = false; - return false; - } - - if (!xvalue.equalTo(*maybe_other_value, offset)) { - result = false; - return false; - } - return true; - }); - return result; - } - - void Object::moveTo(db0::swine_ptr &) { - throw std::runtime_error("Not implemented"); - } - - void Object::setFixture(db0::swine_ptr &new_fixture) - { - if (hasInstance()) { - THROWF(db0::InputException) << "set_prefix failed: object already initialized"; - } - - if (!m_init_manager.getInitializer(*this).trySetFixture(new_fixture)) { - // signal problem with PyErr_BadPrefix - auto fixture = this->getFixture(); - LangToolkit::setError(LangToolkit::getTypeManager().getBadPrefixError(), fixture->getUUID()); - } - } - - void Object::detach() const - { - m_type->detach(); - // invalidate since detach is not supported by the MorphingBIndex - m_kv_index = nullptr; - super_t::detach(); - } - - void Object::commit() const - { - m_type->commit(); - if (m_kv_index) { - m_kv_index->commit(); - } - super_t::commit(); - // reset the silent-mutation flag - m_touched = false; - } - - void Object::unrefMember(db0::swine_ptr &fixture, StorageClass type, Value value) const { - db0::object_model::unrefMember(fixture, type, value); - } - - void Object::unrefMember(db0::swine_ptr &fixture, XValue value) const { - db0::object_model::unrefMember(fixture, value.m_type, value.m_value); - } - - std::shared_ptr Object::unloadType() const - { - auto fixture = this->getFixture(); - return getClassFactory(*fixture).getTypeByClassRef((*this)->getClassRef()).m_class; - } - - Address Object::getAddress() const - { - assert(!isDefunct()); - if (!hasInstance()) { - THROWF(db0::InternalException) << "Object instance does not exist yet (did you forget to use db0.materialized(self) in constructor ?)"; - } - return super_t::getAddress(); - } - - UniqueAddress Object::getUniqueAddress() const - { - if (hasInstance()) { - return super_t::getUniqueAddress(); - } else { - // NOTE: defunct objects don't have a valid address (not assigned yet) - assert(m_flags[ObjectOptions::DROPPED]); - return m_unique_address; - } - } - - bool Object::hasValidClassRef() const - { - if (hasInstance() && m_type) { - return (*this)->getClassRef() == m_type->getClassRef(); - } - return true; - } - - std::shared_ptr Object::getTypeWithHint(const Fixture &fixture, std::uint32_t class_ref, std::shared_ptr type_hint) - { - assert(type_hint); - if (type_hint->getClassRef() == class_ref) { - return type_hint; - } - return getClassFactory(fixture).getTypeByClassRef(class_ref).m_class; - } - - void Object::setDefunct() const { - m_flags.set(ObjectOptions::DEFUNCT); - } - - void Object::touch() - { - if (hasInstance() && !isDefunct()) { - // NOTE: for already modified and small objects we may skip "touch" - if (!super_t::isModified() || this->span() > 1) { - // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag - // this is because the actual change may be missed if performed on a different-then the 1st DP - _touch(); - } - } - } - - void Object::_touch() - { - if (!m_touched) { - // mark the 1st byte of the object as modified (forced-diff) - // this is always the 1st DP occupied by the object - modify(0, 1); - m_touched = true; - } - } - - void Object::addExtRef() const { - ++m_ext_refs; - } - - void Object::removeExtRef() const - { - assert(m_ext_refs > 0); - --m_ext_refs; - } - } \ No newline at end of file diff --git a/src/dbzero/object_model/object/Object.hpp b/src/dbzero/object_model/object/Object.hpp index e19d2dc4..c1c6d7be 100644 --- a/src/dbzero/object_model/object/Object.hpp +++ b/src/dbzero/object_model/object/Object.hpp @@ -1,18 +1,7 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include "ValueTable.hpp" -#include "ObjectInitializer.hpp" -#include -#include -#include -#include -#include "KV_Index.hpp" +#include "ObjectImplBase.hpp" +#include "o_object.hpp" namespace db0 @@ -29,364 +18,19 @@ namespace db0::object_model class Class; using Fixture = db0::Fixture; -DB0_PACKED_BEGIN - class DB0_PACKED_ATTR o_object: public db0::o_base - { - protected: - using super_t = db0::o_base; - - public: - static constexpr unsigned char REALM_ID = 1; - // common object header - o_unique_header m_header; - // optional address of the key-value store (to store extension fields) - KV_Address m_kv_address; - // kv-index type must be stored separately from the address - bindex::type m_kv_type; - // number of auto-assigned type tags - std::uint8_t m_num_type_tags = 0; - - PosVT &pos_vt(); - const PosVT &pos_vt() const; - - const packed_int32 &classRef() const; - std::uint32_t getClassRef() const; - - const IndexVT &index_vt() const; - - IndexVT &index_vt(); - - // ref_counts - the initial reference counts (tags / objects) inherited from the initializer - o_object(std::uint32_t class_ref, std::pair ref_counts, std::uint8_t num_type_tags, - const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin = nullptr, - const XValue *index_vt_end = nullptr); - - static std::size_t measure(std::uint32_t, std::pair, std::uint8_t num_type_tags, - const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin = nullptr, - const XValue *index_vt_end = nullptr); - - template static std::size_t safeSizeOf(BufT buf) - { - return super_t::sizeOfMembers(buf) - (PosVT::type()) - (packed_int32::type()) - (IndexVT::type()); - } - - void incRef(bool is_tag); - bool hasRefs() const; - bool hasAnyRefs() const; - }; -DB0_PACKED_END - - struct FieldLayout - { - std::vector m_pos_vt_fields; - std::vector > m_index_vt_fields; - std::vector > m_kv_index_fields; - }; - - enum class ObjectOptions: std::uint8_t - { - // the dbzero instance has been deleted - DROPPED = 0x01, - // object is defunct - e.g. due to exception on __init__ - DEFUNCT = 0x02 - }; - - using ObjectFlags = db0::FlagSet; - // NOTE: Object instances are created within the realm_id = 1 - using ObjectVType = db0::v_object; - - class Object: public db0::ObjectBase + class Object: public ObjectImplBase { // GC0 specific declarations GC0_Declare public: static constexpr unsigned char REALM_ID = o_object::REALM_ID; - using super_t = db0::ObjectBase; - using LangToolkit = LangConfig::LangToolkit; - using ObjectPtr = typename LangToolkit::ObjectPtr; - using TypeObjectPtr = typename LangToolkit::TypeObjectPtr; - using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; - using TypeManager = typename LangToolkit::TypeManager; - using ObjectStem = ObjectVType; - using TypeInitializer = ObjectInitializer::TypeInitializer; - - // construct as null / dropped object - Object(UniqueAddress, unsigned int ext_refs); - Object(const Object &) = delete; - Object(Object &&) = delete; - - /** - * Construct new Object (uninitialized, without corresponding dbzero instance yet) - */ - Object(std::shared_ptr); - Object(TypeInitializer &&); - - // Unload from address with a known type (possibly a base type) - // NOTE: unload works faster if type_hint is the exact object's type - struct with_type_hint {}; - Object(db0::swine_ptr &, Address, std::shared_ptr type_hint, - with_type_hint, AccessFlags = {}); - - // Unload from stem with a known type (possibly a base type) - // NOTE: unload works faster if type_hint is the exact object's type - Object(db0::swine_ptr &, ObjectStem &&, std::shared_ptr type_hint, with_type_hint); - - Object(db0::swine_ptr &, Address, AccessFlags = {}); - Object(db0::swine_ptr &, std::shared_ptr, std::pair ref_counts, - const PosVT::Data &, unsigned int pos_vt_offset); - Object(db0::swine_ptr &, ObjectStem &&, std::shared_ptr); - - ~Object(); - - // post-init invoked by memo type directly after __init__ - void postInit(FixtureLock &); - - // Destroys an existing instance and constructs a "null" placeholder - void dropInstance(FixtureLock &); - - // Unload the object stem, to retrieve its type - static ObjectStem tryUnloadStem(db0::swine_ptr &, Address, - std::uint16_t instance_id = 0, AccessFlags = {}); - static ObjectStem unloadStem(db0::swine_ptr &, Address, - std::uint16_t instance_id = 0, AccessFlags = {}); - - // Called to finalize adding members - void endInit(); - - // Assign language specific value as a field (to already initialized or uninitialized instance) - // NOTE: if lang_value is nullptr then the member is removed - void set(FixtureLock &, const char *field_name, ObjectPtr lang_value); - void remove(FixtureLock &, const char *field_name); - - // Assign field of an uninitialized instance (assumed as a non-mutating operation) - // NOTE: if lang_value is nullptr then the member is removed - void setPreInit(const char *field_name, ObjectPtr lang_value) const; - void removePreInit(const char *field_name) const; - - ObjectSharedPtr tryGet(const char *field_name) const; - ObjectSharedPtr tryGetAs(const char *field_name, TypeObjectPtr) const; - ObjectSharedPtr get(const char *field_name) const; - - inline std::shared_ptr getClassPtr() const { - return m_type ? m_type : m_init_manager.getInitializer(*this).getClassPtr(); - } - - inline const Class &getType() const { - return m_type ? *m_type : m_init_manager.getInitializer(*this).getClass(); - } - - Class &getType(); - - db0::swine_ptr tryGetFixture() const; - - db0::swine_ptr getFixture() const; - - Memspace &getMemspace() const; - - /** - * Get description of the field layout - */ - FieldLayout getFieldLayout() const; - - // Convert singleton into a regular instance - void unSingleton(FixtureLock &); - - void destroy() const; - - bool isSingleton() const; - - // execute the function for all members (until false is returned from the input lambda) - void forAll(std::function) const; - void forAll(std::function) const; - - // get dbzero member / member names assigned to this object - std::unordered_set getMembers() const; - - /** - * The overloaded incRef implementation is provided to also handle non-fully initialized objects - */ - void incRef(bool is_tag); - bool hasRefs() const; - // check for any refs (including auto-assigned type tags) - bool hasAnyRefs() const; + using super_t = ObjectImplBase; - // check if any references from tags exist (i.e. are any tags assigned) - bool hasTagRefs() const; - - // @return true if reference count was decremented to zero - bool decRef(bool is_tag); - - // Binary (shallow) compare 2 objects or 2 versions of the same memo object (e.g. from different snapshots) - // NOTE: ref-counts are not compared (only user-assigned members) - // @return true if objects are identical - bool equalTo(const Object &) const; - - /** - * Move unreferenced object to a different prefix without changing the instance - * this operations is required for auto-hardening - */ - void moveTo(db0::swine_ptr &); - - /** - * Change fixture of the uninitialized object - * Object must not have any members yet either - */ - void setFixture(db0::swine_ptr &); - - void detach() const; - - void commit() const; - - Address getAddress() const; - UniqueAddress getUniqueAddress() const; - - // NOTE: the operation is marked const because the dbzero state is not affected - void setDefunct() const; - - inline bool isDropped() const { - return m_flags.test(ObjectOptions::DROPPED); - } - - inline bool isDefunct() const { - return m_flags.test(ObjectOptions::DEFUNCT); - } - - // is dropped or defunct - inline bool isDead() const { - return m_flags.any( - static_cast(ObjectOptions::DROPPED) | static_cast(ObjectOptions::DEFUNCT) - ); - } - - // FieldID, is_init_var, fidelity - std::pair findField(const char *name) const; - - // Check if the 2 memo objects are of the same type - bool sameType(const Object &) const; - - // the member called to indicate the object mutation - void touch(); - - void addExtRef() const; - void removeExtRef() const; - - inline std::uint32_t getExtRefs() const { - return m_ext_refs; - } - - private: - // Class will only be assigned after initialization - std::shared_ptr m_type; - // local kv-index instance cache (created at first use) - mutable std::unique_ptr m_kv_index; - static ObjectInitializerManager m_init_manager; - mutable ObjectFlags m_flags; - // reference counter for inner references from language objects - // NOTE: inner references are held by internal dbzero buffers (e.g. TagIndex) - // see also PyEXT_INCREF / PyEXT_DECREF - mutable std::uint32_t m_ext_refs = 0; - // A flag indicating that object's silent mutation has already been reflected - // with the underlying MemLock / ResourceLock - // NOTE: by silent mutation we mean a mutation that does not change data (e.g. +refcount(+-1) + (refcount-1)) - mutable bool m_touched = false; - // NOTE: member assigned only to dropped objects (see replaceWithNull) - // so that we can retrieve the address of the dropped instance after it has been destroyed - const UniqueAddress m_unique_address; - - void setType(std::shared_ptr); - // adjusts to actual type if the type hint is a base class - void setTypeWithHint(std::shared_ptr type_hint); - // @return exists / deleted - std::pair hasValueAt(Value, unsigned int fidelity, unsigned int at) const; - // similar to hasValueAt but assume deleted slot as present - bool slotExists(Value value, unsigned int fidelity, unsigned int at) const; - - // Try retrieving member either from values (initialized) or from the initialization buffer (not initialized yet) - // @return member exists, member deleted flags - std::pair tryGetMemberAt(std::pair, - std::pair &) const; - FieldID tryGetMember(const char *field_name, std::pair &, bool &is_init_var) const; - - // Try resolving field ID of an existing (or deleted) member and also its storage location - // @param pos the member's position in the containing collection - // @return FieldID + containing collection (e.g. pos_vt()) - std::pair tryGetMemberSlot(const MemberID &, unsigned int &pos) const; - - // Try locating a field ID associated slot - std::pair tryGetLoc(FieldID) const; - - inline ObjectInitializer *tryGetInitializer() const { - return m_type ? static_cast(nullptr) : &m_init_manager.getInitializer(*this); + template + Object(Args&&... args) + : super_t(std::forward(args)...) + { } - - /** - * If the KV_Index does not exist yet, create it and add the first value - * otherwise return instance of an existing KV_Index - */ - KV_Index *addKV_First(const XValue &); - - KV_Index *tryGetKV_Index() const; - - void dropMembers(Class &) const; - void dropTags(Class &) const; - - void unrefMember(db0::swine_ptr &, StorageClass, Value) const; - void unrefMember(db0::swine_ptr &, XValue) const; - - using TypeId = db0::bindings::TypeId; - std::pair recognizeType(Fixture &, ObjectPtr lang_value) const; - - // Unload associated type - std::shared_ptr unloadType() const; - - // Retrieve a type by class-ref with a possible match (type_hint) - static std::shared_ptr getTypeWithHint(const Fixture &, std::uint32_t class_ref, std::shared_ptr type_hint); - - bool hasValidClassRef() const; - bool hasKV_Index() const; - - // try retrieving member as XValue - std::optional tryGetX(const char *field_name) const; - void _touch(); - - // Set or update member in a pos_vt - void setPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); - void setIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, - StorageClass, Value); - // Set or update member in kv-index - void setKVIndexValue(FixtureLock &, FieldID, unsigned int fidelity, StorageClass, Value); - - // Set with a specific location (pos_vt, index_vt, kv-index) - void setWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, - StorageClass, Value); - - // Unreference value - // NOTE: storage_class to be assigned can either be DELETED or UNDEFINED - void unrefPosVT(FixtureLock &, FieldID, unsigned int pos, StorageClass, unsigned int fidelity); - void unrefIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, StorageClass, unsigned int fidelity); - void unrefKVIndexValue(FixtureLock &, FieldID, StorageClass, unsigned int fidelity); - - void unrefWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, StorageClass, - unsigned int fidelity); - - // Add a new value - void addToPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); - void addToIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, StorageClass, Value); - void addToKVIndex(FixtureLock &, FieldID, unsigned int fidelity, StorageClass, Value); - - void addWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, - StorageClass, Value); - - // lo-fi member specialized implementation - bool forAll(XValue, std::function) const; - - void getMembersFrom(const Class &this_type, unsigned int index, StorageClass, Value, - std::unordered_set &) const; }; } - -DECLARE_ENUM_VALUES(db0::object_model::ObjectOptions, 2) \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectImplBase.cpp b/src/dbzero/object_model/object/ObjectImplBase.cpp new file mode 100644 index 00000000..836d25b7 --- /dev/null +++ b/src/dbzero/object_model/object/ObjectImplBase.cpp @@ -0,0 +1,1653 @@ +#include "ObjectImplBase.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +DEFINE_ENUM_VALUES(db0::object_model::ObjectOptions, "DROPPED", "DEFUNCT") + +namespace db0::object_model + +{ + + template class ObjectImplBase; + + template + ObjectInitializerManager ObjectImplBase::m_init_manager; + + FlagSet getAccessOptions(const Class &type) { + return type.isNoCache() ? FlagSet { AccessOptions::no_cache } : FlagSet {}; + } + + bool isEqual(const KV_Index *kv_ptr_1, const KV_Index *kv_ptr_2) + { + if (!kv_ptr_1) { + return !kv_ptr_2; + } + + if (!kv_ptr_2) { + return false; + } + + // item-wise comparison + return *kv_ptr_1 == *kv_ptr_2; + } + + template IntT safeCast(unsigned int value, const char *err_msg) + { + if (value > std::numeric_limits::max()) { + THROWF(db0::InputException) << err_msg; + } + return static_cast(value); + } + + template + ObjectImplBase::ObjectImplBase(UniqueAddress addr, unsigned int ext_refs) + : m_flags { ObjectOptions::DROPPED } + , m_ext_refs(ext_refs) + , m_unique_address(addr) + { + } + + template + ObjectImplBase::ObjectImplBase(std::shared_ptr db0_class) + { + // prepare for initialization + m_init_manager.addInitializer(*this, db0_class); + } + + template + ObjectImplBase::ObjectImplBase(TypeInitializer &&type_initializer) + { + // prepare for initialization + m_init_manager.addInitializer(*this, std::move(type_initializer)); + } + + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, std::shared_ptr type, + std::pair ref_counts, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset) + : super_t(fixture, type->getClassRef(), ref_counts, + safeCast(type->getNumBases() + 1, "Too many base classes"), pos_vt_data, pos_vt_offset, nullptr, nullptr, + getAccessOptions(*type)) + , m_type(type) + { + } + + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, Address address, AccessFlags access_mode) + : super_t(super_t::tag_from_address(), fixture, address, access_mode) + { + } + + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type) + : super_t(super_t::tag_from_stem(), fixture, std::move(stem)) + , m_type(type) + { + assert(hasValidClassRef()); + } + + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, Address address, std::shared_ptr type_hint, + with_type_hint, AccessFlags access_mode) + : ObjectImplBase(fixture, address, access_mode) + { + assert(*fixture == *type_hint->getFixture()); + setTypeWithHint(type_hint); + } + + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type_hint, with_type_hint) + : ObjectImplBase(fixture, std::move(stem), getTypeWithHint(*fixture, stem->getClassRef(), type_hint)) + { + } + + template + ObjectImplBase::~ObjectImplBase() + { + // unregister needs to be called before destruction of members + this->unregister(); + if (!this->hasInstance()) { + // release initializer if it exists, object not created + m_init_manager.tryCloseInitializer(*this); + } + } + + template + void ObjectImplBase::dropInstance(FixtureLock &) + { + auto unique_addr = this->getUniqueAddress(); + auto ext_refs = this->getExtRefs(); + this->~ObjectImplBase(); + // construct a null placeholder + new ((void*)this) Object(unique_addr, ext_refs); + } + + template + typename ObjectImplBase::ObjectStem ObjectImplBase::tryUnloadStem(db0::swine_ptr &fixture, + Address address, std::uint16_t instance_id, AccessFlags access_mode) + { + std::size_t size_of; + if (!fixture->isAddressValid(address, REALM_ID, &size_of)) { + return {}; + } + // Unload from a verified address + ObjectVType stem(db0::tag_verified(), fixture->myPtr(address), size_of, access_mode); + if (instance_id && stem->m_header.m_instance_id != instance_id) { + // instance ID validation failed + return {}; + } + return stem; + } + + template + typename ObjectImplBase::ObjectStem ObjectImplBase::unloadStem(db0::swine_ptr &fixture, + Address address, std::uint16_t instance_id, AccessFlags access_mode) + { + auto result = tryUnloadStem(fixture, address, instance_id, access_mode); + if (!result) { + THROWF(db0::InputException) << "Invalid UUID or object has been deleted"; + } + return result; + } + + template + void ObjectImplBase::postInit(FixtureLock &fixture) + { + if (!this->hasInstance()) { + auto &initializer = m_init_manager.getInitializer(*this); + PosVT::Data pos_vt_data; + unsigned int pos_vt_offset = 0; + auto index_vt_data = initializer.getData(pos_vt_data, pos_vt_offset); + + // place object in the same fixture as its class + // construct the dbzero instance & assign to self + m_type = initializer.getClassPtr(); + assert(m_type); + super_t::init(*fixture, m_type->getClassRef(), initializer.getRefCounts(), + safeCast(m_type->getNumBases() + 1, "Too many base classes"), + pos_vt_data, pos_vt_offset, index_vt_data.first, index_vt_data.second, + getAccessOptions(*m_type) + ); + + // reference associated class + m_type->incRef(false); + m_type->updateSchema(pos_vt_offset, pos_vt_data.m_types, pos_vt_data.m_values); + m_type->updateSchema(index_vt_data.first, index_vt_data.second); + + // bind singleton address (now that instance exists) + if (m_type->isSingleton()) { + m_type->setSingletonAddress(*this); + } + initializer.close(); + } + + assert(hasInstance()); + } + + template + std::pair + ObjectImplBase::recognizeType(Fixture &fixture, ObjectPtr lang_value) const + { + auto type_id = LangToolkit::getTypeManager().getTypeId(lang_value); + // NOTE: allow storage as PACK_2 + auto pre_storage_class = TypeUtils::m_storage_class_mapper.getPreStorageClass(type_id, true); + if (type_id == TypeId::MEMO_OBJECT) { + // object reference must be from the same fixture + auto &obj = LangToolkit::getTypeManager().extractObject(lang_value); + if (fixture.getUUID() != obj.getFixture()->getUUID()) { + THROWF(db0::InputException) << "Referencing objects from foreign prefixes is not allowed. Use db0.weak_proxy instead"; + } + } + + // may need to refine the storage class (i.e. long weak ref might be needed instead) + StorageClass storage_class; + if (pre_storage_class == PreStorageClass::OBJECT_WEAK_REF) { + storage_class = db0::getStorageClass(pre_storage_class, fixture, lang_value); + } else { + storage_class = db0::getStorageClass(pre_storage_class); + } + + return { type_id, storage_class }; + } + + template + void ObjectImplBase::removePreInit(const char *field_name) const + { + auto &initializer = m_init_manager.getInitializer(*this); + auto &type = initializer.getClass(); + + // Find an already existing field index + auto member_id = std::get<0>(type.findField(field_name)); + if (!member_id) { + THROWF(db0::InputException) << "Attribute not found: " << field_name; + } + + for (const auto &field_info: member_id) { + assert(field_info.first); + auto loc = field_info.first.getIndexAndOffset(); + // mark as deleted + if (field_info.second == 0) { + initializer.set(loc, StorageClass::DELETED, {}); + } else { + assert(field_info.second == 2 && "Only fidelity == 2 is supported"); + if (member_id.hasFidelity(0)) { + // remove any existing regular initialization + auto loc0 = member_id.get(0).getIndexAndOffset(); + initializer.remove(loc0); + } + initializer.set(loc, StorageClass::PACK_2, Value::DELETED, + lofi_store<2>::mask(loc.second)); + } + } + } + + template + void ObjectImplBase::setPreInit(const char *field_name, ObjectPtr obj_ptr) const + { + assert(!hasInstance()); + if (!LangToolkit::isValid(obj_ptr)) { + removePreInit(field_name); + return; + } + + auto &initializer = m_init_manager.getInitializer(*this); + auto fixture = initializer.getFixture(); + auto &type = initializer.getClass(); + auto [type_id, storage_class] = recognizeType(*fixture, obj_ptr); + auto storage_fidelity = getStorageFidelity(storage_class); + + // Find an already existing field index + auto [member_id, is_init_var] = type.findField(field_name); + // NOTE: even if a field already exists we might need to extend its supported fidelities + if (!member_id || !member_id.hasFidelity(storage_fidelity)) { + // update class definition + // use the default fidelity for the storage class + member_id = type.addField(field_name, storage_fidelity); + } + + if (storage_fidelity == 0) { + if (member_id.hasFidelity(2)) { + // remove any existing lo-fi initialization + auto loc = member_id.get(2).getIndexAndOffset(); + initializer.remove(loc, lofi_store<2>::mask(loc.second)); + } + // register a regular member with the initializer + // NOTE: a new member receives the no-cache flag if set (at the type level) + auto member_flags = type.isNoCache() ? AccessFlags { AccessOptions::no_cache } : AccessFlags(); + initializer.set(member_id.get(0).getIndexAndOffset(), storage_class, + createMember(fixture, type_id, storage_class, obj_ptr, member_flags) + ); + } else { + if (member_id.hasFidelity(0)) { + // remove any existing regular initialization + auto loc = member_id.get(0).getIndexAndOffset(); + initializer.remove(loc); + } + // For now only fidelity == 2 is supported (lo-fi storage) + assert(storage_fidelity == 2); + auto loc = member_id.get(storage_fidelity).getIndexAndOffset(); + // no access flags for lo-fi members + auto value = lofi_store<2>::create(loc.second, + createMember(fixture, type_id, storage_class, obj_ptr, {}).m_store); + // register a lo-fi member with the initializer (using mask) + initializer.set(loc, storage_class, value, lofi_store<2>::mask(loc.second)); + } + } + + template + void ObjectImplBase::remove(FixtureLock &fixture, const char *field_name) + { + assert(hasInstance()); + if (this->span() > 1) { + // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag + // this is because the actual change may be missed if performed on a different-then the 1st DP + _touch(); + } + + assert(m_type); + // Find an already existing field index + auto [member_id, is_init_var] = m_type->findField(field_name); + if (!member_id) { + THROWF(db0::InputException) << "Attribute not found: " << field_name; + } + + unsigned int pos = 0; + auto [field_info, loc_ptr] = tryGetMemberSlot(member_id, pos); + if (!field_info.first) { + THROWF(db0::InputException) << "Attribute not found: " << field_name; + } + + // NOTE: unreference as DELETED + unrefWithLoc(fixture, field_info.first, loc_ptr, pos, StorageClass::DELETED, + field_info.second); + + // the KV-index erase operation must be registered as the potential silent mutation + // but the operation can be avoided if the object is already marked as modified + if (!loc_ptr && !super_t::isModified()) { + this->_touch(); + } + } + + template + void ObjectImplBase::unrefPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, + StorageClass storage_class, unsigned int fidelity) + { + auto &pos_vt = this->modify().pos_vt(); + auto old_storage_class = pos_vt.types()[pos]; + if (fidelity == 0) { + unrefMember(*fixture, old_storage_class, pos_vt.values()[pos]); + // mark member as unreferenced by assigning storage class + pos_vt.set(pos, storage_class, {}); + m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(old_storage_class)); + } else { + assert(fidelity == 2); + auto value = pos_vt.values()[pos]; + auto offset = field_id.getOffset(); + if (storage_class != StorageClass::DELETED && !lofi_store<2>::fromValue(value).isSet(offset)) { + // value is already unset + return; + } + + auto old_type_id = getSchemaTypeId(old_storage_class, lofi_store<2>::fromValue(value).get(offset)); + // either reset or mark as deleted + if (storage_class == StorageClass::DELETED) { + lofi_store<2>::fromValue(value).set(offset, Value::DELETED); + } else { + lofi_store<2>::fromValue(value).reset(offset); + } + pos_vt.set(pos, old_storage_class, value); + m_type->removeFromSchema(field_id, fidelity, old_type_id); + } + } + + template + void ObjectImplBase::unrefIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, + StorageClass storage_class, unsigned int fidelity) + { + auto &index_vt = this->modify().index_vt(); + auto old_storage_class = index_vt.xvalues()[index_vt_pos].m_type; + if (fidelity == 0) { + unrefMember(*fixture, index_vt.xvalues()[index_vt_pos]); + // mark member as unreferenced by assigning storage class + index_vt.set(index_vt_pos, storage_class, {}); + m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(old_storage_class)); + } else { + assert(fidelity == 2); + auto value = index_vt.xvalues()[index_vt_pos].m_value; + auto offset = field_id.getOffset(); + if (storage_class != StorageClass::DELETED && !lofi_store<2>::fromValue(value).isSet(offset)) { + // value is already unset + return; + } + auto old_type_id = getSchemaTypeId(old_storage_class, lofi_store<2>::fromValue(value).get(offset)); + if (storage_class == StorageClass::DELETED) { + lofi_store<2>::fromValue(value).set(offset, Value::DELETED); + } else { + lofi_store<2>::fromValue(value).reset(offset); + } + index_vt.set(index_vt_pos, old_storage_class, value); + m_type->removeFromSchema(field_id, fidelity, old_type_id); + } + } + + template + void ObjectImplBase::unrefKVIndexValue(FixtureLock &fixture, FieldID field_id, + StorageClass storage_class, unsigned int fidelity) + { + auto kv_index_ptr = tryGetKV_Index(); + if (!kv_index_ptr) { + THROWF(db0::InputException) << "Attribute not found"; + } + XValue xvalue(field_id.getIndex()); + if (!kv_index_ptr->findOne(xvalue)) { + THROWF(db0::InputException) << "Attribute not found"; + } + + if (fidelity == 0) { + unrefMember(*fixture, xvalue); + if (storage_class == StorageClass::DELETED) { + // mark as deleted in kv-index + xvalue.m_type = StorageClass::DELETED; + kv_index_ptr->updateExisting(xvalue); + // in case of the IttyIndex updating an element changes the address + // which needs to be updated in the object + if (kv_index_ptr->getIndexType() == bindex::type::itty) { + this->modify().m_kv_address = kv_index_ptr->getAddress(); + } + } else { + auto old_addr = kv_index_ptr->getAddress(); + kv_index_ptr->erase(xvalue); + auto new_addr = kv_index_ptr->getAddress(); + if (new_addr != old_addr) { + // type or address of the kv-index has changed which needs to be reflected + this->modify().m_kv_address = new_addr; + this->modify().m_kv_type = kv_index_ptr->getIndexType(); + } + } + m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(xvalue.m_type)); + } else { + assert(fidelity == 2); + auto value = xvalue.m_value; + auto offset = field_id.getOffset(); + if (storage_class != StorageClass::DELETED && !lofi_store<2>::fromValue(value).isSet(offset)) { + // value is already unset + return; + } + auto old_type_id = getSchemaTypeId(xvalue.m_type, lofi_store<2>::fromValue(value).get(offset)); + if (storage_class == StorageClass::DELETED) { + lofi_store<2>::fromValue(value).set(offset, Value::DELETED); + } else { + lofi_store<2>::fromValue(value).reset(offset); + } + xvalue.m_value = value; + kv_index_ptr->updateExisting(xvalue); + // in case of the IttyIndex updating an element changes the address + // which needs to be updated in the object + if (kv_index_ptr->getIndexType() == bindex::type::itty) { + this->modify().m_kv_address = kv_index_ptr->getAddress(); + } + + m_type->removeFromSchema(field_id, fidelity, old_type_id); + } + } + + template + void ObjectImplBase::unrefWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, + StorageClass storage_class, unsigned int fidelity) + { + if (loc_ptr == &(*this)->pos_vt()) { + unrefPosVT(fixture, field_id, pos, storage_class, fidelity); + } else if (loc_ptr == &(*this)->index_vt()) { + unrefIndexVT(fixture, field_id, pos, storage_class, fidelity); + } else { + unrefKVIndexValue(fixture, field_id, storage_class, fidelity); + } + } + + template + void ObjectImplBase::setPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, + StorageClass storage_class, Value value) + { + auto &pos_vt = modify().pos_vt(); + auto pos_value = pos_vt.values()[pos]; + if (fidelity == 0) { + auto old_storage_class = pos_vt.types()[pos]; + unrefMember(*fixture, old_storage_class, pos_value); + // update attribute stored in the positional value-table + pos_vt.set(pos, storage_class, value); + m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_storage_class), getSchemaTypeId(storage_class)); + } else { + auto offset = field_id.getOffset(); + auto old_type_id = getSchemaTypeId(storage_class, lofi_store<2>::fromValue(pos_value).get(offset)); + lofi_store<2>::fromValue(pos_value).set(offset, value.m_store); + pos_vt.set(pos, storage_class, pos_value); + auto new_type_id = getSchemaTypeId(storage_class, value); + m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); + } + } + + template + void ObjectImplBase::addToPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, + StorageClass storage_class, Value value) + { + auto &pos_vt = modify().pos_vt(); + auto pos_value = pos_vt.values()[pos]; + if (fidelity == 0) { + // update attribute stored in the positional value-table + pos_vt.set(pos, storage_class, value); + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class)); + } else { + unsigned int offset = field_id.getOffset(); + lofi_store<2>::fromValue(pos_value).set(offset, value.m_store); + pos_vt.set(pos, storage_class, pos_value); + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + } + } + + template + void ObjectImplBase::setIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, + unsigned int fidelity, StorageClass storage_class, Value value) + { + auto &index_vt = modify().index_vt(); + if (fidelity == 0) { + auto old_storage_class = index_vt.xvalues()[index_vt_pos].m_type; + unrefMember(*fixture, index_vt.xvalues()[index_vt_pos]); + index_vt.set(index_vt_pos, storage_class, value); + m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_storage_class), getSchemaTypeId(storage_class)); + } else { + auto index_vt_value = index_vt.xvalues()[index_vt_pos].m_value; + auto offset = field_id.getOffset(); + auto old_type_id = getSchemaTypeId(storage_class, lofi_store<2>::fromValue(index_vt_value).get(offset)); + lofi_store<2>::fromValue(index_vt_value).set(offset, value.m_store); + index_vt.set(index_vt_pos, storage_class, index_vt_value); + auto new_type_id = getSchemaTypeId(storage_class, value); + m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); + } + } + + template + void ObjectImplBase::addToIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, + unsigned int fidelity, StorageClass storage_class, Value value) + { + auto &index_vt = this->modify().index_vt(); + if (fidelity == 0) { + index_vt.set(index_vt_pos, storage_class, value); + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class)); + } else { + auto index_vt_value = index_vt.xvalues()[index_vt_pos].m_value; + lofi_store<2>::fromValue(index_vt_value).set(field_id.getOffset(), value.m_store); + index_vt.set(index_vt_pos, storage_class, index_vt_value); + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + } + } + + template + void ObjectImplBase::setKVIndexValue(FixtureLock &fixture, FieldID field_id, unsigned int fidelity, + StorageClass storage_class, Value value) + { + assert(m_type); + XValue xvalue(field_id.getIndex(), storage_class, value); + // encode for lo-fi storage if needed + if (fidelity != 0) { + xvalue.m_value = lofi_store<2>::create(field_id.getOffset(), value.m_store); + } + + auto kv_index_ptr = addKV_First(xvalue); + if (kv_index_ptr) { + // try updating an existing element first + XValue old_value; + if (kv_index_ptr->updateExisting(xvalue, &old_value)) { + if (fidelity == 0) { + unrefMember(*fixture, old_value); + m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_value.m_type), + getSchemaTypeId(storage_class) + ); + } else { + auto kv_value = old_value.m_value; + auto offset = field_id.getOffset(); + auto old_type_id = getSchemaTypeId(old_value.m_type, lofi_store<2>::fromValue(kv_value).get(offset)); + lofi_store<2>::fromValue(kv_value).set(offset, value.m_store); + xvalue.m_value = kv_value; + kv_index_ptr->updateExisting(xvalue); + auto new_type_id = getSchemaTypeId(storage_class, value); + m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); + } + // in case of the IttyIndex updating an element changes the address + // which needs to be updated in the object + if (kv_index_ptr->getIndexType() == bindex::type::itty) { + modify().m_kv_address = kv_index_ptr->getAddress(); + } + } else { + if (kv_index_ptr->insert(xvalue)) { + // type or address of the kv-index has changed which needs to be reflected + this->modify().m_kv_address = kv_index_ptr->getAddress(); + this->modify().m_kv_type = kv_index_ptr->getIndexType(); + } + + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + } + } else { + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + } + } + + template + void ObjectImplBase::addToKVIndex(FixtureLock &fixture, FieldID field_id, unsigned int fidelity, + StorageClass storage_class, Value value) + { + assert(m_type); + XValue xvalue(field_id.getIndex(), storage_class, value); + // encode for lo-fi storage if needed + if (fidelity != 0) { + xvalue.m_value = lofi_store<2>::create(field_id.getOffset(), value.m_store); + } + auto kv_index_ptr = addKV_First(xvalue); + if (kv_index_ptr) { + // NOTE: for fidelity > 0 the element might already exist + XValue old_value; + if (fidelity > 0 && kv_index_ptr->updateExisting(xvalue, &old_value)) { + auto kv_value = old_value.m_value; + lofi_store<2>::fromValue(kv_value).set(field_id.getOffset(), value.m_store); + xvalue.m_value = kv_value; + kv_index_ptr->updateExisting(xvalue); + // in case of the IttyIndex updating an element changes the address + // which needs to be updated in the object + if (kv_index_ptr->getIndexType() == bindex::type::itty) { + modify().m_kv_address = kv_index_ptr->getAddress(); + } + } else { + if (kv_index_ptr->insert(xvalue)) { + // type or address of the kv-index has changed which needs to be reflected + modify().m_kv_address = kv_index_ptr->getAddress(); + modify().m_kv_type = kv_index_ptr->getIndexType(); + } + } + } + + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + } + + template + std::pair + ObjectImplBase::tryGetMemberSlot(const MemberID &member_id, unsigned int &pos) const + { + for (auto &field_info: member_id) { + auto [index, offset] = field_info.first.getIndexAndOffset(); + // pos-vt lookup + if ((*this)->pos_vt().find(index, pos)) { + if (field_info.second == 0 || slotExists((*this)->pos_vt().values()[pos], field_info.second, offset)) { + return { field_info, &(*this)->pos_vt() }; + } else { + continue; + } + } + + // index-vt lookup + if ((*this)->index_vt().find(index, pos)) { + if (field_info.second == 0 || slotExists((*this)->index_vt().xvalues()[pos].m_value, field_info.second, offset)) { + return { field_info, &(*this)->index_vt() }; + } else { + continue; + } + } + + // kv-index lookup + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + XValue value(index); + if (kv_index_ptr->findOne(value)) { + if (field_info.second == 0 || slotExists(value.m_value, field_info.second, offset)) { + return { field_info, nullptr }; + } + } + } + } + + // not found or deleted + return { {}, nullptr }; + } + + template + std::pair ObjectImplBase::tryGetLoc(FieldID field_id) const + { + auto index = field_id.getIndex(); + unsigned int pos = 0; + // pos-vt lookup + if ((*this)->pos_vt().find(index, pos)) { + return { &(*this)->pos_vt(), pos }; + } + // index-vt lookup + if ((*this)->index_vt().find(index, pos)) { + return { &(*this)->index_vt(), pos }; + } + // not found or located in the kv-index + return { nullptr, 0 }; + } + + template + void ObjectImplBase::setWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, + unsigned int fidelity, StorageClass storage_class, Value value) + { + if (loc_ptr == &(*this)->pos_vt()) { + setPosVT(fixture, field_id, pos, fidelity, storage_class, value); + return; + } + + if (loc_ptr == &(*this)->index_vt()) { + setIndexVT(fixture, field_id, pos, fidelity, storage_class, value); + return; + } + + // must be in the kv-index + assert(!loc_ptr); + setKVIndexValue(fixture, field_id, fidelity, storage_class, value); + } + + template + void ObjectImplBase::addWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, + unsigned int fidelity, StorageClass storage_class, Value value) + { + if (loc_ptr == &(*this)->pos_vt()) { + addToPosVT(fixture, field_id, pos, fidelity, storage_class, value); + return; + } + + if (loc_ptr == &(*this)->index_vt()) { + addToIndexVT(fixture, field_id, pos, fidelity, storage_class, value); + return; + } + + assert(!loc_ptr); + addToKVIndex(fixture, field_id, fidelity, storage_class, value); + } + + template + void ObjectImplBase::set(FixtureLock &fixture, const char *field_name, ObjectPtr lang_value) + { + assert(hasInstance()); + // attribute delete operation + if (!PyToolkit::isValid(lang_value)) { + remove(fixture, field_name); + return; + } + + auto [type_id, storage_class] = recognizeType(**fixture, lang_value); + + if (this->span() > 1) { + // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag + // this is because the actual change may be missed if performed on a different-then the 1st DP + _touch(); + } + + assert(m_type); + // find already existing field index + auto [member_id, is_init_var] = m_type->findField(field_name); + auto storage_fidelity = getStorageFidelity(storage_class); + // get field ID matching the required storage fidelity + FieldID field_id; + FieldInfo old_field_info; + unsigned int old_pos = 0; + const void *old_loc_ptr = nullptr; + if (member_id) { + std::tie(old_field_info, old_loc_ptr) = tryGetMemberSlot(member_id, old_pos); + } + + if (!member_id || !(field_id = member_id.tryGet(storage_fidelity))) { + // try mutating the class first + member_id = m_type->addField(field_name, storage_fidelity); + field_id = member_id.get(storage_fidelity); + } + + assert(field_id && member_id); + // NOTE: a new member inherits the parent's no-cache flag + // FIXME: value should be destroyed on exception + auto value = createMember(*fixture, type_id, storage_class, lang_value, getMemberFlags()); + // make sure object address is not null + assert(!(storage_class == StorageClass::OBJECT_REF && value.cast() == 0)); + + if (field_id == old_field_info.first) { + // Set / update value at the existing location + setWithLoc(fixture, field_id, old_loc_ptr, old_pos, storage_fidelity, storage_class, value); + } else { + // must reset / unreference the old value (stored elsewhere) + if (old_field_info.first) { + unrefWithLoc(fixture, old_field_info.first, old_loc_ptr, old_pos, StorageClass::UNDEFINED, + old_field_info.second); + } + + const void *loc_ptr = nullptr; + unsigned int pos = 0; + // NOTE: slot may already exist (pos-vt or index-vt) either for regular or lo-fi storage + std::tie(loc_ptr, pos) = tryGetLoc(field_id); + // Either use existing slot or create a new (kv-index) + addWithLoc(fixture, field_id, loc_ptr, pos, storage_fidelity, storage_class, value); + } + + // the KV-index insert operation must be registered as the potential silent mutation + // but the operation can be avoided if the object is already marked as modified + if (!super_t::isModified()) { + this->_touch(); + } + } + + template + std::pair ObjectImplBase::findField(const char *name) const + { + if (isDropped()) { + // defunct objects should not be accessed + assert(!isDefunct()); + THROWF(db0::InputException) << "Object does not exist"; + } + + auto class_ptr = m_type.get(); + if (!class_ptr) { + // retrieve class from the initializer + class_ptr = &m_init_manager.getInitializer(*this).getClass(); + } + + assert(class_ptr); + return class_ptr->findField(name); + } + + template + FieldID ObjectImplBase::tryGetMember(const char *field_name, std::pair &member, + bool &is_init_var) const + { + MemberID member_id; + std::tie(member_id, is_init_var) = this->findField(field_name); + bool exists, deleted = false; + if (member_id) { + std::tie(exists, deleted) = tryGetMemberAt(member_id.primary(), member); + if (exists) { + assert(!deleted); + return member_id.primary().first; + } + + // the primary slot was not occupied, try with the secondary + bool secondary_deleted = false; + std::tie(exists, secondary_deleted) = tryGetMemberAt(member_id.secondary(), member); + if (exists) { + assert(!secondary_deleted); + return member_id.secondary().first; + } + + deleted |= secondary_deleted; + } + + if (is_init_var) { + // unless explicitly deleted, + // report as None even if the field_id has not been assigned yet + member = { deleted ? StorageClass::DELETED : StorageClass::NONE, Value() }; + } + + // member not found + return {}; + } + + template + std::optional ObjectImplBase::tryGetX(const char *field_name) const + { + auto [member_id, is_init_var] = this->findField(field_name); + bool exists, deleted = false; + if (member_id) { + assert(member_id.primary().first); + std::pair member; + std::tie(exists, deleted) = tryGetMemberAt(member_id.primary(), member); + if (exists) { + assert(!deleted); + return XValue(member_id.primary().first.getIndex(), member.first, member.second); + } + // the primary slot was not occupied, try with the secondary + bool secondary_deleted = false; + std::tie(exists, secondary_deleted) = tryGetMemberAt(member_id.secondary(), member); + if (exists) { + assert(!secondary_deleted); + return XValue(member_id.secondary().first.getIndex(), member.first, member.second); + } + deleted |= secondary_deleted; + } + + if (!deleted && is_init_var) { + // unless explicitly deleted, + // report as None even if the field_id has not been assigned yet + return XValue(member_id.primary().first.getIndex(), StorageClass::NONE, Value()); + } + + return std::nullopt; + } + + template + ObjectImplBase::ObjectSharedPtr ObjectImplBase::tryGet(const char *field_name) const + { + std::pair member; + bool is_init_var = false; + auto field_id = tryGetMember(field_name, member, is_init_var); + // NOTE: init vars are always reported as None if not explicitly set nor explicitly deleted + if (field_id || (is_init_var && member.first != StorageClass::DELETED)) { + auto fixture = this->getFixture(); + // prevent accessing a deleted or undefined member + assert(member.first != StorageClass::DELETED && member.first != StorageClass::UNDEFINED); + // NOTE: offset is required for lo-fi members + return unloadMember( + fixture, member.first, member.second, field_id.maybeOffset(), this->getMemberFlags() + ); + } + + return nullptr; + } + + template + ObjectImplBase::ObjectSharedPtr ObjectImplBase::tryGetAs( + const char *field_name, TypeObjectPtr lang_type) const + { + std::pair member; + bool is_init_var = false; + auto field_id = tryGetMember(field_name, member, is_init_var); + if (field_id || (is_init_var && member.first != StorageClass::DELETED)) { + // prevent accessing a deleted member + assert(member.first != StorageClass::DELETED && member.first != StorageClass::UNDEFINED); + auto fixture = this->getFixture(); + if (member.first == StorageClass::OBJECT_REF) { + auto &class_factory = getClassFactory(*fixture); + return PyToolkit::unloadObject(fixture, member.second.asAddress(), class_factory, lang_type); + } + + // NOTE: offset is required for lo-fi members + return unloadMember( + fixture, member.first, member.second, field_id.getOffset(), this->getMemberFlags() + ); + } + + return nullptr; + } + + template + ObjectImplBase::ObjectSharedPtr ObjectImplBase::get(const char *field_name) const + { + auto obj = tryGet(field_name); + if (!obj) { + if (isDropped()) { + THROWF(db0::InputException) << "Object is no longer accessible"; + } + THROWF(db0::InputException) << "Attribute not found: " << field_name; + } + return obj; + } + + template + bool ObjectImplBase::slotExists(Value value, unsigned int fidelity, unsigned int at) const + { + assert(fidelity != 0 && "Operation only available for lo-fi values"); + // lo-fi value + assert(fidelity == 2); + return lofi_store<2>::fromValue(value).isSet(at); + } + + template + std::pair ObjectImplBase::hasValueAt(Value value, unsigned int fidelity, unsigned int at) const + { + assert(fidelity != 0 && "Operation only available for lo-fi values"); + // lo-fi value + assert(fidelity == 2); + if (lofi_store<2>::fromValue(value).isSet(at)) { + // might be deleted + bool deleted = (lofi_store<2>::fromValue(value).get(at) == Value::DELETED); + return { !deleted, deleted }; + } else { + // NOTE: unset value is assumed as empty / undefined + return { false, false }; + } + } + + template + std::pair ObjectImplBase::tryGetMemberAt(std::pair field_info, + std::pair &result) const + { + if (!field_info.first) { + return { false, false }; + } + + auto loc = field_info.first.getIndexAndOffset(); + if (!hasInstance()) { + // try retrieving from initializer + auto initializer_ptr = m_init_manager.findInitializer(*this); + if (!initializer_ptr) { + return { false, false }; + } + return { initializer_ptr->tryGetAt(loc, result), false }; + } + + // retrieve from positionally encoded values + if ((*this)->pos_vt().find(loc.first, result)) { + // NOTE: removed field slots might be marked as DELETED + if (result.first == StorageClass::DELETED) { + // report as deleted + return { false, true }; + } + + if (field_info.second == 0) { + return { result.first != StorageClass::UNDEFINED, false }; + } else { + return hasValueAt(result.second, field_info.second, loc.second); + } + } + + if ((*this)->index_vt().find(loc.first, result)) { + if (result.first == StorageClass::DELETED) { + // report as deleted + return { false, true }; + } + + if (field_info.second == 0) { + return { result.first != StorageClass::UNDEFINED, false }; + } else { + return hasValueAt(result.second, field_info.second, loc.second); + } + } + + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + XValue xvalue(loc.first); + if (kv_index_ptr->findOne(xvalue)) { + assert(xvalue.getIndex() == loc.first); + if (xvalue.m_type == StorageClass::DELETED) { + // report as deleted + return { false, true }; + } + + // member fetched from the kv_index + result.first = xvalue.m_type; + result.second = xvalue.m_value; + + if (field_info.second == 0) { + return { result.first != StorageClass::UNDEFINED, false }; + } else { + return hasValueAt(result.second, field_info.second, loc.second); + } + } + } + + // Does not exist, not explicitly removed + return { false, false }; + } + + template + db0::swine_ptr ObjectImplBase::tryGetFixture() const + { + if (!hasInstance()) { + if (isDropped()) { + return {}; + } + // retrieve from the initializer + return m_init_manager.getInitializer(*this).tryGetFixture(); + } + return super_t::tryGetFixture(); + } + + template + db0::swine_ptr ObjectImplBase::getFixture() const + { + auto fixture = this->tryGetFixture(); + if (!fixture) { + THROWF(db0::InternalException) << "Object is no longer accessible"; + } + return fixture; + } + + template + Memspace &ObjectImplBase::getMemspace() const { + return *getFixture(); + } + + template + void ObjectImplBase::setType(std::shared_ptr type) + { + assert(!m_type); + m_type = type; + assert(hasValidClassRef()); + } + + template + void ObjectImplBase::setTypeWithHint(std::shared_ptr type_hint) + { + assert(!m_type); + assert(type_hint); + assert(hasInstance()); + if (type_hint->getClassRef() == (*this)->getClassRef()) { + m_type = type_hint; + } else { + m_type = unloadType(); + } + } + + template + bool ObjectImplBase::isSingleton() const { + return getType().isSingleton(); + } + + template + void ObjectImplBase::dropTags(Class &type) const + { + // only drop if any type tags are assigned + if ((*this)->m_header.m_ref_counter.getFirst() > 0) { + auto fixture = this->getFixture(); + assert(fixture); + auto &tag_index = fixture->get(); + const Class *type_ptr = &type; + auto unique_address = this->getUniqueAddress(); + while (type_ptr) { + // remove auto-assigned type (or its base) tag + tag_index.removeTypeTag(unique_address, type_ptr->getAddress()); + // NOTE: no need to decRef since object is being destroyed + type_ptr = type_ptr->getBaseClassPtr(); + } + } + } + + template + void ObjectImplBase::dropMembers(Class &class_ref) const + { + auto fixture = this->getFixture(); + assert(fixture); + // drop pos-vt members first + { + auto &types = (*this)->pos_vt().types(); + auto &values = (*this)->pos_vt().values(); + auto value = values.begin(); + unsigned int index = types.offset(); + for (auto type = types.begin(); type != types.end(); ++type, ++value, ++index) { + if (*type == StorageClass::DELETED || *type == StorageClass::UNDEFINED) { + // skip undefined or deleted members + continue; + } + unrefMember(fixture, *type, *value); + class_ref.removeFromSchema(index, *type, *value); + } + } + // drop index-vt members next + { + auto &xvalues = (*this)->index_vt().xvalues(); + for (auto &xvalue: xvalues) { + if (xvalue.m_type == StorageClass::DELETED || xvalue.m_type == StorageClass::UNDEFINED) { + // skip undefined or deleted members + continue; + } + unrefMember(fixture, xvalue); + class_ref.removeFromSchema(xvalue); + } + } + // finally drop kv-index members + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto it = kv_index_ptr->beginJoin(1); + for (;!it.is_end(); ++it) { + if ((*it).m_type == StorageClass::DELETED || (*it).m_type == StorageClass::UNDEFINED) { + // skip undefined or deleted members + continue; + } + unrefMember(fixture, *it); + class_ref.removeFromSchema(*it); + } + } + } + + template + void ObjectImplBase::unSingleton(FixtureLock &) + { + auto &type = getType(); + // drop reference from the class + if (type.isSingleton()) { + // clear singleton address + type.unlinkSingleton(); + modify().m_header.decRef(false); + } + } + + template + void ObjectImplBase::destroy() const + { + if (hasInstance()) { + // associated class type (may require unloading) + auto type = m_type; + if (!type) { + // retrieve type from the initializer + type = std::const_pointer_cast(unloadType()); + } + + dropTags(*type); + dropMembers(*type); + // dereference associated class + type->decRef(false); + } + super_t::destroy(); + } + + template + FieldLayout ObjectImplBase::getFieldLayout() const + { + FieldLayout layout; + // collect pos-vt information + for (auto type: (*this)->pos_vt().types()) { + layout.m_pos_vt_fields.push_back(type); + } + + // collect index-vt information + for (auto &xvalue: (*this)->index_vt().xvalues()) { + layout.m_index_vt_fields.emplace_back(xvalue.getIndex(), xvalue.m_type); + } + + // collect kv-index information + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto it = kv_index_ptr->beginJoin(1); + for (;!it.is_end(); ++it) { + layout.m_kv_index_fields.emplace_back((*it).getIndex(), (*it).m_type); + } + } + + return layout; + } + + template + KV_Index *ObjectImplBase::addKV_First(const XValue &value) + { + if (!m_kv_index) { + if ((*this)->m_kv_address) { + m_kv_index = std::make_unique( + std::make_pair(&getMemspace(), (*this)->m_kv_address), (*this)->m_kv_type + ); + } else { + // create new kv-index intiialized with the first value + m_kv_index = std::make_unique(getMemspace(), value); + modify().m_kv_address = m_kv_index->getAddress(); + modify().m_kv_type = m_kv_index->getIndexType(); + // return nullptr to indicate that the value has been inserted + return nullptr; + } + } + // return reference without inserting + return m_kv_index.get(); + } + + template + bool ObjectImplBase::hasKV_Index() const { + return m_kv_index || (*this)->m_kv_address; + } + + template + KV_Index *ObjectImplBase::tryGetKV_Index() const + { + // if KV index address has changed, update the cached instance + if (!m_kv_index || m_kv_index->getAddress() != (*this)->m_kv_address) { + if ((*this)->m_kv_address) { + m_kv_index = std::make_unique( + std::make_pair(&getMemspace(), (*this)->m_kv_address), (*this)->m_kv_type + ); + } + } + + return m_kv_index.get(); + } + + template + Class &ObjectImplBase::getType() { + return m_type ? *m_type : m_init_manager.getInitializer(*this).getClass(); + } + + template + void ObjectImplBase::getMembersFrom(const Class &this_type, unsigned int index, StorageClass storage_class, + Value value, std::unordered_set &result) const + { + if (storage_class == StorageClass::DELETED || storage_class == StorageClass::UNDEFINED) { + // skip undefined or deleted members + return; + } + + if (storage_class == StorageClass::PACK_2) { + auto it = lofi_store<2>::fromValue(value).begin(), end = lofi_store<2>::fromValue(value).end(); + for (; it != end; ++it) { + result.insert(this_type.getMember({ index, it.getOffset() }).m_name); + } + } else { + result.insert(this_type.getMember(FieldID::fromIndex(index)).m_name); + } + } + + template + std::unordered_set ObjectImplBase::getMembers() const + { + std::unordered_set result; + // Visit pos-vt members first + auto &obj_type = this->getType(); + { + auto &types = (*this)->pos_vt().types(); + auto &values = (*this)->pos_vt().values(); + unsigned int index = types.offset(); + auto size = types.size(); + for (unsigned int pos = 0;pos < size; ++index, ++pos) { + getMembersFrom(obj_type, index, types[pos], values[pos], result); + } + } + + // Visit index-vt members next + { + auto &xvalues = (*this)->index_vt().xvalues(); + for (auto &xvalue: xvalues) { + auto index = xvalue.getIndex(); + getMembersFrom(obj_type, index, xvalue.m_type, xvalue.m_value, result); + } + } + + // Finally, visit kv-index members + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto it = kv_index_ptr->beginJoin(1); + for (;!it.is_end(); ++it) { + auto index = (*it).getIndex(); + getMembersFrom(obj_type, index, (*it).m_type, (*it).m_value, result); + } + } + return result; + } + + template + void ObjectImplBase::forAll(std::function f) const + { + // Visit pos-vt members first + auto &obj_type = this->getType(); + { + auto &types = (*this)->pos_vt().types(); + auto &values = (*this)->pos_vt().values(); + auto value = values.begin(); + unsigned int index = types.offset(); + for (auto type = types.begin(); type != types.end(); ++type, ++value, ++index) { + if (*type == StorageClass::DELETED || *type == StorageClass::UNDEFINED) { + // skip deleted or undefined members + continue; + } + if (*type == StorageClass::PACK_2) { + // iterate individual lo-fi members + if (!forAll({index, *type, *value}, f)) { + return; + } + } else { + if (!f(obj_type.getMember(FieldID::fromIndex(index)).m_name, { index, *type, *value }, 0)) { + return; + } + } + } + } + + // Visit index-vt members next + { + auto &xvalues = (*this)->index_vt().xvalues(); + for (auto &xvalue: xvalues) { + if (xvalue.m_type == StorageClass::DELETED || xvalue.m_type == StorageClass::UNDEFINED) { + // skip deleted or undefined members + continue; + } + if (xvalue.m_type == StorageClass::PACK_2) { + // iterate individual lo-fi members + if (!forAll(xvalue, f)) { + return; + } + } else { + // regular member + if (!f(obj_type.getMember(FieldID::fromIndex(xvalue.getIndex())).m_name, xvalue, 0)) { + return; + } + } + } + } + + // Finally, visit kv-index members + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto it = kv_index_ptr->beginJoin(1); + for (;!it.is_end(); ++it) { + if ((*it).m_type == StorageClass::DELETED || (*it).m_type == StorageClass::UNDEFINED) { + // skip deleted or undefined members + continue; + } + if ((*it).m_type == StorageClass::PACK_2) { + // iterate individual lo-fi members + if (!forAll(*it, f)) { + return; + } + } else { + if (!f(obj_type.getMember(FieldID::fromIndex((*it).getIndex())).m_name, *it, 0)) { + return; + } + } + } + } + } + + template + void ObjectImplBase::forAll(std::function f) const + { + auto fixture = this->getFixture(); + forAll([&](const std::string &name, const XValue &xvalue, unsigned int offset) -> bool { + // all references convert to UUID + auto py_member = unloadMember( + fixture, xvalue.m_type, xvalue.m_value, offset, this->getMemberFlags() + ); + return f(name, py_member); + }); + } + + template + bool ObjectImplBase::forAll(XValue xvalue, + std::function f) const + { + assert(xvalue.m_type == StorageClass::PACK_2); + unsigned int index = xvalue.getIndex(); + auto _value = xvalue.m_value; + auto it = lofi_store<2>::fromValue(_value).begin(), end = lofi_store<2>::fromValue(_value).end(); + auto &obj_type = this->getType(); + for (; it != end; ++it) { + if (!f(obj_type.getMember(FieldID::fromIndex(index, it.getOffset())).m_name, + xvalue, it.getOffset())) + { + return false; + } + } + return true; + } + + template + void ObjectImplBase::incRef(bool is_tag) + { + if (hasInstance()) { + super_t::incRef(is_tag); + } else { + // incRef with the initializer + m_init_manager.getInitializer(*this).incRef(is_tag); + } + } + + template + bool ObjectImplBase::decRef(bool is_tag) + { + // this operation is a potentially silent mutation + _touch(); + super_t::decRef(is_tag); + return !hasRefs(); + } + + template + bool ObjectImplBase::hasRefs() const + { + assert(hasInstance()); + return (*this)->hasRefs(); + } + + template + bool ObjectImplBase::hasAnyRefs() const { + return (*this)->hasAnyRefs(); + } + + template + bool ObjectImplBase::hasTagRefs() const { + return this->hasInstance() && (*this)->m_header.m_ref_counter.getFirst() > 0; + } + + template + bool ObjectImplBase::equalTo(const ObjectImplBase &other) const + { + if (!hasInstance() || !other.hasInstance()) { + THROWF(db0::InputException) << "Object not initialized"; + } + + if (this->isDefunct() || other.isDefunct()) { + // defunct objects should not be compared + assert(!isDefunct()); + THROWF(db0::InputException) << "Object does not exist"; + } + + if ((*this)->getClassRef() != other->getClassRef()) { + // different types + return false; + } + + if (this->getFixture()->getUUID() == other.getFixture()->getUUID() + && this->getUniqueAddress() == other.getUniqueAddress()) + { + // comparing 2 versions of the same object (fastest) + if (!((*this)->pos_vt() == other->pos_vt())) { + return false; + } + if (!((*this)->index_vt() == other->index_vt())) { + return false; + } + if (!hasKV_Index() && !other.hasKV_Index()) { + return true; + } + return isEqual(this->tryGetKV_Index(), other.tryGetKV_Index()); + } + + // field-wise compare otherwise (slower) + bool result = true; + this->forAll([&](const std::string &name, const XValue &xvalue, unsigned int offset) -> bool { + auto maybe_other_value = other.tryGetX(name.c_str()); + if (!maybe_other_value) { + result = false; + return false; + } + + if (!xvalue.equalTo(*maybe_other_value, offset)) { + result = false; + return false; + } + return true; + }); + return result; + } + + template + void ObjectImplBase::moveTo(db0::swine_ptr &) { + throw std::runtime_error("Not implemented"); + } + + template + void ObjectImplBase::setFixture(db0::swine_ptr &new_fixture) + { + if (hasInstance()) { + THROWF(db0::InputException) << "set_prefix failed: object already initialized"; + } + + if (!m_init_manager.getInitializer(*this).trySetFixture(new_fixture)) { + // signal problem with PyErr_BadPrefix + auto fixture = this->getFixture(); + LangToolkit::setError(LangToolkit::getTypeManager().getBadPrefixError(), fixture->getUUID()); + } + } + + template + void ObjectImplBase::detach() const + { + m_type->detach(); + // invalidate since detach is not supported by the MorphingBIndex + m_kv_index = nullptr; + super_t::detach(); + } + + template + void ObjectImplBase::commit() const + { + m_type->commit(); + if (m_kv_index) { + m_kv_index->commit(); + } + super_t::commit(); + // reset the silent-mutation flag + m_touched = false; + } + + template + void ObjectImplBase::unrefMember(db0::swine_ptr &fixture, StorageClass type, Value value) const { + db0::object_model::unrefMember(fixture, type, value); + } + + template + void ObjectImplBase::unrefMember(db0::swine_ptr &fixture, XValue value) const { + db0::object_model::unrefMember(fixture, value.m_type, value.m_value); + } + + template + std::shared_ptr ObjectImplBase::unloadType() const + { + auto fixture = this->getFixture(); + return getClassFactory(*fixture).getTypeByClassRef((*this)->getClassRef()).m_class; + } + + template + Address ObjectImplBase::getAddress() const + { + assert(!isDefunct()); + if (!hasInstance()) { + THROWF(db0::InternalException) << "Object instance does not exist yet (did you forget to use db0.materialized(self) in constructor ?)"; + } + return super_t::getAddress(); + } + + template + UniqueAddress ObjectImplBase::getUniqueAddress() const + { + if (hasInstance()) { + return super_t::getUniqueAddress(); + } else { + // NOTE: defunct objects don't have a valid address (not assigned yet) + assert(m_flags[ObjectOptions::DROPPED]); + return m_unique_address; + } + } + + template + bool ObjectImplBase::hasValidClassRef() const + { + if (hasInstance() && m_type) { + return (*this)->getClassRef() == m_type->getClassRef(); + } + return true; + } + + template + std::shared_ptr ObjectImplBase::getTypeWithHint(const Fixture &fixture, std::uint32_t class_ref, + std::shared_ptr type_hint) + { + assert(type_hint); + if (type_hint->getClassRef() == class_ref) { + return type_hint; + } + return getClassFactory(fixture).getTypeByClassRef(class_ref).m_class; + } + + template + void ObjectImplBase::setDefunct() const { + m_flags.set(ObjectOptions::DEFUNCT); + } + + template + void ObjectImplBase::touch() + { + if (hasInstance() && !isDefunct()) { + // NOTE: for already modified and small objects we may skip "touch" + if (!super_t::isModified() || this->span() > 1) { + // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag + // this is because the actual change may be missed if performed on a different-then the 1st DP + _touch(); + } + } + } + + template + void ObjectImplBase::_touch() + { + if (!m_touched) { + // mark the 1st byte of the object as modified (forced-diff) + // this is always the 1st DP occupied by the object + modify(0, 1); + m_touched = true; + } + } + + template + void ObjectImplBase::addExtRef() const { + ++m_ext_refs; + } + + template + void ObjectImplBase::removeExtRef() const + { + assert(m_ext_refs > 0); + --m_ext_refs; + } + +} \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectImplBase.hpp b/src/dbzero/object_model/object/ObjectImplBase.hpp new file mode 100644 index 00000000..0201919b --- /dev/null +++ b/src/dbzero/object_model/object/ObjectImplBase.hpp @@ -0,0 +1,343 @@ +#pragma once + +#include +#include +#include +#include +#include +#include "ValueTable.hpp" +#include "ObjectInitializer.hpp" +#include +#include +#include +#include +#include "o_object.hpp" + +namespace db0 + +{ + + class Fixture; + +} + +namespace db0::object_model + +{ + + class Class; + using Fixture = db0::Fixture; + + enum class ObjectOptions: std::uint8_t + { + // the dbzero instance has been deleted + DROPPED = 0x01, + // object is defunct - e.g. due to exception on __init__ + DEFUNCT = 0x02 + }; + + struct FieldLayout + { + std::vector m_pos_vt_fields; + std::vector > m_index_vt_fields; + std::vector > m_kv_index_fields; + }; + + using ObjectFlags = db0::FlagSet; + // NOTE: Object instances are created within the implementation specific realm_id (e.g. =1 for o_object) + template using ObjectVType = db0::v_object; + + template + class ObjectImplBase: public db0::ObjectBase, StorageClass::OBJECT_REF> + { + public: + static constexpr unsigned char REALM_ID = T::REALM_ID; + using super_t = db0::ObjectBase, StorageClass::OBJECT_REF>; + using LangToolkit = LangConfig::LangToolkit; + using ObjectPtr = typename LangToolkit::ObjectPtr; + using TypeObjectPtr = typename LangToolkit::TypeObjectPtr; + using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; + using TypeManager = typename LangToolkit::TypeManager; + using ObjectStem = ObjectVType; + using TypeInitializer = ObjectInitializer::TypeInitializer; + + // Construct as null / dropped object + ObjectImplBase(UniqueAddress, unsigned int ext_refs); + ObjectImplBase(const Object &) = delete; + ObjectImplBase(Object &&) = delete; + + /** + * Construct new Object (uninitialized, without corresponding dbzero instance yet) + */ + ObjectImplBase(std::shared_ptr); + ObjectImplBase(TypeInitializer &&); + + // Unload from address with a known type (possibly a base type) + // NOTE: unload works faster if type_hint is the exact object's type + struct with_type_hint {}; + ObjectImplBase(db0::swine_ptr &, Address, std::shared_ptr type_hint, + with_type_hint, AccessFlags = {}); + + // Unload from stem with a known type (possibly a base type) + // NOTE: unload works faster if type_hint is the exact object's type + ObjectImplBase(db0::swine_ptr &, ObjectStem &&, std::shared_ptr type_hint, + with_type_hint); + + ObjectImplBase(db0::swine_ptr &, Address, AccessFlags = {}); + ObjectImplBase(db0::swine_ptr &, std::shared_ptr, std::pair ref_counts, const PosVT::Data &, unsigned int pos_vt_offset); + ObjectImplBase(db0::swine_ptr &, ObjectStem &&, std::shared_ptr); + + ~ObjectImplBase(); + + // post-init invoked by memo type directly after __init__ + void postInit(FixtureLock &); + + // Destroys an existing instance and constructs a "null" placeholder + void dropInstance(FixtureLock &); + + // Unload the object stem, to retrieve its type + static ObjectStem tryUnloadStem(db0::swine_ptr &, Address, + std::uint16_t instance_id = 0, AccessFlags = {}); + static ObjectStem unloadStem(db0::swine_ptr &, Address, + std::uint16_t instance_id = 0, AccessFlags = {}); + + // Called to finalize adding members + void endInit(); + + // Assign language specific value as a field (to already initialized or uninitialized instance) + // NOTE: if lang_value is nullptr then the member is removed + void set(FixtureLock &, const char *field_name, ObjectPtr lang_value); + void remove(FixtureLock &, const char *field_name); + + // Assign field of an uninitialized instance (assumed as a non-mutating operation) + // NOTE: if lang_value is nullptr then the member is removed + void setPreInit(const char *field_name, ObjectPtr lang_value) const; + void removePreInit(const char *field_name) const; + + ObjectSharedPtr tryGet(const char *field_name) const; + ObjectSharedPtr tryGetAs(const char *field_name, TypeObjectPtr) const; + ObjectSharedPtr get(const char *field_name) const; + + inline std::shared_ptr getClassPtr() const { + return m_type ? m_type : m_init_manager.getInitializer(*this).getClassPtr(); + } + + inline const Class &getType() const { + return m_type ? *m_type : m_init_manager.getInitializer(*this).getClass(); + } + + Class &getType(); + + db0::swine_ptr tryGetFixture() const; + + db0::swine_ptr getFixture() const; + + Memspace &getMemspace() const; + + /** + * Get description of the field layout + */ + FieldLayout getFieldLayout() const; + + // Convert singleton into a regular instance + void unSingleton(FixtureLock &); + + void destroy() const; + + bool isSingleton() const; + + // execute the function for all members (until false is returned from the input lambda) + void forAll(std::function) const; + void forAll(std::function) const; + + // get dbzero member / member names assigned to this object + std::unordered_set getMembers() const; + + /** + * The overloaded incRef implementation is provided to also handle non-fully initialized objects + */ + void incRef(bool is_tag); + bool hasRefs() const; + // check for any refs (including auto-assigned type tags) + bool hasAnyRefs() const; + + // check if any references from tags exist (i.e. are any tags assigned) + bool hasTagRefs() const; + + // @return true if reference count was decremented to zero + bool decRef(bool is_tag); + + // Binary (shallow) compare 2 objects or 2 versions of the same memo object (e.g. from different snapshots) + // NOTE: ref-counts are not compared (only user-assigned members) + // @return true if objects are identical + bool equalTo(const Object &) const; + + /** + * Move unreferenced object to a different prefix without changing the instance + * this operations is required for auto-hardening + */ + void moveTo(db0::swine_ptr &); + + /** + * Change fixture of the uninitialized object + * Object must not have any members yet either + */ + void setFixture(db0::swine_ptr &); + + void detach() const; + + void commit() const; + + Address getAddress() const; + UniqueAddress getUniqueAddress() const; + + // NOTE: the operation is marked const because the dbzero state is not affected + void setDefunct() const; + + inline bool isDropped() const { + return m_flags.test(ObjectOptions::DROPPED); + } + + inline bool isDefunct() const { + return m_flags.test(ObjectOptions::DEFUNCT); + } + + // is dropped or defunct + inline bool isDead() const { + return m_flags.any( + static_cast(ObjectOptions::DROPPED) | static_cast(ObjectOptions::DEFUNCT) + ); + } + + // FieldID, is_init_var, fidelity + std::pair findField(const char *name) const; + + // Check if the 2 memo objects are of the same type + bool sameType(const Object &) const; + + // the member called to indicate the object mutation + void touch(); + + void addExtRef() const; + void removeExtRef() const; + + inline std::uint32_t getExtRefs() const { + return m_ext_refs; + } + + protected: + // Class will only be assigned after initialization + std::shared_ptr m_type; + // local kv-index instance cache (created at first use) + mutable std::unique_ptr m_kv_index; + static ObjectInitializerManager m_init_manager; + mutable ObjectFlags m_flags; + // reference counter for inner references from language objects + // NOTE: inner references are held by internal dbzero buffers (e.g. TagIndex) + // see also PyEXT_INCREF / PyEXT_DECREF + mutable std::uint32_t m_ext_refs = 0; + // A flag indicating that object's silent mutation has already been reflected + // with the underlying MemLock / ResourceLock + // NOTE: by silent mutation we mean a mutation that does not change data (e.g. +refcount(+-1) + (refcount-1)) + mutable bool m_touched = false; + // NOTE: member assigned only to dropped objects (see replaceWithNull) + // so that we can retrieve the address of the dropped instance after it has been destroyed + const UniqueAddress m_unique_address; + + void setType(std::shared_ptr); + // adjusts to actual type if the type hint is a base class + void setTypeWithHint(std::shared_ptr type_hint); + // @return exists / deleted + std::pair hasValueAt(Value, unsigned int fidelity, unsigned int at) const; + // similar to hasValueAt but assume deleted slot as present + bool slotExists(Value value, unsigned int fidelity, unsigned int at) const; + + // Try retrieving member either from values (initialized) or from the initialization buffer (not initialized yet) + // @return member exists, member deleted flags + std::pair tryGetMemberAt(std::pair, + std::pair &) const; + FieldID tryGetMember(const char *field_name, std::pair &, bool &is_init_var) const; + + // Try resolving field ID of an existing (or deleted) member and also its storage location + // @param pos the member's position in the containing collection + // @return FieldID + containing collection (e.g. pos_vt()) + std::pair tryGetMemberSlot(const MemberID &, unsigned int &pos) const; + + // Try locating a field ID associated slot + std::pair tryGetLoc(FieldID) const; + + inline ObjectInitializer *tryGetInitializer() const { + return m_type ? static_cast(nullptr) : &m_init_manager.getInitializer(*this); + } + + /** + * If the KV_Index does not exist yet, create it and add the first value + * otherwise return instance of an existing KV_Index + */ + KV_Index *addKV_First(const XValue &); + + KV_Index *tryGetKV_Index() const; + + void dropMembers(Class &) const; + void dropTags(Class &) const; + + void unrefMember(db0::swine_ptr &, StorageClass, Value) const; + void unrefMember(db0::swine_ptr &, XValue) const; + + using TypeId = db0::bindings::TypeId; + std::pair recognizeType(Fixture &, ObjectPtr lang_value) const; + + // Unload associated type + std::shared_ptr unloadType() const; + + // Retrieve a type by class-ref with a possible match (type_hint) + static std::shared_ptr getTypeWithHint(const Fixture &, std::uint32_t class_ref, std::shared_ptr type_hint); + + bool hasValidClassRef() const; + bool hasKV_Index() const; + + // try retrieving member as XValue + std::optional tryGetX(const char *field_name) const; + void _touch(); + + // Set or update member in a pos_vt + void setPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); + void setIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, + StorageClass, Value); + // Set or update member in kv-index + void setKVIndexValue(FixtureLock &, FieldID, unsigned int fidelity, StorageClass, Value); + + // Set with a specific location (pos_vt, index_vt, kv-index) + void setWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, + StorageClass, Value); + + // Unreference value + // NOTE: storage_class to be assigned can either be DELETED or UNDEFINED + void unrefPosVT(FixtureLock &, FieldID, unsigned int pos, StorageClass, unsigned int fidelity); + void unrefIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, StorageClass, unsigned int fidelity); + void unrefKVIndexValue(FixtureLock &, FieldID, StorageClass, unsigned int fidelity); + + void unrefWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, StorageClass, + unsigned int fidelity); + + // Add a new value + void addToPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); + void addToIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, StorageClass, Value); + void addToKVIndex(FixtureLock &, FieldID, unsigned int fidelity, StorageClass, Value); + + void addWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, + StorageClass, Value); + + // lo-fi member specialized implementation + bool forAll(XValue, std::function) const; + + void getMembersFrom(const Class &this_type, unsigned int index, StorageClass, Value, + std::unordered_set &) const; + }; + + extern template class ObjectImplBase; + +} + +DECLARE_ENUM_VALUES(db0::object_model::ObjectOptions, 2) \ No newline at end of file diff --git a/src/dbzero/object_model/object/o_object.cpp b/src/dbzero/object_model/object/o_object.cpp new file mode 100644 index 00000000..f571b1b3 --- /dev/null +++ b/src/dbzero/object_model/object/o_object.cpp @@ -0,0 +1,76 @@ +#include "o_object.hpp" +#include +#include +#include +#include +#include +#include + +namespace db0::object_model + +{ + + o_object::o_object(std::uint32_t class_ref, std::pair ref_counts, + std::uint8_t num_type_tags, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, + const XValue *index_vt_begin, const XValue *index_vt_end) + : m_header(ref_counts) + , m_num_type_tags(num_type_tags) + { + arrangeMembers() + (PosVT::type(), pos_vt_data, pos_vt_offset) + (packed_int32::type(), class_ref) + (IndexVT::type(), index_vt_begin, index_vt_end); + } + + std::size_t o_object::measure(std::uint32_t class_ref, std::pair, std::uint8_t, + const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, + const XValue *index_vt_begin, const XValue *index_vt_end) + { + return super_t::measureMembers() + (PosVT::type(), pos_vt_data, pos_vt_offset) + (packed_int32::type(), class_ref) + (IndexVT::type(), index_vt_begin, index_vt_end); + } + + const PosVT &o_object::pos_vt() const { + return getDynFirst(PosVT::type()); + } + + PosVT &o_object::pos_vt() { + return getDynFirst(PosVT::type()); + } + + const packed_int32 &o_object::classRef() const { + return getDynAfter(pos_vt(), packed_int32::type()); + } + + std::uint32_t o_object::getClassRef() const { + return classRef().value(); + } + + const IndexVT &o_object::index_vt() const { + return getDynAfter(classRef(), IndexVT::type()); + } + + IndexVT &o_object::index_vt() { + return getDynAfter(classRef(), IndexVT::type()); + } + + void o_object::incRef(bool is_tag) { + m_header.incRef(is_tag); + } + + bool o_object::hasRefs() const + { + // NOTE: type tags are not counted as "proper" references + if (m_header.m_ref_counter.getFirst() > this->m_num_type_tags) { + return true; + } + return m_header.m_ref_counter.getSecond() > 0; + } + + bool o_object::hasAnyRefs() const { + return m_header.hasRefs(); + } + +} \ No newline at end of file diff --git a/src/dbzero/object_model/object/o_object.hpp b/src/dbzero/object_model/object/o_object.hpp new file mode 100644 index 00000000..baea5a5d --- /dev/null +++ b/src/dbzero/object_model/object/o_object.hpp @@ -0,0 +1,75 @@ +#pragma once + +#include +#include +#include "ValueTable.hpp" +#include "ObjectInitializer.hpp" +#include +#include "KV_Index.hpp" + +namespace db0 + +{ + + class Fixture; + +} + +namespace db0::object_model + +{ + + class Class; + using Fixture = db0::Fixture; + +DB0_PACKED_BEGIN + class DB0_PACKED_ATTR o_object: public db0::o_base + { + protected: + using super_t = db0::o_base; + + public: + static constexpr unsigned char REALM_ID = 1; + // common object header + o_unique_header m_header; + // optional address of the key-value store (to store extension fields) + KV_Address m_kv_address; + // kv-index type must be stored separately from the address + bindex::type m_kv_type; + // number of auto-assigned type tags + std::uint8_t m_num_type_tags = 0; + + PosVT &pos_vt(); + const PosVT &pos_vt() const; + + const packed_int32 &classRef() const; + std::uint32_t getClassRef() const; + + const IndexVT &index_vt() const; + + IndexVT &index_vt(); + + // ref_counts - the initial reference counts (tags / objects) inherited from the initializer + o_object(std::uint32_t class_ref, std::pair ref_counts, std::uint8_t num_type_tags, + const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin = nullptr, + const XValue *index_vt_end = nullptr); + + static std::size_t measure(std::uint32_t, std::pair, std::uint8_t num_type_tags, + const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin = nullptr, + const XValue *index_vt_end = nullptr); + + template static std::size_t safeSizeOf(BufT buf) + { + return super_t::sizeOfMembers(buf) + (PosVT::type()) + (packed_int32::type()) + (IndexVT::type()); + } + + void incRef(bool is_tag); + bool hasRefs() const; + bool hasAnyRefs() const; + }; +DB0_PACKED_END + +} \ No newline at end of file From 6cbe123af0dacf2f28722279d7bec3c6f5fc15de Mon Sep 17 00:00:00 2001 From: Wojtek Date: Tue, 4 Nov 2025 09:50:09 +0100 Subject: [PATCH 05/19] refactor --- src/dbzero/object_model/class/Class.cpp | 11 +- src/dbzero/object_model/class/Class.hpp | 20 +- .../object_model/object/ObjectImplBase.cpp | 67 +++--- .../object_model/object/ObjectImplBase.hpp | 8 +- .../object_model/object/ObjectInitializer.cpp | 82 +------- .../object_model/object/ObjectInitializer.hpp | 192 +++++++++++++----- 6 files changed, 193 insertions(+), 187 deletions(-) diff --git a/src/dbzero/object_model/class/Class.cpp b/src/dbzero/object_model/class/Class.cpp index 1c8fb710..d104c588 100644 --- a/src/dbzero/object_model/class/Class.cpp +++ b/src/dbzero/object_model/class/Class.cpp @@ -289,16 +289,7 @@ namespace db0::object_model bool Class::isExistingSingleton() const { return isSingleton() && (*this)->m_singleton_address.isValid(); } - - void Class::setSingletonAddress(Object &object) - { - assert(!(*this)->m_singleton_address.isValid()); - assert(isSingleton()); - // increment reference count in order to prevent singleton object from being destroyed - object.incRef(false); - modify().m_singleton_address = object.getUniqueAddress(); - } - + void Class::onMemberIDUpdated(const MemberID &member_id) const { if (member_id.hasFidelity(PRIMARY_FIDELITY) && member_id.size() > 1) { diff --git a/src/dbzero/object_model/class/Class.hpp b/src/dbzero/object_model/class/Class.hpp index c8ddad71..1dceced2 100644 --- a/src/dbzero/object_model/class/Class.hpp +++ b/src/dbzero/object_model/class/Class.hpp @@ -168,8 +168,8 @@ DB0_PACKED_END // Construct singleton's ObjectId without unloading it ObjectId getSingletonObjectId() const; - - void setSingletonAddress(Object &); + + template void setSingletonAddress(T &); Address getSingletonAddress() const; @@ -258,6 +258,8 @@ DB0_PACKED_END return m_no_cache ? AccessFlags { AccessOptions::no_cache } : AccessFlags {}; } + void unlinkSingleton(); + protected: friend class ClassFactory; friend ClassPtr; @@ -270,9 +272,7 @@ DB0_PACKED_END Class(db0::swine_ptr &, const std::string &name, std::optional module_name, const char *type_id, const char *prefix_name, const std::vector &init_vars, ClassFlags, std::shared_ptr base_class); - - void unlinkSingleton(); - + // Get unique class identifier within its fixture std::uint32_t fetchUID() const; @@ -336,4 +336,14 @@ DB0_PACKED_END std::optional getNameVariant(const Class &, int variant_id); + template + void Class::setSingletonAddress(T &object) + { + assert(!(*this)->m_singleton_address.isValid()); + assert(isSingleton()); + // increment reference count in order to prevent singleton object from being destroyed + object.incRef(false); + modify().m_singleton_address = object.getUniqueAddress(); + } + } diff --git a/src/dbzero/object_model/object/ObjectImplBase.cpp b/src/dbzero/object_model/object/ObjectImplBase.cpp index 836d25b7..6b9b88f5 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.cpp +++ b/src/dbzero/object_model/object/ObjectImplBase.cpp @@ -80,13 +80,13 @@ namespace db0::object_model template ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, Address address, AccessFlags access_mode) - : super_t(super_t::tag_from_address(), fixture, address, access_mode) + : super_t(typename super_t::tag_from_address(), fixture, address, access_mode) { } template ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type) - : super_t(super_t::tag_from_stem(), fixture, std::move(stem)) + : super_t(typename super_t::tag_from_stem(), fixture, std::move(stem)) , m_type(type) { assert(hasValidClassRef()); @@ -137,7 +137,7 @@ namespace db0::object_model return {}; } // Unload from a verified address - ObjectVType stem(db0::tag_verified(), fixture->myPtr(address), size_of, access_mode); + ObjectStem stem(db0::tag_verified(), fixture->myPtr(address), size_of, access_mode); if (instance_id && stem->m_header.m_instance_id != instance_id) { // instance ID validation failed return {}; @@ -473,8 +473,8 @@ namespace db0::object_model template void ObjectImplBase::setPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, StorageClass storage_class, Value value) - { - auto &pos_vt = modify().pos_vt(); + { + auto &pos_vt = this->modify().pos_vt(); auto pos_value = pos_vt.values()[pos]; if (fidelity == 0) { auto old_storage_class = pos_vt.types()[pos]; @@ -496,7 +496,7 @@ namespace db0::object_model void ObjectImplBase::addToPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, StorageClass storage_class, Value value) { - auto &pos_vt = modify().pos_vt(); + auto &pos_vt = this->modify().pos_vt(); auto pos_value = pos_vt.values()[pos]; if (fidelity == 0) { // update attribute stored in the positional value-table @@ -514,7 +514,7 @@ namespace db0::object_model void ObjectImplBase::setIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, unsigned int fidelity, StorageClass storage_class, Value value) { - auto &index_vt = modify().index_vt(); + auto &index_vt = this->modify().index_vt(); if (fidelity == 0) { auto old_storage_class = index_vt.xvalues()[index_vt_pos].m_type; unrefMember(*fixture, index_vt.xvalues()[index_vt_pos]); @@ -581,7 +581,7 @@ namespace db0::object_model // in case of the IttyIndex updating an element changes the address // which needs to be updated in the object if (kv_index_ptr->getIndexType() == bindex::type::itty) { - modify().m_kv_address = kv_index_ptr->getAddress(); + this->modify().m_kv_address = kv_index_ptr->getAddress(); } } else { if (kv_index_ptr->insert(xvalue)) { @@ -619,13 +619,13 @@ namespace db0::object_model // in case of the IttyIndex updating an element changes the address // which needs to be updated in the object if (kv_index_ptr->getIndexType() == bindex::type::itty) { - modify().m_kv_address = kv_index_ptr->getAddress(); + this->modify().m_kv_address = kv_index_ptr->getAddress(); } } else { if (kv_index_ptr->insert(xvalue)) { // type or address of the kv-index has changed which needs to be reflected - modify().m_kv_address = kv_index_ptr->getAddress(); - modify().m_kv_type = kv_index_ptr->getIndexType(); + this->modify().m_kv_address = kv_index_ptr->getAddress(); + this->modify().m_kv_type = kv_index_ptr->getIndexType(); } } } @@ -633,8 +633,7 @@ namespace db0::object_model m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); } - template - std::pair + template std::pair ObjectImplBase::tryGetMemberSlot(const MemberID &member_id, unsigned int &pos) const { for (auto &field_info: member_id) { @@ -767,7 +766,9 @@ namespace db0::object_model assert(field_id && member_id); // NOTE: a new member inherits the parent's no-cache flag // FIXME: value should be destroyed on exception - auto value = createMember(*fixture, type_id, storage_class, lang_value, getMemberFlags()); + auto value = createMember( + *fixture, type_id, storage_class, lang_value, this->getMemberFlags() + ); // make sure object address is not null assert(!(storage_class == StorageClass::OBJECT_REF && value.cast() == 0)); @@ -881,9 +882,9 @@ namespace db0::object_model return std::nullopt; } - + template - ObjectImplBase::ObjectSharedPtr ObjectImplBase::tryGet(const char *field_name) const + typename ObjectImplBase::ObjectSharedPtr ObjectImplBase::tryGet(const char *field_name) const { std::pair member; bool is_init_var = false; @@ -903,7 +904,7 @@ namespace db0::object_model } template - ObjectImplBase::ObjectSharedPtr ObjectImplBase::tryGetAs( + typename ObjectImplBase::ObjectSharedPtr ObjectImplBase::tryGetAs( const char *field_name, TypeObjectPtr lang_type) const { std::pair member; @@ -928,7 +929,7 @@ namespace db0::object_model } template - ObjectImplBase::ObjectSharedPtr ObjectImplBase::get(const char *field_name) const + typename ObjectImplBase::ObjectSharedPtr ObjectImplBase::get(const char *field_name) const { auto obj = tryGet(field_name); if (!obj) { @@ -974,7 +975,7 @@ namespace db0::object_model } auto loc = field_info.first.getIndexAndOffset(); - if (!hasInstance()) { + if (!this->hasInstance()) { // try retrieving from initializer auto initializer_ptr = m_init_manager.findInitializer(*this); if (!initializer_ptr) { @@ -1040,7 +1041,7 @@ namespace db0::object_model template db0::swine_ptr ObjectImplBase::tryGetFixture() const { - if (!hasInstance()) { + if (!this->hasInstance()) { if (isDropped()) { return {}; } @@ -1098,7 +1099,7 @@ namespace db0::object_model if ((*this)->m_header.m_ref_counter.getFirst() > 0) { auto fixture = this->getFixture(); assert(fixture); - auto &tag_index = fixture->get(); + auto &tag_index = fixture->template get(); const Class *type_ptr = &type; auto unique_address = this->getUniqueAddress(); while (type_ptr) { @@ -1165,14 +1166,14 @@ namespace db0::object_model if (type.isSingleton()) { // clear singleton address type.unlinkSingleton(); - modify().m_header.decRef(false); + this->modify().m_header.decRef(false); } } template void ObjectImplBase::destroy() const { - if (hasInstance()) { + if (this->hasInstance()) { // associated class type (may require unloading) auto type = m_type; if (!type) { @@ -1225,8 +1226,8 @@ namespace db0::object_model } else { // create new kv-index intiialized with the first value m_kv_index = std::make_unique(getMemspace(), value); - modify().m_kv_address = m_kv_index->getAddress(); - modify().m_kv_type = m_kv_index->getIndexType(); + this->modify().m_kv_address = m_kv_index->getAddress(); + this->modify().m_kv_type = m_kv_index->getIndexType(); // return nullptr to indicate that the value has been inserted return nullptr; } @@ -1424,7 +1425,7 @@ namespace db0::object_model template void ObjectImplBase::incRef(bool is_tag) { - if (hasInstance()) { + if (this->hasInstance()) { super_t::incRef(is_tag); } else { // incRef with the initializer @@ -1461,7 +1462,7 @@ namespace db0::object_model template bool ObjectImplBase::equalTo(const ObjectImplBase &other) const { - if (!hasInstance() || !other.hasInstance()) { + if (!this->hasInstance() || !other.hasInstance()) { THROWF(db0::InputException) << "Object not initialized"; } @@ -1518,7 +1519,7 @@ namespace db0::object_model template void ObjectImplBase::setFixture(db0::swine_ptr &new_fixture) { - if (hasInstance()) { + if (this->hasInstance()) { THROWF(db0::InputException) << "set_prefix failed: object already initialized"; } @@ -1571,7 +1572,7 @@ namespace db0::object_model Address ObjectImplBase::getAddress() const { assert(!isDefunct()); - if (!hasInstance()) { + if (!this->hasInstance()) { THROWF(db0::InternalException) << "Object instance does not exist yet (did you forget to use db0.materialized(self) in constructor ?)"; } return super_t::getAddress(); @@ -1580,7 +1581,7 @@ namespace db0::object_model template UniqueAddress ObjectImplBase::getUniqueAddress() const { - if (hasInstance()) { + if (this->hasInstance()) { return super_t::getUniqueAddress(); } else { // NOTE: defunct objects don't have a valid address (not assigned yet) @@ -1592,7 +1593,7 @@ namespace db0::object_model template bool ObjectImplBase::hasValidClassRef() const { - if (hasInstance() && m_type) { + if (this->hasInstance() && m_type) { return (*this)->getClassRef() == m_type->getClassRef(); } return true; @@ -1617,7 +1618,7 @@ namespace db0::object_model template void ObjectImplBase::touch() { - if (hasInstance() && !isDefunct()) { + if (this->hasInstance() && !isDefunct()) { // NOTE: for already modified and small objects we may skip "touch" if (!super_t::isModified() || this->span() > 1) { // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag @@ -1633,7 +1634,7 @@ namespace db0::object_model if (!m_touched) { // mark the 1st byte of the object as modified (forced-diff) // this is always the 1st DP occupied by the object - modify(0, 1); + this->modify(0, 1); m_touched = true; } } diff --git a/src/dbzero/object_model/object/ObjectImplBase.hpp b/src/dbzero/object_model/object/ObjectImplBase.hpp index 0201919b..845c6794 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.hpp +++ b/src/dbzero/object_model/object/ObjectImplBase.hpp @@ -63,15 +63,15 @@ namespace db0::object_model // Construct as null / dropped object ObjectImplBase(UniqueAddress, unsigned int ext_refs); - ObjectImplBase(const Object &) = delete; - ObjectImplBase(Object &&) = delete; + ObjectImplBase(const ObjectImplBase &) = delete; + ObjectImplBase(ObjectImplBase &&) = delete; /** * Construct new Object (uninitialized, without corresponding dbzero instance yet) */ ObjectImplBase(std::shared_ptr); ObjectImplBase(TypeInitializer &&); - + // Unload from address with a known type (possibly a base type) // NOTE: unload works faster if type_hint is the exact object's type struct with_type_hint {}; @@ -171,7 +171,7 @@ namespace db0::object_model // Binary (shallow) compare 2 objects or 2 versions of the same memo object (e.g. from different snapshots) // NOTE: ref-counts are not compared (only user-assigned members) // @return true if objects are identical - bool equalTo(const Object &) const; + bool equalTo(const ObjectImplBase &) const; /** * Move unreferenced object to a different prefix without changing the instance diff --git a/src/dbzero/object_model/object/ObjectInitializer.cpp b/src/dbzero/object_model/object/ObjectInitializer.cpp index 740722c2..fd61feab 100644 --- a/src/dbzero/object_model/object/ObjectInitializer.cpp +++ b/src/dbzero/object_model/object/ObjectInitializer.cpp @@ -5,48 +5,7 @@ namespace db0::object_model { - - ObjectInitializer::ObjectInitializer(ObjectInitializerManager &manager, std::uint32_t loc, - Object &object, std::shared_ptr db0_class) - : m_manager(manager) - , m_loc(loc) - , m_closed(false) - , m_object_ptr(&object) - , m_class(db0_class) - // NOTE: limit the dim-2 size to improve performance - , m_has_value(lofi_store<2>::size()) - { - } - - ObjectInitializer::ObjectInitializer(ObjectInitializerManager &manager, std::uint32_t loc, Object &object, - TypeInitializer &&type_initializer) - : m_manager(manager) - , m_loc(loc) - , m_closed(false) - , m_object_ptr(&object) - // NOTE: limit the dim-2 size to improve performance - , m_has_value(lofi_store<2>::size()) - , m_type_initializer(std::move(type_initializer)) - { - } - - void ObjectInitializer::init(Object &object, std::shared_ptr db0_class) - { - assert(m_closed); - m_closed = false; - m_object_ptr = &object; - m_class = db0_class; - } - - void ObjectInitializer::init(Object &object, TypeInitializer &&type_initializer) - { - assert(m_closed); - assert(!m_class); - m_closed = false; - m_object_ptr = &object; - m_type_initializer = std::move(type_initializer); - } - + void ObjectInitializer::close() { m_manager.closeAt(m_loc); } @@ -223,43 +182,6 @@ namespace db0::object_model return true; } - std::shared_ptr ObjectInitializerManager::tryCloseInitializer(const Object &object) - { - for (auto i = 0u; i < m_active_count; ++i) { - if (m_initializers[i]->operator==(object)) { - auto result = m_initializers[i]->getClassPtr(); - closeAt(i); - return result; - } - } - return nullptr; - } - - std::shared_ptr ObjectInitializerManager::closeInitializer(const Object &object) - { - auto result = tryCloseInitializer(object); - if (result) { - return result; - } - THROWF(db0::InternalException) << "Initializer not found" << THROWF_END; - } - - ObjectInitializer *ObjectInitializerManager::findInitializer(const Object &object) const - { - for (auto i = 0u; i < m_active_count; ++i) { - if (m_initializers[i]->operator==(object)) { - // move to front to allow faster lookup the next time - if (i != 0) { - std::swap(m_initializers[i], m_initializers[0]); - *(m_initializers[i]) = i; - *(m_initializers[0]) = 0; - } - return m_initializers[0].get(); - } - } - return nullptr; - } - void ObjectInitializerManager::closeAt(std::uint32_t loc) { auto result = m_initializers[loc]->getClassPtr(); @@ -270,5 +192,5 @@ namespace db0::object_model *(m_initializers[m_active_count - 1]) = m_active_count - 1; --m_active_count; } - + } \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectInitializer.hpp b/src/dbzero/object_model/object/ObjectInitializer.hpp index ac8b8c61..14600810 100644 --- a/src/dbzero/object_model/object/ObjectInitializer.hpp +++ b/src/dbzero/object_model/object/ObjectInitializer.hpp @@ -13,6 +13,7 @@ #include #include "ValueTable.hpp" #include "XValuesVector.hpp" +#include "lofi_store.hpp" namespace db0 @@ -28,9 +29,55 @@ namespace db0::object_model class Class; class Object; - class ObjectInitializerManager; + class ObjectInitializer; using Fixture = db0::Fixture; + /** + * The purpose of this class is to hold Object initializers during the construction process. + * We could simply keep 'initializer' as an Object member, but since this is a short-lived object, it would be a waste of space. + * Also InitializerManager helps us reuse Initializer instances, saving on memory allocations. + */ + class ObjectInitializerManager + { + public: + ObjectInitializerManager() = default; + + template + void addInitializer(T &object, Args&& ...args); + + // Close the initializer and retrieve object's class + template + std::shared_ptr closeInitializer(const T &object); + + // Close the initializer if it exists + template + std::shared_ptr tryCloseInitializer(const T &object); + + template + ObjectInitializer &getInitializer(const T &object) const + { + auto result = findInitializer(object); + if (result) { + return *result; + } + THROWF(InternalException) << "Initializer not found" << THROWF_END; + } + + template + ObjectInitializer *findInitializer(const T &object) const; + + protected: + friend class ObjectInitializer; + void closeAt(std::uint32_t loc); + + private: + mutable std::vector > m_initializers; + // number of active object initializers + std::size_t m_active_count = 0; + // number of non-null initializer instances + std::size_t m_total_count = 0; + }; + /** * Class to store status of the instance / member intialization process (corresponds to __init__) */ @@ -41,12 +88,50 @@ namespace db0::object_model using TypeInitializer = std::function(db0::swine_ptr &)>; // loc - position in the initializer manager's array - ObjectInitializer(ObjectInitializerManager &, std::uint32_t loc, Object &, std::shared_ptr); + template + ObjectInitializer(ObjectInitializerManager &manager, std::uint32_t loc, T &object, std::shared_ptr type) + : m_manager(manager) + , m_loc(loc) + , m_closed(false) + , m_object_ptr(&object) + , m_class(type) + // NOTE: limit the dim-2 size to improve performance + , m_has_value(lofi_store<2>::size()) + { + } + // alternative constructor for lazy type initialization - ObjectInitializer(ObjectInitializerManager &, std::uint32_t loc, Object &, TypeInitializer &&); + template + ObjectInitializer(ObjectInitializerManager &manager, std::uint32_t loc, T &object, TypeInitializer &&type_initializer) + : m_manager(manager) + , m_loc(loc) + , m_closed(false) + , m_object_ptr(&object) + // NOTE: limit the dim-2 size to improve performance + , m_has_value(lofi_store<2>::size()) + , m_type_initializer(std::move(type_initializer)) + { + } - void init(Object &object, std::shared_ptr); - void init(Object &object, TypeInitializer &&); + template + void init(T &object, std::shared_ptr type) + { + assert(m_closed); + m_closed = false; + m_object_ptr = &object; + m_class = type; + } + + template + void init(T &object, TypeInitializer &&type_initializer) + { + assert(m_closed); + assert(!m_class); + m_closed = false; + m_object_ptr = &object; + m_type_initializer = std::move(type_initializer); + + } // @param mask required for lo-fi types (pack-2) void set(std::pair loc, StorageClass storage_class, Value value, @@ -76,7 +161,8 @@ namespace db0::object_model // NOTE always the whole value is retrieved (no mask support) bool tryGetAt(std::pair loc, std::pair &) const; - bool operator==(const Object &other) { + template + bool operator==(const T &other) { return m_object_ptr == &other; } @@ -101,14 +187,15 @@ namespace db0::object_model void reset(); void operator=(std::uint32_t new_loc); - + private: // maximum size of the position-encoded value-block (pos-VT) static constexpr std::size_t POSVT_MAX_SIZE = 128; ObjectInitializerManager &m_manager; std::uint32_t m_loc = std::numeric_limits::max(); bool m_closed = true; - Object *m_object_ptr = nullptr; + // pointer to an implementation-specific type + void *m_object_ptr = nullptr; mutable std::shared_ptr m_class; // indexed initialization values mutable XValuesVector m_values; @@ -119,53 +206,8 @@ namespace db0::object_model mutable TypeInitializer m_type_initializer; }; - /** - * The purpose of this class is to hold Object initializers during the construction process. - * We could simply keep 'initializer' as an Object member, but since this is a short-lived object, it would be a waste of space. - * Also InitializerManager helps us reuse Initializer instances, saving on memory allocations. - */ - class ObjectInitializerManager - { - public: - ObjectInitializerManager() = default; - - template void addInitializer(Object &object, Args&& ...args); - - /** - * Close the initializer and retrieve object's class - */ - std::shared_ptr closeInitializer(const Object &object); - - /** - * Close the initializer if it exists - */ - std::shared_ptr tryCloseInitializer(const Object &object); - - ObjectInitializer &getInitializer(const Object &object) const - { - auto result = findInitializer(object); - if (result) { - return *result; - } - THROWF(InternalException) << "Initializer not found" << THROWF_END; - } - - ObjectInitializer *findInitializer(const Object &object) const; - - protected: - friend class ObjectInitializer; - void closeAt(std::uint32_t loc); - - private: - mutable std::vector > m_initializers; - // number of active object initializers - std::size_t m_active_count = 0; - // number of non-null initializer instances - std::size_t m_total_count = 0; - }; - - template - void ObjectInitializerManager::addInitializer(Object &object, Args&& ...args) + template + void ObjectInitializerManager::addInitializer(T &object, Args&& ...args) { if (m_active_count < m_total_count) { auto loc = m_active_count++; @@ -185,4 +227,44 @@ namespace db0::object_model } } + template + std::shared_ptr ObjectInitializerManager::tryCloseInitializer(const T &object) + { + for (auto i = 0u; i < m_active_count; ++i) { + if (m_initializers[i]->operator==(object)) { + auto result = m_initializers[i]->getClassPtr(); + closeAt(i); + return result; + } + } + return nullptr; + } + + template + std::shared_ptr ObjectInitializerManager::closeInitializer(const T &object) + { + auto result = tryCloseInitializer(object); + if (result) { + return result; + } + THROWF(db0::InternalException) << "Initializer not found" << THROWF_END; + } + + template + ObjectInitializer *ObjectInitializerManager::findInitializer(const T &object) const + { + for (auto i = 0u; i < m_active_count; ++i) { + if (m_initializers[i]->operator==(object)) { + // move to front to allow faster lookup the next time + if (i != 0) { + std::swap(m_initializers[i], m_initializers[0]); + *(m_initializers[i]) = i; + *(m_initializers[0]) = 0; + } + return m_initializers[0].get(); + } + } + return nullptr; + } + } \ No newline at end of file From bb8fa3dd68c2263959e0bc58d4be4dd9ddd38f2a Mon Sep 17 00:00:00 2001 From: Wojtek Date: Tue, 4 Nov 2025 11:12:58 +0100 Subject: [PATCH 06/19] immutable --- python_tests/test_memo_immutable.py | 18 +++++ src/dbzero/bindings/python/Memo.cpp | 17 +++-- src/dbzero/bindings/python/PyToolkit.cpp | 11 ++- src/dbzero/bindings/python/PyToolkit.hpp | 2 + src/dbzero/object_model/class/Class.cpp | 2 +- src/dbzero/object_model/class/Class.hpp | 5 +- .../object_model/class/ClassFactory.cpp | 1 + src/dbzero/object_model/object/Object.hpp | 13 +--- .../object/ObjectImmutableImpl.cpp | 9 +++ .../object/ObjectImmutableImpl.hpp | 25 +++++++ src/dbzero/object_model/object/Options.cpp | 2 +- src/dbzero/object_model/object/Options.hpp | 5 +- .../object/o_immutable_object.cpp | 74 +++++++++++++++++++ .../object/o_immutable_object.hpp | 58 +++++++++++++++ src/dbzero/object_model/object/o_object.cpp | 1 - src/dbzero/object_model/object/o_object.hpp | 11 --- 16 files changed, 217 insertions(+), 37 deletions(-) create mode 100644 python_tests/test_memo_immutable.py create mode 100644 src/dbzero/object_model/object/ObjectImmutableImpl.cpp create mode 100644 src/dbzero/object_model/object/ObjectImmutableImpl.hpp create mode 100644 src/dbzero/object_model/object/o_immutable_object.cpp create mode 100644 src/dbzero/object_model/object/o_immutable_object.hpp diff --git a/python_tests/test_memo_immutable.py b/python_tests/test_memo_immutable.py new file mode 100644 index 00000000..aa128747 --- /dev/null +++ b/python_tests/test_memo_immutable.py @@ -0,0 +1,18 @@ +from random import random +import pytest +import dbzero as db0 +from dataclasses import dataclass +from .conftest import DB0_DIR +import random + + +@db0.memo(immutable=True) +@dataclass +class MemoImmutableClass: + data: str + value: int = 0 + +def test_create_memo_immutable(db0_fixture): + obj_1 = MemoImmutableClass(data="immutable data", value=42) + assert obj_1.data == "immutable data" + assert obj_1.value == 42 diff --git a/src/dbzero/bindings/python/Memo.cpp b/src/dbzero/bindings/python/Memo.cpp index ab4b8c95..712d0d22 100644 --- a/src/dbzero/bindings/python/Memo.cpp +++ b/src/dbzero/bindings/python/Memo.cpp @@ -177,7 +177,7 @@ namespace db0::python { using Class = db0::object_model::Class; using TagIndex = db0::object_model::TagIndex; - + PY_API_FUNC // the instance may already exist (e.g. if this is a singleton) if (!self->ext().hasInstance()) { @@ -457,7 +457,7 @@ namespace db0::python PyObject *wrapPyType(PyTypeObject *base_class, bool is_singleton, bool no_default_tags, const char *prefix_name, const char *type_id, const char *file_name, std::vector &&init_vars, PyObject *py_dyn_prefix_callable, - std::vector &&migrations, bool no_cache) + std::vector &&migrations, bool no_cache, bool immutable) { auto py_class = Py_BORROW(base_class); auto py_module = Py_OWN(findModule(*Py_OWN(PyObject_GetAttrString((PyObject*)*py_class, "__module__")))); @@ -484,6 +484,9 @@ namespace db0::python if (no_cache) { type_flags.set(MemoOptions::NO_CACHE); } + if (immutable) { + type_flags.set(MemoOptions::IMMUTABLE); + } auto type_info = MemoTypeDecoration( py_module, prefix_name, @@ -523,11 +526,12 @@ namespace db0::python // migrations are only processed for singleton types PyObject *py_migrations = nullptr; PyObject *py_no_cache = nullptr; + PyObject *py_immutable = nullptr; static const char *kwlist[] = { "input", "singleton", "no_default_tags", "prefix", "id", "py_file", "py_init_vars", - "py_dyn_prefix", "py_migrations", "no_cache", NULL }; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOOOOOOOO", const_cast(kwlist), &class_obj, &py_singleton, - &py_no_default_tags, &py_prefix_name, &py_type_id, &py_file_name, &py_init_vars, &py_dyn_prefix, &py_migrations, &py_no_cache)) + "py_dyn_prefix", "py_migrations", "no_cache", "immutable", NULL }; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOOOOOOOOO", const_cast(kwlist), &class_obj, &py_singleton, + &py_no_default_tags, &py_prefix_name, &py_type_id, &py_file_name, &py_init_vars, &py_dyn_prefix, &py_migrations, &py_no_cache, &py_immutable)) { return NULL; } @@ -535,6 +539,7 @@ namespace db0::python bool is_singleton = py_singleton && PyObject_IsTrue(py_singleton); bool no_default_tags = py_no_default_tags && PyObject_IsTrue(py_no_default_tags); bool no_cache = py_no_cache && PyObject_IsTrue(py_no_cache); + bool immutable = py_immutable && PyObject_IsTrue(py_immutable); const char *prefix_name = (py_prefix_name && py_prefix_name != Py_None) ? PyUnicode_AsUTF8(py_prefix_name) : nullptr; const char *type_id = py_type_id ? PyUnicode_AsUTF8(py_type_id) : nullptr; const char *file_name = (py_file_name && py_file_name != Py_None) ? PyUnicode_AsUTF8(py_file_name) : nullptr; @@ -569,7 +574,7 @@ namespace db0::python auto migrations = extractMigrations(py_migrations); return wrapPyType(castToType(class_obj), is_singleton, no_default_tags, prefix_name, type_id, file_name, - std::move(init_vars), py_dyn_prefix, std::move(migrations), no_cache + std::move(init_vars), py_dyn_prefix, std::move(migrations), no_cache, immutable ); } diff --git a/src/dbzero/bindings/python/PyToolkit.cpp b/src/dbzero/bindings/python/PyToolkit.cpp index 3acb14e7..9a5d6722 100644 --- a/src/dbzero/bindings/python/PyToolkit.cpp +++ b/src/dbzero/bindings/python/PyToolkit.cpp @@ -555,8 +555,17 @@ namespace db0::python } else { return false; } - } + } + bool PyToolkit::isImmutable(TypeObjectPtr py_type) + { + if (isMemoType(py_type)) { + return MemoTypeDecoration::get(py_type).getFlags()[MemoOptions::IMMUTABLE]; + } else { + return false; + } + } + FlagSet PyToolkit::getMemoFlags(TypeObjectPtr py_type) { if (isMemoType(py_type)) { diff --git a/src/dbzero/bindings/python/PyToolkit.hpp b/src/dbzero/bindings/python/PyToolkit.hpp index 72cf99bb..9c02eb23 100644 --- a/src/dbzero/bindings/python/PyToolkit.hpp +++ b/src/dbzero/bindings/python/PyToolkit.hpp @@ -196,6 +196,8 @@ namespace db0::python // check if a memo type is marked with no_default_tags flag static bool isNoDefaultTags(TypeObjectPtr); static bool isNoCache(TypeObjectPtr); + // type marked as immutable + static bool isImmutable(TypeObjectPtr); static FlagSet getMemoFlags(TypeObjectPtr); inline static void incRef(ObjectPtr py_object) { diff --git a/src/dbzero/object_model/class/Class.cpp b/src/dbzero/object_model/class/Class.cpp index d104c588..bcd6a722 100644 --- a/src/dbzero/object_model/class/Class.cpp +++ b/src/dbzero/object_model/class/Class.cpp @@ -8,7 +8,7 @@ #include #include "Schema.hpp" -DEFINE_ENUM_VALUES(db0::ClassOptions, "SINGLETON", "NO_DEFAULT_TAGS") +DEFINE_ENUM_VALUES(db0::ClassOptions, "SINGLETON", "NO_DEFAULT_TAGS", "IMMUTABLE") namespace db0::object_model diff --git a/src/dbzero/object_model/class/Class.hpp b/src/dbzero/object_model/class/Class.hpp index 1dceced2..5dfa313d 100644 --- a/src/dbzero/object_model/class/Class.hpp +++ b/src/dbzero/object_model/class/Class.hpp @@ -30,13 +30,14 @@ namespace db0 SINGLETON = 0x0001, // instances of this type opted out of auto-assigned type tags NO_DEFAULT_TAGS = 0x0002, + IMMUTABLE = 0x0004 }; using ClassFlags = db0::FlagSet; } -DECLARE_ENUM_VALUES(db0::ClassOptions, 2) +DECLARE_ENUM_VALUES(db0::ClassOptions, 3) namespace db0::object_model @@ -82,7 +83,7 @@ DB0_PACKED_BEGIN ); }; DB0_PACKED_END - + // address <-> class_ref conversion functions // @param type_slot_addr_range the address of the types-specific slot std::uint32_t classRef(const Class &, std::pair type_slot_addr_range); diff --git a/src/dbzero/object_model/class/ClassFactory.cpp b/src/dbzero/object_model/class/ClassFactory.cpp index 2dd140e8..2b9f0d13 100644 --- a/src/dbzero/object_model/class/ClassFactory.cpp +++ b/src/dbzero/object_model/class/ClassFactory.cpp @@ -158,6 +158,7 @@ namespace db0::object_model bool is_singleton = LangToolkit::isSingleton(lang_type); ClassFlags flags { is_singleton ? ClassOptions::SINGLETON : 0 }; flags.set(ClassOptions::NO_DEFAULT_TAGS, LangToolkit::isNoDefaultTags(lang_type)); + flags.set(ClassOptions::IMMUTABLE, LangToolkit::isImmutable(lang_type)); auto memo_base = LangToolkit::getBaseMemoType(lang_type); std::shared_ptr base_class; if (memo_base) { diff --git a/src/dbzero/object_model/object/Object.hpp b/src/dbzero/object_model/object/Object.hpp index c1c6d7be..90866695 100644 --- a/src/dbzero/object_model/object/Object.hpp +++ b/src/dbzero/object_model/object/Object.hpp @@ -3,28 +3,17 @@ #include "ObjectImplBase.hpp" #include "o_object.hpp" -namespace db0 - -{ - - class Fixture; - -} - namespace db0::object_model { - class Class; - using Fixture = db0::Fixture; - class Object: public ObjectImplBase { // GC0 specific declarations GC0_Declare public: static constexpr unsigned char REALM_ID = o_object::REALM_ID; - using super_t = ObjectImplBase; + using super_t = ObjectImplBase; template Object(Args&&... args) diff --git a/src/dbzero/object_model/object/ObjectImmutableImpl.cpp b/src/dbzero/object_model/object/ObjectImmutableImpl.cpp new file mode 100644 index 00000000..490eb1ff --- /dev/null +++ b/src/dbzero/object_model/object/ObjectImmutableImpl.cpp @@ -0,0 +1,9 @@ +#include "ObjectImmutableImpl.hpp" + +namespace db0::object_model + +{ + + GC0_Define(ObjectImmutableImpl) + +} \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectImmutableImpl.hpp b/src/dbzero/object_model/object/ObjectImmutableImpl.hpp new file mode 100644 index 00000000..98d21cdd --- /dev/null +++ b/src/dbzero/object_model/object/ObjectImmutableImpl.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include "ObjectImplBase.hpp" +#include "o_immutable_object.hpp" + +namespace db0::object_model + +{ + + class ObjectImmutableImpl: public ObjectImplBase + { + // GC0 specific declarations + GC0_Declare + public: + static constexpr unsigned char REALM_ID = o_immutable_object::REALM_ID; + using super_t = ObjectImplBase; + + template + ObjectImmutableImpl(Args&&... args) + : super_t(std::forward(args)...) + { + } + }; + +} diff --git a/src/dbzero/object_model/object/Options.cpp b/src/dbzero/object_model/object/Options.cpp index c977faee..fade956e 100644 --- a/src/dbzero/object_model/object/Options.cpp +++ b/src/dbzero/object_model/object/Options.cpp @@ -1,3 +1,3 @@ #include "Options.hpp" -DEFINE_ENUM_VALUES(db0::object_model::MemoOptions, "NO_DEFAULT_TAGS", "NO_CACHE") +DEFINE_ENUM_VALUES(db0::object_model::MemoOptions, "NO_DEFAULT_TAGS", "NO_CACHE", "IMMUTABLE") diff --git a/src/dbzero/object_model/object/Options.hpp b/src/dbzero/object_model/object/Options.hpp index cc0ae8ee..6e95dbe0 100644 --- a/src/dbzero/object_model/object/Options.hpp +++ b/src/dbzero/object_model/object/Options.hpp @@ -12,11 +12,12 @@ namespace db0::object_model // instances of this type opted out of auto-assigned type tags NO_DEFAULT_TAGS = 0x0001, // instances of this type opted out of caching - NO_CACHE = 0x0002 + NO_CACHE = 0x0002, + IMMUTABLE = 0x0004 }; using MemoFlags = db0::FlagSet; } -DECLARE_ENUM_VALUES(db0::object_model::MemoOptions, 2) \ No newline at end of file +DECLARE_ENUM_VALUES(db0::object_model::MemoOptions, 3) \ No newline at end of file diff --git a/src/dbzero/object_model/object/o_immutable_object.cpp b/src/dbzero/object_model/object/o_immutable_object.cpp new file mode 100644 index 00000000..e3e4b54f --- /dev/null +++ b/src/dbzero/object_model/object/o_immutable_object.cpp @@ -0,0 +1,74 @@ +#include "o_immutable_object.hpp" +#include +#include +#include +#include + +namespace db0::object_model + +{ + + o_immutable_object::o_immutable_object(std::uint32_t class_ref, + std::pair ref_counts, std::uint8_t num_type_tags, const PosVT::Data &pos_vt_data, + unsigned int pos_vt_offset, const XValue *index_vt_begin, const XValue *index_vt_end) + : m_header(ref_counts) + , m_num_type_tags(num_type_tags) + { + arrangeMembers() + (PosVT::type(), pos_vt_data, pos_vt_offset) + (packed_int32::type(), class_ref) + (IndexVT::type(), index_vt_begin, index_vt_end); + } + + std::size_t o_immutable_object::measure(std::uint32_t class_ref, + std::pair, std::uint8_t, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, + const XValue *index_vt_begin, const XValue *index_vt_end) + { + return super_t::measureMembers() + (PosVT::type(), pos_vt_data, pos_vt_offset) + (packed_int32::type(), class_ref) + (IndexVT::type(), index_vt_begin, index_vt_end); + } + + const PosVT &o_immutable_object::pos_vt() const { + return getDynFirst(PosVT::type()); + } + + PosVT &o_immutable_object::pos_vt() { + return getDynFirst(PosVT::type()); + } + + const packed_int32 &o_immutable_object::classRef() const { + return getDynAfter(pos_vt(), packed_int32::type()); + } + + std::uint32_t o_immutable_object::getClassRef() const { + return classRef().value(); + } + + const IndexVT &o_immutable_object::index_vt() const { + return getDynAfter(classRef(), IndexVT::type()); + } + + IndexVT &o_immutable_object::index_vt() { + return getDynAfter(classRef(), IndexVT::type()); + } + + void o_immutable_object::incRef(bool is_tag) { + m_header.incRef(is_tag); + } + + bool o_immutable_object::hasRefs() const + { + // NOTE: type tags are not counted as "proper" references + if (m_header.m_ref_counter.getFirst() > this->m_num_type_tags) { + return true; + } + return m_header.m_ref_counter.getSecond() > 0; + } + + bool o_immutable_object::hasAnyRefs() const { + return m_header.hasRefs(); + } + +} \ No newline at end of file diff --git a/src/dbzero/object_model/object/o_immutable_object.hpp b/src/dbzero/object_model/object/o_immutable_object.hpp new file mode 100644 index 00000000..3b09802b --- /dev/null +++ b/src/dbzero/object_model/object/o_immutable_object.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include "ValueTable.hpp" +#include + +namespace db0::object_model + +{ + +DB0_PACKED_BEGIN + class DB0_PACKED_ATTR o_immutable_object: public db0::o_base + { + protected: + using super_t = db0::o_base; + + public: + static constexpr unsigned char REALM_ID = 1; + // common object header + o_unique_header m_header; + // number of auto-assigned type tags + std::uint8_t m_num_type_tags = 0; + + PosVT &pos_vt(); + const PosVT &pos_vt() const; + + const packed_int32 &classRef() const; + std::uint32_t getClassRef() const; + + const IndexVT &index_vt() const; + + IndexVT &index_vt(); + + // ref_counts - the initial reference counts (tags / objects) inherited from the initializer + o_immutable_object(std::uint32_t class_ref, std::pair ref_counts, std::uint8_t num_type_tags, + const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin = nullptr, + const XValue *index_vt_end = nullptr); + + static std::size_t measure(std::uint32_t, std::pair, std::uint8_t num_type_tags, + const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin = nullptr, + const XValue *index_vt_end = nullptr); + + template static std::size_t safeSizeOf(BufT buf) + { + return super_t::sizeOfMembers(buf) + (PosVT::type()) + (packed_int32::type()) + (IndexVT::type()); + } + + void incRef(bool is_tag); + bool hasRefs() const; + bool hasAnyRefs() const; + }; +DB0_PACKED_END + +} \ No newline at end of file diff --git a/src/dbzero/object_model/object/o_object.cpp b/src/dbzero/object_model/object/o_object.cpp index f571b1b3..aa082056 100644 --- a/src/dbzero/object_model/object/o_object.cpp +++ b/src/dbzero/object_model/object/o_object.cpp @@ -1,5 +1,4 @@ #include "o_object.hpp" -#include #include #include #include diff --git a/src/dbzero/object_model/object/o_object.hpp b/src/dbzero/object_model/object/o_object.hpp index baea5a5d..25bd1a18 100644 --- a/src/dbzero/object_model/object/o_object.hpp +++ b/src/dbzero/object_model/object/o_object.hpp @@ -7,21 +7,10 @@ #include #include "KV_Index.hpp" -namespace db0 - -{ - - class Fixture; - -} - namespace db0::object_model { - class Class; - using Fixture = db0::Fixture; - DB0_PACKED_BEGIN class DB0_PACKED_ATTR o_object: public db0::o_base { From aff710f530a6d230a0f2b1d58519f3cc5e37641d Mon Sep 17 00:00:00 2001 From: Wojtek Date: Tue, 4 Nov 2025 15:28:14 +0100 Subject: [PATCH 07/19] save work --- src/dbzero/bindings/python/Memo.cpp | 146 ++-- src/dbzero/bindings/python/Memo.hpp | 21 +- .../bindings/python/MemoTypeDecoration.cpp | 2 +- .../bindings/python/MemoTypeDecoration.hpp | 6 +- src/dbzero/object_model/object/Object.cpp | 330 ++++++++- src/dbzero/object_model/object/Object.hpp | 30 +- .../object/ObjectImmutableImpl.hpp | 6 +- .../object_model/object/ObjectImplBase.cpp | 661 +++++------------- .../object_model/object/ObjectImplBase.hpp | 47 +- 9 files changed, 660 insertions(+), 589 deletions(-) diff --git a/src/dbzero/bindings/python/Memo.cpp b/src/dbzero/bindings/python/Memo.cpp index 712d0d22..adc0f439 100644 --- a/src/dbzero/bindings/python/Memo.cpp +++ b/src/dbzero/bindings/python/Memo.cpp @@ -31,6 +31,7 @@ namespace db0::python { using ObjectSharedPtr = PyTypes::ObjectSharedPtr; + using TypeObjectSharedPtr = PyTypes::TypeObjectSharedPtr; // @return type name / full type name (tp_name) std::pair getMemoTypeName(shared_py_object py_class) @@ -42,9 +43,9 @@ namespace db0::python return { type_name, full_type_name }; } - MemoObject *tryMemoObject_new(PyTypeObject *py_type, PyObject *, PyObject *) - { - auto &decor = MemoTypeDecoration::get(py_type); + template + MemoImplT *tryMemoObject_new(const MemoTypeDecoration &decor, PyTypeObject *py_type, PyObject *, PyObject *) + { // NOTE: read-only fixture access is sufficient here since objects are lazy-initialized // i.e. the actual dbzero instance is created on postInit // this is also important for dynamically scoped clases (where read/write access may not be possible on default fixture) @@ -53,11 +54,11 @@ namespace db0::python auto &class_factory = fixture->get(); // find py type associated dbzero class with the ClassFactory auto type = class_factory.tryGetOrCreateType(py_type); - MemoObject *memo_obj = reinterpret_cast(py_type->tp_alloc(py_type, 0)); + MemoImplT *memo_obj = reinterpret_cast(py_type->tp_alloc(py_type, 0)); - // if type cannot be retrieved due to access mode then deferr this operation (fallback) + // if type cannot be retrieved due to access mode then defer this operation (fallback) if (type) { - // prepare a new DB0 instance of a known db0 class + // prepare a new dbzero instance of a known db0 class memo_obj->makeNew(type); } else { auto type_initializer = [py_type](db0::swine_ptr &fixture) { @@ -71,10 +72,41 @@ namespace db0::python return memo_obj; } - MemoObject *PyAPI_MemoObject_new(PyTypeObject *py_type, PyObject *args, PyObject *kwargs) + Py_hash_t PyAPI_MemoHash(MemoObject *self) { PY_API_FUNC - return reinterpret_cast(runSafe(tryMemoObject_new, py_type, args, kwargs)); + auto fixture = self->ext().getFixture(); + return runSafe(getPyHashImpl, fixture, self); + } + + template + PyObject *tryMemoObject_str(MemoImplT *self) + { + std::stringstream str; + auto &memo = self->ext(); + str << "<" << Py_TYPE(self)->tp_name; + if (memo.hasInstance()) { + str << " instance uuid=" << PyUnicode_AsUTF8(*Py_OWN(tryGetUUID(self))); + } else { + str << " (uninitialized)"; + } + str << ">"; + return PyUnicode_FromString(str.str().c_str()); + } + + template + PyObject *PyAPI_MemoObject_str(MemoImplT *self) + { + PY_API_FUNC + return runSafe(tryMemoObject_str, self); + } + + template + PyObject *PyAPI_MemoObject_new(PyTypeObject *py_type, PyObject *args, PyObject *kwargs) + { + const auto &decor = MemoTypeDecoration::get(py_type); + PY_API_FUNC + return runSafe(tryMemoObject_new, decor, py_type, args, kwargs); } void tryGetScope(PyTypeObject *py_type, PyObject *args, PyObject *kwargs, @@ -177,7 +209,7 @@ namespace db0::python { using Class = db0::object_model::Class; using TagIndex = db0::object_model::TagIndex; - + PY_API_FUNC // the instance may already exist (e.g. if this is a singleton) if (!self->ext().hasInstance()) { @@ -397,8 +429,21 @@ namespace db0::python return new_dict.steal(); } - static PyType_Slot MemoType_common_slots[] = { - {Py_tp_new, (void *)PyAPI_MemoObject_new}, + // Regular memo slots + static PyType_Slot MemoObject_common_slots[] = { + {Py_tp_new, (void *)PyAPI_MemoObject_new}, + {Py_tp_dealloc, (void *)MemoObject_del}, + {Py_tp_init, (void *)PyAPI_MemoObject_init}, + {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro}, + {Py_tp_setattro, (void *)PyAPI_MemoObject_setattro}, + {Py_tp_richcompare, (void *)PyAPI_MemoObject_rq}, + {Py_tp_hash, (void *)PyAPI_MemoHash}, + {0, 0} + }; + + // Immutable memo slots + static PyType_Slot MemoImmutableObject_common_slots[] = { + {Py_tp_new, (void *)PyAPI_MemoObject_new}, {Py_tp_dealloc, (void *)MemoObject_del}, {Py_tp_init, (void *)PyAPI_MemoObject_init}, {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro}, @@ -407,30 +452,42 @@ namespace db0::python {Py_tp_hash, (void *)PyAPI_MemoHash}, {0, 0} }; + + template + PyType_Slot* getCommonSlots(); + + template <> PyType_Slot *getCommonSlots() { + return MemoObject_common_slots; + } + + template <> PyType_Slot *getCommonSlots() { + return MemoImmutableObject_common_slots; + } + template PyTypeObject *PyMemoType_FromSpec(PyTypeObject *base_class, const char *tp_name, bool is_singleton) { // fill-in type specific slots first std::vector slots; // fill-in common slots first - for (auto &slot : MemoType_common_slots) { - if (!slot.slot && !slot.pfunc) { - break; - } - slots.push_back(slot); + auto slot_ptr = getCommonSlots(); + while (slot_ptr->slot || slot_ptr->pfunc) { + slots.push_back(*slot_ptr); + ++slot_ptr; } if (is_singleton) { + // NOTE: singletons are not supported for immutable memo objects slots.push_back({Py_tp_new, (void *)PyAPI_MemoObject_new_singleton}); } else { - slots.push_back({Py_tp_new, (void *)PyAPI_MemoObject_new}); + slots.push_back({Py_tp_new, (void *)PyAPI_MemoObject_new}); } - + slots.push_back({0, 0}); auto type_spec = PyType_Spec { .name = tp_name, - .basicsize = MemoObject::sizeOf(), + .basicsize = MemoImplT::sizeOf(), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE) & ~(Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MANAGED_DICT), .slots = slots.data() }; @@ -443,13 +500,13 @@ namespace db0::python // replace default __str__ and __repr__ implementations if (base_class->tp_str == PyType_Type.tp_str) { - (*tp_result)->tp_str = reinterpret_cast(PyAPI_MemoObject_str); - base_class->tp_str = reinterpret_cast(PyAPI_MemoObject_str); + (*tp_result)->tp_str = reinterpret_cast(PyAPI_MemoObject_str); + base_class->tp_str = reinterpret_cast(PyAPI_MemoObject_str); } - + if (base_class->tp_repr == PyType_Type.tp_repr) { - (*tp_result)->tp_repr = reinterpret_cast(PyAPI_MemoObject_str); - base_class->tp_repr = reinterpret_cast(PyAPI_MemoObject_str); + (*tp_result)->tp_repr = reinterpret_cast(PyAPI_MemoObject_str); + base_class->tp_repr = reinterpret_cast(PyAPI_MemoObject_str); } return tp_result.steal(); @@ -468,13 +525,19 @@ namespace db0::python if ((*py_class)->tp_dict == nullptr) { THROWF(db0::InternalException) << "Type has no tp_dict: " << (*py_class)->tp_name; } - + if ((*py_class)->tp_itemsize != 0) { THROWF(db0::InternalException) << "Variable-length types not supported: " << (*py_class)->tp_name; } auto [type_name, full_type_name] = getMemoTypeName(py_class); - auto new_type = Py_OWN(PyMemoType_FromSpec(base_class, full_type_name.c_str(), is_singleton)); + TypeObjectSharedPtr new_type = nullptr; + // NOTE: MemoObject and MemoImmutableObject have different implementations + if (immutable) { + new_type = Py_OWN(PyMemoType_FromSpec(base_class, full_type_name.c_str(), is_singleton)); + } else { + new_type = Py_OWN(PyMemoType_FromSpec(base_class, full_type_name.c_str(), is_singleton)); + } if (!new_type) { return nullptr; } @@ -656,27 +719,7 @@ namespace db0::python PySafeDict_SetItemString(*py_result, "size_of", Py_OWN(PyLong_FromLong(self->ext()->sizeOf()))); return py_result.steal(); } - - PyObject *tryMemoObject_str(MemoObject *self) - { - std::stringstream str; - auto &memo = self->ext(); - str << "<" << Py_TYPE(self)->tp_name; - if (memo.hasInstance()) { - str << " instance uuid=" << PyUnicode_AsUTF8(*Py_OWN(tryGetUUID(self))); - } else { - str << " (uninitialized)"; - } - str << ">"; - return PyUnicode_FromString(str.str().c_str()); - } - - PyObject *PyAPI_MemoObject_str(MemoObject *self) - { - PY_API_FUNC - return runSafe(tryMemoObject_str, self); - } - + void MemoType_get_info(PyTypeObject *type, PyObject *dict) { auto &decor = MemoTypeDecoration::get(type); @@ -848,14 +891,7 @@ namespace db0::python } Py_RETURN_FALSE; } - - Py_hash_t PyAPI_MemoHash(MemoObject *self) - { - PY_API_FUNC - auto fixture = self->ext().getFixture(); - return runSafe(getPyHashImpl, fixture, self); - } - + PyObject *tryGetSchema(PyTypeObject *py_type) { using SchemaTypeId = db0::object_model::SchemaTypeId; diff --git a/src/dbzero/bindings/python/Memo.hpp b/src/dbzero/bindings/python/Memo.hpp index b9b3fb2f..876681e8 100644 --- a/src/dbzero/bindings/python/Memo.hpp +++ b/src/dbzero/bindings/python/Memo.hpp @@ -9,6 +9,8 @@ #include #include "Migration.hpp" #include "MemoTypeDecoration.hpp" +#include +#include namespace db0::object_model @@ -24,27 +26,21 @@ namespace db0::python using AccessType = db0::AccessType; using MemoObject = PyWrapper; - + using MemoImmutableObject = PyWrapper; + PyObject *PyAPI_wrapPyClass(PyObject *self, PyObject *, PyObject *kwargs); - MemoObject *PyAPI_MemoObject_new(PyTypeObject *type, PyObject * = nullptr, PyObject * = nullptr); // create a memo object stub MemoObject* MemoObjectStub_new(PyTypeObject *type); - PyObject *MemoObject_alloc(PyTypeObject *type, Py_ssize_t nitems); void MemoObject_del(MemoObject* self); void MemoObject_drop(MemoObject* self); - int PyAPI_MemoObject_init(MemoObject* self, PyObject* args, PyObject* kwds); - PyObject *PyAPI_MemoObject_getattro(MemoObject *self, PyObject *attr); - int PyAPI_MemoObject_setattro(MemoObject *self, PyObject *attr, PyObject *value); - Py_hash_t PyAPI_MemoHash(MemoObject *); // check if memo type has been marked as singleton bool PyMemoType_IsSingleton(PyTypeObject *type); PyObject *MemoObject_GetFieldLayout(MemoObject *); - PyObject *MemoObject_DescribeObject(MemoObject *); - PyObject *PyAPI_MemoObject_str(MemoObject *); + PyObject *MemoObject_DescribeObject(MemoObject *); void MemoType_get_info(PyTypeObject *type, PyObject *dict); void MemoType_close(PyTypeObject *type); @@ -65,8 +61,9 @@ namespace db0::python // NOTE: ref-counts are not compared (only user-assigned members) // @return true if objects are identical PyObject *tryCompareMemo(MemoObject *, MemoObject *); - + PyObject *PyAPI_getSchema(PyObject *, PyObject *const *args, Py_ssize_t nargs); - PyObject* executeLoadFunction(PyObject * load_method, PyObject *kwargs, PyObject *py_exclude, - std::unordered_set *load_stack_ptr); + PyObject* executeLoadFunction(PyObject *load_method, PyObject *kwargs, PyObject *py_exclude, + std::unordered_set *load_stack_ptr); + } \ No newline at end of file diff --git a/src/dbzero/bindings/python/MemoTypeDecoration.cpp b/src/dbzero/bindings/python/MemoTypeDecoration.cpp index c71ab4ad..eccbb428 100644 --- a/src/dbzero/bindings/python/MemoTypeDecoration.cpp +++ b/src/dbzero/bindings/python/MemoTypeDecoration.cpp @@ -88,7 +88,7 @@ namespace db0::python } } - std::uint64_t MemoTypeDecoration::getFixtureUUID(std::optional access_type) + std::uint64_t MemoTypeDecoration::getFixtureUUID(std::optional access_type) const { if (!!m_prefix_name && !m_fixture_uuid) { if (!access_type) { diff --git a/src/dbzero/bindings/python/MemoTypeDecoration.hpp b/src/dbzero/bindings/python/MemoTypeDecoration.hpp index ac95941f..7a5e6b30 100644 --- a/src/dbzero/bindings/python/MemoTypeDecoration.hpp +++ b/src/dbzero/bindings/python/MemoTypeDecoration.hpp @@ -71,11 +71,11 @@ namespace db0::python // @param access_type to use for opening the prefix if UUID needs to be resolved by name // note that read-only access cannot later be upgraded to read-write // NOTE: if access type is not provided (std::nullopt), then READ_WRITE will be used as the default - std::uint64_t getFixtureUUID(std::optional access_type = AccessType::READ_WRITE); + std::uint64_t getFixtureUUID(std::optional access_type = AccessType::READ_WRITE) const; // check if the dyn-prefix callable is set bool hasDynPrefix() const; - + // resolve dynamic prefix from the callable std::string getDynPrefix(PyObject *args, PyObject *kwargs) const; @@ -104,7 +104,7 @@ namespace db0::python std::vector m_init_vars; MemoFlags m_flags; // resolved fixture UUID (initialized by the process) - std::atomic m_fixture_uuid = 0; + mutable std::atomic m_fixture_uuid = 0; // dynamic prefix callable shared_py_object m_py_dyn_prefix_callable; std::vector m_migrations; diff --git a/src/dbzero/object_model/object/Object.cpp b/src/dbzero/object_model/object/Object.cpp index dadaf2f2..1efe804c 100644 --- a/src/dbzero/object_model/object/Object.cpp +++ b/src/dbzero/object_model/object/Object.cpp @@ -1,9 +1,337 @@ #include "Object.hpp" +#include +#include namespace db0::object_model { - GC0_Define(Object) + GC0_Define(Object) + + bool isEqual(const KV_Index *kv_ptr_1, const KV_Index *kv_ptr_2) + { + if (!kv_ptr_1) { + return !kv_ptr_2; + } + + if (!kv_ptr_2) { + return false; + } + + // item-wise comparison + return *kv_ptr_1 == *kv_ptr_2; + } + + void Object::set(FixtureLock &fixture, const char *field_name, ObjectPtr lang_value) + { + assert(hasInstance()); + // attribute delete operation + if (!PyToolkit::isValid(lang_value)) { + remove(fixture, field_name); + return; + } + + auto [type_id, storage_class] = recognizeType(**fixture, lang_value); + + if (this->span() > 1) { + // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag + // this is because the actual change may be missed if performed on a different-then the 1st DP + _touch(); + } + + assert(m_type); + // find already existing field index + auto [member_id, is_init_var] = m_type->findField(field_name); + auto storage_fidelity = getStorageFidelity(storage_class); + // get field ID matching the required storage fidelity + FieldID field_id; + FieldInfo old_field_info; + unsigned int old_pos = 0; + const void *old_loc_ptr = nullptr; + if (member_id) { + std::tie(old_field_info, old_loc_ptr) = tryGetMemberSlot(member_id, old_pos); + } + + if (!member_id || !(field_id = member_id.tryGet(storage_fidelity))) { + // try mutating the class first + member_id = m_type->addField(field_name, storage_fidelity); + field_id = member_id.get(storage_fidelity); + } + + assert(field_id && member_id); + // NOTE: a new member inherits the parent's no-cache flag + // FIXME: value should be destroyed on exception + auto value = createMember( + *fixture, type_id, storage_class, lang_value, this->getMemberFlags() + ); + // make sure object address is not null + assert(!(storage_class == StorageClass::OBJECT_REF && value.cast() == 0)); + + if (field_id == old_field_info.first) { + // Set / update value at the existing location + setWithLoc(fixture, field_id, old_loc_ptr, old_pos, storage_fidelity, storage_class, value); + } else { + // must reset / unreference the old value (stored elsewhere) + if (old_field_info.first) { + unrefWithLoc(fixture, old_field_info.first, old_loc_ptr, old_pos, StorageClass::UNDEFINED, + old_field_info.second); + } + + const void *loc_ptr = nullptr; + unsigned int pos = 0; + // NOTE: slot may already exist (pos-vt or index-vt) either for regular or lo-fi storage + std::tie(loc_ptr, pos) = tryGetLoc(field_id); + // Either use existing slot or create a new (kv-index) + addWithLoc(fixture, field_id, loc_ptr, pos, storage_fidelity, storage_class, value); + } + + // the KV-index insert operation must be registered as the potential silent mutation + // but the operation can be avoided if the object is already marked as modified + if (!super_t::isModified()) { + this->_touch(); + } + } + + void Object::remove(FixtureLock &fixture, const char *field_name) + { + assert(hasInstance()); + if (this->span() > 1) { + // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag + // this is because the actual change may be missed if performed on a different-then the 1st DP + _touch(); + } + + assert(m_type); + // Find an already existing field index + auto [member_id, is_init_var] = m_type->findField(field_name); + if (!member_id) { + THROWF(db0::InputException) << "Attribute not found: " << field_name; + } + + unsigned int pos = 0; + auto [field_info, loc_ptr] = tryGetMemberSlot(member_id, pos); + if (!field_info.first) { + THROWF(db0::InputException) << "Attribute not found: " << field_name; + } + + // NOTE: unreference as DELETED + unrefWithLoc(fixture, field_info.first, loc_ptr, pos, StorageClass::DELETED, + field_info.second); + // the KV-index erase operation must be registered as the potential silent mutation + // but the operation can be avoided if the object is already marked as modified + if (!loc_ptr && !super_t::isModified()) { + this->_touch(); + } + } + + KV_Index *Object::addKV_First(const XValue &value) + { + if (!m_kv_index) { + if ((*this)->m_kv_address) { + m_kv_index = std::make_unique( + std::make_pair(&getMemspace(), (*this)->m_kv_address), (*this)->m_kv_type + ); + } else { + // create new kv-index intiialized with the first value + m_kv_index = std::make_unique(getMemspace(), value); + this->modify().m_kv_address = m_kv_index->getAddress(); + this->modify().m_kv_type = m_kv_index->getIndexType(); + // return nullptr to indicate that the value has been inserted + return nullptr; + } + } + // return reference without inserting + return m_kv_index.get(); + } + + bool Object::hasKV_Index() const { + return m_kv_index || (*this)->m_kv_address; + } + + KV_Index *Object::tryGetKV_Index() const + { + // if KV index address has changed, update the cached instance + if (!m_kv_index || m_kv_index->getAddress() != (*this)->m_kv_address) { + if ((*this)->m_kv_address) { + m_kv_index = std::make_unique( + std::make_pair(&getMemspace(), (*this)->m_kv_address), (*this)->m_kv_type + ); + } + } + + return m_kv_index.get(); + } + + void Object::addToKVIndex(FixtureLock &fixture, FieldID field_id, unsigned int fidelity, + StorageClass storage_class, Value value) + { + assert(m_type); + XValue xvalue(field_id.getIndex(), storage_class, value); + // encode for lo-fi storage if needed + if (fidelity != 0) { + xvalue.m_value = lofi_store<2>::create(field_id.getOffset(), value.m_store); + } + auto kv_index_ptr = addKV_First(xvalue); + if (kv_index_ptr) { + // NOTE: for fidelity > 0 the element might already exist + XValue old_value; + if (fidelity > 0 && kv_index_ptr->updateExisting(xvalue, &old_value)) { + auto kv_value = old_value.m_value; + lofi_store<2>::fromValue(kv_value).set(field_id.getOffset(), value.m_store); + xvalue.m_value = kv_value; + kv_index_ptr->updateExisting(xvalue); + // in case of the IttyIndex updating an element changes the address + // which needs to be updated in the object + if (kv_index_ptr->getIndexType() == bindex::type::itty) { + this->modify().m_kv_address = kv_index_ptr->getAddress(); + } + } else { + if (kv_index_ptr->insert(xvalue)) { + // type or address of the kv-index has changed which needs to be reflected + this->modify().m_kv_address = kv_index_ptr->getAddress(); + this->modify().m_kv_type = kv_index_ptr->getIndexType(); + } + } + } + + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + } + + void Object::unrefKVIndexValue(FixtureLock &fixture, FieldID field_id, StorageClass storage_class, + unsigned int fidelity) + { + auto kv_index_ptr = tryGetKV_Index(); + if (!kv_index_ptr) { + THROWF(db0::InputException) << "Attribute not found"; + } + XValue xvalue(field_id.getIndex()); + if (!kv_index_ptr->findOne(xvalue)) { + THROWF(db0::InputException) << "Attribute not found"; + } + + if (fidelity == 0) { + unrefMember(*fixture, xvalue); + if (storage_class == StorageClass::DELETED) { + // mark as deleted in kv-index + xvalue.m_type = StorageClass::DELETED; + kv_index_ptr->updateExisting(xvalue); + // in case of the IttyIndex updating an element changes the address + // which needs to be updated in the object + if (kv_index_ptr->getIndexType() == bindex::type::itty) { + this->modify().m_kv_address = kv_index_ptr->getAddress(); + } + } else { + auto old_addr = kv_index_ptr->getAddress(); + kv_index_ptr->erase(xvalue); + auto new_addr = kv_index_ptr->getAddress(); + if (new_addr != old_addr) { + // type or address of the kv-index has changed which needs to be reflected + this->modify().m_kv_address = new_addr; + this->modify().m_kv_type = kv_index_ptr->getIndexType(); + } + } + m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(xvalue.m_type)); + } else { + assert(fidelity == 2); + auto value = xvalue.m_value; + auto offset = field_id.getOffset(); + if (storage_class != StorageClass::DELETED && !lofi_store<2>::fromValue(value).isSet(offset)) { + // value is already unset + return; + } + auto old_type_id = getSchemaTypeId(xvalue.m_type, lofi_store<2>::fromValue(value).get(offset)); + if (storage_class == StorageClass::DELETED) { + lofi_store<2>::fromValue(value).set(offset, Value::DELETED); + } else { + lofi_store<2>::fromValue(value).reset(offset); + } + xvalue.m_value = value; + kv_index_ptr->updateExisting(xvalue); + // in case of the IttyIndex updating an element changes the address + // which needs to be updated in the object + if (kv_index_ptr->getIndexType() == bindex::type::itty) { + this->modify().m_kv_address = kv_index_ptr->getAddress(); + } + + m_type->removeFromSchema(field_id, fidelity, old_type_id); + } + } + + void Object::dropMembers(db0::swine_ptr &fixture, Class &class_ref) const + { + super_t::dropMembers(fixture, class_ref); + // finally drop kv-index members + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto it = kv_index_ptr->beginJoin(1); + for (;!it.is_end(); ++it) { + if ((*it).m_type == StorageClass::DELETED || (*it).m_type == StorageClass::UNDEFINED) { + // skip undefined or deleted members + continue; + } + unrefMember(fixture, *it); + class_ref.removeFromSchema(*it); + } + } + } + + bool Object::tryUnrefWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, + StorageClass storage_class, unsigned int fidelity) + { + if (!super_t::tryUnrefWithLoc(fixture, field_id, loc_ptr, pos, storage_class, fidelity)) { + unrefKVIndexValue(fixture, field_id, storage_class, fidelity); + } + return true; + } + + void Object::setKVIndexValue(FixtureLock &fixture, FieldID field_id, unsigned int fidelity, + StorageClass storage_class, Value value) + { + assert(m_type); + XValue xvalue(field_id.getIndex(), storage_class, value); + // encode for lo-fi storage if needed + if (fidelity != 0) { + xvalue.m_value = lofi_store<2>::create(field_id.getOffset(), value.m_store); + } + + auto kv_index_ptr = addKV_First(xvalue); + if (kv_index_ptr) { + // try updating an existing element first + XValue old_value; + if (kv_index_ptr->updateExisting(xvalue, &old_value)) { + if (fidelity == 0) { + unrefMember(*fixture, old_value); + m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_value.m_type), + getSchemaTypeId(storage_class) + ); + } else { + auto kv_value = old_value.m_value; + auto offset = field_id.getOffset(); + auto old_type_id = getSchemaTypeId(old_value.m_type, lofi_store<2>::fromValue(kv_value).get(offset)); + lofi_store<2>::fromValue(kv_value).set(offset, value.m_store); + xvalue.m_value = kv_value; + kv_index_ptr->updateExisting(xvalue); + auto new_type_id = getSchemaTypeId(storage_class, value); + m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); + } + // in case of the IttyIndex updating an element changes the address + // which needs to be updated in the object + if (kv_index_ptr->getIndexType() == bindex::type::itty) { + this->modify().m_kv_address = kv_index_ptr->getAddress(); + } + } else { + if (kv_index_ptr->insert(xvalue)) { + // type or address of the kv-index has changed which needs to be reflected + this->modify().m_kv_address = kv_index_ptr->getAddress(); + this->modify().m_kv_type = kv_index_ptr->getIndexType(); + } + + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + } + } else { + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + } + } + } \ No newline at end of file diff --git a/src/dbzero/object_model/object/Object.hpp b/src/dbzero/object_model/object/Object.hpp index 90866695..5d4e5083 100644 --- a/src/dbzero/object_model/object/Object.hpp +++ b/src/dbzero/object_model/object/Object.hpp @@ -7,19 +7,45 @@ namespace db0::object_model { - class Object: public ObjectImplBase + class Object: public ObjectImplBase { // GC0 specific declarations GC0_Declare public: static constexpr unsigned char REALM_ID = o_object::REALM_ID; - using super_t = ObjectImplBase; + using super_t = ObjectImplBase; template Object(Args&&... args) : super_t(std::forward(args)...) { } + + // Assign language specific value as a field (to already initialized or uninitialized instance) + // NOTE: if lang_value is nullptr then the member is removed + void set(FixtureLock &, const char *field_name, ObjectPtr lang_value); + void remove(FixtureLock &, const char *field_name); + + protected: + void dropMembers(db0::swine_ptr &, Class &) const; + + bool tryUnrefWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, StorageClass, + unsigned int fidelity); + + /** + * If the KV_Index does not exist yet, create it and add the first value + * otherwise return instance of an existing KV_Index + */ + KV_Index *addKV_First(const XValue &); + + KV_Index *tryGetKV_Index() const; + + bool hasKV_Index() const; + + void addToKVIndex(FixtureLock &, FieldID, unsigned int fidelity, StorageClass, Value); + void unrefKVIndexValue(FixtureLock &, FieldID, StorageClass, unsigned int fidelity); + // Set or update member in kv-index + void setKVIndexValue(FixtureLock &, FieldID, unsigned int fidelity, StorageClass, Value); }; } diff --git a/src/dbzero/object_model/object/ObjectImmutableImpl.hpp b/src/dbzero/object_model/object/ObjectImmutableImpl.hpp index 98d21cdd..f3d5437c 100644 --- a/src/dbzero/object_model/object/ObjectImmutableImpl.hpp +++ b/src/dbzero/object_model/object/ObjectImmutableImpl.hpp @@ -7,13 +7,13 @@ namespace db0::object_model { - class ObjectImmutableImpl: public ObjectImplBase + class ObjectImmutableImpl: public ObjectImplBase { // GC0 specific declarations GC0_Declare public: static constexpr unsigned char REALM_ID = o_immutable_object::REALM_ID; - using super_t = ObjectImplBase; + using super_t = ObjectImplBase; template ObjectImmutableImpl(Args&&... args) @@ -21,5 +21,5 @@ namespace db0::object_model { } }; - + } diff --git a/src/dbzero/object_model/object/ObjectImplBase.cpp b/src/dbzero/object_model/object/ObjectImplBase.cpp index 6b9b88f5..abbc5838 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.cpp +++ b/src/dbzero/object_model/object/ObjectImplBase.cpp @@ -15,28 +15,15 @@ namespace db0::object_model { - template class ObjectImplBase; + template class ObjectImplBase; + template class ObjectImplBase; - template - ObjectInitializerManager ObjectImplBase::m_init_manager; + template + ObjectInitializerManager ObjectImplBase::m_init_manager; FlagSet getAccessOptions(const Class &type) { return type.isNoCache() ? FlagSet { AccessOptions::no_cache } : FlagSet {}; } - - bool isEqual(const KV_Index *kv_ptr_1, const KV_Index *kv_ptr_2) - { - if (!kv_ptr_1) { - return !kv_ptr_2; - } - - if (!kv_ptr_2) { - return false; - } - - // item-wise comparison - return *kv_ptr_1 == *kv_ptr_2; - } template IntT safeCast(unsigned int value, const char *err_msg) { @@ -46,30 +33,30 @@ namespace db0::object_model return static_cast(value); } - template - ObjectImplBase::ObjectImplBase(UniqueAddress addr, unsigned int ext_refs) + template + ObjectImplBase::ObjectImplBase(UniqueAddress addr, unsigned int ext_refs) : m_flags { ObjectOptions::DROPPED } , m_ext_refs(ext_refs) , m_unique_address(addr) { } - template - ObjectImplBase::ObjectImplBase(std::shared_ptr db0_class) + template + ObjectImplBase::ObjectImplBase(std::shared_ptr db0_class) { // prepare for initialization m_init_manager.addInitializer(*this, db0_class); } - template - ObjectImplBase::ObjectImplBase(TypeInitializer &&type_initializer) + template + ObjectImplBase::ObjectImplBase(TypeInitializer &&type_initializer) { // prepare for initialization m_init_manager.addInitializer(*this, std::move(type_initializer)); } - template - ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, std::shared_ptr type, + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, std::shared_ptr type, std::pair ref_counts, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset) : super_t(fixture, type->getClassRef(), ref_counts, safeCast(type->getNumBases() + 1, "Too many base classes"), pos_vt_data, pos_vt_offset, nullptr, nullptr, @@ -78,37 +65,37 @@ namespace db0::object_model { } - template - ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, Address address, AccessFlags access_mode) + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, Address address, AccessFlags access_mode) : super_t(typename super_t::tag_from_address(), fixture, address, access_mode) { } - template - ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type) + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type) : super_t(typename super_t::tag_from_stem(), fixture, std::move(stem)) , m_type(type) { assert(hasValidClassRef()); } - template - ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, Address address, std::shared_ptr type_hint, + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, Address address, std::shared_ptr type_hint, with_type_hint, AccessFlags access_mode) - : ObjectImplBase(fixture, address, access_mode) - { + : ObjectImplBase(fixture, address, access_mode) + { assert(*fixture == *type_hint->getFixture()); setTypeWithHint(type_hint); } - template - ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type_hint, with_type_hint) - : ObjectImplBase(fixture, std::move(stem), getTypeWithHint(*fixture, stem->getClassRef(), type_hint)) + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type_hint, with_type_hint) + : ObjectImplBase(fixture, std::move(stem), getTypeWithHint(*fixture, stem->getClassRef(), type_hint)) { } - template - ObjectImplBase::~ObjectImplBase() + template + ObjectImplBase::~ObjectImplBase() { // unregister needs to be called before destruction of members this->unregister(); @@ -118,18 +105,18 @@ namespace db0::object_model } } - template - void ObjectImplBase::dropInstance(FixtureLock &) + template + void ObjectImplBase::dropInstance(FixtureLock &) { auto unique_addr = this->getUniqueAddress(); auto ext_refs = this->getExtRefs(); - this->~ObjectImplBase(); + this->~ObjectImplBase(); // construct a null placeholder new ((void*)this) Object(unique_addr, ext_refs); } - template - typename ObjectImplBase::ObjectStem ObjectImplBase::tryUnloadStem(db0::swine_ptr &fixture, + template + typename ObjectImplBase::ObjectStem ObjectImplBase::tryUnloadStem(db0::swine_ptr &fixture, Address address, std::uint16_t instance_id, AccessFlags access_mode) { std::size_t size_of; @@ -145,8 +132,8 @@ namespace db0::object_model return stem; } - template - typename ObjectImplBase::ObjectStem ObjectImplBase::unloadStem(db0::swine_ptr &fixture, + template + typename ObjectImplBase::ObjectStem ObjectImplBase::unloadStem(db0::swine_ptr &fixture, Address address, std::uint16_t instance_id, AccessFlags access_mode) { auto result = tryUnloadStem(fixture, address, instance_id, access_mode); @@ -156,8 +143,8 @@ namespace db0::object_model return result; } - template - void ObjectImplBase::postInit(FixtureLock &fixture) + template + void ObjectImplBase::postInit(FixtureLock &fixture) { if (!this->hasInstance()) { auto &initializer = m_init_manager.getInitializer(*this); @@ -190,9 +177,9 @@ namespace db0::object_model assert(hasInstance()); } - template + template std::pair - ObjectImplBase::recognizeType(Fixture &fixture, ObjectPtr lang_value) const + ObjectImplBase::recognizeType(Fixture &fixture, ObjectPtr lang_value) const { auto type_id = LangToolkit::getTypeManager().getTypeId(lang_value); // NOTE: allow storage as PACK_2 @@ -216,8 +203,8 @@ namespace db0::object_model return { type_id, storage_class }; } - template - void ObjectImplBase::removePreInit(const char *field_name) const + template + void ObjectImplBase::removePreInit(const char *field_name) const { auto &initializer = m_init_manager.getInitializer(*this); auto &type = initializer.getClass(); @@ -247,8 +234,8 @@ namespace db0::object_model } } - template - void ObjectImplBase::setPreInit(const char *field_name, ObjectPtr obj_ptr) const + template + void ObjectImplBase::setPreInit(const char *field_name, ObjectPtr obj_ptr) const { assert(!hasInstance()); if (!LangToolkit::isValid(obj_ptr)) { @@ -300,42 +287,8 @@ namespace db0::object_model } } - template - void ObjectImplBase::remove(FixtureLock &fixture, const char *field_name) - { - assert(hasInstance()); - if (this->span() > 1) { - // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag - // this is because the actual change may be missed if performed on a different-then the 1st DP - _touch(); - } - - assert(m_type); - // Find an already existing field index - auto [member_id, is_init_var] = m_type->findField(field_name); - if (!member_id) { - THROWF(db0::InputException) << "Attribute not found: " << field_name; - } - - unsigned int pos = 0; - auto [field_info, loc_ptr] = tryGetMemberSlot(member_id, pos); - if (!field_info.first) { - THROWF(db0::InputException) << "Attribute not found: " << field_name; - } - - // NOTE: unreference as DELETED - unrefWithLoc(fixture, field_info.first, loc_ptr, pos, StorageClass::DELETED, - field_info.second); - - // the KV-index erase operation must be registered as the potential silent mutation - // but the operation can be avoided if the object is already marked as modified - if (!loc_ptr && !super_t::isModified()) { - this->_touch(); - } - } - - template - void ObjectImplBase::unrefPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, + template + void ObjectImplBase::unrefPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, StorageClass storage_class, unsigned int fidelity) { auto &pos_vt = this->modify().pos_vt(); @@ -366,8 +319,8 @@ namespace db0::object_model } } - template - void ObjectImplBase::unrefIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, + template + void ObjectImplBase::unrefIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, StorageClass storage_class, unsigned int fidelity) { auto &index_vt = this->modify().index_vt(); @@ -395,83 +348,30 @@ namespace db0::object_model m_type->removeFromSchema(field_id, fidelity, old_type_id); } } - - template - void ObjectImplBase::unrefKVIndexValue(FixtureLock &fixture, FieldID field_id, + + template + void ObjectImplBase::unrefWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, StorageClass storage_class, unsigned int fidelity) { - auto kv_index_ptr = tryGetKV_Index(); - if (!kv_index_ptr) { - THROWF(db0::InputException) << "Attribute not found"; - } - XValue xvalue(field_id.getIndex()); - if (!kv_index_ptr->findOne(xvalue)) { - THROWF(db0::InputException) << "Attribute not found"; - } - - if (fidelity == 0) { - unrefMember(*fixture, xvalue); - if (storage_class == StorageClass::DELETED) { - // mark as deleted in kv-index - xvalue.m_type = StorageClass::DELETED; - kv_index_ptr->updateExisting(xvalue); - // in case of the IttyIndex updating an element changes the address - // which needs to be updated in the object - if (kv_index_ptr->getIndexType() == bindex::type::itty) { - this->modify().m_kv_address = kv_index_ptr->getAddress(); - } - } else { - auto old_addr = kv_index_ptr->getAddress(); - kv_index_ptr->erase(xvalue); - auto new_addr = kv_index_ptr->getAddress(); - if (new_addr != old_addr) { - // type or address of the kv-index has changed which needs to be reflected - this->modify().m_kv_address = new_addr; - this->modify().m_kv_type = kv_index_ptr->getIndexType(); - } - } - m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(xvalue.m_type)); - } else { - assert(fidelity == 2); - auto value = xvalue.m_value; - auto offset = field_id.getOffset(); - if (storage_class != StorageClass::DELETED && !lofi_store<2>::fromValue(value).isSet(offset)) { - // value is already unset - return; - } - auto old_type_id = getSchemaTypeId(xvalue.m_type, lofi_store<2>::fromValue(value).get(offset)); - if (storage_class == StorageClass::DELETED) { - lofi_store<2>::fromValue(value).set(offset, Value::DELETED); - } else { - lofi_store<2>::fromValue(value).reset(offset); - } - xvalue.m_value = value; - kv_index_ptr->updateExisting(xvalue); - // in case of the IttyIndex updating an element changes the address - // which needs to be updated in the object - if (kv_index_ptr->getIndexType() == bindex::type::itty) { - this->modify().m_kv_address = kv_index_ptr->getAddress(); - } - - m_type->removeFromSchema(field_id, fidelity, old_type_id); - } + ImplT::unrefWithLoc(fixture, field_id, loc_ptr, pos, storage_class, fidelity); } - template - void ObjectImplBase::unrefWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, + template + bool ObjectImplBase::tryUnrefWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, StorageClass storage_class, unsigned int fidelity) { - if (loc_ptr == &(*this)->pos_vt()) { + if (loc_ptr == &(*this)->pos_vt()) { unrefPosVT(fixture, field_id, pos, storage_class, fidelity); + return true; } else if (loc_ptr == &(*this)->index_vt()) { unrefIndexVT(fixture, field_id, pos, storage_class, fidelity); - } else { - unrefKVIndexValue(fixture, field_id, storage_class, fidelity); + return true; } + return false; } - template - void ObjectImplBase::setPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, + template + void ObjectImplBase::setPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, StorageClass storage_class, Value value) { auto &pos_vt = this->modify().pos_vt(); @@ -492,8 +392,8 @@ namespace db0::object_model } } - template - void ObjectImplBase::addToPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, + template + void ObjectImplBase::addToPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, StorageClass storage_class, Value value) { auto &pos_vt = this->modify().pos_vt(); @@ -509,9 +409,9 @@ namespace db0::object_model m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); } } - - template - void ObjectImplBase::setIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, + + template + void ObjectImplBase::setIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, unsigned int fidelity, StorageClass storage_class, Value value) { auto &index_vt = this->modify().index_vt(); @@ -531,8 +431,8 @@ namespace db0::object_model } } - template - void ObjectImplBase::addToIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, + template + void ObjectImplBase::addToIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, unsigned int fidelity, StorageClass storage_class, Value value) { auto &index_vt = this->modify().index_vt(); @@ -546,95 +446,9 @@ namespace db0::object_model m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); } } - - template - void ObjectImplBase::setKVIndexValue(FixtureLock &fixture, FieldID field_id, unsigned int fidelity, - StorageClass storage_class, Value value) - { - assert(m_type); - XValue xvalue(field_id.getIndex(), storage_class, value); - // encode for lo-fi storage if needed - if (fidelity != 0) { - xvalue.m_value = lofi_store<2>::create(field_id.getOffset(), value.m_store); - } - - auto kv_index_ptr = addKV_First(xvalue); - if (kv_index_ptr) { - // try updating an existing element first - XValue old_value; - if (kv_index_ptr->updateExisting(xvalue, &old_value)) { - if (fidelity == 0) { - unrefMember(*fixture, old_value); - m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_value.m_type), - getSchemaTypeId(storage_class) - ); - } else { - auto kv_value = old_value.m_value; - auto offset = field_id.getOffset(); - auto old_type_id = getSchemaTypeId(old_value.m_type, lofi_store<2>::fromValue(kv_value).get(offset)); - lofi_store<2>::fromValue(kv_value).set(offset, value.m_store); - xvalue.m_value = kv_value; - kv_index_ptr->updateExisting(xvalue); - auto new_type_id = getSchemaTypeId(storage_class, value); - m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); - } - // in case of the IttyIndex updating an element changes the address - // which needs to be updated in the object - if (kv_index_ptr->getIndexType() == bindex::type::itty) { - this->modify().m_kv_address = kv_index_ptr->getAddress(); - } - } else { - if (kv_index_ptr->insert(xvalue)) { - // type or address of the kv-index has changed which needs to be reflected - this->modify().m_kv_address = kv_index_ptr->getAddress(); - this->modify().m_kv_type = kv_index_ptr->getIndexType(); - } - - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); - } - } else { - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); - } - } - - template - void ObjectImplBase::addToKVIndex(FixtureLock &fixture, FieldID field_id, unsigned int fidelity, - StorageClass storage_class, Value value) - { - assert(m_type); - XValue xvalue(field_id.getIndex(), storage_class, value); - // encode for lo-fi storage if needed - if (fidelity != 0) { - xvalue.m_value = lofi_store<2>::create(field_id.getOffset(), value.m_store); - } - auto kv_index_ptr = addKV_First(xvalue); - if (kv_index_ptr) { - // NOTE: for fidelity > 0 the element might already exist - XValue old_value; - if (fidelity > 0 && kv_index_ptr->updateExisting(xvalue, &old_value)) { - auto kv_value = old_value.m_value; - lofi_store<2>::fromValue(kv_value).set(field_id.getOffset(), value.m_store); - xvalue.m_value = kv_value; - kv_index_ptr->updateExisting(xvalue); - // in case of the IttyIndex updating an element changes the address - // which needs to be updated in the object - if (kv_index_ptr->getIndexType() == bindex::type::itty) { - this->modify().m_kv_address = kv_index_ptr->getAddress(); - } - } else { - if (kv_index_ptr->insert(xvalue)) { - // type or address of the kv-index has changed which needs to be reflected - this->modify().m_kv_address = kv_index_ptr->getAddress(); - this->modify().m_kv_type = kv_index_ptr->getIndexType(); - } - } - } - - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); - } - template std::pair - ObjectImplBase::tryGetMemberSlot(const MemberID &member_id, unsigned int &pos) const + template std::pair + ObjectImplBase::tryGetMemberSlot(const MemberID &member_id, unsigned int &pos) const { for (auto &field_info: member_id) { auto [index, offset] = field_info.first.getIndexAndOffset(); @@ -672,8 +486,8 @@ namespace db0::object_model return { {}, nullptr }; } - template - std::pair ObjectImplBase::tryGetLoc(FieldID field_id) const + template + std::pair ObjectImplBase::tryGetLoc(FieldID field_id) const { auto index = field_id.getIndex(); unsigned int pos = 0; @@ -689,8 +503,8 @@ namespace db0::object_model return { nullptr, 0 }; } - template - void ObjectImplBase::setWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, + template + void ObjectImplBase::setWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, unsigned int fidelity, StorageClass storage_class, Value value) { if (loc_ptr == &(*this)->pos_vt()) { @@ -708,8 +522,8 @@ namespace db0::object_model setKVIndexValue(fixture, field_id, fidelity, storage_class, value); } - template - void ObjectImplBase::addWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, + template + void ObjectImplBase::addWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, unsigned int fidelity, StorageClass storage_class, Value value) { if (loc_ptr == &(*this)->pos_vt()) { @@ -725,80 +539,9 @@ namespace db0::object_model assert(!loc_ptr); addToKVIndex(fixture, field_id, fidelity, storage_class, value); } - - template - void ObjectImplBase::set(FixtureLock &fixture, const char *field_name, ObjectPtr lang_value) - { - assert(hasInstance()); - // attribute delete operation - if (!PyToolkit::isValid(lang_value)) { - remove(fixture, field_name); - return; - } - auto [type_id, storage_class] = recognizeType(**fixture, lang_value); - - if (this->span() > 1) { - // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag - // this is because the actual change may be missed if performed on a different-then the 1st DP - _touch(); - } - - assert(m_type); - // find already existing field index - auto [member_id, is_init_var] = m_type->findField(field_name); - auto storage_fidelity = getStorageFidelity(storage_class); - // get field ID matching the required storage fidelity - FieldID field_id; - FieldInfo old_field_info; - unsigned int old_pos = 0; - const void *old_loc_ptr = nullptr; - if (member_id) { - std::tie(old_field_info, old_loc_ptr) = tryGetMemberSlot(member_id, old_pos); - } - - if (!member_id || !(field_id = member_id.tryGet(storage_fidelity))) { - // try mutating the class first - member_id = m_type->addField(field_name, storage_fidelity); - field_id = member_id.get(storage_fidelity); - } - - assert(field_id && member_id); - // NOTE: a new member inherits the parent's no-cache flag - // FIXME: value should be destroyed on exception - auto value = createMember( - *fixture, type_id, storage_class, lang_value, this->getMemberFlags() - ); - // make sure object address is not null - assert(!(storage_class == StorageClass::OBJECT_REF && value.cast() == 0)); - - if (field_id == old_field_info.first) { - // Set / update value at the existing location - setWithLoc(fixture, field_id, old_loc_ptr, old_pos, storage_fidelity, storage_class, value); - } else { - // must reset / unreference the old value (stored elsewhere) - if (old_field_info.first) { - unrefWithLoc(fixture, old_field_info.first, old_loc_ptr, old_pos, StorageClass::UNDEFINED, - old_field_info.second); - } - - const void *loc_ptr = nullptr; - unsigned int pos = 0; - // NOTE: slot may already exist (pos-vt or index-vt) either for regular or lo-fi storage - std::tie(loc_ptr, pos) = tryGetLoc(field_id); - // Either use existing slot or create a new (kv-index) - addWithLoc(fixture, field_id, loc_ptr, pos, storage_fidelity, storage_class, value); - } - - // the KV-index insert operation must be registered as the potential silent mutation - // but the operation can be avoided if the object is already marked as modified - if (!super_t::isModified()) { - this->_touch(); - } - } - - template - std::pair ObjectImplBase::findField(const char *name) const + template + std::pair ObjectImplBase::findField(const char *name) const { if (isDropped()) { // defunct objects should not be accessed @@ -815,11 +558,11 @@ namespace db0::object_model assert(class_ptr); return class_ptr->findField(name); } - - template - FieldID ObjectImplBase::tryGetMember(const char *field_name, std::pair &member, + + template + FieldID ObjectImplBase::tryGetMember(const char *field_name, std::pair &member, bool &is_init_var) const - { + { MemberID member_id; std::tie(member_id, is_init_var) = this->findField(field_name); bool exists, deleted = false; @@ -851,8 +594,8 @@ namespace db0::object_model return {}; } - template - std::optional ObjectImplBase::tryGetX(const char *field_name) const + template + std::optional ObjectImplBase::tryGetX(const char *field_name) const { auto [member_id, is_init_var] = this->findField(field_name); bool exists, deleted = false; @@ -883,8 +626,8 @@ namespace db0::object_model return std::nullopt; } - template - typename ObjectImplBase::ObjectSharedPtr ObjectImplBase::tryGet(const char *field_name) const + template + typename ObjectImplBase::ObjectSharedPtr ObjectImplBase::tryGet(const char *field_name) const { std::pair member; bool is_init_var = false; @@ -903,8 +646,8 @@ namespace db0::object_model return nullptr; } - template - typename ObjectImplBase::ObjectSharedPtr ObjectImplBase::tryGetAs( + template + typename ObjectImplBase::ObjectSharedPtr ObjectImplBase::tryGetAs( const char *field_name, TypeObjectPtr lang_type) const { std::pair member; @@ -928,8 +671,8 @@ namespace db0::object_model return nullptr; } - template - typename ObjectImplBase::ObjectSharedPtr ObjectImplBase::get(const char *field_name) const + template + typename ObjectImplBase::ObjectSharedPtr ObjectImplBase::get(const char *field_name) const { auto obj = tryGet(field_name); if (!obj) { @@ -941,8 +684,8 @@ namespace db0::object_model return obj; } - template - bool ObjectImplBase::slotExists(Value value, unsigned int fidelity, unsigned int at) const + template + bool ObjectImplBase::slotExists(Value value, unsigned int fidelity, unsigned int at) const { assert(fidelity != 0 && "Operation only available for lo-fi values"); // lo-fi value @@ -950,8 +693,8 @@ namespace db0::object_model return lofi_store<2>::fromValue(value).isSet(at); } - template - std::pair ObjectImplBase::hasValueAt(Value value, unsigned int fidelity, unsigned int at) const + template + std::pair ObjectImplBase::hasValueAt(Value value, unsigned int fidelity, unsigned int at) const { assert(fidelity != 0 && "Operation only available for lo-fi values"); // lo-fi value @@ -966,8 +709,8 @@ namespace db0::object_model } } - template - std::pair ObjectImplBase::tryGetMemberAt(std::pair field_info, + template + std::pair ObjectImplBase::tryGetMemberAt(std::pair field_info, std::pair &result) const { if (!field_info.first) { @@ -1038,7 +781,7 @@ namespace db0::object_model return { false, false }; } - template + template db0::swine_ptr ObjectImplBase::tryGetFixture() const { if (!this->hasInstance()) { @@ -1051,8 +794,8 @@ namespace db0::object_model return super_t::tryGetFixture(); } - template - db0::swine_ptr ObjectImplBase::getFixture() const + template + db0::swine_ptr ObjectImplBase::getFixture() const { auto fixture = this->tryGetFixture(); if (!fixture) { @@ -1060,22 +803,22 @@ namespace db0::object_model } return fixture; } - - template - Memspace &ObjectImplBase::getMemspace() const { + + template + Memspace &ObjectImplBase::getMemspace() const { return *getFixture(); } - template - void ObjectImplBase::setType(std::shared_ptr type) + template + void ObjectImplBase::setType(std::shared_ptr type) { assert(!m_type); m_type = type; assert(hasValidClassRef()); } - template - void ObjectImplBase::setTypeWithHint(std::shared_ptr type_hint) + template + void ObjectImplBase::setTypeWithHint(std::shared_ptr type_hint) { assert(!m_type); assert(type_hint); @@ -1087,13 +830,13 @@ namespace db0::object_model } } - template - bool ObjectImplBase::isSingleton() const { + template + bool ObjectImplBase::isSingleton() const { return getType().isSingleton(); } - template - void ObjectImplBase::dropTags(Class &type) const + template + void ObjectImplBase::dropTags(Class &type) const { // only drop if any type tags are assigned if ((*this)->m_header.m_ref_counter.getFirst() > 0) { @@ -1111,11 +854,17 @@ namespace db0::object_model } } - template - void ObjectImplBase::dropMembers(Class &class_ref) const + template + void ObjectImplBase::dropMembers(Class &class_ref) const { auto fixture = this->getFixture(); - assert(fixture); + assert(fixture); + ImplT::dropMembers(*fixture, class_ref); + } + + template + void ObjectImplBase::dropMembers(db0::swine_ptr &fixture, Class &class_ref) const + { // drop pos-vt members first { auto &types = (*this)->pos_vt().types(); @@ -1143,23 +892,10 @@ namespace db0::object_model class_ref.removeFromSchema(xvalue); } } - // finally drop kv-index members - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto it = kv_index_ptr->beginJoin(1); - for (;!it.is_end(); ++it) { - if ((*it).m_type == StorageClass::DELETED || (*it).m_type == StorageClass::UNDEFINED) { - // skip undefined or deleted members - continue; - } - unrefMember(fixture, *it); - class_ref.removeFromSchema(*it); - } - } } - template - void ObjectImplBase::unSingleton(FixtureLock &) + template + void ObjectImplBase::unSingleton(FixtureLock &) { auto &type = getType(); // drop reference from the class @@ -1170,8 +906,8 @@ namespace db0::object_model } } - template - void ObjectImplBase::destroy() const + template + void ObjectImplBase::destroy() const { if (this->hasInstance()) { // associated class type (may require unloading) @@ -1189,8 +925,8 @@ namespace db0::object_model super_t::destroy(); } - template - FieldLayout ObjectImplBase::getFieldLayout() const + template + FieldLayout ObjectImplBase::getFieldLayout() const { FieldLayout layout; // collect pos-vt information @@ -1215,54 +951,13 @@ namespace db0::object_model return layout; } - template - KV_Index *ObjectImplBase::addKV_First(const XValue &value) - { - if (!m_kv_index) { - if ((*this)->m_kv_address) { - m_kv_index = std::make_unique( - std::make_pair(&getMemspace(), (*this)->m_kv_address), (*this)->m_kv_type - ); - } else { - // create new kv-index intiialized with the first value - m_kv_index = std::make_unique(getMemspace(), value); - this->modify().m_kv_address = m_kv_index->getAddress(); - this->modify().m_kv_type = m_kv_index->getIndexType(); - // return nullptr to indicate that the value has been inserted - return nullptr; - } - } - // return reference without inserting - return m_kv_index.get(); - } - - template - bool ObjectImplBase::hasKV_Index() const { - return m_kv_index || (*this)->m_kv_address; - } - - template - KV_Index *ObjectImplBase::tryGetKV_Index() const - { - // if KV index address has changed, update the cached instance - if (!m_kv_index || m_kv_index->getAddress() != (*this)->m_kv_address) { - if ((*this)->m_kv_address) { - m_kv_index = std::make_unique( - std::make_pair(&getMemspace(), (*this)->m_kv_address), (*this)->m_kv_type - ); - } - } - - return m_kv_index.get(); - } - - template - Class &ObjectImplBase::getType() { + template + Class &ObjectImplBase::getType() { return m_type ? *m_type : m_init_manager.getInitializer(*this).getClass(); } - template - void ObjectImplBase::getMembersFrom(const Class &this_type, unsigned int index, StorageClass storage_class, + template + void ObjectImplBase::getMembersFrom(const Class &this_type, unsigned int index, StorageClass storage_class, Value value, std::unordered_set &result) const { if (storage_class == StorageClass::DELETED || storage_class == StorageClass::UNDEFINED) { @@ -1280,8 +975,8 @@ namespace db0::object_model } } - template - std::unordered_set ObjectImplBase::getMembers() const + template + std::unordered_set ObjectImplBase::getMembers() const { std::unordered_set result; // Visit pos-vt members first @@ -1317,8 +1012,8 @@ namespace db0::object_model return result; } - template - void ObjectImplBase::forAll(std::function f) const + template + void ObjectImplBase::forAll(std::function f) const { // Visit pos-vt members first auto &obj_type = this->getType(); @@ -1389,9 +1084,9 @@ namespace db0::object_model } } } - - template - void ObjectImplBase::forAll(std::function f) const + + template + void ObjectImplBase::forAll(std::function f) const { auto fixture = this->getFixture(); forAll([&](const std::string &name, const XValue &xvalue, unsigned int offset) -> bool { @@ -1403,8 +1098,8 @@ namespace db0::object_model }); } - template - bool ObjectImplBase::forAll(XValue xvalue, + template + bool ObjectImplBase::forAll(XValue xvalue, std::function f) const { assert(xvalue.m_type == StorageClass::PACK_2); @@ -1422,8 +1117,8 @@ namespace db0::object_model return true; } - template - void ObjectImplBase::incRef(bool is_tag) + template + void ObjectImplBase::incRef(bool is_tag) { if (this->hasInstance()) { super_t::incRef(is_tag); @@ -1433,8 +1128,8 @@ namespace db0::object_model } } - template - bool ObjectImplBase::decRef(bool is_tag) + template + bool ObjectImplBase::decRef(bool is_tag) { // this operation is a potentially silent mutation _touch(); @@ -1442,25 +1137,25 @@ namespace db0::object_model return !hasRefs(); } - template - bool ObjectImplBase::hasRefs() const + template + bool ObjectImplBase::hasRefs() const { assert(hasInstance()); return (*this)->hasRefs(); } - template - bool ObjectImplBase::hasAnyRefs() const { + template + bool ObjectImplBase::hasAnyRefs() const { return (*this)->hasAnyRefs(); } - template - bool ObjectImplBase::hasTagRefs() const { + template + bool ObjectImplBase::hasTagRefs() const { return this->hasInstance() && (*this)->m_header.m_ref_counter.getFirst() > 0; } - template - bool ObjectImplBase::equalTo(const ObjectImplBase &other) const + template + bool ObjectImplBase::equalTo(const ObjectImplBase &other) const { if (!this->hasInstance() || !other.hasInstance()) { THROWF(db0::InputException) << "Object not initialized"; @@ -1511,13 +1206,13 @@ namespace db0::object_model return result; } - template - void ObjectImplBase::moveTo(db0::swine_ptr &) { + template + void ObjectImplBase::moveTo(db0::swine_ptr &) { throw std::runtime_error("Not implemented"); } - template - void ObjectImplBase::setFixture(db0::swine_ptr &new_fixture) + template + void ObjectImplBase::setFixture(db0::swine_ptr &new_fixture) { if (this->hasInstance()) { THROWF(db0::InputException) << "set_prefix failed: object already initialized"; @@ -1530,8 +1225,8 @@ namespace db0::object_model } } - template - void ObjectImplBase::detach() const + template + void ObjectImplBase::detach() const { m_type->detach(); // invalidate since detach is not supported by the MorphingBIndex @@ -1539,8 +1234,8 @@ namespace db0::object_model super_t::detach(); } - template - void ObjectImplBase::commit() const + template + void ObjectImplBase::commit() const { m_type->commit(); if (m_kv_index) { @@ -1551,25 +1246,25 @@ namespace db0::object_model m_touched = false; } - template - void ObjectImplBase::unrefMember(db0::swine_ptr &fixture, StorageClass type, Value value) const { + template + void ObjectImplBase::unrefMember(db0::swine_ptr &fixture, StorageClass type, Value value) const { db0::object_model::unrefMember(fixture, type, value); } - template - void ObjectImplBase::unrefMember(db0::swine_ptr &fixture, XValue value) const { + template + void ObjectImplBase::unrefMember(db0::swine_ptr &fixture, XValue value) const { db0::object_model::unrefMember(fixture, value.m_type, value.m_value); } - template - std::shared_ptr ObjectImplBase::unloadType() const + template + std::shared_ptr ObjectImplBase::unloadType() const { auto fixture = this->getFixture(); return getClassFactory(*fixture).getTypeByClassRef((*this)->getClassRef()).m_class; } - template - Address ObjectImplBase::getAddress() const + template + Address ObjectImplBase::getAddress() const { assert(!isDefunct()); if (!this->hasInstance()) { @@ -1577,9 +1272,9 @@ namespace db0::object_model } return super_t::getAddress(); } - - template - UniqueAddress ObjectImplBase::getUniqueAddress() const + + template + UniqueAddress ObjectImplBase::getUniqueAddress() const { if (this->hasInstance()) { return super_t::getUniqueAddress(); @@ -1590,8 +1285,8 @@ namespace db0::object_model } } - template - bool ObjectImplBase::hasValidClassRef() const + template + bool ObjectImplBase::hasValidClassRef() const { if (this->hasInstance() && m_type) { return (*this)->getClassRef() == m_type->getClassRef(); @@ -1599,8 +1294,8 @@ namespace db0::object_model return true; } - template - std::shared_ptr ObjectImplBase::getTypeWithHint(const Fixture &fixture, std::uint32_t class_ref, + template + std::shared_ptr ObjectImplBase::getTypeWithHint(const Fixture &fixture, std::uint32_t class_ref, std::shared_ptr type_hint) { assert(type_hint); @@ -1610,13 +1305,13 @@ namespace db0::object_model return getClassFactory(fixture).getTypeByClassRef(class_ref).m_class; } - template - void ObjectImplBase::setDefunct() const { + template + void ObjectImplBase::setDefunct() const { m_flags.set(ObjectOptions::DEFUNCT); } - template - void ObjectImplBase::touch() + template + void ObjectImplBase::touch() { if (this->hasInstance() && !isDefunct()) { // NOTE: for already modified and small objects we may skip "touch" @@ -1628,8 +1323,8 @@ namespace db0::object_model } } - template - void ObjectImplBase::_touch() + template + void ObjectImplBase::_touch() { if (!m_touched) { // mark the 1st byte of the object as modified (forced-diff) @@ -1639,13 +1334,13 @@ namespace db0::object_model } } - template - void ObjectImplBase::addExtRef() const { + template + void ObjectImplBase::addExtRef() const { ++m_ext_refs; } - template - void ObjectImplBase::removeExtRef() const + template + void ObjectImplBase::removeExtRef() const { assert(m_ext_refs > 0); --m_ext_refs; diff --git a/src/dbzero/object_model/object/ObjectImplBase.hpp b/src/dbzero/object_model/object/ObjectImplBase.hpp index 845c6794..562cf502 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.hpp +++ b/src/dbzero/object_model/object/ObjectImplBase.hpp @@ -12,6 +12,7 @@ #include #include #include "o_object.hpp" +#include "o_immutable_object.hpp" namespace db0 @@ -26,6 +27,8 @@ namespace db0::object_model { class Class; + class Object; + class ObjectImmutableImpl; using Fixture = db0::Fixture; enum class ObjectOptions: std::uint8_t @@ -47,12 +50,12 @@ namespace db0::object_model // NOTE: Object instances are created within the implementation specific realm_id (e.g. =1 for o_object) template using ObjectVType = db0::v_object; - template - class ObjectImplBase: public db0::ObjectBase, StorageClass::OBJECT_REF> + template + class ObjectImplBase: public db0::ObjectBase, StorageClass::OBJECT_REF> { public: static constexpr unsigned char REALM_ID = T::REALM_ID; - using super_t = db0::ObjectBase, StorageClass::OBJECT_REF>; + using super_t = db0::ObjectBase, StorageClass::OBJECT_REF>; using LangToolkit = LangConfig::LangToolkit; using ObjectPtr = typename LangToolkit::ObjectPtr; using TypeObjectPtr = typename LangToolkit::TypeObjectPtr; @@ -63,8 +66,8 @@ namespace db0::object_model // Construct as null / dropped object ObjectImplBase(UniqueAddress, unsigned int ext_refs); - ObjectImplBase(const ObjectImplBase &) = delete; - ObjectImplBase(ObjectImplBase &&) = delete; + ObjectImplBase(const ObjectImplBase &) = delete; + ObjectImplBase(ObjectImplBase &&) = delete; /** * Construct new Object (uninitialized, without corresponding dbzero instance yet) @@ -104,12 +107,7 @@ namespace db0::object_model // Called to finalize adding members void endInit(); - - // Assign language specific value as a field (to already initialized or uninitialized instance) - // NOTE: if lang_value is nullptr then the member is removed - void set(FixtureLock &, const char *field_name, ObjectPtr lang_value); - void remove(FixtureLock &, const char *field_name); - + // Assign field of an uninitialized instance (assumed as a non-mutating operation) // NOTE: if lang_value is nullptr then the member is removed void setPreInit(const char *field_name, ObjectPtr lang_value) const; @@ -171,7 +169,7 @@ namespace db0::object_model // Binary (shallow) compare 2 objects or 2 versions of the same memo object (e.g. from different snapshots) // NOTE: ref-counts are not compared (only user-assigned members) // @return true if objects are identical - bool equalTo(const ObjectImplBase &) const; + bool equalTo(const ObjectImplBase &) const; /** * Move unreferenced object to a different prefix without changing the instance @@ -271,14 +269,7 @@ namespace db0::object_model return m_type ? static_cast(nullptr) : &m_init_manager.getInitializer(*this); } - /** - * If the KV_Index does not exist yet, create it and add the first value - * otherwise return instance of an existing KV_Index - */ - KV_Index *addKV_First(const XValue &); - - KV_Index *tryGetKV_Index() const; - + void dropMembers(db0::swine_ptr &, Class &) const; void dropMembers(Class &) const; void dropTags(Class &) const; @@ -295,8 +286,7 @@ namespace db0::object_model static std::shared_ptr getTypeWithHint(const Fixture &, std::uint32_t class_ref, std::shared_ptr type_hint); bool hasValidClassRef() const; - bool hasKV_Index() const; - + // try retrieving member as XValue std::optional tryGetX(const char *field_name) const; void _touch(); @@ -305,8 +295,6 @@ namespace db0::object_model void setPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); void setIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, StorageClass, Value); - // Set or update member in kv-index - void setKVIndexValue(FixtureLock &, FieldID, unsigned int fidelity, StorageClass, Value); // Set with a specific location (pos_vt, index_vt, kv-index) void setWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, @@ -316,16 +304,16 @@ namespace db0::object_model // NOTE: storage_class to be assigned can either be DELETED or UNDEFINED void unrefPosVT(FixtureLock &, FieldID, unsigned int pos, StorageClass, unsigned int fidelity); void unrefIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, StorageClass, unsigned int fidelity); - void unrefKVIndexValue(FixtureLock &, FieldID, StorageClass, unsigned int fidelity); - void unrefWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, StorageClass, + void unrefWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, StorageClass, + unsigned int fidelity); + bool tryUnrefWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, StorageClass, unsigned int fidelity); // Add a new value void addToPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); void addToIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, StorageClass, Value); - void addToKVIndex(FixtureLock &, FieldID, unsigned int fidelity, StorageClass, Value); - + void addWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, StorageClass, Value); @@ -336,7 +324,8 @@ namespace db0::object_model std::unordered_set &) const; }; - extern template class ObjectImplBase; + extern template class ObjectImplBase; + extern template class ObjectImplBase; } From d74cb0cc07c22fa4193f129f04e6f60a89968270 Mon Sep 17 00:00:00 2001 From: Wojtek Date: Tue, 4 Nov 2025 16:56:07 +0100 Subject: [PATCH 08/19] save work --- src/dbzero/bindings/python/Memo.cpp | 2 +- src/dbzero/object_model/object/Object.cpp | 222 +++++++++++- src/dbzero/object_model/object/Object.hpp | 34 +- .../object_model/object/ObjectImplBase.cpp | 328 ++++++------------ .../object_model/object/ObjectImplBase.hpp | 37 +- 5 files changed, 368 insertions(+), 255 deletions(-) diff --git a/src/dbzero/bindings/python/Memo.cpp b/src/dbzero/bindings/python/Memo.cpp index adc0f439..d9739e79 100644 --- a/src/dbzero/bindings/python/Memo.cpp +++ b/src/dbzero/bindings/python/Memo.cpp @@ -671,7 +671,7 @@ namespace db0::python PyObject *MemoObject_GetFieldLayout(MemoObject *self) { - auto field_layout = self->ext().getFieldLayout(); + auto field_layout = self->ext().getFieldLayout(); auto &pos_vt = field_layout.m_pos_vt_fields; auto &index_vt = field_layout.m_index_vt_fields; diff --git a/src/dbzero/object_model/object/Object.cpp b/src/dbzero/object_model/object/Object.cpp index 1efe804c..4d31d430 100644 --- a/src/dbzero/object_model/object/Object.cpp +++ b/src/dbzero/object_model/object/Object.cpp @@ -22,6 +22,68 @@ namespace db0::object_model return *kv_ptr_1 == *kv_ptr_2; } + void Object::getFieldLayoutImpl(FieldLayout &layout) const + { + super_t::getFieldLayoutImpl(layout); + // collect kv-index information + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto it = kv_index_ptr->beginJoin(1); + for (;!it.is_end(); ++it) { + layout.m_kv_index_fields.emplace_back((*it).getIndex(), (*it).m_type); + } + } + } + + void Object::getMembersImpl(std::unordered_set &result) const + { + super_t::getMembersImpl(result); + // Finally, visit kv-index members + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto &obj_type = this->getType(); + auto it = kv_index_ptr->beginJoin(1); + for (;!it.is_end(); ++it) { + auto index = (*it).getIndex(); + getMembersFrom(obj_type, index, (*it).m_type, (*it).m_value, result); + } + } + } + + bool Object::tryFindMemberAt(std::pair field_info, std::pair &result, + std::pair &find_result) const + { + if (!super_t::tryFindMemberAt(field_info, result, find_result)) { + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto loc = field_info.first.getIndexAndOffset(); + XValue xvalue(loc.first); + if (kv_index_ptr->findOne(xvalue)) { + assert(xvalue.getIndex() == loc.first); + if (xvalue.m_type == StorageClass::DELETED) { + // report as deleted + find_result = { false, true }; + return true; + } + + // member fetched from the kv_index + result.first = xvalue.m_type; + result.second = xvalue.m_value; + + if (field_info.second == 0) { + find_result = { result.first != StorageClass::UNDEFINED, false }; + return true; + } else { + find_result = hasValueAt(result.second, field_info.second, loc.second); + return true; + } + } + } + } + + return false; + } + void Object::set(FixtureLock &fixture, const char *field_name, ObjectPtr lang_value) { assert(hasInstance()); @@ -49,7 +111,7 @@ namespace db0::object_model unsigned int old_pos = 0; const void *old_loc_ptr = nullptr; if (member_id) { - std::tie(old_field_info, old_loc_ptr) = tryGetMemberSlot(member_id, old_pos); + std::tie(old_field_info, old_loc_ptr) = super_t::tryGetMemberSlot(member_id, old_pos); } if (!member_id || !(field_id = member_id.tryGet(storage_fidelity))) { @@ -334,4 +396,162 @@ namespace db0::object_model } } + bool Object::tryFindMemberSlot(const std::pair &field_info, unsigned int &pos, + std::pair &result) const + { + if (!super_t::tryFindMemberSlot(field_info, pos, result)) { + // kv-index lookup + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto [index, offset] = field_info.first.getIndexAndOffset(); + XValue value(index); + if (kv_index_ptr->findOne(value)) { + if (field_info.second == 0 || slotExists(value.m_value, field_info.second, offset)) { + result = { field_info, nullptr }; + return true; + } + } + } + } + + return false; + } + + void Object::setPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, + StorageClass storage_class, Value value) + { + auto &pos_vt = this->modify().pos_vt(); + auto pos_value = pos_vt.values()[pos]; + if (fidelity == 0) { + auto old_storage_class = pos_vt.types()[pos]; + unrefMember(*fixture, old_storage_class, pos_value); + // update attribute stored in the positional value-table + pos_vt.set(pos, storage_class, value); + m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_storage_class), getSchemaTypeId(storage_class)); + } else { + auto offset = field_id.getOffset(); + auto old_type_id = getSchemaTypeId(storage_class, lofi_store<2>::fromValue(pos_value).get(offset)); + lofi_store<2>::fromValue(pos_value).set(offset, value.m_store); + pos_vt.set(pos, storage_class, pos_value); + auto new_type_id = getSchemaTypeId(storage_class, value); + m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); + } + } + + void Object::setIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, + unsigned int fidelity, StorageClass storage_class, Value value) + { + auto &index_vt = this->modify().index_vt(); + if (fidelity == 0) { + auto old_storage_class = index_vt.xvalues()[index_vt_pos].m_type; + unrefMember(*fixture, index_vt.xvalues()[index_vt_pos]); + index_vt.set(index_vt_pos, storage_class, value); + m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_storage_class), getSchemaTypeId(storage_class)); + } else { + auto index_vt_value = index_vt.xvalues()[index_vt_pos].m_value; + auto offset = field_id.getOffset(); + auto old_type_id = getSchemaTypeId(storage_class, lofi_store<2>::fromValue(index_vt_value).get(offset)); + lofi_store<2>::fromValue(index_vt_value).set(offset, value.m_store); + index_vt.set(index_vt_pos, storage_class, index_vt_value); + auto new_type_id = getSchemaTypeId(storage_class, value); + m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); + } + } + + void Object::setWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, + unsigned int fidelity, StorageClass storage_class, Value value) + { + if (loc_ptr == &(*this)->pos_vt()) { + setPosVT(fixture, field_id, pos, fidelity, storage_class, value); + return; + } + + if (loc_ptr == &(*this)->index_vt()) { + setIndexVT(fixture, field_id, pos, fidelity, storage_class, value); + return; + } + + // must be in the kv-index + assert(!loc_ptr); + setKVIndexValue(fixture, field_id, fidelity, storage_class, value); + } + + void Object::addToPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, + StorageClass storage_class, Value value) + { + auto &pos_vt = this->modify().pos_vt(); + auto pos_value = pos_vt.values()[pos]; + if (fidelity == 0) { + // update attribute stored in the positional value-table + pos_vt.set(pos, storage_class, value); + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class)); + } else { + unsigned int offset = field_id.getOffset(); + lofi_store<2>::fromValue(pos_value).set(offset, value.m_store); + pos_vt.set(pos, storage_class, pos_value); + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + } + } + + void Object::addToIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, + unsigned int fidelity, StorageClass storage_class, Value value) + { + auto &index_vt = this->modify().index_vt(); + if (fidelity == 0) { + index_vt.set(index_vt_pos, storage_class, value); + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class)); + } else { + auto index_vt_value = index_vt.xvalues()[index_vt_pos].m_value; + lofi_store<2>::fromValue(index_vt_value).set(field_id.getOffset(), value.m_store); + index_vt.set(index_vt_pos, storage_class, index_vt_value); + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + } + } + + void Object::addWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, + unsigned int fidelity, StorageClass storage_class, Value value) + { + if (loc_ptr == &(*this)->pos_vt()) { + addToPosVT(fixture, field_id, pos, fidelity, storage_class, value); + return; + } + + if (loc_ptr == &(*this)->index_vt()) { + addToIndexVT(fixture, field_id, pos, fidelity, storage_class, value); + return; + } + + assert(!loc_ptr); + addToKVIndex(fixture, field_id, fidelity, storage_class, value); + } + + bool Object::forAllImpl(std::function f) const + { + if (super_t::forAllImpl(f)) { + // Finally, visit kv-index members + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto &obj_type = this->getType(); + auto it = kv_index_ptr->beginJoin(1); + for (;!it.is_end(); ++it) { + if ((*it).m_type == StorageClass::DELETED || (*it).m_type == StorageClass::UNDEFINED) { + // skip deleted or undefined members + continue; + } + if ((*it).m_type == StorageClass::PACK_2) { + // iterate individual lo-fi members + if (!forAll(*it, f)) { + return false; + } + } else { + if (!f(obj_type.getMember(FieldID::fromIndex((*it).getIndex())).m_name, *it, 0)) { + return false; + } + } + } + } + } + return true; + } + } \ No newline at end of file diff --git a/src/dbzero/object_model/object/Object.hpp b/src/dbzero/object_model/object/Object.hpp index 5d4e5083..92040e2a 100644 --- a/src/dbzero/object_model/object/Object.hpp +++ b/src/dbzero/object_model/object/Object.hpp @@ -6,7 +6,7 @@ namespace db0::object_model { - + class Object: public ObjectImplBase { // GC0 specific declarations @@ -25,12 +25,38 @@ namespace db0::object_model // NOTE: if lang_value is nullptr then the member is removed void set(FixtureLock &, const char *field_name, ObjectPtr lang_value); void remove(FixtureLock &, const char *field_name); - + protected: + friend super_t; + + bool tryFindMemberAt(std::pair field_info, + std::pair &result, std::pair &find_result) const; + + void getFieldLayoutImpl(FieldLayout &layout) const; + void getMembersImpl(std::unordered_set &) const; + + // Set or update member in a pos_vt + void setPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); + void setIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, + StorageClass, Value); + + // Set with a specific location (pos_vt, index_vt, kv-index) + void setWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, + StorageClass, Value); + + // Add a new value + void addToPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); + void addToIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, StorageClass, Value); + + void addWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, + StorageClass, Value); + void dropMembers(db0::swine_ptr &, Class &) const; bool tryUnrefWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, StorageClass, - unsigned int fidelity); + unsigned int fidelity); + bool tryFindMemberSlot(const std::pair &field_info, unsigned int &pos, + std::pair &result) const; /** * If the KV_Index does not exist yet, create it and add the first value @@ -46,6 +72,8 @@ namespace db0::object_model void unrefKVIndexValue(FixtureLock &, FieldID, StorageClass, unsigned int fidelity); // Set or update member in kv-index void setKVIndexValue(FixtureLock &, FieldID, unsigned int fidelity, StorageClass, Value); + + bool forAllImpl(std::function) const; }; } diff --git a/src/dbzero/object_model/object/ObjectImplBase.cpp b/src/dbzero/object_model/object/ObjectImplBase.cpp index abbc5838..65e5d4f8 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.cpp +++ b/src/dbzero/object_model/object/ObjectImplBase.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include #include @@ -353,7 +355,8 @@ namespace db0::object_model void ObjectImplBase::unrefWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, StorageClass storage_class, unsigned int fidelity) { - ImplT::unrefWithLoc(fixture, field_id, loc_ptr, pos, storage_class, fidelity); + // call the actual implementation + static_cast(this)->tryUnrefWithLoc(fixture, field_id, loc_ptr, pos, storage_class, fidelity); } template @@ -367,123 +370,50 @@ namespace db0::object_model unrefIndexVT(fixture, field_id, pos, storage_class, fidelity); return true; } - return false; + return false; } template - void ObjectImplBase::setPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, - StorageClass storage_class, Value value) - { - auto &pos_vt = this->modify().pos_vt(); - auto pos_value = pos_vt.values()[pos]; - if (fidelity == 0) { - auto old_storage_class = pos_vt.types()[pos]; - unrefMember(*fixture, old_storage_class, pos_value); - // update attribute stored in the positional value-table - pos_vt.set(pos, storage_class, value); - m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_storage_class), getSchemaTypeId(storage_class)); - } else { - auto offset = field_id.getOffset(); - auto old_type_id = getSchemaTypeId(storage_class, lofi_store<2>::fromValue(pos_value).get(offset)); - lofi_store<2>::fromValue(pos_value).set(offset, value.m_store); - pos_vt.set(pos, storage_class, pos_value); - auto new_type_id = getSchemaTypeId(storage_class, value); - m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); - } - } - - template - void ObjectImplBase::addToPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, - StorageClass storage_class, Value value) - { - auto &pos_vt = this->modify().pos_vt(); - auto pos_value = pos_vt.values()[pos]; - if (fidelity == 0) { - // update attribute stored in the positional value-table - pos_vt.set(pos, storage_class, value); - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class)); - } else { - unsigned int offset = field_id.getOffset(); - lofi_store<2>::fromValue(pos_value).set(offset, value.m_store); - pos_vt.set(pos, storage_class, pos_value); - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); - } - } - - template - void ObjectImplBase::setIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, - unsigned int fidelity, StorageClass storage_class, Value value) + bool ObjectImplBase::tryFindMemberSlot(const std::pair &field_info, unsigned int &pos, + std::pair &result) const { - auto &index_vt = this->modify().index_vt(); - if (fidelity == 0) { - auto old_storage_class = index_vt.xvalues()[index_vt_pos].m_type; - unrefMember(*fixture, index_vt.xvalues()[index_vt_pos]); - index_vt.set(index_vt_pos, storage_class, value); - m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_storage_class), getSchemaTypeId(storage_class)); - } else { - auto index_vt_value = index_vt.xvalues()[index_vt_pos].m_value; - auto offset = field_id.getOffset(); - auto old_type_id = getSchemaTypeId(storage_class, lofi_store<2>::fromValue(index_vt_value).get(offset)); - lofi_store<2>::fromValue(index_vt_value).set(offset, value.m_store); - index_vt.set(index_vt_pos, storage_class, index_vt_value); - auto new_type_id = getSchemaTypeId(storage_class, value); - m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); + auto [index, offset] = field_info.first.getIndexAndOffset(); + // pos-vt lookup + if ((*this)->pos_vt().find(index, pos)) { + if (field_info.second == 0 || slotExists((*this)->pos_vt().values()[pos], field_info.second, offset)) { + result = { field_info, &(*this)->pos_vt() }; + return true; + } else { + return false; + } } - } - - template - void ObjectImplBase::addToIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, - unsigned int fidelity, StorageClass storage_class, Value value) - { - auto &index_vt = this->modify().index_vt(); - if (fidelity == 0) { - index_vt.set(index_vt_pos, storage_class, value); - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class)); - } else { - auto index_vt_value = index_vt.xvalues()[index_vt_pos].m_value; - lofi_store<2>::fromValue(index_vt_value).set(field_id.getOffset(), value.m_store); - index_vt.set(index_vt_pos, storage_class, index_vt_value); - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + + // index-vt lookup + if ((*this)->index_vt().find(index, pos)) { + if (field_info.second == 0 || slotExists((*this)->index_vt().xvalues()[pos].m_value, field_info.second, offset)) { + result = { field_info, &(*this)->index_vt() }; + return true; + } else { + return false; + } } + // not found but the lookup may be continued in the kv-index + return true; } template std::pair ObjectImplBase::tryGetMemberSlot(const MemberID &member_id, unsigned int &pos) const { + std::pair result = { {}, nullptr }; for (auto &field_info: member_id) { - auto [index, offset] = field_info.first.getIndexAndOffset(); - // pos-vt lookup - if ((*this)->pos_vt().find(index, pos)) { - if (field_info.second == 0 || slotExists((*this)->pos_vt().values()[pos], field_info.second, offset)) { - return { field_info, &(*this)->pos_vt() }; - } else { - continue; - } - } - - // index-vt lookup - if ((*this)->index_vt().find(index, pos)) { - if (field_info.second == 0 || slotExists((*this)->index_vt().xvalues()[pos].m_value, field_info.second, offset)) { - return { field_info, &(*this)->index_vt() }; - } else { - continue; - } - } - - // kv-index lookup - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - XValue value(index); - if (kv_index_ptr->findOne(value)) { - if (field_info.second == 0 || slotExists(value.m_value, field_info.second, offset)) { - return { field_info, nullptr }; - } - } + // call the actual implementation + if (static_cast(this)->tryFindMemberSlot(field_info, pos, result)) { + break; } } // not found or deleted - return { {}, nullptr }; + return result; } template @@ -503,43 +433,6 @@ namespace db0::object_model return { nullptr, 0 }; } - template - void ObjectImplBase::setWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, - unsigned int fidelity, StorageClass storage_class, Value value) - { - if (loc_ptr == &(*this)->pos_vt()) { - setPosVT(fixture, field_id, pos, fidelity, storage_class, value); - return; - } - - if (loc_ptr == &(*this)->index_vt()) { - setIndexVT(fixture, field_id, pos, fidelity, storage_class, value); - return; - } - - // must be in the kv-index - assert(!loc_ptr); - setKVIndexValue(fixture, field_id, fidelity, storage_class, value); - } - - template - void ObjectImplBase::addWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, - unsigned int fidelity, StorageClass storage_class, Value value) - { - if (loc_ptr == &(*this)->pos_vt()) { - addToPosVT(fixture, field_id, pos, fidelity, storage_class, value); - return; - } - - if (loc_ptr == &(*this)->index_vt()) { - addToIndexVT(fixture, field_id, pos, fidelity, storage_class, value); - return; - } - - assert(!loc_ptr); - addToKVIndex(fixture, field_id, fidelity, storage_class, value); - } - template std::pair ObjectImplBase::findField(const char *name) const { @@ -708,13 +601,14 @@ namespace db0::object_model return { false, false }; } } - + template - std::pair ObjectImplBase::tryGetMemberAt(std::pair field_info, - std::pair &result) const + bool ObjectImplBase::tryFindMemberAt(std::pair field_info, + std::pair &result, std::pair &find_result) const { if (!field_info.first) { - return { false, false }; + find_result = { false, false }; + return true; } auto loc = field_info.first.getIndexAndOffset(); @@ -722,9 +616,11 @@ namespace db0::object_model // try retrieving from initializer auto initializer_ptr = m_init_manager.findInitializer(*this); if (!initializer_ptr) { - return { false, false }; + find_result = { false, false }; + return true; } - return { initializer_ptr->tryGetAt(loc, result), false }; + find_result = { initializer_ptr->tryGetAt(loc, result), false }; + return true; } // retrieve from positionally encoded values @@ -732,57 +628,50 @@ namespace db0::object_model // NOTE: removed field slots might be marked as DELETED if (result.first == StorageClass::DELETED) { // report as deleted - return { false, true }; + find_result = { false, true }; + return true; } if (field_info.second == 0) { - return { result.first != StorageClass::UNDEFINED, false }; + find_result = { result.first != StorageClass::UNDEFINED, false }; } else { - return hasValueAt(result.second, field_info.second, loc.second); + find_result = hasValueAt(result.second, field_info.second, loc.second); } + return true; } if ((*this)->index_vt().find(loc.first, result)) { if (result.first == StorageClass::DELETED) { // report as deleted - return { false, true }; + find_result = { false, true }; + return true; } if (field_info.second == 0) { - return { result.first != StorageClass::UNDEFINED, false }; + find_result = { result.first != StorageClass::UNDEFINED, false }; } else { - return hasValueAt(result.second, field_info.second, loc.second); + find_result = hasValueAt(result.second, field_info.second, loc.second); } + return true; } - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - XValue xvalue(loc.first); - if (kv_index_ptr->findOne(xvalue)) { - assert(xvalue.getIndex() == loc.first); - if (xvalue.m_type == StorageClass::DELETED) { - // report as deleted - return { false, true }; - } - - // member fetched from the kv_index - result.first = xvalue.m_type; - result.second = xvalue.m_value; + return false; + } - if (field_info.second == 0) { - return { result.first != StorageClass::UNDEFINED, false }; - } else { - return hasValueAt(result.second, field_info.second, loc.second); - } - } + template + std::pair ObjectImplBase::tryGetMemberAt(std::pair field_info, + std::pair &result) const + { + std::pair find_result; + if (static_cast(this)->tryFindMemberAt(field_info, result, find_result)) { + return find_result; } - // Does not exist, not explicitly removed - return { false, false }; + return { false, false }; } - + template - db0::swine_ptr ObjectImplBase::tryGetFixture() const + db0::swine_ptr ObjectImplBase::tryGetFixture() const { if (!this->hasInstance()) { if (isDropped()) { @@ -859,7 +748,8 @@ namespace db0::object_model { auto fixture = this->getFixture(); assert(fixture); - ImplT::dropMembers(*fixture, class_ref); + // call the actual implementation + static_cast(this)->dropMembers(fixture, class_ref); } template @@ -924,11 +814,19 @@ namespace db0::object_model } super_t::destroy(); } - + template FieldLayout ObjectImplBase::getFieldLayout() const { FieldLayout layout; + // call the actual implementation + static_cast(this)->getFieldLayoutImpl(layout); + return layout; + } + + template + void ObjectImplBase::getFieldLayoutImpl(FieldLayout &layout) const + { // collect pos-vt information for (auto type: (*this)->pos_vt().types()) { layout.m_pos_vt_fields.push_back(type); @@ -938,17 +836,6 @@ namespace db0::object_model for (auto &xvalue: (*this)->index_vt().xvalues()) { layout.m_index_vt_fields.emplace_back(xvalue.getIndex(), xvalue.m_type); } - - // collect kv-index information - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto it = kv_index_ptr->beginJoin(1); - for (;!it.is_end(); ++it) { - layout.m_kv_index_fields.emplace_back((*it).getIndex(), (*it).m_type); - } - } - - return layout; } template @@ -974,11 +861,19 @@ namespace db0::object_model result.insert(this_type.getMember(FieldID::fromIndex(index)).m_name); } } - + template std::unordered_set ObjectImplBase::getMembers() const { std::unordered_set result; + // call the actual implementation + static_cast(this)->getMembersImpl(result); + return result; + } + + template + void ObjectImplBase::getMembersImpl(std::unordered_set &result) const + { // Visit pos-vt members first auto &obj_type = this->getType(); { @@ -998,22 +893,18 @@ namespace db0::object_model auto index = xvalue.getIndex(); getMembersFrom(obj_type, index, xvalue.m_type, xvalue.m_value, result); } - } - - // Finally, visit kv-index members - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto it = kv_index_ptr->beginJoin(1); - for (;!it.is_end(); ++it) { - auto index = (*it).getIndex(); - getMembersFrom(obj_type, index, (*it).m_type, (*it).m_value, result); - } - } - return result; + } } - + template void ObjectImplBase::forAll(std::function f) const + { + // call the actual implementation + static_cast(this)->forAllImpl(f); + } + + template + bool ObjectImplBase::forAllImpl(std::function f) const { // Visit pos-vt members first auto &obj_type = this->getType(); @@ -1030,11 +921,11 @@ namespace db0::object_model if (*type == StorageClass::PACK_2) { // iterate individual lo-fi members if (!forAll({index, *type, *value}, f)) { - return; + return false; } } else { if (!f(obj_type.getMember(FieldID::fromIndex(index)).m_name, { index, *type, *value }, 0)) { - return; + return false; } } } @@ -1051,38 +942,19 @@ namespace db0::object_model if (xvalue.m_type == StorageClass::PACK_2) { // iterate individual lo-fi members if (!forAll(xvalue, f)) { - return; + return false; } } else { // regular member if (!f(obj_type.getMember(FieldID::fromIndex(xvalue.getIndex())).m_name, xvalue, 0)) { - return; + return false; } } } } - // Finally, visit kv-index members - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto it = kv_index_ptr->beginJoin(1); - for (;!it.is_end(); ++it) { - if ((*it).m_type == StorageClass::DELETED || (*it).m_type == StorageClass::UNDEFINED) { - // skip deleted or undefined members - continue; - } - if ((*it).m_type == StorageClass::PACK_2) { - // iterate individual lo-fi members - if (!forAll(*it, f)) { - return; - } - } else { - if (!f(obj_type.getMember(FieldID::fromIndex((*it).getIndex())).m_name, *it, 0)) { - return; - } - } - } - } + // Continue with kv-index members if any + return true; } template @@ -1154,6 +1026,7 @@ namespace db0::object_model return this->hasInstance() && (*this)->m_header.m_ref_counter.getFirst() > 0; } + /* FIXME: implement template bool ObjectImplBase::equalTo(const ObjectImplBase &other) const { @@ -1205,6 +1078,7 @@ namespace db0::object_model }); return result; } + */ template void ObjectImplBase::moveTo(db0::swine_ptr &) { @@ -1345,5 +1219,5 @@ namespace db0::object_model assert(m_ext_refs > 0); --m_ext_refs; } - + } \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectImplBase.hpp b/src/dbzero/object_model/object/ObjectImplBase.hpp index 562cf502..e561a015 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.hpp +++ b/src/dbzero/object_model/object/ObjectImplBase.hpp @@ -132,17 +132,15 @@ namespace db0::object_model db0::swine_ptr getFixture() const; Memspace &getMemspace() const; - - /** - * Get description of the field layout - */ + + // Get description of the field layout FieldLayout getFieldLayout() const; // Convert singleton into a regular instance void unSingleton(FixtureLock &); void destroy() const; - + bool isSingleton() const; // execute the function for all members (until false is returned from the input lambda) @@ -251,15 +249,23 @@ namespace db0::object_model // similar to hasValueAt but assume deleted slot as present bool slotExists(Value value, unsigned int fidelity, unsigned int at) const; + void getFieldLayoutImpl(FieldLayout &) const; + void getMembersImpl(std::unordered_set &) const; + // Try retrieving member either from values (initialized) or from the initialization buffer (not initialized yet) // @return member exists, member deleted flags + bool tryFindMemberAt(std::pair, std::pair &, + std::pair &find_result) const; std::pair tryGetMemberAt(std::pair, std::pair &) const; FieldID tryGetMember(const char *field_name, std::pair &, bool &is_init_var) const; - + // Try resolving field ID of an existing (or deleted) member and also its storage location // @param pos the member's position in the containing collection // @return FieldID + containing collection (e.g. pos_vt()) + bool tryFindMemberSlot(const std::pair &field_info, unsigned int &pos, + std::pair &result) const; + std::pair tryGetMemberSlot(const MemberID &, unsigned int &pos) const; // Try locating a field ID associated slot @@ -286,20 +292,11 @@ namespace db0::object_model static std::shared_ptr getTypeWithHint(const Fixture &, std::uint32_t class_ref, std::shared_ptr type_hint); bool hasValidClassRef() const; - + // try retrieving member as XValue std::optional tryGetX(const char *field_name) const; void _touch(); - // Set or update member in a pos_vt - void setPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); - void setIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, - StorageClass, Value); - - // Set with a specific location (pos_vt, index_vt, kv-index) - void setWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, - StorageClass, Value); - // Unreference value // NOTE: storage_class to be assigned can either be DELETED or UNDEFINED void unrefPosVT(FixtureLock &, FieldID, unsigned int pos, StorageClass, unsigned int fidelity); @@ -310,13 +307,7 @@ namespace db0::object_model bool tryUnrefWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, StorageClass, unsigned int fidelity); - // Add a new value - void addToPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); - void addToIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, StorageClass, Value); - - void addWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, - StorageClass, Value); - + bool forAllImpl(std::function) const; // lo-fi member specialized implementation bool forAll(XValue, std::function) const; From 1773b6e76c78486b8b70e4fb9d525dac65219c15 Mon Sep 17 00:00:00 2001 From: Wojtek Date: Tue, 4 Nov 2025 20:12:10 +0100 Subject: [PATCH 09/19] WIP: immutable implementation in progress --- src/dbzero/object_model/object/Object.cpp | 103 +++++++++++------- src/dbzero/object_model/object/Object.hpp | 1 + .../object_model/object/ObjectImplBase.cpp | 61 ++++++----- .../object_model/object/ObjectImplBase.hpp | 6 +- 4 files changed, 105 insertions(+), 66 deletions(-) diff --git a/src/dbzero/object_model/object/Object.cpp b/src/dbzero/object_model/object/Object.cpp index 4d31d430..72797e61 100644 --- a/src/dbzero/object_model/object/Object.cpp +++ b/src/dbzero/object_model/object/Object.cpp @@ -50,37 +50,60 @@ namespace db0::object_model } } + bool Object::tryEqualToImpl(const ObjectImplBase &other, bool &result) const + { + if (super_t::tryEqualToImpl(other, result)) { + return true; + } + + if (this->getFixture()->getUUID() == other.getFixture()->getUUID() + && this->getUniqueAddress() == other.getUniqueAddress()) + { + auto &other_obj = static_cast(other); + if (!hasKV_Index() && !other_obj.hasKV_Index()) { + result = true; + return true; + } + result = isEqual(this->tryGetKV_Index(), other_obj.tryGetKV_Index()); + return true; + } + + return false; + } + bool Object::tryFindMemberAt(std::pair field_info, std::pair &result, std::pair &find_result) const { - if (!super_t::tryFindMemberAt(field_info, result, find_result)) { - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto loc = field_info.first.getIndexAndOffset(); - XValue xvalue(loc.first); - if (kv_index_ptr->findOne(xvalue)) { - assert(xvalue.getIndex() == loc.first); - if (xvalue.m_type == StorageClass::DELETED) { - // report as deleted - find_result = { false, true }; - return true; - } + if (super_t::tryFindMemberAt(field_info, result, find_result)) { + return true; + } - // member fetched from the kv_index - result.first = xvalue.m_type; - result.second = xvalue.m_value; + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto loc = field_info.first.getIndexAndOffset(); + XValue xvalue(loc.first); + if (kv_index_ptr->findOne(xvalue)) { + assert(xvalue.getIndex() == loc.first); + if (xvalue.m_type == StorageClass::DELETED) { + // report as deleted + find_result = { false, true }; + return true; + } - if (field_info.second == 0) { - find_result = { result.first != StorageClass::UNDEFINED, false }; - return true; - } else { - find_result = hasValueAt(result.second, field_info.second, loc.second); - return true; - } + // member fetched from the kv_index + result.first = xvalue.m_type; + result.second = xvalue.m_value; + + if (field_info.second == 0) { + find_result = { result.first != StorageClass::UNDEFINED, false }; + return true; + } else { + find_result = hasValueAt(result.second, field_info.second, loc.second); + return true; } } } - + return false; } @@ -111,7 +134,7 @@ namespace db0::object_model unsigned int old_pos = 0; const void *old_loc_ptr = nullptr; if (member_id) { - std::tie(old_field_info, old_loc_ptr) = super_t::tryGetMemberSlot(member_id, old_pos); + std::tie(old_field_info, old_loc_ptr) = tryGetMemberSlot(member_id, old_pos); } if (!member_id || !(field_id = member_id.tryGet(storage_fidelity))) { @@ -128,7 +151,7 @@ namespace db0::object_model ); // make sure object address is not null assert(!(storage_class == StorageClass::OBJECT_REF && value.cast() == 0)); - + if (field_id == old_field_info.first) { // Set / update value at the existing location setWithLoc(fixture, field_id, old_loc_ptr, old_pos, storage_fidelity, storage_class, value); @@ -341,9 +364,10 @@ namespace db0::object_model bool Object::tryUnrefWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, StorageClass storage_class, unsigned int fidelity) { - if (!super_t::tryUnrefWithLoc(fixture, field_id, loc_ptr, pos, storage_class, fidelity)) { - unrefKVIndexValue(fixture, field_id, storage_class, fidelity); + if (super_t::tryUnrefWithLoc(fixture, field_id, loc_ptr, pos, storage_class, fidelity)) { + return true; } + unrefKVIndexValue(fixture, field_id, storage_class, fidelity); return true; } @@ -399,21 +423,24 @@ namespace db0::object_model bool Object::tryFindMemberSlot(const std::pair &field_info, unsigned int &pos, std::pair &result) const { - if (!super_t::tryFindMemberSlot(field_info, pos, result)) { - // kv-index lookup - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto [index, offset] = field_info.first.getIndexAndOffset(); - XValue value(index); - if (kv_index_ptr->findOne(value)) { - if (field_info.second == 0 || slotExists(value.m_value, field_info.second, offset)) { - result = { field_info, nullptr }; - return true; - } + if (super_t::tryFindMemberSlot(field_info, pos, result)) { + return true; + } + + // kv-index lookup + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto [index, offset] = field_info.first.getIndexAndOffset(); + XValue value(index); + if (kv_index_ptr->findOne(value)) { + if (field_info.second == 0 || slotExists(value.m_value, field_info.second, offset)) { + result = { field_info, nullptr }; } + return true; } } + // not found or deleted return false; } diff --git a/src/dbzero/object_model/object/Object.hpp b/src/dbzero/object_model/object/Object.hpp index 92040e2a..744bfcae 100644 --- a/src/dbzero/object_model/object/Object.hpp +++ b/src/dbzero/object_model/object/Object.hpp @@ -34,6 +34,7 @@ namespace db0::object_model void getFieldLayoutImpl(FieldLayout &layout) const; void getMembersImpl(std::unordered_set &) const; + bool tryEqualToImpl(const ObjectImplBase &, bool &result) const; // Set or update member in a pos_vt void setPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); diff --git a/src/dbzero/object_model/object/ObjectImplBase.cpp b/src/dbzero/object_model/object/ObjectImplBase.cpp index 65e5d4f8..18252bd7 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.cpp +++ b/src/dbzero/object_model/object/ObjectImplBase.cpp @@ -158,6 +158,7 @@ namespace db0::object_model // construct the dbzero instance & assign to self m_type = initializer.getClassPtr(); assert(m_type); + super_t::init(*fixture, m_type->getClassRef(), initializer.getRefCounts(), safeCast(m_type->getNumBases() + 1, "Too many base classes"), pos_vt_data, pos_vt_offset, index_vt_data.first, index_vt_data.second, @@ -176,7 +177,7 @@ namespace db0::object_model initializer.close(); } - assert(hasInstance()); + assert(this->hasInstance()); } template @@ -239,7 +240,7 @@ namespace db0::object_model template void ObjectImplBase::setPreInit(const char *field_name, ObjectPtr obj_ptr) const { - assert(!hasInstance()); + assert(!this->hasInstance()); if (!LangToolkit::isValid(obj_ptr)) { removePreInit(field_name); return; @@ -382,38 +383,38 @@ namespace db0::object_model if ((*this)->pos_vt().find(index, pos)) { if (field_info.second == 0 || slotExists((*this)->pos_vt().values()[pos], field_info.second, offset)) { result = { field_info, &(*this)->pos_vt() }; - return true; - } else { - return false; } + return true; } // index-vt lookup if ((*this)->index_vt().find(index, pos)) { if (field_info.second == 0 || slotExists((*this)->index_vt().xvalues()[pos].m_value, field_info.second, offset)) { result = { field_info, &(*this)->index_vt() }; - return true; - } else { - return false; } + return true; + } // not found but the lookup may be continued in the kv-index - return true; + return false; } template std::pair ObjectImplBase::tryGetMemberSlot(const MemberID &member_id, unsigned int &pos) const { - std::pair result = { {}, nullptr }; + std::pair result; for (auto &field_info: member_id) { // call the actual implementation if (static_cast(this)->tryFindMemberSlot(field_info, pos, result)) { - break; + // otherwise continue since member might've been deleted and reassigned to a different slot + if (result.first.first) { + return result; + } } } // not found or deleted - return result; + return { {}, nullptr }; } template @@ -711,7 +712,7 @@ namespace db0::object_model { assert(!m_type); assert(type_hint); - assert(hasInstance()); + assert(this->hasInstance()); if (type_hint->getClassRef() == (*this)->getClassRef()) { m_type = type_hint; } else { @@ -1012,7 +1013,7 @@ namespace db0::object_model template bool ObjectImplBase::hasRefs() const { - assert(hasInstance()); + assert(this->hasInstance()); return (*this)->hasRefs(); } @@ -1026,9 +1027,8 @@ namespace db0::object_model return this->hasInstance() && (*this)->m_header.m_ref_counter.getFirst() > 0; } - /* FIXME: implement template - bool ObjectImplBase::equalTo(const ObjectImplBase &other) const + bool ObjectImplBase::tryEqualToImpl(const ObjectImplBase &other, bool &result) const { if (!this->hasInstance() || !other.hasInstance()) { THROWF(db0::InputException) << "Object not initialized"; @@ -1042,7 +1042,8 @@ namespace db0::object_model if ((*this)->getClassRef() != other->getClassRef()) { // different types - return false; + result = false; + return true; } if (this->getFixture()->getUUID() == other.getFixture()->getUUID() @@ -1050,19 +1051,28 @@ namespace db0::object_model { // comparing 2 versions of the same object (fastest) if (!((*this)->pos_vt() == other->pos_vt())) { - return false; + result = false; + return true; } if (!((*this)->index_vt() == other->index_vt())) { - return false; - } - if (!hasKV_Index() && !other.hasKV_Index()) { + result = false; return true; } - return isEqual(this->tryGetKV_Index(), other.tryGetKV_Index()); + } + // unable to determine + return false; + } + + template + bool ObjectImplBase::equalTo(const ObjectImplBase &other) const + { + bool result; + if (static_cast(this)->tryEqualToImpl(other, result)) { + return result; } // field-wise compare otherwise (slower) - bool result = true; + result = true; this->forAll([&](const std::string &name, const XValue &xvalue, unsigned int offset) -> bool { auto maybe_other_value = other.tryGetX(name.c_str()); if (!maybe_other_value) { @@ -1077,9 +1087,8 @@ namespace db0::object_model return true; }); return result; - } - */ - + } + template void ObjectImplBase::moveTo(db0::swine_ptr &) { throw std::runtime_error("Not implemented"); diff --git a/src/dbzero/object_model/object/ObjectImplBase.hpp b/src/dbzero/object_model/object/ObjectImplBase.hpp index e561a015..5cab0edf 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.hpp +++ b/src/dbzero/object_model/object/ObjectImplBase.hpp @@ -107,7 +107,7 @@ namespace db0::object_model // Called to finalize adding members void endInit(); - + // Assign field of an uninitialized instance (assumed as a non-mutating operation) // NOTE: if lang_value is nullptr then the member is removed void setPreInit(const char *field_name, ObjectPtr lang_value) const; @@ -251,6 +251,7 @@ namespace db0::object_model void getFieldLayoutImpl(FieldLayout &) const; void getMembersImpl(std::unordered_set &) const; + bool tryEqualToImpl(const ObjectImplBase &, bool &result) const; // Try retrieving member either from values (initialized) or from the initialization buffer (not initialized yet) // @return member exists, member deleted flags @@ -289,7 +290,8 @@ namespace db0::object_model std::shared_ptr unloadType() const; // Retrieve a type by class-ref with a possible match (type_hint) - static std::shared_ptr getTypeWithHint(const Fixture &, std::uint32_t class_ref, std::shared_ptr type_hint); + static std::shared_ptr getTypeWithHint(const Fixture &, std::uint32_t class_ref, + std::shared_ptr type_hint); bool hasValidClassRef() const; From 3496bc4b7de81a9e840ef1a9dc2b452dd811a66f Mon Sep 17 00:00:00 2001 From: Wojtek Date: Wed, 5 Nov 2025 11:26:23 +0100 Subject: [PATCH 10/19] WIP: immutable memo integration --- dbzero/dbzero/dbzero.py | 2 +- python_tests/test_memo_immutable.py | 12 ++-- src/dbzero/bindings/TypeId.hpp | 3 +- src/dbzero/bindings/python/Memo.cpp | 70 +++++++++++++++---- src/dbzero/bindings/python/Memo.hpp | 14 +++- .../bindings/python/MemoTypeDecoration.cpp | 12 +--- .../bindings/python/MemoTypeDecoration.hpp | 4 -- src/dbzero/bindings/python/PyInternalAPI.cpp | 14 ++-- .../bindings/python/PyObjectTagManager.cpp | 10 +-- src/dbzero/bindings/python/PyToolkit.cpp | 24 +++---- src/dbzero/bindings/python/PyToolkit.hpp | 5 +- src/dbzero/bindings/python/PyTypeManager.cpp | 9 ++- src/dbzero/bindings/python/PyTypeManager.hpp | 2 + src/dbzero/bindings/python/PyWeakProxy.cpp | 37 +++++++--- src/dbzero/bindings/python/PyWeakProxy.hpp | 15 +++- .../bindings/python/shared_py_object.cpp | 50 +++++++++---- src/dbzero/bindings/python/types/PyTag.cpp | 21 +++--- .../object/ObjectImmutableImpl.cpp | 2 +- .../object_model/object/ObjectImplBase.hpp | 3 - 19 files changed, 207 insertions(+), 102 deletions(-) diff --git a/dbzero/dbzero/dbzero.py b/dbzero/dbzero/dbzero.py index c9e4f4dc..21899e3d 100644 --- a/dbzero/dbzero/dbzero.py +++ b/dbzero/dbzero/dbzero.py @@ -10,7 +10,7 @@ def load_dynamic(name, path): def __bootstrap__(): global __bootstrap__, __loader__, __file__ - paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/release", "/usr/local/lib/python3/dist-packages/dbzero/"] + paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/debug", "/usr/local/lib/python3/dist-packages/dbzero/"] __file__ = None for path in paths: if os.path.isdir(path): diff --git a/python_tests/test_memo_immutable.py b/python_tests/test_memo_immutable.py index aa128747..419026f3 100644 --- a/python_tests/test_memo_immutable.py +++ b/python_tests/test_memo_immutable.py @@ -6,13 +6,13 @@ import random -@db0.memo(immutable=True) +@db0.memo(immutable=True, no_default_tags=True) @dataclass -class MemoImmutableClass: +class MemoImmutableClass1: data: str value: int = 0 - + def test_create_memo_immutable(db0_fixture): - obj_1 = MemoImmutableClass(data="immutable data", value=42) - assert obj_1.data == "immutable data" - assert obj_1.value == 42 + _ = MemoImmutableClass1(data="immutable data", value=42) + + diff --git a/src/dbzero/bindings/TypeId.hpp b/src/dbzero/bindings/TypeId.hpp index c323c40b..be98fc69 100644 --- a/src/dbzero/bindings/TypeId.hpp +++ b/src/dbzero/bindings/TypeId.hpp @@ -53,8 +53,9 @@ namespace db0::bindings MEMO_EXPIRED_REF = 117, // Python type decorated as memo MEMO_TYPE = 118, + MEMO_IMMUTABLE_OBJECT = 119, // COUNT determines size of the type operator arrays - COUNT = 119, + COUNT = 120, // unrecognized type UNKNOWN = std::numeric_limits::max() }; diff --git a/src/dbzero/bindings/python/Memo.cpp b/src/dbzero/bindings/python/Memo.cpp index d9739e79..344af19b 100644 --- a/src/dbzero/bindings/python/Memo.cpp +++ b/src/dbzero/bindings/python/Memo.cpp @@ -197,7 +197,8 @@ namespace db0::python return PyType_GenericAlloc(self, nitems); } - void MemoObject_del(MemoObject* memo_obj) + template + void MemoObject_del(MemoImplT *memo_obj) { PY_API_FUNC // destroy associated db0 Object instance @@ -205,8 +206,9 @@ namespace db0::python Py_TYPE(memo_obj)->tp_free((PyObject*)memo_obj); } - int PyAPI_MemoObject_init(MemoObject* self, PyObject* args, PyObject* kwds) - { + template + int PyAPI_MemoObject_init(MemoImplT *self, PyObject* args, PyObject* kwds) + { using Class = db0::object_model::Class; using TagIndex = db0::object_model::TagIndex; @@ -329,7 +331,12 @@ namespace db0::python return runSafe(tryMemoObject_getattro, self, attr); } - int PyAPI_MemoObject_setattro(MemoObject *self, PyObject *attr, PyObject *value) + template + int PyAPI_MemoObject_setattro(MemoImplT *self, PyObject *attr, PyObject *value); + + // regular memo object specialization + template <> + int PyAPI_MemoObject_setattro(MemoObject *self, PyObject *attr, PyObject *value) { PY_API_FUNC // assign value to a dbzero attribute @@ -357,7 +364,38 @@ namespace db0::python return 0; } - + + // immutable memo object specialization + template <> + int PyAPI_MemoObject_setattro(MemoImmutableObject *self, PyObject *attr, PyObject *value) + { + PY_API_FUNC + // assign value to a dbzero attribute + try { + // must materialize the object before setting as an attribute + if (value && !db0::object_model::isMaterialized(value)) { + db0::FixtureLock lock(self->ext().getFixture()); + db0::object_model::materialize(lock, value); + } + + if (self->ext().hasInstance()) { + PyErr_SetString(PyExc_AttributeError, "Cannot modify an immutable memo object"); + return -1; + } else { + // considered as a non-mutating operation + self->ext().setPreInit(PyUnicode_AsUTF8(attr), value); + } + } catch (const std::exception &e) { + PyErr_SetString(PyExc_AttributeError, e.what()); + return -1; + } catch (...) { + PyErr_SetString(PyExc_AttributeError, "Unknown exception"); + return -1; + } + + return 0; + } + bool isSame(MemoObject *lhs, MemoObject *rhs) { return lhs->ext() == rhs->ext(); } @@ -432,22 +470,23 @@ namespace db0::python // Regular memo slots static PyType_Slot MemoObject_common_slots[] = { {Py_tp_new, (void *)PyAPI_MemoObject_new}, - {Py_tp_dealloc, (void *)MemoObject_del}, - {Py_tp_init, (void *)PyAPI_MemoObject_init}, + {Py_tp_dealloc, (void *)MemoObject_del}, + {Py_tp_init, (void *)PyAPI_MemoObject_init}, {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro}, - {Py_tp_setattro, (void *)PyAPI_MemoObject_setattro}, + {Py_tp_setattro, (void *)PyAPI_MemoObject_setattro}, {Py_tp_richcompare, (void *)PyAPI_MemoObject_rq}, {Py_tp_hash, (void *)PyAPI_MemoHash}, {0, 0} }; - + // Immutable memo slots static PyType_Slot MemoImmutableObject_common_slots[] = { {Py_tp_new, (void *)PyAPI_MemoObject_new}, - {Py_tp_dealloc, (void *)MemoObject_del}, - {Py_tp_init, (void *)PyAPI_MemoObject_init}, + {Py_tp_dealloc, (void *)MemoObject_del}, + {Py_tp_init, (void *)PyAPI_MemoObject_init}, {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro}, - {Py_tp_setattro, (void *)PyAPI_MemoObject_setattro}, + // set available only on pre-initialized objects + {Py_tp_setattro, (void *)PyAPI_MemoObject_setattro}, {Py_tp_richcompare, (void *)PyAPI_MemoObject_rq}, {Py_tp_hash, (void *)PyAPI_MemoHash}, {0, 0} @@ -941,4 +980,11 @@ namespace db0::python return runSafe(tryGetSchema, reinterpret_cast(args[0])); } + bool PyMemoType_Check(PyTypeObject *type) + { + assert(type); + return type->tp_dealloc == reinterpret_cast(MemoObject_del) || + type->tp_dealloc == reinterpret_cast(MemoObject_del); + } + } diff --git a/src/dbzero/bindings/python/Memo.hpp b/src/dbzero/bindings/python/Memo.hpp index 876681e8..b1880925 100644 --- a/src/dbzero/bindings/python/Memo.hpp +++ b/src/dbzero/bindings/python/Memo.hpp @@ -55,6 +55,13 @@ namespace db0::python PyObject *tryLoadMemo(MemoObject *memo_obj, PyObject* kwargs, PyObject* exclude, std::unordered_set *load_stack_ptr = nullptr); + // check for a memo type (i.e. generated by PyAPI_wrapPyClass) + template bool PyMemo_Check(PyObject *); + template bool PyMemoType_Check(PyObject *); + + bool PyAnyMemo_Check(PyObject *); + bool PyAnyMemoType_Check(PyTypeObject *); + PyObject *PyAPI_PyMemo_Check(PyObject *self, PyObject *const * args, Py_ssize_t nargs); // Binary (shallow) compare 2 objects or 2 versions of the same memo object (e.g. from different snapshots) @@ -66,4 +73,9 @@ namespace db0::python PyObject* executeLoadFunction(PyObject *load_method, PyObject *kwargs, PyObject *py_exclude, std::unordered_set *load_stack_ptr); -} \ No newline at end of file + extern template bool PyMemo_Check(PyObject *); + extern template bool PyMemo_Check(PyObject *); + extern template bool PyMemoType_Check(PyObject *); + extern template bool PyMemoType_Check(PyObject *); + +} diff --git a/src/dbzero/bindings/python/MemoTypeDecoration.cpp b/src/dbzero/bindings/python/MemoTypeDecoration.cpp index eccbb428..99a6bc95 100644 --- a/src/dbzero/bindings/python/MemoTypeDecoration.cpp +++ b/src/dbzero/bindings/python/MemoTypeDecoration.cpp @@ -8,16 +8,10 @@ namespace db0::python { - bool PyMemo_Check(PyObject *obj) + bool PyAnyMemo_Check(PyObject *obj) { auto py_type = Py_TYPE(obj); - return py_type && PyMemoType_Check(py_type); - } - - bool PyMemoType_Check(PyTypeObject *type) - { - assert(type); - return type->tp_dealloc == reinterpret_cast(MemoObject_del); + return py_type && PyAnyMemoType_Check(py_type); } MemoTypeDecoration::MemoTypeDecoration(MemoTypeDecoration &&other) @@ -145,7 +139,7 @@ namespace db0::python MemoTypeDecoration &MemoTypeDecoration::get(PyTypeObject *type) { - assert(PyMemoType_Check(type) && "Invalid type (expected memo type)"); + assert(PyAnyMemoType_Check(type) && "Invalid type (expected memo type)"); return PyToolkit::getTypeManager().getMemoTypeDecoration(type); } diff --git a/src/dbzero/bindings/python/MemoTypeDecoration.hpp b/src/dbzero/bindings/python/MemoTypeDecoration.hpp index 7a5e6b30..b48767f6 100644 --- a/src/dbzero/bindings/python/MemoTypeDecoration.hpp +++ b/src/dbzero/bindings/python/MemoTypeDecoration.hpp @@ -21,10 +21,6 @@ namespace db0::python using MemoOptions = db0::object_model::MemoOptions; using MemoFlags = db0::object_model::MemoFlags; - // check for a memo type (i.e. generated by PyAPI_wrapPyClass) - bool PyMemo_Check(PyObject *obj); - bool PyMemoType_Check(PyTypeObject *type); - using AccessType = db0::AccessType; class MemoTypeDecoration diff --git a/src/dbzero/bindings/python/PyInternalAPI.cpp b/src/dbzero/bindings/python/PyInternalAPI.cpp index cde42726..f19a29cf 100644 --- a/src/dbzero/bindings/python/PyInternalAPI.cpp +++ b/src/dbzero/bindings/python/PyInternalAPI.cpp @@ -238,7 +238,8 @@ namespace db0::python PyObject *fetchSingletonObject(db0::Snapshot &snapshot, PyTypeObject *py_type, const char *prefix_name) { - if (!PyMemoType_Check(py_type)) { + // NOTE: singletons only supported for MemoObject types + if (!PyMemoType_Check(py_type)) { THROWF(db0::InternalException) << "Memo type expected for: " << py_type->tp_name << THROWF_END; } @@ -257,7 +258,8 @@ namespace db0::python bool isExistingSingleton(db0::Snapshot &snapshot, PyTypeObject *py_type, const char *prefix_name) { - if (!PyMemoType_Check(py_type)) { + // NOTE: singletons only supported for MemoObject types + if (!PyMemoType_Check(py_type)) { return false; } @@ -276,14 +278,13 @@ namespace db0::python } return isExistingSingleton(fixture, py_type); } - - + void renameMemoClassField(PyTypeObject *py_type, const char *from_name, const char *to_name) { using ClassFactory = db0::object_model::ClassFactory; auto fixture_uuid = MemoTypeDecoration::get(py_type).getFixtureUUID(); - assert(PyMemoType_Check(py_type)); + assert(PyAnyMemoType_Check(py_type)); assert(from_name); assert(to_name); @@ -293,9 +294,8 @@ namespace db0::python auto type = class_factory.getExistingType(py_type); type->renameField(from_name, to_name); } - -#ifndef NDEBUG +#ifndef NDEBUG PyObject *writeBytes(PyObject *self, PyObject *args) { diff --git a/src/dbzero/bindings/python/PyObjectTagManager.cpp b/src/dbzero/bindings/python/PyObjectTagManager.cpp index 23714c9e..769aec35 100644 --- a/src/dbzero/bindings/python/PyObjectTagManager.cpp +++ b/src/dbzero/bindings/python/PyObjectTagManager.cpp @@ -97,12 +97,12 @@ namespace db0::python }; PyObjectTagManager *tryMakeObjectTagManager(PyObject *, PyObject *const *args, Py_ssize_t nargs) - { - // all arguments must be Memo objects + { + // all arguments must be Memo objects for (Py_ssize_t i = 0; i < nargs; ++i) { - if (!PyMemo_Check(args[i])) { - THROWF(db0::InputException) << "All arguments must be memo objects"; - } + if (!PyAnyMemo_Check(args[i])) { + THROWF(db0::InputException) << "All arguments must be dbzero memo objects"; + } } auto tags_obj = Py_OWN(PyObjectTagManager_new(&PyObjectTagManagerType, NULL, NULL)); diff --git a/src/dbzero/bindings/python/PyToolkit.cpp b/src/dbzero/bindings/python/PyToolkit.cpp index 9a5d6722..2a708418 100644 --- a/src/dbzero/bindings/python/PyToolkit.cpp +++ b/src/dbzero/bindings/python/PyToolkit.cpp @@ -69,7 +69,7 @@ namespace db0::python if (result == "__main__") { // for Memo types we can determine the actual module name from the file name // (if stored with the type decoration) - if (PyMemoType_Check(py_type)) { + if (PyAnyMemoType_Check(py_type)) { // file name may not be available in the type decoration auto file_name = MemoTypeDecoration::get(py_type).tryGetFileName(); if (file_name) { @@ -472,8 +472,8 @@ namespace db0::python return PyType_Check(py_object); } - bool PyToolkit::isMemoObject(ObjectPtr py_object) { - return PyMemo_Check(py_object); + bool PyToolkit::isAnyMemoObject(ObjectPtr py_object) { + return PyAnyMemo_Check(py_object); } PyToolkit::ObjectPtr PyToolkit::getUUID(ObjectPtr py_object) { @@ -532,7 +532,7 @@ namespace db0::python std::uint64_t PyToolkit::getFixtureUUID(TypeObjectPtr py_type) { - if (isMemoType(py_type)) { + if (isAnyMemoType(py_type)) { return MemoTypeDecoration::get(py_type).getFixtureUUID(AccessType::READ_ONLY); } else { return 0; @@ -541,7 +541,7 @@ namespace db0::python bool PyToolkit::isNoDefaultTags(TypeObjectPtr py_type) { - if (isMemoType(py_type)) { + if (isAnyMemoType(py_type)) { return MemoTypeDecoration::get(py_type).getFlags()[MemoOptions::NO_DEFAULT_TAGS]; } else { return false; @@ -550,7 +550,7 @@ namespace db0::python bool PyToolkit::isNoCache(TypeObjectPtr py_type) { - if (isMemoType(py_type)) { + if (isAnyMemoType(py_type)) { return MemoTypeDecoration::get(py_type).getFlags()[MemoOptions::NO_CACHE]; } else { return false; @@ -559,7 +559,7 @@ namespace db0::python bool PyToolkit::isImmutable(TypeObjectPtr py_type) { - if (isMemoType(py_type)) { + if (isAnyMemoType(py_type)) { return MemoTypeDecoration::get(py_type).getFlags()[MemoOptions::IMMUTABLE]; } else { return false; @@ -568,7 +568,7 @@ namespace db0::python FlagSet PyToolkit::getMemoFlags(TypeObjectPtr py_type) { - if (isMemoType(py_type)) { + if (isAnyMemoType(py_type)) { return MemoTypeDecoration::get(py_type).getFlags(); } else { return {}; @@ -583,18 +583,18 @@ namespace db0::python const char *PyToolkit::getMemoTypeID(TypeObjectPtr memo_type) { - assert(isMemoType(memo_type)); + assert(isAnyMemoType(memo_type)); return MemoTypeDecoration::get(memo_type).tryGetTypeId(); } const std::vector &PyToolkit::getInitVars(TypeObjectPtr memo_type) { - assert(isMemoType(memo_type)); + assert(isAnyMemoType(memo_type)); return MemoTypeDecoration::get(memo_type).getInitVars(); } - bool PyToolkit::isMemoType(TypeObjectPtr py_type) { - return PyMemoType_Check(py_type); + bool PyToolkit::isAnyMemoType(TypeObjectPtr py_type) { + return PyAnyMemoType_Check(py_type); } void PyToolkit::setError(ObjectPtr err_obj, std::uint64_t err_value) { diff --git a/src/dbzero/bindings/python/PyToolkit.hpp b/src/dbzero/bindings/python/PyToolkit.hpp index 9c02eb23..bf7987c0 100644 --- a/src/dbzero/bindings/python/PyToolkit.hpp +++ b/src/dbzero/bindings/python/PyToolkit.hpp @@ -171,8 +171,9 @@ namespace db0::python static bool isIterable(ObjectPtr py_object); static bool isSequence(ObjectPtr py_object); static bool isType(ObjectPtr py_object); - static bool isMemoType(TypeObjectPtr py_type); - static bool isMemoObject(ObjectPtr py_object); + // either memo or immutable type + static bool isAnyMemoType(TypeObjectPtr py_type); + static bool isAnyMemoObject(ObjectPtr py_object); static bool isEnumValue(ObjectPtr py_object); static bool isFieldDef(ObjectPtr py_object); static bool isClassObject(ObjectPtr py_object); diff --git a/src/dbzero/bindings/python/PyTypeManager.cpp b/src/dbzero/bindings/python/PyTypeManager.cpp index 703828cc..8051fcde 100644 --- a/src/dbzero/bindings/python/PyTypeManager.cpp +++ b/src/dbzero/bindings/python/PyTypeManager.cpp @@ -119,10 +119,13 @@ namespace db0::python } // check if a memo class first - if (PyMemoType_Check(py_type)) { + if (PyMemoType_Check(py_type)) { return TypeId::MEMO_OBJECT; } - + if (PyMemoType_Check(py_type)) { + return TypeId::MEMO_IMMUTABLE_OBJECT; + } + // check with the static types next auto it = m_id_map.find(reinterpret_cast(py_type)); if (it == m_id_map.end()) { @@ -188,7 +191,7 @@ namespace db0::python const db0::object_model::Object &PyTypeManager::extractObject(ObjectPtr memo_ptr) const { - if (PyMemo_Check(memo_ptr)) { + if (PyMemo_Check(memo_ptr)) { return reinterpret_cast(memo_ptr)->ext(); } else if (PyWeakProxy_Check(memo_ptr)) { return reinterpret_cast(reinterpret_cast(memo_ptr)->get())->ext(); diff --git a/src/dbzero/bindings/python/PyTypeManager.hpp b/src/dbzero/bindings/python/PyTypeManager.hpp index ba8473c8..d7dc6b00 100644 --- a/src/dbzero/bindings/python/PyTypeManager.hpp +++ b/src/dbzero/bindings/python/PyTypeManager.hpp @@ -94,6 +94,8 @@ namespace db0::python */ const Object &extractObject(ObjectPtr memo_ptr) const; Object &extractMutableObject(ObjectPtr memo_ptr) const; + ObjectImmutableImpl &extractImmutableObject(ObjectPtr memo_ptr) const; + const Object *tryExtractObject(ObjectPtr memo_ptr) const; Object *tryExtractMutableObject(ObjectPtr memo_ptr) const; const List &extractList(ObjectPtr list_ptr) const; diff --git a/src/dbzero/bindings/python/PyWeakProxy.cpp b/src/dbzero/bindings/python/PyWeakProxy.cpp index 48e37f7d..67669a3c 100644 --- a/src/dbzero/bindings/python/PyWeakProxy.cpp +++ b/src/dbzero/bindings/python/PyWeakProxy.cpp @@ -12,18 +12,30 @@ namespace db0::python { PYVAROBJECT_HEAD_INIT_DESIGNATED, .tp_name = "WeakProxy", - .tp_basicsize = sizeof(PyWeakProxy), + .tp_basicsize = sizeof(PyWeakProxy), .tp_itemsize = 0, - .tp_dealloc = (destructor)PyAPI_PyWeakProxy_del, + .tp_dealloc = (destructor)PyAPI_PyWeakProxy_del, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_new = PyType_GenericNew, }; - MemoObject *PyWeakProxy::get() const { - return reinterpret_cast(m_py_object); + PyTypeObject PyWeakProxyImmutableType = + { + PYVAROBJECT_HEAD_INIT_DESIGNATED, + .tp_name = "WeakProxy", + .tp_basicsize = sizeof(PyWeakProxy), + .tp_itemsize = 0, + .tp_dealloc = (destructor)PyAPI_PyWeakProxy_del, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = PyType_GenericNew, + }; + + template MemoImplT *PyWeakProxy::get() const { + return reinterpret_cast(m_py_object); } - void PyAPI_PyWeakProxy_del(PyWeakProxy *py_weak_proxy) + template + void PyAPI_PyWeakProxy_del(PyWeakProxy *py_weak_proxy) { PY_API_FUNC if (py_weak_proxy->m_py_object) { @@ -35,14 +47,19 @@ namespace db0::python bool PyWeakProxy_Check(PyObject *obj) { return PyObject_TypeCheck(obj, &PyWeakProxyType); } - + PyObject *tryWeakProxy(PyObject *py_obj) { - if (!PyMemo_Check(py_obj)) { - THROWF(db0::InputException) << "Invalid argument type: " << PyToolkit::getTypeName(py_obj) << " (memo expected)"; + PyObject *py_weak_proxy = nullptr; + if (PyMemo_Check(py_obj)) { + py_weak_proxy = PyObject_New(PyWeakProxy, &PyWeakProxyType); + } else if (PyMemo_Check(py_obj)) { + py_weak_proxy = PyObject_New(PyWeakProxy, &PyWeakProxyImmutableType); + } else { + PyErrSetString(PyExc_TypeError, "Expected a memo object"); + return nullptr; } - // new PyWeakProxy - auto py_weak_proxy = PyObject_New(PyWeakProxy, &PyWeakProxyType); + if (!py_weak_proxy) { return nullptr; } diff --git a/src/dbzero/bindings/python/PyWeakProxy.hpp b/src/dbzero/bindings/python/PyWeakProxy.hpp index eabebe4b..a67f0f22 100644 --- a/src/dbzero/bindings/python/PyWeakProxy.hpp +++ b/src/dbzero/bindings/python/PyWeakProxy.hpp @@ -8,21 +8,32 @@ namespace db0::python { + template struct PyWeakProxy { PyObject_HEAD PyObject* m_py_object; // get the underlying memo object - MemoObject *get() const; + MemoImplT *get() const; }; extern PyTypeObject PyWeakProxyType; + // a weak proxy to immutable memo object + extern PyTypeObject PyWeakProxyImmutableType; - void PyAPI_PyWeakProxy_del(PyWeakProxy *self); + bool PyAnyWeakProxy_Check(PyObject *); + + template bool PyWeakProxy_Check(PyObject *obj); PyObject *tryWeakProxy(PyObject *); PyObject *tryExpired(PyObject *); + + extern template struct PyWeakProxy; + extern template struct PyWeakProxy; + + extern template bool PyWeakProxy_Check(PyObject *); + extern template bool PyWeakProxy_Check(PyObject *); } \ No newline at end of file diff --git a/src/dbzero/bindings/python/shared_py_object.cpp b/src/dbzero/bindings/python/shared_py_object.cpp index 4aa42092..8bf0230e 100644 --- a/src/dbzero/bindings/python/shared_py_object.cpp +++ b/src/dbzero/bindings/python/shared_py_object.cpp @@ -4,30 +4,52 @@ namespace db0::python -{ +{ + + template + void incExtRefImpl(PyObject *py_object) { + // increment reference count for memo objects + reinterpret_cast(py_object)->ext().addExtRef(); + } + + template + void decExtRef(PyObject *py_object) { + // decrement reference count for memo objects + reinterpret_cast(py_object)->ext().removeExtRef(); + } + + template + unsigned int getExtRefcount(PyObject *py_object, unsigned int default_count) { + // return reference count for memo objects + return reinterpret_cast(py_object)->ext().getExtRefs(); + } void incExtRef(PyObject *py_object) - { - if (PyMemo_Check(py_object)) { - // increment reference count for memo objects - reinterpret_cast(py_object)->ext().addExtRef(); + { + if (PyMemo_Check(py_object)) { + incExtRefImpl(py_object); + } else if (PyMemo_Check(py_object)) { + incExtRefImpl(py_object); } } void decExtRef(PyObject *py_object) - { - if (PyMemo_Check(py_object)) { - // decrement reference count for memo objects - reinterpret_cast(py_object)->ext().removeExtRef(); - } + { + if (PyMemo_Check(py_object)) { + decExtRef(py_object); + } else if (PyMemo_Check(py_object)) { + decExtRef(py_object); + } } unsigned int getExtRefcount(PyObject *py_object, unsigned int default_count) - { - if (PyMemo_Check(py_object)) { - // return reference count for memo objects - return reinterpret_cast(py_object)->ext().getExtRefs(); + { + if (PyMemo_Check(py_object)) { + return getExtRefcount(py_object, default_count); + } else if (PyMemo_Check(py_object)) { + return getExtRefcount(py_object, default_count); } + // for non-memo objects, return the default count return default_count; } diff --git a/src/dbzero/bindings/python/types/PyTag.cpp b/src/dbzero/bindings/python/types/PyTag.cpp index 25b72a7a..e64aa5f6 100644 --- a/src/dbzero/bindings/python/types/PyTag.cpp +++ b/src/dbzero/bindings/python/types/PyTag.cpp @@ -70,18 +70,19 @@ namespace db0::python return Py_TYPE(py_object) == &PyTagType; } + template PyObject *tryMemoAsTag(PyObject *py_obj) { - assert(PyMemo_Check(py_obj)); - auto &memo_obj = reinterpret_cast(py_obj)->ext(); + assert(PyMemo_Check(py_obj)); + auto &memo_obj = reinterpret_cast(py_obj)->ext(); PyTag *py_tag = PyTagDefault_new(); py_tag->makeNew(memo_obj.getFixture()->getUUID(), memo_obj.getAddress(), py_obj); return py_tag; } - + PyObject *tryMemoTypeAsTag(PyTypeObject *py_type) { - assert(PyMemoType_Check(py_type)); + assert(PyAnyMemoType_Check(py_type)); PyTag *py_tag = PyTagDefault_new(); py_tag->makeNew(py_type, db0::object_model::TagDef::type_as_tag()); return py_tag; @@ -103,18 +104,20 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "as_tag: Expected 1 argument"); return NULL; } - if (PyMemo_Check(args[0])) { - return runSafe(tryMemoAsTag, args[0]); + if (PyMemo_Check(args[0])) { + return runSafe(tryMemoAsTag, args[0]); + } else if (PyMemo_Check(args[0])) { + return runSafe(tryMemoAsTag, args[0]); } else if (PyType_Check(args[0])) { auto *py_type = reinterpret_cast(args[0]); - if (PyMemoType_Check(py_type)) { + if (PyAnyMemoType_Check(py_type)) { return runSafe(tryMemoTypeAsTag, py_type); } } else if (MemoExpiredRef_Check(args[0])) { return runSafe(tryMemoExpiredRefAsTag, args[0]); - } + } PyErr_SetString(PyExc_TypeError, "as_tag: Expected a memo object"); return NULL; } - + } diff --git a/src/dbzero/object_model/object/ObjectImmutableImpl.cpp b/src/dbzero/object_model/object/ObjectImmutableImpl.cpp index 490eb1ff..539d5cd6 100644 --- a/src/dbzero/object_model/object/ObjectImmutableImpl.cpp +++ b/src/dbzero/object_model/object/ObjectImmutableImpl.cpp @@ -5,5 +5,5 @@ namespace db0::object_model { GC0_Define(ObjectImmutableImpl) - + } \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectImplBase.hpp b/src/dbzero/object_model/object/ObjectImplBase.hpp index 5cab0edf..ec88cf5d 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.hpp +++ b/src/dbzero/object_model/object/ObjectImplBase.hpp @@ -209,9 +209,6 @@ namespace db0::object_model // FieldID, is_init_var, fidelity std::pair findField(const char *name) const; - // Check if the 2 memo objects are of the same type - bool sameType(const Object &) const; - // the member called to indicate the object mutation void touch(); From 138f3bf2110cb5b82153fca1605d49fd05d928c0 Mon Sep 17 00:00:00 2001 From: Wojtek Date: Wed, 5 Nov 2025 16:01:01 +0100 Subject: [PATCH 11/19] WIP: immutable object --- src/dbzero/bindings/python/Memo.cpp | 17 ++--- src/dbzero/bindings/python/Memo.hpp | 8 ++- src/dbzero/bindings/python/PySnapshot.cpp | 12 ++-- src/dbzero/bindings/python/PyToolkit.cpp | 25 ++++---- src/dbzero/bindings/python/PyTypeManager.cpp | 27 ++++++-- src/dbzero/bindings/python/PyTypeManager.hpp | 31 ++++++--- src/dbzero/bindings/python/PyWeakProxy.cpp | 52 +++++++++------ src/dbzero/bindings/python/PyWeakProxy.hpp | 3 +- src/dbzero/core/serialization/Ext.hpp | 4 +- src/dbzero/object_model/class/ClassFields.cpp | 4 +- .../object_model/index/IndexBuilder.hpp | 9 ++- .../object_model/object/ObjectCommonImpl.hpp | 22 +++++++ .../object_model/object/ObjectImplBase.cpp | 2 + src/dbzero/object_model/object/o_object.cpp | 7 +- src/dbzero/object_model/object/o_object.hpp | 34 ++++++++-- .../object_model/tags/ObjectTagManager.cpp | 4 +- src/dbzero/object_model/tags/TagIndex.cpp | 64 ++++++++++--------- src/dbzero/object_model/tags/TagIndex.hpp | 24 ++++--- src/dbzero/object_model/value/Member.cpp | 3 +- .../object_model/value/StorageClass.cpp | 2 + src/dbzero/workspace/AtomicContext.cpp | 8 ++- 21 files changed, 240 insertions(+), 122 deletions(-) create mode 100644 src/dbzero/object_model/object/ObjectCommonImpl.hpp diff --git a/src/dbzero/bindings/python/Memo.cpp b/src/dbzero/bindings/python/Memo.cpp index 344af19b..e7787819 100644 --- a/src/dbzero/bindings/python/Memo.cpp +++ b/src/dbzero/bindings/python/Memo.cpp @@ -400,7 +400,8 @@ namespace db0::python return lhs->ext() == rhs->ext(); } - PyObject *PyAPI_MemoObject_rq(MemoObject *memo_obj, PyObject *other, int op) + template + PyObject *PyAPI_MemoObject_rq(MemoImplT *memo_obj, PyObject *other, int op) { PY_API_FUNC PyObject * obj_memo = reinterpret_cast(memo_obj); @@ -408,14 +409,14 @@ namespace db0::python if (obj_memo->ob_type->tp_base->tp_richcompare != PyType_Type.tp_richcompare) { // if the base class richcompare is the same as the memo richcompare don't call the base class richcompare // to avoid infinite recursion - if (obj_memo->ob_type->tp_base->tp_richcompare != (richcmpfunc)PyAPI_MemoObject_rq) { + if (obj_memo->ob_type->tp_base->tp_richcompare != (richcmpfunc)PyAPI_MemoObject_rq) { return obj_memo->ob_type->tp_base->tp_richcompare(reinterpret_cast(memo_obj), other, op); } } bool eq_result = false; - if (PyMemo_Check(other)) { - eq_result = isSame(memo_obj, reinterpret_cast(other)); + if (PyMemo_Check(other)) { + eq_result = isSame(memo_obj, reinterpret_cast(other)); } switch (op) @@ -474,7 +475,7 @@ namespace db0::python {Py_tp_init, (void *)PyAPI_MemoObject_init}, {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro}, {Py_tp_setattro, (void *)PyAPI_MemoObject_setattro}, - {Py_tp_richcompare, (void *)PyAPI_MemoObject_rq}, + {Py_tp_richcompare, (void *)PyAPI_MemoObject_rq}, {Py_tp_hash, (void *)PyAPI_MemoHash}, {0, 0} }; @@ -487,7 +488,7 @@ namespace db0::python {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro}, // set available only on pre-initialized objects {Py_tp_setattro, (void *)PyAPI_MemoObject_setattro}, - {Py_tp_richcompare, (void *)PyAPI_MemoObject_rq}, + {Py_tp_richcompare, (void *)PyAPI_MemoObject_rq}, {Py_tp_hash, (void *)PyAPI_MemoHash}, {0, 0} }; @@ -979,8 +980,8 @@ namespace db0::python PY_API_FUNC return runSafe(tryGetSchema, reinterpret_cast(args[0])); } - - bool PyMemoType_Check(PyTypeObject *type) + + bool PyAnyMemoType_Check(PyTypeObject *type) { assert(type); return type->tp_dealloc == reinterpret_cast(MemoObject_del) || diff --git a/src/dbzero/bindings/python/Memo.hpp b/src/dbzero/bindings/python/Memo.hpp index b1880925..801eefd5 100644 --- a/src/dbzero/bindings/python/Memo.hpp +++ b/src/dbzero/bindings/python/Memo.hpp @@ -11,6 +11,7 @@ #include "MemoTypeDecoration.hpp" #include #include +#include namespace db0::object_model @@ -27,6 +28,7 @@ namespace db0::python using AccessType = db0::AccessType; using MemoObject = PyWrapper; using MemoImmutableObject = PyWrapper; + using MemoCommonObject = PyWrapper; PyObject *PyAPI_wrapPyClass(PyObject *self, PyObject *, PyObject *kwargs); // create a memo object stub @@ -57,7 +59,7 @@ namespace db0::python // check for a memo type (i.e. generated by PyAPI_wrapPyClass) template bool PyMemo_Check(PyObject *); - template bool PyMemoType_Check(PyObject *); + template bool PyMemoType_Check(PyTypeObject *); bool PyAnyMemo_Check(PyObject *); bool PyAnyMemoType_Check(PyTypeObject *); @@ -75,7 +77,7 @@ namespace db0::python extern template bool PyMemo_Check(PyObject *); extern template bool PyMemo_Check(PyObject *); - extern template bool PyMemoType_Check(PyObject *); - extern template bool PyMemoType_Check(PyObject *); + extern template bool PyMemoType_Check(PyTypeObject *); + extern template bool PyMemoType_Check(PyTypeObject *); } diff --git a/src/dbzero/bindings/python/PySnapshot.cpp b/src/dbzero/bindings/python/PySnapshot.cpp index 08749e32..4b499a64 100644 --- a/src/dbzero/bindings/python/PySnapshot.cpp +++ b/src/dbzero/bindings/python/PySnapshot.cpp @@ -238,7 +238,8 @@ namespace db0::python return runSafe(tryPySnapshot_close, self, args); } - PyObject *tryGetSnapshotOf(MemoObject *memo) + template + PyObject *tryGetSnapshotOf(MemoImplT *memo) { auto fixture = memo->ext().getFixture(); auto workspace_view = fixture->getWorkspace().shared_from_this(); @@ -255,11 +256,14 @@ namespace db0::python } PyObject *py_arg = args[0]; - if (!PyMemo_Check(py_arg)) { + if (PyMemo_Check(py_arg)) { + return runSafe(tryGetSnapshotOf, reinterpret_cast(py_arg)); + } else if (PyMemo_Check(py_arg)) { + return runSafe(tryGetSnapshotOf, reinterpret_cast(py_arg)); + } else { PyErr_SetString(PyExc_TypeError, "Invalid argument type (must be a memo object)"); return NULL; - } - return runSafe(tryGetSnapshotOf, reinterpret_cast(py_arg)); + } } static PySequenceMethods PySnapshot_sq = { diff --git a/src/dbzero/bindings/python/PyToolkit.cpp b/src/dbzero/bindings/python/PyToolkit.cpp index 2a708418..d9bbdb43 100644 --- a/src/dbzero/bindings/python/PyToolkit.cpp +++ b/src/dbzero/bindings/python/PyToolkit.cpp @@ -100,15 +100,15 @@ namespace db0::python bool PyToolkit::isExistingObject(db0::swine_ptr &fixture, Address address, std::uint16_t instance_id) { // try unloading from cache first - auto &lang_cache = fixture->getLangCache(); + auto &lang_cache = fixture->getLangCache(); auto obj_ptr = tryUnloadObjectFromCache(lang_cache, address, nullptr); if (obj_ptr.get()) { // only validate instance ID if provided - auto &memo = reinterpret_cast(obj_ptr.get())->ext(); + auto &memo = reinterpret_cast(obj_ptr.get())->ext(); if (instance_id) { // NOTE: we first must check if this is really a memo object - if (!isMemoObject(obj_ptr.get())) { + if (!isAnyMemoObject(obj_ptr.get())) { return false; } @@ -136,11 +136,10 @@ namespace db0::python // only validate instance ID if provided if (instance_id) { // NOTE: we first must check if this is really a memo object - if (!isMemoObject(obj_ptr.get())) { + if (!isAnyMemoObject(obj_ptr.get())) { return {}; } - - if (reinterpret_cast(obj_ptr.get())->ext().getInstanceId() != instance_id) { + if (reinterpret_cast(obj_ptr.get())->ext().getInstanceId() != instance_id) { return {}; } } @@ -517,8 +516,8 @@ namespace db0::python return getFixtureUUID(reinterpret_cast(py_object)); } else if (PyEnumValue_Check(py_object)) { return reinterpret_cast(py_object)->ext().m_fixture.safe_lock()->getUUID(); - } else if (PyMemo_Check(py_object)) { - return reinterpret_cast(py_object)->ext().getFixture()->getUUID(); + } else if (PyAnyMemo_Check(py_object)) { + return reinterpret_cast(py_object)->ext().getFixture()->getUUID(); } else if (PyObjectIterable_Check(py_object)) { return reinterpret_cast(py_object)->ext().getFixture()->getUUID(); } else if (PyObjectIterator_Check(py_object)) { @@ -577,7 +576,7 @@ namespace db0::python const char *PyToolkit::getPrefixName(TypeObjectPtr memo_type) { - assert(isMemoType(memo_type)); + assert(isAnyMemoType(memo_type)); return MemoTypeDecoration::get(memo_type).tryGetPrefixName(); } @@ -685,14 +684,14 @@ namespace db0::python PyToolkit::TypeObjectPtr PyToolkit::getBaseMemoType(TypeObjectPtr py_memo_type) { - assert(isMemoType(py_memo_type)); + assert(isAnyMemoType(py_memo_type)); // first base type is python base. From there we can get the actual base type auto base_py_type = getBaseType(py_memo_type); if (!base_py_type) { return nullptr; } auto memo_base_type = getBaseType(base_py_type); - if (memo_base_type && isMemoType(memo_base_type)) { + if (memo_base_type && isAnyMemoType(memo_base_type)) { return memo_base_type; } return nullptr; @@ -738,8 +737,8 @@ namespace db0::python bool PyToolkit::hasTagRefs(ObjectPtr obj_ptr) { - assert(PyMemo_Check(obj_ptr)); - return reinterpret_cast(obj_ptr)->ext().hasTagRefs(); + assert(PyAnyMemo_Check(obj_ptr)); + return reinterpret_cast(obj_ptr)->ext().hasTagRefs(); } std::unique_ptr PyToolkit::ensureLocked() diff --git a/src/dbzero/bindings/python/PyTypeManager.cpp b/src/dbzero/bindings/python/PyTypeManager.cpp index 8051fcde..45a604b4 100644 --- a/src/dbzero/bindings/python/PyTypeManager.cpp +++ b/src/dbzero/bindings/python/PyTypeManager.cpp @@ -153,7 +153,7 @@ namespace db0::python if (PyType_Check(ptr)) { auto py_type = reinterpret_cast(ptr); - if (PyMemoType_Check(py_type)) { + if (PyAnyMemoType_Check(py_type)) { return TypeId::MEMO_TYPE; } } @@ -189,6 +189,20 @@ namespace db0::python return *type_id; } + /* FIXME: implement + + template + const typename MemoImplT::ExtT &extractObject(ObjectPtr memo_ptr) const + { + if (PyMemo_Check(memo_ptr)) { + return reinterpret_cast(memo_ptr)->ext(); + } else if (PyWeakProxy_Check(memo_ptr)) { + return reinterpret_cast( + reinterpret_cast*>(memo_ptr)->get())->ext(); + } + THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; + } + const db0::object_model::Object &PyTypeManager::extractObject(ObjectPtr memo_ptr) const { if (PyMemo_Check(memo_ptr)) { @@ -198,15 +212,17 @@ namespace db0::python } THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; } - + */ + db0::object_model::Object &PyTypeManager::extractMutableObject(ObjectPtr memo_ptr) const { - if (!PyMemo_Check(memo_ptr)) { + if (!PyMemo_Check(memo_ptr)) { THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; } return reinterpret_cast(memo_ptr)->modifyExt(); } + /* FIXME: const db0::object_model::Object *PyTypeManager::tryExtractObject(ObjectPtr memo_ptr) const { if (!PyMemo_Check(memo_ptr)) { @@ -214,15 +230,16 @@ namespace db0::python } return &reinterpret_cast(memo_ptr)->ext(); } + */ db0::object_model::Object *PyTypeManager::tryExtractMutableObject(ObjectPtr memo_ptr) const { - if (!PyMemo_Check(memo_ptr)) { + if (!PyMemo_Check(memo_ptr)) { return nullptr; } return &reinterpret_cast(memo_ptr)->modifyExt(); } - + const db0::object_model::List &PyTypeManager::extractList(ObjectPtr list_ptr) const { if (!ListObject_Check(list_ptr)) { diff --git a/src/dbzero/bindings/python/PyTypeManager.hpp b/src/dbzero/bindings/python/PyTypeManager.hpp index d7dc6b00..0ec4d287 100644 --- a/src/dbzero/bindings/python/PyTypeManager.hpp +++ b/src/dbzero/bindings/python/PyTypeManager.hpp @@ -14,6 +14,8 @@ namespace db0::object_model { class Object; + class ObjectImmutableImpl; + class ObjectCommonImpl; class Class; class List; class Set; @@ -40,7 +42,9 @@ namespace db0::python { class MemoTypeDecoration; - + using MemoObject = PyWrapper; + using MemoImmutableObject = PyWrapper; + /** * The class dedicated to recognition of Python types */ @@ -52,7 +56,11 @@ namespace db0::python using ObjectSharedPtr = typename PyTypes::ObjectSharedPtr; using TypeObjectPtr = typename PyTypes::TypeObjectPtr; using TypeObjectSharedPtr = typename PyTypes::TypeObjectSharedPtr; + using MemoObject = db0::python::MemoObject; + using MemoImmutableObject = db0::python::MemoImmutableObject; using Object = db0::object_model::Object; + using ObjectImmutableImpl = db0::object_model::ObjectImmutableImpl; + using ObjectCommonImpl = db0::object_model::ObjectCommonImpl; using List = db0::object_model::List; using Set = db0::object_model::Set; using Tuple = db0::object_model::Tuple; @@ -82,22 +90,23 @@ namespace db0::python TypeId getTypeId(TypeObjectPtr py_type) const; std::optional tryGetTypeId(TypeObjectPtr ptr) const; std::string getLangTypeName(TypeObjectPtr) const; - + // Retrieve a Python type object by TypeId (note that a dbzero extension type may be returned) // to return a native type use getTypeObject(asNative(TypeId)) ObjectSharedPtr getTypeObject(TypeId) const; // If the mapping is not found, returns Py_None ObjectSharedPtr tryGetTypeObject(TypeId) const; - /** - * Extracts reference to DB0 object from a memo object - */ - const Object &extractObject(ObjectPtr memo_ptr) const; - Object &extractMutableObject(ObjectPtr memo_ptr) const; - ObjectImmutableImpl &extractImmutableObject(ObjectPtr memo_ptr) const; + // Extracts reference to Object or ObjectImmutableImpl from a memo instance + template + const typename MemoImplT::ExtT &extractObject(ObjectPtr memo_ptr) const; + // Extracts reference to common object part from a memo instance + const ObjectCommonImpl &extractCommonObject(ObjectPtr) const; + Object &extractMutableObject(ObjectPtr memo_ptr) const; const Object *tryExtractObject(ObjectPtr memo_ptr) const; Object *tryExtractMutableObject(ObjectPtr memo_ptr) const; + const List &extractList(ObjectPtr list_ptr) const; List &extractMutableList(ObjectPtr list_ptr) const; const Set &extractSet(ObjectPtr set_ptr) const; @@ -218,4 +227,10 @@ namespace db0::python m_simple_py_type_ids.insert(py_type_id); } + extern template const db0::object_model::Object & + PyTypeManager::extractObject(ObjectPtr) const; + + extern template const db0::object_model::ObjectImmutableImpl & + PyTypeManager::extractObject(ObjectPtr) const; + } \ No newline at end of file diff --git a/src/dbzero/bindings/python/PyWeakProxy.cpp b/src/dbzero/bindings/python/PyWeakProxy.cpp index 67669a3c..e827c97a 100644 --- a/src/dbzero/bindings/python/PyWeakProxy.cpp +++ b/src/dbzero/bindings/python/PyWeakProxy.cpp @@ -8,6 +8,16 @@ namespace db0::python { + template + void PyAPI_PyWeakProxy_del(PyWeakProxy *py_weak_proxy) + { + PY_API_FUNC + if (py_weak_proxy->m_py_object) { + Py_DECREF(py_weak_proxy->m_py_object); + } + Py_TYPE(py_weak_proxy)->tp_free((PyObject*)py_weak_proxy); + } + PyTypeObject PyWeakProxyType = { PYVAROBJECT_HEAD_INIT_DESIGNATED, @@ -30,44 +40,44 @@ namespace db0::python .tp_new = PyType_GenericNew, }; - template MemoImplT *PyWeakProxy::get() const { + template MemoImplT *PyWeakProxy::get() const { return reinterpret_cast(m_py_object); } - template - void PyAPI_PyWeakProxy_del(PyWeakProxy *py_weak_proxy) - { - PY_API_FUNC - if (py_weak_proxy->m_py_object) { - Py_DECREF(py_weak_proxy->m_py_object); - } - Py_TYPE(py_weak_proxy)->tp_free((PyObject*)py_weak_proxy); - } - bool PyWeakProxy_Check(PyObject *obj) { return PyObject_TypeCheck(obj, &PyWeakProxyType); } - PyObject *tryWeakProxy(PyObject *py_obj) + // MemoObject specialization + template <> PyObject *tryWeakProxy(PyObject *py_obj) { - PyObject *py_weak_proxy = nullptr; - if (PyMemo_Check(py_obj)) { - py_weak_proxy = PyObject_New(PyWeakProxy, &PyWeakProxyType); - } else if (PyMemo_Check(py_obj)) { - py_weak_proxy = PyObject_New(PyWeakProxy, &PyWeakProxyImmutableType); - } else { - PyErrSetString(PyExc_TypeError, "Expected a memo object"); - return nullptr; + assert(PyMemo_Check(py_obj)); + auto py_weak_proxy = PyObject_New(PyWeakProxy, &PyWeakProxyType); + + if (!py_weak_proxy) { + return nullptr; } + Py_INCREF(py_obj); + py_weak_proxy->m_py_object = py_obj; + return reinterpret_cast(py_weak_proxy); + } + + // MemoImmutableObject specialization + template <> PyObject *tryWeakProxy(PyObject *py_obj) + { + assert(PyMemo_Check(py_obj)); + auto py_weak_proxy = PyObject_New(PyWeakProxy, &PyWeakProxyImmutableType); + if (!py_weak_proxy) { return nullptr; } + Py_INCREF(py_obj); py_weak_proxy->m_py_object = py_obj; return reinterpret_cast(py_weak_proxy); } - + PyObject *tryExpired(PyObject *py_obj) { if (MemoExpiredRef_Check(py_obj)) { diff --git a/src/dbzero/bindings/python/PyWeakProxy.hpp b/src/dbzero/bindings/python/PyWeakProxy.hpp index a67f0f22..8fd3e121 100644 --- a/src/dbzero/bindings/python/PyWeakProxy.hpp +++ b/src/dbzero/bindings/python/PyWeakProxy.hpp @@ -27,12 +27,13 @@ namespace db0::python template bool PyWeakProxy_Check(PyObject *obj); + template PyObject *tryWeakProxy(PyObject *); PyObject *tryExpired(PyObject *); extern template struct PyWeakProxy; extern template struct PyWeakProxy; - + extern template bool PyWeakProxy_Check(PyObject *); extern template bool PyWeakProxy_Check(PyObject *); diff --git a/src/dbzero/core/serialization/Ext.hpp b/src/dbzero/core/serialization/Ext.hpp index 66497fee..207c4828 100644 --- a/src/dbzero/core/serialization/Ext.hpp +++ b/src/dbzero/core/serialization/Ext.hpp @@ -243,9 +243,9 @@ DB0_PACKED_BEGIN result += true_size_of() - true_size_of(); return result; } - + inline std::size_t sizeOf() const { - return T::safeSizeOf(this); + return T::safeSizeOf(reinterpret_cast(this)); } /** diff --git a/src/dbzero/object_model/class/ClassFields.cpp b/src/dbzero/object_model/class/ClassFields.cpp index 7d7db2cd..29d6e988 100644 --- a/src/dbzero/object_model/class/ClassFields.cpp +++ b/src/dbzero/object_model/class/ClassFields.cpp @@ -11,14 +11,14 @@ namespace db0::object_model ClassFields::ClassFields(TypeObjectPtr lang_type) : m_lang_type(lang_type) { - if (!LangToolkit::isMemoType(lang_type)) { + if (!LangToolkit::isAnyMemoType(lang_type)) { THROWF(db0::InputException) << "Expected Memo type object"; } } void ClassFields::init(TypeObjectPtr lang_type) { - if (!LangToolkit::isMemoType(lang_type)) { + if (!LangToolkit::isAnyMemoType(lang_type)) { THROWF(db0::InputException) << "Expected Memo type object"; } m_lang_type = lang_type; diff --git a/src/dbzero/object_model/index/IndexBuilder.hpp b/src/dbzero/object_model/index/IndexBuilder.hpp index dfb906ad..644bc60f 100644 --- a/src/dbzero/object_model/index/IndexBuilder.hpp +++ b/src/dbzero/object_model/index/IndexBuilder.hpp @@ -48,6 +48,7 @@ namespace db0::object_model mutable std::unordered_map m_object_cache; // add to cache and return object's address + template UniqueAddress addToCache(ObjectPtr); }; @@ -100,13 +101,15 @@ namespace db0::object_model m_object_cache.clear(); } - template UniqueAddress IndexBuilder::addToCache(ObjectPtr obj_ptr) + template + template + UniqueAddress IndexBuilder::addToCache(ObjectPtr obj_ptr) { - auto obj_addr = m_type_manager.extractObject(obj_ptr).getUniqueAddress(); + auto obj_addr = m_type_manager.extractObject(obj_ptr).getUniqueAddress(); if (m_object_cache.find(obj_addr) == m_object_cache.end()) { m_object_cache.emplace(obj_addr, obj_ptr); } return obj_addr; } - + } \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectCommonImpl.hpp b/src/dbzero/object_model/object/ObjectCommonImpl.hpp new file mode 100644 index 00000000..7bee525c --- /dev/null +++ b/src/dbzero/object_model/object/ObjectCommonImpl.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include "ObjectImplBase.hpp" +#include "o_object.hpp" + +namespace db0::object_model + +{ + + // NOTE: ObjectCommonImpl is for reinterpret_cast purposes only + // it allows accessing Object or ObjectImmutableImpl instances under a common base type + class ObjectCommonImpl: public ObjectImplBase + { + public: + static constexpr unsigned char REALM_ID = o_object_base::REALM_ID; + using super_t = ObjectImplBase; + + protected: + friend super_t; + }; + +} diff --git a/src/dbzero/object_model/object/ObjectImplBase.cpp b/src/dbzero/object_model/object/ObjectImplBase.cpp index 18252bd7..ebc2313e 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.cpp +++ b/src/dbzero/object_model/object/ObjectImplBase.cpp @@ -180,6 +180,7 @@ namespace db0::object_model assert(this->hasInstance()); } + /* FIXME: template std::pair ObjectImplBase::recognizeType(Fixture &fixture, ObjectPtr lang_value) const @@ -205,6 +206,7 @@ namespace db0::object_model return { type_id, storage_class }; } + */ template void ObjectImplBase::removePreInit(const char *field_name) const diff --git a/src/dbzero/object_model/object/o_object.cpp b/src/dbzero/object_model/object/o_object.cpp index aa082056..9da3023d 100644 --- a/src/dbzero/object_model/object/o_object.cpp +++ b/src/dbzero/object_model/object/o_object.cpp @@ -8,11 +8,16 @@ namespace db0::object_model { + + o_object_base::o_object_base(std::pair ref_counts) + : m_header(ref_counts) + { + } o_object::o_object(std::uint32_t class_ref, std::pair ref_counts, std::uint8_t num_type_tags, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin, const XValue *index_vt_end) - : m_header(ref_counts) + : super_t(ref_counts) , m_num_type_tags(num_type_tags) { arrangeMembers() diff --git a/src/dbzero/object_model/object/o_object.hpp b/src/dbzero/object_model/object/o_object.hpp index 25bd1a18..275f72cc 100644 --- a/src/dbzero/object_model/object/o_object.hpp +++ b/src/dbzero/object_model/object/o_object.hpp @@ -12,15 +12,41 @@ namespace db0::object_model { DB0_PACKED_BEGIN - class DB0_PACKED_ATTR o_object: public db0::o_base + class DB0_PACKED_ATTR o_object_base: public db0::o_base { protected: - using super_t = db0::o_base; + using super_t = db0::o_base; public: static constexpr unsigned char REALM_ID = 1; // common object header - o_unique_header m_header; + o_unique_header m_header; + + // ref_counts - the initial reference counts (tags / objects) inherited from the initializer + o_object_base(std::pair ref_counts); + + static std::size_t measure(); + static std::size_t measure(std::pair); + + static std::size_t sizeOf(); + + template static std::size_t safeSizeOf(BufT buf) { + return super_t::sizeOfMembers(buf); + } + + void incRef(bool is_tag); + bool hasRefs() const; + bool hasAnyRefs() const; + }; +DB0_PACKED_END + +DB0_PACKED_BEGIN + class DB0_PACKED_ATTR o_object: public db0::o_ext + { + protected: + using super_t = db0::o_ext; + + public: // optional address of the key-value store (to store extension fields) KV_Address m_kv_address; // kv-index type must be stored separately from the address @@ -30,7 +56,7 @@ DB0_PACKED_BEGIN PosVT &pos_vt(); const PosVT &pos_vt() const; - + const packed_int32 &classRef() const; std::uint32_t getClassRef() const; diff --git a/src/dbzero/object_model/tags/ObjectTagManager.cpp b/src/dbzero/object_model/tags/ObjectTagManager.cpp index 9cb054a3..feed36ad 100644 --- a/src/dbzero/object_model/tags/ObjectTagManager.cpp +++ b/src/dbzero/object_model/tags/ObjectTagManager.cpp @@ -44,9 +44,10 @@ namespace db0::object_model // construct as empty return new (at_ptr) ObjectTagManager(); } - return new (at_ptr) ObjectTagManager(memo_ptr, nargs); + return new (at_ptr) ObjectTagManager(memo_ptr, nargs); } + /* FIXME: ObjectTagManager::ObjectInfo::ObjectInfo(ObjectPtr memo_ptr) : m_lang_ptr(memo_ptr) , m_object_ptr(&ObjectTagManager::LangToolkit::getTypeManager().extractObject(memo_ptr)) @@ -56,6 +57,7 @@ namespace db0::object_model , m_has_tags(LangToolkit::hasTagRefs(memo_ptr)) { } + */ void ObjectTagManager::ObjectInfo::add(ObjectPtr const *args, Py_ssize_t nargs) { diff --git a/src/dbzero/object_model/tags/TagIndex.cpp b/src/dbzero/object_model/tags/TagIndex.cpp index 29a66c97..bec6d091 100644 --- a/src/dbzero/object_model/tags/TagIndex.cpp +++ b/src/dbzero/object_model/tags/TagIndex.cpp @@ -165,22 +165,7 @@ namespace db0::object_model "TagIndex::flush() or close() must be called before destruction"); } - FT_BaseIndex::BatchOperationBuilder & - TagIndex::getBatchOperationShort(ObjectPtr memo_ptr, ActiveValueT &result, bool is_type) const - { - if (is_type) { - return getBatchOperation(memo_ptr, m_base_index_short, m_batch_op_types, result); - } else { - return getBatchOperation(memo_ptr, m_base_index_short, m_batch_op_short, result); - } - } - - db0::FT_BaseIndex::BatchOperationBuilder & - TagIndex::getBatchOperationLong(ObjectPtr memo_ptr, ActiveValueT &result) const - { - return getBatchOperation(memo_ptr, m_base_index_long, m_batch_op_long, result); - } - + /* FIXME: implement void TagIndex::addTags(ObjectPtr memo_ptr, ObjectPtr const *args, std::size_t nargs) { using TypeId = db0::bindings::TypeId; @@ -241,6 +226,27 @@ namespace db0::object_model m_mutation_log->onDirty(); } } + */ + + FT_BaseIndex::BatchOperationBuilder & + TagIndex::getBatchOperationShort(ObjectPtr memo_ptr, ActiveValueT &result, bool is_type) const + { + if (is_type) { + return getBatchOperation( + memo_ptr, m_base_index_short, m_batch_op_types, result + ); + } else { + return getBatchOperation( + memo_ptr, m_base_index_short, m_batch_op_short, result + ); + } + } + + db0::FT_BaseIndex::BatchOperationBuilder & + TagIndex::getBatchOperationLong(ObjectPtr memo_ptr, ActiveValueT &result) const + { + return getBatchOperation(memo_ptr, m_base_index_long, m_batch_op_long, result); + } void TagIndex::addTag(ObjectPtr memo_ptr, Address tag_addr, bool is_type) { addTag(memo_ptr, tag_addr.getOffset(), is_type); @@ -252,7 +258,7 @@ namespace db0::object_model auto &batch_operation = getBatchOperationShort(memo_ptr, active_key, is_type); batch_operation->addTags(active_key, TagPtrSequence(&tag, &tag + 1)); m_mutation_log->onDirty(); - } + } void TagIndex::addTag(ObjectPtr memo_ptr, LongTagT tag) { @@ -268,7 +274,7 @@ namespace db0::object_model batch_operation->removeTag({ obj_addr, nullptr }, tag_addr.getOffset()); m_mutation_log->onDirty(); } - + void TagIndex::removeTags(ObjectPtr memo_ptr, ObjectPtr const *args, std::size_t nargs) { if (nargs == 0) { @@ -373,7 +379,7 @@ namespace db0::object_model // NOTE: some object might've been dropped in the meantime, need to be reverted from batch operations for (const auto &item: m_object_cache) { auto obj_ptr = item.second.get(); - auto &memo = type_manager.extractObject(obj_ptr); + auto &memo = type_manager.extractCommonObject(obj_ptr); if (memo.isDead()) { revert(obj_ptr); } @@ -449,7 +455,7 @@ namespace db0::object_model assert(m_active_pre_cache.empty()); for (const auto &item: m_object_cache) { auto obj_ptr = item.second.get(); - auto &memo = type_manager.extractObject(obj_ptr); + auto &memo = type_manager.extractCommonObject(obj_ptr); // NOTE: dropped instances should've already been reverted by now // NOTE: we check for acutal language references (excluding LangCache + TagIndex) if (!memo.isDropped() && !memo.hasAnyRefs() && !LangToolkit::hasAnyLangRefs(obj_ptr, 2)) { @@ -474,7 +480,7 @@ namespace db0::object_model void TagIndex::buildActiveValues() const { for (auto &item: m_active_cache) { - auto &memo = LangToolkit::getTypeManager().extractObject(item.first); + auto &memo = LangToolkit::getTypeManager().extractCommonObject(item.first); // NOTE: defunct objects have to be ignored since they don't have a valid address // NOTE: defunct objects, since no valid unique address is assigned will be auto-reverted on flush if (!memo.isDefunct()) { @@ -553,7 +559,7 @@ namespace db0::object_model // Memo instance is directly fed into the FT_FixedKeyIterator if (type_id == TypeId::MEMO_OBJECT) { - auto addr = LangToolkit::getTypeManager().extractObject(arg).getUniqueAddress(); + auto addr = LangToolkit::getTypeManager().extractCommonObject(arg).getUniqueAddress(); factory.add(std::make_unique >(&addr, &addr + 1)); return true; } @@ -797,11 +803,11 @@ namespace db0::object_model assert(LangToolkit::isString(py_arg)); return LangToolkit::addTagFromString(py_arg, m_string_pool, inc_ref); } - + std::optional TagIndex::tryAddShortTagFromMemo(ObjectPtr py_arg) const { - assert(LangToolkit::isMemoObject(py_arg)); - auto &py_obj = LangToolkit::getTypeManager().extractObject(py_arg); + assert(LangToolkit::isAnyMemoObject(py_arg)); + auto &py_obj = LangToolkit::getTypeManager().extractCommonObject(py_arg); if (py_obj.getFixtureUUID() != m_fixture_uuid) { // must be added as long tag return std::nullopt; @@ -1045,7 +1051,7 @@ namespace db0::object_model // Python Memo type if (LangToolkit::isType(args[args_offset])) { lang_type = type_manager.getTypeObject(args[args_offset++]); - if (LangToolkit::isMemoType(lang_type)) { + if (LangToolkit::isAnyMemoType(lang_type)) { // MemoBase type does not correspond to any find criteria // but we may use its corresponding lang type if (!type_manager.isMemoBase(lang_type)) { @@ -1096,8 +1102,8 @@ namespace db0::object_model LongTagT TagIndex::getLongTagFromMemo(ObjectPtr py_arg) const { - assert(LangToolkit::isMemoObject(py_arg)); - auto &py_obj = LangToolkit::getTypeManager().extractObject(py_arg); + assert(LangToolkit::isAnyMemoObject(py_arg)); + auto &py_obj = LangToolkit::getTypeManager().extractCommonObject(py_arg); return { py_obj.getFixtureUUID(), py_obj.getAddress().getOffset() }; } @@ -1107,7 +1113,7 @@ namespace db0::object_model void TagIndex::revert(ObjectPtr memo_ptr) const { - auto &memo = LangToolkit::getTypeManager().extractObject(memo_ptr); + auto &memo = LangToolkit::getTypeManager().extractCommonObject(memo_ptr); auto addr = memo.getUniqueAddress(); if (m_batch_op_short) { m_batch_op_short->revert(addr); diff --git a/src/dbzero/object_model/tags/TagIndex.hpp b/src/dbzero/object_model/tags/TagIndex.hpp index 4b413db2..5df11462 100644 --- a/src/dbzero/object_model/tags/TagIndex.hpp +++ b/src/dbzero/object_model/tags/TagIndex.hpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -15,21 +15,21 @@ namespace db0::object_model { - -DB0_PACKED_BEGIN - using Object = db0::object_model::Object; + using ObjectCommonImpl = db0::object_model::ObjectCommonImpl; using RC_LimitedStringPool = db0::pools::RC_LimitedStringPool; using LongTagT = db0::LongTagT; class EnumFactory; - + +DB0_PACKED_BEGIN struct DB0_PACKED_ATTR o_tag_index: public o_fixed { Address m_base_index_short_ptr = {}; Address m_base_index_long_ptr = {}; std::array m_reserved = { 0, 0, 0, 0 }; }; - +DB0_PACKED_END + /** * A class to represent a full-text (tag) index and the corresponding batch-update buffer * typically the TagIndex instance is associated with the Class object @@ -38,7 +38,7 @@ DB0_PACKED_BEGIN { public: using super_t = db0::v_object; - using LangToolkit = typename Object::LangToolkit; + using LangToolkit = typename ObjectCommonImpl::LangToolkit; using ObjectPtr = typename LangToolkit::ObjectPtr; using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; using ObjectSharedExtPtr = typename LangToolkit::ObjectSharedExtPtr; @@ -166,7 +166,7 @@ DB0_PACKED_BEGIN db0::FT_BaseIndex::BatchOperationBuilder &getBatchOperationShort(ObjectPtr, ActiveValueT &result, bool is_type) const; - + db0::FT_BaseIndex::BatchOperationBuilder &getBatchOperationLong(ObjectPtr, ActiveValueT &result) const; @@ -250,14 +250,14 @@ DB0_PACKED_BEGIN } return batch_op; } - + template BatchOperationT &TagIndex::getBatchOperation(ObjectPtr memo_ptr, BaseIndexT &base_index, BatchOperationT &batch_op, ActiveValueT &result) const - { + { // prepare the active value only if it's not yet initialized if (!result.first.isValid() && !result.second) { - auto &memo = LangToolkit::getTypeManager().extractObject(memo_ptr); + auto &memo = LangToolkit::getTypeManager().extractCommonObject(memo_ptr); // NOTE: that memo object may not have address before fully initialized (before postInit) if (memo.hasInstance()) { auto object_addr = memo.getUniqueAddress(); @@ -331,6 +331,4 @@ DB0_PACKED_BEGIN // Check if the object is pending update in the TagIndex withih a specific fixture bool isObjectPendingUpdate(db0::swine_ptr &fixture, UniqueAddress); -DB0_PACKED_END - } diff --git a/src/dbzero/object_model/value/Member.cpp b/src/dbzero/object_model/value/Member.cpp index fa632694..2dfce8b9 100644 --- a/src/dbzero/object_model/value/Member.cpp +++ b/src/dbzero/object_model/value/Member.cpp @@ -284,7 +284,8 @@ namespace db0::object_model PyObjectPtr obj_ptr, StorageClass storage_class, AccessFlags) { // NOTE: memo object can be extracted from the weak proxy - const auto &obj = PyToolkit::getTypeManager().extractObject(obj_ptr); + using MemoObject = PyToolkit::TypeManager::MemoObject; + const auto &obj = PyToolkit::getTypeManager().extractObject(obj_ptr); if (storage_class == StorageClass::OBJECT_LONG_WEAK_REF) { LongWeakRef weak_ref(fixture, obj); return weak_ref.getAddress(); diff --git a/src/dbzero/object_model/value/StorageClass.cpp b/src/dbzero/object_model/value/StorageClass.cpp index 2161d985..5d0d80c7 100644 --- a/src/dbzero/object_model/value/StorageClass.cpp +++ b/src/dbzero/object_model/value/StorageClass.cpp @@ -172,6 +172,7 @@ namespace db0 using LangToolkit = db0::object_model::LangConfig::LangToolkit; + /* FIXME: db0::object_model::StorageClass getStorageClass(db0::object_model::PreStorageClass pre_storage_class, db0::swine_ptr &fixture, ObjectPtr lang_value) { @@ -195,5 +196,6 @@ namespace db0 } return StorageClass::OBJECT_WEAK_REF; } + */ } \ No newline at end of file diff --git a/src/dbzero/workspace/AtomicContext.cpp b/src/dbzero/workspace/AtomicContext.cpp index 876bd807..d95755c0 100644 --- a/src/dbzero/workspace/AtomicContext.cpp +++ b/src/dbzero/workspace/AtomicContext.cpp @@ -22,10 +22,12 @@ namespace db0 } // MEMO_OBJECT specialization - template <> void detachObject(PyObjectPtr obj_ptr) { - detachExisting(PyToolkit::getTypeManager().extractObject(obj_ptr)); + template <> void detachObject(PyObjectPtr obj_ptr) + { + using MemoObject = PyToolkit::TypeManager::MemoObject; + detachExisting(PyToolkit::getTypeManager().extractObject(obj_ptr)); } - + // DB0_LIST specialization template <> void detachObject(PyObjectPtr obj_ptr) { detachExisting(PyToolkit::getTypeManager().extractList(obj_ptr)); From 8dd0b9b4d397278e119da7bffcdb5434ffef00cf Mon Sep 17 00:00:00 2001 From: Wojtek Date: Wed, 5 Nov 2025 16:58:44 +0100 Subject: [PATCH 12/19] save work --- src/dbzero/bindings/python/Memo.cpp | 19 ++++--- src/dbzero/bindings/python/Memo.hpp | 9 ++-- src/dbzero/bindings/python/PyAPI.cpp | 29 +++++----- src/dbzero/bindings/python/PyInternalAPI.cpp | 53 +++++++++++-------- src/dbzero/bindings/python/PyToolkit.cpp | 8 +-- src/dbzero/bindings/python/PyTypeManager.hpp | 10 ++-- src/dbzero/core/memory/PrefixCache.cpp | 6 +-- src/dbzero/object_model/index/Index.hpp | 2 +- .../object_model/index/IndexBuilder.hpp | 18 +++---- src/dbzero/object_model/object/Object.cpp | 15 ++++++ src/dbzero/object_model/object/Object.hpp | 4 ++ ...ObjectCommonImpl.hpp => ObjectAnyImpl.hpp} | 6 +-- .../object_model/object/ObjectImplBase.cpp | 19 +------ .../object_model/object/ObjectImplBase.hpp | 5 -- src/dbzero/object_model/tags/TagIndex.hpp | 6 +-- 15 files changed, 112 insertions(+), 97 deletions(-) rename src/dbzero/object_model/object/{ObjectCommonImpl.hpp => ObjectAnyImpl.hpp} (61%) diff --git a/src/dbzero/bindings/python/Memo.cpp b/src/dbzero/bindings/python/Memo.cpp index e7787819..d7a1aaf8 100644 --- a/src/dbzero/bindings/python/Memo.cpp +++ b/src/dbzero/bindings/python/Memo.cpp @@ -396,7 +396,8 @@ namespace db0::python return 0; } - bool isSame(MemoObject *lhs, MemoObject *rhs) { + template + bool isSame(MemoImplT *lhs, MemoImplT *rhs) { return lhs->ext() == rhs->ext(); } @@ -689,7 +690,7 @@ namespace db0::python PyObject *tryPyMemoCheck(PyObject *py_obj) { - if (PyMemo_Check(py_obj) || (PyType_Check(py_obj) && PyMemoType_Check(reinterpret_cast(py_obj)))) { + if (PyAnyMemo_Check(py_obj) || (PyType_Check(py_obj) && PyAnyMemoType_Check(reinterpret_cast(py_obj)))) { Py_RETURN_TRUE; } Py_RETURN_FALSE; @@ -709,7 +710,8 @@ namespace db0::python return type->tp_new == reinterpret_cast(PyAPI_MemoObject_new_singleton); } - PyObject *MemoObject_GetFieldLayout(MemoObject *self) + template + PyObject *MemoObject_GetFieldLayout(MemoImplT *self) { auto field_layout = self->ext().getFieldLayout(); auto &pos_vt = field_layout.m_pos_vt_fields; @@ -749,8 +751,9 @@ namespace db0::python return py_result.steal(); } - PyObject *MemoObject_DescribeObject(MemoObject *self) - { + template + PyObject *MemoObject_DescribeObject(MemoImplT *self) + { auto py_field_layout = Py_OWN(MemoObject_GetFieldLayout(self)); auto py_result = Py_OWN(PyDict_New()); PySafeDict_SetItemString(*py_result, "field_layout", py_field_layout); @@ -759,7 +762,7 @@ namespace db0::python PySafeDict_SetItemString(*py_result, "size_of", Py_OWN(PyLong_FromLong(self->ext()->sizeOf()))); return py_result.steal(); } - + void MemoType_get_info(PyTypeObject *type, PyObject *dict) { auto &decor = MemoTypeDecoration::get(type); @@ -936,7 +939,7 @@ namespace db0::python { using SchemaTypeId = db0::object_model::SchemaTypeId; - if (!PyMemoType_Check(py_type)) { + if (!PyAnyMemoType_Check(py_type)) { PyErr_SetString(PyExc_TypeError, "Expected a memo type"); return nullptr; } @@ -946,7 +949,7 @@ namespace db0::python auto &class_factory = fixture->get(); // find py type associated dbzero class with the ClassFactory auto type = class_factory.getExistingType(py_type); - + auto py_schema = Py_OWN(PyDict_New()); auto &type_manager = PyToolkit::getTypeManager(); type->getSchema([&](const std::string &key, SchemaTypeId primary_type, const std::vector &all_types) { diff --git a/src/dbzero/bindings/python/Memo.hpp b/src/dbzero/bindings/python/Memo.hpp index 801eefd5..38768934 100644 --- a/src/dbzero/bindings/python/Memo.hpp +++ b/src/dbzero/bindings/python/Memo.hpp @@ -11,7 +11,7 @@ #include "MemoTypeDecoration.hpp" #include #include -#include +#include namespace db0::object_model @@ -28,7 +28,7 @@ namespace db0::python using AccessType = db0::AccessType; using MemoObject = PyWrapper; using MemoImmutableObject = PyWrapper; - using MemoCommonObject = PyWrapper; + using MemoAnyObject = PyWrapper; PyObject *PyAPI_wrapPyClass(PyObject *self, PyObject *, PyObject *kwargs); // create a memo object stub @@ -40,9 +40,8 @@ namespace db0::python // check if memo type has been marked as singleton bool PyMemoType_IsSingleton(PyTypeObject *type); - PyObject *MemoObject_GetFieldLayout(MemoObject *); - - PyObject *MemoObject_DescribeObject(MemoObject *); + template + PyObject *MemoObject_DescribeObject(MemoImplT *); void MemoType_get_info(PyTypeObject *type, PyObject *dict); void MemoType_close(PyTypeObject *type); diff --git a/src/dbzero/bindings/python/PyAPI.cpp b/src/dbzero/bindings/python/PyAPI.cpp index 1f0d67bc..4f82a687 100644 --- a/src/dbzero/bindings/python/PyAPI.cpp +++ b/src/dbzero/bindings/python/PyAPI.cpp @@ -413,7 +413,7 @@ namespace db0::python if (PyType_Check(py_object)) { // only memo or enum types can be scoped - if (PyMemoType_Check(reinterpret_cast(py_object))) { + if (PyAnyMemoType_Check(reinterpret_cast(py_object))) { PyTypeObject *py_type = reinterpret_cast(py_object); auto prefix_name = MemoTypeDecoration::get(py_type).tryGetPrefixName(); if (prefix_name) { @@ -548,13 +548,15 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; } - - if (!PyMemo_Check(py_object)) { + + if (PyMemo_Check(py_object)) { + return MemoObject_DescribeObject(reinterpret_cast(py_object)); + } else if (PyMemo_Check(py_object)) { + return MemoObject_DescribeObject(reinterpret_cast(py_object)); + } else { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; - } - - return MemoObject_DescribeObject(reinterpret_cast(py_object)); + } } PyObject *describeObject(PyObject *self, PyObject *args) @@ -593,7 +595,7 @@ namespace db0::python PyObject *TryPyAPI_isSingleton(PyObject *py_object) { - + assert((PyMemo_Check(py_object))); return PyBool_fromBool(reinterpret_cast(py_object)->ext().isSingleton()); } @@ -605,14 +607,17 @@ namespace db0::python return NULL; } - if (!PyMemo_Check(py_object)) { + if (PyMemo_Check(py_object)) { + return runSafe(TryPyAPI_isSingleton, py_object); + } else if (PyMemo_Check(py_object)) { + // immutable memos are never singletons + PY_RETURN_FALSE; + } else { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; - } - - return runSafe(TryPyAPI_isSingleton, py_object); + } } - + PyObject *PyAPI_getRefCount(PyObject *, PyObject *args) { PY_API_FUNC diff --git a/src/dbzero/bindings/python/PyInternalAPI.cpp b/src/dbzero/bindings/python/PyInternalAPI.cpp index f19a29cf..a59deb06 100644 --- a/src/dbzero/bindings/python/PyInternalAPI.cpp +++ b/src/dbzero/bindings/python/PyInternalAPI.cpp @@ -433,26 +433,35 @@ namespace db0::python } return result; } + + template + PyObject *tryMemoGetRefCount(PyObject *py_object) + { + assert(PyMemo_Check(py_object)); + auto &memo = reinterpret_cast(py_object)->ext(); + auto fixture = memo.getFixture(); + // NOTE: there might be tag-removal operations buffered for the requested instance + // in case of a read/write mode conditionally trigger flush in such case + if (db0::object_model::isObjectPendingUpdate(fixture, memo.getUniqueAddress())) { + FixtureLock lock(fixture); + // flush pending updates + lock->flush(); + } + return PyLong_FromLong(getTotal(memo.getRefCounts(), -memo->m_num_type_tags)); + } PyObject *tryGetRefCount(PyObject *py_object) { - if (PyMemo_Check(py_object)) { - auto &memo = reinterpret_cast(py_object)->ext(); - auto fixture = memo.getFixture(); - // NOTE: there might be tag-removal operations buffered for the requested instance - // in case of a read/write mode conditionally trigger flush in such case - if (db0::object_model::isObjectPendingUpdate(fixture, memo.getUniqueAddress())) { - FixtureLock lock(fixture); - // flush pending updates - lock->flush(); - } - return PyLong_FromLong(getTotal(memo.getRefCounts(), -memo->m_num_type_tags)); + if (PyMemo_Check(py_object)) { + return tryMemoGetRefCount(py_object); + } else if (PyMemo_Check(py_object)) { + return tryMemoGetRefCount(py_object); } else if (PyClassObject_Check(py_object)) { auto ref_counts = reinterpret_cast(py_object)->ext().getRefCounts(); return PyLong_FromLong(getTotal(ref_counts, 0)); } else if (PyType_Check(py_object)) { auto py_type = PyToolkit::getTypeManager().getTypeObject(py_object); - if (PyToolkit::isMemoType(py_type)) { + if (PyToolkit::isAnyMemoType(py_type)) { auto &workspace = PyToolkit::getPyWorkspace().getWorkspace(); // sum over all prefixes std::uint64_t ref_counts = 0; @@ -615,8 +624,10 @@ namespace db0::python PyObject *tryGetAddress(PyObject *py_obj) { - if (PyMemo_Check(py_obj)) { - return PyLong_FromUnsignedLongLong(reinterpret_cast(py_obj)->ext().getAddress().getValue()); + if (PyAnyMemo_Check(py_obj)) { + return PyLong_FromUnsignedLongLong( + reinterpret_cast(py_obj)->ext().getAddress().getValue() + ); } // FIXME: implement for other dbzero types @@ -626,8 +637,8 @@ namespace db0::python PyTypeObject *tryGetType(PyObject *py_obj) { - if (PyMemo_Check(py_obj)) { - auto &memo = reinterpret_cast(py_obj)->ext(); + if (PyAnyMemo_Check(py_obj)) { + auto &memo = reinterpret_cast(py_obj)->ext(); auto fixture = memo.getFixture(); auto &class_factory = fixture->get(); if (!class_factory.hasLangType(memo.getType())) { @@ -646,9 +657,9 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; } - + PyTypeObject *py_type = reinterpret_cast(py_object); - if (!PyMemoType_Check(py_type)) { + if (!PyAnyMemoType_Check(py_type)) { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return nullptr; } @@ -664,18 +675,18 @@ namespace db0::python PyObject *tryGetMemoClass(PyObject *py_obj) { - if (!PyMemo_Check(py_obj)) { + if (!PyAnyMemo_Check(py_obj)) { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return nullptr; } - auto &memo_obj = reinterpret_cast(py_obj)->ext(); + auto &memo_obj = reinterpret_cast(py_obj)->ext(); if (!memo_obj.hasInstance()) { PyErr_SetString(PyExc_RuntimeError, "Memo object has no instance"); return nullptr; } return tryGetTypeInfo(memo_obj.getType()); } - + PyObject *tryLoad(PyObject *py_obj, PyObject* kwargs, PyObject *py_exclude, std::unordered_set *load_stack_ptr) { diff --git a/src/dbzero/bindings/python/PyToolkit.cpp b/src/dbzero/bindings/python/PyToolkit.cpp index d9bbdb43..afa6da8a 100644 --- a/src/dbzero/bindings/python/PyToolkit.cpp +++ b/src/dbzero/bindings/python/PyToolkit.cpp @@ -105,7 +105,7 @@ namespace db0::python if (obj_ptr.get()) { // only validate instance ID if provided - auto &memo = reinterpret_cast(obj_ptr.get())->ext(); + auto &memo = reinterpret_cast(obj_ptr.get())->ext(); if (instance_id) { // NOTE: we first must check if this is really a memo object if (!isAnyMemoObject(obj_ptr.get())) { @@ -139,7 +139,7 @@ namespace db0::python if (!isAnyMemoObject(obj_ptr.get())) { return {}; } - if (reinterpret_cast(obj_ptr.get())->ext().getInstanceId() != instance_id) { + if (reinterpret_cast(obj_ptr.get())->ext().getInstanceId() != instance_id) { return {}; } } @@ -517,7 +517,7 @@ namespace db0::python } else if (PyEnumValue_Check(py_object)) { return reinterpret_cast(py_object)->ext().m_fixture.safe_lock()->getUUID(); } else if (PyAnyMemo_Check(py_object)) { - return reinterpret_cast(py_object)->ext().getFixture()->getUUID(); + return reinterpret_cast(py_object)->ext().getFixture()->getUUID(); } else if (PyObjectIterable_Check(py_object)) { return reinterpret_cast(py_object)->ext().getFixture()->getUUID(); } else if (PyObjectIterator_Check(py_object)) { @@ -738,7 +738,7 @@ namespace db0::python bool PyToolkit::hasTagRefs(ObjectPtr obj_ptr) { assert(PyAnyMemo_Check(obj_ptr)); - return reinterpret_cast(obj_ptr)->ext().hasTagRefs(); + return reinterpret_cast(obj_ptr)->ext().hasTagRefs(); } std::unique_ptr PyToolkit::ensureLocked() diff --git a/src/dbzero/bindings/python/PyTypeManager.hpp b/src/dbzero/bindings/python/PyTypeManager.hpp index 0ec4d287..620ee399 100644 --- a/src/dbzero/bindings/python/PyTypeManager.hpp +++ b/src/dbzero/bindings/python/PyTypeManager.hpp @@ -15,7 +15,7 @@ namespace db0::object_model { class Object; class ObjectImmutableImpl; - class ObjectCommonImpl; + class ObjectAnyImpl; class Class; class List; class Set; @@ -60,7 +60,7 @@ namespace db0::python using MemoImmutableObject = db0::python::MemoImmutableObject; using Object = db0::object_model::Object; using ObjectImmutableImpl = db0::object_model::ObjectImmutableImpl; - using ObjectCommonImpl = db0::object_model::ObjectCommonImpl; + using ObjectAnyImpl = db0::object_model::ObjectAnyImpl; using List = db0::object_model::List; using Set = db0::object_model::Set; using Tuple = db0::object_model::Tuple; @@ -102,8 +102,10 @@ namespace db0::python const typename MemoImplT::ExtT &extractObject(ObjectPtr memo_ptr) const; // Extracts reference to common object part from a memo instance - const ObjectCommonImpl &extractCommonObject(ObjectPtr) const; - Object &extractMutableObject(ObjectPtr memo_ptr) const; + const ObjectAnyImpl &extractCommonObject(ObjectPtr) const; + ObjectAnyImpl &extractMutableCommonObject(ObjectPtr) const; + + Object &extractMutableObject(ObjectPtr memo_ptr) const; const Object *tryExtractObject(ObjectPtr memo_ptr) const; Object *tryExtractMutableObject(ObjectPtr memo_ptr) const; diff --git a/src/dbzero/core/memory/PrefixCache.cpp b/src/dbzero/core/memory/PrefixCache.cpp index 1243f2ad..591db096 100644 --- a/src/dbzero/core/memory/PrefixCache.cpp +++ b/src/dbzero/core/memory/PrefixCache.cpp @@ -98,9 +98,9 @@ namespace db0 assert(dp_lock); bool is_volatile = access_mode[AccessOptions::no_flush]; - // Try upgrading the unused lock to the write state - // this is to avoid CoW in a writer process - if (access_mode[AccessOptions::write] && read_state_num != state_num) { + // Try upgrading the unused lock to the write state + // this is to avoid CoW in a writer process + if (access_mode[AccessOptions::write] && read_state_num != state_num) { // unused lock condition (i.e. might only be used by the CacheRecycler) // note that dirty locks cannot be upgraded (otherwise data would be lost) if (dp_lock->allowReuse() && dp_lock.use_count() == (dp_lock->isRecycled() ? 1 : 0) + 1) { diff --git a/src/dbzero/object_model/index/Index.hpp b/src/dbzero/object_model/index/Index.hpp index b158991b..250a9e3f 100644 --- a/src/dbzero/object_model/index/Index.hpp +++ b/src/dbzero/object_model/index/Index.hpp @@ -191,7 +191,7 @@ namespace db0::object_model if (!m_index) { if ((*this)->m_index_addr.isValid()) { // pull existing range tree - m_index = db0::make_shared_void(this->myPtr((*this)->m_index_addr)); + m_index = db0::make_shared_void(this->myPtr((*this)->m_index_addr)); } else { // create a new range tree instance m_index = db0::make_shared_void(this->getMemspace()); diff --git a/src/dbzero/object_model/index/IndexBuilder.hpp b/src/dbzero/object_model/index/IndexBuilder.hpp index 644bc60f..5a586b8b 100644 --- a/src/dbzero/object_model/index/IndexBuilder.hpp +++ b/src/dbzero/object_model/index/IndexBuilder.hpp @@ -1,7 +1,7 @@ #pragma once #include -#include +#include namespace db0::object_model @@ -13,13 +13,13 @@ namespace db0::object_model template class IndexBuilder: public RangeTree::Builder { public: + using super_t = typename RangeTree::Builder; using RangeTreeT = RangeTree; using LangToolkit = typename LangConfig::LangToolkit; using ObjectPtr = typename LangToolkit::ObjectPtr; using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; using ObjectSharedExtPtr = typename LangToolkit::ObjectSharedExtPtr; - using super_t = typename RangeTree::Builder; - + IndexBuilder(); IndexBuilder(std::unordered_set &&remove_null_values, std::unordered_set &&add_null_values, @@ -47,8 +47,7 @@ namespace db0::object_model // NOTE: cache must hold "shared" language reference to prevent object drop (index owns its objects) mutable std::unordered_map m_object_cache; - // add to cache and return object's address - template + // add to cache and return object's address UniqueAddress addToCache(ObjectPtr); }; @@ -88,24 +87,23 @@ namespace db0::object_model std::function add_callback = [&](UniqueAddress address) { auto it = m_object_cache.find(address); assert(it != m_object_cache.end()); - m_type_manager.extractMutableObject(it->second.get()).incRef(false); + m_type_manager.extractMutableCommonObject(it->second.get()).incRef(false); }; std::function erase_callback = [&](UniqueAddress address) { auto it = m_object_cache.find(address); assert(it != m_object_cache.end()); - m_type_manager.extractMutableObject(it->second.get()).decRef(false); + m_type_manager.extractMutableCommonObject(it->second.get()).decRef(false); }; super_t::flush(index, &add_callback, &erase_callback); m_object_cache.clear(); } - template - template + template UniqueAddress IndexBuilder::addToCache(ObjectPtr obj_ptr) { - auto obj_addr = m_type_manager.extractObject(obj_ptr).getUniqueAddress(); + auto obj_addr = m_type_manager.extractCommonObject(obj_ptr).getUniqueAddress(); if (m_object_cache.find(obj_addr) == m_object_cache.end()) { m_object_cache.emplace(obj_addr, obj_ptr); } diff --git a/src/dbzero/object_model/object/Object.cpp b/src/dbzero/object_model/object/Object.cpp index 72797e61..4e97a679 100644 --- a/src/dbzero/object_model/object/Object.cpp +++ b/src/dbzero/object_model/object/Object.cpp @@ -71,6 +71,21 @@ namespace db0::object_model return false; } + void Object::unSingleton(FixtureLock &) + { + auto &type = getType(); + // drop reference from the class + if (type.isSingleton()) { + // clear singleton address + type.unlinkSingleton(); + this->modify().m_header.decRef(false); + } + } + + bool Object::isSingleton() const { + return getType().isSingleton(); + } + bool Object::tryFindMemberAt(std::pair field_info, std::pair &result, std::pair &find_result) const { diff --git a/src/dbzero/object_model/object/Object.hpp b/src/dbzero/object_model/object/Object.hpp index 744bfcae..a0e36d2d 100644 --- a/src/dbzero/object_model/object/Object.hpp +++ b/src/dbzero/object_model/object/Object.hpp @@ -21,6 +21,10 @@ namespace db0::object_model { } + // Convert singleton into a regular instance + void unSingleton(FixtureLock &); + bool isSingleton() const; + // Assign language specific value as a field (to already initialized or uninitialized instance) // NOTE: if lang_value is nullptr then the member is removed void set(FixtureLock &, const char *field_name, ObjectPtr lang_value); diff --git a/src/dbzero/object_model/object/ObjectCommonImpl.hpp b/src/dbzero/object_model/object/ObjectAnyImpl.hpp similarity index 61% rename from src/dbzero/object_model/object/ObjectCommonImpl.hpp rename to src/dbzero/object_model/object/ObjectAnyImpl.hpp index 7bee525c..c27ccf2c 100644 --- a/src/dbzero/object_model/object/ObjectCommonImpl.hpp +++ b/src/dbzero/object_model/object/ObjectAnyImpl.hpp @@ -7,13 +7,13 @@ namespace db0::object_model { - // NOTE: ObjectCommonImpl is for reinterpret_cast purposes only + // NOTE: ObjectAnyImpl is for reinterpret_cast purposes only // it allows accessing Object or ObjectImmutableImpl instances under a common base type - class ObjectCommonImpl: public ObjectImplBase + class ObjectAnyImpl: public ObjectImplBase { public: static constexpr unsigned char REALM_ID = o_object_base::REALM_ID; - using super_t = ObjectImplBase; + using super_t = ObjectImplBase; protected: friend super_t; diff --git a/src/dbzero/object_model/object/ObjectImplBase.cpp b/src/dbzero/object_model/object/ObjectImplBase.cpp index ebc2313e..5ef2a4fa 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.cpp +++ b/src/dbzero/object_model/object/ObjectImplBase.cpp @@ -722,11 +722,6 @@ namespace db0::object_model } } - template - bool ObjectImplBase::isSingleton() const { - return getType().isSingleton(); - } - template void ObjectImplBase::dropTags(Class &type) const { @@ -786,19 +781,7 @@ namespace db0::object_model } } } - - template - void ObjectImplBase::unSingleton(FixtureLock &) - { - auto &type = getType(); - // drop reference from the class - if (type.isSingleton()) { - // clear singleton address - type.unlinkSingleton(); - this->modify().m_header.decRef(false); - } - } - + template void ObjectImplBase::destroy() const { diff --git a/src/dbzero/object_model/object/ObjectImplBase.hpp b/src/dbzero/object_model/object/ObjectImplBase.hpp index ec88cf5d..430cbf53 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.hpp +++ b/src/dbzero/object_model/object/ObjectImplBase.hpp @@ -136,13 +136,8 @@ namespace db0::object_model // Get description of the field layout FieldLayout getFieldLayout() const; - // Convert singleton into a regular instance - void unSingleton(FixtureLock &); - void destroy() const; - bool isSingleton() const; - // execute the function for all members (until false is returned from the input lambda) void forAll(std::function) const; void forAll(std::function) const; diff --git a/src/dbzero/object_model/tags/TagIndex.hpp b/src/dbzero/object_model/tags/TagIndex.hpp index 5df11462..d56b8270 100644 --- a/src/dbzero/object_model/tags/TagIndex.hpp +++ b/src/dbzero/object_model/tags/TagIndex.hpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -16,7 +16,7 @@ namespace db0::object_model { - using ObjectCommonImpl = db0::object_model::ObjectCommonImpl; + using ObjectAnyImpl = db0::object_model::ObjectAnyImpl; using RC_LimitedStringPool = db0::pools::RC_LimitedStringPool; using LongTagT = db0::LongTagT; class EnumFactory; @@ -38,7 +38,7 @@ DB0_PACKED_END { public: using super_t = db0::v_object; - using LangToolkit = typename ObjectCommonImpl::LangToolkit; + using LangToolkit = typename ObjectAnyImpl::LangToolkit; using ObjectPtr = typename LangToolkit::ObjectPtr; using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; using ObjectSharedExtPtr = typename LangToolkit::ObjectSharedExtPtr; From d9b1695fbd13aab8f972edee437cca5dcda9f55a Mon Sep 17 00:00:00 2001 From: Wojtek Date: Wed, 5 Nov 2025 17:51:40 +0100 Subject: [PATCH 13/19] save work --- src/dbzero/bindings/python/Memo.cpp | 30 ++++++---- src/dbzero/bindings/python/Memo.hpp | 10 +++- src/dbzero/bindings/python/PyAPI.cpp | 61 +++++++++++++++----- src/dbzero/bindings/python/PyInternalAPI.cpp | 29 +++++----- src/dbzero/bindings/python/PyInternalAPI.hpp | 3 +- src/dbzero/bindings/python/PyWeakProxy.cpp | 48 ++++----------- src/dbzero/bindings/python/PyWeakProxy.hpp | 24 ++------ 7 files changed, 100 insertions(+), 105 deletions(-) diff --git a/src/dbzero/bindings/python/Memo.cpp b/src/dbzero/bindings/python/Memo.cpp index d7a1aaf8..dbff6b2a 100644 --- a/src/dbzero/bindings/python/Memo.cpp +++ b/src/dbzero/bindings/python/Memo.cpp @@ -307,7 +307,8 @@ namespace db0::python } } - PyObject *tryMemoObject_getattro(MemoObject *memo_obj, PyObject *attr) + template + PyObject *tryMemoObject_getattro(MemoImplT *memo_obj, PyObject *attr) { // The method resolution order for Memo types is following: // 1. User type members (class members such as methods) @@ -325,10 +326,11 @@ namespace db0::python return _PyObject_GenericGetAttrWithDict(reinterpret_cast(memo_obj), attr, NULL, 0); } - PyObject *PyAPI_MemoObject_getattro(MemoObject *self, PyObject *attr) + template + PyObject *PyAPI_MemoObject_getattro(MemoImplT *self, PyObject *attr) { PY_API_FUNC - return runSafe(tryMemoObject_getattro, self, attr); + return runSafe(tryMemoObject_getattro, self, attr); } template @@ -474,7 +476,7 @@ namespace db0::python {Py_tp_new, (void *)PyAPI_MemoObject_new}, {Py_tp_dealloc, (void *)MemoObject_del}, {Py_tp_init, (void *)PyAPI_MemoObject_init}, - {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro}, + {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro}, {Py_tp_setattro, (void *)PyAPI_MemoObject_setattro}, {Py_tp_richcompare, (void *)PyAPI_MemoObject_rq}, {Py_tp_hash, (void *)PyAPI_MemoHash}, @@ -486,14 +488,14 @@ namespace db0::python {Py_tp_new, (void *)PyAPI_MemoObject_new}, {Py_tp_dealloc, (void *)MemoObject_del}, {Py_tp_init, (void *)PyAPI_MemoObject_init}, - {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro}, + {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro}, // set available only on pre-initialized objects {Py_tp_setattro, (void *)PyAPI_MemoObject_setattro}, {Py_tp_richcompare, (void *)PyAPI_MemoObject_rq}, {Py_tp_hash, (void *)PyAPI_MemoHash}, {0, 0} }; - + template PyType_Slot* getCommonSlots(); @@ -774,18 +776,20 @@ namespace db0::python MemoTypeDecoration::get(type).close(); } - PyObject *MemoObject_set_prefix(MemoObject *py_obj, const char *prefix_name) + template + PyObject *MemoObject_set_prefix(MemoImplT *py_obj, const char *prefix_name) { if (prefix_name) { - // can use "ext" since setFixtue is a non-mutating operation - auto &obj = const_cast(py_obj->ext()); + using ObjectT = typename MemoImplT::ExtT; + // can use "ext" since setFixture is a non-mutating operation + auto &obj = const_cast(py_obj->ext()); auto fixture = PyToolkit::getPyWorkspace().getWorkspace().getFixture(prefix_name, AccessType::READ_WRITE); obj.setFixture(fixture); return PyUnicode_FromString(prefix_name); } Py_RETURN_NONE; } - + PyObject *tryGetAttributes(PyTypeObject *type) { auto &decor = MemoTypeDecoration::get(type); @@ -813,7 +817,8 @@ namespace db0::python } } - PyObject *tryGetAttrAs(MemoObject *memo_obj, PyObject *attr, PyTypeObject *py_type) + template + PyObject *tryGetAttrAs(MemoImplT *memo_obj, PyObject *attr, PyTypeObject *py_type) { memo_obj->ext().getFixture()->refreshIfUpdated(); auto member = memo_obj->ext().tryGetAs(PyUnicode_AsUTF8(attr), py_type); @@ -883,7 +888,8 @@ namespace db0::python return tryLoad(*result, kwargs, nullptr, load_stack_ptr); } - PyObject *tryLoadMemo(MemoObject *memo_obj, PyObject *kwargs, PyObject *py_exclude, + template + PyObject *tryLoadMemo(MemoImplT *memo_obj, PyObject *kwargs, PyObject *py_exclude, std::unordered_set *load_stack_ptr) { auto load_method = Py_OWN(tryMemoObject_getattro(memo_obj, *Py_OWN(PyUnicode_FromString("__load__")))); diff --git a/src/dbzero/bindings/python/Memo.hpp b/src/dbzero/bindings/python/Memo.hpp index 38768934..be36e5a9 100644 --- a/src/dbzero/bindings/python/Memo.hpp +++ b/src/dbzero/bindings/python/Memo.hpp @@ -46,14 +46,18 @@ namespace db0::python void MemoType_get_info(PyTypeObject *type, PyObject *dict); void MemoType_close(PyTypeObject *type); - PyObject *MemoObject_set_prefix(MemoObject *, const char *prefix_name); + template + PyObject *MemoObject_set_prefix(MemoImplT *, const char *prefix_name); PyObject *tryGetAttributes(PyTypeObject *type); + // Try retrieving a memo member cast to a specific type // type ignored for non-memo members - PyObject *tryGetAttrAs(MemoObject *, PyObject *attr, PyTypeObject *); + template + PyObject *tryGetAttrAs(MemoImplT *, PyObject *attr, PyTypeObject *); - PyObject *tryLoadMemo(MemoObject *memo_obj, PyObject* kwargs, PyObject* exclude, + template + PyObject *tryLoadMemo(MemoImplT *memo_obj, PyObject* kwargs, PyObject* exclude, std::unordered_set *load_stack_ptr = nullptr); // check for a memo type (i.e. generated by PyAPI_wrapPyClass) diff --git a/src/dbzero/bindings/python/PyAPI.cpp b/src/dbzero/bindings/python/PyAPI.cpp index 4f82a687..f5b8d7ce 100644 --- a/src/dbzero/bindings/python/PyAPI.cpp +++ b/src/dbzero/bindings/python/PyAPI.cpp @@ -611,7 +611,7 @@ namespace db0::python return runSafe(TryPyAPI_isSingleton, py_object); } else if (PyMemo_Check(py_object)) { // immutable memos are never singletons - PY_RETURN_FALSE; + Py_RETURN_FALSE; } else { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; @@ -823,11 +823,11 @@ namespace db0::python Py_RETURN_NONE; } - if (PyMemo_Check(py_object)) { + if (PyAnyMemo_Check(py_object)) { // this operation is handled differently for singletons (as dyn_prefix) // check from type, not instance if (PyMemoType_IsSingleton(Py_TYPE(py_object))) { - if (reinterpret_cast(py_object)->ext().hasInstance()) { + if (reinterpret_cast(py_object)->ext().hasInstance()) { PyErr_SetString(PyExc_TypeError, "Unable to change scope of an existing singleton instance"); return NULL; } @@ -836,7 +836,15 @@ namespace db0::python } Py_RETURN_NONE; } else { - return runSafe(MemoObject_set_prefix, reinterpret_cast(py_object), prefix_name); + if (PyMemo_Check(py_object)) { + return runSafe(MemoObject_set_prefix, + reinterpret_cast(py_object), prefix_name + ); + } else { + return runSafe(MemoObject_set_prefix, + reinterpret_cast(py_object), prefix_name + ); + } } } @@ -949,13 +957,13 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; } - - if (!PyMemoType_Check(py_type)) { + + if (PyAnyMemoType_Check(py_type)) { + return runSafe(tryGetAttributes, py_type); + } else { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; - } - - return runSafe(tryGetAttributes, py_type); + } } PyObject *PyAPI_getAttrAs(PyObject *self, PyObject *const *args, Py_ssize_t nargs) @@ -966,8 +974,8 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "getattr_as requires exactly 2 arguments"); return NULL; } - - if (!PyMemo_Check(args[0])) { + + if (!PyAnyMemo_Check(args[0])) { // fall back to regular getattr if not a memo object return PyObject_GetAttr(args[0], args[1]); } @@ -978,7 +986,11 @@ namespace db0::python } PyTypeObject *py_type = reinterpret_cast(args[2]); - return runSafe(tryGetAttrAs, reinterpret_cast(args[0]), args[1], py_type); + if (PyMemo_Check(args[0])) { + return runSafe(tryGetAttrAs, reinterpret_cast(args[0]), args[1], py_type); + } else { + return runSafe(tryGetAttrAs, reinterpret_cast(args[0]), args[1], py_type); + } } PyObject *PyAPI_getAddress(PyObject *self, PyObject *const *args, Py_ssize_t nargs) @@ -1049,7 +1061,7 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "Invalid argument type. Exclude shoud be a sequence"); return NULL; } - if (!PyMemo_Check(py_object)) { + if (!PyAnyMemo_Check(py_object)) { PyErr_SetString(PyExc_TypeError, "Exclude is only supported for memo objects"); return NULL; } @@ -1083,10 +1095,25 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "materialized requires exactly 1 argument"); return NULL; } + auto py_obj = args[0]; + if (!PyAnyMemo_Check(py_obj)) { + // simply return self if not a memo object + Py_INCREF(py_obj); + return py_obj; + } + PY_API_FUNC - return runSafe(getMaterializedMemoObject, args[0]); + if (PyMemo_Check(py_obj)) { + return runSafe(getMaterializedMemoObject, + reinterpret_cast(py_obj) + ); + } else { + return runSafe(getMaterializedMemoObject, + reinterpret_cast(py_obj) + ); + } } - + PyObject *tryWait(const char *prefix, long state, long timeout) { std::unique_lock api_lock; @@ -1339,6 +1366,7 @@ namespace db0::python return runSafe(tryGetConfig); } + /* FIXME: PyObject *PyAPI_compare(PyObject *, PyObject *args, PyObject *kwargs) { PyObject *py_first = nullptr; @@ -1357,6 +1385,7 @@ namespace db0::python return runSafe(tryCompareMemo, reinterpret_cast(py_first), reinterpret_cast(py_second)); } + */ #ifndef NDEBUG PyObject *PyAPI_startDebugLogs(PyObject *self, PyObject *) @@ -1418,7 +1447,7 @@ namespace db0::python PyObject *PyAPI_touch(PyObject *, PyObject *const *args, Py_ssize_t nargs) { for (Py_ssize_t i = 0; i < nargs; ++i) { - if (!PyMemo_Check(args[i])) { + if (!PyAnyMemo_Check(args[i])) { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; } diff --git a/src/dbzero/bindings/python/PyInternalAPI.cpp b/src/dbzero/bindings/python/PyInternalAPI.cpp index a59deb06..46bf17e6 100644 --- a/src/dbzero/bindings/python/PyInternalAPI.cpp +++ b/src/dbzero/bindings/python/PyInternalAPI.cpp @@ -732,30 +732,27 @@ namespace db0::python return tryLoadEnumValue(reinterpret_cast(py_obj)); } else if (type_id == TypeId::MEMO_OBJECT) { return tryLoadMemo(reinterpret_cast(py_obj), kwargs, py_exclude, load_stack_ptr); + } else if (type_id == TypeId::MEMO_IMMUTABLE_OBJECT) { + return tryLoadMemo(reinterpret_cast(py_obj), kwargs, py_exclude, load_stack_ptr); } else { THROWF(db0::InputException) << "__load__ not implemented for type: " << Py_TYPE(py_obj)->tp_name << THROWF_END; } } - PyObject *getMaterializedMemoObject(PyObject *py_obj) + template + PyObject *getMaterializedMemoObject(MemoImplT *memo_obj) { - if (!PyMemo_Check(py_obj)) { - // simply return self if not a memo object - Py_INCREF(py_obj); - return py_obj; - } - auto memo_obj = reinterpret_cast(py_obj); if (memo_obj->ext().hasInstance()) { - Py_INCREF(py_obj); - return py_obj; + Py_INCREF(memo_obj); + return memo_obj; } db0::FixtureLock lock(memo_obj->ext().getFixture()); // materialize by calling postInit memo_obj->modifyExt().postInit(lock); - Py_INCREF(py_obj); - return py_obj; + Py_INCREF(memo_obj); + return memo_obj; } shared_py_object tryUnloadObjectFromCache(LangCacheView &lang_cache, Address address, @@ -768,10 +765,10 @@ namespace db0::python } if (expected_type) { - if (!PyMemo_Check(obj_ptr.get())) { + if (!PyAnyMemo_Check(obj_ptr.get())) { THROWF(db0::InputException) << "Invalid object type: " << PyToolkit::getTypeName(obj_ptr.get()) << " (Memo expected)"; } - auto &memo = reinterpret_cast(obj_ptr.get())->ext(); + auto &memo = reinterpret_cast(obj_ptr.get())->ext(); // validate type if (memo.getType() != *expected_type) { THROWF(db0::InputException) << "Memo type mismatch"; @@ -893,14 +890,14 @@ namespace db0::python { for (Py_ssize_t i = 0; i < nargs; ++i) { auto py_obj = args[i]; - if (!PyMemo_Check(py_obj)) { + if (!PyAnyMemo_Check(py_obj)) { THROWF(db0::InputException) << "Invalid object type: " << Py_TYPE(py_obj)->tp_name << " (Memo expected)"; } - auto &memo = reinterpret_cast(py_obj)->modifyExt(); + auto &memo = reinterpret_cast(py_obj)->modifyExt(); if (memo.hasInstance()) { db0::FixtureLock lock(memo.getFixture()); memo.touch(); - } + } } Py_RETURN_NONE; diff --git a/src/dbzero/bindings/python/PyInternalAPI.hpp b/src/dbzero/bindings/python/PyInternalAPI.hpp index ddebb5a1..3c0bb5b2 100644 --- a/src/dbzero/bindings/python/PyInternalAPI.hpp +++ b/src/dbzero/bindings/python/PyInternalAPI.hpp @@ -198,7 +198,8 @@ namespace db0::python PyObject *tryLoad(PyObject *, PyObject*, PyObject *py_exlude = nullptr, std::unordered_set *load_stack_ptr = nullptr); - PyObject *getMaterializedMemoObject(PyObject *py_obj); + template + PyObject *getMaterializedMemoObject(MemoImplT *py_obj); // Retrieve prefix (its Fixture objects) from the optional argument "prefix" db0::swine_ptr getOptionalPrefixFromArg(db0::Snapshot &workspace, const char *prefix_name); diff --git a/src/dbzero/bindings/python/PyWeakProxy.cpp b/src/dbzero/bindings/python/PyWeakProxy.cpp index e827c97a..8f55b7ef 100644 --- a/src/dbzero/bindings/python/PyWeakProxy.cpp +++ b/src/dbzero/bindings/python/PyWeakProxy.cpp @@ -7,9 +7,8 @@ namespace db0::python { - - template - void PyAPI_PyWeakProxy_del(PyWeakProxy *py_weak_proxy) + + void PyAPI_PyWeakProxy_del(PyWeakProxy *py_weak_proxy) { PY_API_FUNC if (py_weak_proxy->m_py_object) { @@ -22,37 +21,25 @@ namespace db0::python { PYVAROBJECT_HEAD_INIT_DESIGNATED, .tp_name = "WeakProxy", - .tp_basicsize = sizeof(PyWeakProxy), + .tp_basicsize = sizeof(PyWeakProxy), .tp_itemsize = 0, - .tp_dealloc = (destructor)PyAPI_PyWeakProxy_del, + .tp_dealloc = (destructor)PyAPI_PyWeakProxy_del, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_new = PyType_GenericNew, }; - PyTypeObject PyWeakProxyImmutableType = - { - PYVAROBJECT_HEAD_INIT_DESIGNATED, - .tp_name = "WeakProxy", - .tp_basicsize = sizeof(PyWeakProxy), - .tp_itemsize = 0, - .tp_dealloc = (destructor)PyAPI_PyWeakProxy_del, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = PyType_GenericNew, - }; - - template MemoImplT *PyWeakProxy::get() const { - return reinterpret_cast(m_py_object); + MemoAnyObject *PyWeakProxy::get() const { + return reinterpret_cast(m_py_object); } bool PyWeakProxy_Check(PyObject *obj) { return PyObject_TypeCheck(obj, &PyWeakProxyType); } - // MemoObject specialization - template <> PyObject *tryWeakProxy(PyObject *py_obj) + PyObject *tryWeakProxy(PyObject *py_obj) { - assert(PyMemo_Check(py_obj)); - auto py_weak_proxy = PyObject_New(PyWeakProxy, &PyWeakProxyType); + assert(PyAnyMemo_Check(py_obj)); + auto py_weak_proxy = PyObject_New(PyWeakProxy, &PyWeakProxyType); if (!py_weak_proxy) { return nullptr; @@ -62,22 +49,7 @@ namespace db0::python py_weak_proxy->m_py_object = py_obj; return reinterpret_cast(py_weak_proxy); } - - // MemoImmutableObject specialization - template <> PyObject *tryWeakProxy(PyObject *py_obj) - { - assert(PyMemo_Check(py_obj)); - auto py_weak_proxy = PyObject_New(PyWeakProxy, &PyWeakProxyImmutableType); - - if (!py_weak_proxy) { - return nullptr; - } - - Py_INCREF(py_obj); - py_weak_proxy->m_py_object = py_obj; - return reinterpret_cast(py_weak_proxy); - } - + PyObject *tryExpired(PyObject *py_obj) { if (MemoExpiredRef_Check(py_obj)) { diff --git a/src/dbzero/bindings/python/PyWeakProxy.hpp b/src/dbzero/bindings/python/PyWeakProxy.hpp index 8fd3e121..6cfe1ffa 100644 --- a/src/dbzero/bindings/python/PyWeakProxy.hpp +++ b/src/dbzero/bindings/python/PyWeakProxy.hpp @@ -7,34 +7,20 @@ namespace db0::python { - - template + struct PyWeakProxy { PyObject_HEAD PyObject* m_py_object; - // get the underlying memo object - MemoImplT *get() const; + // get the underlying memo object (as MemoAnyObject) + MemoAnyObject *get() const; }; extern PyTypeObject PyWeakProxyType; - // a weak proxy to immutable memo object - extern PyTypeObject PyWeakProxyImmutableType; - - bool PyAnyWeakProxy_Check(PyObject *); - - template - bool PyWeakProxy_Check(PyObject *obj); - - template + + bool PyWeakProxy_Check(PyObject *obj); PyObject *tryWeakProxy(PyObject *); PyObject *tryExpired(PyObject *); - - extern template struct PyWeakProxy; - extern template struct PyWeakProxy; - - extern template bool PyWeakProxy_Check(PyObject *); - extern template bool PyWeakProxy_Check(PyObject *); } \ No newline at end of file From 0e8065de0a3416aaebcdfd6138895d083371d422 Mon Sep 17 00:00:00 2001 From: Wojtek Date: Wed, 5 Nov 2025 20:48:36 +0100 Subject: [PATCH 14/19] WIP: any base --- src/dbzero/bindings/python/Memo.cpp | 29 ++ src/dbzero/bindings/python/Memo.hpp | 10 + src/dbzero/bindings/python/PyTypeManager.cpp | 18 ++ .../object_model/object/ObjectAnyBase.cpp | 169 +++++++++++ .../object_model/object/ObjectAnyBase.hpp | 159 +++++++++++ .../object_model/object/ObjectAnyImpl.cpp | 9 + .../object_model/object/ObjectAnyImpl.hpp | 6 +- .../object_model/object/ObjectImplBase.cpp | 263 ++++-------------- .../object_model/object/ObjectImplBase.hpp | 123 +------- src/dbzero/object_model/object/o_object.hpp | 6 +- 10 files changed, 464 insertions(+), 328 deletions(-) create mode 100644 src/dbzero/object_model/object/ObjectAnyBase.cpp create mode 100644 src/dbzero/object_model/object/ObjectAnyBase.hpp create mode 100644 src/dbzero/object_model/object/ObjectAnyImpl.cpp diff --git a/src/dbzero/bindings/python/Memo.cpp b/src/dbzero/bindings/python/Memo.cpp index dbff6b2a..6f576fea 100644 --- a/src/dbzero/bindings/python/Memo.cpp +++ b/src/dbzero/bindings/python/Memo.cpp @@ -997,4 +997,33 @@ namespace db0::python type->tp_dealloc == reinterpret_cast(MemoObject_del); } + template + bool PyMemo_Check(PyObject *obj) + { + assert(obj); + return obj->ob_type->tp_dealloc == reinterpret_cast(MemoObject_del); + } + + template + bool PyMemoType_Check(PyTypeObject *type) + { + assert(type); + return type->tp_dealloc == reinterpret_cast(MemoObject_del); + } + + template bool PyMemo_Check(PyObject *); + template bool PyMemo_Check(PyObject *); + template bool PyMemoType_Check(PyTypeObject *); + template bool PyMemoType_Check(PyTypeObject *); + template PyObject *MemoObject_DescribeObject(MemoObject *); + template PyObject *MemoObject_DescribeObject(MemoImmutableObject *); + template PyObject *MemoObject_set_prefix(MemoObject *, const char *); + template PyObject *MemoObject_set_prefix(MemoImmutableObject *, const char *); + template PyObject *tryGetAttrAs(MemoObject *, PyObject *, PyTypeObject *); + template PyObject *tryGetAttrAs(MemoImmutableObject *, PyObject *, PyTypeObject *); + template PyObject *tryLoadMemo(MemoObject *, PyObject*, PyObject*, + std::unordered_set *); + template PyObject *tryLoadMemo(MemoImmutableObject *, PyObject*, PyObject*, + std::unordered_set *); + } diff --git a/src/dbzero/bindings/python/Memo.hpp b/src/dbzero/bindings/python/Memo.hpp index be36e5a9..e9fcf125 100644 --- a/src/dbzero/bindings/python/Memo.hpp +++ b/src/dbzero/bindings/python/Memo.hpp @@ -82,5 +82,15 @@ namespace db0::python extern template bool PyMemo_Check(PyObject *); extern template bool PyMemoType_Check(PyTypeObject *); extern template bool PyMemoType_Check(PyTypeObject *); + extern template PyObject *MemoObject_DescribeObject(MemoObject *); + extern template PyObject *MemoObject_DescribeObject(MemoImmutableObject *); + extern template PyObject *MemoObject_set_prefix(MemoObject *, const char *); + extern template PyObject *MemoObject_set_prefix(MemoImmutableObject *, const char *); + extern template PyObject *tryGetAttrAs(MemoObject *, PyObject *, PyTypeObject *); + extern template PyObject *tryGetAttrAs(MemoImmutableObject *, PyObject *, PyTypeObject *); + extern template PyObject *tryLoadMemo(MemoObject *, PyObject*, PyObject*, + std::unordered_set *); + extern template PyObject *tryLoadMemo(MemoImmutableObject *, PyObject*, PyObject*, + std::unordered_set *); } diff --git a/src/dbzero/bindings/python/PyTypeManager.cpp b/src/dbzero/bindings/python/PyTypeManager.cpp index 45a604b4..3520c24f 100644 --- a/src/dbzero/bindings/python/PyTypeManager.cpp +++ b/src/dbzero/bindings/python/PyTypeManager.cpp @@ -188,6 +188,24 @@ namespace db0::python return *type_id; } + + const PyTypeManager::ObjectAnyImpl &PyTypeManager::extractCommonObject(ObjectPtr obj_ptr) const + { + if (PyAnyMemo_Check(obj_ptr)) { + return reinterpret_cast(obj_ptr)->ext(); + } else if (PyWeakProxy_Check(obj_ptr)) { + return reinterpret_cast(obj_ptr)->get()->ext(); + } + THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; + } + + PyTypeManager::ObjectAnyImpl &PyTypeManager::extractMutableCommonObject(ObjectPtr obj_ptr) const + { + if (!PyAnyMemo_Check(obj_ptr)) { + THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; + } + return reinterpret_cast(obj_ptr)->modifyExt(); + } /* FIXME: implement diff --git a/src/dbzero/object_model/object/ObjectAnyBase.cpp b/src/dbzero/object_model/object/ObjectAnyBase.cpp new file mode 100644 index 00000000..fff2deaf --- /dev/null +++ b/src/dbzero/object_model/object/ObjectAnyBase.cpp @@ -0,0 +1,169 @@ +#include "ObjectAnyBase.hpp" + +namespace db0::object_model + +{ + + template + ObjectInitializerManager ObjectAnyBase::m_init_manager; + + template + ObjectAnyBase::ObjectAnyBase(UniqueAddress addr, unsigned int ext_refs) + : m_flags { ObjectOptions::DROPPED } + , m_ext_refs(ext_refs) + , m_unique_address(addr) + { + } + + template + db0::swine_ptr ObjectAnyBase::tryGetFixture() const + { + if (!this->hasInstance()) { + if (isDropped()) { + return {}; + } + // retrieve from the initializer + return m_init_manager.getInitializer(*this).tryGetFixture(); + } + return super_t::tryGetFixture(); + } + + template + db0::swine_ptr ObjectAnyBase::getFixture() const + { + auto fixture = this->tryGetFixture(); + if (!fixture) { + THROWF(db0::InternalException) << "Object is no longer accessible"; + } + return fixture; + } + + template + Memspace &ObjectAnyBase::getMemspace() const { + return *getFixture(); + } + + template + void ObjectAnyBase::setFixture(db0::swine_ptr &new_fixture) + { + if (this->hasInstance()) { + THROWF(db0::InputException) << "set_prefix failed: object already initialized"; + } + + if (!m_init_manager.getInitializer(*this).trySetFixture(new_fixture)) { + // signal problem with PyErr_BadPrefix + auto fixture = this->getFixture(); + LangToolkit::setError(LangToolkit::getTypeManager().getBadPrefixError(), fixture->getUUID()); + } + } + + template + Address ObjectAnyBase::getAddress() const + { + assert(!isDefunct()); + if (!this->hasInstance()) { + THROWF(db0::InternalException) << "Object instance does not exist yet (did you forget to use db0.materialized(self) in constructor ?)"; + } + return super_t::getAddress(); + } + + template + UniqueAddress ObjectAnyBase::getUniqueAddress() const + { + if (this->hasInstance()) { + return super_t::getUniqueAddress(); + } else { + // NOTE: defunct objects don't have a valid address (not assigned yet) + assert(m_flags[ObjectOptions::DROPPED]); + return m_unique_address; + } + } + + template + void ObjectAnyBase::incRef(bool is_tag) + { + if (this->hasInstance()) { + super_t::incRef(is_tag); + } else { + // incRef with the initializer + super_t::m_init_manager.getInitializer(*this).incRef(is_tag); + } + } + + template + bool ObjectAnyBase::decRef(bool is_tag) + { + // this operation is a potentially silent mutation + _touch(); + super_t::decRef(is_tag); + return !hasRefs(); + } + + template + bool ObjectAnyBase::hasRefs() const + { + assert(this->hasInstance()); + return (*this)->hasRefs(); + } + + template + bool ObjectAnyBase::hasAnyRefs() const { + return (*this)->hasAnyRefs(); + } + + template + bool ObjectAnyBase::hasTagRefs() const { + return this->hasInstance() && (*this)->m_header.m_ref_counter.getFirst() > 0; + } + + template + void ObjectAnyBase::touch() + { + if (this->hasInstance() && !this->isDefunct()) { + // NOTE: for already modified and small objects we may skip "touch" + if (!super_t::isModified() || this->span() > 1) { + // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag + // this is because the actual change may be missed if performed on a different-then the 1st DP + this->_touch(); + } + } + } + + template + void ObjectAnyBase::_touch() + { + if (!m_touched) { + // mark the 1st byte of the object as modified (forced-diff) + // this is always the 1st DP occupied by the object + this->modify(0, 1); + m_touched = true; + } + } + + template + void ObjectAnyBase::addExtRef() const { + ++m_ext_refs; + } + + template + void ObjectAnyBase::removeExtRef() const + { + assert(m_ext_refs > 0); + --m_ext_refs; + } + + template + void ObjectAnyBase::setDefunct() const { + m_flags.set(ObjectOptions::DEFUNCT); + } + + template + Class &ObjectAnyBase::getType() { + return this->m_type ? *this->m_type : m_init_manager.getInitializer(*this).getClass(); + } + + template class ObjectAnyBase; + template class ObjectAnyBase; + template class ObjectAnyBase; + +} diff --git a/src/dbzero/object_model/object/ObjectAnyBase.hpp b/src/dbzero/object_model/object/ObjectAnyBase.hpp new file mode 100644 index 00000000..e116c181 --- /dev/null +++ b/src/dbzero/object_model/object/ObjectAnyBase.hpp @@ -0,0 +1,159 @@ +#pragma once + +#include +#include +#include +#include +#include +#include "ValueTable.hpp" +#include "ObjectInitializer.hpp" +#include +#include +#include +#include +#include "o_object.hpp" +#include "o_immutable_object.hpp" + +namespace db0 + +{ + + class Fixture; + +} + +namespace db0::object_model + +{ + + class Class; + class ObjectAnyImpl; + using Fixture = db0::Fixture; + + enum class ObjectOptions: std::uint8_t + { + // the dbzero instance has been deleted + DROPPED = 0x01, + // object is defunct - e.g. due to exception on __init__ + DEFUNCT = 0x02 + }; + + using ObjectFlags = db0::FlagSet; + // NOTE: Object instances are created within the implementation specific realm_id (e.g. =1 for o_object) + template using ObjectVType = db0::v_object; + + template + class ObjectAnyBase: public db0::ObjectBase, StorageClass::OBJECT_REF> + { + public: + static constexpr unsigned char REALM_ID = T::REALM_ID; + using super_t = db0::ObjectBase, StorageClass::OBJECT_REF>; + using LangToolkit = LangConfig::LangToolkit; + using ObjectPtr = typename LangToolkit::ObjectPtr; + using TypeObjectPtr = typename LangToolkit::TypeObjectPtr; + using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; + using TypeManager = typename LangToolkit::TypeManager; + using ObjectStem = ObjectVType; + using TypeInitializer = ObjectInitializer::TypeInitializer; + + db0::swine_ptr tryGetFixture() const; + db0::swine_ptr getFixture() const; + + Memspace &getMemspace() const; + + inline std::shared_ptr getClassPtr() const { + return this->m_type ? this->m_type : m_init_manager.getInitializer(*this).getClassPtr(); + } + + inline const Class &getType() const { + return this->m_type ? *this->m_type : m_init_manager.getInitializer(*this).getClass(); + } + + Class &getType(); + + /** + * Change fixture of the uninitialized object + * Object must not have any members yet either + */ + void setFixture(db0::swine_ptr &); + + /** + * The overloaded incRef implementation is provided to also handle non-fully initialized objects + */ + void incRef(bool is_tag); + bool hasRefs() const; + // check for any refs (including auto-assigned type tags) + bool hasAnyRefs() const; + + // check if any references from tags exist (i.e. are any tags assigned) + bool hasTagRefs() const; + + // @return true if reference count was decremented to zero + bool decRef(bool is_tag); + + Address getAddress() const; + UniqueAddress getUniqueAddress() const; + + // NOTE: the operation is marked const because the dbzero state is not affected + void setDefunct() const; + + inline bool isDropped() const { + return m_flags.test(ObjectOptions::DROPPED); + } + + inline bool isDefunct() const { + return m_flags.test(ObjectOptions::DEFUNCT); + } + + // is dropped or defunct + inline bool isDead() const { + return m_flags.any( + static_cast(ObjectOptions::DROPPED) | static_cast(ObjectOptions::DEFUNCT) + ); + } + + // the member called to indicate the object mutation + void touch(); + + void addExtRef() const; + void removeExtRef() const; + + inline std::uint32_t getExtRefs() const { + return m_ext_refs; + } + + protected: + static ObjectInitializerManager m_init_manager; + // Class will only be assigned after initialization + std::shared_ptr m_type; + mutable ObjectFlags m_flags; + // reference counter for inner references from language objects + // NOTE: inner references are held by internal dbzero buffers (e.g. TagIndex) + // see also PyEXT_INCREF / PyEXT_DECREF + mutable std::uint32_t m_ext_refs = 0; + // A flag indicating that object's silent mutation has already been reflected + // with the underlying MemLock / ResourceLock + // NOTE: by silent mutation we mean a mutation that does not change data (e.g. +refcount(+-1) + (refcount-1)) + mutable bool m_touched = false; + // NOTE: member assigned only to dropped objects (see replaceWithNull) + // so that we can retrieve the address of the dropped instance after it has been destroyed + const UniqueAddress m_unique_address; + + template ObjectAnyBase(Args&&... args) + : super_t(std::forward(args)...) + { + } + + // As a dropped object + ObjectAnyBase(UniqueAddress addr, unsigned int ext_refs); + + void _touch(); + }; + + extern template class ObjectAnyBase; + extern template class ObjectAnyBase; + extern template class ObjectAnyBase; + +} + +DECLARE_ENUM_VALUES(db0::object_model::ObjectOptions, 2) \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectAnyImpl.cpp b/src/dbzero/object_model/object/ObjectAnyImpl.cpp new file mode 100644 index 00000000..3238ad66 --- /dev/null +++ b/src/dbzero/object_model/object/ObjectAnyImpl.cpp @@ -0,0 +1,9 @@ +#include "ObjectAnyImpl.hpp" +#include "o_object.hpp" + +DEFINE_ENUM_VALUES(db0::object_model::ObjectOptions, "DROPPED", "DEFUNCT") + +namespace db0::object_model + +{ +} diff --git a/src/dbzero/object_model/object/ObjectAnyImpl.hpp b/src/dbzero/object_model/object/ObjectAnyImpl.hpp index c27ccf2c..1b861cfb 100644 --- a/src/dbzero/object_model/object/ObjectAnyImpl.hpp +++ b/src/dbzero/object_model/object/ObjectAnyImpl.hpp @@ -1,6 +1,6 @@ #pragma once -#include "ObjectImplBase.hpp" +#include "ObjectAnyBase.hpp" #include "o_object.hpp" namespace db0::object_model @@ -9,11 +9,11 @@ namespace db0::object_model // NOTE: ObjectAnyImpl is for reinterpret_cast purposes only // it allows accessing Object or ObjectImmutableImpl instances under a common base type - class ObjectAnyImpl: public ObjectImplBase + class ObjectAnyImpl: public ObjectAnyBase { public: static constexpr unsigned char REALM_ID = o_object_base::REALM_ID; - using super_t = ObjectImplBase; + using super_t = ObjectAnyBase; protected: friend super_t; diff --git a/src/dbzero/object_model/object/ObjectImplBase.cpp b/src/dbzero/object_model/object/ObjectImplBase.cpp index 5ef2a4fa..89919478 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.cpp +++ b/src/dbzero/object_model/object/ObjectImplBase.cpp @@ -11,18 +11,10 @@ #include #include -DEFINE_ENUM_VALUES(db0::object_model::ObjectOptions, "DROPPED", "DEFUNCT") - namespace db0::object_model { - - template class ObjectImplBase; - template class ObjectImplBase; - template - ObjectInitializerManager ObjectImplBase::m_init_manager; - FlagSet getAccessOptions(const Class &type) { return type.isNoCache() ? FlagSet { AccessOptions::no_cache } : FlagSet {}; } @@ -37,24 +29,22 @@ namespace db0::object_model template ObjectImplBase::ObjectImplBase(UniqueAddress addr, unsigned int ext_refs) - : m_flags { ObjectOptions::DROPPED } - , m_ext_refs(ext_refs) - , m_unique_address(addr) + : super_t(addr, ext_refs) { } - + template ObjectImplBase::ObjectImplBase(std::shared_ptr db0_class) { // prepare for initialization - m_init_manager.addInitializer(*this, db0_class); + super_t::m_init_manager.addInitializer(*this, db0_class); } - + template ObjectImplBase::ObjectImplBase(TypeInitializer &&type_initializer) { // prepare for initialization - m_init_manager.addInitializer(*this, std::move(type_initializer)); + super_t::m_init_manager.addInitializer(*this, std::move(type_initializer)); } template @@ -62,9 +52,9 @@ namespace db0::object_model std::pair ref_counts, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset) : super_t(fixture, type->getClassRef(), ref_counts, safeCast(type->getNumBases() + 1, "Too many base classes"), pos_vt_data, pos_vt_offset, nullptr, nullptr, - getAccessOptions(*type)) - , m_type(type) + getAccessOptions(*type)) { + this->m_type = type; } template @@ -72,12 +62,12 @@ namespace db0::object_model : super_t(typename super_t::tag_from_address(), fixture, address, access_mode) { } - + template ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type) - : super_t(typename super_t::tag_from_stem(), fixture, std::move(stem)) - , m_type(type) + : super_t(typename super_t::tag_from_stem(), fixture, std::move(stem)) { + this->m_type = type; assert(hasValidClassRef()); } @@ -103,7 +93,7 @@ namespace db0::object_model this->unregister(); if (!this->hasInstance()) { // release initializer if it exists, object not created - m_init_manager.tryCloseInitializer(*this); + super_t::m_init_manager.tryCloseInitializer(*this); } } @@ -122,7 +112,7 @@ namespace db0::object_model Address address, std::uint16_t instance_id, AccessFlags access_mode) { std::size_t size_of; - if (!fixture->isAddressValid(address, REALM_ID, &size_of)) { + if (!fixture->isAddressValid(address, super_t::REALM_ID, &size_of)) { return {}; } // Unload from a verified address @@ -149,30 +139,31 @@ namespace db0::object_model void ObjectImplBase::postInit(FixtureLock &fixture) { if (!this->hasInstance()) { - auto &initializer = m_init_manager.getInitializer(*this); + auto &initializer = super_t::m_init_manager.getInitializer(*this); PosVT::Data pos_vt_data; unsigned int pos_vt_offset = 0; auto index_vt_data = initializer.getData(pos_vt_data, pos_vt_offset); // place object in the same fixture as its class // construct the dbzero instance & assign to self - m_type = initializer.getClassPtr(); - assert(m_type); + this->m_type = initializer.getClassPtr(); + assert(this->m_type); - super_t::init(*fixture, m_type->getClassRef(), initializer.getRefCounts(), - safeCast(m_type->getNumBases() + 1, "Too many base classes"), + auto &type = *this->m_type; + super_t::init(*fixture, type.getClassRef(), initializer.getRefCounts(), + safeCast(type.getNumBases() + 1, "Too many base classes"), pos_vt_data, pos_vt_offset, index_vt_data.first, index_vt_data.second, - getAccessOptions(*m_type) + getAccessOptions(type) ); // reference associated class - m_type->incRef(false); - m_type->updateSchema(pos_vt_offset, pos_vt_data.m_types, pos_vt_data.m_values); - m_type->updateSchema(index_vt_data.first, index_vt_data.second); + type.incRef(false); + type.updateSchema(pos_vt_offset, pos_vt_data.m_types, pos_vt_data.m_values); + type.updateSchema(index_vt_data.first, index_vt_data.second); // bind singleton address (now that instance exists) - if (m_type->isSingleton()) { - m_type->setSingletonAddress(*this); + if (type.isSingleton()) { + type.setSingletonAddress(*this); } initializer.close(); } @@ -211,7 +202,7 @@ namespace db0::object_model template void ObjectImplBase::removePreInit(const char *field_name) const { - auto &initializer = m_init_manager.getInitializer(*this); + auto &initializer = super_t::m_init_manager.getInitializer(*this); auto &type = initializer.getClass(); // Find an already existing field index @@ -248,7 +239,7 @@ namespace db0::object_model return; } - auto &initializer = m_init_manager.getInitializer(*this); + auto &initializer = super_t::m_init_manager.getInitializer(*this); auto fixture = initializer.getFixture(); auto &type = initializer.getClass(); auto [type_id, storage_class] = recognizeType(*fixture, obj_ptr); @@ -302,7 +293,7 @@ namespace db0::object_model unrefMember(*fixture, old_storage_class, pos_vt.values()[pos]); // mark member as unreferenced by assigning storage class pos_vt.set(pos, storage_class, {}); - m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(old_storage_class)); + this->m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(old_storage_class)); } else { assert(fidelity == 2); auto value = pos_vt.values()[pos]; @@ -320,10 +311,10 @@ namespace db0::object_model lofi_store<2>::fromValue(value).reset(offset); } pos_vt.set(pos, old_storage_class, value); - m_type->removeFromSchema(field_id, fidelity, old_type_id); + this->m_type->removeFromSchema(field_id, fidelity, old_type_id); } } - + template void ObjectImplBase::unrefIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, StorageClass storage_class, unsigned int fidelity) @@ -334,7 +325,7 @@ namespace db0::object_model unrefMember(*fixture, index_vt.xvalues()[index_vt_pos]); // mark member as unreferenced by assigning storage class index_vt.set(index_vt_pos, storage_class, {}); - m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(old_storage_class)); + this->m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(old_storage_class)); } else { assert(fidelity == 2); auto value = index_vt.xvalues()[index_vt_pos].m_value; @@ -350,7 +341,7 @@ namespace db0::object_model lofi_store<2>::fromValue(value).reset(offset); } index_vt.set(index_vt_pos, old_storage_class, value); - m_type->removeFromSchema(field_id, fidelity, old_type_id); + this->m_type->removeFromSchema(field_id, fidelity, old_type_id); } } @@ -439,16 +430,16 @@ namespace db0::object_model template std::pair ObjectImplBase::findField(const char *name) const { - if (isDropped()) { + if (this->isDropped()) { // defunct objects should not be accessed - assert(!isDefunct()); + assert(!this->isDefunct()); THROWF(db0::InputException) << "Object does not exist"; } - auto class_ptr = m_type.get(); + auto class_ptr = this->m_type.get(); if (!class_ptr) { // retrieve class from the initializer - class_ptr = &m_init_manager.getInitializer(*this).getClass(); + class_ptr = &super_t::m_init_manager.getInitializer(*this).getClass(); } assert(class_ptr); @@ -572,7 +563,7 @@ namespace db0::object_model { auto obj = tryGet(field_name); if (!obj) { - if (isDropped()) { + if (this->isDropped()) { THROWF(db0::InputException) << "Object is no longer accessible"; } THROWF(db0::InputException) << "Attribute not found: " << field_name; @@ -617,7 +608,7 @@ namespace db0::object_model auto loc = field_info.first.getIndexAndOffset(); if (!this->hasInstance()) { // try retrieving from initializer - auto initializer_ptr = m_init_manager.findInitializer(*this); + auto initializer_ptr = super_t::m_init_manager.findInitializer(*this); if (!initializer_ptr) { find_result = { false, false }; return true; @@ -673,52 +664,24 @@ namespace db0::object_model return { false, false }; } - template - db0::swine_ptr ObjectImplBase::tryGetFixture() const - { - if (!this->hasInstance()) { - if (isDropped()) { - return {}; - } - // retrieve from the initializer - return m_init_manager.getInitializer(*this).tryGetFixture(); - } - return super_t::tryGetFixture(); - } - - template - db0::swine_ptr ObjectImplBase::getFixture() const - { - auto fixture = this->tryGetFixture(); - if (!fixture) { - THROWF(db0::InternalException) << "Object is no longer accessible"; - } - return fixture; - } - - template - Memspace &ObjectImplBase::getMemspace() const { - return *getFixture(); - } - template void ObjectImplBase::setType(std::shared_ptr type) { - assert(!m_type); - m_type = type; + assert(!this->m_type); + this->m_type = type; assert(hasValidClassRef()); } template void ObjectImplBase::setTypeWithHint(std::shared_ptr type_hint) { - assert(!m_type); + assert(!this->m_type); assert(type_hint); assert(this->hasInstance()); if (type_hint->getClassRef() == (*this)->getClassRef()) { - m_type = type_hint; + this->m_type = type_hint; } else { - m_type = unloadType(); + this->m_type = unloadType(); } } @@ -787,7 +750,7 @@ namespace db0::object_model { if (this->hasInstance()) { // associated class type (may require unloading) - auto type = m_type; + auto type = this->m_type; if (!type) { // retrieve type from the initializer type = std::const_pointer_cast(unloadType()); @@ -824,11 +787,6 @@ namespace db0::object_model } } - template - Class &ObjectImplBase::getType() { - return m_type ? *m_type : m_init_manager.getInitializer(*this).getClass(); - } - template void ObjectImplBase::getMembersFrom(const Class &this_type, unsigned int index, StorageClass storage_class, Value value, std::unordered_set &result) const @@ -975,43 +933,6 @@ namespace db0::object_model return true; } - template - void ObjectImplBase::incRef(bool is_tag) - { - if (this->hasInstance()) { - super_t::incRef(is_tag); - } else { - // incRef with the initializer - m_init_manager.getInitializer(*this).incRef(is_tag); - } - } - - template - bool ObjectImplBase::decRef(bool is_tag) - { - // this operation is a potentially silent mutation - _touch(); - super_t::decRef(is_tag); - return !hasRefs(); - } - - template - bool ObjectImplBase::hasRefs() const - { - assert(this->hasInstance()); - return (*this)->hasRefs(); - } - - template - bool ObjectImplBase::hasAnyRefs() const { - return (*this)->hasAnyRefs(); - } - - template - bool ObjectImplBase::hasTagRefs() const { - return this->hasInstance() && (*this)->m_header.m_ref_counter.getFirst() > 0; - } - template bool ObjectImplBase::tryEqualToImpl(const ObjectImplBase &other, bool &result) const { @@ -1021,7 +942,7 @@ namespace db0::object_model if (this->isDefunct() || other.isDefunct()) { // defunct objects should not be compared - assert(!isDefunct()); + assert(!this->isDefunct()); THROWF(db0::InputException) << "Object does not exist"; } @@ -1078,47 +999,33 @@ namespace db0::object_model void ObjectImplBase::moveTo(db0::swine_ptr &) { throw std::runtime_error("Not implemented"); } - - template - void ObjectImplBase::setFixture(db0::swine_ptr &new_fixture) - { - if (this->hasInstance()) { - THROWF(db0::InputException) << "set_prefix failed: object already initialized"; - } - - if (!m_init_manager.getInitializer(*this).trySetFixture(new_fixture)) { - // signal problem with PyErr_BadPrefix - auto fixture = this->getFixture(); - LangToolkit::setError(LangToolkit::getTypeManager().getBadPrefixError(), fixture->getUUID()); - } - } - + template void ObjectImplBase::detach() const { - m_type->detach(); + this->m_type->detach(); // invalidate since detach is not supported by the MorphingBIndex - m_kv_index = nullptr; + this->m_kv_index = nullptr; super_t::detach(); } template void ObjectImplBase::commit() const { - m_type->commit(); + this->m_type->commit(); if (m_kv_index) { m_kv_index->commit(); } super_t::commit(); // reset the silent-mutation flag - m_touched = false; + this->m_touched = false; } template void ObjectImplBase::unrefMember(db0::swine_ptr &fixture, StorageClass type, Value value) const { db0::object_model::unrefMember(fixture, type, value); } - + template void ObjectImplBase::unrefMember(db0::swine_ptr &fixture, XValue value) const { db0::object_model::unrefMember(fixture, value.m_type, value.m_value); @@ -1131,33 +1038,11 @@ namespace db0::object_model return getClassFactory(*fixture).getTypeByClassRef((*this)->getClassRef()).m_class; } - template - Address ObjectImplBase::getAddress() const - { - assert(!isDefunct()); - if (!this->hasInstance()) { - THROWF(db0::InternalException) << "Object instance does not exist yet (did you forget to use db0.materialized(self) in constructor ?)"; - } - return super_t::getAddress(); - } - - template - UniqueAddress ObjectImplBase::getUniqueAddress() const - { - if (this->hasInstance()) { - return super_t::getUniqueAddress(); - } else { - // NOTE: defunct objects don't have a valid address (not assigned yet) - assert(m_flags[ObjectOptions::DROPPED]); - return m_unique_address; - } - } - template bool ObjectImplBase::hasValidClassRef() const { - if (this->hasInstance() && m_type) { - return (*this)->getClassRef() == m_type->getClassRef(); + if (this->hasInstance() && this->m_type) { + return (*this)->getClassRef() == this->m_type->getClassRef(); } return true; } @@ -1173,45 +1058,7 @@ namespace db0::object_model return getClassFactory(fixture).getTypeByClassRef(class_ref).m_class; } - template - void ObjectImplBase::setDefunct() const { - m_flags.set(ObjectOptions::DEFUNCT); - } - - template - void ObjectImplBase::touch() - { - if (this->hasInstance() && !isDefunct()) { - // NOTE: for already modified and small objects we may skip "touch" - if (!super_t::isModified() || this->span() > 1) { - // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag - // this is because the actual change may be missed if performed on a different-then the 1st DP - _touch(); - } - } - } - - template - void ObjectImplBase::_touch() - { - if (!m_touched) { - // mark the 1st byte of the object as modified (forced-diff) - // this is always the 1st DP occupied by the object - this->modify(0, 1); - m_touched = true; - } - } - - template - void ObjectImplBase::addExtRef() const { - ++m_ext_refs; - } - - template - void ObjectImplBase::removeExtRef() const - { - assert(m_ext_refs > 0); - --m_ext_refs; - } - + template class ObjectImplBase; + template class ObjectImplBase; + } \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectImplBase.hpp b/src/dbzero/object_model/object/ObjectImplBase.hpp index 430cbf53..ee70d232 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.hpp +++ b/src/dbzero/object_model/object/ObjectImplBase.hpp @@ -1,16 +1,10 @@ #pragma once +#include "ObjectAnyBase.hpp" #include -#include #include #include #include -#include "ValueTable.hpp" -#include "ObjectInitializer.hpp" -#include -#include -#include -#include #include "o_object.hpp" #include "o_immutable_object.hpp" @@ -31,31 +25,18 @@ namespace db0::object_model class ObjectImmutableImpl; using Fixture = db0::Fixture; - enum class ObjectOptions: std::uint8_t - { - // the dbzero instance has been deleted - DROPPED = 0x01, - // object is defunct - e.g. due to exception on __init__ - DEFUNCT = 0x02 - }; - struct FieldLayout { std::vector m_pos_vt_fields; std::vector > m_index_vt_fields; std::vector > m_kv_index_fields; }; - - using ObjectFlags = db0::FlagSet; - // NOTE: Object instances are created within the implementation specific realm_id (e.g. =1 for o_object) - template using ObjectVType = db0::v_object; template - class ObjectImplBase: public db0::ObjectBase, StorageClass::OBJECT_REF> + class ObjectImplBase: public ObjectAnyBase { public: - static constexpr unsigned char REALM_ID = T::REALM_ID; - using super_t = db0::ObjectBase, StorageClass::OBJECT_REF>; + using super_t = ObjectAnyBase; using LangToolkit = LangConfig::LangToolkit; using ObjectPtr = typename LangToolkit::ObjectPtr; using TypeObjectPtr = typename LangToolkit::TypeObjectPtr; @@ -116,23 +97,7 @@ namespace db0::object_model ObjectSharedPtr tryGet(const char *field_name) const; ObjectSharedPtr tryGetAs(const char *field_name, TypeObjectPtr) const; ObjectSharedPtr get(const char *field_name) const; - - inline std::shared_ptr getClassPtr() const { - return m_type ? m_type : m_init_manager.getInitializer(*this).getClassPtr(); - } - - inline const Class &getType() const { - return m_type ? *m_type : m_init_manager.getInitializer(*this).getClass(); - } - - Class &getType(); - - db0::swine_ptr tryGetFixture() const; - - db0::swine_ptr getFixture() const; - - Memspace &getMemspace() const; - + // Get description of the field layout FieldLayout getFieldLayout() const; @@ -144,21 +109,7 @@ namespace db0::object_model // get dbzero member / member names assigned to this object std::unordered_set getMembers() const; - - /** - * The overloaded incRef implementation is provided to also handle non-fully initialized objects - */ - void incRef(bool is_tag); - bool hasRefs() const; - // check for any refs (including auto-assigned type tags) - bool hasAnyRefs() const; - - // check if any references from tags exist (i.e. are any tags assigned) - bool hasTagRefs() const; - - // @return true if reference count was decremented to zero - bool decRef(bool is_tag); - + // Binary (shallow) compare 2 objects or 2 versions of the same memo object (e.g. from different snapshots) // NOTE: ref-counts are not compared (only user-assigned members) // @return true if objects are identical @@ -170,68 +121,15 @@ namespace db0::object_model */ void moveTo(db0::swine_ptr &); - /** - * Change fixture of the uninitialized object - * Object must not have any members yet either - */ - void setFixture(db0::swine_ptr &); - void detach() const; - void commit() const; - Address getAddress() const; - UniqueAddress getUniqueAddress() const; - - // NOTE: the operation is marked const because the dbzero state is not affected - void setDefunct() const; - - inline bool isDropped() const { - return m_flags.test(ObjectOptions::DROPPED); - } - - inline bool isDefunct() const { - return m_flags.test(ObjectOptions::DEFUNCT); - } - - // is dropped or defunct - inline bool isDead() const { - return m_flags.any( - static_cast(ObjectOptions::DROPPED) | static_cast(ObjectOptions::DEFUNCT) - ); - } - // FieldID, is_init_var, fidelity std::pair findField(const char *name) const; - // the member called to indicate the object mutation - void touch(); - - void addExtRef() const; - void removeExtRef() const; - - inline std::uint32_t getExtRefs() const { - return m_ext_refs; - } - - protected: - // Class will only be assigned after initialization - std::shared_ptr m_type; + protected: // local kv-index instance cache (created at first use) mutable std::unique_ptr m_kv_index; - static ObjectInitializerManager m_init_manager; - mutable ObjectFlags m_flags; - // reference counter for inner references from language objects - // NOTE: inner references are held by internal dbzero buffers (e.g. TagIndex) - // see also PyEXT_INCREF / PyEXT_DECREF - mutable std::uint32_t m_ext_refs = 0; - // A flag indicating that object's silent mutation has already been reflected - // with the underlying MemLock / ResourceLock - // NOTE: by silent mutation we mean a mutation that does not change data (e.g. +refcount(+-1) + (refcount-1)) - mutable bool m_touched = false; - // NOTE: member assigned only to dropped objects (see replaceWithNull) - // so that we can retrieve the address of the dropped instance after it has been destroyed - const UniqueAddress m_unique_address; void setType(std::shared_ptr); // adjusts to actual type if the type hint is a base class @@ -265,7 +163,7 @@ namespace db0::object_model std::pair tryGetLoc(FieldID) const; inline ObjectInitializer *tryGetInitializer() const { - return m_type ? static_cast(nullptr) : &m_init_manager.getInitializer(*this); + return this->m_type ? static_cast(nullptr) : &super_t::m_init_manager.getInitializer(*this); } void dropMembers(db0::swine_ptr &, Class &) const; @@ -288,8 +186,7 @@ namespace db0::object_model bool hasValidClassRef() const; // try retrieving member as XValue - std::optional tryGetX(const char *field_name) const; - void _touch(); + std::optional tryGetX(const char *field_name) const; // Unreference value // NOTE: storage_class to be assigned can either be DELETED or UNDEFINED @@ -311,7 +208,5 @@ namespace db0::object_model extern template class ObjectImplBase; extern template class ObjectImplBase; - + } - -DECLARE_ENUM_VALUES(db0::object_model::ObjectOptions, 2) \ No newline at end of file diff --git a/src/dbzero/object_model/object/o_object.hpp b/src/dbzero/object_model/object/o_object.hpp index 275f72cc..3a0a5fcd 100644 --- a/src/dbzero/object_model/object/o_object.hpp +++ b/src/dbzero/object_model/object/o_object.hpp @@ -27,7 +27,7 @@ DB0_PACKED_BEGIN static std::size_t measure(); static std::size_t measure(std::pair); - + static std::size_t sizeOf(); template static std::size_t safeSizeOf(BufT buf) { @@ -39,7 +39,7 @@ DB0_PACKED_BEGIN bool hasAnyRefs() const; }; DB0_PACKED_END - + DB0_PACKED_BEGIN class DB0_PACKED_ATTR o_object: public db0::o_ext { @@ -79,7 +79,7 @@ DB0_PACKED_BEGIN (PosVT::type()) (packed_int32::type()) (IndexVT::type()); - } + } void incRef(bool is_tag); bool hasRefs() const; From 758f82f04c16f0e02d7e3e3417b8a526069cbfd6 Mon Sep 17 00:00:00 2001 From: Wojtek Date: Wed, 5 Nov 2025 20:50:42 +0100 Subject: [PATCH 15/19] compile fix --- src/dbzero/object_model/object/ObjectAnyBase.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dbzero/object_model/object/ObjectAnyBase.cpp b/src/dbzero/object_model/object/ObjectAnyBase.cpp index fff2deaf..c7caed94 100644 --- a/src/dbzero/object_model/object/ObjectAnyBase.cpp +++ b/src/dbzero/object_model/object/ObjectAnyBase.cpp @@ -86,10 +86,10 @@ namespace db0::object_model super_t::incRef(is_tag); } else { // incRef with the initializer - super_t::m_init_manager.getInitializer(*this).incRef(is_tag); + m_init_manager.getInitializer(*this).incRef(is_tag); } } - + template bool ObjectAnyBase::decRef(bool is_tag) { From 396de55b2d748436a9bc796b99eb1a97d78ca02c Mon Sep 17 00:00:00 2001 From: Wojtek Date: Thu, 6 Nov 2025 11:59:32 +0100 Subject: [PATCH 16/19] Save work --- src/dbzero/bindings/python/PyInternalAPI.cpp | 5 ++- src/dbzero/bindings/python/PyInternalAPI.hpp | 5 ++- src/dbzero/bindings/python/PyToolkit.hpp | 4 +++ src/dbzero/bindings/python/PyTypeManager.cpp | 35 ++++++++----------- src/dbzero/bindings/python/PyTypeManager.hpp | 24 +++++++++---- .../bindings/python/shared_py_object.cpp | 10 +++--- src/dbzero/core/vspace/v_ptr.hpp | 24 +++++++++---- .../object_model/index/IndexBuilder.hpp | 6 ++-- .../object_model/object/ObjectAnyBase.cpp | 12 ++----- .../object_model/object/ObjectAnyBase.hpp | 8 ++--- .../object_model/object/ObjectImplBase.cpp | 7 ++++ .../object_model/object/ObjectImplBase.hpp | 4 +++ src/dbzero/object_model/object/o_object.cpp | 16 ++++----- src/dbzero/object_model/object/o_object.hpp | 12 ++----- src/dbzero/object_model/tags/TagIndex.cpp | 20 +++++------ src/dbzero/object_model/tags/TagIndex.hpp | 2 +- src/dbzero/object_model/value/Member.cpp | 15 +++++--- src/dbzero/object_model/value/Member.hpp | 7 ++-- .../object_model/value/StorageClass.cpp | 11 +++--- 19 files changed, 125 insertions(+), 102 deletions(-) diff --git a/src/dbzero/bindings/python/PyInternalAPI.cpp b/src/dbzero/bindings/python/PyInternalAPI.cpp index 46bf17e6..6eb91fd6 100644 --- a/src/dbzero/bindings/python/PyInternalAPI.cpp +++ b/src/dbzero/bindings/python/PyInternalAPI.cpp @@ -902,5 +902,8 @@ namespace db0::python Py_RETURN_NONE; } - + + template PyObject *getMaterializedMemoObject(MemoObject *); + template PyObject *getMaterializedMemoObject(MemoImmutableObject *); + } \ No newline at end of file diff --git a/src/dbzero/bindings/python/PyInternalAPI.hpp b/src/dbzero/bindings/python/PyInternalAPI.hpp index 3c0bb5b2..52320513 100644 --- a/src/dbzero/bindings/python/PyInternalAPI.hpp +++ b/src/dbzero/bindings/python/PyInternalAPI.hpp @@ -234,6 +234,9 @@ namespace db0::python #endif PyObject *tryAssign(PyObject *targets, PyObject *key_values); - + + extern template PyObject *getMaterializedMemoObject(MemoObject *); + extern template PyObject *getMaterializedMemoObject(MemoImmutableObject *); + } diff --git a/src/dbzero/bindings/python/PyToolkit.hpp b/src/dbzero/bindings/python/PyToolkit.hpp index bf7987c0..9adcace0 100644 --- a/src/dbzero/bindings/python/PyToolkit.hpp +++ b/src/dbzero/bindings/python/PyToolkit.hpp @@ -248,6 +248,10 @@ namespace db0::python // Acquire the interpreter's GIL lock // NOTE: returns nullptr if Python not initialized / defunct static std::unique_ptr ensureLocked(); + + // decRef operation for memo objects + // @return true if reference count was decremented to zero (!hasRefs) + static bool decRefMemo(bool is_tag, ObjectPtr py_object); private: static PyWorkspace m_py_workspace; diff --git a/src/dbzero/bindings/python/PyTypeManager.cpp b/src/dbzero/bindings/python/PyTypeManager.cpp index 3520c24f..190442a2 100644 --- a/src/dbzero/bindings/python/PyTypeManager.cpp +++ b/src/dbzero/bindings/python/PyTypeManager.cpp @@ -189,7 +189,7 @@ namespace db0::python return *type_id; } - const PyTypeManager::ObjectAnyImpl &PyTypeManager::extractCommonObject(ObjectPtr obj_ptr) const + const PyTypeManager::ObjectAnyImpl &PyTypeManager::extractAnyObject(ObjectPtr obj_ptr) const { if (PyAnyMemo_Check(obj_ptr)) { return reinterpret_cast(obj_ptr)->ext(); @@ -199,7 +199,7 @@ namespace db0::python THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; } - PyTypeManager::ObjectAnyImpl &PyTypeManager::extractMutableCommonObject(ObjectPtr obj_ptr) const + PyTypeManager::ObjectAnyImpl &PyTypeManager::extractMutableAnyObject(ObjectPtr obj_ptr) const { if (!PyAnyMemo_Check(obj_ptr)) { THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; @@ -220,7 +220,9 @@ namespace db0::python } THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; } - + */ + + /* const db0::object_model::Object &PyTypeManager::extractObject(ObjectPtr memo_ptr) const { if (PyMemo_Check(memo_ptr)) { @@ -232,30 +234,21 @@ namespace db0::python } */ - db0::object_model::Object &PyTypeManager::extractMutableObject(ObjectPtr memo_ptr) const - { - if (!PyMemo_Check(memo_ptr)) { - THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; - } - return reinterpret_cast(memo_ptr)->modifyExt(); - } - - /* FIXME: - const db0::object_model::Object *PyTypeManager::tryExtractObject(ObjectPtr memo_ptr) const + const PyTypeManager::ObjectAnyImpl *PyTypeManager::tryExtractObject(ObjectPtr memo_ptr) const { - if (!PyMemo_Check(memo_ptr)) { + if (!PyAnyMemo_Check(memo_ptr)) { return nullptr; } - return &reinterpret_cast(memo_ptr)->ext(); - } - */ - - db0::object_model::Object *PyTypeManager::tryExtractMutableObject(ObjectPtr memo_ptr) const + return &reinterpret_cast(memo_ptr)->ext(); + } + + template + typename MemoImplT::ExtT *PyTypeManager::tryExtractMutableObject(ObjectPtr memo_ptr) const { - if (!PyMemo_Check(memo_ptr)) { + if (!PyMemo_Check(memo_ptr)) { return nullptr; } - return &reinterpret_cast(memo_ptr)->modifyExt(); + return &reinterpret_cast(memo_ptr)->modifyExt(); } const db0::object_model::List &PyTypeManager::extractList(ObjectPtr list_ptr) const diff --git a/src/dbzero/bindings/python/PyTypeManager.hpp b/src/dbzero/bindings/python/PyTypeManager.hpp index 620ee399..16493bf3 100644 --- a/src/dbzero/bindings/python/PyTypeManager.hpp +++ b/src/dbzero/bindings/python/PyTypeManager.hpp @@ -100,14 +100,17 @@ namespace db0::python // Extracts reference to Object or ObjectImmutableImpl from a memo instance template const typename MemoImplT::ExtT &extractObject(ObjectPtr memo_ptr) const; + template + typename MemoImplT::ExtT &extractMutableObject(ObjectPtr memo_ptr) const; + + template + typename MemoImplT::ExtT *tryExtractMutableObject(ObjectPtr memo_ptr) const; // Extracts reference to common object part from a memo instance - const ObjectAnyImpl &extractCommonObject(ObjectPtr) const; - ObjectAnyImpl &extractMutableCommonObject(ObjectPtr) const; - - Object &extractMutableObject(ObjectPtr memo_ptr) const; - const Object *tryExtractObject(ObjectPtr memo_ptr) const; - Object *tryExtractMutableObject(ObjectPtr memo_ptr) const; + const ObjectAnyImpl &extractAnyObject(ObjectPtr) const; + ObjectAnyImpl &extractMutableAnyObject(ObjectPtr) const; + + const ObjectAnyImpl *tryExtractObject(ObjectPtr memo_ptr) const; const List &extractList(ObjectPtr list_ptr) const; List &extractMutableList(ObjectPtr list_ptr) const; @@ -235,4 +238,13 @@ namespace db0::python extern template const db0::object_model::ObjectImmutableImpl & PyTypeManager::extractObject(ObjectPtr) const; + extern template db0::object_model::Object & + PyTypeManager::extractMutableObject(ObjectPtr) const; + + extern template db0::object_model::ObjectImmutableImpl * + PyTypeManager::tryExtractMutableObject(ObjectPtr) const; + + extern template db0::object_model::Object * + PyTypeManager::tryExtractMutableObject(ObjectPtr) const; + } \ No newline at end of file diff --git a/src/dbzero/bindings/python/shared_py_object.cpp b/src/dbzero/bindings/python/shared_py_object.cpp index 8bf0230e..43a74810 100644 --- a/src/dbzero/bindings/python/shared_py_object.cpp +++ b/src/dbzero/bindings/python/shared_py_object.cpp @@ -11,7 +11,7 @@ namespace db0::python // increment reference count for memo objects reinterpret_cast(py_object)->ext().addExtRef(); } - + template void decExtRef(PyObject *py_object) { // decrement reference count for memo objects @@ -26,18 +26,18 @@ namespace db0::python void incExtRef(PyObject *py_object) { - if (PyMemo_Check(py_object)) { + if (PyMemo_Check(py_object)) { incExtRefImpl(py_object); - } else if (PyMemo_Check(py_object)) { + } else if (PyMemo_Check(py_object)) { incExtRefImpl(py_object); } } void decExtRef(PyObject *py_object) { - if (PyMemo_Check(py_object)) { + if (PyMemo_Check(py_object)) { decExtRef(py_object); - } else if (PyMemo_Check(py_object)) { + } else if (PyMemo_Check(py_object)) { decExtRef(py_object); } } diff --git a/src/dbzero/core/vspace/v_ptr.hpp b/src/dbzero/core/vspace/v_ptr.hpp index 5d5d87ab..91a95fb9 100644 --- a/src/dbzero/core/vspace/v_ptr.hpp +++ b/src/dbzero/core/vspace/v_ptr.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -50,7 +51,9 @@ namespace db0 mutable std::atomic m_resource_flags = 0; // initial access flags (e.g. read / write / create) FlagSet m_access_mode; - + // NOTE: cached size may speed-up updates but also is relevant for existing vptr's reinterpret casts + mutable std::optional m_cached_size; + /** * Memory mapped range corresponding to this object */ @@ -364,7 +367,7 @@ namespace db0 } assert(m_mem_lock.m_buffer); } - + // version with known size-of (pre-retrieved from the allocator) // we made it as a separate implementation for potential performance gains void assureInitialized(std::size_t size_of) const @@ -384,11 +387,9 @@ namespace db0 assert(m_mem_lock.m_buffer); } - /** - * Resolve the instance size - */ - std::size_t getSize() const - { + // Resolve the instance size + std::uint32_t fetchSize() const + { assert(m_memspace_ptr); if constexpr(metaprog::has_constant_size::value) { // fixed size type @@ -402,6 +403,15 @@ namespace db0 // retrieve from allocator (slowest) return m_memspace_ptr->getAllocator().getAllocSize(m_address, REALM_ID); } + + // Get from cache or fetch size + std::uint32_t getSize() const + { + if (!m_cached_size) { + m_cached_size = fetchSize(); + } + return *m_cached_size; + } }; } diff --git a/src/dbzero/object_model/index/IndexBuilder.hpp b/src/dbzero/object_model/index/IndexBuilder.hpp index 5a586b8b..fa7bf740 100644 --- a/src/dbzero/object_model/index/IndexBuilder.hpp +++ b/src/dbzero/object_model/index/IndexBuilder.hpp @@ -87,13 +87,13 @@ namespace db0::object_model std::function add_callback = [&](UniqueAddress address) { auto it = m_object_cache.find(address); assert(it != m_object_cache.end()); - m_type_manager.extractMutableCommonObject(it->second.get()).incRef(false); + m_type_manager.extractMutableAnyObject(it->second.get()).incRef(false); }; std::function erase_callback = [&](UniqueAddress address) { auto it = m_object_cache.find(address); assert(it != m_object_cache.end()); - m_type_manager.extractMutableCommonObject(it->second.get()).decRef(false); + m_type_manager.extractMutableAnyObject(it->second.get()).decRef(false); }; super_t::flush(index, &add_callback, &erase_callback); @@ -103,7 +103,7 @@ namespace db0::object_model template UniqueAddress IndexBuilder::addToCache(ObjectPtr obj_ptr) { - auto obj_addr = m_type_manager.extractCommonObject(obj_ptr).getUniqueAddress(); + auto obj_addr = m_type_manager.extractAnyObject(obj_ptr).getUniqueAddress(); if (m_object_cache.find(obj_addr) == m_object_cache.end()) { m_object_cache.emplace(obj_addr, obj_ptr); } diff --git a/src/dbzero/object_model/object/ObjectAnyBase.cpp b/src/dbzero/object_model/object/ObjectAnyBase.cpp index c7caed94..d68a3d20 100644 --- a/src/dbzero/object_model/object/ObjectAnyBase.cpp +++ b/src/dbzero/object_model/object/ObjectAnyBase.cpp @@ -91,19 +91,11 @@ namespace db0::object_model } template - bool ObjectAnyBase::decRef(bool is_tag) + void ObjectAnyBase::decRef(bool is_tag) { // this operation is a potentially silent mutation _touch(); - super_t::decRef(is_tag); - return !hasRefs(); - } - - template - bool ObjectAnyBase::hasRefs() const - { - assert(this->hasInstance()); - return (*this)->hasRefs(); + super_t::decRef(is_tag); } template diff --git a/src/dbzero/object_model/object/ObjectAnyBase.hpp b/src/dbzero/object_model/object/ObjectAnyBase.hpp index e116c181..79707436 100644 --- a/src/dbzero/object_model/object/ObjectAnyBase.hpp +++ b/src/dbzero/object_model/object/ObjectAnyBase.hpp @@ -81,16 +81,14 @@ namespace db0::object_model * The overloaded incRef implementation is provided to also handle non-fully initialized objects */ void incRef(bool is_tag); - bool hasRefs() const; + void decRef(bool is_tag); + // check for any refs (including auto-assigned type tags) bool hasAnyRefs() const; // check if any references from tags exist (i.e. are any tags assigned) bool hasTagRefs() const; - - // @return true if reference count was decremented to zero - bool decRef(bool is_tag); - + Address getAddress() const; UniqueAddress getUniqueAddress() const; diff --git a/src/dbzero/object_model/object/ObjectImplBase.cpp b/src/dbzero/object_model/object/ObjectImplBase.cpp index 89919478..b425aa89 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.cpp +++ b/src/dbzero/object_model/object/ObjectImplBase.cpp @@ -1058,6 +1058,13 @@ namespace db0::object_model return getClassFactory(fixture).getTypeByClassRef(class_ref).m_class; } + template + bool ObjectImplBase::hasRefs() const + { + assert(this->hasInstance()); + return (*this)->hasRefs(); + } + template class ObjectImplBase; template class ObjectImplBase; diff --git a/src/dbzero/object_model/object/ObjectImplBase.hpp b/src/dbzero/object_model/object/ObjectImplBase.hpp index ee70d232..46e31fa1 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.hpp +++ b/src/dbzero/object_model/object/ObjectImplBase.hpp @@ -127,6 +127,10 @@ namespace db0::object_model // FieldID, is_init_var, fidelity std::pair findField(const char *name) const; + // NOTE: hasRefs is NOT available in ObjectAnyBase bacause + // of the use of num_type_tags property + bool hasRefs() const; + protected: // local kv-index instance cache (created at first use) mutable std::unique_ptr m_kv_index; diff --git a/src/dbzero/object_model/object/o_object.cpp b/src/dbzero/object_model/object/o_object.cpp index 9da3023d..fb4e5610 100644 --- a/src/dbzero/object_model/object/o_object.cpp +++ b/src/dbzero/object_model/object/o_object.cpp @@ -13,7 +13,15 @@ namespace db0::object_model : m_header(ref_counts) { } + + void o_object_base::incRef(bool is_tag) { + m_header.incRef(is_tag); + } + bool o_object_base::hasAnyRefs() const { + return m_header.hasRefs(); + } + o_object::o_object(std::uint32_t class_ref, std::pair ref_counts, std::uint8_t num_type_tags, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin, const XValue *index_vt_end) @@ -60,10 +68,6 @@ namespace db0::object_model return getDynAfter(classRef(), IndexVT::type()); } - void o_object::incRef(bool is_tag) { - m_header.incRef(is_tag); - } - bool o_object::hasRefs() const { // NOTE: type tags are not counted as "proper" references @@ -73,8 +77,4 @@ namespace db0::object_model return m_header.m_ref_counter.getSecond() > 0; } - bool o_object::hasAnyRefs() const { - return m_header.hasRefs(); - } - } \ No newline at end of file diff --git a/src/dbzero/object_model/object/o_object.hpp b/src/dbzero/object_model/object/o_object.hpp index 3a0a5fcd..4ecb8a12 100644 --- a/src/dbzero/object_model/object/o_object.hpp +++ b/src/dbzero/object_model/object/o_object.hpp @@ -25,17 +25,13 @@ DB0_PACKED_BEGIN // ref_counts - the initial reference counts (tags / objects) inherited from the initializer o_object_base(std::pair ref_counts); - static std::size_t measure(); static std::size_t measure(std::pair); - - static std::size_t sizeOf(); template static std::size_t safeSizeOf(BufT buf) { return super_t::sizeOfMembers(buf); } - + void incRef(bool is_tag); - bool hasRefs() const; bool hasAnyRefs() const; }; DB0_PACKED_END @@ -81,10 +77,8 @@ DB0_PACKED_BEGIN (IndexVT::type()); } - void incRef(bool is_tag); - bool hasRefs() const; - bool hasAnyRefs() const; - }; + bool hasRefs() const; + }; DB0_PACKED_END } \ No newline at end of file diff --git a/src/dbzero/object_model/tags/TagIndex.cpp b/src/dbzero/object_model/tags/TagIndex.cpp index bec6d091..61d32f17 100644 --- a/src/dbzero/object_model/tags/TagIndex.cpp +++ b/src/dbzero/object_model/tags/TagIndex.cpp @@ -379,7 +379,7 @@ namespace db0::object_model // NOTE: some object might've been dropped in the meantime, need to be reverted from batch operations for (const auto &item: m_object_cache) { auto obj_ptr = item.second.get(); - auto &memo = type_manager.extractCommonObject(obj_ptr); + auto &memo = type_manager.extractAnyObject(obj_ptr); if (memo.isDead()) { revert(obj_ptr); } @@ -392,7 +392,7 @@ namespace db0::object_model auto it = m_object_cache.find(obj_addr); assert(it != m_object_cache.end()); // NOTE: inc-ref as tag - type_manager.extractMutableObject(it->second.get()).incRef(true); + type_manager.extractMutableAnyObject(it->second.get()).incRef(true); }; // add_index_callback adds reference to tags (string pool tokens) @@ -407,9 +407,9 @@ namespace db0::object_model // object may not exist if tags are removed post-deletion auto obj_ptr = it->second.get(); if (it != m_object_cache.end()) { - auto &memo = type_manager.extractMutableObject(obj_ptr); // NOTE: we check for acutal language references (excluding LangCache + TagIndex) - if (memo.decRef(true) && !LangToolkit::hasAnyLangRefs(obj_ptr, 2)) { + if (LangToolkit::decRefMemo(true, obj_ptr) && !LangToolkit::hasAnyLangRefs(obj_ptr, 2)) { + auto &memo = type_manager.extractAnyObject(obj_ptr); // if object is pending deletion, remove all type tags as well // we might skip this operation and leave it to Object's dropTags function // but it will be more efficient to do it here @@ -455,7 +455,7 @@ namespace db0::object_model assert(m_active_pre_cache.empty()); for (const auto &item: m_object_cache) { auto obj_ptr = item.second.get(); - auto &memo = type_manager.extractCommonObject(obj_ptr); + auto &memo = type_manager.extractAnyObject(obj_ptr); // NOTE: dropped instances should've already been reverted by now // NOTE: we check for acutal language references (excluding LangCache + TagIndex) if (!memo.isDropped() && !memo.hasAnyRefs() && !LangToolkit::hasAnyLangRefs(obj_ptr, 2)) { @@ -480,7 +480,7 @@ namespace db0::object_model void TagIndex::buildActiveValues() const { for (auto &item: m_active_cache) { - auto &memo = LangToolkit::getTypeManager().extractCommonObject(item.first); + auto &memo = LangToolkit::getTypeManager().extractAnyObject(item.first); // NOTE: defunct objects have to be ignored since they don't have a valid address // NOTE: defunct objects, since no valid unique address is assigned will be auto-reverted on flush if (!memo.isDefunct()) { @@ -559,7 +559,7 @@ namespace db0::object_model // Memo instance is directly fed into the FT_FixedKeyIterator if (type_id == TypeId::MEMO_OBJECT) { - auto addr = LangToolkit::getTypeManager().extractCommonObject(arg).getUniqueAddress(); + auto addr = LangToolkit::getTypeManager().extractAnyObject(arg).getUniqueAddress(); factory.add(std::make_unique >(&addr, &addr + 1)); return true; } @@ -807,7 +807,7 @@ namespace db0::object_model std::optional TagIndex::tryAddShortTagFromMemo(ObjectPtr py_arg) const { assert(LangToolkit::isAnyMemoObject(py_arg)); - auto &py_obj = LangToolkit::getTypeManager().extractCommonObject(py_arg); + auto &py_obj = LangToolkit::getTypeManager().extractAnyObject(py_arg); if (py_obj.getFixtureUUID() != m_fixture_uuid) { // must be added as long tag return std::nullopt; @@ -1103,7 +1103,7 @@ namespace db0::object_model LongTagT TagIndex::getLongTagFromMemo(ObjectPtr py_arg) const { assert(LangToolkit::isAnyMemoObject(py_arg)); - auto &py_obj = LangToolkit::getTypeManager().extractCommonObject(py_arg); + auto &py_obj = LangToolkit::getTypeManager().extractAnyObject(py_arg); return { py_obj.getFixtureUUID(), py_obj.getAddress().getOffset() }; } @@ -1113,7 +1113,7 @@ namespace db0::object_model void TagIndex::revert(ObjectPtr memo_ptr) const { - auto &memo = LangToolkit::getTypeManager().extractCommonObject(memo_ptr); + auto &memo = LangToolkit::getTypeManager().extractAnyObject(memo_ptr); auto addr = memo.getUniqueAddress(); if (m_batch_op_short) { m_batch_op_short->revert(addr); diff --git a/src/dbzero/object_model/tags/TagIndex.hpp b/src/dbzero/object_model/tags/TagIndex.hpp index d56b8270..4a2cc275 100644 --- a/src/dbzero/object_model/tags/TagIndex.hpp +++ b/src/dbzero/object_model/tags/TagIndex.hpp @@ -257,7 +257,7 @@ DB0_PACKED_END { // prepare the active value only if it's not yet initialized if (!result.first.isValid() && !result.second) { - auto &memo = LangToolkit::getTypeManager().extractCommonObject(memo_ptr); + auto &memo = LangToolkit::getTypeManager().extractAnyObject(memo_ptr); // NOTE: that memo object may not have address before fully initialized (before postInit) if (memo.hasInstance()) { auto object_addr = memo.getUniqueAddress(); diff --git a/src/dbzero/object_model/value/Member.cpp b/src/dbzero/object_model/value/Member.cpp index 2dfce8b9..faa979b7 100644 --- a/src/dbzero/object_model/value/Member.cpp +++ b/src/dbzero/object_model/value/Member.cpp @@ -53,11 +53,12 @@ namespace db0::object_model return db0::v_object(*fixture, PyUnicode_AsUTF8(obj_ptr), access_mode).getAddress(); } - // OBJECT specialization + // OBJECT specialization (mutable) template <> Value createMember(db0::swine_ptr &fixture, PyObjectPtr obj_ptr, StorageClass, AccessFlags) { - auto &obj = PyToolkit::getTypeManager().extractMutableObject(obj_ptr); + using MemoObject = PyToolkit::TypeManager::MemoObject; + auto &obj = PyToolkit::getTypeManager().extractMutableObject(obj_ptr); assert(obj.hasInstance()); assureSameFixture(fixture, obj); obj.modify().incRef(false); @@ -647,7 +648,8 @@ namespace db0::object_model // decref cached instance via language specific wrapper type auto lang_wrapper = LangToolkit::template getWrapperTypeOf(obj_ptr.get()); auto &object = lang_wrapper->modifyExt(); - if (object.decRef(false)) { + object.decRef(false); + if (!object.hasRefs()) { // NOTE: we'll drop the object immediately on condition it has no language references if (!LangToolkit::hasLangRefs(*obj_ptr)) { auto unique_addr = object.getUniqueAddress(); @@ -751,12 +753,15 @@ namespace db0::object_model return !object_ptr || object_ptr->hasInstance(); } + /* FIXME: implement dispatching + template void materialize(FixtureLock &fixture, PyObjectPtr obj_ptr) { - auto object_ptr = PyToolkit::getTypeManager().tryExtractMutableObject(obj_ptr); + auto object_ptr = PyToolkit::getTypeManager().tryExtractMutableObject(obj_ptr); if (object_ptr && !object_ptr->hasInstance()) { object_ptr->postInit(fixture); } } - + */ + } \ No newline at end of file diff --git a/src/dbzero/object_model/value/Member.hpp b/src/dbzero/object_model/value/Member.hpp index 10cc0cfa..46f34a59 100644 --- a/src/dbzero/object_model/value/Member.hpp +++ b/src/dbzero/object_model/value/Member.hpp @@ -112,9 +112,8 @@ namespace db0::object_model /** * Invoke materialize before setting obj_ptr as a member * this is to materialize objects (where hasInstance = false) before using them as members - */ - void materialize(FixtureLock &, PyObjectPtr obj_ptr); - + */ + void materialize(FixtureLock &fixture, PyObjectPtr obj_ptr); bool isMaterialized(PyObjectPtr obj_ptr); - + } diff --git a/src/dbzero/object_model/value/StorageClass.cpp b/src/dbzero/object_model/value/StorageClass.cpp index 5d0d80c7..8afe89be 100644 --- a/src/dbzero/object_model/value/StorageClass.cpp +++ b/src/dbzero/object_model/value/StorageClass.cpp @@ -1,6 +1,7 @@ #include "StorageClass.hpp" #include #include +#include namespace db0::object_model @@ -172,12 +173,11 @@ namespace db0 using LangToolkit = db0::object_model::LangConfig::LangToolkit; - /* FIXME: db0::object_model::StorageClass getStorageClass(db0::object_model::PreStorageClass pre_storage_class, db0::swine_ptr &fixture, ObjectPtr lang_value) { assert(pre_storage_class == PreStorageClass::OBJECT_WEAK_REF); - const auto &obj = LangToolkit::getTypeManager().extractObject(lang_value); + const auto &obj = LangToolkit::getTypeManager().extractAnyObject(lang_value); if (*obj.getFixture() != *fixture.get()) { // must use long weak-ref instead, since referenced object is from a foreign prefix return StorageClass::OBJECT_LONG_WEAK_REF; @@ -189,13 +189,12 @@ namespace db0 const db0::Fixture &fixture, ObjectPtr lang_value) { assert(pre_storage_class == PreStorageClass::OBJECT_WEAK_REF); - const auto &obj = LangToolkit::getTypeManager().extractObject(lang_value); + const auto &obj = LangToolkit::getTypeManager().extractAnyObject(lang_value); if (*obj.getFixture() != fixture) { // must use long weak-ref instead, since referenced object is from a foreign prefix return StorageClass::OBJECT_LONG_WEAK_REF; } return StorageClass::OBJECT_WEAK_REF; - } - */ - + } + } \ No newline at end of file From 71a3843873676cab72d8af4069e44b2c9ee87d75 Mon Sep 17 00:00:00 2001 From: Wojtek Date: Thu, 6 Nov 2025 13:38:11 +0100 Subject: [PATCH 17/19] fixes --- src/dbzero/bindings/python/PyAPI.cpp | 9 ++-- src/dbzero/bindings/python/PyToolkit.cpp | 28 ++++++++++ src/dbzero/bindings/python/PyToolkit.hpp | 2 + src/dbzero/bindings/python/PyTypeManager.cpp | 53 +++++++++++-------- .../bindings/python/shared_py_object.cpp | 4 +- src/dbzero/core/serialization/Ext.hpp | 2 +- src/dbzero/core/vspace/v_ptr.cpp | 6 ++- src/dbzero/core/vspace/v_ptr.hpp | 5 +- .../object_model/object/ObjectImplBase.cpp | 6 +-- src/dbzero/object_model/object/o_object.cpp | 10 ++-- src/dbzero/object_model/object/o_object.hpp | 1 + .../object_model/tags/ObjectTagManager.cpp | 6 +-- .../object_model/tags/ObjectTagManager.hpp | 10 ++-- src/dbzero/object_model/tags/TagIndex.cpp | 4 +- src/dbzero/object_model/value/Member.cpp | 22 ++++++-- 15 files changed, 113 insertions(+), 55 deletions(-) diff --git a/src/dbzero/bindings/python/PyAPI.cpp b/src/dbzero/bindings/python/PyAPI.cpp index f5b8d7ce..09154342 100644 --- a/src/dbzero/bindings/python/PyAPI.cpp +++ b/src/dbzero/bindings/python/PyAPI.cpp @@ -1366,7 +1366,6 @@ namespace db0::python return runSafe(tryGetConfig); } - /* FIXME: PyObject *PyAPI_compare(PyObject *, PyObject *args, PyObject *kwargs) { PyObject *py_first = nullptr; @@ -1377,15 +1376,15 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; } - - if (!PyMemo_Check(py_first) || !PyMemo_Check(py_second)) { + + // FIXME: implement for MemoImmutableObject as well + if (!PyMemo_Check(py_first) || !PyMemo_Check(py_second)) { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; } return runSafe(tryCompareMemo, reinterpret_cast(py_first), reinterpret_cast(py_second)); - } - */ + } #ifndef NDEBUG PyObject *PyAPI_startDebugLogs(PyObject *self, PyObject *) diff --git a/src/dbzero/bindings/python/PyToolkit.cpp b/src/dbzero/bindings/python/PyToolkit.cpp index afa6da8a..a4f2a4ce 100644 --- a/src/dbzero/bindings/python/PyToolkit.cpp +++ b/src/dbzero/bindings/python/PyToolkit.cpp @@ -475,6 +475,14 @@ namespace db0::python return PyAnyMemo_Check(py_object); } + bool PyToolkit::isMemoObject(ObjectPtr py_object) { + return PyMemo_Check(py_object); + } + + bool PyToolkit::isMemoImmutableObject(ObjectPtr py_object) { + return PyMemo_Check(py_object); + } + PyToolkit::ObjectPtr PyToolkit::getUUID(ObjectPtr py_object) { return db0::python::tryGetUUID(py_object); } @@ -753,4 +761,24 @@ namespace db0::python return py_object != nullptr; } + template + bool decRefMemoImpl(bool is_tag, MemoImplT *memo_obj) + { + auto &memo = memo_obj->modifyExt(); + memo.decRef(is_tag); + return !memo.hasRefs(); + } + + bool PyToolkit::decRefMemo(bool is_tag, ObjectPtr py_object) + { + if (PyMemo_Check(py_object)) { + return decRefMemoImpl(is_tag, reinterpret_cast(py_object)); + } else if (PyMemo_Check(py_object)) { + return decRefMemoImpl(is_tag, reinterpret_cast(py_object)); + } else { + assert(false); + THROWF(db0::InputException) << "Invalid memo object type for decRefMemo" << THROWF_END; + } + } + } \ No newline at end of file diff --git a/src/dbzero/bindings/python/PyToolkit.hpp b/src/dbzero/bindings/python/PyToolkit.hpp index 9adcace0..04b6700c 100644 --- a/src/dbzero/bindings/python/PyToolkit.hpp +++ b/src/dbzero/bindings/python/PyToolkit.hpp @@ -174,6 +174,8 @@ namespace db0::python // either memo or immutable type static bool isAnyMemoType(TypeObjectPtr py_type); static bool isAnyMemoObject(ObjectPtr py_object); + static bool isMemoObject(ObjectPtr py_object); + static bool isMemoImmutableObject(ObjectPtr py_object); static bool isEnumValue(ObjectPtr py_object); static bool isFieldDef(ObjectPtr py_object); static bool isClassObject(ObjectPtr py_object); diff --git a/src/dbzero/bindings/python/PyTypeManager.cpp b/src/dbzero/bindings/python/PyTypeManager.cpp index 190442a2..80c28bf7 100644 --- a/src/dbzero/bindings/python/PyTypeManager.cpp +++ b/src/dbzero/bindings/python/PyTypeManager.cpp @@ -207,32 +207,25 @@ namespace db0::python return reinterpret_cast(obj_ptr)->modifyExt(); } - /* FIXME: implement - - template - const typename MemoImplT::ExtT &extractObject(ObjectPtr memo_ptr) const - { - if (PyMemo_Check(memo_ptr)) { - return reinterpret_cast(memo_ptr)->ext(); - } else if (PyWeakProxy_Check(memo_ptr)) { - return reinterpret_cast( - reinterpret_cast*>(memo_ptr)->get())->ext(); - } + template typename MemoImplT::ExtT & + PyTypeManager::extractMutableObject(ObjectPtr obj_ptr) const + { + if (!PyMemo_Check(obj_ptr)) { THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; } - */ - - /* - const db0::object_model::Object &PyTypeManager::extractObject(ObjectPtr memo_ptr) const + return reinterpret_cast(obj_ptr)->modifyExt(); + } + + template const typename MemoImplT::ExtT & + PyTypeManager::extractObject(ObjectPtr memo_ptr) const { - if (PyMemo_Check(memo_ptr)) { - return reinterpret_cast(memo_ptr)->ext(); + if (PyMemo_Check(memo_ptr)) { + return reinterpret_cast(memo_ptr)->ext(); } else if (PyWeakProxy_Check(memo_ptr)) { - return reinterpret_cast(reinterpret_cast(memo_ptr)->get())->ext(); + return reinterpret_cast(reinterpret_cast(memo_ptr)->get())->ext(); } THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; - } - */ + } const PyTypeManager::ObjectAnyImpl *PyTypeManager::tryExtractObject(ObjectPtr memo_ptr) const { @@ -682,5 +675,23 @@ namespace db0::python THROWF(db0::InputException) << "Invalid value code: " << val_code << THROWF_END; } } - + + template db0::object_model::Object & + PyTypeManager::extractMutableObject(ObjectPtr) const; + + template db0::object_model::ObjectImmutableImpl & + PyTypeManager::extractMutableObject(ObjectPtr) const; + + template const db0::object_model::Object & + PyTypeManager::extractObject(ObjectPtr) const; + + template const db0::object_model::ObjectImmutableImpl & + PyTypeManager::extractObject(ObjectPtr) const; + + template db0::object_model::ObjectImmutableImpl * + PyTypeManager::tryExtractMutableObject(ObjectPtr) const; + + template db0::object_model::Object * + PyTypeManager::tryExtractMutableObject(ObjectPtr) const; + } \ No newline at end of file diff --git a/src/dbzero/bindings/python/shared_py_object.cpp b/src/dbzero/bindings/python/shared_py_object.cpp index 43a74810..be947b93 100644 --- a/src/dbzero/bindings/python/shared_py_object.cpp +++ b/src/dbzero/bindings/python/shared_py_object.cpp @@ -44,9 +44,9 @@ namespace db0::python unsigned int getExtRefcount(PyObject *py_object, unsigned int default_count) { - if (PyMemo_Check(py_object)) { + if (PyMemo_Check(py_object)) { return getExtRefcount(py_object, default_count); - } else if (PyMemo_Check(py_object)) { + } else if (PyMemo_Check(py_object)) { return getExtRefcount(py_object, default_count); } diff --git a/src/dbzero/core/serialization/Ext.hpp b/src/dbzero/core/serialization/Ext.hpp index 207c4828..28c2fcff 100644 --- a/src/dbzero/core/serialization/Ext.hpp +++ b/src/dbzero/core/serialization/Ext.hpp @@ -236,7 +236,7 @@ DB0_PACKED_BEGIN // measures space requirement of the base overlaid type // plus size of all fixed size members of derived type - template static size_t measureBase(Args&& ...args) + template static Meter measureBase(Args&& ...args) { std::size_t result = super_t::measure(std::forward(args)...); // adjust for fixed size members in derived class diff --git a/src/dbzero/core/vspace/v_ptr.cpp b/src/dbzero/core/vspace/v_ptr.cpp index 4e184ccc..703f9ffb 100644 --- a/src/dbzero/core/vspace/v_ptr.cpp +++ b/src/dbzero/core/vspace/v_ptr.cpp @@ -40,10 +40,11 @@ namespace db0 } vtypeless &vtypeless::operator=(const vtypeless &other) - { + { m_address = other.m_address; m_memspace_ptr = other.m_memspace_ptr; m_access_mode = other.m_access_mode; + m_cached_size = other.m_cached_size; // try locking for copy for (;;) { @@ -63,10 +64,11 @@ namespace db0 } void vtypeless::operator=(vtypeless &&other) - { + { m_address = other.m_address; m_memspace_ptr = other.m_memspace_ptr; m_access_mode = other.m_access_mode; + m_cached_size = other.m_cached_size; // try locking for copy for (;;) { diff --git a/src/dbzero/core/vspace/v_ptr.hpp b/src/dbzero/core/vspace/v_ptr.hpp index 91a95fb9..fb33bb32 100644 --- a/src/dbzero/core/vspace/v_ptr.hpp +++ b/src/dbzero/core/vspace/v_ptr.hpp @@ -213,6 +213,7 @@ namespace db0 m_memspace_ptr->free(m_address); this->m_address = {}; this->m_resource_flags = 0; + this->m_cached_size.reset(); } ContainerT &modify() @@ -403,14 +404,14 @@ namespace db0 // retrieve from allocator (slowest) return m_memspace_ptr->getAllocator().getAllocSize(m_address, REALM_ID); } - + // Get from cache or fetch size std::uint32_t getSize() const { if (!m_cached_size) { m_cached_size = fetchSize(); } - return *m_cached_size; + return *m_cached_size; } }; diff --git a/src/dbzero/object_model/object/ObjectImplBase.cpp b/src/dbzero/object_model/object/ObjectImplBase.cpp index b425aa89..b8fee250 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.cpp +++ b/src/dbzero/object_model/object/ObjectImplBase.cpp @@ -171,7 +171,6 @@ namespace db0::object_model assert(this->hasInstance()); } - /* FIXME: template std::pair ObjectImplBase::recognizeType(Fixture &fixture, ObjectPtr lang_value) const @@ -181,7 +180,7 @@ namespace db0::object_model auto pre_storage_class = TypeUtils::m_storage_class_mapper.getPreStorageClass(type_id, true); if (type_id == TypeId::MEMO_OBJECT) { // object reference must be from the same fixture - auto &obj = LangToolkit::getTypeManager().extractObject(lang_value); + auto &obj = LangToolkit::getTypeManager().extractAnyObject(lang_value); if (fixture.getUUID() != obj.getFixture()->getUUID()) { THROWF(db0::InputException) << "Referencing objects from foreign prefixes is not allowed. Use db0.weak_proxy instead"; } @@ -196,8 +195,7 @@ namespace db0::object_model } return { type_id, storage_class }; - } - */ + } template void ObjectImplBase::removePreInit(const char *field_name) const diff --git a/src/dbzero/object_model/object/o_object.cpp b/src/dbzero/object_model/object/o_object.cpp index fb4e5610..da425c6e 100644 --- a/src/dbzero/object_model/object/o_object.cpp +++ b/src/dbzero/object_model/object/o_object.cpp @@ -22,6 +22,10 @@ namespace db0::object_model return m_header.hasRefs(); } + std::size_t o_object_base::measure() { + return super_t::measureMembers(); + } + o_object::o_object(std::uint32_t class_ref, std::pair ref_counts, std::uint8_t num_type_tags, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin, const XValue *index_vt_end) @@ -35,15 +39,15 @@ namespace db0::object_model } std::size_t o_object::measure(std::uint32_t class_ref, std::pair, std::uint8_t, - const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, + const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin, const XValue *index_vt_end) { - return super_t::measureMembers() + return super_t::measureBase() (PosVT::type(), pos_vt_data, pos_vt_offset) (packed_int32::type(), class_ref) (IndexVT::type(), index_vt_begin, index_vt_end); } - + const PosVT &o_object::pos_vt() const { return getDynFirst(PosVT::type()); } diff --git a/src/dbzero/object_model/object/o_object.hpp b/src/dbzero/object_model/object/o_object.hpp index 4ecb8a12..28bafd10 100644 --- a/src/dbzero/object_model/object/o_object.hpp +++ b/src/dbzero/object_model/object/o_object.hpp @@ -25,6 +25,7 @@ DB0_PACKED_BEGIN // ref_counts - the initial reference counts (tags / objects) inherited from the initializer o_object_base(std::pair ref_counts); + static std::size_t measure(); static std::size_t measure(std::pair); template static std::size_t safeSizeOf(BufT buf) { diff --git a/src/dbzero/object_model/tags/ObjectTagManager.cpp b/src/dbzero/object_model/tags/ObjectTagManager.cpp index feed36ad..a1302ccf 100644 --- a/src/dbzero/object_model/tags/ObjectTagManager.cpp +++ b/src/dbzero/object_model/tags/ObjectTagManager.cpp @@ -47,17 +47,15 @@ namespace db0::object_model return new (at_ptr) ObjectTagManager(memo_ptr, nargs); } - /* FIXME: ObjectTagManager::ObjectInfo::ObjectInfo(ObjectPtr memo_ptr) : m_lang_ptr(memo_ptr) - , m_object_ptr(&ObjectTagManager::LangToolkit::getTypeManager().extractObject(memo_ptr)) + , m_object_ptr(&ObjectTagManager::LangToolkit::getTypeManager().extractAnyObject(memo_ptr)) , m_tag_index_ptr(&m_object_ptr->getFixture()->get()) , m_type(m_object_ptr->getClassPtr()) , m_access_mode(m_object_ptr->getFixture()->getAccessType()) , m_has_tags(LangToolkit::hasTagRefs(memo_ptr)) { - } - */ + } void ObjectTagManager::ObjectInfo::add(ObjectPtr const *args, Py_ssize_t nargs) { diff --git a/src/dbzero/object_model/tags/ObjectTagManager.hpp b/src/dbzero/object_model/tags/ObjectTagManager.hpp index c3bcc8d2..97c81252 100644 --- a/src/dbzero/object_model/tags/ObjectTagManager.hpp +++ b/src/dbzero/object_model/tags/ObjectTagManager.hpp @@ -1,15 +1,15 @@ #pragma once -#include #include #include #include +#include namespace db0::object_model { - - using Object = db0::object_model::Object; + + using ObjectAnyImpl = db0::object_model::ObjectAnyImpl; using Class = db0::object_model::Class; using RC_LimitedStringPool = db0::pools::RC_LimitedStringPool; @@ -20,7 +20,7 @@ namespace db0::object_model class ObjectTagManager { public: - using LangToolkit = typename Object::LangToolkit; + using LangToolkit = typename ObjectAnyImpl::LangToolkit; using ObjectPtr = typename LangToolkit::ObjectPtr; using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; @@ -43,7 +43,7 @@ namespace db0::object_model struct ObjectInfo { ObjectSharedPtr m_lang_ptr; - const Object *m_object_ptr = nullptr; + const ObjectAnyImpl *m_object_ptr = nullptr; TagIndex *m_tag_index_ptr = nullptr; std::shared_ptr m_type; AccessType m_access_mode; diff --git a/src/dbzero/object_model/tags/TagIndex.cpp b/src/dbzero/object_model/tags/TagIndex.cpp index 61d32f17..6d1f1d5e 100644 --- a/src/dbzero/object_model/tags/TagIndex.cpp +++ b/src/dbzero/object_model/tags/TagIndex.cpp @@ -164,8 +164,7 @@ namespace db0::object_model m_batch_op_types.empty() && "TagIndex::flush() or close() must be called before destruction"); } - - /* FIXME: implement + void TagIndex::addTags(ObjectPtr memo_ptr, ObjectPtr const *args, std::size_t nargs) { using TypeId = db0::bindings::TypeId; @@ -226,7 +225,6 @@ namespace db0::object_model m_mutation_log->onDirty(); } } - */ FT_BaseIndex::BatchOperationBuilder & TagIndex::getBatchOperationShort(ObjectPtr memo_ptr, ActiveValueT &result, bool is_type) const diff --git a/src/dbzero/object_model/value/Member.cpp b/src/dbzero/object_model/value/Member.cpp index faa979b7..34d4fbbe 100644 --- a/src/dbzero/object_model/value/Member.cpp +++ b/src/dbzero/object_model/value/Member.cpp @@ -12,6 +12,9 @@ #include // FIXME: remove Python dependency #include +#include +#include +#include namespace db0::object_model @@ -753,15 +756,28 @@ namespace db0::object_model return !object_ptr || object_ptr->hasInstance(); } - /* FIXME: implement dispatching template - void materialize(FixtureLock &fixture, PyObjectPtr obj_ptr) + void materializeImpl(FixtureLock &fixture, PyObjectPtr obj_ptr) { auto object_ptr = PyToolkit::getTypeManager().tryExtractMutableObject(obj_ptr); if (object_ptr && !object_ptr->hasInstance()) { object_ptr->postInit(fixture); } } - */ + + void materialize(FixtureLock &fixture, PyObjectPtr obj_ptr) + { + using MemoObject = PyToolkit::TypeManager::MemoObject; + using MemoImmutableObject = PyToolkit::TypeManager::MemoImmutableObject; + + if (PyToolkit::isMemoObject(obj_ptr)) { + materializeImpl(fixture, obj_ptr); + } else if (PyToolkit::isMemoImmutableObject(obj_ptr)) { + materializeImpl(fixture, obj_ptr); + } else { + assert(false && "Unsupported memo object type"); + THROWF(db0::InputException) << "Unable to materialize non-memo object" << THROWF_END; + } + } } \ No newline at end of file From 4135f8b730db2798eb7191a3ec7ab2200f5217cf Mon Sep 17 00:00:00 2001 From: Wojtek Date: Thu, 6 Nov 2025 14:05:21 +0100 Subject: [PATCH 18/19] fixes --- dbzero/dbzero/dbzero.py | 2 +- python_tests/test_issues_10.py | 2 +- src/dbzero/object_model/object/ObjectAnyBase.cpp | 15 +++++++-------- src/dbzero/object_model/object/ObjectAnyBase.hpp | 14 ++++++++++---- .../object_model/object/ObjectImplBase.cpp | 16 ++++++++-------- .../object_model/object/ObjectImplBase.hpp | 2 +- 6 files changed, 28 insertions(+), 23 deletions(-) diff --git a/dbzero/dbzero/dbzero.py b/dbzero/dbzero/dbzero.py index 21899e3d..c9e4f4dc 100644 --- a/dbzero/dbzero/dbzero.py +++ b/dbzero/dbzero/dbzero.py @@ -10,7 +10,7 @@ def load_dynamic(name, path): def __bootstrap__(): global __bootstrap__, __loader__, __file__ - paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/debug", "/usr/local/lib/python3/dist-packages/dbzero/"] + paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/release", "/usr/local/lib/python3/dist-packages/dbzero/"] __file__ = None for path in paths: if os.path.isdir(path): diff --git a/python_tests/test_issues_10.py b/python_tests/test_issues_10.py index 4281e6e7..82fe2903 100644 --- a/python_tests/test_issues_10.py +++ b/python_tests/test_issues_10.py @@ -71,6 +71,6 @@ def secret(self) -> int: def test_shopping_cart_secret_issue(db0_fixture): cart_1 = ShoppingCart(as_temp=True) assert cart_1.secret is not None - cart_2 = ShoppingCart(as_temp=False) + cart_2 = ShoppingCart(as_temp=False) assert cart_2.secret is None \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectAnyBase.cpp b/src/dbzero/object_model/object/ObjectAnyBase.cpp index d68a3d20..7c846976 100644 --- a/src/dbzero/object_model/object/ObjectAnyBase.cpp +++ b/src/dbzero/object_model/object/ObjectAnyBase.cpp @@ -3,10 +3,9 @@ namespace db0::object_model { + + ObjectInitializerManager InitManager::instance; - template - ObjectInitializerManager ObjectAnyBase::m_init_manager; - template ObjectAnyBase::ObjectAnyBase(UniqueAddress addr, unsigned int ext_refs) : m_flags { ObjectOptions::DROPPED } @@ -23,7 +22,7 @@ namespace db0::object_model return {}; } // retrieve from the initializer - return m_init_manager.getInitializer(*this).tryGetFixture(); + return InitManager::instance.getInitializer(*this).tryGetFixture(); } return super_t::tryGetFixture(); } @@ -50,7 +49,7 @@ namespace db0::object_model THROWF(db0::InputException) << "set_prefix failed: object already initialized"; } - if (!m_init_manager.getInitializer(*this).trySetFixture(new_fixture)) { + if (!InitManager::instance.getInitializer(*this).trySetFixture(new_fixture)) { // signal problem with PyErr_BadPrefix auto fixture = this->getFixture(); LangToolkit::setError(LangToolkit::getTypeManager().getBadPrefixError(), fixture->getUUID()); @@ -86,7 +85,7 @@ namespace db0::object_model super_t::incRef(is_tag); } else { // incRef with the initializer - m_init_manager.getInitializer(*this).incRef(is_tag); + InitManager::instance.getInitializer(*this).incRef(is_tag); } } @@ -148,10 +147,10 @@ namespace db0::object_model void ObjectAnyBase::setDefunct() const { m_flags.set(ObjectOptions::DEFUNCT); } - + template Class &ObjectAnyBase::getType() { - return this->m_type ? *this->m_type : m_init_manager.getInitializer(*this).getClass(); + return this->m_type ? *this->m_type : InitManager::instance.getInitializer(*this).getClass(); } template class ObjectAnyBase; diff --git a/src/dbzero/object_model/object/ObjectAnyBase.hpp b/src/dbzero/object_model/object/ObjectAnyBase.hpp index 79707436..b2950dfd 100644 --- a/src/dbzero/object_model/object/ObjectAnyBase.hpp +++ b/src/dbzero/object_model/object/ObjectAnyBase.hpp @@ -42,6 +42,13 @@ namespace db0::object_model // NOTE: Object instances are created within the implementation specific realm_id (e.g. =1 for o_object) template using ObjectVType = db0::v_object; + // Common init manager for all specializations + class InitManager + { + public: + static ObjectInitializerManager instance; + }; + template class ObjectAnyBase: public db0::ObjectBase, StorageClass::OBJECT_REF> { @@ -62,11 +69,11 @@ namespace db0::object_model Memspace &getMemspace() const; inline std::shared_ptr getClassPtr() const { - return this->m_type ? this->m_type : m_init_manager.getInitializer(*this).getClassPtr(); + return this->m_type ? this->m_type : InitManager::instance.getInitializer(*this).getClassPtr(); } inline const Class &getType() const { - return this->m_type ? *this->m_type : m_init_manager.getInitializer(*this).getClass(); + return this->m_type ? *this->m_type : InitManager::instance.getInitializer(*this).getClass(); } Class &getType(); @@ -120,8 +127,7 @@ namespace db0::object_model return m_ext_refs; } - protected: - static ObjectInitializerManager m_init_manager; + protected: // Class will only be assigned after initialization std::shared_ptr m_type; mutable ObjectFlags m_flags; diff --git a/src/dbzero/object_model/object/ObjectImplBase.cpp b/src/dbzero/object_model/object/ObjectImplBase.cpp index b8fee250..d0963a29 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.cpp +++ b/src/dbzero/object_model/object/ObjectImplBase.cpp @@ -37,14 +37,14 @@ namespace db0::object_model ObjectImplBase::ObjectImplBase(std::shared_ptr db0_class) { // prepare for initialization - super_t::m_init_manager.addInitializer(*this, db0_class); + InitManager::instance.addInitializer(*this, db0_class); } template ObjectImplBase::ObjectImplBase(TypeInitializer &&type_initializer) { // prepare for initialization - super_t::m_init_manager.addInitializer(*this, std::move(type_initializer)); + InitManager::instance.addInitializer(*this, std::move(type_initializer)); } template @@ -93,7 +93,7 @@ namespace db0::object_model this->unregister(); if (!this->hasInstance()) { // release initializer if it exists, object not created - super_t::m_init_manager.tryCloseInitializer(*this); + InitManager::instance.tryCloseInitializer(*this); } } @@ -139,7 +139,7 @@ namespace db0::object_model void ObjectImplBase::postInit(FixtureLock &fixture) { if (!this->hasInstance()) { - auto &initializer = super_t::m_init_manager.getInitializer(*this); + auto &initializer = InitManager::instance.getInitializer(*this); PosVT::Data pos_vt_data; unsigned int pos_vt_offset = 0; auto index_vt_data = initializer.getData(pos_vt_data, pos_vt_offset); @@ -200,7 +200,7 @@ namespace db0::object_model template void ObjectImplBase::removePreInit(const char *field_name) const { - auto &initializer = super_t::m_init_manager.getInitializer(*this); + auto &initializer = InitManager::instance.getInitializer(*this); auto &type = initializer.getClass(); // Find an already existing field index @@ -237,7 +237,7 @@ namespace db0::object_model return; } - auto &initializer = super_t::m_init_manager.getInitializer(*this); + auto &initializer = InitManager::instance.getInitializer(*this); auto fixture = initializer.getFixture(); auto &type = initializer.getClass(); auto [type_id, storage_class] = recognizeType(*fixture, obj_ptr); @@ -437,7 +437,7 @@ namespace db0::object_model auto class_ptr = this->m_type.get(); if (!class_ptr) { // retrieve class from the initializer - class_ptr = &super_t::m_init_manager.getInitializer(*this).getClass(); + class_ptr = &InitManager::instance.getInitializer(*this).getClass(); } assert(class_ptr); @@ -606,7 +606,7 @@ namespace db0::object_model auto loc = field_info.first.getIndexAndOffset(); if (!this->hasInstance()) { // try retrieving from initializer - auto initializer_ptr = super_t::m_init_manager.findInitializer(*this); + auto initializer_ptr = InitManager::instance.findInitializer(*this); if (!initializer_ptr) { find_result = { false, false }; return true; diff --git a/src/dbzero/object_model/object/ObjectImplBase.hpp b/src/dbzero/object_model/object/ObjectImplBase.hpp index 46e31fa1..6dcf5ce6 100644 --- a/src/dbzero/object_model/object/ObjectImplBase.hpp +++ b/src/dbzero/object_model/object/ObjectImplBase.hpp @@ -167,7 +167,7 @@ namespace db0::object_model std::pair tryGetLoc(FieldID) const; inline ObjectInitializer *tryGetInitializer() const { - return this->m_type ? static_cast(nullptr) : &super_t::m_init_manager.getInitializer(*this); + return this->m_type ? static_cast(nullptr) : &InitManager::instance.getInitializer(*this); } void dropMembers(db0::swine_ptr &, Class &) const; From 172dea82732331cf953e3623b608969be1e6bf89 Mon Sep 17 00:00:00 2001 From: Wojtek Date: Thu, 6 Nov 2025 14:56:16 +0100 Subject: [PATCH 19/19] test immutable --- python_tests/test_memo_immutable.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python_tests/test_memo_immutable.py b/python_tests/test_memo_immutable.py index 419026f3..5a14cf4c 100644 --- a/python_tests/test_memo_immutable.py +++ b/python_tests/test_memo_immutable.py @@ -16,3 +16,8 @@ def test_create_memo_immutable(db0_fixture): _ = MemoImmutableClass1(data="immutable data", value=42) +def test_tag_and_find_immutable_instance(db0_fixture): + obj_1 = MemoImmutableClass1(data="immutable data", value=42) + db0.tags(obj_1).add("tag1", "tag2") + assert list(db0.find("tag1")) == [obj_1] + \ No newline at end of file