From ce4d04f31f1910f3fcb3c57ca811a628747b2351 Mon Sep 17 00:00:00 2001 From: Wojtek Date: Fri, 22 May 2026 21:01:20 +0200 Subject: [PATCH 1/2] unreferencing immutable / embedded instances --- design/IMMUTABLE_OBJECTS_DESIGN.md | 1 + python_tests/test_memo_immutable.py | 89 +++++++++++++++++++ src/dbzero/bindings/python/PyTypeManager.cpp | 8 +- src/dbzero/object_model/index/Index.cpp | 10 +-- .../object_model/index/IndexBuilder.hpp | 4 +- .../object/ObjectImmutableImpl.cpp | 9 ++ .../object/ObjectImmutableImpl.hpp | 1 + src/dbzero/object_model/value/Member.cpp | 87 +++++++++++++----- src/dbzero/object_model/value/Member.hpp | 2 + tests/unit_tests/ObjectInitializerTest.cpp | 77 ++++++++++++++++ 10 files changed, 256 insertions(+), 32 deletions(-) diff --git a/design/IMMUTABLE_OBJECTS_DESIGN.md b/design/IMMUTABLE_OBJECTS_DESIGN.md index 09e9dfcb..bcae7ba0 100644 --- a/design/IMMUTABLE_OBJECTS_DESIGN.md +++ b/design/IMMUTABLE_OBJECTS_DESIGN.md @@ -111,6 +111,7 @@ Implementation requirements: - The embedded member is identified by its own address, but that address is inside the allocation and is not the allocation start. - The allocator must be able to recover allocation metadata from an inner address. This allows embedded object addresses to use the same 50-bit representation as regular object addresses. - A parent object can still be referenced by the parent allocation address. +- Strong references to embedded objects increment the root allocation's reference count, and deleting the holding object must decrement that same root reference by resolving the embedded address back to its allocation base. - Root immutable objects store an exact compact index of valid nested embedded-object offsets. Lookup by offset must validate against this index and raise a bad-address error for invalid, out-of-range, or non-object offsets. - The offset index uses packed integer encoding grouped by packed size class so offsets remain compact while supporting logarithmic exact membership checks. Most offsets are expected to fit in 3-4 packed bytes, but the representation must support larger offsets. diff --git a/python_tests/test_memo_immutable.py b/python_tests/test_memo_immutable.py index 9d460b13..53f44f26 100644 --- a/python_tests/test_memo_immutable.py +++ b/python_tests/test_memo_immutable.py @@ -443,6 +443,50 @@ def test_regular_memo_can_reference_embedded_immutable_nested_object(db0_fixture assert reopened.payload.count == 21 +def test_immutable_instance_drops_when_holding_memo_object_is_deleted(db0_fixture): + obj = MemoImmutableClass1(data="held immutable root", value=31) + db0.tags(obj).add("temporary-held-immutable-root") + obj_id = db0.uuid(obj) + holder = MemoSetReferenceHolder(obj) + db0.tags(obj).remove("temporary-held-immutable-root") + del obj + gc.collect() + + db0.commit() + assert db0.exists(obj_id) + + db0.delete(holder) + del holder + gc.collect() + db0.commit() + + with pytest.raises(Exception): + db0.fetch(obj_id) + + +def test_immutable_instance_supported_by_embedded_ref_drops_when_holding_memo_object_is_deleted(db0_fixture): + outer = MemoImmutableNestedHolder(name="held embedded child", count=32, label="root") + db0.tags(outer).add("temporary-held-embedded-root") + outer_id = db0.uuid(outer) + inner = outer.nested + holder = MemoSetReferenceHolder(inner) + db0.tags(outer).remove("temporary-held-embedded-root") + del inner + del outer + gc.collect() + + db0.commit() + assert db0.exists(outer_id) + + db0.delete(holder) + del holder + gc.collect() + db0.commit() + + with pytest.raises(Exception): + db0.fetch(outer_id) + + def test_db0_collections_can_store_embedded_immutable_nested_object_reference(db0_fixture): outer = MemoImmutableNestedHolder(name="collection child", count=22, label="root") db0.tags(outer).add("keep-collection-source") @@ -502,6 +546,51 @@ def test_index_can_store_embedded_immutable_nested_object_reference(db0_fixture) assert retrieved[0].count == 23 +def test_index_remove_unrefs_embedded_immutable_nested_object_reference(db0_fixture): + outer = MemoImmutableNestedHolder(name="index remove child", count=24, label="root") + db0.tags(outer).add("temporary-index-remove-source") + inner = outer.nested + outer_id = db0.uuid(outer) + index = db0.index() + + index.add(1, inner) + index.flush() + db0.tags(outer).remove("temporary-index-remove-source") + del outer + del inner + gc.collect() + + stored = list(index.select())[0] + index.remove(1, stored) + del stored + gc.collect() + db0.commit() + + with pytest.raises(Exception): + db0.fetch(outer_id) + + +def test_index_clear_unrefs_embedded_immutable_nested_object_reference(db0_fixture): + outer = MemoImmutableNestedHolder(name="index clear child", count=25, label="root") + db0.tags(outer).add("temporary-index-clear-source") + inner = outer.nested + outer_id = db0.uuid(outer) + index = db0.index() + + index.add(1, inner) + db0.commit() + db0.tags(outer).remove("temporary-index-clear-source") + del outer + del inner + gc.collect() + + index.clear() + db0.commit() + + with pytest.raises(Exception): + db0.fetch(outer_id) + + def test_read_embedded_tuple_field(db0_fixture): payload = tuple(f"alpha-{index}" for index in range(12)) + (7, b"bytes", None) obj = MemoImmutableTupleHolder(payload) diff --git a/src/dbzero/bindings/python/PyTypeManager.cpp b/src/dbzero/bindings/python/PyTypeManager.cpp index 6d441f82..94770b13 100644 --- a/src/dbzero/bindings/python/PyTypeManager.cpp +++ b/src/dbzero/bindings/python/PyTypeManager.cpp @@ -212,6 +212,8 @@ namespace db0::python { if (PyAnyMemo_Check(obj_ptr)) { return reinterpret_cast(obj_ptr)->ext(); + } else if (PyMemo_Check(obj_ptr)) { + return reinterpret_cast(obj_ptr)->ext(); } else if (PyWeakProxy_Check(obj_ptr)) { return reinterpret_cast(obj_ptr)->get()->ext(); } @@ -220,10 +222,10 @@ namespace db0::python PyTypeManager::ObjectAnyImpl &PyTypeManager::extractMutableAnyObject(ObjectPtr obj_ptr) const { - if (!PyAnyMemo_Check(obj_ptr)) { - THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; + if (PyAnyMemo_Check(obj_ptr) || PyMemo_Check(obj_ptr)) { + return reinterpret_cast(obj_ptr)->modifyExt(); } - return reinterpret_cast(obj_ptr)->modifyExt(); + THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; } db0::swine_ptr PyTypeManager::extractObjectFixture(ObjectPtr obj_ptr) const diff --git a/src/dbzero/object_model/index/Index.cpp b/src/dbzero/object_model/index/Index.cpp index 9370a13b..6afcab88 100644 --- a/src/dbzero/object_model/index/Index.cpp +++ b/src/dbzero/object_model/index/Index.cpp @@ -605,8 +605,8 @@ namespace db0::object_model const_cast(m_builder).rollback(); if (hasRangeTree()) { auto fixture = this->getFixture(); - auto unref_func = [&fixture](Address obj_addr) { - unrefMember(fixture, obj_addr); + auto unref_func = [&fixture](UniqueAddress objAddr) { + unrefAnyMemoObject(fixture, objAddr); }; switch ((*this)->m_data_type) { case IndexDataType::Int64: { @@ -675,8 +675,8 @@ namespace db0::object_model return; } auto fixture = this->getFixture(); - auto unref_func = [&fixture](Address obj_addr) { - unrefMember(fixture, obj_addr); + auto unref_func = [&fixture](UniqueAddress objAddr) { + unrefAnyMemoObject(fixture, objAddr); }; switch ((*this)->m_data_type) { case IndexDataType::Int64: @@ -696,4 +696,4 @@ namespace db0::object_model } -} \ No newline at end of file +} diff --git a/src/dbzero/object_model/index/IndexBuilder.hpp b/src/dbzero/object_model/index/IndexBuilder.hpp index d398c7f0..bcee9639 100644 --- a/src/dbzero/object_model/index/IndexBuilder.hpp +++ b/src/dbzero/object_model/index/IndexBuilder.hpp @@ -9,6 +9,7 @@ namespace db0::object_model { + void unrefAnyMemoObject(db0::swine_ptr &fixture, UniqueAddress address); /** * Wraps extends the RangeTree::Builder providing persistency cache for dbzero instances @@ -101,7 +102,8 @@ namespace db0::object_model std::function erase_callback = [&](UniqueAddress address) { auto it = m_object_cache.find(address); assert(it != m_object_cache.end()); - m_type_manager.extractMutableAnyObject(it->second.get()).decRef(false); + auto fixture = m_type_manager.extractObjectFixture(it->second.get()); + unrefAnyMemoObject(fixture, address); }; super_t::flush(index, &add_callback, &erase_callback); diff --git a/src/dbzero/object_model/object/ObjectImmutableImpl.cpp b/src/dbzero/object_model/object/ObjectImmutableImpl.cpp index b218657b..30421079 100644 --- a/src/dbzero/object_model/object/ObjectImmutableImpl.cpp +++ b/src/dbzero/object_model/object/ObjectImmutableImpl.cpp @@ -252,6 +252,15 @@ namespace db0::object_model super_t::destroy(); } + void ObjectImmutableImpl::dropInstance(FixtureLock &) + { + auto uniqueAddress = this->getUniqueAddress(); + auto extRefs = this->getExtRefs(); + this->destroy(); + this->~ObjectImmutableImpl(); + new ((void *)this) ObjectImmutableImpl(tag_as_dropped(), uniqueAddress, extRefs); + } + ObjectImmutableImpl::ObjectSharedPtr ObjectImmutableImpl::tryGet( MemberLoc memberLoc, bool *isAutoGenerated ) const diff --git a/src/dbzero/object_model/object/ObjectImmutableImpl.hpp b/src/dbzero/object_model/object/ObjectImmutableImpl.hpp index c35134ea..615e04fe 100644 --- a/src/dbzero/object_model/object/ObjectImmutableImpl.hpp +++ b/src/dbzero/object_model/object/ObjectImmutableImpl.hpp @@ -34,6 +34,7 @@ namespace db0::object_model void postInit(FixtureLock &); void setLangObject(ObjectPtr) const; void destroy(); + void dropInstance(FixtureLock &); static ObjectSharedPtr tryGetEmbeddedField( db0::swine_ptr &, ObjectPtr root_object, const o_embedded_object &, diff --git a/src/dbzero/object_model/value/Member.cpp b/src/dbzero/object_model/value/Member.cpp index 35a793e1..e98d7689 100644 --- a/src/dbzero/object_model/value/Member.cpp +++ b/src/dbzero/object_model/value/Member.cpp @@ -19,6 +19,7 @@ #include #include #include +#include namespace db0::object_model @@ -728,33 +729,74 @@ namespace db0::object_model functions[static_cast(StorageClass::CALLABLE)] = unloadMember; } + template + void unrefCachedMemoObject( + db0::swine_ptr &fixture, typename LangToolkit::ObjectPtr objPtr, bool isTag) + { + db0::FixtureLock lock(fixture); + auto langWrapper = reinterpret_cast(objPtr); + auto &object = langWrapper->modifyExt(); + object.decRef(isTag); + if (!object.hasRefs() && !LangToolkit::hasLangRefs(objPtr)) { + auto uniqueAddress = object.getUniqueAddress(); + object.dropInstance(lock); + fixture->getLangCache().erase(uniqueAddress); + } + } + + template + void unrefUncachedMemoObject( + db0::swine_ptr &fixture, typename T::ObjectStem &&stem, bool isTag) + { + auto type = fixture->template get().getTypeByClassRef(stem->getClassRef()).m_class; + T object(fixture, std::move(stem), std::move(type)); + object.decRef(isTag); + } + template void unrefMemoObject(db0::swine_ptr &fixture, Address address) { - auto obj_ptr = fixture->getLangCache().get(address); - if (obj_ptr.get()) { - db0::FixtureLock lock(fixture); - // decref cached instance via language specific wrapper type - auto lang_wrapper = reinterpret_cast(obj_ptr.get()); - auto &object = lang_wrapper->modifyExt(); - object.decRef(false); - if (!object.hasRefs()) { - // NOTE: we'll drop the object immediately on condition it has no language references - if (!LangToolkit::hasLangRefs(*obj_ptr)) { - auto unique_addr = object.getUniqueAddress(); - // drop dbzero instance, replacing it with a "null" placeholder - object.dropInstance(lock); - // might also be removed from lang cache - fixture->getLangCache().erase(unique_addr); - } + auto objPtr = fixture->getLangCache().get(address); + if (objPtr.get()) { + unrefCachedMemoObject(fixture, objPtr.get(), false); + return; + } + + auto allocation = fixture->findAllocation(address, ObjectAnyImpl::REALM_ID); + typename T::ObjectStem stem(db0::tag_verified(), fixture->myPtr(allocation.address), allocation.size); + unrefUncachedMemoObject(fixture, std::move(stem), false); + } + + void unrefAnyMemoObject(db0::swine_ptr &fixture, UniqueAddress address) + { + auto allocation = fixture->findAllocation(address.getAddress(), ObjectAnyImpl::REALM_ID); + auto rootAddress = allocation.address; + + auto objPtr = fixture->getLangCache().get(rootAddress); + if (objPtr.get()) { + if (db0::python::PyMemo_Check(objPtr.get())) { + unrefCachedMemoObject< + ObjectImmutableImpl, PyToolkit::TypeManager::MemoImmutableObject, PyToolkit + >(fixture, objPtr.get(), false); + } else { + unrefCachedMemoObject( + fixture, objPtr.get(), false + ); } + return; + } + + std::size_t sizeOf = allocation.size; + ObjectAnyImpl::ObjectStem commonStem(db0::tag_verified(), fixture->myPtr(rootAddress), sizeOf); + if (commonStem->m_header.isImmutableObject()) { + auto stem = ObjectAnyImpl::castStem(std::move(commonStem)); + unrefUncachedMemoObject(fixture, std::move(stem), false); } else { - T object(fixture, address); - object.decRef(false); - // member will be deleted by GC0 if its ref-count = 0 + auto stem = ObjectAnyImpl::castStem(std::move(commonStem)); + unrefUncachedMemoObject(fixture, std::move(stem), false); } } - + // Unreference any ObjectBase-derived type (except Memo types) template void unrefObjectBase(db0::swine_ptr &fixture, Address address) @@ -792,10 +834,9 @@ namespace db0::object_model } template <> void unrefMember( - db0::swine_ptr &, Value) + db0::swine_ptr &fixture, Value value) { - // Embedded immutable references point inside the root allocation. Retrieval is implemented - // for this feature slice; unreferencing embedded references is intentionally deferred. + unrefAnyMemoObject(fixture, value.asUniqueAddress()); } template <> void unrefMember( diff --git a/src/dbzero/object_model/value/Member.hpp b/src/dbzero/object_model/value/Member.hpp index 64376a12..1d5665cc 100644 --- a/src/dbzero/object_model/value/Member.hpp +++ b/src/dbzero/object_model/value/Member.hpp @@ -93,6 +93,8 @@ namespace db0::object_model // unreference a member (decref / destroy where applicable) template void unrefMember( db0::swine_ptr &fixture, Value value); + + void unrefAnyMemoObject(db0::swine_ptr &fixture, UniqueAddress address); // register StorageClass specializations template void registerUnrefMemberFunctions( diff --git a/tests/unit_tests/ObjectInitializerTest.cpp b/tests/unit_tests/ObjectInitializerTest.cpp index b29baa49..1f67268c 100644 --- a/tests/unit_tests/ObjectInitializerTest.cpp +++ b/tests/unit_tests/ObjectInitializerTest.cpp @@ -1314,6 +1314,83 @@ namespace tests workspace.close(); } + TEST_F( ObjectInitializerTest, testDestroyObjectUnrefsHeldEmbeddedMemoReference ) + { + Py_Initialize(); + + Workspace workspace("", {}, {}, {}, {}, db0::object_model::initializer()); + auto fixture = workspace.getFixture(prefix_name); + auto rootClass = getTestClass(fixture); + auto holderClass = getTestClass(fixture); + auto pyMemoType = makeImmutableMemoType(); + ASSERT_TRUE(pyMemoType.get()); + auto nestedClass = fixture->get().getOrCreateType(pyMemoType.get()); + auto rootLoc = rootClass->addField("child", 0).get(0).getIndexAndOffset(); + auto nestedLoc = nestedClass->addField("value", 0).get(0).getIndexAndOffset(); + auto holderLoc = holderClass->addField("held", 0).get(0).getIndexAndOffset(); + rootClass->flush(); + nestedClass->flush(); + holderClass->flush(); + + { + auto pyNestedMemo = Py_OWN(reinterpret_cast( + db0::python::MemoObjectStub_new(pyMemoType.get()) + )); + pyNestedMemo->makeNew(nestedClass); + auto *nestedInitializer = dynamic_cast( + InitManager::instance.findInitializer(pyNestedMemo->ext()) + ); + ASSERT_NE(nestedInitializer, nullptr); + nestedInitializer->set(nestedLoc, StorageClass::INT64, Value(17)); + + ObjectImmutableImpl root(rootClass); + auto *rootInitializer = dynamic_cast( + InitManager::instance.findInitializer(root) + ); + ASSERT_NE(rootInitializer, nullptr); + rootInitializer->setObject( + rootLoc, StorageClass::OBJECT_REF, Value(0), + ImmutableObjectInitializer::ObjectSharedPtr(reinterpret_cast(pyNestedMemo.get())) + ); + + { + db0::FixtureLock lock(fixture); + root.postInit(lock); + } + root.incRef(false); + ASSERT_EQ(root.getRefCounts().second, 1u); + + auto *embeddedValue = root->variableValue(rootLoc.first); + ASSERT_NE(embeddedValue, nullptr); + ASSERT_EQ(embeddedValue->itemKind(), StorageClass::EMBEDDED_OBJECT); + auto embeddedOffset = offsetOfEmbeddedObject(*root.operator->(), *embeddedValue); + auto embeddedAddress = db0::UniqueAddress( + root.getAddress() + embeddedOffset, root.getUniqueAddress().getInstanceId() + ); + + root.incRef(false); + Object holder(holderClass); + auto *holderInitializer = InitManager::instance.findInitializer(holder); + ASSERT_NE(holderInitializer, nullptr); + holderInitializer->set(holderLoc, StorageClass::EMBEDDED_OBJECT_REF, Value(embeddedAddress)); + { + db0::FixtureLock lock(fixture); + holder.postInit(lock); + } + ASSERT_EQ(root.getRefCounts().second, 2u); + + holder.destroy(); + ASSERT_EQ(root.getRefCounts().second, 1u); + + root.destroy(); + } + + rootClass.reset(); + holderClass.reset(); + nestedClass.reset(); + workspace.close(); + } + TEST_F( ObjectInitializerTest, testImmutablePreInitChangingRegularValueToLoFiClearsEmbeddedObject ) { Py_Initialize(); From b60ba5d1ccf48e290d24b57d546c01c45d5cf2e5 Mon Sep 17 00:00:00 2001 From: Wojtek Date: Fri, 22 May 2026 21:49:50 +0200 Subject: [PATCH 2/2] tagging and untagging of embedded / immutable and nested instances --- python_tests/test_memo_immutable.py | 352 ++++++++++++++++++ src/dbzero/bindings/python/Memo.cpp | 4 +- src/dbzero/bindings/python/PyInternalAPI.cpp | 8 + .../bindings/python/PyObjectTagManager.cpp | 3 +- src/dbzero/bindings/python/PyToolkit.cpp | 13 +- src/dbzero/bindings/python/PyToolkit.hpp | 1 + .../object/ObjectImmutableImpl.cpp | 47 +++ .../object/o_packed_offset_index.cpp | 66 +++- .../object/o_packed_offset_index.hpp | 31 +- .../object_model/tags/ObjectTagManager.cpp | 15 +- .../object_model/tags/ObjectTagManager.hpp | 5 +- src/dbzero/object_model/tags/TagIndex.cpp | 34 +- tests/unit_tests/PackedOffsetIndexTest.cpp | 16 + 13 files changed, 562 insertions(+), 33 deletions(-) diff --git a/python_tests/test_memo_immutable.py b/python_tests/test_memo_immutable.py index 53f44f26..2ab0c8fe 100644 --- a/python_tests/test_memo_immutable.py +++ b/python_tests/test_memo_immutable.py @@ -181,6 +181,56 @@ def __init__(self, name, count): self.label = "deep-root" +@db0.memo(immutable=True) +class MemoImmutableDefaultTagLeaf: + def __init__(self, name, count): + self.name = name + self.count = count + + +@db0.memo(immutable=True) +class MemoImmutableDefaultTagMiddle: + def __init__(self, name, count): + self.name = name + self.leaf = MemoImmutableDefaultTagLeaf(name=f"{name}-leaf", count=count) + + +@db0.memo(immutable=True) +class MemoImmutableDefaultTagRoot: + def __init__(self, name, count): + self.middle = MemoImmutableDefaultTagMiddle(name=name, count=count) + self.label = "default-tag-root" + + +@db0.memo(immutable=True) +class MemoImmutableDefaultTagBase: + def __init__(self, name): + self.name = name + + +@db0.memo(immutable=True) +class MemoImmutableDefaultTagDerived(MemoImmutableDefaultTagBase): + def __init__(self, name, count): + super().__init__(name) + self.count = count + + +@db0.memo(immutable=True) +class MemoImmutableDefaultTagDerivedLeaf(MemoImmutableDefaultTagDerived): + def __init__(self, name, count, marker): + super().__init__(name=name, count=count) + self.marker = marker + + +@db0.memo(immutable=True) +class MemoImmutableDefaultTagInheritanceRoot: + def __init__(self, name, count): + self.child = MemoImmutableDefaultTagDerived(name=name, count=count) + self.leaf = MemoImmutableDefaultTagDerivedLeaf( + name=f"{name}-leaf", count=count + 1, marker="deep-derived" + ) + + @db0.memo(immutable=True, no_default_tags=True) class MemoImmutableTupleHolder: def __init__(self, payload): @@ -337,6 +387,252 @@ def test_tag_and_find_immutable_instance(db0_fixture): assert list(db0.find("tag1")) == [obj_1] +def test_tag_and_find_embedded_immutable_instance(db0_fixture): + root = MemoImmutableNestedHolder(name="tagged embedded child", count=106, label="root") + root = db0.materialized(root) + root_uuid = db0.uuid(root) + nested = root.nested + nested_uuid = db0.uuid(nested) + + db0.tags(nested).add("embedded-tag") + + result = list(db0.find("embedded-tag")) + assert len(result) == 1 + assert isinstance(result[0], MemoImmutableNestedPayload) + assert result[0].name == "tagged embedded child" + assert result[0].count == 106 + assert db0.uuid(result[0]) == nested_uuid + assert db0.uuid(result[0]) != root_uuid + + +def test_typed_find_embedded_immutable_instance(db0_fixture): + root = MemoImmutableNestedHolder(name="typed tagged embedded child", count=107, label="root") + root = db0.materialized(root) + db0.uuid(root) + nested_uuid = db0.uuid(root.nested) + + db0.tags(root.nested).add("embedded-typed-tag") + + typed_result = list(db0.find(MemoImmutableNestedPayload, "embedded-typed-tag")) + root_result = list(db0.find(MemoImmutableNestedHolder, "embedded-typed-tag")) + assert len(typed_result) == 1 + assert typed_result[0].name == "typed tagged embedded child" + assert db0.uuid(typed_result[0]) == nested_uuid + assert root_result == [] + + +def test_tagged_embedded_immutable_instance_survives_reopen(db0_fixture): + root = MemoImmutableNestedHolder(name="reopened tagged embedded child", count=108, label="root") + root = db0.materialized(root) + db0.uuid(root) + nested_uuid = db0.uuid(root.nested) + db0.tags(root.nested).add("embedded-reopen-tag") + + del root + gc.collect() + db0.commit() + db0.close() + db0.init(DB0_DIR) + db0.open("my-test-prefix", "rw") + + result = list(db0.find("embedded-reopen-tag")) + assert len(result) == 1 + assert isinstance(result[0], MemoImmutableNestedPayload) + assert result[0].name == "reopened tagged embedded child" + assert result[0].count == 108 + assert db0.uuid(result[0]) == nested_uuid + + +def test_tag_and_find_deep_embedded_immutable_instance(db0_fixture): + root = MemoImmutableDeepRoot(name="deep tagged embedded", count=109) + root = db0.materialized(root) + db0.uuid(root) + leaf_uuid = db0.uuid(root.middle.leaf) + + db0.tags(root.middle.leaf).add("embedded-deep-tag") + + result = list(db0.find(MemoImmutableDeepLeaf, "embedded-deep-tag")) + assert len(result) == 1 + assert result[0].name == "deep tagged embedded-leaf" + assert result[0].count == 109 + assert db0.uuid(result[0]) == leaf_uuid + + +def test_find_embedded_immutable_instances_by_default_type_tags(db0_fixture): + root = db0.materialized(MemoImmutableDefaultTagRoot(name="type-only embedded", count=125)) + root_uuid = db0.uuid(root) + middle_uuid = db0.uuid(root.middle) + leaf_uuid = db0.uuid(root.middle.leaf) + + root_result = list(db0.find(MemoImmutableDefaultTagRoot)) + middle_result = list(db0.find(MemoImmutableDefaultTagMiddle)) + leaf_result = list(db0.find(MemoImmutableDefaultTagLeaf)) + + assert [db0.uuid(item) for item in root_result] == [root_uuid] + assert [db0.uuid(item) for item in middle_result] == [middle_uuid] + assert middle_result[0].name == "type-only embedded" + assert [db0.uuid(item) for item in leaf_result] == [leaf_uuid] + assert leaf_result[0].name == "type-only embedded-leaf" + assert leaf_result[0].count == 125 + + +def test_find_embedded_immutable_instances_by_type_respects_no_default_tags(db0_fixture): + root = db0.materialized(MemoImmutableDeepRoot(name="no default type-only embedded", count=126)) + assert root.middle.leaf.count == 126 + + assert list(db0.find(MemoImmutableDeepRoot)) == [] + assert list(db0.find(MemoImmutableDeepMiddle)) == [] + assert list(db0.find(MemoImmutableDeepLeaf)) == [] + + +def test_find_embedded_immutable_instances_by_base_type_default_tags(db0_fixture): + root = db0.materialized(MemoImmutableDefaultTagInheritanceRoot(name="base type embedded", count=127)) + child_uuid = db0.uuid(root.child) + leaf_uuid = db0.uuid(root.leaf) + + base_result = list(db0.find(MemoImmutableDefaultTagBase)) + derived_result = list(db0.find(MemoImmutableDefaultTagDerived)) + leaf_result = list(db0.find(MemoImmutableDefaultTagDerivedLeaf)) + + assert [db0.uuid(item) for item in base_result] == [child_uuid, leaf_uuid] + assert [db0.uuid(item) for item in derived_result] == [child_uuid, leaf_uuid] + assert [db0.uuid(item) for item in leaf_result] == [leaf_uuid] + assert isinstance(base_result[0], MemoImmutableDefaultTagDerived) + assert base_result[0].name == "base type embedded" + assert base_result[0].count == 127 + assert isinstance(base_result[1], MemoImmutableDefaultTagDerivedLeaf) + assert base_result[1].name == "base type embedded-leaf" + assert base_result[1].count == 128 + assert base_result[1].marker == "deep-derived" + + +def test_find_mixed_regular_immutable_and_embedded_tagged_instances(db0_fixture): + tag = "mixed-tagged-memo-instances" + regular = MemoRegularFetchUUIDPayload(name="mixed regular", count=111) + immutable = db0.materialized(MemoImmutableClass1(data="mixed immutable", value=112)) + shallow_root = db0.materialized(MemoImmutableNestedHolder( + name="mixed shallow embedded", count=113, label="root" + )) + deep_root = db0.materialized(MemoImmutableDeepRoot(name="mixed deep embedded", count=114)) + shallow_embedded = shallow_root.nested + deep_embedded = deep_root.middle.leaf + + regular_uuid = db0.uuid(regular) + immutable_uuid = db0.uuid(immutable) + shallow_uuid = db0.uuid(shallow_embedded) + deep_uuid = db0.uuid(deep_embedded) + expected_uuids = {regular_uuid, immutable_uuid, shallow_uuid, deep_uuid} + + db0.tags(regular).add(tag) + db0.tags(immutable).add(tag) + db0.tags(shallow_embedded).add(tag) + db0.tags(deep_embedded).add(tag) + + result = list(db0.find(tag)) + by_uuid = {db0.uuid(item): item for item in result} + assert set(by_uuid) == expected_uuids + + assert isinstance(by_uuid[db0.uuid(regular)], MemoRegularFetchUUIDPayload) + assert by_uuid[db0.uuid(regular)].name == "mixed regular" + assert isinstance(by_uuid[db0.uuid(immutable)], MemoImmutableClass1) + assert by_uuid[db0.uuid(immutable)].data == "mixed immutable" + assert isinstance(by_uuid[db0.uuid(shallow_embedded)], MemoImmutableNestedPayload) + assert by_uuid[db0.uuid(shallow_embedded)].name == "mixed shallow embedded" + assert isinstance(by_uuid[db0.uuid(deep_embedded)], MemoImmutableDeepLeaf) + assert by_uuid[db0.uuid(deep_embedded)].name == "mixed deep embedded-leaf" + + assert [db0.uuid(item) for item in db0.find(shallow_embedded, tag)] == [db0.uuid(shallow_embedded)] + assert [db0.uuid(item) for item in db0.find(deep_embedded, tag)] == [db0.uuid(deep_embedded)] + + +def test_remove_tag_from_embedded_immutable_instance(db0_fixture): + root = MemoImmutableNestedHolder(name="remove embedded child", count=110, label="root") + root = db0.materialized(root) + nested_uuid = db0.uuid(root.nested) + db0.tags(root.nested).add("embedded-remove-unsupported-tag") + + assert [db0.uuid(item) for item in db0.find("embedded-remove-unsupported-tag")] == [nested_uuid] + assert [db0.uuid(item) for item in db0.find( + MemoImmutableNestedPayload, "embedded-remove-unsupported-tag" + )] == [nested_uuid] + + db0.tags(root.nested).remove("embedded-remove-unsupported-tag") + + assert list(db0.find("embedded-remove-unsupported-tag")) == [] + assert list(db0.find(MemoImmutableNestedPayload, "embedded-remove-unsupported-tag")) == [] + + +def test_remove_tags_from_deep_embedded_immutable_instance_with_iterable_and_operator(db0_fixture): + root = db0.materialized(MemoImmutableDeepRoot(name="deep untag embedded", count=119)) + leaf_uuid = db0.uuid(root.middle.leaf) + tags = db0.tags(root.middle.leaf) + tags.add(["embedded-remove-one", "embedded-remove-two", "embedded-remove-three"]) + + assert [db0.uuid(item) for item in db0.find("embedded-remove-one")] == [leaf_uuid] + assert [db0.uuid(item) for item in db0.find("embedded-remove-two")] == [leaf_uuid] + assert [db0.uuid(item) for item in db0.find("embedded-remove-three")] == [leaf_uuid] + + tags.remove(["embedded-remove-one", "embedded-remove-two"]) + tags -= "embedded-remove-three" + + assert list(db0.find("embedded-remove-one")) == [] + assert list(db0.find("embedded-remove-two")) == [] + assert list(db0.find("embedded-remove-three")) == [] + + +def test_embedded_immutable_root_drops_after_last_tag_removed(db0_fixture): + root = db0.materialized(MemoImmutableNestedHolder( + name="drop after embedded untag", count=120, label="root" + )) + root_uuid = db0.uuid(root) + nested = root.nested + db0.tags(nested).add("embedded-only-keepalive") + del nested + del root + gc.collect() + + db0.commit() + assert db0.exists(root_uuid) + + nested = next(iter(db0.find("embedded-only-keepalive"))) + db0.tags(nested).remove("embedded-only-keepalive") + del nested + gc.collect() + db0.commit() + + with pytest.raises(Exception): + db0.fetch(root_uuid) + + +def test_removing_embedded_tags_preserves_other_mixed_tagged_instances(db0_fixture): + tag = "mixed-tag-removal" + regular = MemoRegularFetchUUIDPayload(name="mixed remove regular", count=121) + immutable = db0.materialized(MemoImmutableClass1(data="mixed remove immutable", value=122)) + shallow_root = db0.materialized(MemoImmutableNestedHolder( + name="mixed remove shallow embedded", count=123, label="root" + )) + deep_root = db0.materialized(MemoImmutableDeepRoot(name="mixed remove deep embedded", count=124)) + shallow_embedded = shallow_root.nested + deep_embedded = deep_root.middle.leaf + regular_uuid = db0.uuid(regular) + immutable_uuid = db0.uuid(immutable) + shallow_uuid = db0.uuid(shallow_embedded) + deep_uuid = db0.uuid(deep_embedded) + + db0.tags(regular).add(tag) + db0.tags(immutable).add(tag) + db0.tags(shallow_embedded).add(tag) + db0.tags(deep_embedded).add(tag) + assert {db0.uuid(item) for item in db0.find(tag)} == { + regular_uuid, immutable_uuid, shallow_uuid, deep_uuid + } + + db0.tags(shallow_embedded).remove(tag) + db0.tags(deep_embedded).remove(tag) + + assert {db0.uuid(item) for item in db0.find(tag)} == {regular_uuid, immutable_uuid} + + def test_read_embedded_immutable_string_after_reopen(db0_fixture): obj = MemoImmutableClass1(data="small embedded string", value=7) db0.tags(obj).add("keep-embedded-string") @@ -546,6 +842,62 @@ def test_index_can_store_embedded_immutable_nested_object_reference(db0_fixture) assert retrieved[0].count == 23 +def test_index_retrieves_mixed_regular_immutable_and_embedded_instances(db0_fixture): + regular = MemoRegularFetchUUIDPayload(name="index mixed regular", count=115) + immutable = db0.materialized(MemoImmutableClass1(data="index mixed immutable", value=116)) + shallow_root = db0.materialized(MemoImmutableNestedHolder( + name="index mixed shallow embedded", count=117, label="root" + )) + deep_root = db0.materialized(MemoImmutableDeepRoot(name="index mixed deep embedded", count=118)) + shallow_embedded = shallow_root.nested + deep_embedded = deep_root.middle.leaf + + regular_uuid = db0.uuid(regular) + immutable_uuid = db0.uuid(immutable) + shallow_uuid = db0.uuid(shallow_embedded) + deep_uuid = db0.uuid(deep_embedded) + expected_uuids = {regular_uuid, immutable_uuid, shallow_uuid, deep_uuid} + + index = db0.index() + index.add(1, regular) + index.add(2, immutable) + index.add(3, shallow_embedded) + index.add(4, deep_embedded) + index.flush() + + retrieved = list(index.select()) + by_uuid = {db0.uuid(item): item for item in retrieved} + assert set(by_uuid) == expected_uuids + assert isinstance(by_uuid[regular_uuid], MemoRegularFetchUUIDPayload) + assert by_uuid[regular_uuid].name == "index mixed regular" + assert isinstance(by_uuid[immutable_uuid], MemoImmutableClass1) + assert by_uuid[immutable_uuid].data == "index mixed immutable" + assert isinstance(by_uuid[shallow_uuid], MemoImmutableNestedPayload) + assert by_uuid[shallow_uuid].name == "index mixed shallow embedded" + assert isinstance(by_uuid[deep_uuid], MemoImmutableDeepLeaf) + assert by_uuid[deep_uuid].name == "index mixed deep embedded-leaf" + + holder = MemoSetReferenceHolder(index) + db0.tags(holder).add("keep-index-mixed-retrieval") + holder_id = db0.uuid(holder) + del regular, immutable, shallow_embedded, deep_embedded, shallow_root, deep_root, index, holder + gc.collect() + + db0.commit() + db0.close() + db0.init(DB0_DIR) + db0.open("my-test-prefix", "rw") + + reopened_index = db0.fetch(holder_id).payload + reopened = list(reopened_index.select()) + reopened_by_uuid = {db0.uuid(item): item for item in reopened} + assert set(reopened_by_uuid) == expected_uuids + assert reopened_by_uuid[regular_uuid].name == "index mixed regular" + assert reopened_by_uuid[immutable_uuid].data == "index mixed immutable" + assert reopened_by_uuid[shallow_uuid].name == "index mixed shallow embedded" + assert reopened_by_uuid[deep_uuid].name == "index mixed deep embedded-leaf" + + def test_index_remove_unrefs_embedded_immutable_nested_object_reference(db0_fixture): outer = MemoImmutableNestedHolder(name="index remove child", count=24, label="root") db0.tags(outer).add("temporary-index-remove-source") diff --git a/src/dbzero/bindings/python/Memo.cpp b/src/dbzero/bindings/python/Memo.cpp index 6becea76..8c91f38d 100644 --- a/src/dbzero/bindings/python/Memo.cpp +++ b/src/dbzero/bindings/python/Memo.cpp @@ -308,11 +308,11 @@ namespace db0::python // invoke post-init on associated dbzero object auto &object = self->modifyExt(); - db0::FixtureLock fixture(object.getFixture()); - object.postInit(fixture); + db0::FixtureLock fixture(object.getFixture()); if constexpr (std::is_same_v) { object.setLangObject(reinterpret_cast(self)); } + object.postInit(fixture); // need to call modifyExt again after postInit because the instance has just been created // and potentially needs to be included in the AtomicContext diff --git a/src/dbzero/bindings/python/PyInternalAPI.cpp b/src/dbzero/bindings/python/PyInternalAPI.cpp index 6eefe0ec..a5048ac1 100644 --- a/src/dbzero/bindings/python/PyInternalAPI.cpp +++ b/src/dbzero/bindings/python/PyInternalAPI.cpp @@ -833,6 +833,14 @@ namespace db0::python if (!memo_obj->ext().getType().isNoCache()) { fixture->getLangCache().add(memo_obj->ext().getAddress(), memo_obj); } + const object_model::Class *class_ptr = &memo_obj->ext().getType(); + if (class_ptr->assignDefaultTags()) { + auto &tag_index = fixture->get(); + while (class_ptr) { + tag_index.addTag(reinterpret_cast(memo_obj), class_ptr->getAddress(), true); + class_ptr = class_ptr->getBaseClassPtr(); + } + } Py_INCREF(memo_obj); return memo_obj; } diff --git a/src/dbzero/bindings/python/PyObjectTagManager.cpp b/src/dbzero/bindings/python/PyObjectTagManager.cpp index d222c9ef..a958fce1 100644 --- a/src/dbzero/bindings/python/PyObjectTagManager.cpp +++ b/src/dbzero/bindings/python/PyObjectTagManager.cpp @@ -4,6 +4,7 @@ #include "PyObjectTagManager.hpp" #include "Memo.hpp" #include "PyInternalAPI.hpp" +#include "PyToolkit.hpp" namespace db0::python @@ -103,7 +104,7 @@ namespace db0::python { // all arguments must be Memo objects for (Py_ssize_t i = 0; i < nargs; ++i) { - if (!PyAnyMemo_Check(args[i])) { + if (!PyToolkit::isAnyMemoObject(args[i])) { THROWF(db0::InputException) << "All arguments must be dbzero memo objects"; } if (PyMemo_Check(args[i])) { diff --git a/src/dbzero/bindings/python/PyToolkit.cpp b/src/dbzero/bindings/python/PyToolkit.cpp index c177a460..bb646eb2 100644 --- a/src/dbzero/bindings/python/PyToolkit.cpp +++ b/src/dbzero/bindings/python/PyToolkit.cpp @@ -973,6 +973,10 @@ namespace db0::python return PyAnyMemo_Check(py_object) || PyEmbeddedMemo_Check(py_object); } + bool PyToolkit::isEmbeddedMemoObject(ObjectPtr py_object) { + return PyEmbeddedMemo_Check(py_object); + } + bool PyToolkit::isMemoObject(ObjectPtr py_object) { return PyMemo_Check(py_object); } @@ -1330,7 +1334,9 @@ namespace db0::python void PyToolkit::incRefMemo(bool is_tag, ObjectPtr py_object) { - if (PyMemo_Check(py_object)) { + if (PyEmbeddedMemo_Check(py_object)) { + incEmbeddedMemoRef(py_object, is_tag); + } else if (PyMemo_Check(py_object)) { incRefMemoImpl(is_tag, reinterpret_cast(py_object)); } else if (PyMemo_Check(py_object)) { incRefMemoImpl(is_tag, reinterpret_cast(py_object)); @@ -1350,7 +1356,10 @@ namespace db0::python bool PyToolkit::decRefMemo(bool is_tag, ObjectPtr py_object) { - if (PyMemo_Check(py_object)) { + if (PyEmbeddedMemo_Check(py_object)) { + auto *rootObject = getEmbeddedMemoRef(reinterpret_cast(py_object)).rootObject(); + return decRefMemoImpl(is_tag, reinterpret_cast(rootObject)); + } else if (PyMemo_Check(py_object)) { return decRefMemoImpl(is_tag, reinterpret_cast(py_object)); } else if (PyMemo_Check(py_object)) { return decRefMemoImpl(is_tag, reinterpret_cast(py_object)); diff --git a/src/dbzero/bindings/python/PyToolkit.hpp b/src/dbzero/bindings/python/PyToolkit.hpp index 9d2b653a..bca746d3 100644 --- a/src/dbzero/bindings/python/PyToolkit.hpp +++ b/src/dbzero/bindings/python/PyToolkit.hpp @@ -190,6 +190,7 @@ namespace db0::python // either memo or immutable type static bool isAnyMemoType(TypeObjectPtr py_type); static bool isAnyMemoObject(ObjectPtr py_object); + static bool isEmbeddedMemoObject(ObjectPtr py_object); static bool isMemoObject(ObjectPtr py_object); static bool isMemoImmutableObject(ObjectPtr py_object); static bool isEnumValue(ObjectPtr py_object); diff --git a/src/dbzero/object_model/object/ObjectImmutableImpl.cpp b/src/dbzero/object_model/object/ObjectImmutableImpl.cpp index 30421079..86bb24c0 100644 --- a/src/dbzero/object_model/object/ObjectImmutableImpl.cpp +++ b/src/dbzero/object_model/object/ObjectImmutableImpl.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -196,6 +197,47 @@ namespace db0::object_model } } + unsigned int assignDefaultTypeTags( + db0::swine_ptr &fixture, ObjectImmutableImpl::ObjectPtr object + ) + { + auto *classPtr = &db0::python::PyToolkit::getMemoType(object); + if (!classPtr->assignDefaultTags()) { + return 0; + } + + auto &tagIndex = fixture->get(); + unsigned int result = 0; + while (classPtr) { + tagIndex.addTag(object, classPtr->getAddress(), true); + classPtr = classPtr->getBaseClassPtr(); + ++result; + } + return result; + } + + unsigned int assignEmbeddedDefaultTypeTags( + db0::swine_ptr &fixture, ObjectImmutableImpl &object, + ObjectImmutableImpl::ObjectPtr rootObject + ) + { + const auto &offsetIndex = object->getOffsetIndex(); + if (!rootObject) { + if (offsetIndex.size() > 0) { + THROWF(db0::InternalException) + << "Embedded immutable type tag assignment requires an initialized root language object"; + } + return 0; + } + + unsigned int result = 0; + for (auto offset: offsetIndex) { + auto embeddedObject = object.getEmbeddedInstanceAtOffset(offset); + result += assignDefaultTypeTags(fixture, embeddedObject.get()); + } + return result; + } + } void ObjectImmutableImpl::postInit(FixtureLock &fixture) @@ -230,6 +272,11 @@ namespace db0::object_model type.setSingletonAddress(*this); } transformEmbeddedObjectValues(*fixture, *this, m_lang_object, *immutableInitializer); + if (m_lang_object) { + this->modify().m_num_type_tags = safeNumTypeTags( + (*this)->m_num_type_tags + assignEmbeddedDefaultTypeTags(*fixture, *this, m_lang_object) + ); + } initializer.close(); } diff --git a/src/dbzero/object_model/object/o_packed_offset_index.cpp b/src/dbzero/object_model/object/o_packed_offset_index.cpp index 1d5717c2..fcf2f583 100644 --- a/src/dbzero/object_model/object/o_packed_offset_index.cpp +++ b/src/dbzero/object_model/object/o_packed_offset_index.cpp @@ -292,10 +292,72 @@ namespace db0::object_model { } + o_packed_offset_index::const_iterator::const_iterator( + list_t::const_iterator group, list_t::const_iterator end + ) + : m_group(group) + , m_end(end) + { + skipEmptyGroups(); + } + + void o_packed_offset_index::const_iterator::skipEmptyGroups() + { + while (m_group != m_end && m_group->empty()) { + ++m_group; + } + if (m_group == m_end) { + m_index = 0; + } + } + + std::uint64_t o_packed_offset_index::const_iterator::operator*() const + { + return m_group->at(m_index); + } + + o_packed_offset_index::const_iterator &o_packed_offset_index::const_iterator::operator++() + { + ++m_index; + if (m_group != m_end && m_index >= m_group->size()) { + ++m_group; + m_index = 0; + skipEmptyGroups(); + } + return *this; + } + + o_packed_offset_index::const_iterator o_packed_offset_index::const_iterator::operator++(int) + { + auto result = *this; + ++(*this); + return result; + } + + bool o_packed_offset_index::const_iterator::operator==(const const_iterator &other) const + { + return m_group == other.m_group && (m_group == m_end || m_index == other.m_index); + } + + bool o_packed_offset_index::const_iterator::operator!=(const const_iterator &other) const + { + return !(*this == other); + } + + o_packed_offset_index::const_iterator o_packed_offset_index::begin() const + { + return const_iterator(getSuper().begin(), getSuper().end()); + } + + o_packed_offset_index::const_iterator o_packed_offset_index::end() const + { + return const_iterator(getSuper().end(), getSuper().end()); + } + std::size_t o_packed_offset_index::size() const { std::size_t result = 0; - for (auto it = begin(); it != end(); ++it) { + for (auto it = getSuper().begin(); it != getSuper().end(); ++it) { result += it->size(); } return result; @@ -308,7 +370,7 @@ namespace db0::object_model } auto targetPackedSize = checkedPackedSize(db0::packed_int64::measure(value)); - for (auto it = begin(); it != end(); ++it) { + for (auto it = getSuper().begin(); it != getSuper().end(); ++it) { if (targetPackedSize < it->elementSize()) { return false; } diff --git a/src/dbzero/object_model/object/o_packed_offset_index.hpp b/src/dbzero/object_model/object/o_packed_offset_index.hpp index 084d0eff..9bfbf879 100644 --- a/src/dbzero/object_model/object/o_packed_offset_index.hpp +++ b/src/dbzero/object_model/object/o_packed_offset_index.hpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -71,12 +72,40 @@ DB0_PACKED_BEGIN public: using list_t = db0::o_list; using super_t = db0::o_ext; - using const_iterator = list_t::const_iterator; + class const_iterator + { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = std::uint64_t; + using difference_type = std::ptrdiff_t; + using pointer = const std::uint64_t *; + using reference = std::uint64_t; + + const_iterator() = default; + + std::uint64_t operator*() const; + const_iterator &operator++(); + const_iterator operator++(int); + bool operator==(const const_iterator &) const; + bool operator!=(const const_iterator &) const; + + private: + friend class o_packed_offset_index; + + const_iterator(list_t::const_iterator group, list_t::const_iterator end); + void skipEmptyGroups(); + + list_t::const_iterator m_group; + list_t::const_iterator m_end; + std::uint32_t m_index = 0; + }; friend super_t; std::size_t size() const; bool contains(std::uint64_t value) const; + const_iterator begin() const; + const_iterator end() const; static std::size_t measure(const std::vector &offsets); diff --git a/src/dbzero/object_model/tags/ObjectTagManager.cpp b/src/dbzero/object_model/tags/ObjectTagManager.cpp index 92709e56..d3354d47 100644 --- a/src/dbzero/object_model/tags/ObjectTagManager.cpp +++ b/src/dbzero/object_model/tags/ObjectTagManager.cpp @@ -90,11 +90,12 @@ namespace db0::object_model ObjectTagManager::ObjectInfo::ObjectInfo(ObjectPtr memo_ptr) : m_lang_ptr(memo_ptr) - , m_object_ptr(&ObjectTagManager::LangToolkit::getTypeManager().extractAnyObject(memo_ptr)) - , m_tag_index_ptr(&m_object_ptr->getFixture()->get()) - , m_type(m_object_ptr->getClassPtr()) - , m_access_mode(m_object_ptr->getFixture()->getAccessType()) - , m_has_tags(LangToolkit::hasTagRefs(memo_ptr)) + , m_fixture(ObjectTagManager::LangToolkit::getTypeManager().extractObjectFixture(memo_ptr)) + , m_tag_index_ptr(&m_fixture->get()) + , m_type(&LangToolkit::getMemoType(memo_ptr)) + , m_access_mode(m_fixture->getAccessType()) + , m_is_embedded(LangToolkit::isEmbeddedMemoObject(memo_ptr)) + , m_has_tags(!m_is_embedded && LangToolkit::hasTagRefs(memo_ptr)) { } @@ -136,7 +137,7 @@ namespace db0::object_model while (type) { // also add type as tag (once) tag_index.addTag(m_lang_ptr.get(), type->getAddress(), true); - type = type->tryGetBaseClass(); + type = type->getBaseClassPtr(); } m_has_tags = true; } @@ -246,7 +247,7 @@ namespace db0::object_model } db0::swine_ptr ObjectTagManager::ObjectInfo::getFixture() const { - return m_object_ptr ? m_object_ptr->getFixture() : db0::swine_ptr(); + return m_fixture; } void ObjectTagManager::onUpdated() diff --git a/src/dbzero/object_model/tags/ObjectTagManager.hpp b/src/dbzero/object_model/tags/ObjectTagManager.hpp index 92091a61..7a92b321 100644 --- a/src/dbzero/object_model/tags/ObjectTagManager.hpp +++ b/src/dbzero/object_model/tags/ObjectTagManager.hpp @@ -46,10 +46,11 @@ namespace db0::object_model struct ObjectInfo { ObjectSharedPtr m_lang_ptr; - const ObjectAnyImpl *m_object_ptr = nullptr; + db0::swine_ptr m_fixture; TagIndex *m_tag_index_ptr = nullptr; - std::shared_ptr m_type; + const Class *m_type = nullptr; AccessType m_access_mode; + bool m_is_embedded = false; // has any tags already assigned bool m_has_tags = false; diff --git a/src/dbzero/object_model/tags/TagIndex.cpp b/src/dbzero/object_model/tags/TagIndex.cpp index c55d709d..0cd8eb66 100644 --- a/src/dbzero/object_model/tags/TagIndex.cpp +++ b/src/dbzero/object_model/tags/TagIndex.cpp @@ -589,11 +589,10 @@ namespace db0::object_model void TagIndex::buildActiveValues() const { for (auto &item: m_active_cache) { - auto &memo = LangToolkit::getTypeManager().extractAnyObject(item.first); // NOTE: defunct objects and mid-init objects (still in __init__, no address yet) // must be skipped — their placeholder stays zero and is preserved for the next flush. - if (!memo.isDefunct() && memo.hasInstance()) { - auto object_addr = memo.getUniqueAddress(); + if (!LangToolkit::isMemoDead(item.first) && LangToolkit::hasMemoInstance(item.first)) { + auto object_addr = LangToolkit::getMemoUniqueAddress(item.first); assert(object_addr.isValid()); // initialize active value with the actual object address item.second = object_addr; @@ -671,8 +670,8 @@ namespace db0::object_model } // Memo instance is directly fed into the FT_FixedKeyIterator - if (type_id == TypeId::MEMO_OBJECT) { - auto addr = LangToolkit::getTypeManager().extractAnyObject(arg).getUniqueAddress(); + if (type_id == TypeId::MEMO_OBJECT || type_id == TypeId::MEMO_IMMUTABLE_OBJECT) { + auto addr = LangToolkit::getMemoUniqueAddress(arg); factory.add(std::make_unique >(&addr, &addr + 1)); return true; } @@ -825,7 +824,7 @@ namespace db0::object_model using TypeId = db0::bindings::TypeId; auto typeId = LangToolkit::getTypeManager().getTypeId(arg); - if (typeId == TypeId::MEMO_OBJECT) { + if (typeId == TypeId::MEMO_OBJECT || typeId == TypeId::MEMO_IMMUTABLE_OBJECT) { return tryAddShortTagFromMemo(arg); } if (typeId == TypeId::DB0_TAG) { @@ -853,6 +852,8 @@ namespace db0::object_model return getShortTagFromFieldDef(py_arg); } else if (type_id == TypeId::DB0_CLASS) { return getShortTagFromClass(py_arg); + } else if (type_id == TypeId::MEMO_OBJECT || type_id == TypeId::MEMO_IMMUTABLE_OBJECT) { + return LangToolkit::getMemoUniqueAddress(py_arg).getAddress().getOffset(); } THROWF(db0::InputException) << "Unable to interpret object of type: " << LangToolkit::getTypeName(py_arg) << " as a tag" << THROWF_END; @@ -969,7 +970,7 @@ namespace db0::object_model { if (type_id == TypeId::STRING) { return addShortTagFromString(py_arg, inc_ref); - } else if (type_id == TypeId::MEMO_OBJECT) { + } else if (type_id == TypeId::MEMO_OBJECT || type_id == TypeId::MEMO_IMMUTABLE_OBJECT) { return tryAddShortTagFromMemo(py_arg); } else if (type_id == TypeId::DB0_TAG) { return tryAddShortTagFromTag(py_arg); @@ -993,13 +994,12 @@ namespace db0::object_model std::optional TagIndex::tryAddShortTagFromMemo(ObjectPtr py_arg) const { assert(LangToolkit::isAnyMemoObject(py_arg)); - auto &py_obj = LangToolkit::getTypeManager().extractAnyObject(py_arg); - if (py_obj.getFixtureUUID() != m_fixture_uuid) { + if (LangToolkit::getFixtureUUID(py_arg) != m_fixture_uuid) { // must be added as long tag return std::nullopt; } // NOTE: we use only the offset part as tag - to distinguish from enum and class tags (high bits) - return py_obj.getAddress().getOffset(); + return LangToolkit::getMemoUniqueAddress(py_arg).getAddress().getOffset(); } std::optional TagIndex::tryAddShortTagFromTag(ObjectPtr py_arg) const @@ -1025,7 +1025,8 @@ namespace db0::object_model bool TagIndex::isShortTag(ObjectPtr py_arg) const { auto type_id = LangToolkit::getTypeManager().getTypeId(py_arg); - return type_id == TypeId::STRING || type_id == TypeId::MEMO_OBJECT || type_id == TypeId::DB0_ENUM_VALUE || + return type_id == TypeId::STRING || type_id == TypeId::MEMO_OBJECT || + type_id == TypeId::MEMO_IMMUTABLE_OBJECT || type_id == TypeId::DB0_ENUM_VALUE || type_id == TypeId::DB0_FIELD_DEF || type_id == TypeId::DB0_ENUM_VALUE_REPR; } @@ -1089,7 +1090,7 @@ namespace db0::object_model // must check for string since it's is an iterable as well if (type_id == TypeId::DB0_TAG) { return getLongTagFromTag(py_arg); - } else if (type_id == TypeId::MEMO_OBJECT) { + } else if (type_id == TypeId::MEMO_OBJECT || type_id == TypeId::MEMO_IMMUTABLE_OBJECT) { return getLongTagFromMemo(py_arg); } else if (type_id == TypeId::STRING || !LangToolkit::isIterable(py_arg)) { THROWF(db0::InputException) << "Invalid argument (iterable expected)" << THROWF_END; @@ -1322,8 +1323,10 @@ namespace db0::object_model LongTagT TagIndex::getLongTagFromMemo(ObjectPtr py_arg) const { assert(LangToolkit::isAnyMemoObject(py_arg)); - auto &py_obj = LangToolkit::getTypeManager().extractAnyObject(py_arg); - return { py_obj.getFixtureUUID(), py_obj.getAddress().getOffset() }; + return { + LangToolkit::getFixtureUUID(py_arg), + LangToolkit::getMemoUniqueAddress(py_arg).getAddress().getOffset() + }; } bool TagIndex::isPendingUpdate(UniqueAddress addr) const { @@ -1332,8 +1335,7 @@ namespace db0::object_model void TagIndex::revert(ObjectPtr memo_ptr) const { - auto &memo = LangToolkit::getTypeManager().extractAnyObject(memo_ptr); - auto addr = memo.getUniqueAddress(); + auto addr = LangToolkit::getMemoUniqueAddress(memo_ptr); if (m_batch_op_short) { m_batch_op_short->revert(addr); } diff --git a/tests/unit_tests/PackedOffsetIndexTest.cpp b/tests/unit_tests/PackedOffsetIndexTest.cpp index c89e47f4..fdd3b4fe 100644 --- a/tests/unit_tests/PackedOffsetIndexTest.cpp +++ b/tests/unit_tests/PackedOffsetIndexTest.cpp @@ -117,6 +117,22 @@ namespace tests ASSERT_FALSE(index->contains(999'999'999)); } + TEST_F( PackedOffsetIndexTest, testIteratorYieldsOffsetsAcrossPackedGroups ) + { + std::vector offsets { + 1, 2, 127, 128, 16'383, 16'384, 2'097'151, 2'097'152, 268'435'456 + }; + auto memspace = getMemspace(); + v_object index(memspace, offsets); + + std::vector result; + for (auto it = index->begin(); it != index->end(); ++it) { + result.push_back(*it); + } + + ASSERT_EQ(result, offsets); + } + TEST_F( PackedOffsetIndexTest, testWidthBoundaryGroupsUseSortedInput ) { std::vector offsets {