diff --git a/python_tests/test_issues_10.py b/python_tests/test_issues_10.py index 4281e6e7..82fe2903 100644 --- a/python_tests/test_issues_10.py +++ b/python_tests/test_issues_10.py @@ -71,6 +71,6 @@ def secret(self) -> int: def test_shopping_cart_secret_issue(db0_fixture): cart_1 = ShoppingCart(as_temp=True) assert cart_1.secret is not None - cart_2 = ShoppingCart(as_temp=False) + cart_2 = ShoppingCart(as_temp=False) assert cart_2.secret is None \ No newline at end of file diff --git a/python_tests/test_memo_immutable.py b/python_tests/test_memo_immutable.py new file mode 100644 index 00000000..5a14cf4c --- /dev/null +++ b/python_tests/test_memo_immutable.py @@ -0,0 +1,23 @@ +from random import random +import pytest +import dbzero as db0 +from dataclasses import dataclass +from .conftest import DB0_DIR +import random + + +@db0.memo(immutable=True, no_default_tags=True) +@dataclass +class MemoImmutableClass1: + data: str + value: int = 0 + +def test_create_memo_immutable(db0_fixture): + _ = MemoImmutableClass1(data="immutable data", value=42) + + +def test_tag_and_find_immutable_instance(db0_fixture): + obj_1 = MemoImmutableClass1(data="immutable data", value=42) + db0.tags(obj_1).add("tag1", "tag2") + assert list(db0.find("tag1")) == [obj_1] + \ No newline at end of file diff --git a/python_tests/test_memo_no_cache.py b/python_tests/test_memo_no_cache.py index 41b656c9..40fca2c8 100644 --- a/python_tests/test_memo_no_cache.py +++ b/python_tests/test_memo_no_cache.py @@ -115,3 +115,54 @@ def test_find_no_cache_objects(db0_fixture): final_cache_size = db0.get_cache_stats()["size"] # make sure cache utilization is low assert abs(final_cache_size - initial_cache_size) < (300 << 10) + + +def test_fetching_no_cache_objects(db0_fixture): + px_name = db0.get_current_prefix().name + buf = db0.list() + uuid_list = [] + for _ in range(100): + obj = MemoNoCacheClass() + buf.append(obj) + uuid_list.append(db0.uuid(obj)) + + db0.close() + db0.init(DB0_DIR) + db0.open(px_name, "r") + + # now fetch objects by uuid + initial_cache_size = db0.get_cache_stats()["size"] + total_len = 0 + for id in uuid_list: + # NOTE: must fetch with type, otherwise no_cache flag may not be honored + obj = db0.fetch(MemoNoCacheClass, id) + # this forces data retrieval + total_len += len(obj.data) + + final_cache_size = db0.get_cache_stats()["size"] + # make sure cache utilization is low + assert abs(final_cache_size - initial_cache_size) < (300 << 10) + + +def test_find_no_cache_objects(db0_fixture): + px_name = db0.get_current_prefix().name + buf = db0.list() + for _ in range(100): + obj = MemoNoCacheClass() + buf.append(obj) + + db0.close() + db0.init(DB0_DIR) + db0.open(px_name, "r") + + # now retrieve objects using db0.find + initial_cache_size = db0.get_cache_stats()["size"] + total_len = 0 + for obj in db0.find(MemoNoCacheClass): + # this forces data retrieval (but not caching) + total_len += len(obj.data) + + assert total_len > 0 + final_cache_size = db0.get_cache_stats()["size"] + # make sure cache utilization is low + assert abs(final_cache_size - initial_cache_size) < (350 << 10) diff --git a/src/dbzero/bindings/TypeId.hpp b/src/dbzero/bindings/TypeId.hpp index c323c40b..be98fc69 100644 --- a/src/dbzero/bindings/TypeId.hpp +++ b/src/dbzero/bindings/TypeId.hpp @@ -53,8 +53,9 @@ namespace db0::bindings MEMO_EXPIRED_REF = 117, // Python type decorated as memo MEMO_TYPE = 118, + MEMO_IMMUTABLE_OBJECT = 119, // COUNT determines size of the type operator arrays - COUNT = 119, + COUNT = 120, // unrecognized type UNKNOWN = std::numeric_limits::max() }; diff --git a/src/dbzero/bindings/python/Memo.cpp b/src/dbzero/bindings/python/Memo.cpp index ab4b8c95..6f576fea 100644 --- a/src/dbzero/bindings/python/Memo.cpp +++ b/src/dbzero/bindings/python/Memo.cpp @@ -31,6 +31,7 @@ namespace db0::python { using ObjectSharedPtr = PyTypes::ObjectSharedPtr; + using TypeObjectSharedPtr = PyTypes::TypeObjectSharedPtr; // @return type name / full type name (tp_name) std::pair getMemoTypeName(shared_py_object py_class) @@ -42,9 +43,9 @@ namespace db0::python return { type_name, full_type_name }; } - MemoObject *tryMemoObject_new(PyTypeObject *py_type, PyObject *, PyObject *) - { - auto &decor = MemoTypeDecoration::get(py_type); + template + MemoImplT *tryMemoObject_new(const MemoTypeDecoration &decor, PyTypeObject *py_type, PyObject *, PyObject *) + { // NOTE: read-only fixture access is sufficient here since objects are lazy-initialized // i.e. the actual dbzero instance is created on postInit // this is also important for dynamically scoped clases (where read/write access may not be possible on default fixture) @@ -53,11 +54,11 @@ namespace db0::python auto &class_factory = fixture->get(); // find py type associated dbzero class with the ClassFactory auto type = class_factory.tryGetOrCreateType(py_type); - MemoObject *memo_obj = reinterpret_cast(py_type->tp_alloc(py_type, 0)); + MemoImplT *memo_obj = reinterpret_cast(py_type->tp_alloc(py_type, 0)); - // if type cannot be retrieved due to access mode then deferr this operation (fallback) + // if type cannot be retrieved due to access mode then defer this operation (fallback) if (type) { - // prepare a new DB0 instance of a known db0 class + // prepare a new dbzero instance of a known db0 class memo_obj->makeNew(type); } else { auto type_initializer = [py_type](db0::swine_ptr &fixture) { @@ -71,10 +72,41 @@ namespace db0::python return memo_obj; } - MemoObject *PyAPI_MemoObject_new(PyTypeObject *py_type, PyObject *args, PyObject *kwargs) + Py_hash_t PyAPI_MemoHash(MemoObject *self) + { + PY_API_FUNC + auto fixture = self->ext().getFixture(); + return runSafe(getPyHashImpl, fixture, self); + } + + template + PyObject *tryMemoObject_str(MemoImplT *self) + { + std::stringstream str; + auto &memo = self->ext(); + str << "<" << Py_TYPE(self)->tp_name; + if (memo.hasInstance()) { + str << " instance uuid=" << PyUnicode_AsUTF8(*Py_OWN(tryGetUUID(self))); + } else { + str << " (uninitialized)"; + } + str << ">"; + return PyUnicode_FromString(str.str().c_str()); + } + + template + PyObject *PyAPI_MemoObject_str(MemoImplT *self) { PY_API_FUNC - return reinterpret_cast(runSafe(tryMemoObject_new, py_type, args, kwargs)); + return runSafe(tryMemoObject_str, self); + } + + template + PyObject *PyAPI_MemoObject_new(PyTypeObject *py_type, PyObject *args, PyObject *kwargs) + { + const auto &decor = MemoTypeDecoration::get(py_type); + PY_API_FUNC + return runSafe(tryMemoObject_new, decor, py_type, args, kwargs); } void tryGetScope(PyTypeObject *py_type, PyObject *args, PyObject *kwargs, @@ -165,7 +197,8 @@ namespace db0::python return PyType_GenericAlloc(self, nitems); } - void MemoObject_del(MemoObject* memo_obj) + template + void MemoObject_del(MemoImplT *memo_obj) { PY_API_FUNC // destroy associated db0 Object instance @@ -173,8 +206,9 @@ namespace db0::python Py_TYPE(memo_obj)->tp_free((PyObject*)memo_obj); } - int PyAPI_MemoObject_init(MemoObject* self, PyObject* args, PyObject* kwds) - { + template + int PyAPI_MemoObject_init(MemoImplT *self, PyObject* args, PyObject* kwds) + { using Class = db0::object_model::Class; using TagIndex = db0::object_model::TagIndex; @@ -273,7 +307,8 @@ namespace db0::python } } - PyObject *tryMemoObject_getattro(MemoObject *memo_obj, PyObject *attr) + template + PyObject *tryMemoObject_getattro(MemoImplT *memo_obj, PyObject *attr) { // The method resolution order for Memo types is following: // 1. User type members (class members such as methods) @@ -291,13 +326,19 @@ namespace db0::python return _PyObject_GenericGetAttrWithDict(reinterpret_cast(memo_obj), attr, NULL, 0); } - PyObject *PyAPI_MemoObject_getattro(MemoObject *self, PyObject *attr) + template + PyObject *PyAPI_MemoObject_getattro(MemoImplT *self, PyObject *attr) { PY_API_FUNC - return runSafe(tryMemoObject_getattro, self, attr); + return runSafe(tryMemoObject_getattro, self, attr); } - int PyAPI_MemoObject_setattro(MemoObject *self, PyObject *attr, PyObject *value) + template + int PyAPI_MemoObject_setattro(MemoImplT *self, PyObject *attr, PyObject *value); + + // regular memo object specialization + template <> + int PyAPI_MemoObject_setattro(MemoObject *self, PyObject *attr, PyObject *value) { PY_API_FUNC // assign value to a dbzero attribute @@ -325,12 +366,45 @@ namespace db0::python return 0; } - - bool isSame(MemoObject *lhs, MemoObject *rhs) { + + // immutable memo object specialization + template <> + int PyAPI_MemoObject_setattro(MemoImmutableObject *self, PyObject *attr, PyObject *value) + { + PY_API_FUNC + // assign value to a dbzero attribute + try { + // must materialize the object before setting as an attribute + if (value && !db0::object_model::isMaterialized(value)) { + db0::FixtureLock lock(self->ext().getFixture()); + db0::object_model::materialize(lock, value); + } + + if (self->ext().hasInstance()) { + PyErr_SetString(PyExc_AttributeError, "Cannot modify an immutable memo object"); + return -1; + } else { + // considered as a non-mutating operation + self->ext().setPreInit(PyUnicode_AsUTF8(attr), value); + } + } catch (const std::exception &e) { + PyErr_SetString(PyExc_AttributeError, e.what()); + return -1; + } catch (...) { + PyErr_SetString(PyExc_AttributeError, "Unknown exception"); + return -1; + } + + return 0; + } + + template + bool isSame(MemoImplT *lhs, MemoImplT *rhs) { return lhs->ext() == rhs->ext(); } - PyObject *PyAPI_MemoObject_rq(MemoObject *memo_obj, PyObject *other, int op) + template + PyObject *PyAPI_MemoObject_rq(MemoImplT *memo_obj, PyObject *other, int op) { PY_API_FUNC PyObject * obj_memo = reinterpret_cast(memo_obj); @@ -338,14 +412,14 @@ namespace db0::python if (obj_memo->ob_type->tp_base->tp_richcompare != PyType_Type.tp_richcompare) { // if the base class richcompare is the same as the memo richcompare don't call the base class richcompare // to avoid infinite recursion - if (obj_memo->ob_type->tp_base->tp_richcompare != (richcmpfunc)PyAPI_MemoObject_rq) { + if (obj_memo->ob_type->tp_base->tp_richcompare != (richcmpfunc)PyAPI_MemoObject_rq) { return obj_memo->ob_type->tp_base->tp_richcompare(reinterpret_cast(memo_obj), other, op); } } bool eq_result = false; - if (PyMemo_Check(other)) { - eq_result = isSame(memo_obj, reinterpret_cast(other)); + if (PyMemo_Check(other)) { + eq_result = isSame(memo_obj, reinterpret_cast(other)); } switch (op) @@ -397,40 +471,66 @@ namespace db0::python return new_dict.steal(); } - static PyType_Slot MemoType_common_slots[] = { - {Py_tp_new, (void *)PyAPI_MemoObject_new}, - {Py_tp_dealloc, (void *)MemoObject_del}, - {Py_tp_init, (void *)PyAPI_MemoObject_init}, - {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro}, - {Py_tp_setattro, (void *)PyAPI_MemoObject_setattro}, - {Py_tp_richcompare, (void *)PyAPI_MemoObject_rq}, + // Regular memo slots + static PyType_Slot MemoObject_common_slots[] = { + {Py_tp_new, (void *)PyAPI_MemoObject_new}, + {Py_tp_dealloc, (void *)MemoObject_del}, + {Py_tp_init, (void *)PyAPI_MemoObject_init}, + {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro}, + {Py_tp_setattro, (void *)PyAPI_MemoObject_setattro}, + {Py_tp_richcompare, (void *)PyAPI_MemoObject_rq}, {Py_tp_hash, (void *)PyAPI_MemoHash}, {0, 0} }; + // Immutable memo slots + static PyType_Slot MemoImmutableObject_common_slots[] = { + {Py_tp_new, (void *)PyAPI_MemoObject_new}, + {Py_tp_dealloc, (void *)MemoObject_del}, + {Py_tp_init, (void *)PyAPI_MemoObject_init}, + {Py_tp_getattro, (void *)PyAPI_MemoObject_getattro}, + // set available only on pre-initialized objects + {Py_tp_setattro, (void *)PyAPI_MemoObject_setattro}, + {Py_tp_richcompare, (void *)PyAPI_MemoObject_rq}, + {Py_tp_hash, (void *)PyAPI_MemoHash}, + {0, 0} + }; + + template + PyType_Slot* getCommonSlots(); + + template <> PyType_Slot *getCommonSlots() { + return MemoObject_common_slots; + } + + template <> PyType_Slot *getCommonSlots() { + return MemoImmutableObject_common_slots; + } + + template PyTypeObject *PyMemoType_FromSpec(PyTypeObject *base_class, const char *tp_name, bool is_singleton) { // fill-in type specific slots first std::vector slots; // fill-in common slots first - for (auto &slot : MemoType_common_slots) { - if (!slot.slot && !slot.pfunc) { - break; - } - slots.push_back(slot); + auto slot_ptr = getCommonSlots(); + while (slot_ptr->slot || slot_ptr->pfunc) { + slots.push_back(*slot_ptr); + ++slot_ptr; } if (is_singleton) { + // NOTE: singletons are not supported for immutable memo objects slots.push_back({Py_tp_new, (void *)PyAPI_MemoObject_new_singleton}); } else { - slots.push_back({Py_tp_new, (void *)PyAPI_MemoObject_new}); + slots.push_back({Py_tp_new, (void *)PyAPI_MemoObject_new}); } - + slots.push_back({0, 0}); auto type_spec = PyType_Spec { .name = tp_name, - .basicsize = MemoObject::sizeOf(), + .basicsize = MemoImplT::sizeOf(), .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE) & ~(Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MANAGED_DICT), .slots = slots.data() }; @@ -443,13 +543,13 @@ namespace db0::python // replace default __str__ and __repr__ implementations if (base_class->tp_str == PyType_Type.tp_str) { - (*tp_result)->tp_str = reinterpret_cast(PyAPI_MemoObject_str); - base_class->tp_str = reinterpret_cast(PyAPI_MemoObject_str); + (*tp_result)->tp_str = reinterpret_cast(PyAPI_MemoObject_str); + base_class->tp_str = reinterpret_cast(PyAPI_MemoObject_str); } - + if (base_class->tp_repr == PyType_Type.tp_repr) { - (*tp_result)->tp_repr = reinterpret_cast(PyAPI_MemoObject_str); - base_class->tp_repr = reinterpret_cast(PyAPI_MemoObject_str); + (*tp_result)->tp_repr = reinterpret_cast(PyAPI_MemoObject_str); + base_class->tp_repr = reinterpret_cast(PyAPI_MemoObject_str); } return tp_result.steal(); @@ -457,7 +557,7 @@ namespace db0::python PyObject *wrapPyType(PyTypeObject *base_class, bool is_singleton, bool no_default_tags, const char *prefix_name, const char *type_id, const char *file_name, std::vector &&init_vars, PyObject *py_dyn_prefix_callable, - std::vector &&migrations, bool no_cache) + std::vector &&migrations, bool no_cache, bool immutable) { auto py_class = Py_BORROW(base_class); auto py_module = Py_OWN(findModule(*Py_OWN(PyObject_GetAttrString((PyObject*)*py_class, "__module__")))); @@ -468,13 +568,19 @@ namespace db0::python if ((*py_class)->tp_dict == nullptr) { THROWF(db0::InternalException) << "Type has no tp_dict: " << (*py_class)->tp_name; } - + if ((*py_class)->tp_itemsize != 0) { THROWF(db0::InternalException) << "Variable-length types not supported: " << (*py_class)->tp_name; } auto [type_name, full_type_name] = getMemoTypeName(py_class); - auto new_type = Py_OWN(PyMemoType_FromSpec(base_class, full_type_name.c_str(), is_singleton)); + TypeObjectSharedPtr new_type = nullptr; + // NOTE: MemoObject and MemoImmutableObject have different implementations + if (immutable) { + new_type = Py_OWN(PyMemoType_FromSpec(base_class, full_type_name.c_str(), is_singleton)); + } else { + new_type = Py_OWN(PyMemoType_FromSpec(base_class, full_type_name.c_str(), is_singleton)); + } if (!new_type) { return nullptr; } @@ -484,6 +590,9 @@ namespace db0::python if (no_cache) { type_flags.set(MemoOptions::NO_CACHE); } + if (immutable) { + type_flags.set(MemoOptions::IMMUTABLE); + } auto type_info = MemoTypeDecoration( py_module, prefix_name, @@ -523,11 +632,12 @@ namespace db0::python // migrations are only processed for singleton types PyObject *py_migrations = nullptr; PyObject *py_no_cache = nullptr; + PyObject *py_immutable = nullptr; static const char *kwlist[] = { "input", "singleton", "no_default_tags", "prefix", "id", "py_file", "py_init_vars", - "py_dyn_prefix", "py_migrations", "no_cache", NULL }; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOOOOOOOO", const_cast(kwlist), &class_obj, &py_singleton, - &py_no_default_tags, &py_prefix_name, &py_type_id, &py_file_name, &py_init_vars, &py_dyn_prefix, &py_migrations, &py_no_cache)) + "py_dyn_prefix", "py_migrations", "no_cache", "immutable", NULL }; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOOOOOOOOO", const_cast(kwlist), &class_obj, &py_singleton, + &py_no_default_tags, &py_prefix_name, &py_type_id, &py_file_name, &py_init_vars, &py_dyn_prefix, &py_migrations, &py_no_cache, &py_immutable)) { return NULL; } @@ -535,6 +645,7 @@ namespace db0::python bool is_singleton = py_singleton && PyObject_IsTrue(py_singleton); bool no_default_tags = py_no_default_tags && PyObject_IsTrue(py_no_default_tags); bool no_cache = py_no_cache && PyObject_IsTrue(py_no_cache); + bool immutable = py_immutable && PyObject_IsTrue(py_immutable); const char *prefix_name = (py_prefix_name && py_prefix_name != Py_None) ? PyUnicode_AsUTF8(py_prefix_name) : nullptr; const char *type_id = py_type_id ? PyUnicode_AsUTF8(py_type_id) : nullptr; const char *file_name = (py_file_name && py_file_name != Py_None) ? PyUnicode_AsUTF8(py_file_name) : nullptr; @@ -569,7 +680,7 @@ namespace db0::python auto migrations = extractMigrations(py_migrations); return wrapPyType(castToType(class_obj), is_singleton, no_default_tags, prefix_name, type_id, file_name, - std::move(init_vars), py_dyn_prefix, std::move(migrations), no_cache + std::move(init_vars), py_dyn_prefix, std::move(migrations), no_cache, immutable ); } @@ -581,7 +692,7 @@ namespace db0::python PyObject *tryPyMemoCheck(PyObject *py_obj) { - if (PyMemo_Check(py_obj) || (PyType_Check(py_obj) && PyMemoType_Check(reinterpret_cast(py_obj)))) { + if (PyAnyMemo_Check(py_obj) || (PyType_Check(py_obj) && PyAnyMemoType_Check(reinterpret_cast(py_obj)))) { Py_RETURN_TRUE; } Py_RETURN_FALSE; @@ -601,9 +712,10 @@ namespace db0::python return type->tp_new == reinterpret_cast(PyAPI_MemoObject_new_singleton); } - PyObject *MemoObject_GetFieldLayout(MemoObject *self) + template + PyObject *MemoObject_GetFieldLayout(MemoImplT *self) { - auto field_layout = self->ext().getFieldLayout(); + auto field_layout = self->ext().getFieldLayout(); auto &pos_vt = field_layout.m_pos_vt_fields; auto &index_vt = field_layout.m_index_vt_fields; @@ -641,8 +753,9 @@ namespace db0::python return py_result.steal(); } - PyObject *MemoObject_DescribeObject(MemoObject *self) - { + template + PyObject *MemoObject_DescribeObject(MemoImplT *self) + { auto py_field_layout = Py_OWN(MemoObject_GetFieldLayout(self)); auto py_result = Py_OWN(PyDict_New()); PySafeDict_SetItemString(*py_result, "field_layout", py_field_layout); @@ -652,26 +765,6 @@ namespace db0::python return py_result.steal(); } - PyObject *tryMemoObject_str(MemoObject *self) - { - std::stringstream str; - auto &memo = self->ext(); - str << "<" << Py_TYPE(self)->tp_name; - if (memo.hasInstance()) { - str << " instance uuid=" << PyUnicode_AsUTF8(*Py_OWN(tryGetUUID(self))); - } else { - str << " (uninitialized)"; - } - str << ">"; - return PyUnicode_FromString(str.str().c_str()); - } - - PyObject *PyAPI_MemoObject_str(MemoObject *self) - { - PY_API_FUNC - return runSafe(tryMemoObject_str, self); - } - void MemoType_get_info(PyTypeObject *type, PyObject *dict) { auto &decor = MemoTypeDecoration::get(type); @@ -683,18 +776,20 @@ namespace db0::python MemoTypeDecoration::get(type).close(); } - PyObject *MemoObject_set_prefix(MemoObject *py_obj, const char *prefix_name) + template + PyObject *MemoObject_set_prefix(MemoImplT *py_obj, const char *prefix_name) { if (prefix_name) { - // can use "ext" since setFixtue is a non-mutating operation - auto &obj = const_cast(py_obj->ext()); + using ObjectT = typename MemoImplT::ExtT; + // can use "ext" since setFixture is a non-mutating operation + auto &obj = const_cast(py_obj->ext()); auto fixture = PyToolkit::getPyWorkspace().getWorkspace().getFixture(prefix_name, AccessType::READ_WRITE); obj.setFixture(fixture); return PyUnicode_FromString(prefix_name); } Py_RETURN_NONE; } - + PyObject *tryGetAttributes(PyTypeObject *type) { auto &decor = MemoTypeDecoration::get(type); @@ -722,7 +817,8 @@ namespace db0::python } } - PyObject *tryGetAttrAs(MemoObject *memo_obj, PyObject *attr, PyTypeObject *py_type) + template + PyObject *tryGetAttrAs(MemoImplT *memo_obj, PyObject *attr, PyTypeObject *py_type) { memo_obj->ext().getFixture()->refreshIfUpdated(); auto member = memo_obj->ext().tryGetAs(PyUnicode_AsUTF8(attr), py_type); @@ -792,7 +888,8 @@ namespace db0::python return tryLoad(*result, kwargs, nullptr, load_stack_ptr); } - PyObject *tryLoadMemo(MemoObject *memo_obj, PyObject *kwargs, PyObject *py_exclude, + template + PyObject *tryLoadMemo(MemoImplT *memo_obj, PyObject *kwargs, PyObject *py_exclude, std::unordered_set *load_stack_ptr) { auto load_method = Py_OWN(tryMemoObject_getattro(memo_obj, *Py_OWN(PyUnicode_FromString("__load__")))); @@ -843,19 +940,12 @@ namespace db0::python } Py_RETURN_FALSE; } - - Py_hash_t PyAPI_MemoHash(MemoObject *self) - { - PY_API_FUNC - auto fixture = self->ext().getFixture(); - return runSafe(getPyHashImpl, fixture, self); - } - + PyObject *tryGetSchema(PyTypeObject *py_type) { using SchemaTypeId = db0::object_model::SchemaTypeId; - if (!PyMemoType_Check(py_type)) { + if (!PyAnyMemoType_Check(py_type)) { PyErr_SetString(PyExc_TypeError, "Expected a memo type"); return nullptr; } @@ -865,7 +955,7 @@ namespace db0::python auto &class_factory = fixture->get(); // find py type associated dbzero class with the ClassFactory auto type = class_factory.getExistingType(py_type); - + auto py_schema = Py_OWN(PyDict_New()); auto &type_manager = PyToolkit::getTypeManager(); type->getSchema([&](const std::string &key, SchemaTypeId primary_type, const std::vector &all_types) { @@ -899,5 +989,41 @@ namespace db0::python PY_API_FUNC return runSafe(tryGetSchema, reinterpret_cast(args[0])); } + + bool PyAnyMemoType_Check(PyTypeObject *type) + { + assert(type); + return type->tp_dealloc == reinterpret_cast(MemoObject_del) || + type->tp_dealloc == reinterpret_cast(MemoObject_del); + } + template + bool PyMemo_Check(PyObject *obj) + { + assert(obj); + return obj->ob_type->tp_dealloc == reinterpret_cast(MemoObject_del); + } + + template + bool PyMemoType_Check(PyTypeObject *type) + { + assert(type); + return type->tp_dealloc == reinterpret_cast(MemoObject_del); + } + + template bool PyMemo_Check(PyObject *); + template bool PyMemo_Check(PyObject *); + template bool PyMemoType_Check(PyTypeObject *); + template bool PyMemoType_Check(PyTypeObject *); + template PyObject *MemoObject_DescribeObject(MemoObject *); + template PyObject *MemoObject_DescribeObject(MemoImmutableObject *); + template PyObject *MemoObject_set_prefix(MemoObject *, const char *); + template PyObject *MemoObject_set_prefix(MemoImmutableObject *, const char *); + template PyObject *tryGetAttrAs(MemoObject *, PyObject *, PyTypeObject *); + template PyObject *tryGetAttrAs(MemoImmutableObject *, PyObject *, PyTypeObject *); + template PyObject *tryLoadMemo(MemoObject *, PyObject*, PyObject*, + std::unordered_set *); + template PyObject *tryLoadMemo(MemoImmutableObject *, PyObject*, PyObject*, + std::unordered_set *); + } diff --git a/src/dbzero/bindings/python/Memo.hpp b/src/dbzero/bindings/python/Memo.hpp index b9b3fb2f..e9fcf125 100644 --- a/src/dbzero/bindings/python/Memo.hpp +++ b/src/dbzero/bindings/python/Memo.hpp @@ -9,6 +9,9 @@ #include #include "Migration.hpp" #include "MemoTypeDecoration.hpp" +#include +#include +#include namespace db0::object_model @@ -24,49 +27,70 @@ namespace db0::python using AccessType = db0::AccessType; using MemoObject = PyWrapper; - + using MemoImmutableObject = PyWrapper; + using MemoAnyObject = PyWrapper; + PyObject *PyAPI_wrapPyClass(PyObject *self, PyObject *, PyObject *kwargs); - MemoObject *PyAPI_MemoObject_new(PyTypeObject *type, PyObject * = nullptr, PyObject * = nullptr); // create a memo object stub MemoObject* MemoObjectStub_new(PyTypeObject *type); - PyObject *MemoObject_alloc(PyTypeObject *type, Py_ssize_t nitems); void MemoObject_del(MemoObject* self); void MemoObject_drop(MemoObject* self); - int PyAPI_MemoObject_init(MemoObject* self, PyObject* args, PyObject* kwds); - PyObject *PyAPI_MemoObject_getattro(MemoObject *self, PyObject *attr); - int PyAPI_MemoObject_setattro(MemoObject *self, PyObject *attr, PyObject *value); - Py_hash_t PyAPI_MemoHash(MemoObject *); // check if memo type has been marked as singleton bool PyMemoType_IsSingleton(PyTypeObject *type); - PyObject *MemoObject_GetFieldLayout(MemoObject *); - - PyObject *MemoObject_DescribeObject(MemoObject *); - PyObject *PyAPI_MemoObject_str(MemoObject *); + template + PyObject *MemoObject_DescribeObject(MemoImplT *); void MemoType_get_info(PyTypeObject *type, PyObject *dict); void MemoType_close(PyTypeObject *type); - PyObject *MemoObject_set_prefix(MemoObject *, const char *prefix_name); + template + PyObject *MemoObject_set_prefix(MemoImplT *, const char *prefix_name); PyObject *tryGetAttributes(PyTypeObject *type); + // Try retrieving a memo member cast to a specific type // type ignored for non-memo members - PyObject *tryGetAttrAs(MemoObject *, PyObject *attr, PyTypeObject *); + template + PyObject *tryGetAttrAs(MemoImplT *, PyObject *attr, PyTypeObject *); - PyObject *tryLoadMemo(MemoObject *memo_obj, PyObject* kwargs, PyObject* exclude, + template + PyObject *tryLoadMemo(MemoImplT *memo_obj, PyObject* kwargs, PyObject* exclude, std::unordered_set *load_stack_ptr = nullptr); + // check for a memo type (i.e. generated by PyAPI_wrapPyClass) + template bool PyMemo_Check(PyObject *); + template bool PyMemoType_Check(PyTypeObject *); + + bool PyAnyMemo_Check(PyObject *); + bool PyAnyMemoType_Check(PyTypeObject *); + PyObject *PyAPI_PyMemo_Check(PyObject *self, PyObject *const * args, Py_ssize_t nargs); // Binary (shallow) compare 2 objects or 2 versions of the same memo object (e.g. from different snapshots) // NOTE: ref-counts are not compared (only user-assigned members) // @return true if objects are identical PyObject *tryCompareMemo(MemoObject *, MemoObject *); - + PyObject *PyAPI_getSchema(PyObject *, PyObject *const *args, Py_ssize_t nargs); - PyObject* executeLoadFunction(PyObject * load_method, PyObject *kwargs, PyObject *py_exclude, - std::unordered_set *load_stack_ptr); -} \ No newline at end of file + PyObject* executeLoadFunction(PyObject *load_method, PyObject *kwargs, PyObject *py_exclude, + std::unordered_set *load_stack_ptr); + + extern template bool PyMemo_Check(PyObject *); + extern template bool PyMemo_Check(PyObject *); + extern template bool PyMemoType_Check(PyTypeObject *); + extern template bool PyMemoType_Check(PyTypeObject *); + extern template PyObject *MemoObject_DescribeObject(MemoObject *); + extern template PyObject *MemoObject_DescribeObject(MemoImmutableObject *); + extern template PyObject *MemoObject_set_prefix(MemoObject *, const char *); + extern template PyObject *MemoObject_set_prefix(MemoImmutableObject *, const char *); + extern template PyObject *tryGetAttrAs(MemoObject *, PyObject *, PyTypeObject *); + extern template PyObject *tryGetAttrAs(MemoImmutableObject *, PyObject *, PyTypeObject *); + extern template PyObject *tryLoadMemo(MemoObject *, PyObject*, PyObject*, + std::unordered_set *); + extern template PyObject *tryLoadMemo(MemoImmutableObject *, PyObject*, PyObject*, + std::unordered_set *); + +} diff --git a/src/dbzero/bindings/python/MemoTypeDecoration.cpp b/src/dbzero/bindings/python/MemoTypeDecoration.cpp index c71ab4ad..99a6bc95 100644 --- a/src/dbzero/bindings/python/MemoTypeDecoration.cpp +++ b/src/dbzero/bindings/python/MemoTypeDecoration.cpp @@ -8,16 +8,10 @@ namespace db0::python { - bool PyMemo_Check(PyObject *obj) + bool PyAnyMemo_Check(PyObject *obj) { auto py_type = Py_TYPE(obj); - return py_type && PyMemoType_Check(py_type); - } - - bool PyMemoType_Check(PyTypeObject *type) - { - assert(type); - return type->tp_dealloc == reinterpret_cast(MemoObject_del); + return py_type && PyAnyMemoType_Check(py_type); } MemoTypeDecoration::MemoTypeDecoration(MemoTypeDecoration &&other) @@ -88,7 +82,7 @@ namespace db0::python } } - std::uint64_t MemoTypeDecoration::getFixtureUUID(std::optional access_type) + std::uint64_t MemoTypeDecoration::getFixtureUUID(std::optional access_type) const { if (!!m_prefix_name && !m_fixture_uuid) { if (!access_type) { @@ -145,7 +139,7 @@ namespace db0::python MemoTypeDecoration &MemoTypeDecoration::get(PyTypeObject *type) { - assert(PyMemoType_Check(type) && "Invalid type (expected memo type)"); + assert(PyAnyMemoType_Check(type) && "Invalid type (expected memo type)"); return PyToolkit::getTypeManager().getMemoTypeDecoration(type); } diff --git a/src/dbzero/bindings/python/MemoTypeDecoration.hpp b/src/dbzero/bindings/python/MemoTypeDecoration.hpp index ac95941f..b48767f6 100644 --- a/src/dbzero/bindings/python/MemoTypeDecoration.hpp +++ b/src/dbzero/bindings/python/MemoTypeDecoration.hpp @@ -21,10 +21,6 @@ namespace db0::python using MemoOptions = db0::object_model::MemoOptions; using MemoFlags = db0::object_model::MemoFlags; - // check for a memo type (i.e. generated by PyAPI_wrapPyClass) - bool PyMemo_Check(PyObject *obj); - bool PyMemoType_Check(PyTypeObject *type); - using AccessType = db0::AccessType; class MemoTypeDecoration @@ -71,11 +67,11 @@ namespace db0::python // @param access_type to use for opening the prefix if UUID needs to be resolved by name // note that read-only access cannot later be upgraded to read-write // NOTE: if access type is not provided (std::nullopt), then READ_WRITE will be used as the default - std::uint64_t getFixtureUUID(std::optional access_type = AccessType::READ_WRITE); + std::uint64_t getFixtureUUID(std::optional access_type = AccessType::READ_WRITE) const; // check if the dyn-prefix callable is set bool hasDynPrefix() const; - + // resolve dynamic prefix from the callable std::string getDynPrefix(PyObject *args, PyObject *kwargs) const; @@ -104,7 +100,7 @@ namespace db0::python std::vector m_init_vars; MemoFlags m_flags; // resolved fixture UUID (initialized by the process) - std::atomic m_fixture_uuid = 0; + mutable std::atomic m_fixture_uuid = 0; // dynamic prefix callable shared_py_object m_py_dyn_prefix_callable; std::vector m_migrations; diff --git a/src/dbzero/bindings/python/PyAPI.cpp b/src/dbzero/bindings/python/PyAPI.cpp index 1f0d67bc..09154342 100644 --- a/src/dbzero/bindings/python/PyAPI.cpp +++ b/src/dbzero/bindings/python/PyAPI.cpp @@ -413,7 +413,7 @@ namespace db0::python if (PyType_Check(py_object)) { // only memo or enum types can be scoped - if (PyMemoType_Check(reinterpret_cast(py_object))) { + if (PyAnyMemoType_Check(reinterpret_cast(py_object))) { PyTypeObject *py_type = reinterpret_cast(py_object); auto prefix_name = MemoTypeDecoration::get(py_type).tryGetPrefixName(); if (prefix_name) { @@ -548,13 +548,15 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; } - - if (!PyMemo_Check(py_object)) { + + if (PyMemo_Check(py_object)) { + return MemoObject_DescribeObject(reinterpret_cast(py_object)); + } else if (PyMemo_Check(py_object)) { + return MemoObject_DescribeObject(reinterpret_cast(py_object)); + } else { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; - } - - return MemoObject_DescribeObject(reinterpret_cast(py_object)); + } } PyObject *describeObject(PyObject *self, PyObject *args) @@ -593,7 +595,7 @@ namespace db0::python PyObject *TryPyAPI_isSingleton(PyObject *py_object) { - + assert((PyMemo_Check(py_object))); return PyBool_fromBool(reinterpret_cast(py_object)->ext().isSingleton()); } @@ -605,14 +607,17 @@ namespace db0::python return NULL; } - if (!PyMemo_Check(py_object)) { + if (PyMemo_Check(py_object)) { + return runSafe(TryPyAPI_isSingleton, py_object); + } else if (PyMemo_Check(py_object)) { + // immutable memos are never singletons + Py_RETURN_FALSE; + } else { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; - } - - return runSafe(TryPyAPI_isSingleton, py_object); + } } - + PyObject *PyAPI_getRefCount(PyObject *, PyObject *args) { PY_API_FUNC @@ -818,11 +823,11 @@ namespace db0::python Py_RETURN_NONE; } - if (PyMemo_Check(py_object)) { + if (PyAnyMemo_Check(py_object)) { // this operation is handled differently for singletons (as dyn_prefix) // check from type, not instance if (PyMemoType_IsSingleton(Py_TYPE(py_object))) { - if (reinterpret_cast(py_object)->ext().hasInstance()) { + if (reinterpret_cast(py_object)->ext().hasInstance()) { PyErr_SetString(PyExc_TypeError, "Unable to change scope of an existing singleton instance"); return NULL; } @@ -831,7 +836,15 @@ namespace db0::python } Py_RETURN_NONE; } else { - return runSafe(MemoObject_set_prefix, reinterpret_cast(py_object), prefix_name); + if (PyMemo_Check(py_object)) { + return runSafe(MemoObject_set_prefix, + reinterpret_cast(py_object), prefix_name + ); + } else { + return runSafe(MemoObject_set_prefix, + reinterpret_cast(py_object), prefix_name + ); + } } } @@ -944,13 +957,13 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; } - - if (!PyMemoType_Check(py_type)) { + + if (PyAnyMemoType_Check(py_type)) { + return runSafe(tryGetAttributes, py_type); + } else { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; - } - - return runSafe(tryGetAttributes, py_type); + } } PyObject *PyAPI_getAttrAs(PyObject *self, PyObject *const *args, Py_ssize_t nargs) @@ -961,8 +974,8 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "getattr_as requires exactly 2 arguments"); return NULL; } - - if (!PyMemo_Check(args[0])) { + + if (!PyAnyMemo_Check(args[0])) { // fall back to regular getattr if not a memo object return PyObject_GetAttr(args[0], args[1]); } @@ -973,7 +986,11 @@ namespace db0::python } PyTypeObject *py_type = reinterpret_cast(args[2]); - return runSafe(tryGetAttrAs, reinterpret_cast(args[0]), args[1], py_type); + if (PyMemo_Check(args[0])) { + return runSafe(tryGetAttrAs, reinterpret_cast(args[0]), args[1], py_type); + } else { + return runSafe(tryGetAttrAs, reinterpret_cast(args[0]), args[1], py_type); + } } PyObject *PyAPI_getAddress(PyObject *self, PyObject *const *args, Py_ssize_t nargs) @@ -1044,7 +1061,7 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "Invalid argument type. Exclude shoud be a sequence"); return NULL; } - if (!PyMemo_Check(py_object)) { + if (!PyAnyMemo_Check(py_object)) { PyErr_SetString(PyExc_TypeError, "Exclude is only supported for memo objects"); return NULL; } @@ -1078,10 +1095,25 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "materialized requires exactly 1 argument"); return NULL; } + auto py_obj = args[0]; + if (!PyAnyMemo_Check(py_obj)) { + // simply return self if not a memo object + Py_INCREF(py_obj); + return py_obj; + } + PY_API_FUNC - return runSafe(getMaterializedMemoObject, args[0]); + if (PyMemo_Check(py_obj)) { + return runSafe(getMaterializedMemoObject, + reinterpret_cast(py_obj) + ); + } else { + return runSafe(getMaterializedMemoObject, + reinterpret_cast(py_obj) + ); + } } - + PyObject *tryWait(const char *prefix, long state, long timeout) { std::unique_lock api_lock; @@ -1344,14 +1376,15 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; } - - if (!PyMemo_Check(py_first) || !PyMemo_Check(py_second)) { + + // FIXME: implement for MemoImmutableObject as well + if (!PyMemo_Check(py_first) || !PyMemo_Check(py_second)) { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; } return runSafe(tryCompareMemo, reinterpret_cast(py_first), reinterpret_cast(py_second)); - } + } #ifndef NDEBUG PyObject *PyAPI_startDebugLogs(PyObject *self, PyObject *) @@ -1413,7 +1446,7 @@ namespace db0::python PyObject *PyAPI_touch(PyObject *, PyObject *const *args, Py_ssize_t nargs) { for (Py_ssize_t i = 0; i < nargs; ++i) { - if (!PyMemo_Check(args[i])) { + if (!PyAnyMemo_Check(args[i])) { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; } diff --git a/src/dbzero/bindings/python/PyInternalAPI.cpp b/src/dbzero/bindings/python/PyInternalAPI.cpp index cde42726..6eb91fd6 100644 --- a/src/dbzero/bindings/python/PyInternalAPI.cpp +++ b/src/dbzero/bindings/python/PyInternalAPI.cpp @@ -238,7 +238,8 @@ namespace db0::python PyObject *fetchSingletonObject(db0::Snapshot &snapshot, PyTypeObject *py_type, const char *prefix_name) { - if (!PyMemoType_Check(py_type)) { + // NOTE: singletons only supported for MemoObject types + if (!PyMemoType_Check(py_type)) { THROWF(db0::InternalException) << "Memo type expected for: " << py_type->tp_name << THROWF_END; } @@ -257,7 +258,8 @@ namespace db0::python bool isExistingSingleton(db0::Snapshot &snapshot, PyTypeObject *py_type, const char *prefix_name) { - if (!PyMemoType_Check(py_type)) { + // NOTE: singletons only supported for MemoObject types + if (!PyMemoType_Check(py_type)) { return false; } @@ -276,14 +278,13 @@ namespace db0::python } return isExistingSingleton(fixture, py_type); } - - + void renameMemoClassField(PyTypeObject *py_type, const char *from_name, const char *to_name) { using ClassFactory = db0::object_model::ClassFactory; auto fixture_uuid = MemoTypeDecoration::get(py_type).getFixtureUUID(); - assert(PyMemoType_Check(py_type)); + assert(PyAnyMemoType_Check(py_type)); assert(from_name); assert(to_name); @@ -293,9 +294,8 @@ namespace db0::python auto type = class_factory.getExistingType(py_type); type->renameField(from_name, to_name); } - -#ifndef NDEBUG +#ifndef NDEBUG PyObject *writeBytes(PyObject *self, PyObject *args) { @@ -433,26 +433,35 @@ namespace db0::python } return result; } + + template + PyObject *tryMemoGetRefCount(PyObject *py_object) + { + assert(PyMemo_Check(py_object)); + auto &memo = reinterpret_cast(py_object)->ext(); + auto fixture = memo.getFixture(); + // NOTE: there might be tag-removal operations buffered for the requested instance + // in case of a read/write mode conditionally trigger flush in such case + if (db0::object_model::isObjectPendingUpdate(fixture, memo.getUniqueAddress())) { + FixtureLock lock(fixture); + // flush pending updates + lock->flush(); + } + return PyLong_FromLong(getTotal(memo.getRefCounts(), -memo->m_num_type_tags)); + } PyObject *tryGetRefCount(PyObject *py_object) { - if (PyMemo_Check(py_object)) { - auto &memo = reinterpret_cast(py_object)->ext(); - auto fixture = memo.getFixture(); - // NOTE: there might be tag-removal operations buffered for the requested instance - // in case of a read/write mode conditionally trigger flush in such case - if (db0::object_model::isObjectPendingUpdate(fixture, memo.getUniqueAddress())) { - FixtureLock lock(fixture); - // flush pending updates - lock->flush(); - } - return PyLong_FromLong(getTotal(memo.getRefCounts(), -memo->m_num_type_tags)); + if (PyMemo_Check(py_object)) { + return tryMemoGetRefCount(py_object); + } else if (PyMemo_Check(py_object)) { + return tryMemoGetRefCount(py_object); } else if (PyClassObject_Check(py_object)) { auto ref_counts = reinterpret_cast(py_object)->ext().getRefCounts(); return PyLong_FromLong(getTotal(ref_counts, 0)); } else if (PyType_Check(py_object)) { auto py_type = PyToolkit::getTypeManager().getTypeObject(py_object); - if (PyToolkit::isMemoType(py_type)) { + if (PyToolkit::isAnyMemoType(py_type)) { auto &workspace = PyToolkit::getPyWorkspace().getWorkspace(); // sum over all prefixes std::uint64_t ref_counts = 0; @@ -615,8 +624,10 @@ namespace db0::python PyObject *tryGetAddress(PyObject *py_obj) { - if (PyMemo_Check(py_obj)) { - return PyLong_FromUnsignedLongLong(reinterpret_cast(py_obj)->ext().getAddress().getValue()); + if (PyAnyMemo_Check(py_obj)) { + return PyLong_FromUnsignedLongLong( + reinterpret_cast(py_obj)->ext().getAddress().getValue() + ); } // FIXME: implement for other dbzero types @@ -626,8 +637,8 @@ namespace db0::python PyTypeObject *tryGetType(PyObject *py_obj) { - if (PyMemo_Check(py_obj)) { - auto &memo = reinterpret_cast(py_obj)->ext(); + if (PyAnyMemo_Check(py_obj)) { + auto &memo = reinterpret_cast(py_obj)->ext(); auto fixture = memo.getFixture(); auto &class_factory = fixture->get(); if (!class_factory.hasLangType(memo.getType())) { @@ -646,9 +657,9 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return NULL; } - + PyTypeObject *py_type = reinterpret_cast(py_object); - if (!PyMemoType_Check(py_type)) { + if (!PyAnyMemoType_Check(py_type)) { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return nullptr; } @@ -664,18 +675,18 @@ namespace db0::python PyObject *tryGetMemoClass(PyObject *py_obj) { - if (!PyMemo_Check(py_obj)) { + if (!PyAnyMemo_Check(py_obj)) { PyErr_SetString(PyExc_TypeError, "Invalid argument type"); return nullptr; } - auto &memo_obj = reinterpret_cast(py_obj)->ext(); + auto &memo_obj = reinterpret_cast(py_obj)->ext(); if (!memo_obj.hasInstance()) { PyErr_SetString(PyExc_RuntimeError, "Memo object has no instance"); return nullptr; } return tryGetTypeInfo(memo_obj.getType()); } - + PyObject *tryLoad(PyObject *py_obj, PyObject* kwargs, PyObject *py_exclude, std::unordered_set *load_stack_ptr) { @@ -721,30 +732,27 @@ namespace db0::python return tryLoadEnumValue(reinterpret_cast(py_obj)); } else if (type_id == TypeId::MEMO_OBJECT) { return tryLoadMemo(reinterpret_cast(py_obj), kwargs, py_exclude, load_stack_ptr); + } else if (type_id == TypeId::MEMO_IMMUTABLE_OBJECT) { + return tryLoadMemo(reinterpret_cast(py_obj), kwargs, py_exclude, load_stack_ptr); } else { THROWF(db0::InputException) << "__load__ not implemented for type: " << Py_TYPE(py_obj)->tp_name << THROWF_END; } } - PyObject *getMaterializedMemoObject(PyObject *py_obj) + template + PyObject *getMaterializedMemoObject(MemoImplT *memo_obj) { - if (!PyMemo_Check(py_obj)) { - // simply return self if not a memo object - Py_INCREF(py_obj); - return py_obj; - } - auto memo_obj = reinterpret_cast(py_obj); if (memo_obj->ext().hasInstance()) { - Py_INCREF(py_obj); - return py_obj; + Py_INCREF(memo_obj); + return memo_obj; } db0::FixtureLock lock(memo_obj->ext().getFixture()); // materialize by calling postInit memo_obj->modifyExt().postInit(lock); - Py_INCREF(py_obj); - return py_obj; + Py_INCREF(memo_obj); + return memo_obj; } shared_py_object tryUnloadObjectFromCache(LangCacheView &lang_cache, Address address, @@ -757,10 +765,10 @@ namespace db0::python } if (expected_type) { - if (!PyMemo_Check(obj_ptr.get())) { + if (!PyAnyMemo_Check(obj_ptr.get())) { THROWF(db0::InputException) << "Invalid object type: " << PyToolkit::getTypeName(obj_ptr.get()) << " (Memo expected)"; } - auto &memo = reinterpret_cast(obj_ptr.get())->ext(); + auto &memo = reinterpret_cast(obj_ptr.get())->ext(); // validate type if (memo.getType() != *expected_type) { THROWF(db0::InputException) << "Memo type mismatch"; @@ -882,17 +890,20 @@ namespace db0::python { for (Py_ssize_t i = 0; i < nargs; ++i) { auto py_obj = args[i]; - if (!PyMemo_Check(py_obj)) { + if (!PyAnyMemo_Check(py_obj)) { THROWF(db0::InputException) << "Invalid object type: " << Py_TYPE(py_obj)->tp_name << " (Memo expected)"; } - auto &memo = reinterpret_cast(py_obj)->modifyExt(); + auto &memo = reinterpret_cast(py_obj)->modifyExt(); if (memo.hasInstance()) { db0::FixtureLock lock(memo.getFixture()); memo.touch(); - } + } } Py_RETURN_NONE; } - + + template PyObject *getMaterializedMemoObject(MemoObject *); + template PyObject *getMaterializedMemoObject(MemoImmutableObject *); + } \ No newline at end of file diff --git a/src/dbzero/bindings/python/PyInternalAPI.hpp b/src/dbzero/bindings/python/PyInternalAPI.hpp index ddebb5a1..52320513 100644 --- a/src/dbzero/bindings/python/PyInternalAPI.hpp +++ b/src/dbzero/bindings/python/PyInternalAPI.hpp @@ -198,7 +198,8 @@ namespace db0::python PyObject *tryLoad(PyObject *, PyObject*, PyObject *py_exlude = nullptr, std::unordered_set *load_stack_ptr = nullptr); - PyObject *getMaterializedMemoObject(PyObject *py_obj); + template + PyObject *getMaterializedMemoObject(MemoImplT *py_obj); // Retrieve prefix (its Fixture objects) from the optional argument "prefix" db0::swine_ptr getOptionalPrefixFromArg(db0::Snapshot &workspace, const char *prefix_name); @@ -233,6 +234,9 @@ namespace db0::python #endif PyObject *tryAssign(PyObject *targets, PyObject *key_values); - + + extern template PyObject *getMaterializedMemoObject(MemoObject *); + extern template PyObject *getMaterializedMemoObject(MemoImmutableObject *); + } diff --git a/src/dbzero/bindings/python/PyObjectTagManager.cpp b/src/dbzero/bindings/python/PyObjectTagManager.cpp index 23714c9e..769aec35 100644 --- a/src/dbzero/bindings/python/PyObjectTagManager.cpp +++ b/src/dbzero/bindings/python/PyObjectTagManager.cpp @@ -97,12 +97,12 @@ namespace db0::python }; PyObjectTagManager *tryMakeObjectTagManager(PyObject *, PyObject *const *args, Py_ssize_t nargs) - { - // all arguments must be Memo objects + { + // all arguments must be Memo objects for (Py_ssize_t i = 0; i < nargs; ++i) { - if (!PyMemo_Check(args[i])) { - THROWF(db0::InputException) << "All arguments must be memo objects"; - } + if (!PyAnyMemo_Check(args[i])) { + THROWF(db0::InputException) << "All arguments must be dbzero memo objects"; + } } auto tags_obj = Py_OWN(PyObjectTagManager_new(&PyObjectTagManagerType, NULL, NULL)); diff --git a/src/dbzero/bindings/python/PySnapshot.cpp b/src/dbzero/bindings/python/PySnapshot.cpp index 08749e32..4b499a64 100644 --- a/src/dbzero/bindings/python/PySnapshot.cpp +++ b/src/dbzero/bindings/python/PySnapshot.cpp @@ -238,7 +238,8 @@ namespace db0::python return runSafe(tryPySnapshot_close, self, args); } - PyObject *tryGetSnapshotOf(MemoObject *memo) + template + PyObject *tryGetSnapshotOf(MemoImplT *memo) { auto fixture = memo->ext().getFixture(); auto workspace_view = fixture->getWorkspace().shared_from_this(); @@ -255,11 +256,14 @@ namespace db0::python } PyObject *py_arg = args[0]; - if (!PyMemo_Check(py_arg)) { + if (PyMemo_Check(py_arg)) { + return runSafe(tryGetSnapshotOf, reinterpret_cast(py_arg)); + } else if (PyMemo_Check(py_arg)) { + return runSafe(tryGetSnapshotOf, reinterpret_cast(py_arg)); + } else { PyErr_SetString(PyExc_TypeError, "Invalid argument type (must be a memo object)"); return NULL; - } - return runSafe(tryGetSnapshotOf, reinterpret_cast(py_arg)); + } } static PySequenceMethods PySnapshot_sq = { diff --git a/src/dbzero/bindings/python/PyToolkit.cpp b/src/dbzero/bindings/python/PyToolkit.cpp index 3acb14e7..a4f2a4ce 100644 --- a/src/dbzero/bindings/python/PyToolkit.cpp +++ b/src/dbzero/bindings/python/PyToolkit.cpp @@ -69,7 +69,7 @@ namespace db0::python if (result == "__main__") { // for Memo types we can determine the actual module name from the file name // (if stored with the type decoration) - if (PyMemoType_Check(py_type)) { + if (PyAnyMemoType_Check(py_type)) { // file name may not be available in the type decoration auto file_name = MemoTypeDecoration::get(py_type).tryGetFileName(); if (file_name) { @@ -100,15 +100,15 @@ namespace db0::python bool PyToolkit::isExistingObject(db0::swine_ptr &fixture, Address address, std::uint16_t instance_id) { // try unloading from cache first - auto &lang_cache = fixture->getLangCache(); + auto &lang_cache = fixture->getLangCache(); auto obj_ptr = tryUnloadObjectFromCache(lang_cache, address, nullptr); if (obj_ptr.get()) { // only validate instance ID if provided - auto &memo = reinterpret_cast(obj_ptr.get())->ext(); + auto &memo = reinterpret_cast(obj_ptr.get())->ext(); if (instance_id) { // NOTE: we first must check if this is really a memo object - if (!isMemoObject(obj_ptr.get())) { + if (!isAnyMemoObject(obj_ptr.get())) { return false; } @@ -136,11 +136,10 @@ namespace db0::python // only validate instance ID if provided if (instance_id) { // NOTE: we first must check if this is really a memo object - if (!isMemoObject(obj_ptr.get())) { + if (!isAnyMemoObject(obj_ptr.get())) { return {}; } - - if (reinterpret_cast(obj_ptr.get())->ext().getInstanceId() != instance_id) { + if (reinterpret_cast(obj_ptr.get())->ext().getInstanceId() != instance_id) { return {}; } } @@ -472,10 +471,18 @@ namespace db0::python return PyType_Check(py_object); } + bool PyToolkit::isAnyMemoObject(ObjectPtr py_object) { + return PyAnyMemo_Check(py_object); + } + bool PyToolkit::isMemoObject(ObjectPtr py_object) { - return PyMemo_Check(py_object); + return PyMemo_Check(py_object); } + bool PyToolkit::isMemoImmutableObject(ObjectPtr py_object) { + return PyMemo_Check(py_object); + } + PyToolkit::ObjectPtr PyToolkit::getUUID(ObjectPtr py_object) { return db0::python::tryGetUUID(py_object); } @@ -517,8 +524,8 @@ namespace db0::python return getFixtureUUID(reinterpret_cast(py_object)); } else if (PyEnumValue_Check(py_object)) { return reinterpret_cast(py_object)->ext().m_fixture.safe_lock()->getUUID(); - } else if (PyMemo_Check(py_object)) { - return reinterpret_cast(py_object)->ext().getFixture()->getUUID(); + } else if (PyAnyMemo_Check(py_object)) { + return reinterpret_cast(py_object)->ext().getFixture()->getUUID(); } else if (PyObjectIterable_Check(py_object)) { return reinterpret_cast(py_object)->ext().getFixture()->getUUID(); } else if (PyObjectIterator_Check(py_object)) { @@ -532,7 +539,7 @@ namespace db0::python std::uint64_t PyToolkit::getFixtureUUID(TypeObjectPtr py_type) { - if (isMemoType(py_type)) { + if (isAnyMemoType(py_type)) { return MemoTypeDecoration::get(py_type).getFixtureUUID(AccessType::READ_ONLY); } else { return 0; @@ -541,7 +548,7 @@ namespace db0::python bool PyToolkit::isNoDefaultTags(TypeObjectPtr py_type) { - if (isMemoType(py_type)) { + if (isAnyMemoType(py_type)) { return MemoTypeDecoration::get(py_type).getFlags()[MemoOptions::NO_DEFAULT_TAGS]; } else { return false; @@ -550,16 +557,25 @@ namespace db0::python bool PyToolkit::isNoCache(TypeObjectPtr py_type) { - if (isMemoType(py_type)) { + if (isAnyMemoType(py_type)) { return MemoTypeDecoration::get(py_type).getFlags()[MemoOptions::NO_CACHE]; } else { return false; } - } + } + bool PyToolkit::isImmutable(TypeObjectPtr py_type) + { + if (isAnyMemoType(py_type)) { + return MemoTypeDecoration::get(py_type).getFlags()[MemoOptions::IMMUTABLE]; + } else { + return false; + } + } + FlagSet PyToolkit::getMemoFlags(TypeObjectPtr py_type) { - if (isMemoType(py_type)) { + if (isAnyMemoType(py_type)) { return MemoTypeDecoration::get(py_type).getFlags(); } else { return {}; @@ -568,24 +584,24 @@ namespace db0::python const char *PyToolkit::getPrefixName(TypeObjectPtr memo_type) { - assert(isMemoType(memo_type)); + assert(isAnyMemoType(memo_type)); return MemoTypeDecoration::get(memo_type).tryGetPrefixName(); } const char *PyToolkit::getMemoTypeID(TypeObjectPtr memo_type) { - assert(isMemoType(memo_type)); + assert(isAnyMemoType(memo_type)); return MemoTypeDecoration::get(memo_type).tryGetTypeId(); } const std::vector &PyToolkit::getInitVars(TypeObjectPtr memo_type) { - assert(isMemoType(memo_type)); + assert(isAnyMemoType(memo_type)); return MemoTypeDecoration::get(memo_type).getInitVars(); } - bool PyToolkit::isMemoType(TypeObjectPtr py_type) { - return PyMemoType_Check(py_type); + bool PyToolkit::isAnyMemoType(TypeObjectPtr py_type) { + return PyAnyMemoType_Check(py_type); } void PyToolkit::setError(ObjectPtr err_obj, std::uint64_t err_value) { @@ -676,14 +692,14 @@ namespace db0::python PyToolkit::TypeObjectPtr PyToolkit::getBaseMemoType(TypeObjectPtr py_memo_type) { - assert(isMemoType(py_memo_type)); + assert(isAnyMemoType(py_memo_type)); // first base type is python base. From there we can get the actual base type auto base_py_type = getBaseType(py_memo_type); if (!base_py_type) { return nullptr; } auto memo_base_type = getBaseType(base_py_type); - if (memo_base_type && isMemoType(memo_base_type)) { + if (memo_base_type && isAnyMemoType(memo_base_type)) { return memo_base_type; } return nullptr; @@ -729,8 +745,8 @@ namespace db0::python bool PyToolkit::hasTagRefs(ObjectPtr obj_ptr) { - assert(PyMemo_Check(obj_ptr)); - return reinterpret_cast(obj_ptr)->ext().hasTagRefs(); + assert(PyAnyMemo_Check(obj_ptr)); + return reinterpret_cast(obj_ptr)->ext().hasTagRefs(); } std::unique_ptr PyToolkit::ensureLocked() @@ -745,4 +761,24 @@ namespace db0::python return py_object != nullptr; } + template + bool decRefMemoImpl(bool is_tag, MemoImplT *memo_obj) + { + auto &memo = memo_obj->modifyExt(); + memo.decRef(is_tag); + return !memo.hasRefs(); + } + + bool PyToolkit::decRefMemo(bool is_tag, ObjectPtr py_object) + { + if (PyMemo_Check(py_object)) { + return decRefMemoImpl(is_tag, reinterpret_cast(py_object)); + } else if (PyMemo_Check(py_object)) { + return decRefMemoImpl(is_tag, reinterpret_cast(py_object)); + } else { + assert(false); + THROWF(db0::InputException) << "Invalid memo object type for decRefMemo" << THROWF_END; + } + } + } \ No newline at end of file diff --git a/src/dbzero/bindings/python/PyToolkit.hpp b/src/dbzero/bindings/python/PyToolkit.hpp index 72cf99bb..04b6700c 100644 --- a/src/dbzero/bindings/python/PyToolkit.hpp +++ b/src/dbzero/bindings/python/PyToolkit.hpp @@ -171,8 +171,11 @@ namespace db0::python static bool isIterable(ObjectPtr py_object); static bool isSequence(ObjectPtr py_object); static bool isType(ObjectPtr py_object); - static bool isMemoType(TypeObjectPtr py_type); + // either memo or immutable type + static bool isAnyMemoType(TypeObjectPtr py_type); + static bool isAnyMemoObject(ObjectPtr py_object); static bool isMemoObject(ObjectPtr py_object); + static bool isMemoImmutableObject(ObjectPtr py_object); static bool isEnumValue(ObjectPtr py_object); static bool isFieldDef(ObjectPtr py_object); static bool isClassObject(ObjectPtr py_object); @@ -196,6 +199,8 @@ namespace db0::python // check if a memo type is marked with no_default_tags flag static bool isNoDefaultTags(TypeObjectPtr); static bool isNoCache(TypeObjectPtr); + // type marked as immutable + static bool isImmutable(TypeObjectPtr); static FlagSet getMemoFlags(TypeObjectPtr); inline static void incRef(ObjectPtr py_object) { @@ -245,6 +250,10 @@ namespace db0::python // Acquire the interpreter's GIL lock // NOTE: returns nullptr if Python not initialized / defunct static std::unique_ptr ensureLocked(); + + // decRef operation for memo objects + // @return true if reference count was decremented to zero (!hasRefs) + static bool decRefMemo(bool is_tag, ObjectPtr py_object); private: static PyWorkspace m_py_workspace; diff --git a/src/dbzero/bindings/python/PyTypeManager.cpp b/src/dbzero/bindings/python/PyTypeManager.cpp index 703828cc..80c28bf7 100644 --- a/src/dbzero/bindings/python/PyTypeManager.cpp +++ b/src/dbzero/bindings/python/PyTypeManager.cpp @@ -119,10 +119,13 @@ namespace db0::python } // check if a memo class first - if (PyMemoType_Check(py_type)) { + if (PyMemoType_Check(py_type)) { return TypeId::MEMO_OBJECT; } - + if (PyMemoType_Check(py_type)) { + return TypeId::MEMO_IMMUTABLE_OBJECT; + } + // check with the static types next auto it = m_id_map.find(reinterpret_cast(py_type)); if (it == m_id_map.end()) { @@ -150,7 +153,7 @@ namespace db0::python if (PyType_Check(ptr)) { auto py_type = reinterpret_cast(ptr); - if (PyMemoType_Check(py_type)) { + if (PyAnyMemoType_Check(py_type)) { return TypeId::MEMO_TYPE; } } @@ -185,41 +188,62 @@ namespace db0::python return *type_id; } - - const db0::object_model::Object &PyTypeManager::extractObject(ObjectPtr memo_ptr) const + + const PyTypeManager::ObjectAnyImpl &PyTypeManager::extractAnyObject(ObjectPtr obj_ptr) const { - if (PyMemo_Check(memo_ptr)) { - return reinterpret_cast(memo_ptr)->ext(); - } else if (PyWeakProxy_Check(memo_ptr)) { - return reinterpret_cast(reinterpret_cast(memo_ptr)->get())->ext(); + if (PyAnyMemo_Check(obj_ptr)) { + return reinterpret_cast(obj_ptr)->ext(); + } else if (PyWeakProxy_Check(obj_ptr)) { + return reinterpret_cast(obj_ptr)->get()->ext(); } THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; } - db0::object_model::Object &PyTypeManager::extractMutableObject(ObjectPtr memo_ptr) const + PyTypeManager::ObjectAnyImpl &PyTypeManager::extractMutableAnyObject(ObjectPtr obj_ptr) const { - if (!PyMemo_Check(memo_ptr)) { + if (!PyAnyMemo_Check(obj_ptr)) { THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; } - return reinterpret_cast(memo_ptr)->modifyExt(); + return reinterpret_cast(obj_ptr)->modifyExt(); } - - const db0::object_model::Object *PyTypeManager::tryExtractObject(ObjectPtr memo_ptr) const + + template typename MemoImplT::ExtT & + PyTypeManager::extractMutableObject(ObjectPtr obj_ptr) const { - if (!PyMemo_Check(memo_ptr)) { - return nullptr; + if (!PyMemo_Check(obj_ptr)) { + THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; } - return &reinterpret_cast(memo_ptr)->ext(); + return reinterpret_cast(obj_ptr)->modifyExt(); } - db0::object_model::Object *PyTypeManager::tryExtractMutableObject(ObjectPtr memo_ptr) const + template const typename MemoImplT::ExtT & + PyTypeManager::extractObject(ObjectPtr memo_ptr) const + { + if (PyMemo_Check(memo_ptr)) { + return reinterpret_cast(memo_ptr)->ext(); + } else if (PyWeakProxy_Check(memo_ptr)) { + return reinterpret_cast(reinterpret_cast(memo_ptr)->get())->ext(); + } + THROWF(db0::InputException) << "Expected a memo object" << THROWF_END; + } + + const PyTypeManager::ObjectAnyImpl *PyTypeManager::tryExtractObject(ObjectPtr memo_ptr) const { - if (!PyMemo_Check(memo_ptr)) { + if (!PyAnyMemo_Check(memo_ptr)) { return nullptr; } - return &reinterpret_cast(memo_ptr)->modifyExt(); + return &reinterpret_cast(memo_ptr)->ext(); + } + + template + typename MemoImplT::ExtT *PyTypeManager::tryExtractMutableObject(ObjectPtr memo_ptr) const + { + if (!PyMemo_Check(memo_ptr)) { + return nullptr; + } + return &reinterpret_cast(memo_ptr)->modifyExt(); } - + const db0::object_model::List &PyTypeManager::extractList(ObjectPtr list_ptr) const { if (!ListObject_Check(list_ptr)) { @@ -651,5 +675,23 @@ namespace db0::python THROWF(db0::InputException) << "Invalid value code: " << val_code << THROWF_END; } } - + + template db0::object_model::Object & + PyTypeManager::extractMutableObject(ObjectPtr) const; + + template db0::object_model::ObjectImmutableImpl & + PyTypeManager::extractMutableObject(ObjectPtr) const; + + template const db0::object_model::Object & + PyTypeManager::extractObject(ObjectPtr) const; + + template const db0::object_model::ObjectImmutableImpl & + PyTypeManager::extractObject(ObjectPtr) const; + + template db0::object_model::ObjectImmutableImpl * + PyTypeManager::tryExtractMutableObject(ObjectPtr) const; + + template db0::object_model::Object * + PyTypeManager::tryExtractMutableObject(ObjectPtr) const; + } \ No newline at end of file diff --git a/src/dbzero/bindings/python/PyTypeManager.hpp b/src/dbzero/bindings/python/PyTypeManager.hpp index ba8473c8..16493bf3 100644 --- a/src/dbzero/bindings/python/PyTypeManager.hpp +++ b/src/dbzero/bindings/python/PyTypeManager.hpp @@ -14,6 +14,8 @@ namespace db0::object_model { class Object; + class ObjectImmutableImpl; + class ObjectAnyImpl; class Class; class List; class Set; @@ -40,7 +42,9 @@ namespace db0::python { class MemoTypeDecoration; - + using MemoObject = PyWrapper; + using MemoImmutableObject = PyWrapper; + /** * The class dedicated to recognition of Python types */ @@ -52,7 +56,11 @@ namespace db0::python using ObjectSharedPtr = typename PyTypes::ObjectSharedPtr; using TypeObjectPtr = typename PyTypes::TypeObjectPtr; using TypeObjectSharedPtr = typename PyTypes::TypeObjectSharedPtr; + using MemoObject = db0::python::MemoObject; + using MemoImmutableObject = db0::python::MemoImmutableObject; using Object = db0::object_model::Object; + using ObjectImmutableImpl = db0::object_model::ObjectImmutableImpl; + using ObjectAnyImpl = db0::object_model::ObjectAnyImpl; using List = db0::object_model::List; using Set = db0::object_model::Set; using Tuple = db0::object_model::Tuple; @@ -82,20 +90,28 @@ namespace db0::python TypeId getTypeId(TypeObjectPtr py_type) const; std::optional tryGetTypeId(TypeObjectPtr ptr) const; std::string getLangTypeName(TypeObjectPtr) const; - + // Retrieve a Python type object by TypeId (note that a dbzero extension type may be returned) // to return a native type use getTypeObject(asNative(TypeId)) ObjectSharedPtr getTypeObject(TypeId) const; // If the mapping is not found, returns Py_None ObjectSharedPtr tryGetTypeObject(TypeId) const; - /** - * Extracts reference to DB0 object from a memo object - */ - const Object &extractObject(ObjectPtr memo_ptr) const; - Object &extractMutableObject(ObjectPtr memo_ptr) const; - const Object *tryExtractObject(ObjectPtr memo_ptr) const; - Object *tryExtractMutableObject(ObjectPtr memo_ptr) const; + // Extracts reference to Object or ObjectImmutableImpl from a memo instance + template + const typename MemoImplT::ExtT &extractObject(ObjectPtr memo_ptr) const; + template + typename MemoImplT::ExtT &extractMutableObject(ObjectPtr memo_ptr) const; + + template + typename MemoImplT::ExtT *tryExtractMutableObject(ObjectPtr memo_ptr) const; + + // Extracts reference to common object part from a memo instance + const ObjectAnyImpl &extractAnyObject(ObjectPtr) const; + ObjectAnyImpl &extractMutableAnyObject(ObjectPtr) const; + + const ObjectAnyImpl *tryExtractObject(ObjectPtr memo_ptr) const; + const List &extractList(ObjectPtr list_ptr) const; List &extractMutableList(ObjectPtr list_ptr) const; const Set &extractSet(ObjectPtr set_ptr) const; @@ -216,4 +232,19 @@ namespace db0::python m_simple_py_type_ids.insert(py_type_id); } + extern template const db0::object_model::Object & + PyTypeManager::extractObject(ObjectPtr) const; + + extern template const db0::object_model::ObjectImmutableImpl & + PyTypeManager::extractObject(ObjectPtr) const; + + extern template db0::object_model::Object & + PyTypeManager::extractMutableObject(ObjectPtr) const; + + extern template db0::object_model::ObjectImmutableImpl * + PyTypeManager::tryExtractMutableObject(ObjectPtr) const; + + extern template db0::object_model::Object * + PyTypeManager::tryExtractMutableObject(ObjectPtr) const; + } \ No newline at end of file diff --git a/src/dbzero/bindings/python/PyWeakProxy.cpp b/src/dbzero/bindings/python/PyWeakProxy.cpp index 48e37f7d..8f55b7ef 100644 --- a/src/dbzero/bindings/python/PyWeakProxy.cpp +++ b/src/dbzero/bindings/python/PyWeakProxy.cpp @@ -7,6 +7,15 @@ namespace db0::python { + + void PyAPI_PyWeakProxy_del(PyWeakProxy *py_weak_proxy) + { + PY_API_FUNC + if (py_weak_proxy->m_py_object) { + Py_DECREF(py_weak_proxy->m_py_object); + } + Py_TYPE(py_weak_proxy)->tp_free((PyObject*)py_weak_proxy); + } PyTypeObject PyWeakProxyType = { @@ -19,33 +28,23 @@ namespace db0::python .tp_new = PyType_GenericNew, }; - MemoObject *PyWeakProxy::get() const { - return reinterpret_cast(m_py_object); + MemoAnyObject *PyWeakProxy::get() const { + return reinterpret_cast(m_py_object); } - void PyAPI_PyWeakProxy_del(PyWeakProxy *py_weak_proxy) - { - PY_API_FUNC - if (py_weak_proxy->m_py_object) { - Py_DECREF(py_weak_proxy->m_py_object); - } - Py_TYPE(py_weak_proxy)->tp_free((PyObject*)py_weak_proxy); - } - bool PyWeakProxy_Check(PyObject *obj) { return PyObject_TypeCheck(obj, &PyWeakProxyType); } - + PyObject *tryWeakProxy(PyObject *py_obj) { - if (!PyMemo_Check(py_obj)) { - THROWF(db0::InputException) << "Invalid argument type: " << PyToolkit::getTypeName(py_obj) << " (memo expected)"; - } - // new PyWeakProxy + assert(PyAnyMemo_Check(py_obj)); auto py_weak_proxy = PyObject_New(PyWeakProxy, &PyWeakProxyType); + if (!py_weak_proxy) { return nullptr; } + Py_INCREF(py_obj); py_weak_proxy->m_py_object = py_obj; return reinterpret_cast(py_weak_proxy); diff --git a/src/dbzero/bindings/python/PyWeakProxy.hpp b/src/dbzero/bindings/python/PyWeakProxy.hpp index eabebe4b..6cfe1ffa 100644 --- a/src/dbzero/bindings/python/PyWeakProxy.hpp +++ b/src/dbzero/bindings/python/PyWeakProxy.hpp @@ -7,21 +7,19 @@ namespace db0::python { - + struct PyWeakProxy { PyObject_HEAD PyObject* m_py_object; - // get the underlying memo object - MemoObject *get() const; + // get the underlying memo object (as MemoAnyObject) + MemoAnyObject *get() const; }; extern PyTypeObject PyWeakProxyType; - - void PyAPI_PyWeakProxy_del(PyWeakProxy *self); - bool PyWeakProxy_Check(PyObject *obj); - + + bool PyWeakProxy_Check(PyObject *obj); PyObject *tryWeakProxy(PyObject *); PyObject *tryExpired(PyObject *); diff --git a/src/dbzero/bindings/python/shared_py_object.cpp b/src/dbzero/bindings/python/shared_py_object.cpp index 4aa42092..be947b93 100644 --- a/src/dbzero/bindings/python/shared_py_object.cpp +++ b/src/dbzero/bindings/python/shared_py_object.cpp @@ -4,30 +4,52 @@ namespace db0::python -{ +{ + + template + void incExtRefImpl(PyObject *py_object) { + // increment reference count for memo objects + reinterpret_cast(py_object)->ext().addExtRef(); + } + + template + void decExtRef(PyObject *py_object) { + // decrement reference count for memo objects + reinterpret_cast(py_object)->ext().removeExtRef(); + } + + template + unsigned int getExtRefcount(PyObject *py_object, unsigned int default_count) { + // return reference count for memo objects + return reinterpret_cast(py_object)->ext().getExtRefs(); + } void incExtRef(PyObject *py_object) - { - if (PyMemo_Check(py_object)) { - // increment reference count for memo objects - reinterpret_cast(py_object)->ext().addExtRef(); + { + if (PyMemo_Check(py_object)) { + incExtRefImpl(py_object); + } else if (PyMemo_Check(py_object)) { + incExtRefImpl(py_object); } } void decExtRef(PyObject *py_object) - { - if (PyMemo_Check(py_object)) { - // decrement reference count for memo objects - reinterpret_cast(py_object)->ext().removeExtRef(); - } + { + if (PyMemo_Check(py_object)) { + decExtRef(py_object); + } else if (PyMemo_Check(py_object)) { + decExtRef(py_object); + } } unsigned int getExtRefcount(PyObject *py_object, unsigned int default_count) - { - if (PyMemo_Check(py_object)) { - // return reference count for memo objects - return reinterpret_cast(py_object)->ext().getExtRefs(); + { + if (PyMemo_Check(py_object)) { + return getExtRefcount(py_object, default_count); + } else if (PyMemo_Check(py_object)) { + return getExtRefcount(py_object, default_count); } + // for non-memo objects, return the default count return default_count; } diff --git a/src/dbzero/bindings/python/types/PyTag.cpp b/src/dbzero/bindings/python/types/PyTag.cpp index 25b72a7a..e64aa5f6 100644 --- a/src/dbzero/bindings/python/types/PyTag.cpp +++ b/src/dbzero/bindings/python/types/PyTag.cpp @@ -70,18 +70,19 @@ namespace db0::python return Py_TYPE(py_object) == &PyTagType; } + template PyObject *tryMemoAsTag(PyObject *py_obj) { - assert(PyMemo_Check(py_obj)); - auto &memo_obj = reinterpret_cast(py_obj)->ext(); + assert(PyMemo_Check(py_obj)); + auto &memo_obj = reinterpret_cast(py_obj)->ext(); PyTag *py_tag = PyTagDefault_new(); py_tag->makeNew(memo_obj.getFixture()->getUUID(), memo_obj.getAddress(), py_obj); return py_tag; } - + PyObject *tryMemoTypeAsTag(PyTypeObject *py_type) { - assert(PyMemoType_Check(py_type)); + assert(PyAnyMemoType_Check(py_type)); PyTag *py_tag = PyTagDefault_new(); py_tag->makeNew(py_type, db0::object_model::TagDef::type_as_tag()); return py_tag; @@ -103,18 +104,20 @@ namespace db0::python PyErr_SetString(PyExc_TypeError, "as_tag: Expected 1 argument"); return NULL; } - if (PyMemo_Check(args[0])) { - return runSafe(tryMemoAsTag, args[0]); + if (PyMemo_Check(args[0])) { + return runSafe(tryMemoAsTag, args[0]); + } else if (PyMemo_Check(args[0])) { + return runSafe(tryMemoAsTag, args[0]); } else if (PyType_Check(args[0])) { auto *py_type = reinterpret_cast(args[0]); - if (PyMemoType_Check(py_type)) { + if (PyAnyMemoType_Check(py_type)) { return runSafe(tryMemoTypeAsTag, py_type); } } else if (MemoExpiredRef_Check(args[0])) { return runSafe(tryMemoExpiredRefAsTag, args[0]); - } + } PyErr_SetString(PyExc_TypeError, "as_tag: Expected a memo object"); return NULL; } - + } diff --git a/src/dbzero/core/memory/PrefixCache.cpp b/src/dbzero/core/memory/PrefixCache.cpp index 1243f2ad..591db096 100644 --- a/src/dbzero/core/memory/PrefixCache.cpp +++ b/src/dbzero/core/memory/PrefixCache.cpp @@ -98,9 +98,9 @@ namespace db0 assert(dp_lock); bool is_volatile = access_mode[AccessOptions::no_flush]; - // Try upgrading the unused lock to the write state - // this is to avoid CoW in a writer process - if (access_mode[AccessOptions::write] && read_state_num != state_num) { + // Try upgrading the unused lock to the write state + // this is to avoid CoW in a writer process + if (access_mode[AccessOptions::write] && read_state_num != state_num) { // unused lock condition (i.e. might only be used by the CacheRecycler) // note that dirty locks cannot be upgraded (otherwise data would be lost) if (dp_lock->allowReuse() && dp_lock.use_count() == (dp_lock->isRecycled() ? 1 : 0) + 1) { diff --git a/src/dbzero/core/serialization/Ext.hpp b/src/dbzero/core/serialization/Ext.hpp index 66497fee..28c2fcff 100644 --- a/src/dbzero/core/serialization/Ext.hpp +++ b/src/dbzero/core/serialization/Ext.hpp @@ -236,16 +236,16 @@ DB0_PACKED_BEGIN // measures space requirement of the base overlaid type // plus size of all fixed size members of derived type - template static size_t measureBase(Args&& ...args) + template static Meter measureBase(Args&& ...args) { std::size_t result = super_t::measure(std::forward(args)...); // adjust for fixed size members in derived class result += true_size_of() - true_size_of(); return result; } - + inline std::size_t sizeOf() const { - return T::safeSizeOf(this); + return T::safeSizeOf(reinterpret_cast(this)); } /** diff --git a/src/dbzero/core/vspace/v_ptr.cpp b/src/dbzero/core/vspace/v_ptr.cpp index 4e184ccc..703f9ffb 100644 --- a/src/dbzero/core/vspace/v_ptr.cpp +++ b/src/dbzero/core/vspace/v_ptr.cpp @@ -40,10 +40,11 @@ namespace db0 } vtypeless &vtypeless::operator=(const vtypeless &other) - { + { m_address = other.m_address; m_memspace_ptr = other.m_memspace_ptr; m_access_mode = other.m_access_mode; + m_cached_size = other.m_cached_size; // try locking for copy for (;;) { @@ -63,10 +64,11 @@ namespace db0 } void vtypeless::operator=(vtypeless &&other) - { + { m_address = other.m_address; m_memspace_ptr = other.m_memspace_ptr; m_access_mode = other.m_access_mode; + m_cached_size = other.m_cached_size; // try locking for copy for (;;) { diff --git a/src/dbzero/core/vspace/v_ptr.hpp b/src/dbzero/core/vspace/v_ptr.hpp index 5d5d87ab..fb33bb32 100644 --- a/src/dbzero/core/vspace/v_ptr.hpp +++ b/src/dbzero/core/vspace/v_ptr.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -50,7 +51,9 @@ namespace db0 mutable std::atomic m_resource_flags = 0; // initial access flags (e.g. read / write / create) FlagSet m_access_mode; - + // NOTE: cached size may speed-up updates but also is relevant for existing vptr's reinterpret casts + mutable std::optional m_cached_size; + /** * Memory mapped range corresponding to this object */ @@ -210,6 +213,7 @@ namespace db0 m_memspace_ptr->free(m_address); this->m_address = {}; this->m_resource_flags = 0; + this->m_cached_size.reset(); } ContainerT &modify() @@ -364,7 +368,7 @@ namespace db0 } assert(m_mem_lock.m_buffer); } - + // version with known size-of (pre-retrieved from the allocator) // we made it as a separate implementation for potential performance gains void assureInitialized(std::size_t size_of) const @@ -384,11 +388,9 @@ namespace db0 assert(m_mem_lock.m_buffer); } - /** - * Resolve the instance size - */ - std::size_t getSize() const - { + // Resolve the instance size + std::uint32_t fetchSize() const + { assert(m_memspace_ptr); if constexpr(metaprog::has_constant_size::value) { // fixed size type @@ -402,6 +404,15 @@ namespace db0 // retrieve from allocator (slowest) return m_memspace_ptr->getAllocator().getAllocSize(m_address, REALM_ID); } + + // Get from cache or fetch size + std::uint32_t getSize() const + { + if (!m_cached_size) { + m_cached_size = fetchSize(); + } + return *m_cached_size; + } }; } diff --git a/src/dbzero/object_model/class/Class.cpp b/src/dbzero/object_model/class/Class.cpp index 1c8fb710..bcd6a722 100644 --- a/src/dbzero/object_model/class/Class.cpp +++ b/src/dbzero/object_model/class/Class.cpp @@ -8,7 +8,7 @@ #include #include "Schema.hpp" -DEFINE_ENUM_VALUES(db0::ClassOptions, "SINGLETON", "NO_DEFAULT_TAGS") +DEFINE_ENUM_VALUES(db0::ClassOptions, "SINGLETON", "NO_DEFAULT_TAGS", "IMMUTABLE") namespace db0::object_model @@ -289,16 +289,7 @@ namespace db0::object_model bool Class::isExistingSingleton() const { return isSingleton() && (*this)->m_singleton_address.isValid(); } - - void Class::setSingletonAddress(Object &object) - { - assert(!(*this)->m_singleton_address.isValid()); - assert(isSingleton()); - // increment reference count in order to prevent singleton object from being destroyed - object.incRef(false); - modify().m_singleton_address = object.getUniqueAddress(); - } - + void Class::onMemberIDUpdated(const MemberID &member_id) const { if (member_id.hasFidelity(PRIMARY_FIDELITY) && member_id.size() > 1) { diff --git a/src/dbzero/object_model/class/Class.hpp b/src/dbzero/object_model/class/Class.hpp index c8ddad71..300d0525 100644 --- a/src/dbzero/object_model/class/Class.hpp +++ b/src/dbzero/object_model/class/Class.hpp @@ -30,13 +30,14 @@ namespace db0 SINGLETON = 0x0001, // instances of this type opted out of auto-assigned type tags NO_DEFAULT_TAGS = 0x0002, + IMMUTABLE = 0x0004 }; using ClassFlags = db0::FlagSet; } -DECLARE_ENUM_VALUES(db0::ClassOptions, 2) +DECLARE_ENUM_VALUES(db0::ClassOptions, 3) namespace db0::object_model @@ -47,6 +48,8 @@ namespace db0::object_model using Fixture = db0::Fixture; using ClassFlags = db0::ClassFlags; class Object; + class ObjectImmutableImpl; + class ObjectAnyImpl; class Class; struct ObjectId; @@ -82,7 +85,7 @@ DB0_PACKED_BEGIN ); }; DB0_PACKED_END - + // address <-> class_ref conversion functions // @param type_slot_addr_range the address of the types-specific slot std::uint32_t classRef(const Class &, std::pair type_slot_addr_range); @@ -168,8 +171,8 @@ DB0_PACKED_END // Construct singleton's ObjectId without unloading it ObjectId getSingletonObjectId() const; - - void setSingletonAddress(Object &); + + template void setSingletonAddress(T &); Address getSingletonAddress() const; @@ -257,22 +260,24 @@ DB0_PACKED_END inline AccessFlags getInstanceFlags() const { return m_no_cache ? AccessFlags { AccessOptions::no_cache } : AccessFlags {}; } - + protected: friend class ClassFactory; friend ClassPtr; friend class Object; + friend class ObjectImmutableImpl; + friend class ObjectAnyImpl; friend super_t; + void unlinkSingleton(); + // dbzero class instances should only be created by the ClassFactory // construct a new dbzero class // NOTE: module name may not be available in some contexts (e.g. classes defined in notebooks) Class(db0::swine_ptr &, const std::string &name, std::optional module_name, const char *type_id, const char *prefix_name, const std::vector &init_vars, ClassFlags, std::shared_ptr base_class); - - void unlinkSingleton(); - + // Get unique class identifier within its fixture std::uint32_t fetchUID() const; @@ -336,4 +341,14 @@ DB0_PACKED_END std::optional getNameVariant(const Class &, int variant_id); + template + void Class::setSingletonAddress(T &object) + { + assert(!(*this)->m_singleton_address.isValid()); + assert(isSingleton()); + // increment reference count in order to prevent singleton object from being destroyed + object.incRef(false); + modify().m_singleton_address = object.getUniqueAddress(); + } + } diff --git a/src/dbzero/object_model/class/ClassFactory.cpp b/src/dbzero/object_model/class/ClassFactory.cpp index 2dd140e8..2b9f0d13 100644 --- a/src/dbzero/object_model/class/ClassFactory.cpp +++ b/src/dbzero/object_model/class/ClassFactory.cpp @@ -158,6 +158,7 @@ namespace db0::object_model bool is_singleton = LangToolkit::isSingleton(lang_type); ClassFlags flags { is_singleton ? ClassOptions::SINGLETON : 0 }; flags.set(ClassOptions::NO_DEFAULT_TAGS, LangToolkit::isNoDefaultTags(lang_type)); + flags.set(ClassOptions::IMMUTABLE, LangToolkit::isImmutable(lang_type)); auto memo_base = LangToolkit::getBaseMemoType(lang_type); std::shared_ptr base_class; if (memo_base) { diff --git a/src/dbzero/object_model/class/ClassFields.cpp b/src/dbzero/object_model/class/ClassFields.cpp index 7d7db2cd..29d6e988 100644 --- a/src/dbzero/object_model/class/ClassFields.cpp +++ b/src/dbzero/object_model/class/ClassFields.cpp @@ -11,14 +11,14 @@ namespace db0::object_model ClassFields::ClassFields(TypeObjectPtr lang_type) : m_lang_type(lang_type) { - if (!LangToolkit::isMemoType(lang_type)) { + if (!LangToolkit::isAnyMemoType(lang_type)) { THROWF(db0::InputException) << "Expected Memo type object"; } } void ClassFields::init(TypeObjectPtr lang_type) { - if (!LangToolkit::isMemoType(lang_type)) { + if (!LangToolkit::isAnyMemoType(lang_type)) { THROWF(db0::InputException) << "Expected Memo type object"; } m_lang_type = lang_type; diff --git a/src/dbzero/object_model/index/Index.hpp b/src/dbzero/object_model/index/Index.hpp index b158991b..250a9e3f 100644 --- a/src/dbzero/object_model/index/Index.hpp +++ b/src/dbzero/object_model/index/Index.hpp @@ -191,7 +191,7 @@ namespace db0::object_model if (!m_index) { if ((*this)->m_index_addr.isValid()) { // pull existing range tree - m_index = db0::make_shared_void(this->myPtr((*this)->m_index_addr)); + m_index = db0::make_shared_void(this->myPtr((*this)->m_index_addr)); } else { // create a new range tree instance m_index = db0::make_shared_void(this->getMemspace()); diff --git a/src/dbzero/object_model/index/IndexBuilder.hpp b/src/dbzero/object_model/index/IndexBuilder.hpp index dfb906ad..fa7bf740 100644 --- a/src/dbzero/object_model/index/IndexBuilder.hpp +++ b/src/dbzero/object_model/index/IndexBuilder.hpp @@ -1,7 +1,7 @@ #pragma once #include -#include +#include namespace db0::object_model @@ -13,13 +13,13 @@ namespace db0::object_model template class IndexBuilder: public RangeTree::Builder { public: + using super_t = typename RangeTree::Builder; using RangeTreeT = RangeTree; using LangToolkit = typename LangConfig::LangToolkit; using ObjectPtr = typename LangToolkit::ObjectPtr; using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; using ObjectSharedExtPtr = typename LangToolkit::ObjectSharedExtPtr; - using super_t = typename RangeTree::Builder; - + IndexBuilder(); IndexBuilder(std::unordered_set &&remove_null_values, std::unordered_set &&add_null_values, @@ -47,7 +47,7 @@ namespace db0::object_model // NOTE: cache must hold "shared" language reference to prevent object drop (index owns its objects) mutable std::unordered_map m_object_cache; - // add to cache and return object's address + // add to cache and return object's address UniqueAddress addToCache(ObjectPtr); }; @@ -87,26 +87,27 @@ namespace db0::object_model std::function add_callback = [&](UniqueAddress address) { auto it = m_object_cache.find(address); assert(it != m_object_cache.end()); - m_type_manager.extractMutableObject(it->second.get()).incRef(false); + m_type_manager.extractMutableAnyObject(it->second.get()).incRef(false); }; std::function erase_callback = [&](UniqueAddress address) { auto it = m_object_cache.find(address); assert(it != m_object_cache.end()); - m_type_manager.extractMutableObject(it->second.get()).decRef(false); + m_type_manager.extractMutableAnyObject(it->second.get()).decRef(false); }; super_t::flush(index, &add_callback, &erase_callback); m_object_cache.clear(); } - template UniqueAddress IndexBuilder::addToCache(ObjectPtr obj_ptr) + template + UniqueAddress IndexBuilder::addToCache(ObjectPtr obj_ptr) { - auto obj_addr = m_type_manager.extractObject(obj_ptr).getUniqueAddress(); + auto obj_addr = m_type_manager.extractAnyObject(obj_ptr).getUniqueAddress(); if (m_object_cache.find(obj_addr) == m_object_cache.end()) { m_object_cache.emplace(obj_addr, obj_ptr); } return obj_addr; } - + } \ No newline at end of file diff --git a/src/dbzero/object_model/object/Object.cpp b/src/dbzero/object_model/object/Object.cpp index 59025bd9..5f0c8915 100644 --- a/src/dbzero/object_model/object/Object.cpp +++ b/src/dbzero/object_model/object/Object.cpp @@ -1,27 +1,13 @@ #include "Object.hpp" -#include -#include -#include -#include #include -#include -#include -#include -#include - -DEFINE_ENUM_VALUES(db0::object_model::ObjectOptions, "DROPPED", "DEFUNCT") +#include namespace db0::object_model { GC0_Define(Object) - ObjectInitializerManager Object::m_init_manager; - - FlagSet getAccessOptions(const Class &type) { - return type.isNoCache() ? FlagSet { AccessOptions::no_cache } : FlagSet {}; - } - + bool isEqual(const KV_Index *kv_ptr_1, const KV_Index *kv_ptr_2) { if (!kv_ptr_1) { @@ -36,311 +22,174 @@ namespace db0::object_model return *kv_ptr_1 == *kv_ptr_2; } - template IntT safeCast(unsigned int value, const char *err_msg) + void Object::getFieldLayoutImpl(FieldLayout &layout) const { - if (value > std::numeric_limits::max()) { - THROWF(db0::InputException) << err_msg; + super_t::getFieldLayoutImpl(layout); + // collect kv-index information + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto it = kv_index_ptr->beginJoin(1); + for (;!it.is_end(); ++it) { + layout.m_kv_index_fields.emplace_back((*it).getIndex(), (*it).m_type); + } } - return static_cast(value); } - o_object::o_object(std::uint32_t class_ref, std::pair ref_counts, - std::uint8_t num_type_tags, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, - const XValue *index_vt_begin, const XValue *index_vt_end) - : m_header(ref_counts) - , m_num_type_tags(num_type_tags) + void Object::getMembersImpl(std::unordered_set &result) const { - arrangeMembers() - (PosVT::type(), pos_vt_data, pos_vt_offset) - (packed_int32::type(), class_ref) - (IndexVT::type(), index_vt_begin, index_vt_end); + super_t::getMembersImpl(result); + // Finally, visit kv-index members + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto &obj_type = this->getType(); + auto it = kv_index_ptr->beginJoin(1); + for (;!it.is_end(); ++it) { + auto index = (*it).getIndex(); + getMembersFrom(obj_type, index, (*it).m_type, (*it).m_value, result); + } + } } - std::size_t o_object::measure(std::uint32_t class_ref, std::pair, std::uint8_t, - const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, - const XValue *index_vt_begin, const XValue *index_vt_end) + bool Object::tryEqualToImpl(const ObjectImplBase &other, bool &result) const { - return super_t::measureMembers() - (PosVT::type(), pos_vt_data, pos_vt_offset) - (packed_int32::type(), class_ref) - (IndexVT::type(), index_vt_begin, index_vt_end); - } - - const PosVT &o_object::pos_vt() const { - return getDynFirst(PosVT::type()); - } - - PosVT &o_object::pos_vt() { - return getDynFirst(PosVT::type()); - } - - const packed_int32 &o_object::classRef() const { - return getDynAfter(pos_vt(), packed_int32::type()); - } - - std::uint32_t o_object::getClassRef() const { - return classRef().value(); - } + if (super_t::tryEqualToImpl(other, result)) { + return true; + } - const IndexVT &o_object::index_vt() const { - return getDynAfter(classRef(), IndexVT::type()); - } - - IndexVT &o_object::index_vt() { - return getDynAfter(classRef(), IndexVT::type()); - } - - void o_object::incRef(bool is_tag) { - m_header.incRef(is_tag); + if (this->getFixture()->getUUID() == other.getFixture()->getUUID() + && this->getUniqueAddress() == other.getUniqueAddress()) + { + auto &other_obj = static_cast(other); + if (!hasKV_Index() && !other_obj.hasKV_Index()) { + result = true; + return true; + } + result = isEqual(this->tryGetKV_Index(), other_obj.tryGetKV_Index()); + return true; + } + + return false; } - bool o_object::hasRefs() const + void Object::unSingleton(FixtureLock &) { - // NOTE: type tags are not counted as "proper" references - if (m_header.m_ref_counter.getFirst() > this->m_num_type_tags) { - return true; + auto &type = getType(); + // drop reference from the class + if (type.isSingleton()) { + // clear singleton address + type.unlinkSingleton(); + this->modify().m_header.decRef(false); } - return m_header.m_ref_counter.getSecond() > 0; - } - - bool o_object::hasAnyRefs() const { - return m_header.hasRefs(); } - Object::Object(UniqueAddress addr, unsigned int ext_refs) - : m_flags { ObjectOptions::DROPPED } - , m_ext_refs(ext_refs) - , m_unique_address(addr) - { + bool Object::isSingleton() const { + return getType().isSingleton(); } - Object::Object(std::shared_ptr db0_class) + bool Object::tryFindMemberAt(std::pair field_info, std::pair &result, + std::pair &find_result) const { - // prepare for initialization - m_init_manager.addInitializer(*this, db0_class); - } + if (super_t::tryFindMemberAt(field_info, result, find_result)) { + return true; + } - Object::Object(TypeInitializer &&type_initializer) - { - // prepare for initialization - m_init_manager.addInitializer(*this, std::move(type_initializer)); - } + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto loc = field_info.first.getIndexAndOffset(); + XValue xvalue(loc.first); + if (kv_index_ptr->findOne(xvalue)) { + assert(xvalue.getIndex() == loc.first); + if (xvalue.m_type == StorageClass::DELETED) { + // report as deleted + find_result = { false, true }; + return true; + } - Object::Object(db0::swine_ptr &fixture, std::shared_ptr type, - std::pair ref_counts, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset) - : super_t(fixture, type->getClassRef(), ref_counts, - safeCast(type->getNumBases() + 1, "Too many base classes"), pos_vt_data, pos_vt_offset, nullptr, nullptr, - getAccessOptions(*type)) - , m_type(type) - { - } + // member fetched from the kv_index + result.first = xvalue.m_type; + result.second = xvalue.m_value; - Object::Object(db0::swine_ptr &fixture, Address address, AccessFlags access_mode) - : super_t(super_t::tag_from_address(), fixture, address, access_mode) - { - } - - Object::Object(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type) - : super_t(super_t::tag_from_stem(), fixture, std::move(stem)) - , m_type(type) - { - assert(hasValidClassRef()); - } - - Object::Object(db0::swine_ptr &fixture, Address address, std::shared_ptr type_hint, with_type_hint, AccessFlags access_mode) - : Object(fixture, address, access_mode) - { - assert(*fixture == *type_hint->getFixture()); - setTypeWithHint(type_hint); - } - - Object::Object(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type_hint, with_type_hint) - : Object(fixture, std::move(stem), getTypeWithHint(*fixture, stem->getClassRef(), type_hint)) - { - } - - Object::~Object() - { - // unregister needs to be called before destruction of members - unregister(); - if (!hasInstance()) { - // release initializer if it exists, object not created - m_init_manager.tryCloseInitializer(*this); - } - } - - void Object::dropInstance(FixtureLock &) - { - auto unique_addr = this->getUniqueAddress(); - auto ext_refs = this->getExtRefs(); - this->~Object(); - // construct a null placeholder - new ((void*)this) Object(unique_addr, ext_refs); - } - - Object::ObjectStem Object::tryUnloadStem(db0::swine_ptr &fixture, Address address, - std::uint16_t instance_id, AccessFlags access_mode) - { - std::size_t size_of; - if (!fixture->isAddressValid(address, REALM_ID, &size_of)) { - return {}; - } - // Unload from a verified address - ObjectVType stem(db0::tag_verified(), fixture->myPtr(address), size_of, access_mode); - if (instance_id && stem->m_header.m_instance_id != instance_id) { - // instance ID validation failed - return {}; - } - return stem; - } - - Object::ObjectStem Object::unloadStem(db0::swine_ptr &fixture, Address address, - std::uint16_t instance_id, AccessFlags access_mode) - { - auto result = tryUnloadStem(fixture, address, instance_id, access_mode); - if (!result) { - THROWF(db0::InputException) << "Invalid UUID or object has been deleted"; - } - return result; - } - - void Object::postInit(FixtureLock &fixture) - { - if (!hasInstance()) { - auto &initializer = m_init_manager.getInitializer(*this); - PosVT::Data pos_vt_data; - unsigned int pos_vt_offset = 0; - auto index_vt_data = initializer.getData(pos_vt_data, pos_vt_offset); - - // place object in the same fixture as its class - // construct the dbzero instance & assign to self - m_type = initializer.getClassPtr(); - assert(m_type); - super_t::init(*fixture, m_type->getClassRef(), initializer.getRefCounts(), - safeCast(m_type->getNumBases() + 1, "Too many base classes"), - pos_vt_data, pos_vt_offset, index_vt_data.first, index_vt_data.second, - getAccessOptions(*m_type) - ); - - // reference associated class - m_type->incRef(false); - m_type->updateSchema(pos_vt_offset, pos_vt_data.m_types, pos_vt_data.m_values); - m_type->updateSchema(index_vt_data.first, index_vt_data.second); - - // bind singleton address (now that instance exists) - if (m_type->isSingleton()) { - m_type->setSingletonAddress(*this); + if (field_info.second == 0) { + find_result = { result.first != StorageClass::UNDEFINED, false }; + return true; + } else { + find_result = hasValueAt(result.second, field_info.second, loc.second); + return true; + } } - initializer.close(); } - assert(hasInstance()); + return false; } - - std::pair Object::recognizeType(Fixture &fixture, ObjectPtr lang_value) const - { - auto type_id = LangToolkit::getTypeManager().getTypeId(lang_value); - // NOTE: allow storage as PACK_2 - auto pre_storage_class = TypeUtils::m_storage_class_mapper.getPreStorageClass(type_id, true); - if (type_id == TypeId::MEMO_OBJECT) { - // object reference must be from the same fixture - auto &obj = LangToolkit::getTypeManager().extractObject(lang_value); - if (fixture.getUUID() != obj.getFixture()->getUUID()) { - THROWF(db0::InputException) << "Referencing objects from foreign prefixes is not allowed. Use db0.weak_proxy instead"; - } + + void Object::set(FixtureLock &fixture, const char *field_name, ObjectPtr lang_value) + { + assert(hasInstance()); + // attribute delete operation + if (!PyToolkit::isValid(lang_value)) { + remove(fixture, field_name); + return; } + + auto [type_id, storage_class] = recognizeType(**fixture, lang_value); - // may need to refine the storage class (i.e. long weak ref might be needed instead) - StorageClass storage_class; - if (pre_storage_class == PreStorageClass::OBJECT_WEAK_REF) { - storage_class = db0::getStorageClass(pre_storage_class, fixture, lang_value); - } else { - storage_class = db0::getStorageClass(pre_storage_class); + if (this->span() > 1) { + // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag + // this is because the actual change may be missed if performed on a different-then the 1st DP + _touch(); } - return { type_id, storage_class }; - } - - void Object::removePreInit(const char *field_name) const - { - auto &initializer = m_init_manager.getInitializer(*this); - auto &type = initializer.getClass(); - - // Find an already existing field index - auto member_id = std::get<0>(type.findField(field_name)); - if (!member_id) { - THROWF(db0::InputException) << "Attribute not found: " << field_name; + assert(m_type); + // find already existing field index + auto [member_id, is_init_var] = m_type->findField(field_name); + auto storage_fidelity = getStorageFidelity(storage_class); + // get field ID matching the required storage fidelity + FieldID field_id; + FieldInfo old_field_info; + unsigned int old_pos = 0; + const void *old_loc_ptr = nullptr; + if (member_id) { + std::tie(old_field_info, old_loc_ptr) = tryGetMemberSlot(member_id, old_pos); } - for (const auto &field_info: member_id) { - assert(field_info.first); - auto loc = field_info.first.getIndexAndOffset(); - // mark as deleted - if (field_info.second == 0) { - initializer.set(loc, StorageClass::DELETED, {}); - } else { - assert(field_info.second == 2 && "Only fidelity == 2 is supported"); - if (member_id.hasFidelity(0)) { - // remove any existing regular initialization - auto loc0 = member_id.get(0).getIndexAndOffset(); - initializer.remove(loc0); - } - initializer.set(loc, StorageClass::PACK_2, Value::DELETED, - lofi_store<2>::mask(loc.second)); - } - } - } - - void Object::setPreInit(const char *field_name, ObjectPtr obj_ptr) const - { - assert(!hasInstance()); - if (!LangToolkit::isValid(obj_ptr)) { - removePreInit(field_name); - return; + if (!member_id || !(field_id = member_id.tryGet(storage_fidelity))) { + // try mutating the class first + member_id = m_type->addField(field_name, storage_fidelity); + field_id = member_id.get(storage_fidelity); } - - auto &initializer = m_init_manager.getInitializer(*this); - auto fixture = initializer.getFixture(); - auto &type = initializer.getClass(); - auto [type_id, storage_class] = recognizeType(*fixture, obj_ptr); - auto storage_fidelity = getStorageFidelity(storage_class); - // Find an already existing field index - auto [member_id, is_init_var] = type.findField(field_name); - // NOTE: even if a field already exists we might need to extend its supported fidelities - if (!member_id || !member_id.hasFidelity(storage_fidelity)) { - // update class definition - // use the default fidelity for the storage class - member_id = type.addField(field_name, storage_fidelity); - } + assert(field_id && member_id); + // NOTE: a new member inherits the parent's no-cache flag + // FIXME: value should be destroyed on exception + auto value = createMember( + *fixture, type_id, storage_class, lang_value, this->getMemberFlags() + ); + // make sure object address is not null + assert(!(storage_class == StorageClass::OBJECT_REF && value.cast() == 0)); - if (storage_fidelity == 0) { - if (member_id.hasFidelity(2)) { - // remove any existing lo-fi initialization - auto loc = member_id.get(2).getIndexAndOffset(); - initializer.remove(loc, lofi_store<2>::mask(loc.second)); - } - // register a regular member with the initializer - // NOTE: a new member receives the no-cache flag if set (at the type level) - auto member_flags = type.isNoCache() ? AccessFlags { AccessOptions::no_cache } : AccessFlags(); - initializer.set(member_id.get(0).getIndexAndOffset(), storage_class, - createMember(fixture, type_id, storage_class, obj_ptr, member_flags) - ); + if (field_id == old_field_info.first) { + // Set / update value at the existing location + setWithLoc(fixture, field_id, old_loc_ptr, old_pos, storage_fidelity, storage_class, value); } else { - if (member_id.hasFidelity(0)) { - // remove any existing regular initialization - auto loc = member_id.get(0).getIndexAndOffset(); - initializer.remove(loc); + // must reset / unreference the old value (stored elsewhere) + if (old_field_info.first) { + unrefWithLoc(fixture, old_field_info.first, old_loc_ptr, old_pos, StorageClass::UNDEFINED, + old_field_info.second); } - // For now only fidelity == 2 is supported (lo-fi storage) - assert(storage_fidelity == 2); - auto loc = member_id.get(storage_fidelity).getIndexAndOffset(); - // no access flags for lo-fi members - auto value = lofi_store<2>::create(loc.second, - createMember(fixture, type_id, storage_class, obj_ptr, {}).m_store); - // register a lo-fi member with the initializer (using mask) - initializer.set(loc, storage_class, value, lofi_store<2>::mask(loc.second)); + + const void *loc_ptr = nullptr; + unsigned int pos = 0; + // NOTE: slot may already exist (pos-vt or index-vt) either for regular or lo-fi storage + std::tie(loc_ptr, pos) = tryGetLoc(field_id); + // Either use existing slot or create a new (kv-index) + addWithLoc(fixture, field_id, loc_ptr, pos, storage_fidelity, storage_class, value); } + + // the KV-index insert operation must be registered as the potential silent mutation + // but the operation can be avoided if the object is already marked as modified + if (!super_t::isModified()) { + this->_touch(); + } } void Object::remove(FixtureLock &fixture, const char *field_name) @@ -375,74 +224,87 @@ namespace db0::object_model this->_touch(); } } - - void Object::unrefPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, StorageClass storage_class, - unsigned int fidelity) - { - auto &pos_vt = modify().pos_vt(); - auto old_storage_class = pos_vt.types()[pos]; - if (fidelity == 0) { - unrefMember(*fixture, old_storage_class, pos_vt.values()[pos]); - // mark member as unreferenced by assigning storage class - pos_vt.set(pos, storage_class, {}); - m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(old_storage_class)); - } else { - assert(fidelity == 2); - auto value = pos_vt.values()[pos]; - auto offset = field_id.getOffset(); - if (storage_class != StorageClass::DELETED && !lofi_store<2>::fromValue(value).isSet(offset)) { - // value is already unset - return; - } - auto old_type_id = getSchemaTypeId(old_storage_class, lofi_store<2>::fromValue(value).get(offset)); - // either reset or mark as deleted - if (storage_class == StorageClass::DELETED) { - lofi_store<2>::fromValue(value).set(offset, Value::DELETED); + KV_Index *Object::addKV_First(const XValue &value) + { + if (!m_kv_index) { + if ((*this)->m_kv_address) { + m_kv_index = std::make_unique( + std::make_pair(&getMemspace(), (*this)->m_kv_address), (*this)->m_kv_type + ); } else { - lofi_store<2>::fromValue(value).reset(offset); + // create new kv-index intiialized with the first value + m_kv_index = std::make_unique(getMemspace(), value); + this->modify().m_kv_address = m_kv_index->getAddress(); + this->modify().m_kv_type = m_kv_index->getIndexType(); + // return nullptr to indicate that the value has been inserted + return nullptr; } - pos_vt.set(pos, old_storage_class, value); - m_type->removeFromSchema(field_id, fidelity, old_type_id); } + // return reference without inserting + return m_kv_index.get(); } - - void Object::unrefIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, - StorageClass storage_class, unsigned int fidelity) + + bool Object::hasKV_Index() const { + return m_kv_index || (*this)->m_kv_address; + } + + KV_Index *Object::tryGetKV_Index() const { - auto &index_vt = modify().index_vt(); - auto old_storage_class = index_vt.xvalues()[index_vt_pos].m_type; - if (fidelity == 0) { - unrefMember(*fixture, index_vt.xvalues()[index_vt_pos]); - // mark member as unreferenced by assigning storage class - index_vt.set(index_vt_pos, storage_class, {}); - m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(old_storage_class)); - } else { - assert(fidelity == 2); - auto value = index_vt.xvalues()[index_vt_pos].m_value; - auto offset = field_id.getOffset(); - if (storage_class != StorageClass::DELETED && !lofi_store<2>::fromValue(value).isSet(offset)) { - // value is already unset - return; + // if KV index address has changed, update the cached instance + if (!m_kv_index || m_kv_index->getAddress() != (*this)->m_kv_address) { + if ((*this)->m_kv_address) { + m_kv_index = std::make_unique( + std::make_pair(&getMemspace(), (*this)->m_kv_address), (*this)->m_kv_type + ); } - auto old_type_id = getSchemaTypeId(old_storage_class, lofi_store<2>::fromValue(value).get(offset)); - if (storage_class == StorageClass::DELETED) { - lofi_store<2>::fromValue(value).set(offset, Value::DELETED); - } else { - lofi_store<2>::fromValue(value).reset(offset); - } - index_vt.set(index_vt_pos, old_storage_class, value); - m_type->removeFromSchema(field_id, fidelity, old_type_id); } + + return m_kv_index.get(); } - - void Object::unrefKVIndexValue(FixtureLock &fixture, FieldID field_id, StorageClass storage_class, - unsigned int fidelity) + + void Object::addToKVIndex(FixtureLock &fixture, FieldID field_id, unsigned int fidelity, + StorageClass storage_class, Value value) { - auto kv_index_ptr = tryGetKV_Index(); - if (!kv_index_ptr) { - THROWF(db0::InputException) << "Attribute not found"; - } + assert(m_type); + XValue xvalue(field_id.getIndex(), storage_class, value); + // encode for lo-fi storage if needed + if (fidelity != 0) { + xvalue.m_value = lofi_store<2>::create(field_id.getOffset(), value.m_store); + } + auto kv_index_ptr = addKV_First(xvalue); + if (kv_index_ptr) { + // NOTE: for fidelity > 0 the element might already exist + XValue old_value; + if (fidelity > 0 && kv_index_ptr->updateExisting(xvalue, &old_value)) { + auto kv_value = old_value.m_value; + lofi_store<2>::fromValue(kv_value).set(field_id.getOffset(), value.m_store); + xvalue.m_value = kv_value; + kv_index_ptr->updateExisting(xvalue); + // in case of the IttyIndex updating an element changes the address + // which needs to be updated in the object + if (kv_index_ptr->getIndexType() == bindex::type::itty) { + this->modify().m_kv_address = kv_index_ptr->getAddress(); + } + } else { + if (kv_index_ptr->insert(xvalue)) { + // type or address of the kv-index has changed which needs to be reflected + this->modify().m_kv_address = kv_index_ptr->getAddress(); + this->modify().m_kv_type = kv_index_ptr->getIndexType(); + } + } + } + + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + } + + void Object::unrefKVIndexValue(FixtureLock &fixture, FieldID field_id, StorageClass storage_class, + unsigned int fidelity) + { + auto kv_index_ptr = tryGetKV_Index(); + if (!kv_index_ptr) { + THROWF(db0::InputException) << "Attribute not found"; + } XValue xvalue(field_id.getIndex()); if (!kv_index_ptr->findOne(xvalue)) { THROWF(db0::InputException) << "Attribute not found"; @@ -457,7 +319,7 @@ namespace db0::object_model // in case of the IttyIndex updating an element changes the address // which needs to be updated in the object if (kv_index_ptr->getIndexType() == bindex::type::itty) { - modify().m_kv_address = kv_index_ptr->getAddress(); + this->modify().m_kv_address = kv_index_ptr->getAddress(); } } else { auto old_addr = kv_index_ptr->getAddress(); @@ -465,8 +327,8 @@ namespace db0::object_model auto new_addr = kv_index_ptr->getAddress(); if (new_addr != old_addr) { // type or address of the kv-index has changed which needs to be reflected - modify().m_kv_address = new_addr; - modify().m_kv_type = kv_index_ptr->getIndexType(); + this->modify().m_kv_address = new_addr; + this->modify().m_kv_type = kv_index_ptr->getIndexType(); } } m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(xvalue.m_type)); @@ -489,98 +351,41 @@ namespace db0::object_model // in case of the IttyIndex updating an element changes the address // which needs to be updated in the object if (kv_index_ptr->getIndexType() == bindex::type::itty) { - modify().m_kv_address = kv_index_ptr->getAddress(); + this->modify().m_kv_address = kv_index_ptr->getAddress(); } m_type->removeFromSchema(field_id, fidelity, old_type_id); } } - void Object::unrefWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, - StorageClass storage_class, unsigned int fidelity) - { - if (loc_ptr == &(*this)->pos_vt()) { - unrefPosVT(fixture, field_id, pos, storage_class, fidelity); - } else if (loc_ptr == &(*this)->index_vt()) { - unrefIndexVT(fixture, field_id, pos, storage_class, fidelity); - } else { - unrefKVIndexValue(fixture, field_id, storage_class, fidelity); - } - } - - void Object::setPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, - StorageClass storage_class, Value value) - { - auto &pos_vt = modify().pos_vt(); - auto pos_value = pos_vt.values()[pos]; - if (fidelity == 0) { - auto old_storage_class = pos_vt.types()[pos]; - unrefMember(*fixture, old_storage_class, pos_value); - // update attribute stored in the positional value-table - pos_vt.set(pos, storage_class, value); - m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_storage_class), getSchemaTypeId(storage_class)); - } else { - auto offset = field_id.getOffset(); - auto old_type_id = getSchemaTypeId(storage_class, lofi_store<2>::fromValue(pos_value).get(offset)); - lofi_store<2>::fromValue(pos_value).set(offset, value.m_store); - pos_vt.set(pos, storage_class, pos_value); - auto new_type_id = getSchemaTypeId(storage_class, value); - m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); - } - } - - void Object::addToPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, - StorageClass storage_class, Value value) + void Object::dropMembers(db0::swine_ptr &fixture, Class &class_ref) const { - auto &pos_vt = modify().pos_vt(); - auto pos_value = pos_vt.values()[pos]; - if (fidelity == 0) { - // update attribute stored in the positional value-table - pos_vt.set(pos, storage_class, value); - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class)); - } else { - unsigned int offset = field_id.getOffset(); - lofi_store<2>::fromValue(pos_value).set(offset, value.m_store); - pos_vt.set(pos, storage_class, pos_value); - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + super_t::dropMembers(fixture, class_ref); + // finally drop kv-index members + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto it = kv_index_ptr->beginJoin(1); + for (;!it.is_end(); ++it) { + if ((*it).m_type == StorageClass::DELETED || (*it).m_type == StorageClass::UNDEFINED) { + // skip undefined or deleted members + continue; + } + unrefMember(fixture, *it); + class_ref.removeFromSchema(*it); + } } } - void Object::setIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, - unsigned int fidelity, StorageClass storage_class, Value value) - { - auto &index_vt = modify().index_vt(); - if (fidelity == 0) { - auto old_storage_class = index_vt.xvalues()[index_vt_pos].m_type; - unrefMember(*fixture, index_vt.xvalues()[index_vt_pos]); - index_vt.set(index_vt_pos, storage_class, value); - m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_storage_class), getSchemaTypeId(storage_class)); - } else { - auto index_vt_value = index_vt.xvalues()[index_vt_pos].m_value; - auto offset = field_id.getOffset(); - auto old_type_id = getSchemaTypeId(storage_class, lofi_store<2>::fromValue(index_vt_value).get(offset)); - lofi_store<2>::fromValue(index_vt_value).set(offset, value.m_store); - index_vt.set(index_vt_pos, storage_class, index_vt_value); - auto new_type_id = getSchemaTypeId(storage_class, value); - m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); - } - } - - void Object::addToIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, - unsigned int fidelity, StorageClass storage_class, Value value) + bool Object::tryUnrefWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, + StorageClass storage_class, unsigned int fidelity) { - auto &index_vt = modify().index_vt(); - if (fidelity == 0) { - index_vt.set(index_vt_pos, storage_class, value); - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class)); - } else { - auto index_vt_value = index_vt.xvalues()[index_vt_pos].m_value; - lofi_store<2>::fromValue(index_vt_value).set(field_id.getOffset(), value.m_store); - index_vt.set(index_vt_pos, storage_class, index_vt_value); - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + if (super_t::tryUnrefWithLoc(fixture, field_id, loc_ptr, pos, storage_class, fidelity)) { + return true; } + unrefKVIndexValue(fixture, field_id, storage_class, fidelity); + return true; } - + void Object::setKVIndexValue(FixtureLock &fixture, FieldID field_id, unsigned int fidelity, StorageClass storage_class, Value value) { @@ -614,13 +419,13 @@ namespace db0::object_model // in case of the IttyIndex updating an element changes the address // which needs to be updated in the object if (kv_index_ptr->getIndexType() == bindex::type::itty) { - modify().m_kv_address = kv_index_ptr->getAddress(); + this->modify().m_kv_address = kv_index_ptr->getAddress(); } } else { if (kv_index_ptr->insert(xvalue)) { // type or address of the kv-index has changed which needs to be reflected - modify().m_kv_address = kv_index_ptr->getAddress(); - modify().m_kv_type = kv_index_ptr->getIndexType(); + this->modify().m_kv_address = kv_index_ptr->getAddress(); + this->modify().m_kv_type = kv_index_ptr->getIndexType(); } m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); @@ -629,97 +434,72 @@ namespace db0::object_model m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); } } - - void Object::addToKVIndex(FixtureLock &fixture, FieldID field_id, unsigned int fidelity, - StorageClass storage_class, Value value) + + bool Object::tryFindMemberSlot(const std::pair &field_info, unsigned int &pos, + std::pair &result) const { - assert(m_type); - XValue xvalue(field_id.getIndex(), storage_class, value); - // encode for lo-fi storage if needed - if (fidelity != 0) { - xvalue.m_value = lofi_store<2>::create(field_id.getOffset(), value.m_store); + if (super_t::tryFindMemberSlot(field_info, pos, result)) { + return true; } - auto kv_index_ptr = addKV_First(xvalue); + + // kv-index lookup + auto kv_index_ptr = tryGetKV_Index(); if (kv_index_ptr) { - // NOTE: for fidelity > 0 the element might already exist - XValue old_value; - if (fidelity > 0 && kv_index_ptr->updateExisting(xvalue, &old_value)) { - auto kv_value = old_value.m_value; - lofi_store<2>::fromValue(kv_value).set(field_id.getOffset(), value.m_store); - xvalue.m_value = kv_value; - kv_index_ptr->updateExisting(xvalue); - // in case of the IttyIndex updating an element changes the address - // which needs to be updated in the object - if (kv_index_ptr->getIndexType() == bindex::type::itty) { - modify().m_kv_address = kv_index_ptr->getAddress(); - } - } else { - if (kv_index_ptr->insert(xvalue)) { - // type or address of the kv-index has changed which needs to be reflected - modify().m_kv_address = kv_index_ptr->getAddress(); - modify().m_kv_type = kv_index_ptr->getIndexType(); - } - } - } - - m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); - } - - std::pair Object::tryGetMemberSlot(const MemberID &member_id, - unsigned int &pos) const - { - for (auto &field_info: member_id) { auto [index, offset] = field_info.first.getIndexAndOffset(); - // pos-vt lookup - if ((*this)->pos_vt().find(index, pos)) { - if (field_info.second == 0 || slotExists((*this)->pos_vt().values()[pos], field_info.second, offset)) { - return { field_info, &(*this)->pos_vt() }; - } else { - continue; - } - } - - // index-vt lookup - if ((*this)->index_vt().find(index, pos)) { - if (field_info.second == 0 || slotExists((*this)->index_vt().xvalues()[pos].m_value, field_info.second, offset)) { - return { field_info, &(*this)->index_vt() }; - } else { - continue; - } - } - - // kv-index lookup - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - XValue value(index); - if (kv_index_ptr->findOne(value)) { - if (field_info.second == 0 || slotExists(value.m_value, field_info.second, offset)) { - return { field_info, nullptr }; - } + XValue value(index); + if (kv_index_ptr->findOne(value)) { + if (field_info.second == 0 || slotExists(value.m_value, field_info.second, offset)) { + result = { field_info, nullptr }; } + return true; } } - + // not found or deleted - return { {}, nullptr }; + return false; + } + + void Object::setPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, + StorageClass storage_class, Value value) + { + auto &pos_vt = this->modify().pos_vt(); + auto pos_value = pos_vt.values()[pos]; + if (fidelity == 0) { + auto old_storage_class = pos_vt.types()[pos]; + unrefMember(*fixture, old_storage_class, pos_value); + // update attribute stored in the positional value-table + pos_vt.set(pos, storage_class, value); + m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_storage_class), getSchemaTypeId(storage_class)); + } else { + auto offset = field_id.getOffset(); + auto old_type_id = getSchemaTypeId(storage_class, lofi_store<2>::fromValue(pos_value).get(offset)); + lofi_store<2>::fromValue(pos_value).set(offset, value.m_store); + pos_vt.set(pos, storage_class, pos_value); + auto new_type_id = getSchemaTypeId(storage_class, value); + m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); + } } - std::pair Object::tryGetLoc(FieldID field_id) const + void Object::setIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, + unsigned int fidelity, StorageClass storage_class, Value value) { - auto index = field_id.getIndex(); - unsigned int pos = 0; - // pos-vt lookup - if ((*this)->pos_vt().find(index, pos)) { - return { &(*this)->pos_vt(), pos }; - } - // index-vt lookup - if ((*this)->index_vt().find(index, pos)) { - return { &(*this)->index_vt(), pos }; + auto &index_vt = this->modify().index_vt(); + if (fidelity == 0) { + auto old_storage_class = index_vt.xvalues()[index_vt_pos].m_type; + unrefMember(*fixture, index_vt.xvalues()[index_vt_pos]); + index_vt.set(index_vt_pos, storage_class, value); + m_type->updateSchema(field_id, fidelity, getSchemaTypeId(old_storage_class), getSchemaTypeId(storage_class)); + } else { + auto index_vt_value = index_vt.xvalues()[index_vt_pos].m_value; + auto offset = field_id.getOffset(); + auto old_type_id = getSchemaTypeId(storage_class, lofi_store<2>::fromValue(index_vt_value).get(offset)); + lofi_store<2>::fromValue(index_vt_value).set(offset, value.m_store); + index_vt.set(index_vt_pos, storage_class, index_vt_value); + auto new_type_id = getSchemaTypeId(storage_class, value); + m_type->updateSchema(field_id, fidelity, old_type_id, new_type_id); } - // not found or located in the kv-index - return { nullptr, 0 }; } - + void Object::setWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, unsigned int fidelity, StorageClass storage_class, Value value) { @@ -737,7 +517,39 @@ namespace db0::object_model assert(!loc_ptr); setKVIndexValue(fixture, field_id, fidelity, storage_class, value); } - + + void Object::addToPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, unsigned int fidelity, + StorageClass storage_class, Value value) + { + auto &pos_vt = this->modify().pos_vt(); + auto pos_value = pos_vt.values()[pos]; + if (fidelity == 0) { + // update attribute stored in the positional value-table + pos_vt.set(pos, storage_class, value); + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class)); + } else { + unsigned int offset = field_id.getOffset(); + lofi_store<2>::fromValue(pos_value).set(offset, value.m_store); + pos_vt.set(pos, storage_class, pos_value); + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + } + } + + void Object::addToIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, + unsigned int fidelity, StorageClass storage_class, Value value) + { + auto &index_vt = this->modify().index_vt(); + if (fidelity == 0) { + index_vt.set(index_vt_pos, storage_class, value); + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class)); + } else { + auto index_vt_value = index_vt.xvalues()[index_vt_pos].m_value; + lofi_store<2>::fromValue(index_vt_value).set(field_id.getOffset(), value.m_store); + index_vt.set(index_vt_pos, storage_class, index_vt_value); + m_type->addToSchema(field_id, fidelity, getSchemaTypeId(storage_class, value)); + } + } + void Object::addWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, unsigned int fidelity, StorageClass storage_class, Value value) { @@ -755,872 +567,33 @@ namespace db0::object_model addToKVIndex(fixture, field_id, fidelity, storage_class, value); } - void Object::set(FixtureLock &fixture, const char *field_name, ObjectPtr lang_value) - { - assert(hasInstance()); - // attribute delete operation - if (!PyToolkit::isValid(lang_value)) { - remove(fixture, field_name); - return; - } - - auto [type_id, storage_class] = recognizeType(**fixture, lang_value); - - if (this->span() > 1) { - // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag - // this is because the actual change may be missed if performed on a different-then the 1st DP - _touch(); - } - - assert(m_type); - // find already existing field index - auto [member_id, is_init_var] = m_type->findField(field_name); - auto storage_fidelity = getStorageFidelity(storage_class); - // get field ID matching the required storage fidelity - FieldID field_id; - FieldInfo old_field_info; - unsigned int old_pos = 0; - const void *old_loc_ptr = nullptr; - if (member_id) { - std::tie(old_field_info, old_loc_ptr) = tryGetMemberSlot(member_id, old_pos); - } - - if (!member_id || !(field_id = member_id.tryGet(storage_fidelity))) { - // try mutating the class first - member_id = m_type->addField(field_name, storage_fidelity); - field_id = member_id.get(storage_fidelity); - } - - assert(field_id && member_id); - // NOTE: a new member inherits the parent's no-cache flag - // FIXME: value should be destroyed on exception - auto value = createMember(*fixture, type_id, storage_class, lang_value, getMemberFlags()); - // make sure object address is not null - assert(!(storage_class == StorageClass::OBJECT_REF && value.cast() == 0)); - - if (field_id == old_field_info.first) { - // Set / update value at the existing location - setWithLoc(fixture, field_id, old_loc_ptr, old_pos, storage_fidelity, storage_class, value); - } else { - // must reset / unreference the old value (stored elsewhere) - if (old_field_info.first) { - unrefWithLoc(fixture, old_field_info.first, old_loc_ptr, old_pos, StorageClass::UNDEFINED, - old_field_info.second); - } - - const void *loc_ptr = nullptr; - unsigned int pos = 0; - // NOTE: slot may already exist (pos-vt or index-vt) either for regular or lo-fi storage - std::tie(loc_ptr, pos) = tryGetLoc(field_id); - // Either use existing slot or create a new (kv-index) - addWithLoc(fixture, field_id, loc_ptr, pos, storage_fidelity, storage_class, value); - } - - // the KV-index insert operation must be registered as the potential silent mutation - // but the operation can be avoided if the object is already marked as modified - if (!super_t::isModified()) { - this->_touch(); - } - } - - std::pair Object::findField(const char *name) const - { - if (isDropped()) { - // defunct objects should not be accessed - assert(!isDefunct()); - THROWF(db0::InputException) << "Object does not exist"; - } - - auto class_ptr = m_type.get(); - if (!class_ptr) { - // retrieve class from the initializer - class_ptr = &m_init_manager.getInitializer(*this).getClass(); - } - - assert(class_ptr); - return class_ptr->findField(name); - } - - FieldID Object::tryGetMember(const char *field_name, std::pair &member, - bool &is_init_var) const - { - MemberID member_id; - std::tie(member_id, is_init_var) = this->findField(field_name); - bool exists, deleted = false; - if (member_id) { - std::tie(exists, deleted) = tryGetMemberAt(member_id.primary(), member); - if (exists) { - assert(!deleted); - return member_id.primary().first; - } - - // the primary slot was not occupied, try with the secondary - bool secondary_deleted = false; - std::tie(exists, secondary_deleted) = tryGetMemberAt(member_id.secondary(), member); - if (exists) { - assert(!secondary_deleted); - return member_id.secondary().first; - } - - deleted |= secondary_deleted; - } - - if (is_init_var) { - // unless explicitly deleted, - // report as None even if the field_id has not been assigned yet - member = { deleted ? StorageClass::DELETED : StorageClass::NONE, Value() }; - } - - // member not found - return {}; - } - - std::optional Object::tryGetX(const char *field_name) const - { - auto [member_id, is_init_var] = this->findField(field_name); - bool exists, deleted = false; - if (member_id) { - assert(member_id.primary().first); - std::pair member; - std::tie(exists, deleted) = tryGetMemberAt(member_id.primary(), member); - if (exists) { - assert(!deleted); - return XValue(member_id.primary().first.getIndex(), member.first, member.second); - } - // the primary slot was not occupied, try with the secondary - bool secondary_deleted = false; - std::tie(exists, secondary_deleted) = tryGetMemberAt(member_id.secondary(), member); - if (exists) { - assert(!secondary_deleted); - return XValue(member_id.secondary().first.getIndex(), member.first, member.second); - } - deleted |= secondary_deleted; - } - - if (!deleted && is_init_var) { - // unless explicitly deleted, - // report as None even if the field_id has not been assigned yet - return XValue(member_id.primary().first.getIndex(), StorageClass::NONE, Value()); - } - - return std::nullopt; - } - - Object::ObjectSharedPtr Object::tryGet(const char *field_name) const - { - std::pair member; - bool is_init_var = false; - auto field_id = tryGetMember(field_name, member, is_init_var); - // NOTE: init vars are always reported as None if not explicitly set nor explicitly deleted - if (field_id || (is_init_var && member.first != StorageClass::DELETED)) { - auto fixture = this->getFixture(); - // prevent accessing a deleted or undefined member - assert(member.first != StorageClass::DELETED && member.first != StorageClass::UNDEFINED); - // NOTE: offset is required for lo-fi members - return unloadMember( - fixture, member.first, member.second, field_id.maybeOffset(), this->getMemberFlags() - ); - } - - return nullptr; - } - - Object::ObjectSharedPtr Object::tryGetAs(const char *field_name, TypeObjectPtr lang_type) const - { - std::pair member; - bool is_init_var = false; - auto field_id = tryGetMember(field_name, member, is_init_var); - if (field_id || (is_init_var && member.first != StorageClass::DELETED)) { - // prevent accessing a deleted member - assert(member.first != StorageClass::DELETED && member.first != StorageClass::UNDEFINED); - auto fixture = this->getFixture(); - if (member.first == StorageClass::OBJECT_REF) { - auto &class_factory = getClassFactory(*fixture); - return PyToolkit::unloadObject(fixture, member.second.asAddress(), class_factory, lang_type); - } - - // NOTE: offset is required for lo-fi members - return unloadMember( - fixture, member.first, member.second, field_id.getOffset(), this->getMemberFlags() - ); - } - - return nullptr; - } - - Object::ObjectSharedPtr Object::get(const char *field_name) const - { - auto obj = tryGet(field_name); - if (!obj) { - if (isDropped()) { - THROWF(db0::InputException) << "Object is no longer accessible"; - } - THROWF(db0::InputException) << "Attribute not found: " << field_name; - } - return obj; - } - - bool Object::slotExists(Value value, unsigned int fidelity, unsigned int at) const - { - assert(fidelity != 0 && "Operation only available for lo-fi values"); - // lo-fi value - assert(fidelity == 2); - return lofi_store<2>::fromValue(value).isSet(at); - } - - std::pair Object::hasValueAt(Value value, unsigned int fidelity, unsigned int at) const - { - assert(fidelity != 0 && "Operation only available for lo-fi values"); - // lo-fi value - assert(fidelity == 2); - if (lofi_store<2>::fromValue(value).isSet(at)) { - // might be deleted - bool deleted = (lofi_store<2>::fromValue(value).get(at) == Value::DELETED); - return { !deleted, deleted }; - } else { - // NOTE: unset value is assumed as empty / undefined - return { false, false }; - } - } - - std::pair Object::tryGetMemberAt(std::pair field_info, - std::pair &result) const - { - if (!field_info.first) { - return { false, false }; - } - - auto loc = field_info.first.getIndexAndOffset(); - if (!hasInstance()) { - // try retrieving from initializer - auto initializer_ptr = m_init_manager.findInitializer(*this); - if (!initializer_ptr) { - return { false, false }; - } - return { initializer_ptr->tryGetAt(loc, result), false }; - } - - // retrieve from positionally encoded values - if ((*this)->pos_vt().find(loc.first, result)) { - // NOTE: removed field slots might be marked as DELETED - if (result.first == StorageClass::DELETED) { - // report as deleted - return { false, true }; - } - - if (field_info.second == 0) { - return { result.first != StorageClass::UNDEFINED, false }; - } else { - return hasValueAt(result.second, field_info.second, loc.second); - } - } - - if ((*this)->index_vt().find(loc.first, result)) { - if (result.first == StorageClass::DELETED) { - // report as deleted - return { false, true }; - } - - if (field_info.second == 0) { - return { result.first != StorageClass::UNDEFINED, false }; - } else { - return hasValueAt(result.second, field_info.second, loc.second); - } - } - - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - XValue xvalue(loc.first); - if (kv_index_ptr->findOne(xvalue)) { - assert(xvalue.getIndex() == loc.first); - if (xvalue.m_type == StorageClass::DELETED) { - // report as deleted - return { false, true }; - } - - // member fetched from the kv_index - result.first = xvalue.m_type; - result.second = xvalue.m_value; - - if (field_info.second == 0) { - return { result.first != StorageClass::UNDEFINED, false }; - } else { - return hasValueAt(result.second, field_info.second, loc.second); - } - } - } - - // Does not exist, not explicitly removed - return { false, false }; - } - - db0::swine_ptr Object::tryGetFixture() const - { - if (!hasInstance()) { - if (isDropped()) { - return {}; - } - // retrieve from the initializer - return m_init_manager.getInitializer(*this).tryGetFixture(); - } - return super_t::tryGetFixture(); - } - - db0::swine_ptr Object::getFixture() const - { - auto fixture = this->tryGetFixture(); - if (!fixture) { - THROWF(db0::InternalException) << "Object is no longer accessible"; - } - return fixture; - } - - Memspace &Object::getMemspace() const { - return *getFixture(); - } - - void Object::setType(std::shared_ptr type) - { - assert(!m_type); - m_type = type; - assert(hasValidClassRef()); - } - - void Object::setTypeWithHint(std::shared_ptr type_hint) - { - assert(!m_type); - assert(type_hint); - assert(hasInstance()); - if (type_hint->getClassRef() == (*this)->getClassRef()) { - m_type = type_hint; - } else { - m_type = unloadType(); - } - } - - bool Object::isSingleton() const { - return getType().isSingleton(); - } - - void Object::dropTags(Class &type) const - { - // only drop if any type tags are assigned - if ((*this)->m_header.m_ref_counter.getFirst() > 0) { - auto fixture = this->getFixture(); - assert(fixture); - auto &tag_index = fixture->get(); - const Class *type_ptr = &type; - auto unique_address = this->getUniqueAddress(); - while (type_ptr) { - // remove auto-assigned type (or its base) tag - tag_index.removeTypeTag(unique_address, type_ptr->getAddress()); - // NOTE: no need to decRef since object is being destroyed - type_ptr = type_ptr->getBaseClassPtr(); - } - } - } - - void Object::dropMembers(Class &class_ref) const - { - auto fixture = this->getFixture(); - assert(fixture); - // drop pos-vt members first - { - auto &types = (*this)->pos_vt().types(); - auto &values = (*this)->pos_vt().values(); - auto value = values.begin(); - unsigned int index = types.offset(); - for (auto type = types.begin(); type != types.end(); ++type, ++value, ++index) { - if (*type == StorageClass::DELETED || *type == StorageClass::UNDEFINED) { - // skip undefined or deleted members - continue; - } - unrefMember(fixture, *type, *value); - class_ref.removeFromSchema(index, *type, *value); - } - } - // drop index-vt members next - { - auto &xvalues = (*this)->index_vt().xvalues(); - for (auto &xvalue: xvalues) { - if (xvalue.m_type == StorageClass::DELETED || xvalue.m_type == StorageClass::UNDEFINED) { - // skip undefined or deleted members - continue; - } - unrefMember(fixture, xvalue); - class_ref.removeFromSchema(xvalue); - } - } - // finally drop kv-index members - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto it = kv_index_ptr->beginJoin(1); - for (;!it.is_end(); ++it) { - if ((*it).m_type == StorageClass::DELETED || (*it).m_type == StorageClass::UNDEFINED) { - // skip undefined or deleted members - continue; - } - unrefMember(fixture, *it); - class_ref.removeFromSchema(*it); - } - } - } - - void Object::unSingleton(FixtureLock &) - { - auto &type = getType(); - // drop reference from the class - if (type.isSingleton()) { - // clear singleton address - type.unlinkSingleton(); - modify().m_header.decRef(false); - } - } - - void Object::destroy() const - { - if (hasInstance()) { - // associated class type (may require unloading) - auto type = m_type; - if (!type) { - // retrieve type from the initializer - type = std::const_pointer_cast(unloadType()); - } - - dropTags(*type); - dropMembers(*type); - // dereference associated class - type->decRef(false); - } - super_t::destroy(); - } - - FieldLayout Object::getFieldLayout() const - { - FieldLayout layout; - // collect pos-vt information - for (auto type: (*this)->pos_vt().types()) { - layout.m_pos_vt_fields.push_back(type); - } - - // collect index-vt information - for (auto &xvalue: (*this)->index_vt().xvalues()) { - layout.m_index_vt_fields.emplace_back(xvalue.getIndex(), xvalue.m_type); - } - - // collect kv-index information - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto it = kv_index_ptr->beginJoin(1); - for (;!it.is_end(); ++it) { - layout.m_kv_index_fields.emplace_back((*it).getIndex(), (*it).m_type); - } - } - - return layout; - } - - KV_Index *Object::addKV_First(const XValue &value) - { - if (!m_kv_index) { - if ((*this)->m_kv_address) { - m_kv_index = std::make_unique( - std::make_pair(&getMemspace(), (*this)->m_kv_address), (*this)->m_kv_type - ); - } else { - // create new kv-index intiialized with the first value - m_kv_index = std::make_unique(getMemspace(), value); - modify().m_kv_address = m_kv_index->getAddress(); - modify().m_kv_type = m_kv_index->getIndexType(); - // return nullptr to indicate that the value has been inserted - return nullptr; - } - } - // return reference without inserting - return m_kv_index.get(); - } - - bool Object::hasKV_Index() const { - return m_kv_index || (*this)->m_kv_address; - } - - KV_Index *Object::tryGetKV_Index() const - { - // if KV index address has changed, update the cached instance - if (!m_kv_index || m_kv_index->getAddress() != (*this)->m_kv_address) { - if ((*this)->m_kv_address) { - m_kv_index = std::make_unique( - std::make_pair(&getMemspace(), (*this)->m_kv_address), (*this)->m_kv_type - ); - } - } - - return m_kv_index.get(); - } - - Class &Object::getType() { - return m_type ? *m_type : m_init_manager.getInitializer(*this).getClass(); - } - - void Object::getMembersFrom(const Class &this_type, unsigned int index, StorageClass storage_class, - Value value, std::unordered_set &result) const - { - if (storage_class == StorageClass::DELETED || storage_class == StorageClass::UNDEFINED) { - // skip undefined or deleted members - return; - } - - if (storage_class == StorageClass::PACK_2) { - auto it = lofi_store<2>::fromValue(value).begin(), end = lofi_store<2>::fromValue(value).end(); - for (; it != end; ++it) { - result.insert(this_type.getMember({ index, it.getOffset() }).m_name); - } - } else { - result.insert(this_type.getMember(FieldID::fromIndex(index)).m_name); - } - } - - std::unordered_set Object::getMembers() const - { - std::unordered_set result; - // Visit pos-vt members first - auto &obj_type = this->getType(); - { - auto &types = (*this)->pos_vt().types(); - auto &values = (*this)->pos_vt().values(); - unsigned int index = types.offset(); - auto size = types.size(); - for (unsigned int pos = 0;pos < size; ++index, ++pos) { - getMembersFrom(obj_type, index, types[pos], values[pos], result); - } - } - - // Visit index-vt members next - { - auto &xvalues = (*this)->index_vt().xvalues(); - for (auto &xvalue: xvalues) { - auto index = xvalue.getIndex(); - getMembersFrom(obj_type, index, xvalue.m_type, xvalue.m_value, result); - } - } - - // Finally, visit kv-index members - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto it = kv_index_ptr->beginJoin(1); - for (;!it.is_end(); ++it) { - auto index = (*it).getIndex(); - getMembersFrom(obj_type, index, (*it).m_type, (*it).m_value, result); - } - } - return result; - } - - void Object::forAll(std::function f) const - { - // Visit pos-vt members first - auto &obj_type = this->getType(); - { - auto &types = (*this)->pos_vt().types(); - auto &values = (*this)->pos_vt().values(); - auto value = values.begin(); - unsigned int index = types.offset(); - for (auto type = types.begin(); type != types.end(); ++type, ++value, ++index) { - if (*type == StorageClass::DELETED || *type == StorageClass::UNDEFINED) { - // skip deleted or undefined members - continue; - } - if (*type == StorageClass::PACK_2) { - // iterate individual lo-fi members - if (!forAll({index, *type, *value}, f)) { - return; - } - } else { - if (!f(obj_type.getMember(FieldID::fromIndex(index)).m_name, { index, *type, *value }, 0)) { - return; - } - } - } - } - - // Visit index-vt members next - { - auto &xvalues = (*this)->index_vt().xvalues(); - for (auto &xvalue: xvalues) { - if (xvalue.m_type == StorageClass::DELETED || xvalue.m_type == StorageClass::UNDEFINED) { - // skip deleted or undefined members - continue; - } - if (xvalue.m_type == StorageClass::PACK_2) { - // iterate individual lo-fi members - if (!forAll(xvalue, f)) { - return; - } - } else { - // regular member - if (!f(obj_type.getMember(FieldID::fromIndex(xvalue.getIndex())).m_name, xvalue, 0)) { - return; - } - } - } - } - - // Finally, visit kv-index members - auto kv_index_ptr = tryGetKV_Index(); - if (kv_index_ptr) { - auto it = kv_index_ptr->beginJoin(1); - for (;!it.is_end(); ++it) { - if ((*it).m_type == StorageClass::DELETED || (*it).m_type == StorageClass::UNDEFINED) { - // skip deleted or undefined members - continue; - } - if ((*it).m_type == StorageClass::PACK_2) { - // iterate individual lo-fi members - if (!forAll(*it, f)) { - return; - } - } else { - if (!f(obj_type.getMember(FieldID::fromIndex((*it).getIndex())).m_name, *it, 0)) { - return; - } - } - } - } - } - - void Object::forAll(std::function f) const - { - auto fixture = this->getFixture(); - forAll([&](const std::string &name, const XValue &xvalue, unsigned int offset) -> bool { - // all references convert to UUID - auto py_member = unloadMember( - fixture, xvalue.m_type, xvalue.m_value, offset, this->getMemberFlags() - ); - return f(name, py_member); - }); - } - - bool Object::forAll(XValue xvalue, std::function f) const - { - assert(xvalue.m_type == StorageClass::PACK_2); - unsigned int index = xvalue.getIndex(); - auto _value = xvalue.m_value; - auto it = lofi_store<2>::fromValue(_value).begin(), end = lofi_store<2>::fromValue(_value).end(); - auto &obj_type = this->getType(); - for (; it != end; ++it) { - if (!f(obj_type.getMember(FieldID::fromIndex(index, it.getOffset())).m_name, - xvalue, it.getOffset())) - { - return false; - } + bool Object::forAllImpl(std::function f) const + { + if (super_t::forAllImpl(f)) { + // Finally, visit kv-index members + auto kv_index_ptr = tryGetKV_Index(); + if (kv_index_ptr) { + auto &obj_type = this->getType(); + auto it = kv_index_ptr->beginJoin(1); + for (;!it.is_end(); ++it) { + if ((*it).m_type == StorageClass::DELETED || (*it).m_type == StorageClass::UNDEFINED) { + // skip deleted or undefined members + continue; + } + if ((*it).m_type == StorageClass::PACK_2) { + // iterate individual lo-fi members + if (!forAll(*it, f)) { + return false; + } + } else { + if (!f(obj_type.getMember(FieldID::fromIndex((*it).getIndex())).m_name, *it, 0)) { + return false; + } + } + } + } } return true; } - - void Object::incRef(bool is_tag) - { - if (hasInstance()) { - super_t::incRef(is_tag); - } else { - // incRef with the initializer - m_init_manager.getInitializer(*this).incRef(is_tag); - } - } - - bool Object::decRef(bool is_tag) - { - // this operation is a potentially silent mutation - _touch(); - super_t::decRef(is_tag); - return !hasRefs(); - } - - bool Object::hasRefs() const - { - assert(hasInstance()); - return (*this)->hasRefs(); - } - - bool Object::hasAnyRefs() const { - return (*this)->hasAnyRefs(); - } - - bool Object::hasTagRefs() const { - return this->hasInstance() && (*this)->m_header.m_ref_counter.getFirst() > 0; - } - - bool Object::equalTo(const Object &other) const - { - if (!hasInstance() || !other.hasInstance()) { - THROWF(db0::InputException) << "Object not initialized"; - } - - if (this->isDefunct() || other.isDefunct()) { - // defunct objects should not be compared - assert(!isDefunct()); - THROWF(db0::InputException) << "Object does not exist"; - } - - if ((*this)->getClassRef() != other->getClassRef()) { - // different types - return false; - } - - if (this->getFixture()->getUUID() == other.getFixture()->getUUID() - && this->getUniqueAddress() == other.getUniqueAddress()) - { - // comparing 2 versions of the same object (fastest) - if (!((*this)->pos_vt() == other->pos_vt())) { - return false; - } - if (!((*this)->index_vt() == other->index_vt())) { - return false; - } - if (!hasKV_Index() && !other.hasKV_Index()) { - return true; - } - return isEqual(this->tryGetKV_Index(), other.tryGetKV_Index()); - } - - // field-wise compare otherwise (slower) - bool result = true; - this->forAll([&](const std::string &name, const XValue &xvalue, unsigned int offset) -> bool { - auto maybe_other_value = other.tryGetX(name.c_str()); - if (!maybe_other_value) { - result = false; - return false; - } - - if (!xvalue.equalTo(*maybe_other_value, offset)) { - result = false; - return false; - } - return true; - }); - return result; - } - - void Object::moveTo(db0::swine_ptr &) { - throw std::runtime_error("Not implemented"); - } - - void Object::setFixture(db0::swine_ptr &new_fixture) - { - if (hasInstance()) { - THROWF(db0::InputException) << "set_prefix failed: object already initialized"; - } - - if (!m_init_manager.getInitializer(*this).trySetFixture(new_fixture)) { - // signal problem with PyErr_BadPrefix - auto fixture = this->getFixture(); - LangToolkit::setError(LangToolkit::getTypeManager().getBadPrefixError(), fixture->getUUID()); - } - } - - void Object::detach() const - { - m_type->detach(); - // invalidate since detach is not supported by the MorphingBIndex - m_kv_index = nullptr; - super_t::detach(); - } - - void Object::commit() const - { - m_type->commit(); - if (m_kv_index) { - m_kv_index->commit(); - } - super_t::commit(); - // reset the silent-mutation flag - m_touched = false; - } - - void Object::unrefMember(db0::swine_ptr &fixture, StorageClass type, Value value) const { - db0::object_model::unrefMember(fixture, type, value); - } - - void Object::unrefMember(db0::swine_ptr &fixture, XValue value) const { - db0::object_model::unrefMember(fixture, value.m_type, value.m_value); - } - - std::shared_ptr Object::unloadType() const - { - auto fixture = this->getFixture(); - return getClassFactory(*fixture).getTypeByClassRef((*this)->getClassRef()).m_class; - } - - Address Object::getAddress() const - { - assert(!isDefunct()); - if (!hasInstance()) { - THROWF(db0::InternalException) << "Object instance does not exist yet (did you forget to use db0.materialized(self) in constructor ?)"; - } - return super_t::getAddress(); - } - - UniqueAddress Object::getUniqueAddress() const - { - if (hasInstance()) { - return super_t::getUniqueAddress(); - } else { - // NOTE: defunct objects don't have a valid address (not assigned yet) - assert(m_flags[ObjectOptions::DROPPED]); - return m_unique_address; - } - } - - bool Object::hasValidClassRef() const - { - if (hasInstance() && m_type) { - return (*this)->getClassRef() == m_type->getClassRef(); - } - return true; - } - - std::shared_ptr Object::getTypeWithHint(const Fixture &fixture, std::uint32_t class_ref, std::shared_ptr type_hint) - { - assert(type_hint); - if (type_hint->getClassRef() == class_ref) { - return type_hint; - } - return getClassFactory(fixture).getTypeByClassRef(class_ref).m_class; - } - - void Object::setDefunct() const { - m_flags.set(ObjectOptions::DEFUNCT); - } - - void Object::touch() - { - if (hasInstance() && !isDefunct()) { - // NOTE: for already modified and small objects we may skip "touch" - if (!super_t::isModified() || this->span() > 1) { - // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag - // this is because the actual change may be missed if performed on a different-then the 1st DP - _touch(); - } - } - } - - void Object::_touch() - { - if (!m_touched) { - // mark the 1st byte of the object as modified (forced-diff) - // this is always the 1st DP occupied by the object - modify(0, 1); - m_touched = true; - } - } - - void Object::addExtRef() const { - ++m_ext_refs; - } - - void Object::removeExtRef() const - { - assert(m_ext_refs > 0); - --m_ext_refs; - } } \ No newline at end of file diff --git a/src/dbzero/object_model/object/Object.hpp b/src/dbzero/object_model/object/Object.hpp index e19d2dc4..a0e36d2d 100644 --- a/src/dbzero/object_model/object/Object.hpp +++ b/src/dbzero/object_model/object/Object.hpp @@ -1,326 +1,67 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include "ValueTable.hpp" -#include "ObjectInitializer.hpp" -#include -#include -#include -#include -#include "KV_Index.hpp" - -namespace db0 - -{ - - class Fixture; - -} +#include "ObjectImplBase.hpp" +#include "o_object.hpp" namespace db0::object_model { - - class Class; - using Fixture = db0::Fixture; - -DB0_PACKED_BEGIN - class DB0_PACKED_ATTR o_object: public db0::o_base - { - protected: - using super_t = db0::o_base; - - public: - static constexpr unsigned char REALM_ID = 1; - // common object header - o_unique_header m_header; - // optional address of the key-value store (to store extension fields) - KV_Address m_kv_address; - // kv-index type must be stored separately from the address - bindex::type m_kv_type; - // number of auto-assigned type tags - std::uint8_t m_num_type_tags = 0; - - PosVT &pos_vt(); - const PosVT &pos_vt() const; - - const packed_int32 &classRef() const; - std::uint32_t getClassRef() const; - - const IndexVT &index_vt() const; - - IndexVT &index_vt(); - - // ref_counts - the initial reference counts (tags / objects) inherited from the initializer - o_object(std::uint32_t class_ref, std::pair ref_counts, std::uint8_t num_type_tags, - const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin = nullptr, - const XValue *index_vt_end = nullptr); - - static std::size_t measure(std::uint32_t, std::pair, std::uint8_t num_type_tags, - const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin = nullptr, - const XValue *index_vt_end = nullptr); - - template static std::size_t safeSizeOf(BufT buf) - { - return super_t::sizeOfMembers(buf) - (PosVT::type()) - (packed_int32::type()) - (IndexVT::type()); - } - - void incRef(bool is_tag); - bool hasRefs() const; - bool hasAnyRefs() const; - }; -DB0_PACKED_END - - struct FieldLayout - { - std::vector m_pos_vt_fields; - std::vector > m_index_vt_fields; - std::vector > m_kv_index_fields; - }; - - enum class ObjectOptions: std::uint8_t - { - // the dbzero instance has been deleted - DROPPED = 0x01, - // object is defunct - e.g. due to exception on __init__ - DEFUNCT = 0x02 - }; - - using ObjectFlags = db0::FlagSet; - // NOTE: Object instances are created within the realm_id = 1 - using ObjectVType = db0::v_object; - class Object: public db0::ObjectBase + class Object: public ObjectImplBase { // GC0 specific declarations GC0_Declare public: static constexpr unsigned char REALM_ID = o_object::REALM_ID; - using super_t = db0::ObjectBase; - using LangToolkit = LangConfig::LangToolkit; - using ObjectPtr = typename LangToolkit::ObjectPtr; - using TypeObjectPtr = typename LangToolkit::TypeObjectPtr; - using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; - using TypeManager = typename LangToolkit::TypeManager; - using ObjectStem = ObjectVType; - using TypeInitializer = ObjectInitializer::TypeInitializer; - - // construct as null / dropped object - Object(UniqueAddress, unsigned int ext_refs); - Object(const Object &) = delete; - Object(Object &&) = delete; - - /** - * Construct new Object (uninitialized, without corresponding dbzero instance yet) - */ - Object(std::shared_ptr); - Object(TypeInitializer &&); - - // Unload from address with a known type (possibly a base type) - // NOTE: unload works faster if type_hint is the exact object's type - struct with_type_hint {}; - Object(db0::swine_ptr &, Address, std::shared_ptr type_hint, - with_type_hint, AccessFlags = {}); + using super_t = ObjectImplBase; - // Unload from stem with a known type (possibly a base type) - // NOTE: unload works faster if type_hint is the exact object's type - Object(db0::swine_ptr &, ObjectStem &&, std::shared_ptr type_hint, with_type_hint); - - Object(db0::swine_ptr &, Address, AccessFlags = {}); - Object(db0::swine_ptr &, std::shared_ptr, std::pair ref_counts, - const PosVT::Data &, unsigned int pos_vt_offset); - Object(db0::swine_ptr &, ObjectStem &&, std::shared_ptr); - - ~Object(); - - // post-init invoked by memo type directly after __init__ - void postInit(FixtureLock &); - - // Destroys an existing instance and constructs a "null" placeholder - void dropInstance(FixtureLock &); - - // Unload the object stem, to retrieve its type - static ObjectStem tryUnloadStem(db0::swine_ptr &, Address, - std::uint16_t instance_id = 0, AccessFlags = {}); - static ObjectStem unloadStem(db0::swine_ptr &, Address, - std::uint16_t instance_id = 0, AccessFlags = {}); - - // Called to finalize adding members - void endInit(); - - // Assign language specific value as a field (to already initialized or uninitialized instance) - // NOTE: if lang_value is nullptr then the member is removed - void set(FixtureLock &, const char *field_name, ObjectPtr lang_value); - void remove(FixtureLock &, const char *field_name); - - // Assign field of an uninitialized instance (assumed as a non-mutating operation) - // NOTE: if lang_value is nullptr then the member is removed - void setPreInit(const char *field_name, ObjectPtr lang_value) const; - void removePreInit(const char *field_name) const; - - ObjectSharedPtr tryGet(const char *field_name) const; - ObjectSharedPtr tryGetAs(const char *field_name, TypeObjectPtr) const; - ObjectSharedPtr get(const char *field_name) const; - - inline std::shared_ptr getClassPtr() const { - return m_type ? m_type : m_init_manager.getInitializer(*this).getClassPtr(); - } - - inline const Class &getType() const { - return m_type ? *m_type : m_init_manager.getInitializer(*this).getClass(); + template + Object(Args&&... args) + : super_t(std::forward(args)...) + { } - Class &getType(); - - db0::swine_ptr tryGetFixture() const; - - db0::swine_ptr getFixture() const; - - Memspace &getMemspace() const; - - /** - * Get description of the field layout - */ - FieldLayout getFieldLayout() const; - // Convert singleton into a regular instance void unSingleton(FixtureLock &); - - void destroy() const; - bool isSingleton() const; - - // execute the function for all members (until false is returned from the input lambda) - void forAll(std::function) const; - void forAll(std::function) const; - - // get dbzero member / member names assigned to this object - std::unordered_set getMembers() const; - - /** - * The overloaded incRef implementation is provided to also handle non-fully initialized objects - */ - void incRef(bool is_tag); - bool hasRefs() const; - // check for any refs (including auto-assigned type tags) - bool hasAnyRefs() const; - - // check if any references from tags exist (i.e. are any tags assigned) - bool hasTagRefs() const; - - // @return true if reference count was decremented to zero - bool decRef(bool is_tag); - - // Binary (shallow) compare 2 objects or 2 versions of the same memo object (e.g. from different snapshots) - // NOTE: ref-counts are not compared (only user-assigned members) - // @return true if objects are identical - bool equalTo(const Object &) const; - - /** - * Move unreferenced object to a different prefix without changing the instance - * this operations is required for auto-hardening - */ - void moveTo(db0::swine_ptr &); - - /** - * Change fixture of the uninitialized object - * Object must not have any members yet either - */ - void setFixture(db0::swine_ptr &); - - void detach() const; - - void commit() const; - - Address getAddress() const; - UniqueAddress getUniqueAddress() const; - // NOTE: the operation is marked const because the dbzero state is not affected - void setDefunct() const; - - inline bool isDropped() const { - return m_flags.test(ObjectOptions::DROPPED); - } - - inline bool isDefunct() const { - return m_flags.test(ObjectOptions::DEFUNCT); - } - - // is dropped or defunct - inline bool isDead() const { - return m_flags.any( - static_cast(ObjectOptions::DROPPED) | static_cast(ObjectOptions::DEFUNCT) - ); - } - - // FieldID, is_init_var, fidelity - std::pair findField(const char *name) const; + // Assign language specific value as a field (to already initialized or uninitialized instance) + // NOTE: if lang_value is nullptr then the member is removed + void set(FixtureLock &, const char *field_name, ObjectPtr lang_value); + void remove(FixtureLock &, const char *field_name); - // Check if the 2 memo objects are of the same type - bool sameType(const Object &) const; + protected: + friend super_t; - // the member called to indicate the object mutation - void touch(); + bool tryFindMemberAt(std::pair field_info, + std::pair &result, std::pair &find_result) const; - void addExtRef() const; - void removeExtRef() const; + void getFieldLayoutImpl(FieldLayout &layout) const; + void getMembersImpl(std::unordered_set &) const; + bool tryEqualToImpl(const ObjectImplBase &, bool &result) const; - inline std::uint32_t getExtRefs() const { - return m_ext_refs; - } - - private: - // Class will only be assigned after initialization - std::shared_ptr m_type; - // local kv-index instance cache (created at first use) - mutable std::unique_ptr m_kv_index; - static ObjectInitializerManager m_init_manager; - mutable ObjectFlags m_flags; - // reference counter for inner references from language objects - // NOTE: inner references are held by internal dbzero buffers (e.g. TagIndex) - // see also PyEXT_INCREF / PyEXT_DECREF - mutable std::uint32_t m_ext_refs = 0; - // A flag indicating that object's silent mutation has already been reflected - // with the underlying MemLock / ResourceLock - // NOTE: by silent mutation we mean a mutation that does not change data (e.g. +refcount(+-1) + (refcount-1)) - mutable bool m_touched = false; - // NOTE: member assigned only to dropped objects (see replaceWithNull) - // so that we can retrieve the address of the dropped instance after it has been destroyed - const UniqueAddress m_unique_address; + // Set or update member in a pos_vt + void setPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); + void setIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, + StorageClass, Value); - void setType(std::shared_ptr); - // adjusts to actual type if the type hint is a base class - void setTypeWithHint(std::shared_ptr type_hint); - // @return exists / deleted - std::pair hasValueAt(Value, unsigned int fidelity, unsigned int at) const; - // similar to hasValueAt but assume deleted slot as present - bool slotExists(Value value, unsigned int fidelity, unsigned int at) const; + // Set with a specific location (pos_vt, index_vt, kv-index) + void setWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, + StorageClass, Value); - // Try retrieving member either from values (initialized) or from the initialization buffer (not initialized yet) - // @return member exists, member deleted flags - std::pair tryGetMemberAt(std::pair, - std::pair &) const; - FieldID tryGetMember(const char *field_name, std::pair &, bool &is_init_var) const; + // Add a new value + void addToPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); + void addToIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, StorageClass, Value); - // Try resolving field ID of an existing (or deleted) member and also its storage location - // @param pos the member's position in the containing collection - // @return FieldID + containing collection (e.g. pos_vt()) - std::pair tryGetMemberSlot(const MemberID &, unsigned int &pos) const; + void addWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, + StorageClass, Value); - // Try locating a field ID associated slot - std::pair tryGetLoc(FieldID) const; + void dropMembers(db0::swine_ptr &, Class &) const; - inline ObjectInitializer *tryGetInitializer() const { - return m_type ? static_cast(nullptr) : &m_init_manager.getInitializer(*this); - } + bool tryUnrefWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, StorageClass, + unsigned int fidelity); + bool tryFindMemberSlot(const std::pair &field_info, unsigned int &pos, + std::pair &result) const; /** * If the KV_Index does not exist yet, create it and add the first value @@ -330,63 +71,14 @@ DB0_PACKED_END KV_Index *tryGetKV_Index() const; - void dropMembers(Class &) const; - void dropTags(Class &) const; - - void unrefMember(db0::swine_ptr &, StorageClass, Value) const; - void unrefMember(db0::swine_ptr &, XValue) const; - - using TypeId = db0::bindings::TypeId; - std::pair recognizeType(Fixture &, ObjectPtr lang_value) const; - - // Unload associated type - std::shared_ptr unloadType() const; - - // Retrieve a type by class-ref with a possible match (type_hint) - static std::shared_ptr getTypeWithHint(const Fixture &, std::uint32_t class_ref, std::shared_ptr type_hint); - - bool hasValidClassRef() const; bool hasKV_Index() const; - - // try retrieving member as XValue - std::optional tryGetX(const char *field_name) const; - void _touch(); - - // Set or update member in a pos_vt - void setPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); - void setIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, - StorageClass, Value); + + void addToKVIndex(FixtureLock &, FieldID, unsigned int fidelity, StorageClass, Value); + void unrefKVIndexValue(FixtureLock &, FieldID, StorageClass, unsigned int fidelity); // Set or update member in kv-index void setKVIndexValue(FixtureLock &, FieldID, unsigned int fidelity, StorageClass, Value); - // Set with a specific location (pos_vt, index_vt, kv-index) - void setWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, - StorageClass, Value); - - // Unreference value - // NOTE: storage_class to be assigned can either be DELETED or UNDEFINED - void unrefPosVT(FixtureLock &, FieldID, unsigned int pos, StorageClass, unsigned int fidelity); - void unrefIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, StorageClass, unsigned int fidelity); - void unrefKVIndexValue(FixtureLock &, FieldID, StorageClass, unsigned int fidelity); - - void unrefWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, StorageClass, - unsigned int fidelity); - - // Add a new value - void addToPosVT(FixtureLock &, FieldID, unsigned int pos, unsigned int fidelity, StorageClass, Value); - void addToIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, unsigned int fidelity, StorageClass, Value); - void addToKVIndex(FixtureLock &, FieldID, unsigned int fidelity, StorageClass, Value); - - void addWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, unsigned int fidelity, - StorageClass, Value); - - // lo-fi member specialized implementation - bool forAll(XValue, std::function) const; - - void getMembersFrom(const Class &this_type, unsigned int index, StorageClass, Value, - std::unordered_set &) const; + bool forAllImpl(std::function) const; }; } - -DECLARE_ENUM_VALUES(db0::object_model::ObjectOptions, 2) \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectAnyBase.cpp b/src/dbzero/object_model/object/ObjectAnyBase.cpp new file mode 100644 index 00000000..7c846976 --- /dev/null +++ b/src/dbzero/object_model/object/ObjectAnyBase.cpp @@ -0,0 +1,160 @@ +#include "ObjectAnyBase.hpp" + +namespace db0::object_model + +{ + + ObjectInitializerManager InitManager::instance; + + template + ObjectAnyBase::ObjectAnyBase(UniqueAddress addr, unsigned int ext_refs) + : m_flags { ObjectOptions::DROPPED } + , m_ext_refs(ext_refs) + , m_unique_address(addr) + { + } + + template + db0::swine_ptr ObjectAnyBase::tryGetFixture() const + { + if (!this->hasInstance()) { + if (isDropped()) { + return {}; + } + // retrieve from the initializer + return InitManager::instance.getInitializer(*this).tryGetFixture(); + } + return super_t::tryGetFixture(); + } + + template + db0::swine_ptr ObjectAnyBase::getFixture() const + { + auto fixture = this->tryGetFixture(); + if (!fixture) { + THROWF(db0::InternalException) << "Object is no longer accessible"; + } + return fixture; + } + + template + Memspace &ObjectAnyBase::getMemspace() const { + return *getFixture(); + } + + template + void ObjectAnyBase::setFixture(db0::swine_ptr &new_fixture) + { + if (this->hasInstance()) { + THROWF(db0::InputException) << "set_prefix failed: object already initialized"; + } + + if (!InitManager::instance.getInitializer(*this).trySetFixture(new_fixture)) { + // signal problem with PyErr_BadPrefix + auto fixture = this->getFixture(); + LangToolkit::setError(LangToolkit::getTypeManager().getBadPrefixError(), fixture->getUUID()); + } + } + + template + Address ObjectAnyBase::getAddress() const + { + assert(!isDefunct()); + if (!this->hasInstance()) { + THROWF(db0::InternalException) << "Object instance does not exist yet (did you forget to use db0.materialized(self) in constructor ?)"; + } + return super_t::getAddress(); + } + + template + UniqueAddress ObjectAnyBase::getUniqueAddress() const + { + if (this->hasInstance()) { + return super_t::getUniqueAddress(); + } else { + // NOTE: defunct objects don't have a valid address (not assigned yet) + assert(m_flags[ObjectOptions::DROPPED]); + return m_unique_address; + } + } + + template + void ObjectAnyBase::incRef(bool is_tag) + { + if (this->hasInstance()) { + super_t::incRef(is_tag); + } else { + // incRef with the initializer + InitManager::instance.getInitializer(*this).incRef(is_tag); + } + } + + template + void ObjectAnyBase::decRef(bool is_tag) + { + // this operation is a potentially silent mutation + _touch(); + super_t::decRef(is_tag); + } + + template + bool ObjectAnyBase::hasAnyRefs() const { + return (*this)->hasAnyRefs(); + } + + template + bool ObjectAnyBase::hasTagRefs() const { + return this->hasInstance() && (*this)->m_header.m_ref_counter.getFirst() > 0; + } + + template + void ObjectAnyBase::touch() + { + if (this->hasInstance() && !this->isDefunct()) { + // NOTE: for already modified and small objects we may skip "touch" + if (!super_t::isModified() || this->span() > 1) { + // NOTE: large objects i.e. with span > 1 must always be marked with a silent mutation flag + // this is because the actual change may be missed if performed on a different-then the 1st DP + this->_touch(); + } + } + } + + template + void ObjectAnyBase::_touch() + { + if (!m_touched) { + // mark the 1st byte of the object as modified (forced-diff) + // this is always the 1st DP occupied by the object + this->modify(0, 1); + m_touched = true; + } + } + + template + void ObjectAnyBase::addExtRef() const { + ++m_ext_refs; + } + + template + void ObjectAnyBase::removeExtRef() const + { + assert(m_ext_refs > 0); + --m_ext_refs; + } + + template + void ObjectAnyBase::setDefunct() const { + m_flags.set(ObjectOptions::DEFUNCT); + } + + template + Class &ObjectAnyBase::getType() { + return this->m_type ? *this->m_type : InitManager::instance.getInitializer(*this).getClass(); + } + + template class ObjectAnyBase; + template class ObjectAnyBase; + template class ObjectAnyBase; + +} diff --git a/src/dbzero/object_model/object/ObjectAnyBase.hpp b/src/dbzero/object_model/object/ObjectAnyBase.hpp new file mode 100644 index 00000000..b2950dfd --- /dev/null +++ b/src/dbzero/object_model/object/ObjectAnyBase.hpp @@ -0,0 +1,163 @@ +#pragma once + +#include +#include +#include +#include +#include +#include "ValueTable.hpp" +#include "ObjectInitializer.hpp" +#include +#include +#include +#include +#include "o_object.hpp" +#include "o_immutable_object.hpp" + +namespace db0 + +{ + + class Fixture; + +} + +namespace db0::object_model + +{ + + class Class; + class ObjectAnyImpl; + using Fixture = db0::Fixture; + + enum class ObjectOptions: std::uint8_t + { + // the dbzero instance has been deleted + DROPPED = 0x01, + // object is defunct - e.g. due to exception on __init__ + DEFUNCT = 0x02 + }; + + using ObjectFlags = db0::FlagSet; + // NOTE: Object instances are created within the implementation specific realm_id (e.g. =1 for o_object) + template using ObjectVType = db0::v_object; + + // Common init manager for all specializations + class InitManager + { + public: + static ObjectInitializerManager instance; + }; + + template + class ObjectAnyBase: public db0::ObjectBase, StorageClass::OBJECT_REF> + { + public: + static constexpr unsigned char REALM_ID = T::REALM_ID; + using super_t = db0::ObjectBase, StorageClass::OBJECT_REF>; + using LangToolkit = LangConfig::LangToolkit; + using ObjectPtr = typename LangToolkit::ObjectPtr; + using TypeObjectPtr = typename LangToolkit::TypeObjectPtr; + using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; + using TypeManager = typename LangToolkit::TypeManager; + using ObjectStem = ObjectVType; + using TypeInitializer = ObjectInitializer::TypeInitializer; + + db0::swine_ptr tryGetFixture() const; + db0::swine_ptr getFixture() const; + + Memspace &getMemspace() const; + + inline std::shared_ptr getClassPtr() const { + return this->m_type ? this->m_type : InitManager::instance.getInitializer(*this).getClassPtr(); + } + + inline const Class &getType() const { + return this->m_type ? *this->m_type : InitManager::instance.getInitializer(*this).getClass(); + } + + Class &getType(); + + /** + * Change fixture of the uninitialized object + * Object must not have any members yet either + */ + void setFixture(db0::swine_ptr &); + + /** + * The overloaded incRef implementation is provided to also handle non-fully initialized objects + */ + void incRef(bool is_tag); + void decRef(bool is_tag); + + // check for any refs (including auto-assigned type tags) + bool hasAnyRefs() const; + + // check if any references from tags exist (i.e. are any tags assigned) + bool hasTagRefs() const; + + Address getAddress() const; + UniqueAddress getUniqueAddress() const; + + // NOTE: the operation is marked const because the dbzero state is not affected + void setDefunct() const; + + inline bool isDropped() const { + return m_flags.test(ObjectOptions::DROPPED); + } + + inline bool isDefunct() const { + return m_flags.test(ObjectOptions::DEFUNCT); + } + + // is dropped or defunct + inline bool isDead() const { + return m_flags.any( + static_cast(ObjectOptions::DROPPED) | static_cast(ObjectOptions::DEFUNCT) + ); + } + + // the member called to indicate the object mutation + void touch(); + + void addExtRef() const; + void removeExtRef() const; + + inline std::uint32_t getExtRefs() const { + return m_ext_refs; + } + + protected: + // Class will only be assigned after initialization + std::shared_ptr m_type; + mutable ObjectFlags m_flags; + // reference counter for inner references from language objects + // NOTE: inner references are held by internal dbzero buffers (e.g. TagIndex) + // see also PyEXT_INCREF / PyEXT_DECREF + mutable std::uint32_t m_ext_refs = 0; + // A flag indicating that object's silent mutation has already been reflected + // with the underlying MemLock / ResourceLock + // NOTE: by silent mutation we mean a mutation that does not change data (e.g. +refcount(+-1) + (refcount-1)) + mutable bool m_touched = false; + // NOTE: member assigned only to dropped objects (see replaceWithNull) + // so that we can retrieve the address of the dropped instance after it has been destroyed + const UniqueAddress m_unique_address; + + template ObjectAnyBase(Args&&... args) + : super_t(std::forward(args)...) + { + } + + // As a dropped object + ObjectAnyBase(UniqueAddress addr, unsigned int ext_refs); + + void _touch(); + }; + + extern template class ObjectAnyBase; + extern template class ObjectAnyBase; + extern template class ObjectAnyBase; + +} + +DECLARE_ENUM_VALUES(db0::object_model::ObjectOptions, 2) \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectAnyImpl.cpp b/src/dbzero/object_model/object/ObjectAnyImpl.cpp new file mode 100644 index 00000000..3238ad66 --- /dev/null +++ b/src/dbzero/object_model/object/ObjectAnyImpl.cpp @@ -0,0 +1,9 @@ +#include "ObjectAnyImpl.hpp" +#include "o_object.hpp" + +DEFINE_ENUM_VALUES(db0::object_model::ObjectOptions, "DROPPED", "DEFUNCT") + +namespace db0::object_model + +{ +} diff --git a/src/dbzero/object_model/object/ObjectAnyImpl.hpp b/src/dbzero/object_model/object/ObjectAnyImpl.hpp new file mode 100644 index 00000000..1b861cfb --- /dev/null +++ b/src/dbzero/object_model/object/ObjectAnyImpl.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include "ObjectAnyBase.hpp" +#include "o_object.hpp" + +namespace db0::object_model + +{ + + // NOTE: ObjectAnyImpl is for reinterpret_cast purposes only + // it allows accessing Object or ObjectImmutableImpl instances under a common base type + class ObjectAnyImpl: public ObjectAnyBase + { + public: + static constexpr unsigned char REALM_ID = o_object_base::REALM_ID; + using super_t = ObjectAnyBase; + + protected: + friend super_t; + }; + +} diff --git a/src/dbzero/object_model/object/ObjectImmutableImpl.cpp b/src/dbzero/object_model/object/ObjectImmutableImpl.cpp new file mode 100644 index 00000000..539d5cd6 --- /dev/null +++ b/src/dbzero/object_model/object/ObjectImmutableImpl.cpp @@ -0,0 +1,9 @@ +#include "ObjectImmutableImpl.hpp" + +namespace db0::object_model + +{ + + GC0_Define(ObjectImmutableImpl) + +} \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectImmutableImpl.hpp b/src/dbzero/object_model/object/ObjectImmutableImpl.hpp new file mode 100644 index 00000000..f3d5437c --- /dev/null +++ b/src/dbzero/object_model/object/ObjectImmutableImpl.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include "ObjectImplBase.hpp" +#include "o_immutable_object.hpp" + +namespace db0::object_model + +{ + + class ObjectImmutableImpl: public ObjectImplBase + { + // GC0 specific declarations + GC0_Declare + public: + static constexpr unsigned char REALM_ID = o_immutable_object::REALM_ID; + using super_t = ObjectImplBase; + + template + ObjectImmutableImpl(Args&&... args) + : super_t(std::forward(args)...) + { + } + }; + +} diff --git a/src/dbzero/object_model/object/ObjectImplBase.cpp b/src/dbzero/object_model/object/ObjectImplBase.cpp new file mode 100644 index 00000000..d0963a29 --- /dev/null +++ b/src/dbzero/object_model/object/ObjectImplBase.cpp @@ -0,0 +1,1069 @@ +#include "ObjectImplBase.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace db0::object_model + +{ + + FlagSet getAccessOptions(const Class &type) { + return type.isNoCache() ? FlagSet { AccessOptions::no_cache } : FlagSet {}; + } + + template IntT safeCast(unsigned int value, const char *err_msg) + { + if (value > std::numeric_limits::max()) { + THROWF(db0::InputException) << err_msg; + } + return static_cast(value); + } + + template + ObjectImplBase::ObjectImplBase(UniqueAddress addr, unsigned int ext_refs) + : super_t(addr, ext_refs) + { + } + + template + ObjectImplBase::ObjectImplBase(std::shared_ptr db0_class) + { + // prepare for initialization + InitManager::instance.addInitializer(*this, db0_class); + } + + template + ObjectImplBase::ObjectImplBase(TypeInitializer &&type_initializer) + { + // prepare for initialization + InitManager::instance.addInitializer(*this, std::move(type_initializer)); + } + + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, std::shared_ptr type, + std::pair ref_counts, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset) + : super_t(fixture, type->getClassRef(), ref_counts, + safeCast(type->getNumBases() + 1, "Too many base classes"), pos_vt_data, pos_vt_offset, nullptr, nullptr, + getAccessOptions(*type)) + { + this->m_type = type; + } + + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, Address address, AccessFlags access_mode) + : super_t(typename super_t::tag_from_address(), fixture, address, access_mode) + { + } + + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type) + : super_t(typename super_t::tag_from_stem(), fixture, std::move(stem)) + { + this->m_type = type; + assert(hasValidClassRef()); + } + + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, Address address, std::shared_ptr type_hint, + with_type_hint, AccessFlags access_mode) + : ObjectImplBase(fixture, address, access_mode) + { + assert(*fixture == *type_hint->getFixture()); + setTypeWithHint(type_hint); + } + + template + ObjectImplBase::ObjectImplBase(db0::swine_ptr &fixture, ObjectStem &&stem, std::shared_ptr type_hint, with_type_hint) + : ObjectImplBase(fixture, std::move(stem), getTypeWithHint(*fixture, stem->getClassRef(), type_hint)) + { + } + + template + ObjectImplBase::~ObjectImplBase() + { + // unregister needs to be called before destruction of members + this->unregister(); + if (!this->hasInstance()) { + // release initializer if it exists, object not created + InitManager::instance.tryCloseInitializer(*this); + } + } + + template + void ObjectImplBase::dropInstance(FixtureLock &) + { + auto unique_addr = this->getUniqueAddress(); + auto ext_refs = this->getExtRefs(); + this->~ObjectImplBase(); + // construct a null placeholder + new ((void*)this) Object(unique_addr, ext_refs); + } + + template + typename ObjectImplBase::ObjectStem ObjectImplBase::tryUnloadStem(db0::swine_ptr &fixture, + Address address, std::uint16_t instance_id, AccessFlags access_mode) + { + std::size_t size_of; + if (!fixture->isAddressValid(address, super_t::REALM_ID, &size_of)) { + return {}; + } + // Unload from a verified address + ObjectStem stem(db0::tag_verified(), fixture->myPtr(address), size_of, access_mode); + if (instance_id && stem->m_header.m_instance_id != instance_id) { + // instance ID validation failed + return {}; + } + return stem; + } + + template + typename ObjectImplBase::ObjectStem ObjectImplBase::unloadStem(db0::swine_ptr &fixture, + Address address, std::uint16_t instance_id, AccessFlags access_mode) + { + auto result = tryUnloadStem(fixture, address, instance_id, access_mode); + if (!result) { + THROWF(db0::InputException) << "Invalid UUID or object has been deleted"; + } + return result; + } + + template + void ObjectImplBase::postInit(FixtureLock &fixture) + { + if (!this->hasInstance()) { + auto &initializer = InitManager::instance.getInitializer(*this); + PosVT::Data pos_vt_data; + unsigned int pos_vt_offset = 0; + auto index_vt_data = initializer.getData(pos_vt_data, pos_vt_offset); + + // place object in the same fixture as its class + // construct the dbzero instance & assign to self + this->m_type = initializer.getClassPtr(); + assert(this->m_type); + + auto &type = *this->m_type; + super_t::init(*fixture, type.getClassRef(), initializer.getRefCounts(), + safeCast(type.getNumBases() + 1, "Too many base classes"), + pos_vt_data, pos_vt_offset, index_vt_data.first, index_vt_data.second, + getAccessOptions(type) + ); + + // reference associated class + type.incRef(false); + type.updateSchema(pos_vt_offset, pos_vt_data.m_types, pos_vt_data.m_values); + type.updateSchema(index_vt_data.first, index_vt_data.second); + + // bind singleton address (now that instance exists) + if (type.isSingleton()) { + type.setSingletonAddress(*this); + } + initializer.close(); + } + + assert(this->hasInstance()); + } + + template + std::pair + ObjectImplBase::recognizeType(Fixture &fixture, ObjectPtr lang_value) const + { + auto type_id = LangToolkit::getTypeManager().getTypeId(lang_value); + // NOTE: allow storage as PACK_2 + auto pre_storage_class = TypeUtils::m_storage_class_mapper.getPreStorageClass(type_id, true); + if (type_id == TypeId::MEMO_OBJECT) { + // object reference must be from the same fixture + auto &obj = LangToolkit::getTypeManager().extractAnyObject(lang_value); + if (fixture.getUUID() != obj.getFixture()->getUUID()) { + THROWF(db0::InputException) << "Referencing objects from foreign prefixes is not allowed. Use db0.weak_proxy instead"; + } + } + + // may need to refine the storage class (i.e. long weak ref might be needed instead) + StorageClass storage_class; + if (pre_storage_class == PreStorageClass::OBJECT_WEAK_REF) { + storage_class = db0::getStorageClass(pre_storage_class, fixture, lang_value); + } else { + storage_class = db0::getStorageClass(pre_storage_class); + } + + return { type_id, storage_class }; + } + + template + void ObjectImplBase::removePreInit(const char *field_name) const + { + auto &initializer = InitManager::instance.getInitializer(*this); + auto &type = initializer.getClass(); + + // Find an already existing field index + auto member_id = std::get<0>(type.findField(field_name)); + if (!member_id) { + THROWF(db0::InputException) << "Attribute not found: " << field_name; + } + + for (const auto &field_info: member_id) { + assert(field_info.first); + auto loc = field_info.first.getIndexAndOffset(); + // mark as deleted + if (field_info.second == 0) { + initializer.set(loc, StorageClass::DELETED, {}); + } else { + assert(field_info.second == 2 && "Only fidelity == 2 is supported"); + if (member_id.hasFidelity(0)) { + // remove any existing regular initialization + auto loc0 = member_id.get(0).getIndexAndOffset(); + initializer.remove(loc0); + } + initializer.set(loc, StorageClass::PACK_2, Value::DELETED, + lofi_store<2>::mask(loc.second)); + } + } + } + + template + void ObjectImplBase::setPreInit(const char *field_name, ObjectPtr obj_ptr) const + { + assert(!this->hasInstance()); + if (!LangToolkit::isValid(obj_ptr)) { + removePreInit(field_name); + return; + } + + auto &initializer = InitManager::instance.getInitializer(*this); + auto fixture = initializer.getFixture(); + auto &type = initializer.getClass(); + auto [type_id, storage_class] = recognizeType(*fixture, obj_ptr); + auto storage_fidelity = getStorageFidelity(storage_class); + + // Find an already existing field index + auto [member_id, is_init_var] = type.findField(field_name); + // NOTE: even if a field already exists we might need to extend its supported fidelities + if (!member_id || !member_id.hasFidelity(storage_fidelity)) { + // update class definition + // use the default fidelity for the storage class + member_id = type.addField(field_name, storage_fidelity); + } + + if (storage_fidelity == 0) { + if (member_id.hasFidelity(2)) { + // remove any existing lo-fi initialization + auto loc = member_id.get(2).getIndexAndOffset(); + initializer.remove(loc, lofi_store<2>::mask(loc.second)); + } + // register a regular member with the initializer + // NOTE: a new member receives the no-cache flag if set (at the type level) + auto member_flags = type.isNoCache() ? AccessFlags { AccessOptions::no_cache } : AccessFlags(); + initializer.set(member_id.get(0).getIndexAndOffset(), storage_class, + createMember(fixture, type_id, storage_class, obj_ptr, member_flags) + ); + } else { + if (member_id.hasFidelity(0)) { + // remove any existing regular initialization + auto loc = member_id.get(0).getIndexAndOffset(); + initializer.remove(loc); + } + // For now only fidelity == 2 is supported (lo-fi storage) + assert(storage_fidelity == 2); + auto loc = member_id.get(storage_fidelity).getIndexAndOffset(); + // no access flags for lo-fi members + auto value = lofi_store<2>::create(loc.second, + createMember(fixture, type_id, storage_class, obj_ptr, {}).m_store); + // register a lo-fi member with the initializer (using mask) + initializer.set(loc, storage_class, value, lofi_store<2>::mask(loc.second)); + } + } + + template + void ObjectImplBase::unrefPosVT(FixtureLock &fixture, FieldID field_id, unsigned int pos, + StorageClass storage_class, unsigned int fidelity) + { + auto &pos_vt = this->modify().pos_vt(); + auto old_storage_class = pos_vt.types()[pos]; + if (fidelity == 0) { + unrefMember(*fixture, old_storage_class, pos_vt.values()[pos]); + // mark member as unreferenced by assigning storage class + pos_vt.set(pos, storage_class, {}); + this->m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(old_storage_class)); + } else { + assert(fidelity == 2); + auto value = pos_vt.values()[pos]; + auto offset = field_id.getOffset(); + if (storage_class != StorageClass::DELETED && !lofi_store<2>::fromValue(value).isSet(offset)) { + // value is already unset + return; + } + + auto old_type_id = getSchemaTypeId(old_storage_class, lofi_store<2>::fromValue(value).get(offset)); + // either reset or mark as deleted + if (storage_class == StorageClass::DELETED) { + lofi_store<2>::fromValue(value).set(offset, Value::DELETED); + } else { + lofi_store<2>::fromValue(value).reset(offset); + } + pos_vt.set(pos, old_storage_class, value); + this->m_type->removeFromSchema(field_id, fidelity, old_type_id); + } + } + + template + void ObjectImplBase::unrefIndexVT(FixtureLock &fixture, FieldID field_id, unsigned int index_vt_pos, + StorageClass storage_class, unsigned int fidelity) + { + auto &index_vt = this->modify().index_vt(); + auto old_storage_class = index_vt.xvalues()[index_vt_pos].m_type; + if (fidelity == 0) { + unrefMember(*fixture, index_vt.xvalues()[index_vt_pos]); + // mark member as unreferenced by assigning storage class + index_vt.set(index_vt_pos, storage_class, {}); + this->m_type->removeFromSchema(field_id, fidelity, getSchemaTypeId(old_storage_class)); + } else { + assert(fidelity == 2); + auto value = index_vt.xvalues()[index_vt_pos].m_value; + auto offset = field_id.getOffset(); + if (storage_class != StorageClass::DELETED && !lofi_store<2>::fromValue(value).isSet(offset)) { + // value is already unset + return; + } + auto old_type_id = getSchemaTypeId(old_storage_class, lofi_store<2>::fromValue(value).get(offset)); + if (storage_class == StorageClass::DELETED) { + lofi_store<2>::fromValue(value).set(offset, Value::DELETED); + } else { + lofi_store<2>::fromValue(value).reset(offset); + } + index_vt.set(index_vt_pos, old_storage_class, value); + this->m_type->removeFromSchema(field_id, fidelity, old_type_id); + } + } + + template + void ObjectImplBase::unrefWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, + StorageClass storage_class, unsigned int fidelity) + { + // call the actual implementation + static_cast(this)->tryUnrefWithLoc(fixture, field_id, loc_ptr, pos, storage_class, fidelity); + } + + template + bool ObjectImplBase::tryUnrefWithLoc(FixtureLock &fixture, FieldID field_id, const void *loc_ptr, unsigned int pos, + StorageClass storage_class, unsigned int fidelity) + { + if (loc_ptr == &(*this)->pos_vt()) { + unrefPosVT(fixture, field_id, pos, storage_class, fidelity); + return true; + } else if (loc_ptr == &(*this)->index_vt()) { + unrefIndexVT(fixture, field_id, pos, storage_class, fidelity); + return true; + } + return false; + } + + template + bool ObjectImplBase::tryFindMemberSlot(const std::pair &field_info, unsigned int &pos, + std::pair &result) const + { + auto [index, offset] = field_info.first.getIndexAndOffset(); + // pos-vt lookup + if ((*this)->pos_vt().find(index, pos)) { + if (field_info.second == 0 || slotExists((*this)->pos_vt().values()[pos], field_info.second, offset)) { + result = { field_info, &(*this)->pos_vt() }; + } + return true; + } + + // index-vt lookup + if ((*this)->index_vt().find(index, pos)) { + if (field_info.second == 0 || slotExists((*this)->index_vt().xvalues()[pos].m_value, field_info.second, offset)) { + result = { field_info, &(*this)->index_vt() }; + } + return true; + + } + // not found but the lookup may be continued in the kv-index + return false; + } + + template std::pair + ObjectImplBase::tryGetMemberSlot(const MemberID &member_id, unsigned int &pos) const + { + std::pair result; + for (auto &field_info: member_id) { + // call the actual implementation + if (static_cast(this)->tryFindMemberSlot(field_info, pos, result)) { + // otherwise continue since member might've been deleted and reassigned to a different slot + if (result.first.first) { + return result; + } + } + } + + // not found or deleted + return { {}, nullptr }; + } + + template + std::pair ObjectImplBase::tryGetLoc(FieldID field_id) const + { + auto index = field_id.getIndex(); + unsigned int pos = 0; + // pos-vt lookup + if ((*this)->pos_vt().find(index, pos)) { + return { &(*this)->pos_vt(), pos }; + } + // index-vt lookup + if ((*this)->index_vt().find(index, pos)) { + return { &(*this)->index_vt(), pos }; + } + // not found or located in the kv-index + return { nullptr, 0 }; + } + + template + std::pair ObjectImplBase::findField(const char *name) const + { + if (this->isDropped()) { + // defunct objects should not be accessed + assert(!this->isDefunct()); + THROWF(db0::InputException) << "Object does not exist"; + } + + auto class_ptr = this->m_type.get(); + if (!class_ptr) { + // retrieve class from the initializer + class_ptr = &InitManager::instance.getInitializer(*this).getClass(); + } + + assert(class_ptr); + return class_ptr->findField(name); + } + + template + FieldID ObjectImplBase::tryGetMember(const char *field_name, std::pair &member, + bool &is_init_var) const + { + MemberID member_id; + std::tie(member_id, is_init_var) = this->findField(field_name); + bool exists, deleted = false; + if (member_id) { + std::tie(exists, deleted) = tryGetMemberAt(member_id.primary(), member); + if (exists) { + assert(!deleted); + return member_id.primary().first; + } + + // the primary slot was not occupied, try with the secondary + bool secondary_deleted = false; + std::tie(exists, secondary_deleted) = tryGetMemberAt(member_id.secondary(), member); + if (exists) { + assert(!secondary_deleted); + return member_id.secondary().first; + } + + deleted |= secondary_deleted; + } + + if (is_init_var) { + // unless explicitly deleted, + // report as None even if the field_id has not been assigned yet + member = { deleted ? StorageClass::DELETED : StorageClass::NONE, Value() }; + } + + // member not found + return {}; + } + + template + std::optional ObjectImplBase::tryGetX(const char *field_name) const + { + auto [member_id, is_init_var] = this->findField(field_name); + bool exists, deleted = false; + if (member_id) { + assert(member_id.primary().first); + std::pair member; + std::tie(exists, deleted) = tryGetMemberAt(member_id.primary(), member); + if (exists) { + assert(!deleted); + return XValue(member_id.primary().first.getIndex(), member.first, member.second); + } + // the primary slot was not occupied, try with the secondary + bool secondary_deleted = false; + std::tie(exists, secondary_deleted) = tryGetMemberAt(member_id.secondary(), member); + if (exists) { + assert(!secondary_deleted); + return XValue(member_id.secondary().first.getIndex(), member.first, member.second); + } + deleted |= secondary_deleted; + } + + if (!deleted && is_init_var) { + // unless explicitly deleted, + // report as None even if the field_id has not been assigned yet + return XValue(member_id.primary().first.getIndex(), StorageClass::NONE, Value()); + } + + return std::nullopt; + } + + template + typename ObjectImplBase::ObjectSharedPtr ObjectImplBase::tryGet(const char *field_name) const + { + std::pair member; + bool is_init_var = false; + auto field_id = tryGetMember(field_name, member, is_init_var); + // NOTE: init vars are always reported as None if not explicitly set nor explicitly deleted + if (field_id || (is_init_var && member.first != StorageClass::DELETED)) { + auto fixture = this->getFixture(); + // prevent accessing a deleted or undefined member + assert(member.first != StorageClass::DELETED && member.first != StorageClass::UNDEFINED); + // NOTE: offset is required for lo-fi members + return unloadMember( + fixture, member.first, member.second, field_id.maybeOffset(), this->getMemberFlags() + ); + } + + return nullptr; + } + + template + typename ObjectImplBase::ObjectSharedPtr ObjectImplBase::tryGetAs( + const char *field_name, TypeObjectPtr lang_type) const + { + std::pair member; + bool is_init_var = false; + auto field_id = tryGetMember(field_name, member, is_init_var); + if (field_id || (is_init_var && member.first != StorageClass::DELETED)) { + // prevent accessing a deleted member + assert(member.first != StorageClass::DELETED && member.first != StorageClass::UNDEFINED); + auto fixture = this->getFixture(); + if (member.first == StorageClass::OBJECT_REF) { + auto &class_factory = getClassFactory(*fixture); + return PyToolkit::unloadObject(fixture, member.second.asAddress(), class_factory, lang_type); + } + + // NOTE: offset is required for lo-fi members + return unloadMember( + fixture, member.first, member.second, field_id.getOffset(), this->getMemberFlags() + ); + } + + return nullptr; + } + + template + typename ObjectImplBase::ObjectSharedPtr ObjectImplBase::get(const char *field_name) const + { + auto obj = tryGet(field_name); + if (!obj) { + if (this->isDropped()) { + THROWF(db0::InputException) << "Object is no longer accessible"; + } + THROWF(db0::InputException) << "Attribute not found: " << field_name; + } + return obj; + } + + template + bool ObjectImplBase::slotExists(Value value, unsigned int fidelity, unsigned int at) const + { + assert(fidelity != 0 && "Operation only available for lo-fi values"); + // lo-fi value + assert(fidelity == 2); + return lofi_store<2>::fromValue(value).isSet(at); + } + + template + std::pair ObjectImplBase::hasValueAt(Value value, unsigned int fidelity, unsigned int at) const + { + assert(fidelity != 0 && "Operation only available for lo-fi values"); + // lo-fi value + assert(fidelity == 2); + if (lofi_store<2>::fromValue(value).isSet(at)) { + // might be deleted + bool deleted = (lofi_store<2>::fromValue(value).get(at) == Value::DELETED); + return { !deleted, deleted }; + } else { + // NOTE: unset value is assumed as empty / undefined + return { false, false }; + } + } + + template + bool ObjectImplBase::tryFindMemberAt(std::pair field_info, + std::pair &result, std::pair &find_result) const + { + if (!field_info.first) { + find_result = { false, false }; + return true; + } + + auto loc = field_info.first.getIndexAndOffset(); + if (!this->hasInstance()) { + // try retrieving from initializer + auto initializer_ptr = InitManager::instance.findInitializer(*this); + if (!initializer_ptr) { + find_result = { false, false }; + return true; + } + find_result = { initializer_ptr->tryGetAt(loc, result), false }; + return true; + } + + // retrieve from positionally encoded values + if ((*this)->pos_vt().find(loc.first, result)) { + // NOTE: removed field slots might be marked as DELETED + if (result.first == StorageClass::DELETED) { + // report as deleted + find_result = { false, true }; + return true; + } + + if (field_info.second == 0) { + find_result = { result.first != StorageClass::UNDEFINED, false }; + } else { + find_result = hasValueAt(result.second, field_info.second, loc.second); + } + return true; + } + + if ((*this)->index_vt().find(loc.first, result)) { + if (result.first == StorageClass::DELETED) { + // report as deleted + find_result = { false, true }; + return true; + } + + if (field_info.second == 0) { + find_result = { result.first != StorageClass::UNDEFINED, false }; + } else { + find_result = hasValueAt(result.second, field_info.second, loc.second); + } + return true; + } + + return false; + } + + template + std::pair ObjectImplBase::tryGetMemberAt(std::pair field_info, + std::pair &result) const + { + std::pair find_result; + if (static_cast(this)->tryFindMemberAt(field_info, result, find_result)) { + return find_result; + } + // Does not exist, not explicitly removed + return { false, false }; + } + + template + void ObjectImplBase::setType(std::shared_ptr type) + { + assert(!this->m_type); + this->m_type = type; + assert(hasValidClassRef()); + } + + template + void ObjectImplBase::setTypeWithHint(std::shared_ptr type_hint) + { + assert(!this->m_type); + assert(type_hint); + assert(this->hasInstance()); + if (type_hint->getClassRef() == (*this)->getClassRef()) { + this->m_type = type_hint; + } else { + this->m_type = unloadType(); + } + } + + template + void ObjectImplBase::dropTags(Class &type) const + { + // only drop if any type tags are assigned + if ((*this)->m_header.m_ref_counter.getFirst() > 0) { + auto fixture = this->getFixture(); + assert(fixture); + auto &tag_index = fixture->template get(); + const Class *type_ptr = &type; + auto unique_address = this->getUniqueAddress(); + while (type_ptr) { + // remove auto-assigned type (or its base) tag + tag_index.removeTypeTag(unique_address, type_ptr->getAddress()); + // NOTE: no need to decRef since object is being destroyed + type_ptr = type_ptr->getBaseClassPtr(); + } + } + } + + template + void ObjectImplBase::dropMembers(Class &class_ref) const + { + auto fixture = this->getFixture(); + assert(fixture); + // call the actual implementation + static_cast(this)->dropMembers(fixture, class_ref); + } + + template + void ObjectImplBase::dropMembers(db0::swine_ptr &fixture, Class &class_ref) const + { + // drop pos-vt members first + { + auto &types = (*this)->pos_vt().types(); + auto &values = (*this)->pos_vt().values(); + auto value = values.begin(); + unsigned int index = types.offset(); + for (auto type = types.begin(); type != types.end(); ++type, ++value, ++index) { + if (*type == StorageClass::DELETED || *type == StorageClass::UNDEFINED) { + // skip undefined or deleted members + continue; + } + unrefMember(fixture, *type, *value); + class_ref.removeFromSchema(index, *type, *value); + } + } + // drop index-vt members next + { + auto &xvalues = (*this)->index_vt().xvalues(); + for (auto &xvalue: xvalues) { + if (xvalue.m_type == StorageClass::DELETED || xvalue.m_type == StorageClass::UNDEFINED) { + // skip undefined or deleted members + continue; + } + unrefMember(fixture, xvalue); + class_ref.removeFromSchema(xvalue); + } + } + } + + template + void ObjectImplBase::destroy() const + { + if (this->hasInstance()) { + // associated class type (may require unloading) + auto type = this->m_type; + if (!type) { + // retrieve type from the initializer + type = std::const_pointer_cast(unloadType()); + } + + dropTags(*type); + dropMembers(*type); + // dereference associated class + type->decRef(false); + } + super_t::destroy(); + } + + template + FieldLayout ObjectImplBase::getFieldLayout() const + { + FieldLayout layout; + // call the actual implementation + static_cast(this)->getFieldLayoutImpl(layout); + return layout; + } + + template + void ObjectImplBase::getFieldLayoutImpl(FieldLayout &layout) const + { + // collect pos-vt information + for (auto type: (*this)->pos_vt().types()) { + layout.m_pos_vt_fields.push_back(type); + } + + // collect index-vt information + for (auto &xvalue: (*this)->index_vt().xvalues()) { + layout.m_index_vt_fields.emplace_back(xvalue.getIndex(), xvalue.m_type); + } + } + + template + void ObjectImplBase::getMembersFrom(const Class &this_type, unsigned int index, StorageClass storage_class, + Value value, std::unordered_set &result) const + { + if (storage_class == StorageClass::DELETED || storage_class == StorageClass::UNDEFINED) { + // skip undefined or deleted members + return; + } + + if (storage_class == StorageClass::PACK_2) { + auto it = lofi_store<2>::fromValue(value).begin(), end = lofi_store<2>::fromValue(value).end(); + for (; it != end; ++it) { + result.insert(this_type.getMember({ index, it.getOffset() }).m_name); + } + } else { + result.insert(this_type.getMember(FieldID::fromIndex(index)).m_name); + } + } + + template + std::unordered_set ObjectImplBase::getMembers() const + { + std::unordered_set result; + // call the actual implementation + static_cast(this)->getMembersImpl(result); + return result; + } + + template + void ObjectImplBase::getMembersImpl(std::unordered_set &result) const + { + // Visit pos-vt members first + auto &obj_type = this->getType(); + { + auto &types = (*this)->pos_vt().types(); + auto &values = (*this)->pos_vt().values(); + unsigned int index = types.offset(); + auto size = types.size(); + for (unsigned int pos = 0;pos < size; ++index, ++pos) { + getMembersFrom(obj_type, index, types[pos], values[pos], result); + } + } + + // Visit index-vt members next + { + auto &xvalues = (*this)->index_vt().xvalues(); + for (auto &xvalue: xvalues) { + auto index = xvalue.getIndex(); + getMembersFrom(obj_type, index, xvalue.m_type, xvalue.m_value, result); + } + } + } + + template + void ObjectImplBase::forAll(std::function f) const + { + // call the actual implementation + static_cast(this)->forAllImpl(f); + } + + template + bool ObjectImplBase::forAllImpl(std::function f) const + { + // Visit pos-vt members first + auto &obj_type = this->getType(); + { + auto &types = (*this)->pos_vt().types(); + auto &values = (*this)->pos_vt().values(); + auto value = values.begin(); + unsigned int index = types.offset(); + for (auto type = types.begin(); type != types.end(); ++type, ++value, ++index) { + if (*type == StorageClass::DELETED || *type == StorageClass::UNDEFINED) { + // skip deleted or undefined members + continue; + } + if (*type == StorageClass::PACK_2) { + // iterate individual lo-fi members + if (!forAll({index, *type, *value}, f)) { + return false; + } + } else { + if (!f(obj_type.getMember(FieldID::fromIndex(index)).m_name, { index, *type, *value }, 0)) { + return false; + } + } + } + } + + // Visit index-vt members next + { + auto &xvalues = (*this)->index_vt().xvalues(); + for (auto &xvalue: xvalues) { + if (xvalue.m_type == StorageClass::DELETED || xvalue.m_type == StorageClass::UNDEFINED) { + // skip deleted or undefined members + continue; + } + if (xvalue.m_type == StorageClass::PACK_2) { + // iterate individual lo-fi members + if (!forAll(xvalue, f)) { + return false; + } + } else { + // regular member + if (!f(obj_type.getMember(FieldID::fromIndex(xvalue.getIndex())).m_name, xvalue, 0)) { + return false; + } + } + } + } + + // Continue with kv-index members if any + return true; + } + + template + void ObjectImplBase::forAll(std::function f) const + { + auto fixture = this->getFixture(); + forAll([&](const std::string &name, const XValue &xvalue, unsigned int offset) -> bool { + // all references convert to UUID + auto py_member = unloadMember( + fixture, xvalue.m_type, xvalue.m_value, offset, this->getMemberFlags() + ); + return f(name, py_member); + }); + } + + template + bool ObjectImplBase::forAll(XValue xvalue, + std::function f) const + { + assert(xvalue.m_type == StorageClass::PACK_2); + unsigned int index = xvalue.getIndex(); + auto _value = xvalue.m_value; + auto it = lofi_store<2>::fromValue(_value).begin(), end = lofi_store<2>::fromValue(_value).end(); + auto &obj_type = this->getType(); + for (; it != end; ++it) { + if (!f(obj_type.getMember(FieldID::fromIndex(index, it.getOffset())).m_name, + xvalue, it.getOffset())) + { + return false; + } + } + return true; + } + + template + bool ObjectImplBase::tryEqualToImpl(const ObjectImplBase &other, bool &result) const + { + if (!this->hasInstance() || !other.hasInstance()) { + THROWF(db0::InputException) << "Object not initialized"; + } + + if (this->isDefunct() || other.isDefunct()) { + // defunct objects should not be compared + assert(!this->isDefunct()); + THROWF(db0::InputException) << "Object does not exist"; + } + + if ((*this)->getClassRef() != other->getClassRef()) { + // different types + result = false; + return true; + } + + if (this->getFixture()->getUUID() == other.getFixture()->getUUID() + && this->getUniqueAddress() == other.getUniqueAddress()) + { + // comparing 2 versions of the same object (fastest) + if (!((*this)->pos_vt() == other->pos_vt())) { + result = false; + return true; + } + if (!((*this)->index_vt() == other->index_vt())) { + result = false; + return true; + } + } + // unable to determine + return false; + } + + template + bool ObjectImplBase::equalTo(const ObjectImplBase &other) const + { + bool result; + if (static_cast(this)->tryEqualToImpl(other, result)) { + return result; + } + + // field-wise compare otherwise (slower) + result = true; + this->forAll([&](const std::string &name, const XValue &xvalue, unsigned int offset) -> bool { + auto maybe_other_value = other.tryGetX(name.c_str()); + if (!maybe_other_value) { + result = false; + return false; + } + + if (!xvalue.equalTo(*maybe_other_value, offset)) { + result = false; + return false; + } + return true; + }); + return result; + } + + template + void ObjectImplBase::moveTo(db0::swine_ptr &) { + throw std::runtime_error("Not implemented"); + } + + template + void ObjectImplBase::detach() const + { + this->m_type->detach(); + // invalidate since detach is not supported by the MorphingBIndex + this->m_kv_index = nullptr; + super_t::detach(); + } + + template + void ObjectImplBase::commit() const + { + this->m_type->commit(); + if (m_kv_index) { + m_kv_index->commit(); + } + super_t::commit(); + // reset the silent-mutation flag + this->m_touched = false; + } + + template + void ObjectImplBase::unrefMember(db0::swine_ptr &fixture, StorageClass type, Value value) const { + db0::object_model::unrefMember(fixture, type, value); + } + + template + void ObjectImplBase::unrefMember(db0::swine_ptr &fixture, XValue value) const { + db0::object_model::unrefMember(fixture, value.m_type, value.m_value); + } + + template + std::shared_ptr ObjectImplBase::unloadType() const + { + auto fixture = this->getFixture(); + return getClassFactory(*fixture).getTypeByClassRef((*this)->getClassRef()).m_class; + } + + template + bool ObjectImplBase::hasValidClassRef() const + { + if (this->hasInstance() && this->m_type) { + return (*this)->getClassRef() == this->m_type->getClassRef(); + } + return true; + } + + template + std::shared_ptr ObjectImplBase::getTypeWithHint(const Fixture &fixture, std::uint32_t class_ref, + std::shared_ptr type_hint) + { + assert(type_hint); + if (type_hint->getClassRef() == class_ref) { + return type_hint; + } + return getClassFactory(fixture).getTypeByClassRef(class_ref).m_class; + } + + template + bool ObjectImplBase::hasRefs() const + { + assert(this->hasInstance()); + return (*this)->hasRefs(); + } + + template class ObjectImplBase; + template class ObjectImplBase; + +} \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectImplBase.hpp b/src/dbzero/object_model/object/ObjectImplBase.hpp new file mode 100644 index 00000000..6dcf5ce6 --- /dev/null +++ b/src/dbzero/object_model/object/ObjectImplBase.hpp @@ -0,0 +1,216 @@ +#pragma once + +#include "ObjectAnyBase.hpp" +#include +#include +#include +#include +#include "o_object.hpp" +#include "o_immutable_object.hpp" + +namespace db0 + +{ + + class Fixture; + +} + +namespace db0::object_model + +{ + + class Class; + class Object; + class ObjectImmutableImpl; + using Fixture = db0::Fixture; + + struct FieldLayout + { + std::vector m_pos_vt_fields; + std::vector > m_index_vt_fields; + std::vector > m_kv_index_fields; + }; + + template + class ObjectImplBase: public ObjectAnyBase + { + public: + using super_t = ObjectAnyBase; + using LangToolkit = LangConfig::LangToolkit; + using ObjectPtr = typename LangToolkit::ObjectPtr; + using TypeObjectPtr = typename LangToolkit::TypeObjectPtr; + using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; + using TypeManager = typename LangToolkit::TypeManager; + using ObjectStem = ObjectVType; + using TypeInitializer = ObjectInitializer::TypeInitializer; + + // Construct as null / dropped object + ObjectImplBase(UniqueAddress, unsigned int ext_refs); + ObjectImplBase(const ObjectImplBase &) = delete; + ObjectImplBase(ObjectImplBase &&) = delete; + + /** + * Construct new Object (uninitialized, without corresponding dbzero instance yet) + */ + ObjectImplBase(std::shared_ptr); + ObjectImplBase(TypeInitializer &&); + + // Unload from address with a known type (possibly a base type) + // NOTE: unload works faster if type_hint is the exact object's type + struct with_type_hint {}; + ObjectImplBase(db0::swine_ptr &, Address, std::shared_ptr type_hint, + with_type_hint, AccessFlags = {}); + + // Unload from stem with a known type (possibly a base type) + // NOTE: unload works faster if type_hint is the exact object's type + ObjectImplBase(db0::swine_ptr &, ObjectStem &&, std::shared_ptr type_hint, + with_type_hint); + + ObjectImplBase(db0::swine_ptr &, Address, AccessFlags = {}); + ObjectImplBase(db0::swine_ptr &, std::shared_ptr, std::pair ref_counts, const PosVT::Data &, unsigned int pos_vt_offset); + ObjectImplBase(db0::swine_ptr &, ObjectStem &&, std::shared_ptr); + + ~ObjectImplBase(); + + // post-init invoked by memo type directly after __init__ + void postInit(FixtureLock &); + + // Destroys an existing instance and constructs a "null" placeholder + void dropInstance(FixtureLock &); + + // Unload the object stem, to retrieve its type + static ObjectStem tryUnloadStem(db0::swine_ptr &, Address, + std::uint16_t instance_id = 0, AccessFlags = {}); + static ObjectStem unloadStem(db0::swine_ptr &, Address, + std::uint16_t instance_id = 0, AccessFlags = {}); + + // Called to finalize adding members + void endInit(); + + // Assign field of an uninitialized instance (assumed as a non-mutating operation) + // NOTE: if lang_value is nullptr then the member is removed + void setPreInit(const char *field_name, ObjectPtr lang_value) const; + void removePreInit(const char *field_name) const; + + ObjectSharedPtr tryGet(const char *field_name) const; + ObjectSharedPtr tryGetAs(const char *field_name, TypeObjectPtr) const; + ObjectSharedPtr get(const char *field_name) const; + + // Get description of the field layout + FieldLayout getFieldLayout() const; + + void destroy() const; + + // execute the function for all members (until false is returned from the input lambda) + void forAll(std::function) const; + void forAll(std::function) const; + + // get dbzero member / member names assigned to this object + std::unordered_set getMembers() const; + + // Binary (shallow) compare 2 objects or 2 versions of the same memo object (e.g. from different snapshots) + // NOTE: ref-counts are not compared (only user-assigned members) + // @return true if objects are identical + bool equalTo(const ObjectImplBase &) const; + + /** + * Move unreferenced object to a different prefix without changing the instance + * this operations is required for auto-hardening + */ + void moveTo(db0::swine_ptr &); + + void detach() const; + void commit() const; + + // FieldID, is_init_var, fidelity + std::pair findField(const char *name) const; + + // NOTE: hasRefs is NOT available in ObjectAnyBase bacause + // of the use of num_type_tags property + bool hasRefs() const; + + protected: + // local kv-index instance cache (created at first use) + mutable std::unique_ptr m_kv_index; + + void setType(std::shared_ptr); + // adjusts to actual type if the type hint is a base class + void setTypeWithHint(std::shared_ptr type_hint); + // @return exists / deleted + std::pair hasValueAt(Value, unsigned int fidelity, unsigned int at) const; + // similar to hasValueAt but assume deleted slot as present + bool slotExists(Value value, unsigned int fidelity, unsigned int at) const; + + void getFieldLayoutImpl(FieldLayout &) const; + void getMembersImpl(std::unordered_set &) const; + bool tryEqualToImpl(const ObjectImplBase &, bool &result) const; + + // Try retrieving member either from values (initialized) or from the initialization buffer (not initialized yet) + // @return member exists, member deleted flags + bool tryFindMemberAt(std::pair, std::pair &, + std::pair &find_result) const; + std::pair tryGetMemberAt(std::pair, + std::pair &) const; + FieldID tryGetMember(const char *field_name, std::pair &, bool &is_init_var) const; + + // Try resolving field ID of an existing (or deleted) member and also its storage location + // @param pos the member's position in the containing collection + // @return FieldID + containing collection (e.g. pos_vt()) + bool tryFindMemberSlot(const std::pair &field_info, unsigned int &pos, + std::pair &result) const; + + std::pair tryGetMemberSlot(const MemberID &, unsigned int &pos) const; + + // Try locating a field ID associated slot + std::pair tryGetLoc(FieldID) const; + + inline ObjectInitializer *tryGetInitializer() const { + return this->m_type ? static_cast(nullptr) : &InitManager::instance.getInitializer(*this); + } + + void dropMembers(db0::swine_ptr &, Class &) const; + void dropMembers(Class &) const; + void dropTags(Class &) const; + + void unrefMember(db0::swine_ptr &, StorageClass, Value) const; + void unrefMember(db0::swine_ptr &, XValue) const; + + using TypeId = db0::bindings::TypeId; + std::pair recognizeType(Fixture &, ObjectPtr lang_value) const; + + // Unload associated type + std::shared_ptr unloadType() const; + + // Retrieve a type by class-ref with a possible match (type_hint) + static std::shared_ptr getTypeWithHint(const Fixture &, std::uint32_t class_ref, + std::shared_ptr type_hint); + + bool hasValidClassRef() const; + + // try retrieving member as XValue + std::optional tryGetX(const char *field_name) const; + + // Unreference value + // NOTE: storage_class to be assigned can either be DELETED or UNDEFINED + void unrefPosVT(FixtureLock &, FieldID, unsigned int pos, StorageClass, unsigned int fidelity); + void unrefIndexVT(FixtureLock &, FieldID, unsigned int index_vt_pos, StorageClass, unsigned int fidelity); + + void unrefWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, StorageClass, + unsigned int fidelity); + bool tryUnrefWithLoc(FixtureLock &, FieldID, const void *, unsigned int pos, StorageClass, + unsigned int fidelity); + + bool forAllImpl(std::function) const; + // lo-fi member specialized implementation + bool forAll(XValue, std::function) const; + + void getMembersFrom(const Class &this_type, unsigned int index, StorageClass, Value, + std::unordered_set &) const; + }; + + extern template class ObjectImplBase; + extern template class ObjectImplBase; + +} diff --git a/src/dbzero/object_model/object/ObjectInitializer.cpp b/src/dbzero/object_model/object/ObjectInitializer.cpp index 740722c2..fd61feab 100644 --- a/src/dbzero/object_model/object/ObjectInitializer.cpp +++ b/src/dbzero/object_model/object/ObjectInitializer.cpp @@ -5,48 +5,7 @@ namespace db0::object_model { - - ObjectInitializer::ObjectInitializer(ObjectInitializerManager &manager, std::uint32_t loc, - Object &object, std::shared_ptr db0_class) - : m_manager(manager) - , m_loc(loc) - , m_closed(false) - , m_object_ptr(&object) - , m_class(db0_class) - // NOTE: limit the dim-2 size to improve performance - , m_has_value(lofi_store<2>::size()) - { - } - - ObjectInitializer::ObjectInitializer(ObjectInitializerManager &manager, std::uint32_t loc, Object &object, - TypeInitializer &&type_initializer) - : m_manager(manager) - , m_loc(loc) - , m_closed(false) - , m_object_ptr(&object) - // NOTE: limit the dim-2 size to improve performance - , m_has_value(lofi_store<2>::size()) - , m_type_initializer(std::move(type_initializer)) - { - } - - void ObjectInitializer::init(Object &object, std::shared_ptr db0_class) - { - assert(m_closed); - m_closed = false; - m_object_ptr = &object; - m_class = db0_class; - } - - void ObjectInitializer::init(Object &object, TypeInitializer &&type_initializer) - { - assert(m_closed); - assert(!m_class); - m_closed = false; - m_object_ptr = &object; - m_type_initializer = std::move(type_initializer); - } - + void ObjectInitializer::close() { m_manager.closeAt(m_loc); } @@ -223,43 +182,6 @@ namespace db0::object_model return true; } - std::shared_ptr ObjectInitializerManager::tryCloseInitializer(const Object &object) - { - for (auto i = 0u; i < m_active_count; ++i) { - if (m_initializers[i]->operator==(object)) { - auto result = m_initializers[i]->getClassPtr(); - closeAt(i); - return result; - } - } - return nullptr; - } - - std::shared_ptr ObjectInitializerManager::closeInitializer(const Object &object) - { - auto result = tryCloseInitializer(object); - if (result) { - return result; - } - THROWF(db0::InternalException) << "Initializer not found" << THROWF_END; - } - - ObjectInitializer *ObjectInitializerManager::findInitializer(const Object &object) const - { - for (auto i = 0u; i < m_active_count; ++i) { - if (m_initializers[i]->operator==(object)) { - // move to front to allow faster lookup the next time - if (i != 0) { - std::swap(m_initializers[i], m_initializers[0]); - *(m_initializers[i]) = i; - *(m_initializers[0]) = 0; - } - return m_initializers[0].get(); - } - } - return nullptr; - } - void ObjectInitializerManager::closeAt(std::uint32_t loc) { auto result = m_initializers[loc]->getClassPtr(); @@ -270,5 +192,5 @@ namespace db0::object_model *(m_initializers[m_active_count - 1]) = m_active_count - 1; --m_active_count; } - + } \ No newline at end of file diff --git a/src/dbzero/object_model/object/ObjectInitializer.hpp b/src/dbzero/object_model/object/ObjectInitializer.hpp index ac8b8c61..14600810 100644 --- a/src/dbzero/object_model/object/ObjectInitializer.hpp +++ b/src/dbzero/object_model/object/ObjectInitializer.hpp @@ -13,6 +13,7 @@ #include #include "ValueTable.hpp" #include "XValuesVector.hpp" +#include "lofi_store.hpp" namespace db0 @@ -28,9 +29,55 @@ namespace db0::object_model class Class; class Object; - class ObjectInitializerManager; + class ObjectInitializer; using Fixture = db0::Fixture; + /** + * The purpose of this class is to hold Object initializers during the construction process. + * We could simply keep 'initializer' as an Object member, but since this is a short-lived object, it would be a waste of space. + * Also InitializerManager helps us reuse Initializer instances, saving on memory allocations. + */ + class ObjectInitializerManager + { + public: + ObjectInitializerManager() = default; + + template + void addInitializer(T &object, Args&& ...args); + + // Close the initializer and retrieve object's class + template + std::shared_ptr closeInitializer(const T &object); + + // Close the initializer if it exists + template + std::shared_ptr tryCloseInitializer(const T &object); + + template + ObjectInitializer &getInitializer(const T &object) const + { + auto result = findInitializer(object); + if (result) { + return *result; + } + THROWF(InternalException) << "Initializer not found" << THROWF_END; + } + + template + ObjectInitializer *findInitializer(const T &object) const; + + protected: + friend class ObjectInitializer; + void closeAt(std::uint32_t loc); + + private: + mutable std::vector > m_initializers; + // number of active object initializers + std::size_t m_active_count = 0; + // number of non-null initializer instances + std::size_t m_total_count = 0; + }; + /** * Class to store status of the instance / member intialization process (corresponds to __init__) */ @@ -41,12 +88,50 @@ namespace db0::object_model using TypeInitializer = std::function(db0::swine_ptr &)>; // loc - position in the initializer manager's array - ObjectInitializer(ObjectInitializerManager &, std::uint32_t loc, Object &, std::shared_ptr); + template + ObjectInitializer(ObjectInitializerManager &manager, std::uint32_t loc, T &object, std::shared_ptr type) + : m_manager(manager) + , m_loc(loc) + , m_closed(false) + , m_object_ptr(&object) + , m_class(type) + // NOTE: limit the dim-2 size to improve performance + , m_has_value(lofi_store<2>::size()) + { + } + // alternative constructor for lazy type initialization - ObjectInitializer(ObjectInitializerManager &, std::uint32_t loc, Object &, TypeInitializer &&); + template + ObjectInitializer(ObjectInitializerManager &manager, std::uint32_t loc, T &object, TypeInitializer &&type_initializer) + : m_manager(manager) + , m_loc(loc) + , m_closed(false) + , m_object_ptr(&object) + // NOTE: limit the dim-2 size to improve performance + , m_has_value(lofi_store<2>::size()) + , m_type_initializer(std::move(type_initializer)) + { + } - void init(Object &object, std::shared_ptr); - void init(Object &object, TypeInitializer &&); + template + void init(T &object, std::shared_ptr type) + { + assert(m_closed); + m_closed = false; + m_object_ptr = &object; + m_class = type; + } + + template + void init(T &object, TypeInitializer &&type_initializer) + { + assert(m_closed); + assert(!m_class); + m_closed = false; + m_object_ptr = &object; + m_type_initializer = std::move(type_initializer); + + } // @param mask required for lo-fi types (pack-2) void set(std::pair loc, StorageClass storage_class, Value value, @@ -76,7 +161,8 @@ namespace db0::object_model // NOTE always the whole value is retrieved (no mask support) bool tryGetAt(std::pair loc, std::pair &) const; - bool operator==(const Object &other) { + template + bool operator==(const T &other) { return m_object_ptr == &other; } @@ -101,14 +187,15 @@ namespace db0::object_model void reset(); void operator=(std::uint32_t new_loc); - + private: // maximum size of the position-encoded value-block (pos-VT) static constexpr std::size_t POSVT_MAX_SIZE = 128; ObjectInitializerManager &m_manager; std::uint32_t m_loc = std::numeric_limits::max(); bool m_closed = true; - Object *m_object_ptr = nullptr; + // pointer to an implementation-specific type + void *m_object_ptr = nullptr; mutable std::shared_ptr m_class; // indexed initialization values mutable XValuesVector m_values; @@ -119,53 +206,8 @@ namespace db0::object_model mutable TypeInitializer m_type_initializer; }; - /** - * The purpose of this class is to hold Object initializers during the construction process. - * We could simply keep 'initializer' as an Object member, but since this is a short-lived object, it would be a waste of space. - * Also InitializerManager helps us reuse Initializer instances, saving on memory allocations. - */ - class ObjectInitializerManager - { - public: - ObjectInitializerManager() = default; - - template void addInitializer(Object &object, Args&& ...args); - - /** - * Close the initializer and retrieve object's class - */ - std::shared_ptr closeInitializer(const Object &object); - - /** - * Close the initializer if it exists - */ - std::shared_ptr tryCloseInitializer(const Object &object); - - ObjectInitializer &getInitializer(const Object &object) const - { - auto result = findInitializer(object); - if (result) { - return *result; - } - THROWF(InternalException) << "Initializer not found" << THROWF_END; - } - - ObjectInitializer *findInitializer(const Object &object) const; - - protected: - friend class ObjectInitializer; - void closeAt(std::uint32_t loc); - - private: - mutable std::vector > m_initializers; - // number of active object initializers - std::size_t m_active_count = 0; - // number of non-null initializer instances - std::size_t m_total_count = 0; - }; - - template - void ObjectInitializerManager::addInitializer(Object &object, Args&& ...args) + template + void ObjectInitializerManager::addInitializer(T &object, Args&& ...args) { if (m_active_count < m_total_count) { auto loc = m_active_count++; @@ -185,4 +227,44 @@ namespace db0::object_model } } + template + std::shared_ptr ObjectInitializerManager::tryCloseInitializer(const T &object) + { + for (auto i = 0u; i < m_active_count; ++i) { + if (m_initializers[i]->operator==(object)) { + auto result = m_initializers[i]->getClassPtr(); + closeAt(i); + return result; + } + } + return nullptr; + } + + template + std::shared_ptr ObjectInitializerManager::closeInitializer(const T &object) + { + auto result = tryCloseInitializer(object); + if (result) { + return result; + } + THROWF(db0::InternalException) << "Initializer not found" << THROWF_END; + } + + template + ObjectInitializer *ObjectInitializerManager::findInitializer(const T &object) const + { + for (auto i = 0u; i < m_active_count; ++i) { + if (m_initializers[i]->operator==(object)) { + // move to front to allow faster lookup the next time + if (i != 0) { + std::swap(m_initializers[i], m_initializers[0]); + *(m_initializers[i]) = i; + *(m_initializers[0]) = 0; + } + return m_initializers[0].get(); + } + } + return nullptr; + } + } \ No newline at end of file diff --git a/src/dbzero/object_model/object/Options.cpp b/src/dbzero/object_model/object/Options.cpp index c977faee..fade956e 100644 --- a/src/dbzero/object_model/object/Options.cpp +++ b/src/dbzero/object_model/object/Options.cpp @@ -1,3 +1,3 @@ #include "Options.hpp" -DEFINE_ENUM_VALUES(db0::object_model::MemoOptions, "NO_DEFAULT_TAGS", "NO_CACHE") +DEFINE_ENUM_VALUES(db0::object_model::MemoOptions, "NO_DEFAULT_TAGS", "NO_CACHE", "IMMUTABLE") diff --git a/src/dbzero/object_model/object/Options.hpp b/src/dbzero/object_model/object/Options.hpp index cc0ae8ee..6e95dbe0 100644 --- a/src/dbzero/object_model/object/Options.hpp +++ b/src/dbzero/object_model/object/Options.hpp @@ -12,11 +12,12 @@ namespace db0::object_model // instances of this type opted out of auto-assigned type tags NO_DEFAULT_TAGS = 0x0001, // instances of this type opted out of caching - NO_CACHE = 0x0002 + NO_CACHE = 0x0002, + IMMUTABLE = 0x0004 }; using MemoFlags = db0::FlagSet; } -DECLARE_ENUM_VALUES(db0::object_model::MemoOptions, 2) \ No newline at end of file +DECLARE_ENUM_VALUES(db0::object_model::MemoOptions, 3) \ No newline at end of file diff --git a/src/dbzero/object_model/object/o_immutable_object.cpp b/src/dbzero/object_model/object/o_immutable_object.cpp new file mode 100644 index 00000000..e3e4b54f --- /dev/null +++ b/src/dbzero/object_model/object/o_immutable_object.cpp @@ -0,0 +1,74 @@ +#include "o_immutable_object.hpp" +#include +#include +#include +#include + +namespace db0::object_model + +{ + + o_immutable_object::o_immutable_object(std::uint32_t class_ref, + std::pair ref_counts, std::uint8_t num_type_tags, const PosVT::Data &pos_vt_data, + unsigned int pos_vt_offset, const XValue *index_vt_begin, const XValue *index_vt_end) + : m_header(ref_counts) + , m_num_type_tags(num_type_tags) + { + arrangeMembers() + (PosVT::type(), pos_vt_data, pos_vt_offset) + (packed_int32::type(), class_ref) + (IndexVT::type(), index_vt_begin, index_vt_end); + } + + std::size_t o_immutable_object::measure(std::uint32_t class_ref, + std::pair, std::uint8_t, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, + const XValue *index_vt_begin, const XValue *index_vt_end) + { + return super_t::measureMembers() + (PosVT::type(), pos_vt_data, pos_vt_offset) + (packed_int32::type(), class_ref) + (IndexVT::type(), index_vt_begin, index_vt_end); + } + + const PosVT &o_immutable_object::pos_vt() const { + return getDynFirst(PosVT::type()); + } + + PosVT &o_immutable_object::pos_vt() { + return getDynFirst(PosVT::type()); + } + + const packed_int32 &o_immutable_object::classRef() const { + return getDynAfter(pos_vt(), packed_int32::type()); + } + + std::uint32_t o_immutable_object::getClassRef() const { + return classRef().value(); + } + + const IndexVT &o_immutable_object::index_vt() const { + return getDynAfter(classRef(), IndexVT::type()); + } + + IndexVT &o_immutable_object::index_vt() { + return getDynAfter(classRef(), IndexVT::type()); + } + + void o_immutable_object::incRef(bool is_tag) { + m_header.incRef(is_tag); + } + + bool o_immutable_object::hasRefs() const + { + // NOTE: type tags are not counted as "proper" references + if (m_header.m_ref_counter.getFirst() > this->m_num_type_tags) { + return true; + } + return m_header.m_ref_counter.getSecond() > 0; + } + + bool o_immutable_object::hasAnyRefs() const { + return m_header.hasRefs(); + } + +} \ No newline at end of file diff --git a/src/dbzero/object_model/object/o_immutable_object.hpp b/src/dbzero/object_model/object/o_immutable_object.hpp new file mode 100644 index 00000000..3b09802b --- /dev/null +++ b/src/dbzero/object_model/object/o_immutable_object.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include "ValueTable.hpp" +#include + +namespace db0::object_model + +{ + +DB0_PACKED_BEGIN + class DB0_PACKED_ATTR o_immutable_object: public db0::o_base + { + protected: + using super_t = db0::o_base; + + public: + static constexpr unsigned char REALM_ID = 1; + // common object header + o_unique_header m_header; + // number of auto-assigned type tags + std::uint8_t m_num_type_tags = 0; + + PosVT &pos_vt(); + const PosVT &pos_vt() const; + + const packed_int32 &classRef() const; + std::uint32_t getClassRef() const; + + const IndexVT &index_vt() const; + + IndexVT &index_vt(); + + // ref_counts - the initial reference counts (tags / objects) inherited from the initializer + o_immutable_object(std::uint32_t class_ref, std::pair ref_counts, std::uint8_t num_type_tags, + const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin = nullptr, + const XValue *index_vt_end = nullptr); + + static std::size_t measure(std::uint32_t, std::pair, std::uint8_t num_type_tags, + const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin = nullptr, + const XValue *index_vt_end = nullptr); + + template static std::size_t safeSizeOf(BufT buf) + { + return super_t::sizeOfMembers(buf) + (PosVT::type()) + (packed_int32::type()) + (IndexVT::type()); + } + + void incRef(bool is_tag); + bool hasRefs() const; + bool hasAnyRefs() const; + }; +DB0_PACKED_END + +} \ No newline at end of file diff --git a/src/dbzero/object_model/object/o_object.cpp b/src/dbzero/object_model/object/o_object.cpp new file mode 100644 index 00000000..da425c6e --- /dev/null +++ b/src/dbzero/object_model/object/o_object.cpp @@ -0,0 +1,84 @@ +#include "o_object.hpp" +#include +#include +#include +#include +#include + +namespace db0::object_model + +{ + + o_object_base::o_object_base(std::pair ref_counts) + : m_header(ref_counts) + { + } + + void o_object_base::incRef(bool is_tag) { + m_header.incRef(is_tag); + } + + bool o_object_base::hasAnyRefs() const { + return m_header.hasRefs(); + } + + std::size_t o_object_base::measure() { + return super_t::measureMembers(); + } + + o_object::o_object(std::uint32_t class_ref, std::pair ref_counts, + std::uint8_t num_type_tags, const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, + const XValue *index_vt_begin, const XValue *index_vt_end) + : super_t(ref_counts) + , m_num_type_tags(num_type_tags) + { + arrangeMembers() + (PosVT::type(), pos_vt_data, pos_vt_offset) + (packed_int32::type(), class_ref) + (IndexVT::type(), index_vt_begin, index_vt_end); + } + + std::size_t o_object::measure(std::uint32_t class_ref, std::pair, std::uint8_t, + const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, + const XValue *index_vt_begin, const XValue *index_vt_end) + { + return super_t::measureBase() + (PosVT::type(), pos_vt_data, pos_vt_offset) + (packed_int32::type(), class_ref) + (IndexVT::type(), index_vt_begin, index_vt_end); + } + + const PosVT &o_object::pos_vt() const { + return getDynFirst(PosVT::type()); + } + + PosVT &o_object::pos_vt() { + return getDynFirst(PosVT::type()); + } + + const packed_int32 &o_object::classRef() const { + return getDynAfter(pos_vt(), packed_int32::type()); + } + + std::uint32_t o_object::getClassRef() const { + return classRef().value(); + } + + const IndexVT &o_object::index_vt() const { + return getDynAfter(classRef(), IndexVT::type()); + } + + IndexVT &o_object::index_vt() { + return getDynAfter(classRef(), IndexVT::type()); + } + + bool o_object::hasRefs() const + { + // NOTE: type tags are not counted as "proper" references + if (m_header.m_ref_counter.getFirst() > this->m_num_type_tags) { + return true; + } + return m_header.m_ref_counter.getSecond() > 0; + } + +} \ No newline at end of file diff --git a/src/dbzero/object_model/object/o_object.hpp b/src/dbzero/object_model/object/o_object.hpp new file mode 100644 index 00000000..28bafd10 --- /dev/null +++ b/src/dbzero/object_model/object/o_object.hpp @@ -0,0 +1,85 @@ +#pragma once + +#include +#include +#include "ValueTable.hpp" +#include "ObjectInitializer.hpp" +#include +#include "KV_Index.hpp" + +namespace db0::object_model + +{ + +DB0_PACKED_BEGIN + class DB0_PACKED_ATTR o_object_base: public db0::o_base + { + protected: + using super_t = db0::o_base; + + public: + static constexpr unsigned char REALM_ID = 1; + // common object header + o_unique_header m_header; + + // ref_counts - the initial reference counts (tags / objects) inherited from the initializer + o_object_base(std::pair ref_counts); + + static std::size_t measure(); + static std::size_t measure(std::pair); + + template static std::size_t safeSizeOf(BufT buf) { + return super_t::sizeOfMembers(buf); + } + + void incRef(bool is_tag); + bool hasAnyRefs() const; + }; +DB0_PACKED_END + +DB0_PACKED_BEGIN + class DB0_PACKED_ATTR o_object: public db0::o_ext + { + protected: + using super_t = db0::o_ext; + + public: + // optional address of the key-value store (to store extension fields) + KV_Address m_kv_address; + // kv-index type must be stored separately from the address + bindex::type m_kv_type; + // number of auto-assigned type tags + std::uint8_t m_num_type_tags = 0; + + PosVT &pos_vt(); + const PosVT &pos_vt() const; + + const packed_int32 &classRef() const; + std::uint32_t getClassRef() const; + + const IndexVT &index_vt() const; + + IndexVT &index_vt(); + + // ref_counts - the initial reference counts (tags / objects) inherited from the initializer + o_object(std::uint32_t class_ref, std::pair ref_counts, std::uint8_t num_type_tags, + const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin = nullptr, + const XValue *index_vt_end = nullptr); + + static std::size_t measure(std::uint32_t, std::pair, std::uint8_t num_type_tags, + const PosVT::Data &pos_vt_data, unsigned int pos_vt_offset, const XValue *index_vt_begin = nullptr, + const XValue *index_vt_end = nullptr); + + template static std::size_t safeSizeOf(BufT buf) + { + return super_t::sizeOfMembers(buf) + (PosVT::type()) + (packed_int32::type()) + (IndexVT::type()); + } + + bool hasRefs() const; + }; +DB0_PACKED_END + +} \ No newline at end of file diff --git a/src/dbzero/object_model/tags/ObjectTagManager.cpp b/src/dbzero/object_model/tags/ObjectTagManager.cpp index 9cb054a3..a1302ccf 100644 --- a/src/dbzero/object_model/tags/ObjectTagManager.cpp +++ b/src/dbzero/object_model/tags/ObjectTagManager.cpp @@ -44,18 +44,18 @@ namespace db0::object_model // construct as empty return new (at_ptr) ObjectTagManager(); } - return new (at_ptr) ObjectTagManager(memo_ptr, nargs); + return new (at_ptr) ObjectTagManager(memo_ptr, nargs); } ObjectTagManager::ObjectInfo::ObjectInfo(ObjectPtr memo_ptr) : m_lang_ptr(memo_ptr) - , m_object_ptr(&ObjectTagManager::LangToolkit::getTypeManager().extractObject(memo_ptr)) + , m_object_ptr(&ObjectTagManager::LangToolkit::getTypeManager().extractAnyObject(memo_ptr)) , m_tag_index_ptr(&m_object_ptr->getFixture()->get()) , m_type(m_object_ptr->getClassPtr()) , m_access_mode(m_object_ptr->getFixture()->getAccessType()) , m_has_tags(LangToolkit::hasTagRefs(memo_ptr)) { - } + } void ObjectTagManager::ObjectInfo::add(ObjectPtr const *args, Py_ssize_t nargs) { diff --git a/src/dbzero/object_model/tags/ObjectTagManager.hpp b/src/dbzero/object_model/tags/ObjectTagManager.hpp index c3bcc8d2..97c81252 100644 --- a/src/dbzero/object_model/tags/ObjectTagManager.hpp +++ b/src/dbzero/object_model/tags/ObjectTagManager.hpp @@ -1,15 +1,15 @@ #pragma once -#include #include #include #include +#include namespace db0::object_model { - - using Object = db0::object_model::Object; + + using ObjectAnyImpl = db0::object_model::ObjectAnyImpl; using Class = db0::object_model::Class; using RC_LimitedStringPool = db0::pools::RC_LimitedStringPool; @@ -20,7 +20,7 @@ namespace db0::object_model class ObjectTagManager { public: - using LangToolkit = typename Object::LangToolkit; + using LangToolkit = typename ObjectAnyImpl::LangToolkit; using ObjectPtr = typename LangToolkit::ObjectPtr; using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; @@ -43,7 +43,7 @@ namespace db0::object_model struct ObjectInfo { ObjectSharedPtr m_lang_ptr; - const Object *m_object_ptr = nullptr; + const ObjectAnyImpl *m_object_ptr = nullptr; TagIndex *m_tag_index_ptr = nullptr; std::shared_ptr m_type; AccessType m_access_mode; diff --git a/src/dbzero/object_model/tags/TagIndex.cpp b/src/dbzero/object_model/tags/TagIndex.cpp index 29a66c97..6d1f1d5e 100644 --- a/src/dbzero/object_model/tags/TagIndex.cpp +++ b/src/dbzero/object_model/tags/TagIndex.cpp @@ -164,23 +164,7 @@ namespace db0::object_model m_batch_op_types.empty() && "TagIndex::flush() or close() must be called before destruction"); } - - FT_BaseIndex::BatchOperationBuilder & - TagIndex::getBatchOperationShort(ObjectPtr memo_ptr, ActiveValueT &result, bool is_type) const - { - if (is_type) { - return getBatchOperation(memo_ptr, m_base_index_short, m_batch_op_types, result); - } else { - return getBatchOperation(memo_ptr, m_base_index_short, m_batch_op_short, result); - } - } - - db0::FT_BaseIndex::BatchOperationBuilder & - TagIndex::getBatchOperationLong(ObjectPtr memo_ptr, ActiveValueT &result) const - { - return getBatchOperation(memo_ptr, m_base_index_long, m_batch_op_long, result); - } - + void TagIndex::addTags(ObjectPtr memo_ptr, ObjectPtr const *args, std::size_t nargs) { using TypeId = db0::bindings::TypeId; @@ -242,6 +226,26 @@ namespace db0::object_model } } + FT_BaseIndex::BatchOperationBuilder & + TagIndex::getBatchOperationShort(ObjectPtr memo_ptr, ActiveValueT &result, bool is_type) const + { + if (is_type) { + return getBatchOperation( + memo_ptr, m_base_index_short, m_batch_op_types, result + ); + } else { + return getBatchOperation( + memo_ptr, m_base_index_short, m_batch_op_short, result + ); + } + } + + db0::FT_BaseIndex::BatchOperationBuilder & + TagIndex::getBatchOperationLong(ObjectPtr memo_ptr, ActiveValueT &result) const + { + return getBatchOperation(memo_ptr, m_base_index_long, m_batch_op_long, result); + } + void TagIndex::addTag(ObjectPtr memo_ptr, Address tag_addr, bool is_type) { addTag(memo_ptr, tag_addr.getOffset(), is_type); } @@ -252,7 +256,7 @@ namespace db0::object_model auto &batch_operation = getBatchOperationShort(memo_ptr, active_key, is_type); batch_operation->addTags(active_key, TagPtrSequence(&tag, &tag + 1)); m_mutation_log->onDirty(); - } + } void TagIndex::addTag(ObjectPtr memo_ptr, LongTagT tag) { @@ -268,7 +272,7 @@ namespace db0::object_model batch_operation->removeTag({ obj_addr, nullptr }, tag_addr.getOffset()); m_mutation_log->onDirty(); } - + void TagIndex::removeTags(ObjectPtr memo_ptr, ObjectPtr const *args, std::size_t nargs) { if (nargs == 0) { @@ -373,7 +377,7 @@ namespace db0::object_model // NOTE: some object might've been dropped in the meantime, need to be reverted from batch operations for (const auto &item: m_object_cache) { auto obj_ptr = item.second.get(); - auto &memo = type_manager.extractObject(obj_ptr); + auto &memo = type_manager.extractAnyObject(obj_ptr); if (memo.isDead()) { revert(obj_ptr); } @@ -386,7 +390,7 @@ namespace db0::object_model auto it = m_object_cache.find(obj_addr); assert(it != m_object_cache.end()); // NOTE: inc-ref as tag - type_manager.extractMutableObject(it->second.get()).incRef(true); + type_manager.extractMutableAnyObject(it->second.get()).incRef(true); }; // add_index_callback adds reference to tags (string pool tokens) @@ -401,9 +405,9 @@ namespace db0::object_model // object may not exist if tags are removed post-deletion auto obj_ptr = it->second.get(); if (it != m_object_cache.end()) { - auto &memo = type_manager.extractMutableObject(obj_ptr); // NOTE: we check for acutal language references (excluding LangCache + TagIndex) - if (memo.decRef(true) && !LangToolkit::hasAnyLangRefs(obj_ptr, 2)) { + if (LangToolkit::decRefMemo(true, obj_ptr) && !LangToolkit::hasAnyLangRefs(obj_ptr, 2)) { + auto &memo = type_manager.extractAnyObject(obj_ptr); // if object is pending deletion, remove all type tags as well // we might skip this operation and leave it to Object's dropTags function // but it will be more efficient to do it here @@ -449,7 +453,7 @@ namespace db0::object_model assert(m_active_pre_cache.empty()); for (const auto &item: m_object_cache) { auto obj_ptr = item.second.get(); - auto &memo = type_manager.extractObject(obj_ptr); + auto &memo = type_manager.extractAnyObject(obj_ptr); // NOTE: dropped instances should've already been reverted by now // NOTE: we check for acutal language references (excluding LangCache + TagIndex) if (!memo.isDropped() && !memo.hasAnyRefs() && !LangToolkit::hasAnyLangRefs(obj_ptr, 2)) { @@ -474,7 +478,7 @@ namespace db0::object_model void TagIndex::buildActiveValues() const { for (auto &item: m_active_cache) { - auto &memo = LangToolkit::getTypeManager().extractObject(item.first); + auto &memo = LangToolkit::getTypeManager().extractAnyObject(item.first); // NOTE: defunct objects have to be ignored since they don't have a valid address // NOTE: defunct objects, since no valid unique address is assigned will be auto-reverted on flush if (!memo.isDefunct()) { @@ -553,7 +557,7 @@ namespace db0::object_model // Memo instance is directly fed into the FT_FixedKeyIterator if (type_id == TypeId::MEMO_OBJECT) { - auto addr = LangToolkit::getTypeManager().extractObject(arg).getUniqueAddress(); + auto addr = LangToolkit::getTypeManager().extractAnyObject(arg).getUniqueAddress(); factory.add(std::make_unique >(&addr, &addr + 1)); return true; } @@ -797,11 +801,11 @@ namespace db0::object_model assert(LangToolkit::isString(py_arg)); return LangToolkit::addTagFromString(py_arg, m_string_pool, inc_ref); } - + std::optional TagIndex::tryAddShortTagFromMemo(ObjectPtr py_arg) const { - assert(LangToolkit::isMemoObject(py_arg)); - auto &py_obj = LangToolkit::getTypeManager().extractObject(py_arg); + assert(LangToolkit::isAnyMemoObject(py_arg)); + auto &py_obj = LangToolkit::getTypeManager().extractAnyObject(py_arg); if (py_obj.getFixtureUUID() != m_fixture_uuid) { // must be added as long tag return std::nullopt; @@ -1045,7 +1049,7 @@ namespace db0::object_model // Python Memo type if (LangToolkit::isType(args[args_offset])) { lang_type = type_manager.getTypeObject(args[args_offset++]); - if (LangToolkit::isMemoType(lang_type)) { + if (LangToolkit::isAnyMemoType(lang_type)) { // MemoBase type does not correspond to any find criteria // but we may use its corresponding lang type if (!type_manager.isMemoBase(lang_type)) { @@ -1096,8 +1100,8 @@ namespace db0::object_model LongTagT TagIndex::getLongTagFromMemo(ObjectPtr py_arg) const { - assert(LangToolkit::isMemoObject(py_arg)); - auto &py_obj = LangToolkit::getTypeManager().extractObject(py_arg); + assert(LangToolkit::isAnyMemoObject(py_arg)); + auto &py_obj = LangToolkit::getTypeManager().extractAnyObject(py_arg); return { py_obj.getFixtureUUID(), py_obj.getAddress().getOffset() }; } @@ -1107,7 +1111,7 @@ namespace db0::object_model void TagIndex::revert(ObjectPtr memo_ptr) const { - auto &memo = LangToolkit::getTypeManager().extractObject(memo_ptr); + auto &memo = LangToolkit::getTypeManager().extractAnyObject(memo_ptr); auto addr = memo.getUniqueAddress(); if (m_batch_op_short) { m_batch_op_short->revert(addr); diff --git a/src/dbzero/object_model/tags/TagIndex.hpp b/src/dbzero/object_model/tags/TagIndex.hpp index 4b413db2..4a2cc275 100644 --- a/src/dbzero/object_model/tags/TagIndex.hpp +++ b/src/dbzero/object_model/tags/TagIndex.hpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -15,21 +15,21 @@ namespace db0::object_model { - -DB0_PACKED_BEGIN - using Object = db0::object_model::Object; + using ObjectAnyImpl = db0::object_model::ObjectAnyImpl; using RC_LimitedStringPool = db0::pools::RC_LimitedStringPool; using LongTagT = db0::LongTagT; class EnumFactory; - + +DB0_PACKED_BEGIN struct DB0_PACKED_ATTR o_tag_index: public o_fixed { Address m_base_index_short_ptr = {}; Address m_base_index_long_ptr = {}; std::array m_reserved = { 0, 0, 0, 0 }; }; - +DB0_PACKED_END + /** * A class to represent a full-text (tag) index and the corresponding batch-update buffer * typically the TagIndex instance is associated with the Class object @@ -38,7 +38,7 @@ DB0_PACKED_BEGIN { public: using super_t = db0::v_object; - using LangToolkit = typename Object::LangToolkit; + using LangToolkit = typename ObjectAnyImpl::LangToolkit; using ObjectPtr = typename LangToolkit::ObjectPtr; using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; using ObjectSharedExtPtr = typename LangToolkit::ObjectSharedExtPtr; @@ -166,7 +166,7 @@ DB0_PACKED_BEGIN db0::FT_BaseIndex::BatchOperationBuilder &getBatchOperationShort(ObjectPtr, ActiveValueT &result, bool is_type) const; - + db0::FT_BaseIndex::BatchOperationBuilder &getBatchOperationLong(ObjectPtr, ActiveValueT &result) const; @@ -250,14 +250,14 @@ DB0_PACKED_BEGIN } return batch_op; } - + template BatchOperationT &TagIndex::getBatchOperation(ObjectPtr memo_ptr, BaseIndexT &base_index, BatchOperationT &batch_op, ActiveValueT &result) const - { + { // prepare the active value only if it's not yet initialized if (!result.first.isValid() && !result.second) { - auto &memo = LangToolkit::getTypeManager().extractObject(memo_ptr); + auto &memo = LangToolkit::getTypeManager().extractAnyObject(memo_ptr); // NOTE: that memo object may not have address before fully initialized (before postInit) if (memo.hasInstance()) { auto object_addr = memo.getUniqueAddress(); @@ -331,6 +331,4 @@ DB0_PACKED_BEGIN // Check if the object is pending update in the TagIndex withih a specific fixture bool isObjectPendingUpdate(db0::swine_ptr &fixture, UniqueAddress); -DB0_PACKED_END - } diff --git a/src/dbzero/object_model/value/Member.cpp b/src/dbzero/object_model/value/Member.cpp index fa632694..34d4fbbe 100644 --- a/src/dbzero/object_model/value/Member.cpp +++ b/src/dbzero/object_model/value/Member.cpp @@ -12,6 +12,9 @@ #include // FIXME: remove Python dependency #include +#include +#include +#include namespace db0::object_model @@ -53,11 +56,12 @@ namespace db0::object_model return db0::v_object(*fixture, PyUnicode_AsUTF8(obj_ptr), access_mode).getAddress(); } - // OBJECT specialization + // OBJECT specialization (mutable) template <> Value createMember(db0::swine_ptr &fixture, PyObjectPtr obj_ptr, StorageClass, AccessFlags) { - auto &obj = PyToolkit::getTypeManager().extractMutableObject(obj_ptr); + using MemoObject = PyToolkit::TypeManager::MemoObject; + auto &obj = PyToolkit::getTypeManager().extractMutableObject(obj_ptr); assert(obj.hasInstance()); assureSameFixture(fixture, obj); obj.modify().incRef(false); @@ -284,7 +288,8 @@ namespace db0::object_model PyObjectPtr obj_ptr, StorageClass storage_class, AccessFlags) { // NOTE: memo object can be extracted from the weak proxy - const auto &obj = PyToolkit::getTypeManager().extractObject(obj_ptr); + using MemoObject = PyToolkit::TypeManager::MemoObject; + const auto &obj = PyToolkit::getTypeManager().extractObject(obj_ptr); if (storage_class == StorageClass::OBJECT_LONG_WEAK_REF) { LongWeakRef weak_ref(fixture, obj); return weak_ref.getAddress(); @@ -646,7 +651,8 @@ namespace db0::object_model // decref cached instance via language specific wrapper type auto lang_wrapper = LangToolkit::template getWrapperTypeOf(obj_ptr.get()); auto &object = lang_wrapper->modifyExt(); - if (object.decRef(false)) { + object.decRef(false); + if (!object.hasRefs()) { // NOTE: we'll drop the object immediately on condition it has no language references if (!LangToolkit::hasLangRefs(*obj_ptr)) { auto unique_addr = object.getUniqueAddress(); @@ -750,12 +756,28 @@ namespace db0::object_model return !object_ptr || object_ptr->hasInstance(); } - void materialize(FixtureLock &fixture, PyObjectPtr obj_ptr) + template + void materializeImpl(FixtureLock &fixture, PyObjectPtr obj_ptr) { - auto object_ptr = PyToolkit::getTypeManager().tryExtractMutableObject(obj_ptr); + auto object_ptr = PyToolkit::getTypeManager().tryExtractMutableObject(obj_ptr); if (object_ptr && !object_ptr->hasInstance()) { object_ptr->postInit(fixture); } } + + void materialize(FixtureLock &fixture, PyObjectPtr obj_ptr) + { + using MemoObject = PyToolkit::TypeManager::MemoObject; + using MemoImmutableObject = PyToolkit::TypeManager::MemoImmutableObject; + if (PyToolkit::isMemoObject(obj_ptr)) { + materializeImpl(fixture, obj_ptr); + } else if (PyToolkit::isMemoImmutableObject(obj_ptr)) { + materializeImpl(fixture, obj_ptr); + } else { + assert(false && "Unsupported memo object type"); + THROWF(db0::InputException) << "Unable to materialize non-memo object" << THROWF_END; + } + } + } \ No newline at end of file diff --git a/src/dbzero/object_model/value/Member.hpp b/src/dbzero/object_model/value/Member.hpp index 10cc0cfa..46f34a59 100644 --- a/src/dbzero/object_model/value/Member.hpp +++ b/src/dbzero/object_model/value/Member.hpp @@ -112,9 +112,8 @@ namespace db0::object_model /** * Invoke materialize before setting obj_ptr as a member * this is to materialize objects (where hasInstance = false) before using them as members - */ - void materialize(FixtureLock &, PyObjectPtr obj_ptr); - + */ + void materialize(FixtureLock &fixture, PyObjectPtr obj_ptr); bool isMaterialized(PyObjectPtr obj_ptr); - + } diff --git a/src/dbzero/object_model/value/StorageClass.cpp b/src/dbzero/object_model/value/StorageClass.cpp index 2161d985..8afe89be 100644 --- a/src/dbzero/object_model/value/StorageClass.cpp +++ b/src/dbzero/object_model/value/StorageClass.cpp @@ -1,6 +1,7 @@ #include "StorageClass.hpp" #include #include +#include namespace db0::object_model @@ -176,7 +177,7 @@ namespace db0 db0::swine_ptr &fixture, ObjectPtr lang_value) { assert(pre_storage_class == PreStorageClass::OBJECT_WEAK_REF); - const auto &obj = LangToolkit::getTypeManager().extractObject(lang_value); + const auto &obj = LangToolkit::getTypeManager().extractAnyObject(lang_value); if (*obj.getFixture() != *fixture.get()) { // must use long weak-ref instead, since referenced object is from a foreign prefix return StorageClass::OBJECT_LONG_WEAK_REF; @@ -188,12 +189,12 @@ namespace db0 const db0::Fixture &fixture, ObjectPtr lang_value) { assert(pre_storage_class == PreStorageClass::OBJECT_WEAK_REF); - const auto &obj = LangToolkit::getTypeManager().extractObject(lang_value); + const auto &obj = LangToolkit::getTypeManager().extractAnyObject(lang_value); if (*obj.getFixture() != fixture) { // must use long weak-ref instead, since referenced object is from a foreign prefix return StorageClass::OBJECT_LONG_WEAK_REF; } return StorageClass::OBJECT_WEAK_REF; - } - + } + } \ No newline at end of file diff --git a/src/dbzero/workspace/AtomicContext.cpp b/src/dbzero/workspace/AtomicContext.cpp index 876bd807..d95755c0 100644 --- a/src/dbzero/workspace/AtomicContext.cpp +++ b/src/dbzero/workspace/AtomicContext.cpp @@ -22,10 +22,12 @@ namespace db0 } // MEMO_OBJECT specialization - template <> void detachObject(PyObjectPtr obj_ptr) { - detachExisting(PyToolkit::getTypeManager().extractObject(obj_ptr)); + template <> void detachObject(PyObjectPtr obj_ptr) + { + using MemoObject = PyToolkit::TypeManager::MemoObject; + detachExisting(PyToolkit::getTypeManager().extractObject(obj_ptr)); } - + // DB0_LIST specialization template <> void detachObject(PyObjectPtr obj_ptr) { detachExisting(PyToolkit::getTypeManager().extractList(obj_ptr));