diff --git a/AGENTS.md b/AGENTS.md index 540b5aea..c080ee9f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,18 +16,24 @@ All tests must pass before a change is considered complete. ### Building -- Debug build: `./docker/dbzero-build.sh` -- Release build: `./docker/dbzero-build.sh -r` +- Debug build: `./scripts/build.sh` +- Release build: `./scripts/build.sh -r` +- Release build with C++ unit test binary: `./scripts/build.sh -r -t` ### Running tests - Python tests: `./scripts/run_tests.sh` +- C++ tests after a `-t` build: `./build/release/tests.x` - If any C++ source under the native/core part of the project was modified, also run the C++ test suite (do not rely on the Python tests alone to cover native changes). Never mark a task done while tests are failing. ## Implementation notes +### C++ style + +- Use camelCase for local helper variables, lambdas, and method names in C++ code. + ### MorphingBIndex: address and type can change on mutation A `MorphingBIndex` does not behave like a typical container. On mutation (`insert`, `erase`) it may morph into a different internal storage variant (itty / array_2..4 / vector / bindex), and the morph can change both its **address** and its **type**. diff --git a/docker/Dockerfile-claude b/docker/Dockerfile-claude deleted file mode 100644 index 5f6db0c5..00000000 --- a/docker/Dockerfile-claude +++ /dev/null @@ -1,88 +0,0 @@ -FROM python:3.12-bullseye - -ARG NODE_MAJOR=20 -ARG CLAUDE_CODE_VERSION=latest -ARG DEV_USER=claude -ARG DEV_UID=1000 -ARG DEV_GID=1000 - -ENV DEBIAN_FRONTEND=noninteractive -ENV HOME=/home/${DEV_USER} -ENV CLAUDE_CONFIG_DIR=/home/${DEV_USER}/.claude -ENV PATH=/home/${DEV_USER}/.local/bin:${PATH} - -# Install development dependencies, Node.js 20, and Claude Code. -RUN apt-get update && apt-get install -y \ - ca-certificates \ - cmake \ - curl \ - gdb \ - gettext-base \ - git \ - gnupg \ - jq \ - less \ - meson \ - ninja-build \ - psmisc \ - python3-dbg \ - python3-pip \ - python3-venv \ - ripgrep \ - rsync \ - screen \ - unzip \ - valgrind \ - && mkdir -p /etc/apt/keyrings \ - && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \ - | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ - && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" \ - > /etc/apt/sources.list.d/nodesource.list \ - && apt-get update && apt-get install -y nodejs \ - && npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" \ - && node "$(npm root -g)/@anthropic-ai/claude-code/install.cjs" \ - && npm cache clean --force \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -RUN groupadd --gid ${DEV_GID} ${DEV_USER} \ - && useradd --uid ${DEV_UID} --gid ${DEV_GID} --create-home --shell /bin/bash ${DEV_USER} - -# Install Python build tooling and project requirements. -RUN python3 -m pip install --no-cache-dir --upgrade pip setuptools wheel build - -COPY requirements.txt /usr/src/dbzero/ -RUN python3 -m pip install --no-cache-dir --upgrade -r /usr/src/dbzero/requirements.txt - -# Seed Claude Code defaults and make them available in interactive shells. -COPY docker/claude-settings.json /opt/claude/settings.json -COPY docker/claude-shell-init.sh /usr/local/bin/claude-shell-init -COPY docker/dbzero-build.sh /usr/local/bin/dbzero-build -COPY docker/dbzero-build-package.sh /usr/local/bin/dbzero-build-package -RUN chmod +x /usr/local/bin/claude-shell-init \ - && chmod +x /usr/local/bin/dbzero-build /usr/local/bin/dbzero-build-package \ - && mkdir -p ${CLAUDE_CONFIG_DIR} /etc/profile.d \ - && chmod 700 ${CLAUDE_CONFIG_DIR} \ - && cp /opt/claude/settings.json ${CLAUDE_CONFIG_DIR}/settings.json \ - && chmod 600 ${CLAUDE_CONFIG_DIR}/settings.json \ - && chown -R ${DEV_UID}:${DEV_GID} ${HOME} /opt/claude \ - && printf '%s\n' \ - 'export CLAUDE_CONFIG_DIR="${CLAUDE_CONFIG_DIR:-$HOME/.claude}"' \ - > /etc/profile.d/claude-code.sh \ - && printf '%s\n' \ - 'export CLAUDE_CONFIG_DIR="${CLAUDE_CONFIG_DIR:-$HOME/.claude}"' \ - >> ${HOME}/.bashrc \ - && claude --version - -RUN ulimit -c unlimited - -COPY --chown=${DEV_UID}:${DEV_GID} . /usr/src/dbzero -WORKDIR /usr/src/dbzero - -RUN python3 scripts/generate_meson.py ./src/dbzero/ core -RUN python3 scripts/generate_meson_tests.py tests/ - -USER ${DEV_USER} - -ENTRYPOINT ["/usr/local/bin/claude-shell-init"] -CMD ["/bin/bash"] \ No newline at end of file diff --git a/docker/Dockerfile-dev b/docker/Dockerfile-dev index c78aac07..e9306a6d 100644 --- a/docker/Dockerfile-dev +++ b/docker/Dockerfile-dev @@ -1,4 +1,8 @@ -FROM gcc:12-bullseye +FROM python:3.11-bullseye + +ARG USERNAME=dbzero +ARG USER_UID=1000 +ARG USER_GID=1000 RUN apt-get update && apt-get install -y \ cmake \ @@ -6,15 +10,23 @@ RUN apt-get update && apt-get install -y \ RUN apt-get update RUN apt-get install psmisc -RUN apt-get install python3.9-dev -y RUN apt-get install python3-pip -y RUN apt-get install gdb -y RUN apt-get install screen -y RUN apt-get install rsync -y +RUN apt-get install ripgrep -y RUN apt-get install meson ninja-build -y RUN apt-get install python3-venv -y RUN apt-get install gettext-base -y RUN apt-get install valgrind -y +RUN apt-get install curl ca-certificates gnupg -y + +RUN mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_22.x nodistro main" > /etc/apt/sources.list.d/nodesource.list \ + && apt-get update \ + && apt-get install nodejs -y \ + && npm install -g @openai/codex # for demo purposes only (bokeh is a charts package) RUN pip3 install jupyter @@ -31,11 +43,13 @@ RUN ulimit -c unlimited RUN mkdir -p "$(cat /proc/sys/kernel/core_pattern | sed 's/%.*//')" RUN chmod 777 "$(cat /proc/sys/kernel/core_pattern | sed 's/%.*//')" -COPY . /usr/src/dbzero -WORKDIR /usr/src/dbzero +RUN groupadd --gid "$USER_GID" "$USERNAME" \ + && useradd --uid "$USER_UID" --gid "$USER_GID" --create-home --shell /bin/bash "$USERNAME" -RUN python3 scripts/generate_meson.py ./src/dbzero/ core -RUN python3 scripts/generate_meson_tests.py tests/ +RUN chown -R "$USERNAME:$USERNAME" /usr/local/lib/python3.11/site-packages /usr/local/bin + +WORKDIR /usr/src/dbzero +USER $USERNAME # RUN ./build.sh -r # WORKDIR /usr/src/dbzero/build/release/ # RUN meson install diff --git a/python_tests/test_composite_tags.py b/python_tests/test_composite_tags.py new file mode 100644 index 00000000..1a5f2e75 --- /dev/null +++ b/python_tests/test_composite_tags.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# Copyright (c) 2026 DBZero Software sp. z o.o. + +import dbzero as db0 +import pytest + + +@db0.memo +class CompositeTagUser: + def __init__(self, name): + self.name = name + + +@db0.memo +class CompositeTagDocument: + def __init__(self, title): + self.title = title + + +def test_can_add_tuple_based_composite_tag(db0_fixture): + user = CompositeTagUser("user-1") + document = CompositeTagDocument("doc-1") + + db0.tags(document).add(("GRANT-READ", user)) + db0.commit() + + +def test_can_add_as_tag_composite_tag(db0_fixture): + user = CompositeTagUser("user-1") + document = CompositeTagDocument("doc-1") + + db0.tags(document).add(db0.as_tag("GRANT-READ", user)) + db0.commit() + + +def test_can_add_as_tag_tuple_composite_tag(db0_fixture): + user = CompositeTagUser("user-1") + document = CompositeTagDocument("doc-1") + + db0.tags(document).add(db0.as_tag(("GRANT-READ", user))) + db0.commit() + + +def test_can_remove_as_tag_composite_tag(db0_fixture): + user = CompositeTagUser("user-1") + document = CompositeTagDocument("doc-1") + composite_tag = db0.as_tag("GRANT-READ", user) + + db0.tags(document).add(composite_tag) + db0.tags(document).remove(composite_tag) + db0.commit() + + +def test_find_by_as_tag_composite_tag(db0_fixture): + user_1 = CompositeTagUser("user-1") + user_2 = CompositeTagUser("user-2") + document_1 = CompositeTagDocument("doc-1") + document_2 = CompositeTagDocument("doc-2") + document_3 = CompositeTagDocument("doc-3") + + db0.tags(document_1).add(db0.as_tag("GRANT-READ", user_1)) + db0.tags(document_2).add(db0.as_tag("GRANT-READ", user_2)) + db0.tags(document_3).add("GRANT-READ") + + assert [doc.title for doc in db0.find(db0.as_tag("GRANT-READ", user_1))] == ["doc-1"] + assert [doc.title for doc in db0.find(db0.as_tag(("GRANT-READ", user_2)))] == ["doc-2"] + + +def test_find_by_composite_tag_with_type_filter(db0_fixture): + user = CompositeTagUser("user-1") + document = CompositeTagDocument("doc-1") + other = CompositeTagUser("not-a-document") + + db0.tags(document).add(db0.as_tag("GRANT-READ", user)) + db0.tags(other).add(db0.as_tag("GRANT-READ", user)) + + assert [doc.title for doc in db0.find(CompositeTagDocument, db0.as_tag("GRANT-READ", user))] == ["doc-1"] + + +def test_find_by_composite_tag_combines_with_simple_tags(db0_fixture): + user = CompositeTagUser("user-1") + document_1 = CompositeTagDocument("doc-1") + document_2 = CompositeTagDocument("doc-2") + + db0.tags(document_1).add(db0.as_tag("GRANT-READ", user), "active") + db0.tags(document_2).add(db0.as_tag("GRANT-READ", user), "archived") + + assert [doc.title for doc in db0.find(db0.as_tag("GRANT-READ", user), "active")] == ["doc-1"] + + +def test_find_by_missing_composite_leaf_returns_empty(db0_fixture): + user_1 = CompositeTagUser("user-1") + user_2 = CompositeTagUser("user-2") + document = CompositeTagDocument("doc-1") + + db0.tags(document).add(db0.as_tag("GRANT-READ", user_1)) + + assert list(db0.find(db0.as_tag("GRANT-READ", user_2))) == [] + + +def test_find_by_composite_tag_inside_or_clause(db0_fixture): + user = CompositeTagUser("user-1") + document_1 = CompositeTagDocument("doc-1") + document_2 = CompositeTagDocument("doc-2") + + db0.tags(document_1).add(db0.as_tag("GRANT-READ", user)) + db0.tags(document_2).add("fallback") + + assert {doc.title for doc in db0.find([db0.as_tag("GRANT-READ", user), "fallback"])} == {"doc-1", "doc-2"} + + +def test_find_by_composite_tag_with_negation(db0_fixture): + user = CompositeTagUser("user-1") + document_1 = CompositeTagDocument("doc-1") + document_2 = CompositeTagDocument("doc-2") + + db0.tags(document_1).add(db0.as_tag("GRANT-READ", user)) + db0.tags(document_2).add("visible") + + assert [doc.title for doc in db0.find(CompositeTagDocument, db0.no(db0.as_tag("GRANT-READ", user)))] == ["doc-2"] + + +def test_composite_tags_with_many_objects_and_many_tags(db0_fixture): + users = [CompositeTagUser(f"user-{i}") for i in range(12)] + documents = [CompositeTagDocument(f"doc-{i}") for i in range(60)] + predicates = [f"perm-{i}" for i in range(15)] + removed_pairs = set() + + def pairs_for_document(index): + return { + (index % len(predicates), index % len(users)), + ((index + 1) % len(predicates), (index + 3) % len(users)), + } + + def titles_for(predicate_index, user_index): + return { + documents[index].title + for index in range(len(documents)) + if (predicate_index, user_index) in pairs_for_document(index) + and (index, predicate_index, user_index) not in removed_pairs + } + + def query_titles(*args): + return {doc.title for doc in db0.find(*args)} + + for index, document in enumerate(documents): + composite_tags = [ + db0.as_tag(predicates[predicate_index], users[user_index]) + for predicate_index, user_index in sorted(pairs_for_document(index)) + ] + simple_tags = ["active" if index % 2 == 0 else "archived"] + if index % 3 == 0: + simple_tags.append("reviewed") + db0.tags(document).add(*composite_tags, *simple_tags) + + assert query_titles(db0.as_tag(predicates[4], users[4])) == titles_for(4, 4) + assert query_titles(db0.as_tag((predicates[7], users[10]))) == titles_for(7, 10) + + active_expected = { + title + for title in titles_for(4, 4) + if int(title.split("-")[1]) % 2 == 0 + } + assert query_titles(CompositeTagDocument, db0.as_tag(predicates[4], users[4]), "active") == active_expected + + reviewed_expected = {document.title for index, document in enumerate(documents) if index % 3 == 0} + assert query_titles([db0.as_tag(predicates[7], users[10]), "reviewed"]) == ( + titles_for(7, 10) | reviewed_expected + ) + + denied_expected = {document.title for document in documents} - titles_for(4, 4) + assert query_titles(CompositeTagDocument, db0.no(db0.as_tag(predicates[4], users[4]))) == denied_expected + + for index in range(0, len(documents), 5): + predicate_index, user_index = next(iter(pairs_for_document(index))) + db0.tags(documents[index]).remove(db0.as_tag(predicates[predicate_index], users[user_index])) + removed_pairs.add((index, predicate_index, user_index)) + + assert query_titles(db0.as_tag(predicates[0], users[0])) == titles_for(0, 0) + assert query_titles(db0.as_tag(predicates[5], users[5])) == titles_for(5, 5) + assert query_titles(CompositeTagDocument, db0.as_tag(predicates[10], users[10])) == titles_for(10, 10) + + +def test_rejects_nested_composite_tag_before_update(db0_fixture): + user = CompositeTagUser("user-1") + document = CompositeTagDocument("doc-1") + + with pytest.raises(Exception): + db0.tags(document).add("simple-tag", ("GRANT-READ", ("nested", user))) + + db0.tags(document).add(("GRANT-READ", user)) + db0.commit() + + +def test_rejects_nested_composite_tag_on_remove(db0_fixture): + user = CompositeTagUser("user-1") + document = CompositeTagDocument("doc-1") + + db0.tags(document).add(("GRANT-READ", user)) + + with pytest.raises(Exception): + db0.tags(document).remove("simple-tag", ("GRANT-READ", ("nested", user))) + + db0.commit() diff --git a/python_tests/test_tags.py b/python_tests/test_tags.py index 3cecfd07..3dafd910 100644 --- a/python_tests/test_tags.py +++ b/python_tests/test_tags.py @@ -84,6 +84,39 @@ def test_tags_can_be_applied_to_multiple_objects(db0_fixture): assert len(list(db0.find("tag1"))) == 3 +def test_adding_same_tag_after_reopen_without_querying_updates_inverted_list(db0_fixture): + prefix_name = db0.get_current_prefix().name + initial_values = range(64) + for value in initial_values: + db0.tags(MemoClassForTags(value)).add("tag1") + + db0.close() + db0.init(DB0_DIR) + db0.open(prefix_name) + + additional_values = range(64, 80) + for value in additional_values: + db0.tags(MemoClassForTags(value)).add("tag1") + + assert {obj.value for obj in db0.find(MemoClassForTags, "tag1")} == set(range(80)) + + +def test_adding_same_tag_after_reopen_with_existing_object_in_cache(db0_fixture): + prefix_name = db0.get_current_prefix().name + object_1 = MemoClassForTags(1) + object_1_uuid = db0.uuid(object_1) + db0.tags(object_1).add("tag1") + + db0.close() + db0.init(DB0_DIR) + db0.open(prefix_name) + + assert db0.fetch(object_1_uuid).value == 1 + db0.tags(MemoClassForTags(2)).add("tag1") + + assert {obj.value for obj in db0.find(MemoClassForTags, "tag1")} == {1, 2} + + def test_tag_queries_can_use_or_filters(db0_fixture): objects = [MemoClassForTags(i) for i in range(10)] db0.tags(objects[4]).add(["tag1", "tag2"]) diff --git a/src/dbzero/bindings/TypeId.hpp b/src/dbzero/bindings/TypeId.hpp index 6aee9403..4710730f 100644 --- a/src/dbzero/bindings/TypeId.hpp +++ b/src/dbzero/bindings/TypeId.hpp @@ -59,8 +59,9 @@ namespace db0::bindings MEMO_TYPE = 118, MEMO_IMMUTABLE_OBJECT = 119, DB0_WEAK_SET = 120, + DB0_COMPOSITE_TAG = 121, // COUNT determines size of the type operator arrays - COUNT = 121, + COUNT = 122, // unrecognized type UNKNOWN = std::numeric_limits::max() }; diff --git a/src/dbzero/bindings/python/PyTypeManager.cpp b/src/dbzero/bindings/python/PyTypeManager.cpp index a86662f3..df271baf 100644 --- a/src/dbzero/bindings/python/PyTypeManager.cpp +++ b/src/dbzero/bindings/python/PyTypeManager.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -84,6 +85,7 @@ namespace db0::python addStaticSimpleType(decimal_type, TypeId::DECIMAL); // dbzero extension types addStaticdbzeroType(&PyTagType, TypeId::DB0_TAG); + addStaticdbzeroType(&PyCompositeTagType, TypeId::DB0_COMPOSITE_TAG); addStaticdbzeroType(&TagSetType, TypeId::DB0_TAG_SET); addStaticdbzeroType(&IndexObjectType, TypeId::DB0_INDEX); addStaticdbzeroType(&ListObjectType, TypeId::DB0_LIST); @@ -596,6 +598,12 @@ namespace db0::python assert(PyTag_Check(py_tag)); return reinterpret_cast(py_tag)->ext(); } + + const PyTypeManager::CompositeTagDef &PyTypeManager::extractCompositeTag(ObjectPtr py_tag) const + { + assert(PyCompositeTag_Check(py_tag)); + return reinterpret_cast(py_tag)->ext(); + } const MemoTypeDecoration &PyTypeManager::getMemoTypeDecoration(TypeObjectPtr py_type) const { @@ -724,4 +732,4 @@ namespace db0::python template db0::object_model::Object * PyTypeManager::tryExtractMutableObject(ObjectPtr) const; -} \ No newline at end of file +} diff --git a/src/dbzero/bindings/python/PyTypeManager.hpp b/src/dbzero/bindings/python/PyTypeManager.hpp index f7394dfd..211882d0 100644 --- a/src/dbzero/bindings/python/PyTypeManager.hpp +++ b/src/dbzero/bindings/python/PyTypeManager.hpp @@ -41,6 +41,7 @@ namespace db0::object_model { struct EnumValueRepr; struct FieldDef; class TagDef; + class CompositeTagDef; class ByteArray; class PyWeakProxy; @@ -86,6 +87,7 @@ namespace db0::python using FieldDef = db0::object_model::FieldDef; using Class = db0::object_model::Class; using TagDef = db0::object_model::TagDef; + using CompositeTagDef = db0::object_model::CompositeTagDef; using ByteArray = db0::object_model::ByteArray; PyTypeManager(); @@ -151,6 +153,7 @@ namespace db0::python ObjectPtr getLangObject(TypeObjectPtr py_type) const; std::shared_ptr extractConstClass(ObjectPtr py_class) const; const TagDef &extractTag(ObjectPtr py_tag) const; + const CompositeTagDef &extractCompositeTag(ObjectPtr py_tag) const; ByteArray &extractMutableByteArray(ObjectPtr) const; ObjectPtr getBadPrefixError() const; @@ -262,4 +265,4 @@ namespace db0::python extern template db0::object_model::Object * PyTypeManager::tryExtractMutableObject(ObjectPtr) const; -} \ No newline at end of file +} diff --git a/src/dbzero/bindings/python/dbzero.cpp b/src/dbzero/bindings/python/dbzero.cpp index 3606e68d..0c0ec1d9 100644 --- a/src/dbzero/bindings/python/dbzero.cpp +++ b/src/dbzero/bindings/python/dbzero.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include namespace py = db0::python; @@ -212,6 +213,7 @@ PyMODINIT_FUNC PyInit_dbzero(void) &py::TagSetType, &py::PyAtomicType, &py::PyTagType, + &py::PyCompositeTagType, &py::PyLockedType, &py::PyWeakProxyType, }; diff --git a/src/dbzero/bindings/python/types/PyCompositeTag.cpp b/src/dbzero/bindings/python/types/PyCompositeTag.cpp new file mode 100644 index 00000000..f1e38714 --- /dev/null +++ b/src/dbzero/bindings/python/types/PyCompositeTag.cpp @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright (c) 2026 DBZero Software sp. z o.o. + +#include "PyCompositeTag.hpp" +#include +#include + +namespace db0::python + +{ + + PyCompositeTag *PyCompositeTag_new(PyTypeObject *type, PyObject *, PyObject *) + { + return reinterpret_cast(type->tp_alloc(type, 0)); + } + + PyCompositeTag *PyCompositeTagDefault_new() + { + return PyCompositeTag_new(&PyCompositeTagType, NULL, NULL); + } + + void PyCompositeTag_del(PyCompositeTag *self) + { + PY_API_FUNC + self->destroy(); + Py_TYPE(self)->tp_free((PyObject*)self); + } + + static PyObject *tryPyAPI_PyCompositeTag_richcompare(PyCompositeTag *self, PyObject *other, int op) + { + bool result = false; + if (PyCompositeTag_Check(other)) { + auto *other_tag = reinterpret_cast(other); + result = self->ext() == other_tag->ext(); + } + + switch (op) + { + case Py_EQ: + return PyBool_fromBool(result); + case Py_NE: + return PyBool_fromBool(!result); + default: + Py_RETURN_NOTIMPLEMENTED; + } + } + + static PyObject *PyAPI_PyCompositeTag_richcompare(PyCompositeTag *self, PyObject *other, int op) + { + PY_API_FUNC + return runSafe(tryPyAPI_PyCompositeTag_richcompare, self, other, op); + } + + static Py_hash_t TryPyAPI_PyCompositeTag_hash(PyCompositeTag *self) + { + return self->ext().getHash(); + } + + static Py_hash_t PyAPI_PyCompositeTag_hash(PyCompositeTag *self) + { + PY_API_FUNC + return runSafe(TryPyAPI_PyCompositeTag_hash, self); + } + + PyTypeObject PyCompositeTagType = { + PYVAROBJECT_HEAD_INIT_DESIGNATED, + .tp_name = "dbzero.CompositeTag", + .tp_basicsize = PyCompositeTag::sizeOf(), + .tp_itemsize = 0, + .tp_dealloc = (destructor)PyCompositeTag_del, + .tp_hash = (hashfunc)PyAPI_PyCompositeTag_hash, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_richcompare = (richcmpfunc)PyAPI_PyCompositeTag_richcompare, + .tp_alloc = PyType_GenericAlloc, + .tp_new = (newfunc)PyCompositeTag_new, + .tp_free = PyObject_Free, + }; + + bool PyCompositeTag_Check(PyObject *py_object) + { + return Py_TYPE(py_object) == &PyCompositeTagType; + } + +} diff --git a/src/dbzero/bindings/python/types/PyCompositeTag.hpp b/src/dbzero/bindings/python/types/PyCompositeTag.hpp new file mode 100644 index 00000000..8e19334e --- /dev/null +++ b/src/dbzero/bindings/python/types/PyCompositeTag.hpp @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright (c) 2026 DBZero Software sp. z o.o. + +#pragma once + +#include +#include +#include + +namespace db0::python + +{ + + using CompositeTagDef = db0::object_model::CompositeTagDef; + using PyCompositeTag = PyWrapper; + + PyCompositeTag *PyCompositeTag_new(PyTypeObject *type, PyObject *, PyObject *); + PyCompositeTag *PyCompositeTagDefault_new(); + + void PyCompositeTag_del(PyCompositeTag *); + extern PyTypeObject PyCompositeTagType; + + bool PyCompositeTag_Check(PyObject *); + +} diff --git a/src/dbzero/bindings/python/types/PyTag.cpp b/src/dbzero/bindings/python/types/PyTag.cpp index 3f62145f..a77b30eb 100644 --- a/src/dbzero/bindings/python/types/PyTag.cpp +++ b/src/dbzero/bindings/python/types/PyTag.cpp @@ -2,6 +2,7 @@ // Copyright (c) 2025 DBZero Software sp. z o.o. #include "PyTag.hpp" +#include "PyCompositeTag.hpp" #include #include #include @@ -99,14 +100,78 @@ namespace db0::python py_tag->makeNew(expired_ref.getFixtureUUID(), expired_ref.getAddress(), py_obj); return py_tag; } + + CompositeTagDef::ObjectSharedPtr makeCompositeItem(PyObject *arg) + { + if (PyTag_Check(arg) || PyCompositeTag_Check(arg)) { + return CompositeTagDef::ObjectSharedPtr(arg); + } + if (PyMemo_Check(arg)) { + return CompositeTagDef::ObjectSharedPtr(tryMemoAsTag(arg), false); + } + if (PyMemo_Check(arg)) { + return CompositeTagDef::ObjectSharedPtr(tryMemoAsTag(arg), false); + } + if (PyType_Check(arg)) { + auto *py_type = reinterpret_cast(arg); + if (PyAnyMemoType_Check(py_type)) { + return CompositeTagDef::ObjectSharedPtr(tryMemoTypeAsTag(py_type), false); + } + } + if (MemoExpiredRef_Check(arg)) { + return CompositeTagDef::ObjectSharedPtr(tryMemoExpiredRefAsTag(arg), false); + } + return CompositeTagDef::ObjectSharedPtr(arg); + } + + PyObject *tryCompositeAsTag(PyObject *const *args, Py_ssize_t nargs) + { + std::vector items; + items.reserve(nargs); + for (Py_ssize_t i = 0; i < nargs; ++i) { + items.push_back(makeCompositeItem(args[i])); + } + if (items.size() < 2) { + THROWF(db0::InputException) << "as_tag: composite tag requires at least 2 items" << THROWF_END; + } + auto *py_tag = PyCompositeTagDefault_new(); + py_tag->makeNew(std::move(items)); + return reinterpret_cast(py_tag); + } + + PyObject *tryCompositeTupleAsTag(PyObject *arg) + { + auto length = PyTuple_Size(arg); + if (length < 0) { + THROWF(db0::InputException) << "as_tag: unable to read tuple" << THROWF_END; + } + + std::vector items; + items.reserve(length); + for (Py_ssize_t i = 0; i < length; ++i) { + items.push_back(makeCompositeItem(PyTuple_GET_ITEM(arg, i))); + } + if (items.size() < 2) { + THROWF(db0::InputException) << "as_tag: composite tag requires at least 2 items" << THROWF_END; + } + auto *py_tag = PyCompositeTagDefault_new(); + py_tag->makeNew(std::move(items)); + return reinterpret_cast(py_tag); + } PyObject *PyAPI_as_tag(PyObject *, PyObject *const *args, Py_ssize_t nargs) { PY_API_FUNC - if (nargs != 1) { - PyErr_SetString(PyExc_TypeError, "as_tag: Expected 1 argument"); + if (nargs == 0) { + PyErr_SetString(PyExc_TypeError, "as_tag: Expected at least 1 argument"); return NULL; } + if (nargs > 1) { + return runSafe(tryCompositeAsTag, args, nargs); + } + if (PyTuple_Check(args[0])) { + return runSafe(tryCompositeTupleAsTag, args[0]); + } if (PyMemo_Check(args[0])) { return runSafe(tryMemoAsTag, args[0]); } else if (PyMemo_Check(args[0])) { diff --git a/src/dbzero/core/collections/full_text/InvertedIndex.hpp b/src/dbzero/core/collections/full_text/InvertedIndex.hpp index 4f982dc8..2ef643a6 100644 --- a/src/dbzero/core/collections/full_text/InvertedIndex.hpp +++ b/src/dbzero/core/collections/full_text/InvertedIndex.hpp @@ -29,9 +29,13 @@ namespace db0 // use high 4-bits for index type auto index_type = static_cast(address.getOffset() >> 60); auto mb_addr = Address::fromOffset(address & 0x0FFFFFFFFFFFFFFF); - // NOTE: first address is for cache, the latter for MorphingBIndex + // NOTE: first address is for cache, the latter for MorphingBIndex. + // This pulls a MorphingBIndex wrapper over existing state; the constructor + // opens the encoded address/type instead of creating a new persisted index. // NOTE: MorphingBIndex does not provide detach functionality - return cache.findOrCreate >(mb_addr, false, mb_addr, index_type); + return cache.findOrPull >( + mb_addr, false, cache.getMemspace(), mb_addr, index_type + ); } template @@ -62,13 +66,6 @@ namespace db0 InvertedIndex(InvertedIndex &&); - /** - * Pull existing or create new key inverted list - * @param key key to retrieve/create the inverted list by - * @return the inverted list object - */ - std::shared_ptr findOrCreateInvertedList(IndexKeyT key); - /** * Similar as getObjectIndex but performed in a bulk operation for all provided keys * @param keys ***MUST BE SORTED*** ascendingS @@ -110,11 +107,6 @@ namespace db0 return result; } - /** - * Pull list by map item - */ - std::shared_ptr getInvertedList(const MapItemT &item) const; - /** * Pull existing index from under iterator (or create new) */ @@ -166,39 +158,14 @@ namespace db0 { } - template - std::shared_ptr::ListT> - InvertedIndex::findOrCreateInvertedList(IndexKeyT key) - { - MapItemT item(key); - auto it = super_t::find(item); - if (it == super_t::end()) { - // construct as empty (pull through cache) - auto list_ptr = m_cache.create(); - item.value = m_value_function(*list_ptr); - super_t::insert(item); - return list_ptr; - } else { - // fetch existing - return m_list_function(m_cache, it->value); - } - } - - template - std::shared_ptr::ListT> - InvertedIndex::getInvertedList(const MapItemT &item) const - { - // pull dbzero existing - return m_list_function(m_cache, item.value); - } - template std::shared_ptr::ListT> InvertedIndex::getInvertedList(iterator &it) { if ((*it).value == ValueT()) { - // assume ListT as non-detachable (e.g. MorphingBIndex) - auto list_ptr = m_cache.create(false); + // Empty map value means this tag has no inverted list yet. + // Pulling ListT with no extra args constructs a new empty MorphingBIndex. + auto list_ptr = m_cache.pull(false, this->getMemspace()); it.modifyItem().value = m_value_function(*list_ptr); return list_ptr; } else { @@ -240,4 +207,4 @@ namespace db0 } } -} \ No newline at end of file +} diff --git a/src/dbzero/core/collections/map/VInstanceMap.hpp b/src/dbzero/core/collections/map/VInstanceMap.hpp new file mode 100644 index 00000000..398cb3bf --- /dev/null +++ b/src/dbzero/core/collections/map/VInstanceMap.hpp @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright (c) 2026 DBZero Software sp. z o.o. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace db0 + +{ + + /** + * Ordered numeric-key map from KeyT to persisted instances of ValueT. + * + * Values are stored as v-object addresses in the underlying v_bindex and + * materialized through VObjectCache on lookup. + */ + template + class VInstanceMap: public db0::v_bindex, Address> + { + public: + using MapItemT = key_value; + using super_t = db0::v_bindex; + using iterator = typename super_t::iterator; + using const_iterator = typename super_t::const_iterator; + + VInstanceMap() = default; + + VInstanceMap(Memspace &memspace, VObjectCache &cache) + : super_t(memspace) + , m_cache(&cache) + { + } + + VInstanceMap(mptr ptr, VObjectCache &cache) + : super_t(ptr, ptr.getPageSize()) + , m_cache(&cache) + { + } + + VInstanceMap(VInstanceMap &&other) + : super_t(std::move(other)) + , m_cache(other.m_cache) + , m_active_instances(std::move(other.m_active_instances)) + { + } + + VInstanceMap &operator=(VInstanceMap &&other) + { + super_t::operator=(std::move(other)); + m_cache = other.m_cache; + m_active_instances = std::move(other.m_active_instances); + return *this; + } + + template + std::shared_ptr insert(KeyT key, Args&&... args) + { + // This is a new map value: constructor args are creation args for ValueT, + // so pull allocates new persisted state and caches the wrapper. + auto value = m_cache->template pull(true, this->getMemspace(), std::forward(args)...); + MapItemT item(key, value->getAddress()); + MapItemT old_item(key); + if (super_t::updateExisting(item, &old_item)) { + destroyValue(old_item.value, std::forward(args)...); + m_cache->erase(old_item.value); + m_active_instances.erase(cacheKey(old_item.value)); + } else { + super_t::insert(item); + } + track(value); + return value; + } + + template + std::shared_ptr findOrCreate(KeyT key, Args&&... args) + { + MapItemT item(key); + if (super_t::findOne(item)) { + return openExisting(item.value, std::forward(args)...); + } + return insert(key, std::forward(args)...); + } + + template + std::shared_ptr tryGet(KeyT key, Args&&... args) const + { + MapItemT item(key); + if (!super_t::findOne(item)) { + return nullptr; + } + return open(item.value, std::forward(args)...); + } + + template + std::shared_ptr get(KeyT key, Args&&... args) const + { + auto result = tryGet(key, std::forward(args)...); + if (!result) { + THROWF(db0::InputException) << "VInstanceMap key not found" << THROWF_END; + } + return result; + } + + template + bool erase(KeyT key, Args&&... args) + { + auto it = super_t::find(MapItemT(key)); + if (it == super_t::end()) { + return false; + } + + auto address = (*it).value; + auto value = open(address, std::forward(args)...); + value->destroy(); + super_t::erase(it); + m_cache->erase(address); + m_active_instances.erase(cacheKey(address)); + return true; + } + + template + std::size_t forEachActive(F &&f) const + { + std::size_t active_count = 0; + for (auto it = m_active_instances.begin(); it != m_active_instances.end();) { + auto value = it->second.lock(); + if (!value) { + it = m_active_instances.erase(it); + continue; + } + + ++active_count; + ++it; + if constexpr(std::is_void_v >) { + f(*value); + } else { + static_assert(std::is_convertible_v, bool>, + "VInstanceMap::forEachActive callback must return void or bool"); + if (!f(*value)) { + break; + } + } + } + return active_count; + } + + private: + template + void destroyValue(ValueAddrT address, Args&&... args) + { + auto tracked_value = tryFindTracked(address); + if (tracked_value) { + tracked_value->destroy(); + return; + } + + auto cached_value = m_cache->template tryFind(address); + if (cached_value) { + cached_value->destroy(); + return; + } + + if constexpr(std::is_constructible_v) { + open(address, std::forward(args)...)->destroy(); + } else { + static_assert(std::is_constructible_v, + "ValueT must be constructible from mptr or still present in VObjectCache to destroy replaced values"); + open(address)->destroy(); + } + } + + template + std::shared_ptr open(ValueAddrT address, Args&&... args) const + { + // This map item already stores a persisted ValueT address. The numeric + // address is the cache key; the mptr is the constructor argument that + // makes pull open the existing object instead of allocating a replacement. + auto ptr = this->getMemspace().myPtr(address); + auto value = m_cache->template findOrPull( + address.getOffset(), + true, + ptr, + std::forward(args)... + ); + track(value); + return value; + } + + template + std::shared_ptr openExisting(ValueAddrT address, Args&&... args) const + { + if constexpr(std::is_constructible_v) { + return open(address, std::forward(args)...); + } else { + static_assert(std::is_constructible_v, + "ValueT must be constructible from mptr or from mptr plus the supplied arguments"); + return open(address); + } + } + + void track(const std::shared_ptr &value) const + { + m_active_instances[value->getAddress().getOffset()] = value; + } + + std::shared_ptr tryFindTracked(ValueAddrT address) const + { + auto it = m_active_instances.find(cacheKey(address)); + if (it == m_active_instances.end()) { + return nullptr; + } + auto value = it->second.lock(); + if (!value) { + m_active_instances.erase(it); + } + return value; + } + + static std::uint64_t cacheKey(ValueAddrT address) + { + return address.getOffset(); + } + + VObjectCache *m_cache = nullptr; + mutable std::unordered_map > m_active_instances; + }; + +} diff --git a/src/dbzero/core/memory/VObjectCache.hpp b/src/dbzero/core/memory/VObjectCache.hpp index 1da2804a..f704d6de 100644 --- a/src/dbzero/core/memory/VObjectCache.hpp +++ b/src/dbzero/core/memory/VObjectCache.hpp @@ -58,11 +58,18 @@ namespace db0 VObjectCache(Memspace &, FixedObjectList &shared_object_list); /** - * Create a new v_object instance add add to cache + * Pull an object wrapper into the cache. + * + * The actual behavior depends on T's constructor selected by Args: + * - constructors taking Memspace plus creation args allocate/create new persisted state; + * - constructors taking Memspace plus an existing address/type open existing persisted state; + * - constructors taking mptr open an existing pointed object. + * + * Call sites should pass arguments that make the intended behavior explicit. * @param has_detach whether the object can be detached * @return the v_object's shared_ptr */ - template std::shared_ptr create(bool has_detach, Args&&... args); + template std::shared_ptr pull(bool has_detach, Args&&... args); /** * Try locating an existing instance in cache @@ -72,12 +79,15 @@ namespace db0 std::shared_ptr tryFind(std::uint64_t address) const; /** - * Either locate existing instance in cache or create a new one + * Either locate existing instance in cache or pull a wrapper using the supplied + * constructor arguments. This is intended for types such as MorphingBIndex where + * construction with the forwarded arguments opens existing state instead of + * allocating new persisted state. * @param address the instance address - * @param has_detach whether the object can be detached + * @param has_detach whether the object can be detached * @return the v_object's shared_ptr */ - template std::shared_ptr findOrCreate(std::uint64_t address, + template std::shared_ptr findOrPull(std::uint64_t address, bool has_detach, Args&&... args); /** @@ -94,6 +104,10 @@ namespace db0 FixedObjectList &getSharedObjectList() const; + Memspace &getMemspace() const { + return m_memspace; + } + void beginAtomic(); void endAtomic(); void cancelAtomic(); @@ -111,14 +125,14 @@ namespace db0 }; template - std::shared_ptr VObjectCache::create(bool has_detach, Args&&... args) + std::shared_ptr VObjectCache::pull(bool has_detach, Args&&... args) { if (m_shared_object_list.full()) { // remove 1/4 of cached objects once the max_size is reached m_shared_object_list.eraseItems((m_shared_object_list.size() >> 2) + 1); } - auto ptr = make_shared_void(m_memspace, std::forward(args)...); + auto ptr = make_shared_void(std::forward(args)...); // note that the index may be at any moment released and reused by other item auto index = m_shared_object_list.append(ptr); auto result_ptr = std::static_pointer_cast(ptr); @@ -141,7 +155,7 @@ namespace db0 } template - std::shared_ptr VObjectCache::findOrCreate(std::uint64_t address, bool has_detach, Args&&... args) + std::shared_ptr VObjectCache::findOrPull(std::uint64_t address, bool has_detach, Args&&... args) { auto it = m_cache.find(address); if (it != m_cache.end()) { @@ -157,9 +171,9 @@ namespace db0 m_cache.erase(it); } } - return create(has_detach, std::forward(args)...); + return pull(has_detach, std::forward(args)...); } - + template std::shared_ptr VObjectCache::tryFind(std::uint64_t address) const { @@ -179,4 +193,4 @@ namespace db0 return std::static_pointer_cast(lock); } -} \ No newline at end of file +} diff --git a/src/dbzero/object_model/tags/CompositeTagDef.cpp b/src/dbzero/object_model/tags/CompositeTagDef.cpp new file mode 100644 index 00000000..4aa62322 --- /dev/null +++ b/src/dbzero/object_model/tags/CompositeTagDef.cpp @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright (c) 2026 DBZero Software sp. z o.o. + +#include "CompositeTagDef.hpp" +#include + +namespace db0::object_model + +{ + + CompositeTagDef::CompositeTagDef(std::vector &&items) + : m_items(std::move(items)) + { + } + + const std::vector &CompositeTagDef::getItems() const + { + return m_items; + } + + std::size_t CompositeTagDef::size() const + { + return m_items.size(); + } + + bool CompositeTagDef::operator==(const CompositeTagDef &other) const + { + if (m_items.size() != other.m_items.size()) { + return false; + } + for (std::size_t i = 0; i < m_items.size(); ++i) { + if (m_items[i].get() != other.m_items[i].get()) { + return false; + } + } + return true; + } + + bool CompositeTagDef::operator!=(const CompositeTagDef &other) const + { + return !(*this == other); + } + + std::uint64_t CompositeTagDef::getHash() const + { + std::uint64_t result = 1469598103934665603ull; + for (auto const &item: m_items) { + result ^= static_cast(reinterpret_cast(item.get())); + result *= 1099511628211ull; + } + return result; + } + +} diff --git a/src/dbzero/object_model/tags/CompositeTagDef.hpp b/src/dbzero/object_model/tags/CompositeTagDef.hpp new file mode 100644 index 00000000..d790be00 --- /dev/null +++ b/src/dbzero/object_model/tags/CompositeTagDef.hpp @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright (c) 2026 DBZero Software sp. z o.o. + +#pragma once + +#include +#include +#include + +namespace db0::object_model + +{ + + class CompositeTagDef + { + public: + using LangToolkit = LangConfig::LangToolkit; + using ObjectPtr = typename LangToolkit::ObjectPtr; + using ObjectSharedPtr = typename LangToolkit::ObjectSharedPtr; + + CompositeTagDef() = default; + explicit CompositeTagDef(std::vector &&items); + + const std::vector &getItems() const; + std::size_t size() const; + + bool operator==(const CompositeTagDef &other) const; + bool operator!=(const CompositeTagDef &other) const; + std::uint64_t getHash() const; + + private: + std::vector m_items; + }; + +} diff --git a/src/dbzero/object_model/tags/ObjectTagManager.cpp b/src/dbzero/object_model/tags/ObjectTagManager.cpp index 07c8847d..92709e56 100644 --- a/src/dbzero/object_model/tags/ObjectTagManager.cpp +++ b/src/dbzero/object_model/tags/ObjectTagManager.cpp @@ -5,10 +5,48 @@ #include #include #include +#include +#include namespace db0::object_model { + namespace + { + bool isCompositeTag(ObjectTagManager::ObjectPtr arg) + { + return db0::python::PyCompositeTag_Check(arg) || + (PyTuple_Check(arg) && ObjectTagManager::LangToolkit::length(arg) >= 2); + } + + std::size_t compositeTagSize(ObjectTagManager::ObjectPtr arg) + { + if (db0::python::PyCompositeTag_Check(arg)) { + return reinterpret_cast(arg)->ext().size(); + } + return ObjectTagManager::LangToolkit::length(arg); + } + + ObjectTagManager::ObjectSharedPtr getCompositeItem(ObjectTagManager::ObjectPtr arg, std::size_t index) + { + if (db0::python::PyCompositeTag_Check(arg)) { + return reinterpret_cast(arg)->ext().getItems()[index]; + } + return ObjectTagManager::LangToolkit::getItem(arg, index); + } + + void validateCompositeTag(ObjectTagManager::ObjectPtr arg) + { + auto length = compositeTagSize(arg); + for (std::size_t i = 0; i < length; ++i) { + auto item = getCompositeItem(arg, i); + if (isCompositeTag(item.get())) { + THROWF(db0::InputException) << "Nested composite tags are not supported" << THROWF_END; + } + } + } + + } ObjectTagManager::ObjectTagManager(ObjectPtr const *memo_ptr, std::size_t nargs) : m_info(memo_ptr[0]) @@ -59,18 +97,45 @@ namespace db0::object_model , m_has_tags(LangToolkit::hasTagRefs(memo_ptr)) { } + + bool ObjectTagManager::ObjectInfo::hasCompositeTags(ObjectPtr const *args, Py_ssize_t nargs) const + { + for (Py_ssize_t i = 0; i < nargs; ++i) { + if (isCompositeTag(args[i])) { + return true; + } + } + return false; + } void ObjectTagManager::ObjectInfo::add(ObjectPtr const *args, Py_ssize_t nargs) { assert(m_tag_index_ptr); + auto &tag_index = *m_tag_index_ptr; assert(m_access_mode == AccessType::READ_WRITE); - m_tag_index_ptr->addTags(m_lang_ptr.get(), args, nargs); + + if (!hasCompositeTags(args, nargs)) { + tag_index.addTags(m_lang_ptr.get(), args, nargs); + } else { + for (Py_ssize_t i = 0; i < nargs; ++i) { + if (isCompositeTag(args[i])) { + validateCompositeTag(args[i]); + } + } + for (Py_ssize_t i = 0; i < nargs; ++i) { + if (isCompositeTag(args[i])) { + addComposite(args[i]); + } else { + tag_index.addTags(m_lang_ptr.get(), args + i, 1); + } + } + } // assign default tags (only when adding the first tag) if (!m_has_tags) { auto type = m_type; while (type) { // also add type as tag (once) - m_tag_index_ptr->addTag(m_lang_ptr.get(), type->getAddress(), true); + tag_index.addTag(m_lang_ptr.get(), type->getAddress(), true); type = type->tryGetBaseClass(); } m_has_tags = true; @@ -80,7 +145,72 @@ namespace db0::object_model void ObjectTagManager::ObjectInfo::remove(ObjectPtr const *args, Py_ssize_t nargs) { assert(m_access_mode == AccessType::READ_WRITE); - m_tag_index_ptr->removeTags(m_lang_ptr.get(), args, nargs); + assert(m_tag_index_ptr); + auto &tag_index = *m_tag_index_ptr; + + if (!hasCompositeTags(args, nargs)) { + tag_index.removeTags(m_lang_ptr.get(), args, nargs); + } else { + for (Py_ssize_t i = 0; i < nargs; ++i) { + if (isCompositeTag(args[i])) { + validateCompositeTag(args[i]); + } + } + for (Py_ssize_t i = 0; i < nargs; ++i) { + if (isCompositeTag(args[i])) { + removeComposite(args[i]); + } else { + tag_index.removeTags(m_lang_ptr.get(), args + i, 1); + } + } + } + } + + void ObjectTagManager::ObjectInfo::addComposite(ObjectPtr arg) + { + assert(m_tag_index_ptr); + assert(isCompositeTag(arg)); + auto length = compositeTagSize(arg); + assert(length >= 2); + validateCompositeTag(arg); + + std::shared_ptr currentTagIndexPtr; + auto *currentTagIndex = m_tag_index_ptr; + for (std::size_t i = 0; i + 1 < length; ++i) { + auto item = getCompositeItem(arg, i); + auto key = currentTagIndex->addCompositeKey(item.get()); + currentTagIndexPtr = currentTagIndex->addComposite(m_lang_ptr.get(), key); + currentTagIndex = currentTagIndexPtr.get(); + } + + auto tagPtr = getCompositeItem(arg, length - 1); + ObjectPtr tag = tagPtr.get(); + currentTagIndex->addTags(m_lang_ptr.get(), &tag, 1); + } + + void ObjectTagManager::ObjectInfo::removeComposite(ObjectPtr arg) + { + assert(m_tag_index_ptr); + assert(isCompositeTag(arg)); + auto length = compositeTagSize(arg); + assert(length >= 2); + validateCompositeTag(arg); + + std::shared_ptr currentTagIndexPtr; + auto *currentTagIndex = m_tag_index_ptr; + for (std::size_t i = 0; i + 1 < length; ++i) { + auto item = getCompositeItem(arg, i); + auto key = currentTagIndex->getCompositeKey(item.get()); + currentTagIndexPtr = currentTagIndex->tryUpdateComposite(m_lang_ptr.get(), key); + if (!currentTagIndexPtr) { + return; + } + currentTagIndex = currentTagIndexPtr.get(); + } + + auto tagPtr = getCompositeItem(arg, length - 1); + ObjectPtr tag = tagPtr.get(); + currentTagIndex->removeTags(m_lang_ptr.get(), &tag, 1); } void ObjectTagManager::add(ObjectPtr const *args, Py_ssize_t nargs) @@ -133,4 +263,4 @@ namespace db0::object_model } } -} \ No newline at end of file +} diff --git a/src/dbzero/object_model/tags/ObjectTagManager.hpp b/src/dbzero/object_model/tags/ObjectTagManager.hpp index ea185e4e..92091a61 100644 --- a/src/dbzero/object_model/tags/ObjectTagManager.hpp +++ b/src/dbzero/object_model/tags/ObjectTagManager.hpp @@ -58,6 +58,9 @@ namespace db0::object_model void add(ObjectPtr const *args, Py_ssize_t nargs); void remove(ObjectPtr const *args, Py_ssize_t nargs); + bool hasCompositeTags(ObjectPtr const *args, Py_ssize_t nargs) const; + void addComposite(ObjectPtr); + void removeComposite(ObjectPtr); db0::swine_ptr getFixture() const; }; diff --git a/src/dbzero/object_model/tags/TagIndex.cpp b/src/dbzero/object_model/tags/TagIndex.cpp index c1850611..c0bb1800 100644 --- a/src/dbzero/object_model/tags/TagIndex.cpp +++ b/src/dbzero/object_model/tags/TagIndex.cpp @@ -4,6 +4,7 @@ #include "TagIndex.hpp" #include "ObjectIterator.hpp" #include "OR_QueryObserver.hpp" +#include "CompositeTagDef.hpp" #include #include #include @@ -133,8 +134,10 @@ namespace db0::object_model , m_string_pool(string_pool) , m_class_factory(class_factory) , m_enum_factory(enum_factory) + , m_cache(cache) , m_base_index_short(memspace, cache) , m_base_index_long(memspace, cache) + , m_short_tag_index_map(std::make_unique(memspace, cache)) , m_fixture(enum_factory.getFixture()) , m_fixture_uuid(enum_factory.getFixture()->getUUID()) , m_mutation_log(mutation_log) @@ -142,6 +145,7 @@ namespace db0::object_model assert(mutation_log); modify().m_base_index_short_ptr = m_base_index_short.getAddress(); modify().m_base_index_long_ptr = m_base_index_long.getAddress(); + modify().m_reserved[0] = m_short_tag_index_map->getAddress().getOffset(); } TagIndex::TagIndex(mptr ptr, ClassFactory &class_factory, EnumFactory &enum_factory, @@ -150,6 +154,7 @@ namespace db0::object_model , m_string_pool(string_pool) , m_class_factory(class_factory) , m_enum_factory(enum_factory) + , m_cache(cache) , m_base_index_short(myPtr((*this)->m_base_index_short_ptr), cache) , m_base_index_long(myPtr((*this)->m_base_index_long_ptr), cache) , m_fixture(enum_factory.getFixture()) @@ -157,6 +162,12 @@ namespace db0::object_model , m_mutation_log(mutation_log) { assert(mutation_log); + if ((*this)->m_reserved[0] != 0) { + m_short_tag_index_map = std::make_unique( + myPtr(Address::fromOffset((*this)->m_reserved[0])), + cache + ); + } } TagIndex::~TagIndex() @@ -268,6 +279,56 @@ namespace db0::object_model batch_operation->addTags(active_key, TagPtrSequence(&tag, &tag + 1)); m_mutation_log->onDirty(); } + + std::shared_ptr TagIndex::addComposite(ObjectPtr, ShortTagT tag) + { + return getShortTagIndexMap().findOrCreate( + tag, + m_class_factory, + m_enum_factory, + m_string_pool, + m_cache, + m_mutation_log + ); + } + + std::shared_ptr TagIndex::tryUpdateComposite(ObjectPtr, ShortTagT tag) + { + if (!m_short_tag_index_map) { + return nullptr; + } + + return m_short_tag_index_map->tryGet( + tag, + m_class_factory, + m_enum_factory, + m_string_pool, + m_cache, + m_mutation_log + ); + } + + TagIndex::ShortTagT TagIndex::addCompositeKey(ObjectPtr arg) + { + bool inc_ref = false; + auto result = tryAddShortTag(arg, inc_ref); + if (!result) { + THROWF(db0::InputException) << "Unable to add foreign composite tag"; + } + if (inc_ref) { + m_inc_refed_tags.insert(*result); + } + return *result; + } + + TagIndex::ShortTagT TagIndex::getCompositeKey(ObjectPtr arg) const + { + auto result = tryGetCompositeKey(arg); + if (!result) { + THROWF(db0::InputException) << "Unable to resolve foreign composite tag key" << THROWF_END; + } + return *result; + } void TagIndex::removeTypeTag(UniqueAddress obj_addr, Address tag_addr) { @@ -291,12 +352,12 @@ namespace db0::object_model if (type_id != TypeId::STRING && LangToolkit::isIterable(args[i])) { batch_operation->removeTags(active_key, IterableSequence(LangToolkit::getIterator(args[i]), ForwardIterator::end(), [&](ObjectSharedPtr arg) { - return getShortTag(arg.get()); + return getCompositeKey(arg.get()); }) ); m_mutation_log->onDirty(); } else { - batch_operation->removeTag(active_key, getShortTag(type_id, args[i])); + batch_operation->removeTag(active_key, getCompositeKey(args[i])); m_mutation_log->onDirty(); } } @@ -304,6 +365,12 @@ namespace db0::object_model void TagIndex::rollback() { + if (m_short_tag_index_map) { + m_short_tag_index_map->forEachActive([](TagIndex &tag_index) { + tag_index.rollback(); + }); + } + // Reject any pending updates if (m_batch_op_short) { m_batch_op_short.reset(); @@ -333,6 +400,12 @@ namespace db0::object_model void TagIndex::close() { + if (m_short_tag_index_map) { + m_short_tag_index_map->forEachActive([](TagIndex &tag_index) { + tag_index.close(); + }); + } + if (m_batch_op_short) { m_batch_op_short.reset(); } @@ -364,14 +437,47 @@ namespace db0::object_model } } - void TagIndex::flush() const + bool TagIndex::flush() const { using ShortBatchOperationBulder = db0::FT_BaseIndex::BatchOperationBuilder; + + bool composite_indexes_contain_values = false; + auto hasPersistedValues = [&]() { + return !m_base_index_short.empty() || !m_base_index_long.empty() || composite_indexes_contain_values; + }; + + if (m_short_tag_index_map) { + std::vector emptyCompositeTags; + m_short_tag_index_map->forEachActive([this, &composite_indexes_contain_values, &emptyCompositeTags](TagIndex &tag_index) { + if (tag_index.flush()) { + composite_indexes_contain_values = true; + return; + } + + auto tag_index_address = tag_index.getAddress(); + for (auto it = m_short_tag_index_map->begin(); it != m_short_tag_index_map->end(); ++it) { + if ((*it).value == tag_index_address) { + emptyCompositeTags.push_back((*it).key); + return; + } + } + }); + for (auto tag: emptyCompositeTags) { + m_short_tag_index_map->erase( + tag, + m_class_factory, + m_enum_factory, + m_string_pool, + m_cache, + m_mutation_log + ); + } + } if (empty()) { - return; + return hasPersistedValues(); } - + // this is to resolve addresses of incomplete objects (must be done before flushing) buildActiveValues(); auto &type_manager = LangToolkit::getTypeManager(); @@ -480,6 +586,7 @@ namespace db0::object_model } } m_inc_refed_tags.clear(); + return hasPersistedValues(); } void TagIndex::buildActiveValues() const @@ -550,6 +657,10 @@ namespace db0::object_model using IterableSequence = TagMakerSequence; auto type_id = LangToolkit::getTypeManager().getTypeId(arg); + if (type_id == TypeId::DB0_COMPOSITE_TAG) { + return addCompositeIterator(LangToolkit::getTypeManager().extractCompositeTag(arg), factory, query_observers); + } + // simple tag-convertible type if (type_id == TypeId::STRING || type_id == TypeId::DB0_TAG || type_id == TypeId::DB0_ENUM_VALUE || type_id == TypeId::DB0_CLASS) @@ -657,6 +768,79 @@ namespace db0::object_model THROWF(db0::InputException) << "Unable to interpret object of type: " << LangToolkit::getTypeName(arg) << " as a query" << THROWF_END; } + + bool TagIndex::addCompositeIterator(const CompositeTagDef &tag, + db0::FT_IteratorFactory &factory, + std::vector > &query_observers) const + { + if (tag.size() < 2 || !m_short_tag_index_map) { + return false; + } + + auto const &items = tag.getItems(); + auto firstKey = tryGetCompositeKey(items[0].get()); + if (!firstKey) { + return false; + } + + auto currentTagIndexPtr = m_short_tag_index_map->tryGet( + *firstKey, + m_class_factory, + m_enum_factory, + m_string_pool, + m_cache, + m_mutation_log + ); + if (!currentTagIndexPtr) { + return false; + } + + auto *currentTagIndex = currentTagIndexPtr.get(); + for (std::size_t i = 1; i + 1 < items.size(); ++i) { + auto key = currentTagIndex->tryGetCompositeKey(items[i].get()); + if (!key || !currentTagIndex->m_short_tag_index_map) { + return false; + } + currentTagIndexPtr = currentTagIndex->m_short_tag_index_map->tryGet( + *key, + currentTagIndex->m_class_factory, + currentTagIndex->m_enum_factory, + currentTagIndex->m_string_pool, + currentTagIndex->m_cache, + currentTagIndex->m_mutation_log + ); + if (!currentTagIndexPtr) { + return false; + } + currentTagIndex = currentTagIndexPtr.get(); + } + + std::vector > negIterators; + auto result = currentTagIndex->addIterator(items.back().get(), factory, negIterators, query_observers); + if (!negIterators.empty()) { + THROWF(db0::InputException) << "Negated composite tag leaves are not supported" << THROWF_END; + } + return result; + } + + std::optional TagIndex::tryGetCompositeKey(ObjectPtr arg) const + { + using TypeId = db0::bindings::TypeId; + + auto typeId = LangToolkit::getTypeManager().getTypeId(arg); + if (typeId == TypeId::MEMO_OBJECT) { + return tryAddShortTagFromMemo(arg); + } + if (typeId == TypeId::DB0_TAG) { + return tryAddShortTagFromTag(arg); + } + if (typeId == TypeId::STRING || typeId == TypeId::DB0_ENUM_VALUE || typeId == TypeId::DB0_ENUM_VALUE_REPR || + typeId == TypeId::DB0_FIELD_DEF || typeId == TypeId::DB0_CLASS) + { + return getShortTag(typeId, arg); + } + return std::nullopt; + } TagIndex::ShortTagT TagIndex::getShortTag(TypeId type_id, ObjectPtr py_arg, ObjectSharedPtr *alt_repr) const { @@ -949,6 +1133,10 @@ namespace db0::object_model flush(); m_base_index_short.commit(); m_base_index_long.commit(); + // no need to commit invdividual nested intances since they're managed by cache + if (m_short_tag_index_map) { + m_short_tag_index_map->commit(); + } super_t::commit(); } @@ -956,6 +1144,10 @@ namespace db0::object_model { m_base_index_short.detach(); m_base_index_long.detach(); + // no need to detach invdividual nested intances since they're managed by cache + if (m_short_tag_index_map) { + m_short_tag_index_map->detach(); + } super_t::detach(); } @@ -971,6 +1163,31 @@ namespace db0::object_model return m_base_index_long; } + TagIndex::ShortTagIndexMap *TagIndex::tryGetShortTagIndexMap() { + return m_short_tag_index_map.get(); + } + + const TagIndex::ShortTagIndexMap *TagIndex::tryGetShortTagIndexMap() const { + return m_short_tag_index_map.get(); + } + + TagIndex::ShortTagIndexMap &TagIndex::getShortTagIndexMap() + { + if (!m_short_tag_index_map) { + m_short_tag_index_map = std::make_unique(getMemspace(), m_cache); + modify().m_reserved[0] = m_short_tag_index_map->getAddress().getOffset(); + m_mutation_log->onDirty(); + } + return *m_short_tag_index_map; + } + + const TagIndex::ShortTagIndexMap &TagIndex::getShortTagIndexMap() const { + if (!m_short_tag_index_map) { + THROWF(db0::InternalException) << "Short tag index map not initialized" << THROWF_END; + } + return *m_short_tag_index_map; + } + std::unique_ptr TagIndex::makeIterator(ObjectPtr obj_ptr) const { assert(obj_ptr); return makeIterator(getShortTag(obj_ptr)); @@ -1132,17 +1349,35 @@ namespace db0::object_model } bool TagIndex::empty() const { - return m_batch_op_short.empty() && m_batch_op_long.empty() && m_batch_op_types.empty(); + if (!m_batch_op_short.empty() || !m_batch_op_long.empty() || !m_batch_op_types.empty()) { + return false; + } + + if (m_short_tag_index_map) { + bool all_composite_indexes_empty = true; + m_short_tag_index_map->forEachActive([&all_composite_indexes_empty](TagIndex &tag_index) { + if (!tag_index.empty()) { + all_composite_indexes_empty = false; + return false; + } + return true; + }); + return all_composite_indexes_empty; + } + + return true; } bool TagIndex::assureEmpty() const { - if (empty()) { - m_batch_op_short.clear(); - m_batch_op_long.clear(); - m_batch_op_types.clear(); + if (!m_batch_op_short.empty() || !m_batch_op_long.empty() || !m_batch_op_types.empty()) { + return false; } - return false; + + m_batch_op_short.clear(); + m_batch_op_long.clear(); + m_batch_op_types.clear(); + return true; } bool isObjectPendingUpdate(db0::swine_ptr &fixture, UniqueAddress addr) @@ -1182,4 +1417,4 @@ namespace db0::object_model ); } -} \ No newline at end of file +} diff --git a/src/dbzero/object_model/tags/TagIndex.hpp b/src/dbzero/object_model/tags/TagIndex.hpp index 6600b959..dc456534 100644 --- a/src/dbzero/object_model/tags/TagIndex.hpp +++ b/src/dbzero/object_model/tags/TagIndex.hpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ namespace db0::object_model using RC_LimitedStringPool = db0::pools::RC_LimitedStringPool; using LongTagT = db0::LongTagT; class EnumFactory; + class CompositeTagDef; DB0_PACKED_BEGIN struct DB0_PACKED_ATTR o_tag_index: public o_fixed_versioned @@ -52,6 +54,7 @@ DB0_PACKED_END using TP_Iterator = TagProduct; // string tokens and classes are represented as short tags using ShortTagT = std::uint64_t; + using ShortTagIndexMap = db0::VInstanceMap; TagIndex(Memspace &memspace, ClassFactory &, EnumFactory &, RC_LimitedStringPool &, VObjectCache &, std::shared_ptr mutation_log); @@ -63,6 +66,12 @@ DB0_PACKED_END // @param is_type true for implicilty assigned type tags void addTag(ObjectPtr memo_ptr, ShortTagT tag_addr, bool is_type); void addTag(ObjectPtr memo_ptr, Address tag_addr, bool is_type); + // add the high-order part of the composite tag + std::shared_ptr addComposite(ObjectPtr memo_ptr, ShortTagT); + // For modifying composite tags + std::shared_ptr tryUpdateComposite(ObjectPtr memo_ptr, ShortTagT); + ShortTagT addCompositeKey(ObjectPtr); + ShortTagT getCompositeKey(ObjectPtr) const; // add a tag using long identifier void addTag(ObjectPtr memo_ptr, LongTagT tag_addr); @@ -96,8 +105,9 @@ DB0_PACKED_END // Clears the uncommited contents (rollback) void rollback(); - // Flush any pending updates from the internal buffers - void flush() const; + // Flush any pending updates from the internal buffers. + // @return true if this index or any active composite child index contains persisted entries after flushing. + bool flush() const; // Close tag index without flushing any pending updates void close(); @@ -109,6 +119,10 @@ DB0_PACKED_END db0::FT_BaseIndex &getBaseIndexShort(); const db0::FT_BaseIndex &getBaseIndexShort() const; const db0::FT_BaseIndex &getBaseIndexLong() const; + ShortTagIndexMap *tryGetShortTagIndexMap(); + const ShortTagIndexMap *tryGetShortTagIndexMap() const; + ShortTagIndexMap &getShortTagIndexMap(); + const ShortTagIndexMap &getShortTagIndexMap() const; // add a defunct object (failed on __init__) void addDefunct(ObjectPtr memo_ptr) const; @@ -139,8 +153,11 @@ DB0_PACKED_END RC_LimitedStringPool &m_string_pool; ClassFactory &m_class_factory; EnumFactory &m_enum_factory; + VObjectCache &m_cache; db0::FT_BaseIndex m_base_index_short; db0::FT_BaseIndex m_base_index_long; + // For composite tags + std::unique_ptr m_short_tag_index_map; // Current batch-operation buffer (may not be initialized) mutable db0::FT_BaseIndex::BatchOperationBuilder m_batch_op_short; mutable db0::FT_BaseIndex::BatchOperationBuilder m_batch_op_long; @@ -210,6 +227,9 @@ DB0_PACKED_END bool addIterator(ObjectPtr, db0::FT_IteratorFactory &factory, std::vector > &neg_iterators, std::vector > &query_observers) const; + bool addCompositeIterator(const CompositeTagDef &, db0::FT_IteratorFactory &factory, + std::vector > &query_observers) const; + std::optional tryGetCompositeKey(ObjectPtr) const; bool isShortTag(ObjectPtr) const; bool isShortTag(ObjectSharedPtr) const; diff --git a/tests/unit_tests/VInstanceMapTest.cpp b/tests/unit_tests/VInstanceMapTest.cpp new file mode 100644 index 00000000..717a845f --- /dev/null +++ b/tests/unit_tests/VInstanceMapTest.cpp @@ -0,0 +1,682 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +// Copyright (c) 2026 DBZero Software sp. z o.o. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" PyObject *PyInit_dbzero(void); + +namespace tests + +{ + + using namespace db0; + +DB0_PACKED_BEGIN + struct DB0_PACKED_ATTR o_v_instance_map_value: db0::o_fixed + { + std::uint32_t m_value = 0; + std::uint32_t m_extra = 0; + + o_v_instance_map_value() = default; + + o_v_instance_map_value(std::uint32_t value, std::uint32_t extra) + : m_value(value) + , m_extra(extra) + { + } + }; +DB0_PACKED_END + + class VInstanceMapTest: public MemspaceTestBase + { + protected: + VInstanceMapTest() + : m_shared_object_list(1) + , m_memspace(getMemspace()) + , m_cache(m_memspace, m_shared_object_list) + { + } + + FixedObjectList m_shared_object_list; + Memspace m_memspace; + VObjectCache m_cache; + }; + + TEST_F(VInstanceMapTest, testCanInsertAndLookupInstancesByNumericKey) + { + using ValueT = db0::v_object; + + VInstanceMap cut(m_memspace, m_cache); + auto inserted = cut.insert(42, 691, 17); + + auto found = cut.get(42); + ASSERT_TRUE(found); + ASSERT_EQ(inserted->getAddress(), found->getAddress()); + ASSERT_EQ(691u, (*found)->m_value); + ASSERT_EQ(17u, (*found)->m_extra); + } + + TEST_F(VInstanceMapTest, testKeysAreStoredInAscendingOrder) + { + using ValueT = db0::v_object; + + VInstanceMap cut(m_memspace, m_cache); + cut.insert(30, 3, 30); + cut.insert(10, 1, 10); + cut.insert(20, 2, 20); + + auto it = cut.begin(); + ASSERT_EQ(10u, (*it).key); + ++it; + ASSERT_EQ(20u, (*it).key); + ++it; + ASSERT_EQ(30u, (*it).key); + } + + TEST_F(VInstanceMapTest, testEraseKeyRemovesTheMapping) + { + using ValueT = db0::v_object; + + VInstanceMap cut(m_memspace, m_cache); + cut.insert(1, 10, 1); + cut.insert(2, 20, 2); + + ASSERT_TRUE(cut.erase(1)); + ASSERT_FALSE(cut.tryGet(1)); + ASSERT_TRUE(cut.tryGet(2)); + ASSERT_FALSE(cut.erase(3)); + } + + TEST_F(VInstanceMapTest, testEraseKeyDestroysUnderlyingInstance) + { + using ValueT = db0::v_object; + + VInstanceMap cut(m_memspace, m_cache); + Address value_address; + { + auto inserted = cut.insert(1, 10, 1); + value_address = inserted->getAddress(); + } + + ASSERT_NO_THROW(m_memspace.getAllocator().getAllocSize(value_address)); + ASSERT_TRUE(cut.erase(1)); + ASSERT_FALSE(cut.tryGet(1)); + ASSERT_ANY_THROW(m_memspace.getAllocator().getAllocSize(value_address)); + } + + TEST_F(VInstanceMapTest, testReplacingExistingKeyDestroysOldInstance) + { + using ValueT = db0::v_object; + + VInstanceMap cut(m_memspace, m_cache); + auto first = cut.insert(1, 10, 1); + auto first_address = first->getAddress(); + auto second = cut.insert(1, 20, 2); + + ASSERT_NE(first_address, second->getAddress()); + ASSERT_EQ(second->getAddress(), cut.get(1)->getAddress()); + ASSERT_ANY_THROW(m_memspace.getAllocator().getAllocSize(first_address)); + } + + TEST_F(VInstanceMapTest, testCanReopenInstanceAfterWeakCacheEntryExpired) + { + using ValueT = db0::v_object; + + VInstanceMap cut(m_memspace, m_cache); + { + auto inserted = cut.insert(5, 81392, 99); + ASSERT_EQ(81392u, (*inserted)->m_value); + } + + auto reopened = cut.get(5); + ASSERT_TRUE(reopened); + ASSERT_EQ(81392u, (*reopened)->m_value); + } + + TEST_F(VInstanceMapTest, testFindOrCreateReusesExistingValue) + { + using ValueT = db0::v_object; + + VInstanceMap cut(m_memspace, m_cache); + auto first = cut.findOrCreate(7, 100, 1); + auto second = cut.findOrCreate(7, 200, 2); + + ASSERT_EQ(first->getAddress(), second->getAddress()); + ASSERT_EQ(100u, (*second)->m_value); + ASSERT_EQ(1u, (*second)->m_extra); + } + + TEST_F(VInstanceMapTest, testForAllVisitsActiveValuesEvenAfterVObjectCacheEviction) + { + using ValueT = db0::v_object; + + VInstanceMap cut(m_memspace, m_cache); + auto first = cut.insert(1, 10, 1); + auto second = cut.insert(2, 20, 2); + + std::set values; + auto active_count = cut.forEachActive([&](ValueT &value) { + values.insert(value->m_value); + }); + + ASSERT_EQ(2u, active_count); + ASSERT_EQ(values, (std::set { 10, 20 })); + ASSERT_TRUE(first); + ASSERT_TRUE(second); + } + + TEST_F(VInstanceMapTest, testForAllCleansExpiredWeakValues) + { + using ValueT = db0::v_object; + + VInstanceMap cut(m_memspace, m_cache); + { + auto first = cut.insert(1, 10, 1); + ASSERT_TRUE(first); + } + auto second = cut.insert(2, 20, 2); + + std::vector values; + auto active_count = cut.forEachActive([&](ValueT &value) { + values.push_back(value->m_value); + }); + + ASSERT_EQ(1u, active_count); + ASSERT_EQ(values, (std::vector { 20 })); + + std::size_t second_pass_count = cut.forEachActive([](ValueT &) {}); + ASSERT_EQ(1u, second_pass_count); + ASSERT_TRUE(second); + } + + TEST_F(VInstanceMapTest, testForAllCanStopEarly) + { + using ValueT = db0::v_object; + + VInstanceMap cut(m_memspace, m_cache); + auto first = cut.insert(10, 1, 10); + auto second = cut.insert(20, 2, 20); + + std::size_t visited = 0; + auto active_count = cut.forEachActive([&](ValueT &) { + ++visited; + return false; + }); + + ASSERT_EQ(1u, active_count); + ASSERT_EQ(1u, visited); + ASSERT_TRUE(first); + ASSERT_TRUE(second); + } + + class VInstanceMapTagIndexTest: public FixtureTestBase + { + }; + + void initializeDbzeroPythonBindingsOnce() + { + static bool py_dbzero_initialized = false; + if (!py_dbzero_initialized) { + auto py_dbzero_module = Py_OWN(PyInit_dbzero()); + ASSERT_TRUE(py_dbzero_module.get()); + py_dbzero_initialized = true; + } + } + + db0::python::shared_py_object makeMaterializedMemo(db0::swine_ptr &fixture) + { + initializeDbzeroPythonBindingsOnce(); + + static std::uint64_t memo_type_index = 0; + auto class_name = "VInstanceMapCompositeMemo" + std::to_string(memo_type_index); + auto type_id = "tests/" + class_name; + ++memo_type_index; + + if (PyRun_SimpleString(("class " + class_name + ": pass\n").c_str()) != 0) { + ADD_FAILURE() << "Failed to define Python memo test class"; + return {}; + } + auto main_module = Py_BORROW(PyImport_AddModule("__main__")); + EXPECT_TRUE(main_module.get()); + auto py_class = Py_OWN(PyObject_GetAttrString(*main_module, class_name.c_str())); + EXPECT_TRUE(py_class.get()); + auto args = Py_OWN(PyTuple_Pack(1, py_class.get())); + auto kwargs = Py_OWN(PyDict_New()); + EXPECT_TRUE(args.get()); + EXPECT_TRUE(kwargs.get()); + auto py_type_id = Py_OWN(PyUnicode_FromString(type_id.c_str())); + EXPECT_TRUE(py_type_id.get()); + if (PyDict_SetItemString(*kwargs, "id", py_type_id.get()) != 0) { + ADD_FAILURE() << "Failed to set Python memo type id"; + return {}; + } + auto py_memo_type = Py_OWN(db0::python::PyAPI_wrapPyClass(nullptr, args.get(), kwargs.get())); + EXPECT_TRUE(py_memo_type.get()); + + auto object_type = getTestClass(fixture); + auto memo_ptr = Py_OWN(db0::python::MemoObjectStub_new(reinterpret_cast(py_memo_type.get()))); + memo_ptr->makeNew(object_type); + { + FixtureLock lock(fixture); + memo_ptr->modifyExt().postInit(lock); + } + return memo_ptr; + } + + void assertTagIndexContainsObject(db0::object_model::TagIndex &tag_index, + db0::object_model::TagIndex::ShortTagT tag, UniqueAddress object_address) + { + auto query = tag_index.makeIterator(tag); + ASSERT_FALSE(query->isEnd()); + UniqueAddress result; + query->next(&result); + ASSERT_EQ(object_address, result); + ASSERT_TRUE(query->isEnd()); + } + + void assertTagIndexDoesNotContainObject(db0::object_model::TagIndex &tag_index, + db0::object_model::TagIndex::ShortTagT tag) + { + auto query = tag_index.makeIterator(tag); + ASSERT_TRUE(query->isEnd()); + } + + TEST_F(VInstanceMapTagIndexTest, testCanStoreTagIndexInstances) + { + using TagIndex = db0::object_model::TagIndex; + + auto fixture = getFixture(); + db0::object_model::ClassFactory class_factory(fixture); + db0::object_model::EnumFactory enum_factory(fixture); + auto mutation_log = fixture->addMutationHandler(); + + VInstanceMap cut(*fixture, fixture->getVObjectCache()); + auto inserted = cut.insert( + 11, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + + auto found = cut.get( + 11, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + + ASSERT_TRUE(found); + ASSERT_EQ(inserted->getAddress(), found->getAddress()); + ASSERT_TRUE(found->empty()); + } + + TEST_F(VInstanceMapTagIndexTest, testTagIndexPersistsShortTagIndexMapAddress) + { + using TagIndex = db0::object_model::TagIndex; + + auto fixture = getFixture(); + db0::object_model::ClassFactory class_factory(fixture); + db0::object_model::EnumFactory enum_factory(fixture); + auto mutation_log = fixture->addMutationHandler(); + + TagIndex tag_index( + *fixture, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + + ASSERT_TRUE(tag_index.tryGetShortTagIndexMap()); + ASSERT_NE(0u, tag_index->m_reserved[0]); + ASSERT_EQ(tag_index.getShortTagIndexMap().getAddress().getOffset(), tag_index->m_reserved[0]); + + TagIndex reopened( + fixture->myPtr(tag_index.getAddress()), + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + + ASSERT_TRUE(reopened.tryGetShortTagIndexMap()); + ASSERT_EQ(tag_index->m_reserved[0], reopened.getShortTagIndexMap().getAddress().getOffset()); + } + + TEST_F(VInstanceMapTagIndexTest, testTagIndexLazilyCreatesShortTagIndexMapForLegacyZeroAddress) + { + using TagIndex = db0::object_model::TagIndex; + + auto fixture = getFixture(); + db0::object_model::ClassFactory class_factory(fixture); + db0::object_model::EnumFactory enum_factory(fixture); + auto mutation_log = fixture->addMutationHandler(); + + TagIndex tag_index( + *fixture, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + tag_index.modify().m_reserved[0] = 0; + + TagIndex reopened( + fixture->myPtr(tag_index.getAddress()), + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + + ASSERT_FALSE(reopened.tryGetShortTagIndexMap()); + const TagIndex &const_reopened = reopened; + ASSERT_ANY_THROW(const_reopened.getShortTagIndexMap()); + + auto &short_tag_index_map = reopened.getShortTagIndexMap(); + ASSERT_TRUE(reopened.tryGetShortTagIndexMap()); + ASSERT_NE(0u, reopened->m_reserved[0]); + ASSERT_EQ(short_tag_index_map.getAddress().getOffset(), reopened->m_reserved[0]); + } + + TEST_F(VInstanceMapTagIndexTest, testAddCompositeCreatesChildTagIndex) + { + using TagIndex = db0::object_model::TagIndex; + + Workspace workspace("", {}, {}, {}, {}, db0::object_model::initializer()); + auto fixture = workspace.getFixture("v-instance-map-composite-tag"); + db0::object_model::ClassFactory class_factory(fixture); + db0::object_model::EnumFactory enum_factory(fixture); + auto mutation_log = fixture->addMutationHandler(); + auto memo_ptr = makeMaterializedMemo(fixture); + ASSERT_TRUE(memo_ptr.get()); + + TagIndex tag_index( + *fixture, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + + constexpr TagIndex::ShortTagT composite_tag = 12345; + ASSERT_FALSE(tag_index.tryUpdateComposite(reinterpret_cast(memo_ptr.get()), composite_tag)); + auto child_tag_index = tag_index.addComposite(reinterpret_cast(memo_ptr.get()), composite_tag); + + ASSERT_TRUE(child_tag_index); + ASSERT_EQ( + child_tag_index->getAddress(), + tag_index.getShortTagIndexMap().get( + composite_tag, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + )->getAddress() + ); + + auto update_child_tag_index = tag_index.tryUpdateComposite(reinterpret_cast(memo_ptr.get()), composite_tag); + ASSERT_TRUE(update_child_tag_index); + ASSERT_EQ(child_tag_index->getAddress(), update_child_tag_index->getAddress()); + + ASSERT_TRUE(child_tag_index->empty()); + update_child_tag_index.reset(); + child_tag_index.reset(); + memo_ptr.reset(); + workspace.close(); + } + + TEST_F(VInstanceMapTagIndexTest, testFlushForwardsToCompositeTagIndexes) + { + using TagIndex = db0::object_model::TagIndex; + + Workspace workspace("", {}, {}, {}, {}, db0::object_model::initializer()); + auto fixture = workspace.getFixture("v-instance-map-composite-flush"); + db0::object_model::ClassFactory class_factory(fixture); + db0::object_model::EnumFactory enum_factory(fixture); + auto mutation_log = fixture->addMutationHandler(); + auto memo_ptr = makeMaterializedMemo(fixture); + ASSERT_TRUE(memo_ptr.get()); + + TagIndex tag_index( + *fixture, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + + constexpr TagIndex::ShortTagT composite_tag = 12346; + auto child_tag_index = tag_index.addComposite(reinterpret_cast(memo_ptr.get()), composite_tag); + child_tag_index->addTag(reinterpret_cast(memo_ptr.get()), composite_tag, false); + ASSERT_FALSE(tag_index.empty()); + + ASSERT_TRUE(tag_index.flush()); + + ASSERT_TRUE(tag_index.empty()); + assertTagIndexContainsObject(*child_tag_index, composite_tag, memo_ptr->ext().getUniqueAddress()); + child_tag_index.reset(); + memo_ptr.reset(); + workspace.close(); + } + + TEST_F(VInstanceMapTagIndexTest, testFlushReturnsFalseWhenTagIndexContainsNoElements) + { + using TagIndex = db0::object_model::TagIndex; + + Workspace workspace("", {}, {}, {}, {}, db0::object_model::initializer()); + auto fixture = workspace.getFixture("v-instance-map-composite-flush-empty"); + db0::object_model::ClassFactory class_factory(fixture); + db0::object_model::EnumFactory enum_factory(fixture); + auto mutation_log = fixture->addMutationHandler(); + auto memo_ptr = makeMaterializedMemo(fixture); + ASSERT_TRUE(memo_ptr.get()); + + TagIndex tag_index( + *fixture, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + + ASSERT_FALSE(tag_index.flush()); + + constexpr TagIndex::ShortTagT composite_tag = 12351; + auto child_tag_index = tag_index.addComposite(reinterpret_cast(memo_ptr.get()), composite_tag); + ASSERT_TRUE(child_tag_index->empty()); + ASSERT_FALSE(tag_index.flush()); + ASSERT_FALSE(tag_index.getShortTagIndexMap().tryGet( + composite_tag, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + )); + + child_tag_index.reset(); + memo_ptr.reset(); + workspace.close(); + } + + TEST_F(VInstanceMapTagIndexTest, testRollbackForwardsToCompositeTagIndexes) + { + using TagIndex = db0::object_model::TagIndex; + + Workspace workspace("", {}, {}, {}, {}, db0::object_model::initializer()); + auto fixture = workspace.getFixture("v-instance-map-composite-rollback"); + db0::object_model::ClassFactory class_factory(fixture); + db0::object_model::EnumFactory enum_factory(fixture); + auto mutation_log = fixture->addMutationHandler(); + auto memo_ptr = makeMaterializedMemo(fixture); + ASSERT_TRUE(memo_ptr.get()); + + TagIndex tag_index( + *fixture, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + + constexpr TagIndex::ShortTagT composite_tag = 12347; + auto child_tag_index = tag_index.addComposite(reinterpret_cast(memo_ptr.get()), composite_tag); + child_tag_index->addTag(reinterpret_cast(memo_ptr.get()), composite_tag, false); + ASSERT_FALSE(tag_index.empty()); + + tag_index.rollback(); + + ASSERT_TRUE(tag_index.empty()); + ASSERT_TRUE(child_tag_index->empty()); + child_tag_index.reset(); + memo_ptr.reset(); + workspace.close(); + } + + TEST_F(VInstanceMapTagIndexTest, testCloseForwardsToCompositeTagIndexes) + { + using TagIndex = db0::object_model::TagIndex; + + Workspace workspace("", {}, {}, {}, {}, db0::object_model::initializer()); + auto fixture = workspace.getFixture("v-instance-map-composite-close"); + db0::object_model::ClassFactory class_factory(fixture); + db0::object_model::EnumFactory enum_factory(fixture); + auto mutation_log = fixture->addMutationHandler(); + auto memo_ptr = makeMaterializedMemo(fixture); + ASSERT_TRUE(memo_ptr.get()); + + TagIndex tag_index( + *fixture, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + + constexpr TagIndex::ShortTagT composite_tag = 12348; + auto child_tag_index = tag_index.addComposite(reinterpret_cast(memo_ptr.get()), composite_tag); + child_tag_index->addTag(reinterpret_cast(memo_ptr.get()), composite_tag, false); + ASSERT_FALSE(tag_index.empty()); + + tag_index.close(); + + ASSERT_TRUE(tag_index.empty()); + ASSERT_TRUE(child_tag_index->empty()); + child_tag_index.reset(); + memo_ptr.reset(); + workspace.close(); + } + + TEST_F(VInstanceMapTagIndexTest, testCommitForwardsToCompositeTagIndexes) + { + using TagIndex = db0::object_model::TagIndex; + + Workspace workspace("", {}, {}, {}, {}, db0::object_model::initializer()); + auto fixture = workspace.getFixture("v-instance-map-composite-commit"); + db0::object_model::ClassFactory class_factory(fixture); + db0::object_model::EnumFactory enum_factory(fixture); + auto mutation_log = fixture->addMutationHandler(); + auto memo_ptr = makeMaterializedMemo(fixture); + ASSERT_TRUE(memo_ptr.get()); + + TagIndex tag_index( + *fixture, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + + constexpr TagIndex::ShortTagT composite_tag = 12349; + auto child_tag_index = tag_index.addComposite(reinterpret_cast(memo_ptr.get()), composite_tag); + child_tag_index->addTag(reinterpret_cast(memo_ptr.get()), composite_tag, false); + auto child_address = child_tag_index->getAddress(); + + tag_index.commit(); + + auto reopened_child = tag_index.getShortTagIndexMap().get( + composite_tag, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + ASSERT_EQ(child_address, reopened_child->getAddress()); + assertTagIndexContainsObject(*reopened_child, composite_tag, memo_ptr->ext().getUniqueAddress()); + child_tag_index.reset(); + reopened_child.reset(); + memo_ptr.reset(); + workspace.close(); + } + + TEST_F(VInstanceMapTagIndexTest, testDetachForwardsToCompositeTagIndexes) + { + using TagIndex = db0::object_model::TagIndex; + + Workspace workspace("", {}, {}, {}, {}, db0::object_model::initializer()); + auto fixture = workspace.getFixture("v-instance-map-composite-detach"); + db0::object_model::ClassFactory class_factory(fixture); + db0::object_model::EnumFactory enum_factory(fixture); + auto mutation_log = fixture->addMutationHandler(); + auto memo_ptr = makeMaterializedMemo(fixture); + ASSERT_TRUE(memo_ptr.get()); + + TagIndex tag_index( + *fixture, + class_factory, + enum_factory, + fixture->getLimitedStringPool(), + fixture->getVObjectCache(), + mutation_log + ); + + constexpr TagIndex::ShortTagT composite_tag = 12350; + auto child_tag_index = tag_index.addComposite(reinterpret_cast(memo_ptr.get()), composite_tag); + child_tag_index->addTag(reinterpret_cast(memo_ptr.get()), composite_tag, false); + + tag_index.flush(); + tag_index.detach(); + + assertTagIndexContainsObject(*child_tag_index, composite_tag, memo_ptr->ext().getUniqueAddress()); + child_tag_index.reset(); + memo_ptr.reset(); + workspace.close(); + } + +} diff --git a/tests/unit_tests/VObjectCacheTest.cpp b/tests/unit_tests/VObjectCacheTest.cpp index 24b00c8e..5eb7c65c 100644 --- a/tests/unit_tests/VObjectCacheTest.cpp +++ b/tests/unit_tests/VObjectCacheTest.cpp @@ -37,21 +37,21 @@ namespace tests } }; - TEST_F( VObjectCacheTests , testVObjecCacheCanCreateNewInstance ) + TEST_F( VObjectCacheTests , testVObjecCacheCanPullNewInstance ) { using VType = db0::v_object; auto memspace = getMemspace(); VObjectCache cut(memspace, m_sol); - auto obj = cut.create(true); + auto obj = cut.pull(true, memspace); ASSERT_EQ(691u, (*obj)->m_value); } - TEST_F( VObjectCacheTests , testVObjecCacheCanCreateNewInstanceWithArguments ) + TEST_F( VObjectCacheTests , testVObjecCacheCanPullNewInstanceWithArguments ) { using VType = db0::v_object; auto memspace = getMemspace(); VObjectCache cut(memspace, m_sol); - auto obj = cut.create(true, 81392); + auto obj = cut.pull(true, memspace, 81392); ASSERT_EQ(81392u, (*obj)->m_value); } @@ -61,7 +61,7 @@ namespace tests auto memspace = getMemspace(); VObjectCache cut(memspace, m_sol); // add to cache first - auto address = cut.create(true, 81392)->getAddress(); + auto address = cut.pull(true, memspace, 81392)->getAddress(); auto obj = cut.tryFind(address); ASSERT_TRUE(obj); ASSERT_EQ(81392u, (*obj)->m_value); @@ -73,11 +73,11 @@ namespace tests auto memspace = getMemspace(); VObjectCache cut(memspace, m_sol); std::unordered_set addresses; - addresses.insert(cut.create(true, 81392)->getAddress()); + addresses.insert(cut.pull(true, memspace, 81392)->getAddress()); // add more instances so that the cache size is exhausted // use twice the cache size because extra capacity may be assigned by the implementation for (std::size_t i = 0; i < CACHE_SIZE * 2; ++i) { - addresses.insert(cut.create(i)->getAddress()); + addresses.insert(cut.pull(true, memspace, i)->getAddress()); } int existing_count = 0;