From 97f2ad7f43056a7c8eb7d6d725dcd22c46fa298b Mon Sep 17 00:00:00 2001 From: Adrian Zawadzki Date: Tue, 18 Nov 2025 11:07:52 +0100 Subject: [PATCH 1/3] added exception for cache recycler and cleanup for LangCacheView in onAutocommit --- src/dbzero/core/exception/Exceptions.cpp | 5 +++++ src/dbzero/core/exception/Exceptions.hpp | 7 +++++++ src/dbzero/core/memory/CacheRecycler.cpp | 4 ++++ src/dbzero/workspace/Fixture.cpp | 2 +- 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/dbzero/core/exception/Exceptions.cpp b/src/dbzero/core/exception/Exceptions.cpp index 9f5b33ec..2e6c4b2a 100644 --- a/src/dbzero/core/exception/Exceptions.cpp +++ b/src/dbzero/core/exception/Exceptions.cpp @@ -64,4 +64,9 @@ namespace db0 { } + CacheException::CacheException() + : CriticalException(exception_id) + { + } + } \ No newline at end of file diff --git a/src/dbzero/core/exception/Exceptions.hpp b/src/dbzero/core/exception/Exceptions.hpp index 250f5c43..58e7a50b 100644 --- a/src/dbzero/core/exception/Exceptions.hpp +++ b/src/dbzero/core/exception/Exceptions.hpp @@ -118,4 +118,11 @@ namespace db0 BadAddressException(); }; + class CacheException: public CriticalException + { + public: + static constexpr int exception_id = EXCEPTION_ID_PREFIX::BASIC | 0x0f; + CacheException(); + }; + } \ No newline at end of file diff --git a/src/dbzero/core/memory/CacheRecycler.cpp b/src/dbzero/core/memory/CacheRecycler.cpp index 2851ffab..75584811 100644 --- a/src/dbzero/core/memory/CacheRecycler.cpp +++ b/src/dbzero/core/memory/CacheRecycler.cpp @@ -131,6 +131,10 @@ namespace db0 updateSize(lock, m_capacity - flush_size); flushed = true; flush_result = m_current_size[priority] <= (m_capacity - flush_size); + if(getCurrentSize() >= m_capacity){ + THROWF(db0::CacheException) << "CacheRecycler: unable to free sufficient cache space. " + << "To many memo objects in python scope" << THROWF_END; + } } // resize is a costly operation but cannot be avoided if the number of locked // resources exceeds the assumed limit diff --git a/src/dbzero/workspace/Fixture.cpp b/src/dbzero/workspace/Fixture.cpp index 80582486..3300709f 100644 --- a/src/dbzero/workspace/Fixture.cpp +++ b/src/dbzero/workspace/Fixture.cpp @@ -385,7 +385,7 @@ namespace db0 for (auto &handler: m_flush_handlers) { handler(); } - + m_lang_cache.clear(true); // lock for exclusive access { std::unique_lock lock(m_commit_mutex); From 4bf973c1ebb7ce48a0fba453f3c79c570795205f Mon Sep 17 00:00:00 2001 From: Adrian Zawadzki Date: Tue, 18 Nov 2025 22:09:40 +0100 Subject: [PATCH 2/3] added config parameter --- dbzero/dbzero/initialization.py | 2 +- python_tests/conftest.py | 4 ++-- src/dbzero/bindings/python/PyAPI.cpp | 3 ++- src/dbzero/core/memory/CacheRecycler.cpp | 9 +++++---- src/dbzero/core/memory/CacheRecycler.hpp | 4 +++- src/dbzero/workspace/Workspace.cpp | 9 ++++++--- src/dbzero/workspace/Workspace.hpp | 3 ++- 7 files changed, 21 insertions(+), 13 deletions(-) diff --git a/dbzero/dbzero/initialization.py b/dbzero/dbzero/initialization.py index 169b7b28..95d65c43 100644 --- a/dbzero/dbzero/initialization.py +++ b/dbzero/dbzero/initialization.py @@ -34,7 +34,7 @@ def init(dbzero_root: str, **kwargs) -> None: init_kwargs = {} - config_keys = ("autocommit", "autocommit_interval", "cache_size", "lang_cache_size") + config_keys = ("autocommit", "autocommit_interval", "cache_size", "lang_cache_size", "throw_on_dist_memory_overflow") config = {} for key in config_keys: if key in kwargs: diff --git a/python_tests/conftest.py b/python_tests/conftest.py index 2fa181c1..e3c1b2d5 100644 --- a/python_tests/conftest.py +++ b/python_tests/conftest.py @@ -73,7 +73,7 @@ def db0_autocommit_fixture(request): shutil.rmtree(DB0_DIR) # create empty directory os.mkdir(DB0_DIR) - db0.init(DB0_DIR, autocommit=True, autocommit_interval=request.param) + db0.init(DB0_DIR, autocommit=True, autocommit_interval=request.param, throw_on_dist_memory_overflow=False) db0.open("my-test-prefix") yield db0 gc.collect() @@ -92,7 +92,7 @@ def db0_no_autocommit(): # create empty directory os.mkdir(DB0_DIR) # disable autocommit on all prefixes - db0.init(DB0_DIR, autocommit=False) + db0.init(DB0_DIR, autocommit=False, throw_on_dist_memory_overflow=False) db0.open("my-test-prefix") yield db0 db0.close() diff --git a/src/dbzero/bindings/python/PyAPI.cpp b/src/dbzero/bindings/python/PyAPI.cpp index f0b4df80..6c5eaa19 100644 --- a/src/dbzero/bindings/python/PyAPI.cpp +++ b/src/dbzero/bindings/python/PyAPI.cpp @@ -288,7 +288,8 @@ namespace db0::python {"cache_size", []{ return PyLong_FromUnsignedLongLong(BaseWorkspace::DEFAULT_CACHE_SIZE); }}, {"lang_cache_size", []{ return PyLong_FromUnsignedLongLong(LangCache::DEFAULT_CAPACITY); }}, {"autocommit", []{ Py_RETURN_TRUE; }}, - {"autocommit_interval", []{ return PyLong_FromUnsignedLongLong(Workspace::DEFAULT_AUTOCOMMIT_INTERVAL_MS); }} + {"autocommit_interval", []{ return PyLong_FromUnsignedLongLong(Workspace::DEFAULT_AUTOCOMMIT_INTERVAL_MS); }}, + {"throw_on_dist_memory_overflow", []{ Py_RETURN_TRUE; }} }; for (const auto &[key_str, default_fn] : defaults) { // Populate default values so then can be easily accessed with get_config diff --git a/src/dbzero/core/memory/CacheRecycler.cpp b/src/dbzero/core/memory/CacheRecycler.cpp index 75584811..654ac608 100644 --- a/src/dbzero/core/memory/CacheRecycler.cpp +++ b/src/dbzero/core/memory/CacheRecycler.cpp @@ -27,7 +27,8 @@ namespace db0 CacheRecycler::CacheRecycler(std::size_t capacity, const std::atomic &dirty_meter, std::optional flush_size, std::function flush_dirty, - std::function flush_callback) + std::function flush_callback, + bool throw_on_dist_memory_overflow) : m_capacity(capacity) // NOTE: buffers are overprovisioned , m_res_bufs { getMaxSize(m_capacity), getMaxSize(m_capacity) } @@ -36,6 +37,7 @@ namespace db0 , m_flush_size(flush_size.value_or(DEFAULT_FLUSH_SIZE)) , m_flush_dirty(flush_dirty) , m_flush_callback(flush_callback) + , m_throw_on_dist_memory_overflow(throw_on_dist_memory_overflow) { } @@ -131,9 +133,8 @@ namespace db0 updateSize(lock, m_capacity - flush_size); flushed = true; flush_result = m_current_size[priority] <= (m_capacity - flush_size); - if(getCurrentSize() >= m_capacity){ - THROWF(db0::CacheException) << "CacheRecycler: unable to free sufficient cache space. " - << "To many memo objects in python scope" << THROWF_END; + if(getCurrentSize() >= m_capacity && m_throw_on_dist_memory_overflow){ + THROWF(db0::CacheException) << "DIST Memory Overflow. Too many Python objects" << THROWF_END; } } // resize is a costly operation but cannot be avoided if the number of locked diff --git a/src/dbzero/core/memory/CacheRecycler.hpp b/src/dbzero/core/memory/CacheRecycler.hpp index fae00a1e..d4ce50c6 100644 --- a/src/dbzero/core/memory/CacheRecycler.hpp +++ b/src/dbzero/core/memory/CacheRecycler.hpp @@ -29,7 +29,8 @@ namespace db0 */ CacheRecycler(std::size_t capacity, const std::atomic &dirty_meter, std::optional flush_size = {}, std::function flush_dirty = {}, - std::function flush_callback = {}); + std::function flush_callback = {}, + bool throw_on_dist_memory_overflow = true); void update(std::shared_ptr res_lock); @@ -90,6 +91,7 @@ namespace db0 std::function m_flush_dirty; std::function m_flush_callback; std::pair m_last_flush_callback_result = {true, false}; + bool m_throw_on_dist_memory_overflow = true; void resize(std::unique_lock &, std::size_t new_size, int priority); diff --git a/src/dbzero/workspace/Workspace.cpp b/src/dbzero/workspace/Workspace.cpp index 4987a1dd..8ad44ca2 100644 --- a/src/dbzero/workspace/Workspace.cpp +++ b/src/dbzero/workspace/Workspace.cpp @@ -11,7 +11,8 @@ namespace db0 { BaseWorkspace::BaseWorkspace(const std::string &root_path, std::optional cache_size, - std::optional slab_cache_size, std::optional flush_size, std::optional default_lock_flags) + std::optional slab_cache_size, std::optional flush_size, + std::optional default_lock_flags, std::optional throw_on_dist_memory_overflow) : m_prefix_catalog(root_path) , m_default_lock_flags(default_lock_flags ? *default_lock_flags : LockFlags()) , m_cache_recycler(cache_size ? *cache_size : DEFAULT_CACHE_SIZE, m_dirty_meter, flush_size, @@ -20,7 +21,8 @@ namespace db0 }, [this](bool threshold_reached) -> bool { return this->onCacheFlushed(threshold_reached); - } + }, + throw_on_dist_memory_overflow ? *throw_on_dist_memory_overflow : true ) , m_slab_recycler(slab_cache_size ? *slab_cache_size : DEFAULT_SLAB_CACHE_SIZE) { @@ -228,7 +230,8 @@ namespace db0 std::optional slab_cache_size, std::optional vobject_cache_size, std::optional flush_size, std::function &, bool, bool, bool)> fixture_initializer, std::shared_ptr config, std::optional default_lock_flags) - : BaseWorkspace(root_path, cache_size, slab_cache_size, flush_size, default_lock_flags) + : BaseWorkspace(root_path, cache_size, slab_cache_size, flush_size, + default_lock_flags, m_config ? config->get("throw_on_dist_memory_overflow") : true) , m_config(config) , m_fixture_catalog(m_prefix_catalog) , m_fixture_initializer(fixture_initializer) diff --git a/src/dbzero/workspace/Workspace.hpp b/src/dbzero/workspace/Workspace.hpp index 742b2124..712f6cf3 100644 --- a/src/dbzero/workspace/Workspace.hpp +++ b/src/dbzero/workspace/Workspace.hpp @@ -52,7 +52,8 @@ namespace db0 **/ BaseWorkspace(const std::string &root_path = "", std::optional cache_size = {}, std::optional slab_cache_size = {}, std::optional flush_size = {}, - std::optional default_lock_flags = {}); + std::optional default_lock_flags = {}, + std::optional throw_on_dist_memory_overflow = {}); virtual ~BaseWorkspace()= default; /** From db2d6ced562dbacbf7e9d9aebdbdfefaaa750aeb Mon Sep 17 00:00:00 2001 From: Adrian Zawadzki Date: Thu, 20 Nov 2025 16:25:59 +0100 Subject: [PATCH 3/3] merge request fixes --- dbzero/dbzero/dbzero.py | 2 +- dbzero/dbzero/initialization.py | 2 +- python_tests/conftest.py | 4 ++-- src/dbzero/bindings/python/PyAPI.cpp | 2 +- src/dbzero/core/memory/CacheRecycler.cpp | 6 +++--- src/dbzero/core/memory/CacheRecycler.hpp | 4 ++-- src/dbzero/workspace/Workspace.cpp | 6 +++--- src/dbzero/workspace/Workspace.hpp | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dbzero/dbzero/dbzero.py b/dbzero/dbzero/dbzero.py index 21899e3d..c9e4f4dc 100644 --- a/dbzero/dbzero/dbzero.py +++ b/dbzero/dbzero/dbzero.py @@ -10,7 +10,7 @@ def load_dynamic(name, path): def __bootstrap__(): global __bootstrap__, __loader__, __file__ - paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/debug", "/usr/local/lib/python3/dist-packages/dbzero/"] + paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/release", "/usr/local/lib/python3/dist-packages/dbzero/"] __file__ = None for path in paths: if os.path.isdir(path): diff --git a/dbzero/dbzero/initialization.py b/dbzero/dbzero/initialization.py index 95d65c43..ebfb7f26 100644 --- a/dbzero/dbzero/initialization.py +++ b/dbzero/dbzero/initialization.py @@ -34,7 +34,7 @@ def init(dbzero_root: str, **kwargs) -> None: init_kwargs = {} - config_keys = ("autocommit", "autocommit_interval", "cache_size", "lang_cache_size", "throw_on_dist_memory_overflow") + config_keys = ("autocommit", "autocommit_interval", "cache_size", "lang_cache_size", "suppress_dist_overflow_error") config = {} for key in config_keys: if key in kwargs: diff --git a/python_tests/conftest.py b/python_tests/conftest.py index e3c1b2d5..a8e38124 100644 --- a/python_tests/conftest.py +++ b/python_tests/conftest.py @@ -73,7 +73,7 @@ def db0_autocommit_fixture(request): shutil.rmtree(DB0_DIR) # create empty directory os.mkdir(DB0_DIR) - db0.init(DB0_DIR, autocommit=True, autocommit_interval=request.param, throw_on_dist_memory_overflow=False) + db0.init(DB0_DIR, autocommit=True, autocommit_interval=request.param, suppress_dist_overflow_error=True) db0.open("my-test-prefix") yield db0 gc.collect() @@ -92,7 +92,7 @@ def db0_no_autocommit(): # create empty directory os.mkdir(DB0_DIR) # disable autocommit on all prefixes - db0.init(DB0_DIR, autocommit=False, throw_on_dist_memory_overflow=False) + db0.init(DB0_DIR, autocommit=False, suppress_dist_overflow_error=True) db0.open("my-test-prefix") yield db0 db0.close() diff --git a/src/dbzero/bindings/python/PyAPI.cpp b/src/dbzero/bindings/python/PyAPI.cpp index 6c5eaa19..cdf6ea9c 100644 --- a/src/dbzero/bindings/python/PyAPI.cpp +++ b/src/dbzero/bindings/python/PyAPI.cpp @@ -289,7 +289,7 @@ namespace db0::python {"lang_cache_size", []{ return PyLong_FromUnsignedLongLong(LangCache::DEFAULT_CAPACITY); }}, {"autocommit", []{ Py_RETURN_TRUE; }}, {"autocommit_interval", []{ return PyLong_FromUnsignedLongLong(Workspace::DEFAULT_AUTOCOMMIT_INTERVAL_MS); }}, - {"throw_on_dist_memory_overflow", []{ Py_RETURN_TRUE; }} + {"suppress_dist_overflow_error", []{ Py_RETURN_FALSE; }} }; for (const auto &[key_str, default_fn] : defaults) { // Populate default values so then can be easily accessed with get_config diff --git a/src/dbzero/core/memory/CacheRecycler.cpp b/src/dbzero/core/memory/CacheRecycler.cpp index 654ac608..4b21ebc7 100644 --- a/src/dbzero/core/memory/CacheRecycler.cpp +++ b/src/dbzero/core/memory/CacheRecycler.cpp @@ -28,7 +28,7 @@ namespace db0 std::optional flush_size, std::function flush_dirty, std::function flush_callback, - bool throw_on_dist_memory_overflow) + bool suppress_dist_overflow_error) : m_capacity(capacity) // NOTE: buffers are overprovisioned , m_res_bufs { getMaxSize(m_capacity), getMaxSize(m_capacity) } @@ -37,7 +37,7 @@ namespace db0 , m_flush_size(flush_size.value_or(DEFAULT_FLUSH_SIZE)) , m_flush_dirty(flush_dirty) , m_flush_callback(flush_callback) - , m_throw_on_dist_memory_overflow(throw_on_dist_memory_overflow) + , m_suppress_dist_overflow_error(suppress_dist_overflow_error) { } @@ -133,7 +133,7 @@ namespace db0 updateSize(lock, m_capacity - flush_size); flushed = true; flush_result = m_current_size[priority] <= (m_capacity - flush_size); - if(getCurrentSize() >= m_capacity && m_throw_on_dist_memory_overflow){ + if(getCurrentSize() >= m_capacity && !m_suppress_dist_overflow_error){ THROWF(db0::CacheException) << "DIST Memory Overflow. Too many Python objects" << THROWF_END; } } diff --git a/src/dbzero/core/memory/CacheRecycler.hpp b/src/dbzero/core/memory/CacheRecycler.hpp index d4ce50c6..219aa6d2 100644 --- a/src/dbzero/core/memory/CacheRecycler.hpp +++ b/src/dbzero/core/memory/CacheRecycler.hpp @@ -30,7 +30,7 @@ namespace db0 CacheRecycler(std::size_t capacity, const std::atomic &dirty_meter, std::optional flush_size = {}, std::function flush_dirty = {}, std::function flush_callback = {}, - bool throw_on_dist_memory_overflow = true); + bool suppress_dist_overflow_error = false); void update(std::shared_ptr res_lock); @@ -91,7 +91,7 @@ namespace db0 std::function m_flush_dirty; std::function m_flush_callback; std::pair m_last_flush_callback_result = {true, false}; - bool m_throw_on_dist_memory_overflow = true; + bool m_suppress_dist_overflow_error = false; void resize(std::unique_lock &, std::size_t new_size, int priority); diff --git a/src/dbzero/workspace/Workspace.cpp b/src/dbzero/workspace/Workspace.cpp index 8ad44ca2..c590191b 100644 --- a/src/dbzero/workspace/Workspace.cpp +++ b/src/dbzero/workspace/Workspace.cpp @@ -12,7 +12,7 @@ namespace db0 BaseWorkspace::BaseWorkspace(const std::string &root_path, std::optional cache_size, std::optional slab_cache_size, std::optional flush_size, - std::optional default_lock_flags, std::optional throw_on_dist_memory_overflow) + std::optional default_lock_flags, std::optional suppress_dist_overflow_error) : m_prefix_catalog(root_path) , m_default_lock_flags(default_lock_flags ? *default_lock_flags : LockFlags()) , m_cache_recycler(cache_size ? *cache_size : DEFAULT_CACHE_SIZE, m_dirty_meter, flush_size, @@ -22,7 +22,7 @@ namespace db0 [this](bool threshold_reached) -> bool { return this->onCacheFlushed(threshold_reached); }, - throw_on_dist_memory_overflow ? *throw_on_dist_memory_overflow : true + suppress_dist_overflow_error ? *suppress_dist_overflow_error : false ) , m_slab_recycler(slab_cache_size ? *slab_cache_size : DEFAULT_SLAB_CACHE_SIZE) { @@ -231,7 +231,7 @@ namespace db0 std::optional flush_size, std::function &, bool, bool, bool)> fixture_initializer, std::shared_ptr config, std::optional default_lock_flags) : BaseWorkspace(root_path, cache_size, slab_cache_size, flush_size, - default_lock_flags, m_config ? config->get("throw_on_dist_memory_overflow") : true) + default_lock_flags, m_config ? config->get("suppress_dist_overflow_error") : false) , m_config(config) , m_fixture_catalog(m_prefix_catalog) , m_fixture_initializer(fixture_initializer) diff --git a/src/dbzero/workspace/Workspace.hpp b/src/dbzero/workspace/Workspace.hpp index 712f6cf3..91a21b45 100644 --- a/src/dbzero/workspace/Workspace.hpp +++ b/src/dbzero/workspace/Workspace.hpp @@ -53,7 +53,7 @@ namespace db0 BaseWorkspace(const std::string &root_path = "", std::optional cache_size = {}, std::optional slab_cache_size = {}, std::optional flush_size = {}, std::optional default_lock_flags = {}, - std::optional throw_on_dist_memory_overflow = {}); + std::optional suppress_dist_overflow_error = {}); virtual ~BaseWorkspace()= default; /**