diff --git a/dbzero/dbzero/dbzero.py b/dbzero/dbzero/dbzero.py index 21899e3d..c9e4f4dc 100644 --- a/dbzero/dbzero/dbzero.py +++ b/dbzero/dbzero/dbzero.py @@ -10,7 +10,7 @@ def load_dynamic(name, path): def __bootstrap__(): global __bootstrap__, __loader__, __file__ - paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/debug", "/usr/local/lib/python3/dist-packages/dbzero/"] + paths = [os.path.join(os.path.split(__file__)[0]), "/src/dev/build/release", "/usr/local/lib/python3/dist-packages/dbzero/"] __file__ = None for path in paths: if os.path.isdir(path): diff --git a/dbzero/dbzero/initialization.py b/dbzero/dbzero/initialization.py index 169b7b28..ebfb7f26 100644 --- a/dbzero/dbzero/initialization.py +++ b/dbzero/dbzero/initialization.py @@ -34,7 +34,7 @@ def init(dbzero_root: str, **kwargs) -> None: init_kwargs = {} - config_keys = ("autocommit", "autocommit_interval", "cache_size", "lang_cache_size") + config_keys = ("autocommit", "autocommit_interval", "cache_size", "lang_cache_size", "suppress_dist_overflow_error") config = {} for key in config_keys: if key in kwargs: diff --git a/python_tests/conftest.py b/python_tests/conftest.py index 2fa181c1..a8e38124 100644 --- a/python_tests/conftest.py +++ b/python_tests/conftest.py @@ -73,7 +73,7 @@ def db0_autocommit_fixture(request): shutil.rmtree(DB0_DIR) # create empty directory os.mkdir(DB0_DIR) - db0.init(DB0_DIR, autocommit=True, autocommit_interval=request.param) + db0.init(DB0_DIR, autocommit=True, autocommit_interval=request.param, suppress_dist_overflow_error=True) db0.open("my-test-prefix") yield db0 gc.collect() @@ -92,7 +92,7 @@ def db0_no_autocommit(): # create empty directory os.mkdir(DB0_DIR) # disable autocommit on all prefixes - db0.init(DB0_DIR, autocommit=False) + db0.init(DB0_DIR, autocommit=False, suppress_dist_overflow_error=True) db0.open("my-test-prefix") yield db0 db0.close() diff --git a/src/dbzero/bindings/python/PyAPI.cpp b/src/dbzero/bindings/python/PyAPI.cpp index f0b4df80..cdf6ea9c 100644 --- a/src/dbzero/bindings/python/PyAPI.cpp +++ b/src/dbzero/bindings/python/PyAPI.cpp @@ -288,7 +288,8 @@ namespace db0::python {"cache_size", []{ return PyLong_FromUnsignedLongLong(BaseWorkspace::DEFAULT_CACHE_SIZE); }}, {"lang_cache_size", []{ return PyLong_FromUnsignedLongLong(LangCache::DEFAULT_CAPACITY); }}, {"autocommit", []{ Py_RETURN_TRUE; }}, - {"autocommit_interval", []{ return PyLong_FromUnsignedLongLong(Workspace::DEFAULT_AUTOCOMMIT_INTERVAL_MS); }} + {"autocommit_interval", []{ return PyLong_FromUnsignedLongLong(Workspace::DEFAULT_AUTOCOMMIT_INTERVAL_MS); }}, + {"suppress_dist_overflow_error", []{ Py_RETURN_FALSE; }} }; for (const auto &[key_str, default_fn] : defaults) { // Populate default values so then can be easily accessed with get_config diff --git a/src/dbzero/core/exception/Exceptions.cpp b/src/dbzero/core/exception/Exceptions.cpp index 9f5b33ec..2e6c4b2a 100644 --- a/src/dbzero/core/exception/Exceptions.cpp +++ b/src/dbzero/core/exception/Exceptions.cpp @@ -64,4 +64,9 @@ namespace db0 { } + CacheException::CacheException() + : CriticalException(exception_id) + { + } + } \ No newline at end of file diff --git a/src/dbzero/core/exception/Exceptions.hpp b/src/dbzero/core/exception/Exceptions.hpp index 250f5c43..58e7a50b 100644 --- a/src/dbzero/core/exception/Exceptions.hpp +++ b/src/dbzero/core/exception/Exceptions.hpp @@ -118,4 +118,11 @@ namespace db0 BadAddressException(); }; + class CacheException: public CriticalException + { + public: + static constexpr int exception_id = EXCEPTION_ID_PREFIX::BASIC | 0x0f; + CacheException(); + }; + } \ No newline at end of file diff --git a/src/dbzero/core/memory/CacheRecycler.cpp b/src/dbzero/core/memory/CacheRecycler.cpp index 2851ffab..4b21ebc7 100644 --- a/src/dbzero/core/memory/CacheRecycler.cpp +++ b/src/dbzero/core/memory/CacheRecycler.cpp @@ -27,7 +27,8 @@ namespace db0 CacheRecycler::CacheRecycler(std::size_t capacity, const std::atomic &dirty_meter, std::optional flush_size, std::function flush_dirty, - std::function flush_callback) + std::function flush_callback, + bool suppress_dist_overflow_error) : m_capacity(capacity) // NOTE: buffers are overprovisioned , m_res_bufs { getMaxSize(m_capacity), getMaxSize(m_capacity) } @@ -36,6 +37,7 @@ namespace db0 , m_flush_size(flush_size.value_or(DEFAULT_FLUSH_SIZE)) , m_flush_dirty(flush_dirty) , m_flush_callback(flush_callback) + , m_suppress_dist_overflow_error(suppress_dist_overflow_error) { } @@ -131,6 +133,9 @@ namespace db0 updateSize(lock, m_capacity - flush_size); flushed = true; flush_result = m_current_size[priority] <= (m_capacity - flush_size); + if(getCurrentSize() >= m_capacity && !m_suppress_dist_overflow_error){ + THROWF(db0::CacheException) << "DIST Memory Overflow. Too many Python objects" << THROWF_END; + } } // resize is a costly operation but cannot be avoided if the number of locked // resources exceeds the assumed limit diff --git a/src/dbzero/core/memory/CacheRecycler.hpp b/src/dbzero/core/memory/CacheRecycler.hpp index fae00a1e..219aa6d2 100644 --- a/src/dbzero/core/memory/CacheRecycler.hpp +++ b/src/dbzero/core/memory/CacheRecycler.hpp @@ -29,7 +29,8 @@ namespace db0 */ CacheRecycler(std::size_t capacity, const std::atomic &dirty_meter, std::optional flush_size = {}, std::function flush_dirty = {}, - std::function flush_callback = {}); + std::function flush_callback = {}, + bool suppress_dist_overflow_error = false); void update(std::shared_ptr res_lock); @@ -90,6 +91,7 @@ namespace db0 std::function m_flush_dirty; std::function m_flush_callback; std::pair m_last_flush_callback_result = {true, false}; + bool m_suppress_dist_overflow_error = false; void resize(std::unique_lock &, std::size_t new_size, int priority); diff --git a/src/dbzero/workspace/Fixture.cpp b/src/dbzero/workspace/Fixture.cpp index e8e49078..1214ae27 100644 --- a/src/dbzero/workspace/Fixture.cpp +++ b/src/dbzero/workspace/Fixture.cpp @@ -385,7 +385,7 @@ namespace db0 for (auto &handler: m_flush_handlers) { handler(); } - + m_lang_cache.clear(true); // lock for exclusive access { std::unique_lock lock(m_commit_mutex); diff --git a/src/dbzero/workspace/Workspace.cpp b/src/dbzero/workspace/Workspace.cpp index 4987a1dd..c590191b 100644 --- a/src/dbzero/workspace/Workspace.cpp +++ b/src/dbzero/workspace/Workspace.cpp @@ -11,7 +11,8 @@ namespace db0 { BaseWorkspace::BaseWorkspace(const std::string &root_path, std::optional cache_size, - std::optional slab_cache_size, std::optional flush_size, std::optional default_lock_flags) + std::optional slab_cache_size, std::optional flush_size, + std::optional default_lock_flags, std::optional suppress_dist_overflow_error) : m_prefix_catalog(root_path) , m_default_lock_flags(default_lock_flags ? *default_lock_flags : LockFlags()) , m_cache_recycler(cache_size ? *cache_size : DEFAULT_CACHE_SIZE, m_dirty_meter, flush_size, @@ -20,7 +21,8 @@ namespace db0 }, [this](bool threshold_reached) -> bool { return this->onCacheFlushed(threshold_reached); - } + }, + suppress_dist_overflow_error ? *suppress_dist_overflow_error : false ) , m_slab_recycler(slab_cache_size ? *slab_cache_size : DEFAULT_SLAB_CACHE_SIZE) { @@ -228,7 +230,8 @@ namespace db0 std::optional slab_cache_size, std::optional vobject_cache_size, std::optional flush_size, std::function &, bool, bool, bool)> fixture_initializer, std::shared_ptr config, std::optional default_lock_flags) - : BaseWorkspace(root_path, cache_size, slab_cache_size, flush_size, default_lock_flags) + : BaseWorkspace(root_path, cache_size, slab_cache_size, flush_size, + default_lock_flags, m_config ? config->get("suppress_dist_overflow_error") : false) , m_config(config) , m_fixture_catalog(m_prefix_catalog) , m_fixture_initializer(fixture_initializer) diff --git a/src/dbzero/workspace/Workspace.hpp b/src/dbzero/workspace/Workspace.hpp index 742b2124..91a21b45 100644 --- a/src/dbzero/workspace/Workspace.hpp +++ b/src/dbzero/workspace/Workspace.hpp @@ -52,7 +52,8 @@ namespace db0 **/ BaseWorkspace(const std::string &root_path = "", std::optional cache_size = {}, std::optional slab_cache_size = {}, std::optional flush_size = {}, - std::optional default_lock_flags = {}); + std::optional default_lock_flags = {}, + std::optional suppress_dist_overflow_error = {}); virtual ~BaseWorkspace()= default; /**