diff --git a/python_tests/conftest.py b/python_tests/conftest.py index b9442d3f..0ea0eb45 100644 --- a/python_tests/conftest.py +++ b/python_tests/conftest.py @@ -24,8 +24,7 @@ def db0_fixture(request): shutil.rmtree(DB0_DIR) os.mkdir(DB0_DIR) db0.init( - DB0_DIR, - suppress_dist_overflow_error=__extract_param(request, "suppress_dist_overflow_error", False), + DB0_DIR ) db0.open("my-test-prefix") yield db0 @@ -61,8 +60,7 @@ def db0_slab_size(request): db0.init( DB0_DIR, autocommit=request.param.get("autocommit", True), - autocommit_interval=request.param.get("autocommit_interval", 250), - suppress_dist_overflow_error=__extract_param(request, "suppress_dist_overflow_error", False), + autocommit_interval=request.param.get("autocommit_interval", 250) ) db0.open("my-test-prefix", slab_size=request.param["slab_size"]) yield db0 @@ -81,7 +79,7 @@ def db0_autocommit_fixture(request): shutil.rmtree(DB0_DIR) # create empty directory os.mkdir(DB0_DIR) - db0.init(DB0_DIR, autocommit=True, autocommit_interval=request.param, suppress_dist_overflow_error=True) + db0.init(DB0_DIR, autocommit=True, autocommit_interval=request.param) db0.open("my-test-prefix") yield db0 gc.collect() @@ -100,7 +98,7 @@ def db0_no_autocommit(): # create empty directory os.mkdir(DB0_DIR) # disable autocommit on all prefixes - db0.init(DB0_DIR, autocommit=False, suppress_dist_overflow_error=True) + db0.init(DB0_DIR, autocommit=False) db0.open("my-test-prefix") yield db0 db0.close() diff --git a/python_tests/test_durability.py b/python_tests/test_durability.py index a0462e20..886b1703 100644 --- a/python_tests/test_durability.py +++ b/python_tests/test_durability.py @@ -283,7 +283,7 @@ def rand_string(str_len): @pytest.mark.stress_test @pytest.mark.parametrize("db0_slab_size", - [{"slab_size": 2 << 20, "autocommit": True, "suppress_dist_overflow_error": True}], indirect=True) + [{"slab_size": 2 << 20, "autocommit": True}], indirect=True) def test_low_cache_bad_address_issue1(db0_slab_size): """ Test was failing with: CRDT_Allocator internal error: blank not found after ~6k operations diff --git a/python_tests/test_issues_12.py b/python_tests/test_issues_12.py index 104c75b8..74a21d22 100644 --- a/python_tests/test_issues_12.py +++ b/python_tests/test_issues_12.py @@ -7,7 +7,7 @@ import random import time from typing import Dict, List - +import sys @db0.memo @dataclass @@ -17,7 +17,7 @@ class Issuer: inv_index: db0.index -@db0.memo(no_cache=True) +@db0.memo() @dataclass class Invoice: tax_id: int @@ -25,6 +25,21 @@ class Invoice: data: bytes + + +@db0.memo(no_cache=True) +@dataclass +class InvoiceNoCache: + tax_id: int + issue_dt: datetime + data: bytes + +@db0.memo +@dataclass +class SimpleIssuer: + inv_object: InvoiceNoCache + inv_index: db0.index + def get_random_tax_id(tax_ids_set=set()): tax_id = random.randint(1000000000, 9999999999) while tax_id in tax_ids_set: @@ -98,3 +113,13 @@ def test_no_cache_allocator_issue(db0_slab_size): if (now - start) > execution_time: break + +@pytest.mark.skip(reason="need to fix: issues/533") +def test_free_issue_with_index_and_no_cache_object(db0_fixture): + index = db0.index() + new_issuer = SimpleIssuer(inv_object=None, inv_index=index) + db0.commit() + RANDOM_BYTES = b'DB0'*5 + invoice = InvoiceNoCache(tax_id=None, issue_dt=datetime.now(), data=RANDOM_BYTES) + new_issuer.inv_object = invoice + new_issuer.inv_index.add(datetime.now(), invoice) diff --git a/python_tests/test_volume_stress.py b/python_tests/test_volume_stress.py index 27be0896..35d47696 100644 --- a/python_tests/test_volume_stress.py +++ b/python_tests/test_volume_stress.py @@ -60,7 +60,7 @@ def test_create_large_objects_cache_recycler_issue_1(db0_slab_size): @pytest.mark.stress_test @pytest.mark.parametrize("db0_slab_size", - [{"slab_size": 1024 * 1024 * 1024, "suppress_dist_overflow_error": True}], indirect=True) + [{"slab_size": 1024 * 1024 * 1024}], indirect=True) def test_create_and_drop_simple_memo_objects(db0_slab_size): """ Run on valgrind with --tool=massif to detect memory problems diff --git a/src/dbzero/bindings/python/PyAPI.cpp b/src/dbzero/bindings/python/PyAPI.cpp index cdf6ea9c..8fb15f87 100644 --- a/src/dbzero/bindings/python/PyAPI.cpp +++ b/src/dbzero/bindings/python/PyAPI.cpp @@ -289,7 +289,6 @@ namespace db0::python {"lang_cache_size", []{ return PyLong_FromUnsignedLongLong(LangCache::DEFAULT_CAPACITY); }}, {"autocommit", []{ Py_RETURN_TRUE; }}, {"autocommit_interval", []{ return PyLong_FromUnsignedLongLong(Workspace::DEFAULT_AUTOCOMMIT_INTERVAL_MS); }}, - {"suppress_dist_overflow_error", []{ Py_RETURN_FALSE; }} }; for (const auto &[key_str, default_fn] : defaults) { // Populate default values so then can be easily accessed with get_config diff --git a/src/dbzero/core/memory/CacheRecycler.cpp b/src/dbzero/core/memory/CacheRecycler.cpp index 1b845593..ce791759 100644 --- a/src/dbzero/core/memory/CacheRecycler.cpp +++ b/src/dbzero/core/memory/CacheRecycler.cpp @@ -27,8 +27,7 @@ namespace db0 CacheRecycler::CacheRecycler(std::size_t capacity, const std::atomic &dirty_meter, std::optional flush_size, std::function flush_dirty, - std::function flush_callback, - bool suppress_dist_overflow_error) + std::function flush_callback) : m_capacity(capacity) // NOTE: buffers are overprovisioned , m_res_bufs { getMaxSize(m_capacity), getMaxSize(m_capacity) } @@ -37,7 +36,6 @@ namespace db0 , m_flush_size(flush_size.value_or(DEFAULT_FLUSH_SIZE)) , m_flush_dirty(flush_dirty) , m_flush_callback(flush_callback) - , m_suppress_dist_overflow_error(suppress_dist_overflow_error) { } @@ -132,10 +130,7 @@ namespace db0 auto flush_size = std::min(m_capacity >> 1, m_flush_size); updateSize(lock, m_capacity - flush_size); flushed = true; - flush_result = m_current_size[priority] <= (m_capacity - flush_size); - if (getCurrentSize() >= m_capacity && !m_suppress_dist_overflow_error) { - THROWF(db0::CacheException) << "DIST Memory Overflow. Too many Python objects"; - } + flush_result = m_current_size[priority] <= (m_capacity - flush_size); } // resize is a costly operation but cannot be avoided if the number of locked // resources exceeds the assumed limit diff --git a/src/dbzero/core/memory/CacheRecycler.hpp b/src/dbzero/core/memory/CacheRecycler.hpp index 219aa6d2..fae00a1e 100644 --- a/src/dbzero/core/memory/CacheRecycler.hpp +++ b/src/dbzero/core/memory/CacheRecycler.hpp @@ -29,8 +29,7 @@ namespace db0 */ CacheRecycler(std::size_t capacity, const std::atomic &dirty_meter, std::optional flush_size = {}, std::function flush_dirty = {}, - std::function flush_callback = {}, - bool suppress_dist_overflow_error = false); + std::function flush_callback = {}); void update(std::shared_ptr res_lock); @@ -91,7 +90,6 @@ namespace db0 std::function m_flush_dirty; std::function m_flush_callback; std::pair m_last_flush_callback_result = {true, false}; - bool m_suppress_dist_overflow_error = false; void resize(std::unique_lock &, std::size_t new_size, int priority); diff --git a/src/dbzero/object_model/index/Index.cpp b/src/dbzero/object_model/index/Index.cpp index de4ae54f..d31b81bf 100644 --- a/src/dbzero/object_model/index/Index.cpp +++ b/src/dbzero/object_model/index/Index.cpp @@ -72,11 +72,7 @@ namespace db0::object_model // after unregister object might still have unflushed data, we need to flush them if (hasInstance() && isDirty()) { - try { - _flush(); - } catch (CacheException &) { - // suppress cache exception, other exceptions will terminate - } + _flush(); } } diff --git a/src/dbzero/workspace/Workspace.cpp b/src/dbzero/workspace/Workspace.cpp index 3094324c..c70f9048 100644 --- a/src/dbzero/workspace/Workspace.cpp +++ b/src/dbzero/workspace/Workspace.cpp @@ -12,7 +12,7 @@ namespace db0 BaseWorkspace::BaseWorkspace(const std::string &root_path, std::optional cache_size, std::optional slab_cache_size, std::optional flush_size, - std::optional default_lock_flags, std::optional suppress_dist_overflow_error) + std::optional default_lock_flags) : m_prefix_catalog(root_path) , m_default_lock_flags(default_lock_flags ? *default_lock_flags : LockFlags()) , m_cache_recycler(cache_size ? *cache_size : DEFAULT_CACHE_SIZE, m_dirty_meter, flush_size, @@ -21,8 +21,7 @@ namespace db0 }, [this](bool threshold_reached) -> bool { return this->onCacheFlushed(threshold_reached); - }, - suppress_dist_overflow_error ? *suppress_dist_overflow_error : false + } ) , m_slab_recycler(slab_cache_size ? *slab_cache_size : DEFAULT_SLAB_CACHE_SIZE) { @@ -230,8 +229,8 @@ namespace db0 std::optional slab_cache_size, std::optional vobject_cache_size, std::optional flush_size, std::function &, bool, bool, bool)> fixture_initializer, std::shared_ptr config, std::optional default_lock_flags) - : BaseWorkspace(root_path, cache_size, slab_cache_size, flush_size, - default_lock_flags, config ? config->get("suppress_dist_overflow_error") : false) + : BaseWorkspace(root_path, cache_size, slab_cache_size, flush_size, + default_lock_flags) , m_config(config) , m_fixture_catalog(m_prefix_catalog) , m_fixture_initializer(fixture_initializer) diff --git a/src/dbzero/workspace/Workspace.hpp b/src/dbzero/workspace/Workspace.hpp index 91a21b45..742b2124 100644 --- a/src/dbzero/workspace/Workspace.hpp +++ b/src/dbzero/workspace/Workspace.hpp @@ -52,8 +52,7 @@ namespace db0 **/ BaseWorkspace(const std::string &root_path = "", std::optional cache_size = {}, std::optional slab_cache_size = {}, std::optional flush_size = {}, - std::optional default_lock_flags = {}, - std::optional suppress_dist_overflow_error = {}); + std::optional default_lock_flags = {}); virtual ~BaseWorkspace()= default; /**