Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions python_tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@ def db0_fixture(request):
shutil.rmtree(DB0_DIR)
os.mkdir(DB0_DIR)
db0.init(
DB0_DIR,
suppress_dist_overflow_error=__extract_param(request, "suppress_dist_overflow_error", False),
DB0_DIR
)
db0.open("my-test-prefix")
yield db0
Expand Down Expand Up @@ -61,8 +60,7 @@ def db0_slab_size(request):
db0.init(
DB0_DIR,
autocommit=request.param.get("autocommit", True),
autocommit_interval=request.param.get("autocommit_interval", 250),
suppress_dist_overflow_error=__extract_param(request, "suppress_dist_overflow_error", False),
autocommit_interval=request.param.get("autocommit_interval", 250)
)
db0.open("my-test-prefix", slab_size=request.param["slab_size"])
yield db0
Expand All @@ -81,7 +79,7 @@ def db0_autocommit_fixture(request):
shutil.rmtree(DB0_DIR)
# create empty directory
os.mkdir(DB0_DIR)
db0.init(DB0_DIR, autocommit=True, autocommit_interval=request.param, suppress_dist_overflow_error=True)
db0.init(DB0_DIR, autocommit=True, autocommit_interval=request.param)
db0.open("my-test-prefix")
yield db0
gc.collect()
Expand All @@ -100,7 +98,7 @@ def db0_no_autocommit():
# create empty directory
os.mkdir(DB0_DIR)
# disable autocommit on all prefixes
db0.init(DB0_DIR, autocommit=False, suppress_dist_overflow_error=True)
db0.init(DB0_DIR, autocommit=False)
db0.open("my-test-prefix")
yield db0
db0.close()
Expand Down
2 changes: 1 addition & 1 deletion python_tests/test_durability.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,7 @@ def rand_string(str_len):

@pytest.mark.stress_test
@pytest.mark.parametrize("db0_slab_size",
[{"slab_size": 2 << 20, "autocommit": True, "suppress_dist_overflow_error": True}], indirect=True)
[{"slab_size": 2 << 20, "autocommit": True}], indirect=True)
def test_low_cache_bad_address_issue1(db0_slab_size):
"""
Test was failing with: CRDT_Allocator internal error: blank not found after ~6k operations
Expand Down
29 changes: 27 additions & 2 deletions python_tests/test_issues_12.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import random
import time
from typing import Dict, List

import sys

@db0.memo
@dataclass
Expand All @@ -17,14 +17,29 @@ class Issuer:
inv_index: db0.index


@db0.memo(no_cache=True)
@db0.memo()
@dataclass
class Invoice:
tax_id: int
issue_dt: datetime
data: bytes




@db0.memo(no_cache=True)
@dataclass
class InvoiceNoCache:
tax_id: int
issue_dt: datetime
data: bytes

@db0.memo
@dataclass
class SimpleIssuer:
inv_object: InvoiceNoCache
inv_index: db0.index

def get_random_tax_id(tax_ids_set=set()):
tax_id = random.randint(1000000000, 9999999999)
while tax_id in tax_ids_set:
Expand Down Expand Up @@ -98,3 +113,13 @@ def test_no_cache_allocator_issue(db0_slab_size):

if (now - start) > execution_time:
break

@pytest.mark.skip(reason="need to fix: issues/533")
def test_free_issue_with_index_and_no_cache_object(db0_fixture):
index = db0.index()
new_issuer = SimpleIssuer(inv_object=None, inv_index=index)
db0.commit()
RANDOM_BYTES = b'DB0'*5
invoice = InvoiceNoCache(tax_id=None, issue_dt=datetime.now(), data=RANDOM_BYTES)
new_issuer.inv_object = invoice
new_issuer.inv_index.add(datetime.now(), invoice)
2 changes: 1 addition & 1 deletion python_tests/test_volume_stress.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def test_create_large_objects_cache_recycler_issue_1(db0_slab_size):

@pytest.mark.stress_test
@pytest.mark.parametrize("db0_slab_size",
[{"slab_size": 1024 * 1024 * 1024, "suppress_dist_overflow_error": True}], indirect=True)
[{"slab_size": 1024 * 1024 * 1024}], indirect=True)
def test_create_and_drop_simple_memo_objects(db0_slab_size):
"""
Run on valgrind with --tool=massif to detect memory problems
Expand Down
1 change: 0 additions & 1 deletion src/dbzero/bindings/python/PyAPI.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,6 @@ namespace db0::python
{"lang_cache_size", []{ return PyLong_FromUnsignedLongLong(LangCache::DEFAULT_CAPACITY); }},
{"autocommit", []{ Py_RETURN_TRUE; }},
{"autocommit_interval", []{ return PyLong_FromUnsignedLongLong(Workspace::DEFAULT_AUTOCOMMIT_INTERVAL_MS); }},
{"suppress_dist_overflow_error", []{ Py_RETURN_FALSE; }}
};
for (const auto &[key_str, default_fn] : defaults) {
// Populate default values so then can be easily accessed with get_config
Expand Down
9 changes: 2 additions & 7 deletions src/dbzero/core/memory/CacheRecycler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,7 @@ namespace db0
CacheRecycler::CacheRecycler(std::size_t capacity, const std::atomic<std::size_t> &dirty_meter,
std::optional<std::size_t> flush_size,
std::function<void(std::size_t limit)> flush_dirty,
std::function<bool(bool threshold_reached)> flush_callback,
bool suppress_dist_overflow_error)
std::function<bool(bool threshold_reached)> flush_callback)
: m_capacity(capacity)
// NOTE: buffers are overprovisioned
, m_res_bufs { getMaxSize(m_capacity), getMaxSize(m_capacity) }
Expand All @@ -37,7 +36,6 @@ namespace db0
, m_flush_size(flush_size.value_or(DEFAULT_FLUSH_SIZE))
, m_flush_dirty(flush_dirty)
, m_flush_callback(flush_callback)
, m_suppress_dist_overflow_error(suppress_dist_overflow_error)
{
}

Expand Down Expand Up @@ -132,10 +130,7 @@ namespace db0
auto flush_size = std::min(m_capacity >> 1, m_flush_size);
updateSize(lock, m_capacity - flush_size);
flushed = true;
flush_result = m_current_size[priority] <= (m_capacity - flush_size);
if (getCurrentSize() >= m_capacity && !m_suppress_dist_overflow_error) {
THROWF(db0::CacheException) << "DIST Memory Overflow. Too many Python objects";
}
flush_result = m_current_size[priority] <= (m_capacity - flush_size);
}
// resize is a costly operation but cannot be avoided if the number of locked
// resources exceeds the assumed limit
Expand Down
4 changes: 1 addition & 3 deletions src/dbzero/core/memory/CacheRecycler.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,7 @@ namespace db0
*/
CacheRecycler(std::size_t capacity, const std::atomic<std::size_t> &dirty_meter, std::optional<std::size_t> flush_size = {},
std::function<void(std::size_t limit)> flush_dirty = {},
std::function<bool(bool threshold_reached)> flush_callback = {},
bool suppress_dist_overflow_error = false);
std::function<bool(bool threshold_reached)> flush_callback = {});

void update(std::shared_ptr<ResourceLock> res_lock);

Expand Down Expand Up @@ -91,7 +90,6 @@ namespace db0
std::function<void(std::size_t limit)> m_flush_dirty;
std::function<bool(bool)> m_flush_callback;
std::pair<bool, bool> m_last_flush_callback_result = {true, false};
bool m_suppress_dist_overflow_error = false;

void resize(std::unique_lock<std::mutex> &, std::size_t new_size, int priority);

Expand Down
6 changes: 1 addition & 5 deletions src/dbzero/object_model/index/Index.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,7 @@ namespace db0::object_model

// after unregister object might still have unflushed data, we need to flush them
if (hasInstance() && isDirty()) {
try {
_flush();
} catch (CacheException &) {
// suppress cache exception, other exceptions will terminate
}
_flush();
}
}

Expand Down
9 changes: 4 additions & 5 deletions src/dbzero/workspace/Workspace.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ namespace db0

BaseWorkspace::BaseWorkspace(const std::string &root_path, std::optional<std::size_t> cache_size,
std::optional<std::size_t> slab_cache_size, std::optional<std::size_t> flush_size,
std::optional<LockFlags> default_lock_flags, std::optional<bool> suppress_dist_overflow_error)
std::optional<LockFlags> default_lock_flags)
: m_prefix_catalog(root_path)
, m_default_lock_flags(default_lock_flags ? *default_lock_flags : LockFlags())
, m_cache_recycler(cache_size ? *cache_size : DEFAULT_CACHE_SIZE, m_dirty_meter, flush_size,
Expand All @@ -21,8 +21,7 @@ namespace db0
},
[this](bool threshold_reached) -> bool {
return this->onCacheFlushed(threshold_reached);
},
suppress_dist_overflow_error ? *suppress_dist_overflow_error : false
}
)
, m_slab_recycler(slab_cache_size ? *slab_cache_size : DEFAULT_SLAB_CACHE_SIZE)
{
Expand Down Expand Up @@ -230,8 +229,8 @@ namespace db0
std::optional<std::size_t> slab_cache_size, std::optional<std::size_t> vobject_cache_size,
std::optional<std::size_t> flush_size, std::function<void(db0::swine_ptr<Fixture> &, bool, bool, bool)> fixture_initializer,
std::shared_ptr<Config> config, std::optional<LockFlags> default_lock_flags)
: BaseWorkspace(root_path, cache_size, slab_cache_size, flush_size,
default_lock_flags, config ? config->get<bool>("suppress_dist_overflow_error") : false)
: BaseWorkspace(root_path, cache_size, slab_cache_size, flush_size,
default_lock_flags)
, m_config(config)
, m_fixture_catalog(m_prefix_catalog)
, m_fixture_initializer(fixture_initializer)
Expand Down
3 changes: 1 addition & 2 deletions src/dbzero/workspace/Workspace.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,7 @@ namespace db0
**/
BaseWorkspace(const std::string &root_path = "", std::optional<std::size_t> cache_size = {},
std::optional<std::size_t> slab_cache_size = {}, std::optional<std::size_t> flush_size = {},
std::optional<LockFlags> default_lock_flags = {},
std::optional<bool> suppress_dist_overflow_error = {});
std::optional<LockFlags> default_lock_flags = {});
virtual ~BaseWorkspace()= default;

/**
Expand Down