diff --git a/AGENTS.md b/AGENTS.md index 0d8fbae0..9d47f194 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,10 @@ Never mark a task done while tests are failing. ## Implementation notes +### Scope discipline + +Never make cosmetic changes unless they are explicitly requested by the task. + ### Temporary immutable objects design When working on immutable objects, use `design/IMMUTABLE_OBJECTS_DESIGN.md` as the implementation design reference. This file is temporary and should be removed after the feature is complete. diff --git a/src/dbzero/core/crdt/CRDT_Allocator.cpp b/src/dbzero/core/crdt/CRDT_Allocator.cpp index f89fdc2a..106cf81a 100644 --- a/src/dbzero/core/crdt/CRDT_Allocator.cpp +++ b/src/dbzero/core/crdt/CRDT_Allocator.cpp @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -21,7 +22,7 @@ namespace db0 : m_allocs(allocs) { } - + std::optional tryAlloc(std::size_t size, std::optional addr_bound) { auto hint = &m_hints[0]; @@ -39,9 +40,9 @@ namespace db0 } return std::nullopt; } - + void addMutable(AllocIterator new_alloc, Alloc *) - { + { auto hint = &m_hints[0]; auto new_hint = new_alloc.first->getHint(); for (auto &alloc : m_cache) { @@ -53,9 +54,9 @@ namespace db0 std::swap(alloc, new_alloc); std::swap(*hint, new_hint); ++hint; - } + } } - + /** * Invalidate all cached iterators */ @@ -68,7 +69,7 @@ namespace db0 std::array m_cache; std::array, N> m_hints; }; - + std::uint32_t getMinAlignedAllocSize(std::optional min_aligned_alloc_size, std::uint32_t page_size) { // use page_size + 1 as the default minimum @@ -96,14 +97,14 @@ namespace db0 assert(!m_bounds_fn || std::get<1>(m_bounds_fn()) < std::get<2>(m_bounds_fn())); if (!m_allocs.empty()) { m_max_addr = m_allocs.find_max().first->endAddr(); - } + } assert(!m_bounds_fn || m_max_addr <= std::get<2>(m_bounds_fn())); } CRDT_Allocator::~CRDT_Allocator() { } - + CRDT_Allocator::Alloc::Alloc(std::uint32_t address, std::uint32_t stride, std::uint32_t size, bool has_stripe) : m_address(address) , m_stride(stride) @@ -115,11 +116,11 @@ namespace db0 std::uint32_t CRDT_Allocator::Alloc::allocUnit() { return m_address + m_stride * m_fill_map.allocUnit(); } - + void CRDT_Allocator::Alloc::setHasStripe(bool has_stripe) { m_fill_map.setHasStripe(has_stripe); } - + void CRDT_Allocator::Alloc::setLostStripe() { m_fill_map.setLostStripe(); } @@ -150,7 +151,7 @@ namespace db0 if (isFull()) { return std::nullopt; } - + auto revert = m_fill_map; auto hint_revert = hint_index; auto result = m_address + m_stride * m_fill_map.allocUnit(end_index, hint_index); @@ -160,7 +161,7 @@ namespace db0 hint_index = hint_revert; return std::nullopt; } - return result; + return result; } std::uint32_t CRDT_Allocator::Alloc::endAddr() const { @@ -170,7 +171,7 @@ namespace db0 std::uint32_t CRDT_Allocator::Alloc::span() const { return m_fill_map.span() * m_stride; } - + std::uint32_t CRDT_Allocator::Alloc::getAllocSize(std::uint32_t address) const { // Get allocation size under a specific address or throw exception @@ -179,7 +180,7 @@ namespace db0 } return m_stride; } - + bool CRDT_Allocator::Alloc::isAllocated(std::uint32_t address, std::size_t *size_of_result) const { if (((address >= m_address) && ((address - m_address) % m_stride == 0) && (address < m_address + m_stride * m_fill_map.size()) && @@ -193,6 +194,18 @@ namespace db0 return false; } + std::pair CRDT_Allocator::Alloc::findAllocation(std::uint32_t address) const + { + if (address < m_address || address >= m_address + m_stride * m_fill_map.size()) { + THROWF(db0::BadAddressException) << "Invalid address: " << address; + } + auto index = static_cast((address - m_address) / m_stride); + if (!m_fill_map[index]) { + THROWF(db0::BadAddressException) << "Invalid address: " << address; + } + return std::make_pair(m_address + index * m_stride, m_stride); + } + bool CRDT_Allocator::Alloc::deallocUnit(std::uint32_t address) { if ((address >= m_address) && ((address - m_address) % m_stride == 0) && (address < m_address + m_stride * m_fill_map.size())) { @@ -204,8 +217,8 @@ namespace db0 } return !m_fill_map.empty(); } - - CRDT_Allocator::Blank CRDT_Allocator::Alloc::reclaimSpace(std::uint32_t min_size) + + CRDT_Allocator::Blank CRDT_Allocator::Alloc::reclaimSpace(std::uint32_t min_size) { auto old_size = size(); auto unit_count = (min_size - 1) / m_stride + 1; @@ -223,7 +236,7 @@ namespace db0 CRDT_Allocator::Stripe CRDT_Allocator::Alloc::toStripe() const { return Stripe(m_stride, m_address); } - + CRDT_Allocator::FillMap::FillMap(std::uint32_t size, bool has_stripe) : m_data(has_stripe ? db0::crdt::HAS_STRIPE_BIT : 0) { @@ -262,7 +275,7 @@ namespace db0 } return unused; } - + unsigned int CRDT_Allocator::FillMap::tryDownsize(unsigned int min_units) { auto unused_units = unused(); @@ -284,7 +297,7 @@ namespace db0 // unable to reclaim any space return 0; } - + bool CRDT_Allocator::FillMap::canDownsize(unsigned int min_units) const { auto unused_units = unused(); @@ -302,11 +315,11 @@ namespace db0 // unable to reclaim any space return false; } - + void CRDT_Allocator::FillMap::setLostStripe() { - m_data |= crdt::LOST_STRIPE_BIT; + m_data |= crdt::LOST_STRIPE_BIT; } - + void CRDT_Allocator::FillMap::setHasStripe(bool has_stripe) { if (has_stripe) { @@ -332,7 +345,7 @@ namespace db0 } std::uint32_t CRDT_Allocator::FillMap::allocUnit(std::uint32_t end_index, std::uint32_t &hint_index) - { + { crdt::bitarray_t m_mask = (crdt::bitarray_t)0x01 << hint_index; for (;hint_index != end_index; m_mask <<= 1, ++hint_index) { if (!(m_data & m_mask)) { @@ -343,11 +356,11 @@ namespace db0 hint_index = allocUnit(); return hint_index++; } - + std::pair CRDT_Allocator::FillMap::getHint() const { return { size(), 0 }; } - + bool CRDT_Allocator::FillMap::empty() const { return !(m_data & crdt::NRESERVED_MASK()); } @@ -355,7 +368,7 @@ namespace db0 void CRDT_Allocator::FillMap::reset(unsigned int index) { m_data &= ~((crdt::bitarray_t)0x01 << index); } - + CRDT_Allocator::Stripe::Stripe(std::uint32_t stride, std::uint32_t address) : m_stride(stride) , m_address(address) @@ -387,7 +400,7 @@ namespace db0 return *result; } } - + // try raclaiming aligned space (at least size + 1DP - 1) // FIXME: blocked due to performance issues // if (!tryReclaimSpaceFromStripes(std::max(size, static_cast(size * m_page_size - 1)))) { @@ -395,7 +408,7 @@ namespace db0 // } break; } - + // out of memory return std::nullopt; } @@ -409,7 +422,7 @@ namespace db0 } std::uint32_t last_stripe_units = 0; - // aligned ranges cannot be allocated from stripes + // aligned ranges cannot be allocated from stripes auto result = tryAllocFromStripe(size, last_stripe_units); if (result) { // address must be within the dynamic bounds (below red limit) @@ -427,7 +440,7 @@ namespace db0 unsigned int start_index = crdt::NSIZE - 1; while (start_index > 0 && last_stripe_units >= crdt::SIZE_MAP[start_index]) --start_index; - + for (;;) { if (!m_blanks.empty() || !m_aligned_blanks.empty()) { std::optional max_blank_size; @@ -447,10 +460,10 @@ namespace db0 auto max = m_blanks.find_max(); assert(max.first); max_blank_size = max.first->m_size; - } + } } } - + // FIXME: blocked due to performance issues // if (!tryReclaimSpaceFromStripes(size)) { // break; @@ -459,9 +472,9 @@ namespace db0 } // out of memory - return std::nullopt; + return std::nullopt; } - + void CRDT_Allocator::eraseBlank(const Blank &blank) { // primary index, holds all blanks @@ -471,7 +484,7 @@ namespace db0 eraseBlank(m_aligned_blanks, blank); } } - + void CRDT_Allocator::insertBlank(const Blank &blank) { m_blanks.insert(blank); @@ -479,7 +492,7 @@ namespace db0 m_aligned_blanks.insert(blank); } } - + void CRDT_Allocator::insertBlank(BlankSetT &blanks, AlignedBlankSetT &aligned_blanks, const Blank &blank, std::uint32_t page_size, std::optional min_aligned_alloc_size) { @@ -488,11 +501,11 @@ namespace db0 aligned_blanks.insert(blank); } } - + bool CRDT_Allocator::tryReclaimSpaceFromStripes(std::uint32_t min_size) { using AllocWindowT = typename CRDT_Allocator::AllocSetT::WindowT; - + if (!greenZone()) { // operation disallowed when not in the "green zone" return false; @@ -513,7 +526,7 @@ namespace db0 if (!m_allocs.lower_equal_window(stripe.m_address, alloc_window)) { THROWF(db0::BadAddressException) << "Invalid address: " << stripe.m_address; } - + // NOTE: modify invalidates the entire window, therefore dedicated "modify" version is used assert(!alloc_window[1].isEnd()); // the additional check is to avoid unnecessary modifications @@ -548,7 +561,7 @@ namespace db0 eraseBlank(*b1); blank.m_size += b1->m_size; } - + // space has been successfully reclaimed, now add the newly created blank insertBlank(blank); // remove the stripe if this alloc is full @@ -570,7 +583,7 @@ namespace db0 using AllocWindowT = typename CRDT_Allocator::AllocSetT::WindowT; AllocWindowT alloc_window; if (!m_allocs.lower_equal_window(address, alloc_window)) { - THROWF(db0::BadAddressException) << "Invalid address: " << address; + THROWF(db0::BadAddressException) << "Invalid address: " << address; } assert(!alloc_window[1].isEnd()); const auto alloc = *alloc_window[1].first; @@ -592,7 +605,7 @@ namespace db0 // NOTE: we're unable to perform this operation when in the "red zone" // this will result in stripe-related address space irrecoverably lost if (redZone()) { - // need to mark the entire stripe's space as used (since it's unreachable to future allocs) + // need to mark the entire stripe's space as used (since it's unreachable to future allocs) m_allocs.modify(alloc_window)->setLostStripe(); m_alloc_delta += alloc.capacity(); m_loss_delta += alloc.capacity(); @@ -604,7 +617,7 @@ namespace db0 // just deallocated a single unit return; } - + // if the associated stripe exists then remove it if (criticalZone()) { // Do not perform any cleanups when in the critical zone @@ -615,11 +628,11 @@ namespace db0 if (alloc.hasStripe()) { auto stripe = m_stripes.find_equal(alloc.toStripe()); - assert(stripe.first); - m_stripes.erase(stripe); + assert(stripe.first); + m_stripes.erase(stripe); // NOTE: no need to remove the "has stripe" flag since alloc is to be removed } - + // we need to remove the alloc entry since it's empty std::optional b0, b1; if (!alloc_window[0].isEnd()) { @@ -648,7 +661,7 @@ namespace db0 b1 = Blank(m_size - alloc.m_address - alloc.size(), alloc.m_address + alloc.size()); } } - + // remove blanks if (b0) { eraseBlank(*b0); @@ -656,7 +669,7 @@ namespace db0 if (b1) { eraseBlank(*b1); } - + // L0 cache must be invalidated m_cache->clear(); // remove the allocation @@ -674,21 +687,21 @@ namespace db0 if (!b1) { b1 = Blank(alloc.size(), alloc.m_address); } - + // the combined blanks size auto blank_size = b1->m_address + b1->m_size - b0->m_address; insertBlank({ blank_size, b0->m_address }); } - + std::size_t CRDT_Allocator::getAllocSize(std::uint64_t address) const { auto alloc = m_allocs.lower_equal_bound(address); if (alloc.isEnd()) { - THROWF(db0::BadAddressException) << "Invalid address: " << address; + THROWF(db0::BadAddressException) << "Invalid address: " << address; } return alloc.first->getAllocSize(address); } - + bool CRDT_Allocator::isAllocated(std::uint64_t address, std::size_t *size_of_result) const { auto alloc = m_allocs.lower_equal_bound(address); @@ -698,6 +711,19 @@ namespace db0 return alloc.first->isAllocated(address, size_of_result); } + std::pair CRDT_Allocator::findAllocation(std::uint64_t address) const + { + if (address > std::numeric_limits::max()) { + THROWF(db0::BadAddressException) << "Invalid address: " << address; + } + auto alloc = m_allocs.lower_equal_bound(static_cast(address)); + if (alloc.isEnd()) { + THROWF(db0::BadAddressException) << "Invalid address: " << address; + } + auto result = alloc.first->findAllocation(static_cast(address)); + return std::make_pair(static_cast(result.first), static_cast(result.second)); + } + std::optional CRDT_Allocator::tryAlignedAllocFromBlanks(std::uint32_t size) { // operation only allowed when in the "green zone" @@ -708,7 +734,7 @@ namespace db0 if (size < m_page_size * ALIGNED_INDEX_THRESHOLD) { blank = tryPullBlank(m_aligned_blanks, size); } - // if not present, then resort to regular blanks using adjusted blank size (to guarantee alignment) + // if not present, then resort to regular blanks using adjusted blank size (to guarantee alignment) if (!blank) { // blank size must be at least size + page size - 1 blank = tryPullBlank(m_blanks, size + m_page_size - 1); @@ -717,15 +743,15 @@ namespace db0 if (!blank) { return std::nullopt; } - + // L0 cache must be invalidated m_cache->clear(); assert(blank->getAlignedSize(m_mask, m_page_size) >= size); auto addr = blank->getAlignedAddress(m_mask, m_page_size); - + // max_addr must be updated before any updates to allocator's metadata m_max_addr = std::max(m_max_addr, addr + size); - + assert(addr >= blank->m_address); assert(addr + size <= blank->m_address + blank->m_size); // NOTE: has_stripe flag is set here @@ -734,7 +760,7 @@ namespace db0 assert(alloc.first->endAddr() <= m_max_addr); assert(!redZone()); m_stripes.insert(alloc.first->toStripe()); - + // give back the part of the blank before the allocated (aligned) address if (blank->m_address < addr) { insertBlank({ addr - blank->m_address, blank->m_address }); @@ -748,7 +774,7 @@ namespace db0 return result; } - + std::optional CRDT_Allocator::tryAllocFromBlanks(std::uint32_t stride, std::uint32_t count) { // operation only allowed when in the "green zone" @@ -760,7 +786,7 @@ namespace db0 if (!blank) { return std::nullopt; } - + // L0 cache must be invalidated m_cache->clear(); @@ -775,18 +801,18 @@ namespace db0 // register with L0 cache m_cache->addMutable(alloc, alloc.first); } - // register the new alloc with stripes (even if count == 1) + // register the new alloc with stripes (even if count == 1) m_stripes.insert(alloc.first->toStripe()); if (blank->m_size > min_size) { // register remaining part of the blank // note that the remaining part is registered even if it falls outside of the dynamic bounds - // this is by design since the dynamic bounds may change in the future + // this is by design since the dynamic bounds may change in the future insertBlank({ blank->m_size - stride * count, blank->m_address + stride * count }); } return result; } - + std::optional CRDT_Allocator::tryAllocFromStripe(typename StripeSetT::ConstItemIterator &stripe, std::uint32_t &last_stripe_units, std::optional &addr_bound) { @@ -796,7 +822,7 @@ namespace db0 assert(false); THROWF(db0::InternalException) << "CRDT_Allocator internal error: alloc not found"; } - + if (alloc.first->isFull()) { last_stripe_units = alloc.first->getUnitCount(); assert(alloc.first->hasStripe()); @@ -808,7 +834,7 @@ namespace db0 alloc_ptr->setHasStripe(false); return std::nullopt; } - + // allocate from existing stripe auto alloc_ptr = m_allocs.modify(alloc); auto result = alloc_ptr->tryAllocUnit(addr_bound); @@ -816,22 +842,22 @@ namespace db0 // register with cache for fast future retrieval m_cache->addMutable(alloc, alloc_ptr); } - + return result; } - + std::optional CRDT_Allocator::tryAllocFromStripe(std::uint32_t size, std::uint32_t &last_stripe_units) { std::optional addr_bound; if (m_bounds_fn) { addr_bound = std::get<0>(m_bounds_fn()); } - + auto result = m_cache->tryAlloc(size, addr_bound); if (result) { return result; } - + // Find stripe of exactly matching size last_stripe_units = 0; auto stripe_ptr = m_stripes.lower_equal_bound(size); @@ -839,12 +865,12 @@ namespace db0 if (stripe_ptr.isEnd() || stripe_ptr.first->m_stride != size) { return std::nullopt; } - + result = tryAllocFromStripe(stripe_ptr, last_stripe_units, addr_bound); if (last_stripe_units > 0 || result) { return result; } - + // try with other registered stripes (which might be within the dynamic bounds) auto it = m_stripes.upper_slice(stripe_ptr); auto node = it.get().second; @@ -865,7 +891,7 @@ namespace db0 } return result; } - + std::uint64_t CRDT_Allocator::alloc(std::size_t size, bool align) { auto result = tryAlloc(size, align); @@ -874,7 +900,7 @@ namespace db0 } return *result; } - + void CRDT_Allocator::setDynamicBound(BoundsFunctionT bounds_fn) { this->m_bounds_fn = bounds_fn; } @@ -882,15 +908,15 @@ namespace db0 std::uint64_t CRDT_Allocator::getFirstAddress() { return 0; } - + void CRDT_Allocator::commit() const { m_cache->clear(); } - + void CRDT_Allocator::detach() const { m_cache->clear(); } - + std::uint32_t CRDT_Allocator::Blank::getAlignedAddress(std::uint32_t mask, std::uint32_t page_size) const { if (m_address & mask) { @@ -913,11 +939,11 @@ namespace db0 // non-aligned blank return 0; } - } + } // already aligned return m_size; } - + bool CRDT_Allocator::isAligned(const Blank &blank) const { auto aligned_size = blank.getAlignedSize(m_mask, m_page_size); @@ -930,17 +956,17 @@ namespace db0 auto aligned_size = blank.getAlignedSize(getPageMask(page_size), page_size); return aligned_size > getMinAlignedAllocSize(min_aligned_alloc_size, page_size) && aligned_size < page_size * ALIGNED_INDEX_THRESHOLD; } - + bool CRDT_Allocator::redZone() const { assert(!m_bounds_fn || m_max_addr <= std::get<2>(m_bounds_fn())); return m_bounds_fn && m_max_addr >= std::get<1>(m_bounds_fn()); } - + bool CRDT_Allocator::greenZone() const { return !m_bounds_fn || m_max_addr < std::get<0>(m_bounds_fn()); } - + bool CRDT_Allocator::criticalZone() const { return m_bounds_fn && m_max_addr > (std::get<2>(m_bounds_fn()) - m_critical_margin); } @@ -952,9 +978,9 @@ namespace std { ostream &operator<<(ostream &os, const db0::CRDT_Allocator::Blank &blank) { - return os << "size=" << blank.m_size << ", address=" << blank.m_address; + return os << "size=" << blank.m_size << ", address=" << blank.m_address; } - + ostream &operator<<(ostream &os, const db0::CRDT_Allocator::Alloc &alloc) { return os << "address=" << alloc.m_address << ", stride=" << alloc.m_stride << ", size=" << alloc.size(); } @@ -962,5 +988,5 @@ namespace std ostream &operator<<(ostream &os, const db0::CRDT_Allocator::Stripe &stripe) { return os << "stride=" << stripe.m_stride << ", address=" << stripe.m_address; } - + } diff --git a/src/dbzero/core/crdt/CRDT_Allocator.hpp b/src/dbzero/core/crdt/CRDT_Allocator.hpp index 1112982f..a03cf36a 100644 --- a/src/dbzero/core/crdt/CRDT_Allocator.hpp +++ b/src/dbzero/core/crdt/CRDT_Allocator.hpp @@ -13,10 +13,10 @@ namespace db0 { - namespace crdt - + namespace crdt + { - + using bitarray_t = std::uint64_t; static constexpr std::uint32_t SIZE_MAP[4] = { 56, 24, 8, 1 }; static constexpr bitarray_t NSIZE = sizeof(SIZE_MAP) / sizeof(std::uint32_t); @@ -24,15 +24,15 @@ namespace db0 static constexpr std::uint64_t HAS_STRIPE_BIT = (bitarray_t)0x01 << (SIZE_MAP[0] + 2); // flag indicating the "lost stripe" (see free method) static constexpr std::uint64_t LOST_STRIPE_BIT = HAS_STRIPE_BIT << 1; - + static constexpr bitarray_t mask(unsigned int index) { return ((bitarray_t)0x01 << SIZE_MAP[index]) - 1; }; - + static constexpr bitarray_t MASKS[4] = { mask(0), mask(1), mask(2), mask(3) }; - + // Fill-map's size encoding bits static constexpr bitarray_t SIZE_MASK() { return (bitarray_t)0x03 << SIZE_MAP[0]; @@ -40,19 +40,19 @@ namespace db0 static constexpr bitarray_t NSIZE_MASK() { return ~SIZE_MASK(); - } + } // Fill-map reserved bits (i.e. size + flags) static constexpr bitarray_t RESERVED_MASK() { return ((bitarray_t)0x03 << SIZE_MAP[0]) | HAS_STRIPE_BIT | LOST_STRIPE_BIT; } - + static constexpr bitarray_t NRESERVED_MASK() { return ~RESERVED_MASK(); } - + } - + // very small buffer with N recent alloc stripes template class L0_Cache; @@ -62,7 +62,7 @@ namespace db0 * The interface is compatible with the db0::Allocator */ class CRDT_Allocator - { + { public: // upper blank size (as the number of pages) to be registered in the aligned blank index // higher values allow eliminating "bind spots" in the allocator, but may incur storage & performance overhead @@ -70,7 +70,7 @@ namespace db0 DB0_PACKED_BEGIN struct DB0_PACKED_ATTR FillMap - { + { // the low 1 - 56 bits are used to encode unit allocations // the high 2 bits (i.e. 57 - 58) are used to encode size (56, 24, 8, or 1) // bit #59 is used to indicate that the associated stripe exists @@ -79,13 +79,13 @@ DB0_PACKED_BEGIN FillMap() = default; FillMap(std::uint32_t size, bool has_stripe); - + bool operator[](unsigned int index) const; inline std::uint32_t sizeId() const { return (m_data & crdt::SIZE_MASK()) >> crdt::SIZE_MAP[0]; } - + inline std::uint32_t size() const { return crdt::SIZE_MAP[sizeId()]; } @@ -94,12 +94,12 @@ DB0_PACKED_BEGIN inline bool all() const { return (m_data & crdt::NRESERVED_MASK()) == crdt::MASKS[this->sizeId()]; } - + bool empty() const; - + /** * Allocate a single unit - * + * * @param hint the likely index where to allocate the unit (updated) * @return the index of the allocated unit or throw exception */ @@ -116,23 +116,23 @@ DB0_PACKED_BEGIN /** * Try downsize by at least min_units - * + * * @return the number of units by which the resize was done or 0 if not resized */ unsigned int tryDownsize(unsigned int min_units); // Check if downsize by a specific number of units is possible bool canDownsize(unsigned int min_units) const; - + // Get the number of unused units (high order bits) unsigned int unused() const; std::uint32_t span() const; - + // Set or reset the "has stripe" flag void setHasStripe(bool has_stripe); void setLostStripe(); - + inline bool hasStripe() const { return m_data & crdt::HAS_STRIPE_BIT; } @@ -155,17 +155,17 @@ DB0_PACKED_BEGIN std::uint32_t m_address = 0; std::uint32_t m_stride = 0; FillMap m_fill_map; - + Alloc() = default; Alloc(std::uint32_t address, std::uint32_t stride, std::uint32_t size, bool has_stripe); - + static inline std::uint32_t getKey(const Alloc &alloc) { return alloc.m_address; } - + // Extract key from construction args static inline std::uint32_t getKey(std::uint32_t address, std::uint32_t, std::uint32_t, bool) { - return address; + return address; } struct CompT @@ -173,7 +173,7 @@ DB0_PACKED_BEGIN inline bool operator()(const Alloc &lhs, const Alloc &rhs) const { return lhs.m_address < rhs.m_address; } - + inline bool operator()(const Alloc &lhs, std::uint32_t rhs) const { return lhs.m_address < rhs; } @@ -188,7 +188,7 @@ DB0_PACKED_BEGIN inline bool operator()(const Alloc &lhs, const Alloc &rhs) const { return lhs.m_address == rhs.m_address; } - + inline bool operator()(const Alloc &lhs, std::uint32_t rhs) const { return lhs.m_address == rhs; } @@ -197,7 +197,7 @@ DB0_PACKED_BEGIN return lhs == rhs.m_address; } }; - + /** * The reserved size in bytes */ @@ -209,8 +209,8 @@ DB0_PACKED_BEGIN * Get span of the allocated addresses (as number of bytes) */ std::uint32_t span() const; - - // The reserved number of units + + // The reserved number of units std::uint32_t getUnitCount() const { return m_fill_map.size(); } @@ -222,11 +222,11 @@ DB0_PACKED_BEGIN inline bool isFull() const { return m_fill_map.all(); } - + // Set or reset the "has stripe" flag void setHasStripe(bool has_stripe); void setLostStripe(); - + inline bool hasStripe() const { return m_fill_map.hasStripe(); } @@ -237,19 +237,19 @@ DB0_PACKED_BEGIN /** * Allocate a single unit - * + * * @return the address of the allocated unit */ std::uint32_t allocUnit(); - + /** * Try bounded unit allocation */ std::optional tryAllocUnit(std::optional addr_bound); - - std::optional tryAllocUnit(std::optional addr_bound, unsigned int end_index, + + std::optional tryAllocUnit(std::optional addr_bound, unsigned int end_index, unsigned int &hint_index); - + std::pair getHint() const { return m_fill_map.getHint(); } @@ -263,28 +263,30 @@ DB0_PACKED_BEGIN * Validate address and retrieve alloc size (i.e. stride) */ std::uint32_t getAllocSize(std::uint32_t address) const; - + // Validate address bool isAllocated(std::uint32_t address, std::size_t *size_of_result = nullptr) const; - + + std::pair findAllocation(std::uint32_t address) const; + Stripe toStripe() const; /** * Try reducing this alloc to reclaim at least min_size of space - * + * * @param min_size the minimum size to reclaim - * @return the blank corresponding to reclaimed space or an empty blank if no space was reclaimed + * @return the blank corresponding to reclaimed space or an empty blank if no space was reclaimed */ Blank reclaimSpace(std::uint32_t min_size); - + // Check if the alloc can be downsized by at least min_size bool canReclaimSpace(std::uint32_t min_size) const; - + /** * Calculate the address past the last allocated unit */ std::uint32_t endAddr() const; - + // Get total capacity std::uint32_t capacity() const; }; @@ -312,10 +314,10 @@ DB0_PACKED_BEGIN // Get first aligned address within the blank (must satisfy aligned size > 0) std::uint32_t getAlignedAddress(std::uint32_t mask, std::uint32_t page_size) const; - + // Get size of the aligned block within the blank std::uint32_t getAlignedSize(std::uint32_t mask, std::uint32_t page_size) const; - + struct CompT { inline bool operator()(const Blank &lhs, const Blank &rhs) const @@ -327,7 +329,7 @@ DB0_PACKED_BEGIN } else { return lhs.m_address < rhs.m_address; } - } + } }; // Comparator by aligned blank size @@ -339,7 +341,7 @@ DB0_PACKED_BEGIN AlignedCompT(std::uint32_t page_size) : m_mask(getPageMask(page_size)) , m_page_size(page_size) - { + { } inline bool operator()(const Blank &lhs, const Blank &rhs) const @@ -377,7 +379,7 @@ DB0_PACKED_BEGIN static inline Stripe getKey(const Stripe &stripe) { return stripe; } - + // Extract key from construction args static inline Stripe getKey(std::uint32_t stride, std::uint32_t address) { return { stride, address }; @@ -396,7 +398,7 @@ DB0_PACKED_BEGIN return lhs.m_address < rhs.m_address; } } - + inline bool operator()(const Stripe &lhs, std::uint32_t rhs) const { return lhs.m_stride < rhs; } @@ -405,16 +407,16 @@ DB0_PACKED_BEGIN return lhs < rhs.m_stride; } }; - + struct EqualT { inline bool operator()(const Stripe &lhs, const Stripe &rhs) const { return lhs.m_stride == rhs.m_stride && lhs.m_address == rhs.m_address; - } + } inline bool operator()(const Stripe &lhs, std::uint32_t rhs) const { return lhs.m_stride == rhs; - } + } inline bool operator()(std::uint32_t lhs, const Stripe &rhs) const { return lhs == rhs.m_stride; @@ -427,7 +429,7 @@ DB0_PACKED_END using BlankSetT = db0::SGB_Tree; using AlignedBlankSetT = db0::SGB_Tree; using StripeSetT = db0::SGB_Tree; - + public: // The bounds function returns the yellow, red and black bounds // The "yellow" bound includes a margin allowing ALL underlying collections to grow to accomodate at least 1 new allocation @@ -447,32 +449,34 @@ DB0_PACKED_END * @param align if true, the allocation will be aligned to the page boundary */ std::uint64_t alloc(std::size_t size, bool aligned = false); - + /** * @param align if true, the allocation will be aligned to the page boundary */ std::optional tryAlloc(std::size_t size, bool aligned = false); - + void free(std::uint64_t address); - + std::size_t getAllocSize(std::uint64_t address) const; bool isAllocated(std::uint64_t address, std::size_t *size_of_result = nullptr) const; + std::pair findAllocation(std::uint64_t address) const; + /** * This function allows checking the space available to this allocator dynamically * The bounds_fn function is called to assure the allocated address is within the currently available bounds * this functionality is helpful when integrating with other data structures sharing the high-rages of the same address space * Note that checks are performed only during the allocations / not when setting the funtion - * + * * @param bounds_fn the size bounds checking function */ void setDynamicBound(BoundsFunctionT bounds_fn); - + inline std::uint32_t getMaxAddr() const { return m_max_addr; } - + /** * Get cummulative size of allocations / dellocations performed since creation of this instance */ @@ -483,7 +487,7 @@ DB0_PACKED_END std::int64_t getLossDelta() const { return m_loss_delta; } - + /** * Get address of the 1st allocation */ @@ -498,11 +502,11 @@ DB0_PACKED_END static void insertBlank(BlankSetT &blanks, AlignedBlankSetT &aligned_blanks, const Blank &blank, std::uint32_t page_size, std::optional min_aligned_alloc_size = {}); static bool isAligned(const Blank &blank, std::uint32_t page_size, std::optional min_aligned_alloc_size = {}); - + private: AllocSetT &m_allocs; BlankSetT &m_blanks; - // additional index which organizes blanks (only where 0 < aligned size < 2DP) + // additional index which organizes blanks (only where 0 < aligned size < 2DP) AlignedBlankSetT &m_aligned_blanks; StripeSetT &m_stripes; // size of the space available to the allocator (i.e. a single slab) @@ -523,27 +527,27 @@ DB0_PACKED_END std::int64_t m_loss_delta = 0; // the purpose of this cache is to speed up allocations of identical size sequences mutable std::unique_ptr > m_cache; - + std::optional tryAlignedAlloc(std::size_t size); std::optional tryAllocFromStripe(std::uint32_t size, std::uint32_t &last_stripe_units); std::optional tryAllocFromStripe(typename StripeSetT::ConstItemIterator &, std::uint32_t &last_stripe_units, std::optional &cache); - + /** * Try making an allocation from registered blanks - * + * * @param stride the stride size (in bytes) - * @param count the number of strides to pre-allocate + * @param count the number of strides to pre-allocate * @return the address of the allocated unit or std::nullopt if unable to allocate */ - std::optional tryAllocFromBlanks(std::uint32_t stride, std::uint32_t count); + std::optional tryAllocFromBlanks(std::uint32_t stride, std::uint32_t count); std::optional tryAlignedAllocFromBlanks(std::uint32_t size); - + /** * Try reclaiming at least min_size bytes from registered stripes - * + * * @return true if space was reclaimed */ bool tryReclaimSpaceFromStripes(std::uint32_t min_size); @@ -551,40 +555,40 @@ DB0_PACKED_END /** * Check if this blank is within current dynamic bounds (if set) */ - template bool inBounds(const IndexT &, const Blank &, std::uint32_t min_size, + template bool inBounds(const IndexT &, const Blank &, std::uint32_t min_size, std::optional &cache) const; // Get first accessible address from the blank from a given index // the purpose of this function is to be able to retrieve aligned addresses if blank comes from the aligned index template std::uint32_t getFirstAddress(const IndexT &, const Blank &) const; - + /** * Check if the unit taken from this allocation would be within current dynamic bounds (if set) */ bool inBounds(const Alloc &, std::optional &cache) const; - + // Erase either from blanks or from aligned blanks template void eraseBlank(IndexT &index, const Blank &); template bool isAlignedIndex(const IndexT &index) const; - + // Find blank of a given minimal size and remove it from a specific index template std::optional tryPullBlank(IndexT &index, std::uint32_t min_size); - + // Erase record from the corresponding indexes (m_blanks / m_aligned_blanks) void eraseBlank(const Blank &); // Insert with proper indexes (m_blanks / m_aligned_blanks) void insertBlank(const Blank &); - + // Checks if the aligned size is in the range 0 < aligned size < 1DP bool isAligned(const Blank &) const; - + bool greenZone() const; // check if max_addr crossed the dynamic bounds (red watermark) bool redZone() const; // critical zone is when no more meta-data can be safely appended bool criticalZone() const; }; - + template bool CRDT_Allocator::inBounds(const IndexT &index, const Blank &blank, std::uint32_t size, std::optional &cache) const { @@ -597,13 +601,13 @@ DB0_PACKED_END } return true; } - + template bool CRDT_Allocator::isAlignedIndex(const IndexT &index) const { return reinterpret_cast(&index) == reinterpret_cast(&m_aligned_blanks); } - + template void CRDT_Allocator::eraseBlank(IndexT &index, const Blank &blank) { @@ -611,12 +615,12 @@ DB0_PACKED_END if (!it.first) { assert(false && "CRDT_Allocator internal error: blank not found"); THROWF(db0::InternalException) - << "CRDT_Allocator internal error: blank not found (address = " << blank.m_address + << "CRDT_Allocator internal error: blank not found (address = " << blank.m_address << ", size = " << blank.m_size << ")" << THROWF_END; } index.erase(it); } - + template std::optional CRDT_Allocator::tryPullBlank(IndexT &index, std::uint32_t min_size) { @@ -626,7 +630,7 @@ DB0_PACKED_END if (blank_ptr.isEnd()) { return std::nullopt; } - + // validate dynamic bounds if such exist std::optional cache; if (inBounds(index, *blank_ptr.first, min_size, cache)) { @@ -657,7 +661,7 @@ DB0_PACKED_END if (it.is_end()) { return std::nullopt; } - + Blank blank = *it; eraseBlank(index, blank); if (isAlignedIndex(index)) { @@ -672,7 +676,7 @@ DB0_PACKED_END return blank; } } - + template std::uint32_t CRDT_Allocator::getFirstAddress(const IndexT &index, const Blank &blank) const { @@ -684,7 +688,7 @@ DB0_PACKED_END } -namespace std +namespace std { @@ -692,4 +696,4 @@ namespace std ostream &operator<<(ostream &os, const db0::CRDT_Allocator::Blank &); ostream &operator<<(ostream &os, const db0::CRDT_Allocator::Stripe &); -} \ No newline at end of file +} diff --git a/src/dbzero/core/dram/DRAM_Allocator.cpp b/src/dbzero/core/dram/DRAM_Allocator.cpp index 87866070..a02a8613 100644 --- a/src/dbzero/core/dram/DRAM_Allocator.cpp +++ b/src/dbzero/core/dram/DRAM_Allocator.cpp @@ -94,9 +94,9 @@ namespace db0 { // address validity not checked here auto offset = address % m_page_size; - return m_page_size - offset; + return m_page_size - offset; } - + bool DRAM_Allocator::isAllocated(Address address, std::size_t *size_of_result) const { auto page_id = address / m_page_size; @@ -113,15 +113,24 @@ namespace db0 } if (size_of_result) { auto offset = address % m_page_size; - *size_of_result = m_page_size - offset; + *size_of_result = m_page_size - offset; } return true; } - + + Allocator::AllocationInfo DRAM_Allocator::findAllocation(Address address) const + { + auto pageId = address / m_page_size; + if (pageId < FIRST_PAGE_ID || pageId >= m_next_page_id || m_free_pages.find(pageId) != m_free_pages.end()) { + THROWF(db0::BadAddressException) << "Invalid address: " << address; + } + return AllocationInfo { Address::fromOffset(pageId * m_page_size), m_page_size }; + } + Address DRAM_Allocator::firstAlloc() const { return Address::fromOffset(FIRST_PAGE_ID * m_page_size); } - + void DRAM_Allocator::commit() const { } @@ -130,4 +139,4 @@ namespace db0 { } -} \ No newline at end of file +} diff --git a/src/dbzero/core/dram/DRAM_Allocator.hpp b/src/dbzero/core/dram/DRAM_Allocator.hpp index dce3553c..8dbfa110 100644 --- a/src/dbzero/core/dram/DRAM_Allocator.hpp +++ b/src/dbzero/core/dram/DRAM_Allocator.hpp @@ -11,7 +11,7 @@ namespace db0 { /** - * In-memory only allocator, allocates only whole memory pages + * In-memory only allocator, allocates only whole memory pages */ class DRAM_Allocator: public Allocator { @@ -22,25 +22,27 @@ namespace db0 * Create pre-populated with existing allocations */ DRAM_Allocator(const std::unordered_set &allocs, std::size_t page_size); - + /** * Update with externally provided list of allocations (add new allocations) */ void update(const std::unordered_set &allocs); - std::optional
tryAlloc(std::size_t size, std::uint32_t slot_num = 0, + std::optional
tryAlloc(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0) override; - + void free(Address) override; std::size_t getAllocSize(Address) const override; bool isAllocated(Address, std::size_t *size_of_result = nullptr) const override; - + + AllocationInfo findAllocation(Address) const override; + void commit() const override; void detach() const override; - + /** * Get address of the 1st allocation */ @@ -53,5 +55,5 @@ namespace db0 std::size_t m_next_page_id = FIRST_PAGE_ID; std::unordered_set m_free_pages; }; - -} \ No newline at end of file + +} diff --git a/src/dbzero/core/memory/AlgoAllocator.cpp b/src/dbzero/core/memory/AlgoAllocator.cpp index 60fb1b14..40be0a00 100644 --- a/src/dbzero/core/memory/AlgoAllocator.cpp +++ b/src/dbzero/core/memory/AlgoAllocator.cpp @@ -12,10 +12,10 @@ namespace db0 AlgoAllocator::AlgoAllocator(AddressPoolF f, ReverseAddressPoolF rf, std::size_t alloc_size) : m_address_pool_f(f) , m_reverse_address_pool_f(rf) - , m_alloc_size(alloc_size) + , m_alloc_size(alloc_size) { } - + std::optional
AlgoAllocator::tryAlloc(std::size_t size, std::uint32_t slot_num, bool aligned, unsigned char, unsigned char) { @@ -24,7 +24,7 @@ namespace db0 assert(size == m_alloc_size && "AlgoAllocator: invalid alloc size requested"); return m_address_pool_f(m_next_i++); } - + void AlgoAllocator::free(Address address) { if (address % m_alloc_size != 0) { @@ -39,7 +39,7 @@ namespace db0 --m_next_i; } } - + std::size_t AlgoAllocator::getAllocSize(Address address) const { auto offset = address % m_alloc_size; @@ -49,7 +49,7 @@ namespace db0 } return m_alloc_size - offset; } - + bool AlgoAllocator::isAllocated(Address address, std::size_t *size_of_result) const { auto offset = address % m_alloc_size; @@ -62,27 +62,43 @@ namespace db0 } return true; } - + + Allocator::AllocationInfo AlgoAllocator::findAllocation(Address address) const + { + auto offset = address % m_alloc_size; + auto baseAddress = address - offset; + unsigned int i; + try { + i = m_reverse_address_pool_f(baseAddress); + } catch (const db0::AbstractException &) { + THROWF(db0::BadAddressException) << "Invalid address: " << address << THROWF_END; + } + if (i >= m_next_i) { + THROWF(db0::BadAddressException) << "Invalid address: " << address; + } + return AllocationInfo { baseAddress, m_alloc_size }; + } + void AlgoAllocator::reset() { m_next_i = 0; } - + Address AlgoAllocator::getRootAddress() const { return m_address_pool_f(0); } - + void AlgoAllocator::setMaxAddress(Address max_address) { auto offset = max_address % m_alloc_size; m_next_i = m_reverse_address_pool_f(max_address - offset) + 1; } - + void AlgoAllocator::commit() const { } - + void AlgoAllocator::detach() const { } -} \ No newline at end of file +} diff --git a/src/dbzero/core/memory/AlgoAllocator.hpp b/src/dbzero/core/memory/AlgoAllocator.hpp index 61792c87..170d2f4a 100644 --- a/src/dbzero/core/memory/AlgoAllocator.hpp +++ b/src/dbzero/core/memory/AlgoAllocator.hpp @@ -22,13 +22,15 @@ namespace db0 std::optional
tryAlloc(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0) override; - + void free(Address) override; std::size_t getAllocSize(Address) const override; bool isAllocated(Address, std::size_t *size_of_result = nullptr) const override; - + + AllocationInfo findAllocation(Address) const override; + void commit() const override; void detach() const override; @@ -42,12 +44,12 @@ namespace db0 * Reset the allocator to the initial state (as if no allocation was done) */ void reset(); - + /** * Get the first assigned i.e. the root address */ Address getRootAddress() const; - + private: AddressPoolF m_address_pool_f; ReverseAddressPoolF m_reverse_address_pool_f; @@ -55,4 +57,4 @@ namespace db0 unsigned int m_next_i = 0; }; -} \ No newline at end of file +} diff --git a/src/dbzero/core/memory/Allocator.cpp b/src/dbzero/core/memory/Allocator.cpp index 0e92a7bd..3a2b781f 100644 --- a/src/dbzero/core/memory/Allocator.cpp +++ b/src/dbzero/core/memory/Allocator.cpp @@ -7,15 +7,15 @@ namespace db0 { - + std::optional Allocator::tryAllocUnique( std::size_t, std::uint32_t, bool, unsigned char, unsigned char) { - THROWF(InternalException) + THROWF(InternalException) << "Allocator: unique allocation not supported by: " << typeid(*this).name() << THROWF_END; } - - Address Allocator::alloc(std::size_t size, std::uint32_t slot_num, bool aligned, + + Address Allocator::alloc(std::size_t size, std::uint32_t slot_num, bool aligned, unsigned char realm_id, unsigned char locality) { auto result = tryAlloc(size, slot_num, aligned, realm_id, locality); @@ -24,8 +24,8 @@ namespace db0 } return *result; } - - UniqueAddress Allocator::allocUnique(std::size_t size, std::uint32_t slot_num, bool aligned, + + UniqueAddress Allocator::allocUnique(std::size_t size, std::uint32_t slot_num, bool aligned, unsigned char realm_id, unsigned char locality) { auto result = tryAllocUnique(size, slot_num, aligned, realm_id, locality); @@ -34,26 +34,36 @@ namespace db0 } return *result; } - + void Allocator::flush() const { } - + bool Allocator::inRange(Address) const { return true; } - + std::size_t Allocator::getAllocSize(Address address, unsigned char realm_id) const { return getAllocSize(address); } - + bool Allocator::isAllocated(Address address, unsigned char, std::size_t *size_of_result) const { return isAllocated(address, size_of_result); } - + + Allocator::AllocationInfo Allocator::findAllocation(Address) const + { + THROWF(InternalException) + << "Allocator: allocation lookup by inner address not supported by: " << typeid(*this).name() << THROWF_END; + } + + Allocator::AllocationInfo Allocator::findAllocation(Address address, unsigned char) const { + return findAllocation(address); + } + std::pair > Allocator::getRange(std::uint32_t slot_num) const { if (slot_num != 0) { - THROWF(InternalException) << "Invalid / unsupported slot number"; + THROWF(InternalException) << "Invalid / unsupported slot number"; } return { Address::fromOffset(0), std::nullopt }; } @@ -62,4 +72,4 @@ namespace db0 { } -} \ No newline at end of file +} diff --git a/src/dbzero/core/memory/Allocator.hpp b/src/dbzero/core/memory/Allocator.hpp index c425f6d8..4246ecc4 100644 --- a/src/dbzero/core/memory/Allocator.hpp +++ b/src/dbzero/core/memory/Allocator.hpp @@ -13,14 +13,20 @@ namespace db0 { - + /** * The DB0 allocator interface * NOTE: allocators may return logical adddress which needs to be converted to physical one */ class Allocator { - public: + public: + struct AllocationInfo + { + Address address; + std::size_t size; + }; + /** * @param the allocation size in bytes * @param slot_num optional slot number to allocate from (slot_num = 0 means any slot). @@ -33,29 +39,29 @@ namespace db0 */ virtual std::optional
tryAlloc(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0) = 0; - + // Try allocating a unique, never repeating address // NOTE: this functionality is only supported by some allocators // The default throwing implementation is provided virtual std::optional tryAllocUnique(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0); - + /** * Free previously allocated address * @param address the address previously returned by alloc (the memory offset part) */ virtual void free(Address) = 0; - + /** * Retrieve size of the range allocated under a specific address - * + * * @param address the address previously returned by alloc * @return the range size in bytes */ virtual std::size_t getAllocSize(Address) const = 0; // getAllocSize with realm_id validatation (where unsupported simply forwards to the non-realm version) virtual std::size_t getAllocSize(Address, unsigned char realm_id) const; - + /** * Check if the address is a valid allocation address with this allocator * size_of retrieved on request (if size_of_result is not null) @@ -63,7 +69,16 @@ namespace db0 virtual bool isAllocated(Address, std::size_t *size_of_result = nullptr) const = 0; // isAllocated version with realm_id validation virtual bool isAllocated(Address, unsigned char realm_id, std::size_t *size_of_result = nullptr) const; - + + /** + * Find a live allocation containing the given address. + * @return the allocation base address and full allocation size + * @throws BadAddressException if no allocation contains address + */ + virtual AllocationInfo findAllocation(Address) const; + // findAllocation version with realm_id validation + virtual AllocationInfo findAllocation(Address, unsigned char realm_id) const; + /** * Prepare the allocator for the next transaction */ @@ -74,30 +89,30 @@ namespace db0 // Flush any pending deferred operations (e.g. deferred free) // the default empty implementation is provieded virtual void flush() const; - + /** * Allocate a new continuous range of a given size - * + * * @param size size (in bytes) of the range to be allocated * @param slot_num optional slot number to allocate from (slot_num = 0 means any slot). * @return the address of the range */ - Address alloc(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, + Address alloc(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0); - - UniqueAddress allocUnique(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, + + UniqueAddress allocUnique(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0); - + // Check if the address is within the range managed by the allocator // (only applicable to limited allocators - e.g. SlabAllocator) virtual bool inRange(Address) const; - + // Get range covered by the allocator or a specific slot // @return begin / end (which might be undefined for unlimited allocators) virtual std::pair > getRange(std::uint32_t slot_num = 0) const; - + // To be implemented where it makes sense virtual void close(); }; - -} \ No newline at end of file + +} diff --git a/src/dbzero/core/memory/BitsetAllocator.hpp b/src/dbzero/core/memory/BitsetAllocator.hpp index 47292f04..93e99a56 100644 --- a/src/dbzero/core/memory/BitsetAllocator.hpp +++ b/src/dbzero/core/memory/BitsetAllocator.hpp @@ -22,23 +22,25 @@ namespace db0 * @param bitset the underlying bit container compatible with FixedBitset * @param base_address the begin / end address of the managed range (depends on direction) * @param alloc_size the allowed allocation size, typically equal data page size - * @param direction either 1 or -1 (the direction in which the addresses are allocated) + * @param direction either 1 or -1 (the direction in which the addresses are allocated) */ BitsetAllocator(BitSetT &&bitset, Address base_addr, std::size_t alloc_size, int direction); std::optional
tryAlloc(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0) override; - + void free(Address) override; std::size_t getAllocSize(Address) const override; - + bool isAllocated(Address, std::size_t *size_of_result = nullptr) const override; - + + AllocationInfo findAllocation(Address) const override; + void commit() const override; void detach() const override; - + /// Get the total number of allocations std::size_t getAllocCount() const; @@ -47,13 +49,13 @@ namespace db0 Address getBaseAddress() const { return m_base_addr; } - + /// Get total size of the area occupied by allocations (as the number of allocations) std::size_t span() const; /** * Enable dynamic bounds checking - * + * * @param bounds_fn the function to return the address threshold to not be exceeded */ void setDynamicBounds(std::function bounds_fn); @@ -72,7 +74,7 @@ namespace db0 std::function m_bounds_fn; // pre-calculated span std::size_t m_span = 0; - + unsigned int indexOf(std::uint64_t address) const { if (m_direction > 0) { @@ -92,16 +94,16 @@ namespace db0 , m_alloc_size(alloc_size) , m_base_addr(base_addr) , m_direction(direction) - , m_shift(direction > 0 ? 0 : alloc_size) - , m_span(calculateSpan()) + , m_shift(direction > 0 ? 0 : alloc_size) + , m_span(calculateSpan()) { } - + template std::optional
BitsetAllocator::tryAlloc(std::size_t size, std::uint32_t slot_num, bool aligned, unsigned char, unsigned char) { assert(slot_num == 0); - // all BitSetAllocator allocations are aligned + // all BitSetAllocator allocations are aligned assert(size == m_alloc_size && "BitsetAllocator: invalid alloc size requested"); auto index = m_bitset->firstIndexOf(false); if (index == m_bitset.npos) { @@ -122,7 +124,7 @@ namespace db0 } m_bitset.modify().set(index, true); - m_span = calculateSpan(); + m_span = calculateSpan(); return Address::fromOffset(addressOf(index)); } @@ -130,7 +132,7 @@ namespace db0 { if (address % m_alloc_size != 0) { // do not dealloc sub-addresses - return; + return; } std::uint64_t index = indexOf(address); if (index >= m_bitset.npos || !m_bitset->get(index)) { @@ -139,7 +141,7 @@ namespace db0 m_bitset.modify().set(index, false); m_span = calculateSpan(); } - + template std::size_t BitsetAllocator::getAllocSize(Address address) const { auto inner_offset = address % m_alloc_size; @@ -165,6 +167,24 @@ namespace db0 return true; } + template + Allocator::AllocationInfo BitsetAllocator::findAllocation(Address address) const + { + auto innerOffset = address % m_alloc_size; + auto baseAddress = address - innerOffset; + if (m_direction > 0 && baseAddress < m_base_addr) { + THROWF(db0::BadAddressException) << "Invalid address: " << address; + } + if (m_direction < 0 && !(baseAddress < m_base_addr)) { + THROWF(db0::BadAddressException) << "Invalid address: " << address; + } + auto index = indexOf(baseAddress); + if (index >= m_bitset.npos || !m_bitset->get(index)) { + THROWF(db0::BadAddressException) << "Invalid address: " << address; + } + return AllocationInfo { baseAddress, m_alloc_size }; + } + template std::size_t BitsetAllocator::getAllocCount() const { return m_bitset->count(true); } @@ -177,14 +197,14 @@ namespace db0 return m_bitset->lastIndexOf(true) + 1; } - template void BitsetAllocator::setDynamicBounds(std::function bounds_fn) { + template void BitsetAllocator::setDynamicBounds(std::function bounds_fn) { m_bounds_fn = bounds_fn; } - + template std::size_t BitsetAllocator::span() const { return m_span; } - + template void BitsetAllocator::commit() const { m_bitset.commit(); } @@ -193,4 +213,4 @@ namespace db0 m_bitset.detach(); } -} \ No newline at end of file +} diff --git a/src/dbzero/core/memory/MetaAllocator.cpp b/src/dbzero/core/memory/MetaAllocator.cpp index 098bf196..60e158c6 100644 --- a/src/dbzero/core/memory/MetaAllocator.cpp +++ b/src/dbzero/core/memory/MetaAllocator.cpp @@ -13,7 +13,7 @@ namespace db0 { static constexpr double MIN_FILL_RATE = 0.25; - + inline unsigned char getRealmID(std::uint32_t slab_id) { return slab_id & MetaAllocator::REALM_MASK; } @@ -32,9 +32,9 @@ namespace db0 std::size_t align(std::size_t address, std::size_t page_size) { return ((address + page_size - 1) / page_size) * page_size; } - - std::function MetaAllocator::getAddressPool(std::size_t offset, std::size_t page_size, - std::size_t slab_size) + + std::function MetaAllocator::getAddressPool(std::size_t offset, std::size_t page_size, + std::size_t slab_size) { auto slab_count = getSlabCount(page_size, slab_size); // make offset page-aligned @@ -51,7 +51,7 @@ namespace db0 } // Construct the reverse address pool function - std::function MetaAllocator::getReverseAddressPool(std::size_t offset, + std::function MetaAllocator::getReverseAddressPool(std::size_t offset, std::size_t page_size, std::size_t slab_size) { auto slab_count = getSlabCount(page_size, slab_size); @@ -69,7 +69,7 @@ namespace db0 return x + (d / page_size); }; } - + std::function MetaAllocator::getSlabIdFunction(std::size_t offset, std::size_t page_size, std::size_t slab_size) { @@ -81,9 +81,9 @@ namespace db0 auto block_id = (address - offset) / block_size; auto slab_num = (address - offset - block_id * block_size - MP * page_size) / slab_size; return block_id * slab_count + slab_num; - }; + }; } - + // Get function to translate slab id to slab address std::function getSlabAddressFunction(std::size_t offset, std::size_t page_size, std::size_t slab_size) { @@ -98,13 +98,13 @@ namespace db0 return Address::fromOffset(offset + block_id * block_size + MP * page_size + slab_num * slab_size); }; } - + o_meta_header::o_meta_header(std::uint32_t page_size, std::uint32_t slab_size) : m_page_size(page_size) , m_slab_size(slab_size) { } - + std::uint64_t MetaAllocator::Realm::getSlabMaxAddress() const { // take max of the 2 collections @@ -120,7 +120,7 @@ namespace db0 } return max_addr; } - + MetaAllocator::MetaAllocator(std::shared_ptr prefix, SlabRecycler *recycler, bool deferred_free) : m_prefix(prefix) , m_header(getMetaHeader(prefix)) @@ -131,7 +131,7 @@ namespace db0 ) , m_metaspace(createMetaspace()) , m_realms(m_metaspace, m_prefix, recycler, m_header, NUM_REALMS, deferred_free) - , m_recycler_ptr(recycler) + , m_recycler_ptr(recycler) , m_slab_id_function(getSlabIdFunction(o_meta_header::sizeOf(), m_header.m_page_size, m_header.m_slab_size)) { auto max_addr = m_realms.getSlabMaxAddress(); @@ -140,11 +140,11 @@ namespace db0 m_algo_allocator.setMaxAddress(Address::fromOffset(max_addr)); } } - + MetaAllocator::~MetaAllocator() { } - + MetaAllocator::Realm::Realm(Memspace &metaspace, std::shared_ptr prefix, SlabRecycler *slab_recycler, o_realm realm, std::uint32_t slab_size, std::uint32_t page_size, unsigned char realm_id, bool deferred_free) : m_slab_defs(metaspace.myPtr(realm.m_slab_defs_ptr), page_size) @@ -159,7 +159,7 @@ namespace db0 )) { } - + Memspace MetaAllocator::createMetaspace() const { // this is to temporarily initialize for unlimited reading @@ -167,7 +167,7 @@ namespace db0 m_algo_allocator.setMaxAddress(get_address(std::numeric_limits::max() - 1)); return { Memspace::tag_from_reference(), m_prefix, m_algo_allocator }; } - + void MetaAllocator::formatPrefix(std::shared_ptr prefix, std::size_t page_size, std::size_t slab_size) { // create the meta-header at the address 0x0 @@ -177,11 +177,11 @@ namespace db0 auto offset = o_meta_header::sizeOf(); // Construct the meta-space for the slab tree AlgoAllocator algo_allocator( - getAddressPool(offset, page_size, slab_size), - getReverseAddressPool(offset, page_size, slab_size), + getAddressPool(offset, page_size, slab_size), + getReverseAddressPool(offset, page_size, slab_size), page_size); Memspace meta_space(Memspace::tag_from_reference(), prefix, algo_allocator); - + // initialize realms in the meta-header for (unsigned int i = 0; i < NUM_REALMS; ++i) { // Create the empty slab-defs and capacity items trees on the meta-space @@ -193,7 +193,7 @@ namespace db0 meta_header.modify().m_realms[i].m_capacity_items_ptr = Address::fromOffset(capacity_items.getAddress()); } } - + o_meta_header MetaAllocator::getMetaHeader(std::shared_ptr prefix) { // meta-header is located at the fixed address 0x0 @@ -203,14 +203,14 @@ namespace db0 v_object meta_header(memspace.myPtr(header_addr)); return meta_header.const_ref(); } - - std::optional
MetaAllocator::tryAlloc(std::size_t size, std::uint32_t slot_num, + + std::optional
MetaAllocator::tryAlloc(std::size_t size, std::uint32_t slot_num, bool aligned, unsigned char realm_id, unsigned char locality) { std::uint16_t instance_id; return tryAllocImpl(size, slot_num, aligned, false, instance_id, realm_id, locality); } - + std::optional MetaAllocator::tryAllocUnique(std::size_t size, std::uint32_t slot_num, bool aligned, unsigned char realm_id, unsigned char locality) { @@ -221,29 +221,29 @@ namespace db0 } return {}; } - + std::optional
MetaAllocator::tryAllocImpl(std::size_t size, std::uint32_t slot_num, bool aligned, bool unique, std::uint16_t &instance_id, unsigned char realm_id, unsigned char locality) { assert(slot_num == 0); - assert(size > 0); + assert(size > 0); return m_realms[realm_id].tryAlloc(size, slot_num, aligned, unique, instance_id, locality); } - + void MetaAllocator::free(Address address) - { + { auto slab_id = m_slab_id_function(address); auto realm_id = getRealmID(slab_id); m_realms[realm_id].free(address, slab_id); } std::size_t MetaAllocator::getAllocSize(Address address) const - { + { auto slab_id = m_slab_id_function(address); auto realm_id = getRealmID(slab_id); return m_realms[realm_id].getAllocSize(address, slab_id); } - + std::size_t MetaAllocator::getAllocSize(Address address, unsigned char realm_id) const { auto slab_id = m_slab_id_function(address); @@ -252,9 +252,9 @@ namespace db0 } return m_realms[realm_id].getAllocSize(address, slab_id); } - + bool MetaAllocator::isAllocated(Address address, std::size_t *size_of_result) const - { + { auto slab_id = m_slab_id_function(address); auto realm_id = getRealmID(slab_id); return m_realms[realm_id].isAllocated(address, slab_id, size_of_result); @@ -268,7 +268,23 @@ namespace db0 } return m_realms[realm_id].isAllocated(address, slab_id, size_of_result); } - + + Allocator::AllocationInfo MetaAllocator::findAllocation(Address address) const + { + auto slab_id = m_slab_id_function(address); + auto realm_id = getRealmID(slab_id); + return m_realms[realm_id].findAllocation(address, slab_id); + } + + Allocator::AllocationInfo MetaAllocator::findAllocation(Address address, unsigned char realm_id) const + { + auto slab_id = m_slab_id_function(address); + if (realm_id != getRealmID(slab_id)) { + THROWF(db0::BadAddressException) << "Invalid address accessed"; + } + return m_realms[realm_id].findAllocation(address, slab_id); + } + unsigned int MetaAllocator::getSlabCount() const { unsigned int total_slab_count = 0; @@ -277,13 +293,13 @@ namespace db0 } return total_slab_count; } - + std::uint32_t MetaAllocator::getRemainingCapacity(std::uint32_t slab_id) const { auto realm_id = slab_id & MetaAllocator::REALM_MASK; return m_realms[realm_id].getRemainingCapacity(slab_id); } - + void MetaAllocator::close() { if (m_recycler_ptr) { @@ -294,11 +310,11 @@ namespace db0 } m_realms.close(); } - + std::shared_ptr MetaAllocator::reserveNewSlab(unsigned char realm_id) { return m_realms[realm_id].reserveNewSlab(); } - + Address MetaAllocator::getFirstAddress() const { return m_realms[0].getFirstAddress(); } @@ -311,16 +327,16 @@ namespace db0 assert(result->size() == size); return result; } - + void MetaAllocator::Realm::commit() const { // NOTE: slab manager must commit first (important!) // this is because it may perform modifications to the slab defs and capacity items m_slab_manager->commit(); m_slab_defs.commit(); - m_capacity_items.commit(); + m_capacity_items.commit(); } - + void MetaAllocator::Realm::detach() const { m_slab_defs.detach(); @@ -331,7 +347,7 @@ namespace db0 void MetaAllocator::commit() const { // NOTE: if atomic operation is in progress, the deferred free operations are not flushed - // this is not a finalized and potentially reversible commit + // this is not a finalized and potentially reversible commit if (m_atomic_depth == 0) { flush(); } @@ -341,41 +357,41 @@ namespace db0 void MetaAllocator::detach() const { m_realms.detach(); } - + SlabRecycler *MetaAllocator::getSlabRecyclerPtr() const { return m_recycler_ptr; } - + void MetaAllocator::forAllSlabs(std::function f) const { - m_realms.forAllSlabs(f); + m_realms.forAllSlabs(f); } - + void MetaAllocator::flush() const { assert(m_atomic_depth == 0); m_realms.flush(); } - + void MetaAllocator::beginAtomic() - { + { ++m_atomic_depth; m_realms.beginAtomic(); } - + void MetaAllocator::endAtomic() - { + { assert(m_atomic_depth > 0); --m_atomic_depth; m_realms.endAtomic(); } - + void MetaAllocator::cancelAtomic() { assert(m_atomic_depth > 0); --m_atomic_depth; m_realms.cancelAtomic(); } - + MetaAllocator::RealmsVector::RealmsVector(Memspace &metaspace, std::shared_ptr prefix, SlabRecycler *slab_recycler, o_meta_header &meta_header, unsigned int size, bool deferred_free) { @@ -383,26 +399,26 @@ namespace db0 auto slab_size = meta_header.m_slab_size; auto page_size = meta_header.m_page_size; for (unsigned int i = 0; i < size; ++i) { - emplace_back(metaspace, prefix, slab_recycler, meta_header.m_realms[i], slab_size, + emplace_back(metaspace, prefix, slab_recycler, meta_header.m_realms[i], slab_size, page_size, static_cast(i), deferred_free ); } } - + void MetaAllocator::RealmsVector::forAllSlabs(std::function f) const { for (const auto &realm: *this) { realm->forAllSlabs(f); } } - + void MetaAllocator::RealmsVector::detach() const { for (const auto &realm: *this) { realm.detach(); } } - + void MetaAllocator::RealmsVector::commit() const { for (const auto &realm: *this) { @@ -439,7 +455,7 @@ namespace db0 } std::uint64_t MetaAllocator::RealmsVector::getSlabMaxAddress() const - { + { std::uint64_t max_addr = 0; for (const auto &realm : *this) { max_addr = std::max(max_addr, realm.getSlabMaxAddress()); @@ -451,7 +467,7 @@ namespace db0 { for (const auto &realm : *this) { realm->flush(); - } + } } std::size_t MetaAllocator::RealmsVector::getDeferredFreeCount() const @@ -462,11 +478,11 @@ namespace db0 } return result; } - + std::uint32_t MetaAllocator::getSlabId(Address address) const { return m_slab_id_function(address); } - + std::size_t MetaAllocator::getDeferredFreeCount() const { return m_realms.getDeferredFreeCount(); } diff --git a/src/dbzero/core/memory/MetaAllocator.hpp b/src/dbzero/core/memory/MetaAllocator.hpp index 912e8644..76d3ca9a 100644 --- a/src/dbzero/core/memory/MetaAllocator.hpp +++ b/src/dbzero/core/memory/MetaAllocator.hpp @@ -21,21 +21,21 @@ namespace db0 { - + class SlabManager; using SlabRecycler = db0::Recycler; - + DB0_PACKED_BEGIN struct DB0_PACKED_ATTR o_realm: public o_fixed_versioned { Address m_slab_defs_ptr; Address m_capacity_items_ptr; - + o_realm() = default; o_realm(const std::pair &); }; DB0_PACKED_END - + DB0_PACKED_BEGIN struct DB0_PACKED_ATTR o_meta_header: public o_fixed_versioned { @@ -46,11 +46,11 @@ DB0_PACKED_BEGIN // slab size in bytes std::uint32_t m_slab_size; o_realm m_realms[NUM_REALMS]; - + o_meta_header(std::uint32_t page_size, std::uint32_t slab_size); }; DB0_PACKED_END - + class MetaAllocator: public Allocator { public: @@ -61,25 +61,25 @@ DB0_PACKED_END * Opens an existing instance of a MetaAllocator over a specific prefix * @param deferred_free if true, free operations are deferred until commit (see Transactional Allocator) */ - MetaAllocator(std::shared_ptr prefix, SlabRecycler *recycler = nullptr, + MetaAllocator(std::shared_ptr prefix, SlabRecycler *recycler = nullptr, bool deferred_free = true); - + virtual ~MetaAllocator(); /** * Initialize a new MetaAllocator instance over an empty prefix */ static void formatPrefix(std::shared_ptr prefix, std::size_t page_size, std::size_t slab_size); - + using CapacityTreeT = SGB_Tree; using SlabTreeT = SGB_Tree; - + std::optional
tryAlloc(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0) override; - + std::optional tryAllocUnique(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0) override; - + void free(Address) override; std::size_t getAllocSize(Address) const override; @@ -87,14 +87,17 @@ DB0_PACKED_END bool isAllocated(Address, std::size_t *size_of_result = nullptr) const override; bool isAllocated(Address, unsigned char realm_id, std::size_t *size_of_result = nullptr) const override; - + + AllocationInfo findAllocation(Address) const override; + AllocationInfo findAllocation(Address, unsigned char realm_id) const override; + void commit() const override; void detach() const override; // Flush any pending "deferred" free operations void flush() const override; - + /** * Calculate the number of slabs which can be annotated by a single page pair */ @@ -102,46 +105,46 @@ DB0_PACKED_END static std::function getAddressPool(std::size_t offset, std::size_t page_size, std::size_t slab_size); - + static std::function getReverseAddressPool(std::size_t offset, std::size_t page_size, std::size_t slab_size); static std::function getSlabIdFunction(std::size_t offset, std::size_t page_size, std::size_t slab_size); - + unsigned int getSlabCount() const; - + /** * Retrieve information about the remaining space available to the Slab */ std::uint32_t getRemainingCapacity(std::uint32_t slab_id) const; - + /** * Retrieve a new slab reserved for private use * note that this slab will not be available for allocations from MetaAllocator and has to be used directly */ std::shared_ptr reserveNewSlab(unsigned char realm_id = 0); - + /** * Open existing slab for private use (reserved slab) */ std::shared_ptr openReservedSlab(Address, std::size_t size) const; - + /** * Close the allocator and flush all modifications with backed */ void close() override; - + /** * Get address of the 1st allocation (irrespective of whether it was performed by the MetaAllocator or not) */ Address getFirstAddress() const; - + SlabRecycler *getSlabRecyclerPtr() const; - + // Visit all underlying slabs void forAllSlabs(std::function) const; - + // Get the number of queued defferred free operations std::size_t getDeferredFreeCount() const; @@ -149,17 +152,17 @@ DB0_PACKED_END void beginAtomic(); void endAtomic(); void cancelAtomic(); - + protected: // Calculate slab ID for the given address std::uint32_t getSlabId(Address) const; - - private: + + private: std::shared_ptr m_prefix; o_meta_header m_header; mutable AlgoAllocator m_algo_allocator; Memspace m_metaspace; - + struct Realm { SlabTreeT m_slab_defs; @@ -181,14 +184,14 @@ DB0_PACKED_END const SlabManager *operator->() const { return m_slab_manager.get(); - } + } }; - + struct RealmsVector: protected std::vector { - RealmsVector(Memspace &, std::shared_ptr, SlabRecycler *, o_meta_header &, + RealmsVector(Memspace &, std::shared_ptr, SlabRecycler *, o_meta_header &, unsigned int size, bool deferred_free); - + // evaluate the max address from all realms std::uint64_t getSlabMaxAddress() const; std::size_t getDeferredFreeCount() const; @@ -209,22 +212,22 @@ DB0_PACKED_END void beginAtomic(); void endAtomic(); void cancelAtomic(); - + void flush() const; void close(); }; - + RealmsVector m_realms; - SlabRecycler *m_recycler_ptr; + SlabRecycler *m_recycler_ptr; std::function m_slab_id_function; // atomic operation nesting depth std::uint32_t m_atomic_depth = 0; - + /** * Reads header information from the prefix */ o_meta_header getMetaHeader(std::shared_ptr prefix); - + Memspace createMetaspace() const; /** @@ -232,10 +235,10 @@ DB0_PACKED_END * if not found then create a new slab */ std::shared_ptr getSlabAllocator(std::size_t min_capacity); - + // NOTE: instance ID will only be populated when unique = true - std::optional
tryAllocImpl(std::size_t size, std::uint32_t slot_num, bool aligned, bool unique, + std::optional
tryAllocImpl(std::size_t size, std::uint32_t slot_num, bool aligned, bool unique, std::uint16_t &instance_id, unsigned char realm_id, unsigned char locality); }; - + } diff --git a/src/dbzero/core/memory/OneShotAllocator.cpp b/src/dbzero/core/memory/OneShotAllocator.cpp index 12ef284e..c9c8b131 100644 --- a/src/dbzero/core/memory/OneShotAllocator.cpp +++ b/src/dbzero/core/memory/OneShotAllocator.cpp @@ -10,23 +10,23 @@ namespace db0 { OneShotAllocator::OneShotAllocator(Address addr, std::size_t size) - : m_addr(addr) + : m_addr(addr) , m_size(size) { } - + std::optional
OneShotAllocator::tryAlloc(std::size_t size, std::uint32_t slot_num, bool aligned, unsigned char, unsigned char) { assert(slot_num == 0); - assert(!aligned && "OneShotAllocator: aligned allocation not supported"); + assert(!aligned && "OneShotAllocator: aligned allocation not supported"); if (size != m_size || m_allocated) { return std::nullopt; } m_allocated = true; return m_addr; } - + void OneShotAllocator::free(Address address) { if (address != m_addr || !m_allocated) { @@ -34,15 +34,15 @@ namespace db0 } m_allocated = false; } - - std::size_t OneShotAllocator::getAllocSize(Address address) const + + std::size_t OneShotAllocator::getAllocSize(Address address) const { if (address != m_addr || !m_allocated) { THROWF(db0::BadAddressException) << "OneShotAllocator invalid address: " << address; } return m_size; } - + bool OneShotAllocator::isAllocated(Address address, std::size_t *size_of_result) const { if (address != m_addr || !m_allocated) { @@ -53,7 +53,15 @@ namespace db0 } return true; } - + + Allocator::AllocationInfo OneShotAllocator::findAllocation(Address address) const + { + if (!m_allocated || address < m_addr || address.getOffset() >= m_addr.getOffset() + m_size) { + THROWF(db0::BadAddressException) << "Invalid address: " << address; + } + return AllocationInfo { m_addr, m_size }; + } + void OneShotAllocator::commit() const { // nothing to do } @@ -62,4 +70,4 @@ namespace db0 // nothing to do } -} \ No newline at end of file +} diff --git a/src/dbzero/core/memory/OneShotAllocator.hpp b/src/dbzero/core/memory/OneShotAllocator.hpp index 9f1fc9a3..d08aa3ff 100644 --- a/src/dbzero/core/memory/OneShotAllocator.hpp +++ b/src/dbzero/core/memory/OneShotAllocator.hpp @@ -16,16 +16,18 @@ namespace db0 { public: OneShotAllocator(Address addr, std::size_t size); - + std::optional
tryAlloc(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0) override; - + void free(Address) override; std::size_t getAllocSize(Address) const override; bool isAllocated(Address, std::size_t *size_of_result = nullptr) const override; - + + AllocationInfo findAllocation(Address) const override; + void commit() const override; void detach() const override; diff --git a/src/dbzero/core/memory/SlabAllocator.cpp b/src/dbzero/core/memory/SlabAllocator.cpp index 99b7102f..5896ff80 100644 --- a/src/dbzero/core/memory/SlabAllocator.cpp +++ b/src/dbzero/core/memory/SlabAllocator.cpp @@ -38,7 +38,7 @@ namespace db0 { // For aligned allocations the begin address must be also aligned assert(m_begin_addr % m_page_size == 0); - + // apply dynamic bound on the CRDT allocator to prevent allocating addresses overlapping with the admin space // include ADMIN_MARGIN bitspace allocations to allow margin for the admin space to grow // NOTE: CRDT allocator's dynamic bounds are specified as relative to the allocator's base address @@ -51,18 +51,18 @@ namespace db0 std::uint32_t b0 = (b1 >= admin_margin_bytes) ? b1 - admin_margin_bytes : 0; return std::make_tuple(b0, b1, b2); }); - + // provide CRDT allocator's dynamic bound to the bitspace (this is for address validation/ collision prevention purposes) // NOTE: the bitspace bounds use the absolute address m_bitspace.setDynamicBounds([this]() { return m_begin_addr + static_cast(m_allocator.getMaxAddr()); }); } - + SlabAllocator::~SlabAllocator() { } - + std::optional
SlabAllocator::tryAlloc(std::size_t size, std::uint32_t slot_num, bool aligned, unsigned char, unsigned char) { @@ -76,11 +76,11 @@ namespace db0 } return std::nullopt; } - + void SlabAllocator::free(Address address) { m_allocator.free(makeRelative(address)); } - + std::size_t SlabAllocator::getAllocSize(Address address) const { return m_allocator.getAllocSize(makeRelative(address)); } @@ -92,16 +92,29 @@ namespace db0 bool SlabAllocator::isAllocated(Address address, std::size_t *size_of_result) const { return m_allocator.isAllocated(makeRelative(address), size_of_result); } - + bool SlabAllocator::isAllocated(Address address, unsigned char, std::size_t *size_of_result) const { return m_allocator.isAllocated(makeRelative(address), size_of_result); } - + + Allocator::AllocationInfo SlabAllocator::findAllocation(Address address) const + { + if (!inRange(address)) { + THROWF(db0::BadAddressException) << "Invalid address: " << address; + } + auto result = m_allocator.findAllocation(makeRelative(address)); + return AllocationInfo { makeAbsolute(static_cast(result.first)), result.second }; + } + + Allocator::AllocationInfo SlabAllocator::findAllocation(Address address, unsigned char) const { + return findAllocation(address); + } + Address SlabAllocator::headerAddr(Address begin_addr, std::uint32_t size) { return begin_addr + static_cast(size) - static_cast(o_slab_header::sizeOf()); } - - std::size_t SlabAllocator::formatSlab(std::shared_ptr prefix, Address begin_addr, + + std::size_t SlabAllocator::formatSlab(std::shared_ptr prefix, Address begin_addr, std::uint32_t size, std::size_t page_size) { auto admin_size = calculateAdminSpaceSize(page_size); @@ -109,11 +122,11 @@ namespace db0 if (size <= admin_size + admin_margin_bytes) { THROWF(db0::InternalException) << "Slab size too small: " << size; } - + if (size % page_size != 0) { THROWF(db0::InternalException) << "Slab size not multiple of page size: " << size << " % " << page_size; } - + // put bitspace right before the header (at the end of the slab) BitSpace::create( prefix, headerAddr(begin_addr, size), page_size, -1 @@ -123,21 +136,21 @@ namespace db0 BitSpace bitspace( prefix, headerAddr(begin_addr, size), page_size, -1 ); - + // Create the CRDT allocator data structures on top of the bitspace AllocSetT allocs(bitspace, page_size); BlankSetT blanks(bitspace, page_size); AlignedBlankSetT aligned_blanks(bitspace, page_size, CompT(page_size), page_size); StripeSetT stripes(bitspace, page_size); - LimitedVector alloc_counter(bitspace, page_size); + LimitedVector alloc_counter(bitspace, page_size); alloc_counter.reserve(SlabAllocatorConfig::SLAB_BITSPACE_SIZE()); // calculate size initially available to CRTD allocator std::uint32_t crdt_size = static_cast(size - admin_size - admin_margin_bytes); assert(crdt_size > 0); - + // register the initial blank - associated with the relative address = 0 CRDT_Allocator::insertBlank(blanks, aligned_blanks, { crdt_size, 0 }, page_size); - + // create a temporary memspace only to allocate the header under a known address OneShotAllocator osa(headerAddr(begin_addr, size), o_slab_header::sizeOf()); Memspace memspace(Memspace::tag_from_reference(), prefix, osa); @@ -154,11 +167,11 @@ namespace db0 ); return crdt_size; } - + const std::size_t SlabAllocator::getSlabSize() const { return m_slab_size; } - + const std::size_t SlabAllocator::getAdminSpaceSize(bool include_margin) const { auto result = m_begin_addr.getOffset() + m_slab_size - m_bitspace.getBaseAddress() + m_bitspace.span() * m_page_size; @@ -168,7 +181,7 @@ namespace db0 } return result; } - + std::size_t SlabAllocator::calculateAdminSpaceSize(std::size_t page_size) { auto result = BitSpace::sizeOf() + o_slab_header::sizeOf(); @@ -180,11 +193,11 @@ namespace db0 result += LimitedVectorT::DP_REQ(SlabAllocatorConfig::SLAB_BITSPACE_SIZE(), page_size) * page_size; return result; } - + std::size_t SlabAllocator::getMaxAllocSize() const { return m_slab_size - calculateAdminSpaceSize(m_page_size) - ADMIN_MARGIN() * m_page_size; } - + std::size_t SlabAllocator::getLostCapacity() const { if (!m_initial_lost_capacity) { @@ -194,7 +207,7 @@ namespace db0 assert(result >= 0); return static_cast(result); } - + std::size_t SlabAllocator::getRemainingCapacity() const { if (!m_initial_remaining_capacity) { @@ -203,11 +216,11 @@ namespace db0 std::int64_t result = (std::int64_t)*m_initial_remaining_capacity - m_allocator.getAllocDelta() - (getAdminSpaceSize(true) - m_initial_admin_size); return result > 0 ? result : 0; } - + const Prefix &SlabAllocator::getPrefix() const { return *m_prefix; } - + bool SlabAllocator::empty() const { return m_allocs.empty(); } @@ -223,7 +236,7 @@ namespace db0 Address SlabAllocator::getFirstAddress() { return Address::fromOffset(CRDT_Allocator::getFirstAddress()); } - + void SlabAllocator::commit() const { m_header.commit(); @@ -233,9 +246,9 @@ namespace db0 m_aligned_blanks.commit(); m_stripes.commit(); m_alloc_counter.commit(); - m_allocator.commit(); + m_allocator.commit(); } - + void SlabAllocator::detach() const { m_header.detach(); @@ -247,7 +260,7 @@ namespace db0 m_alloc_counter.detach(); m_allocator.detach(); } - + bool SlabAllocator::tryMakeAddressUnique(Address address, std::uint16_t &instance_id) { // make sure high 14 bits are 0 @@ -259,10 +272,10 @@ namespace db0 } assert(instance_id > 0); assert(instance_id <= UniqueAddress::INSTANCE_ID_MAX); - + return true; } - + UniqueAddress SlabAllocator::tryMakeAddressUnique(Address address) { std::uint16_t instance_id; @@ -272,17 +285,17 @@ namespace db0 // unable to make the address unique return {}; } - - bool SlabAllocator::inRange(Address address) const + + bool SlabAllocator::inRange(Address address) const { - return (address.getOffset() >= m_begin_addr.getOffset()) && + return (address.getOffset() >= m_begin_addr.getOffset()) && (address.getOffset() < m_begin_addr.getOffset() + m_slab_size); } - + std::pair > SlabAllocator::getRange(std::uint32_t slot_num) const { assert(!slot_num && "SlabAllocator does not support slots"); return { m_begin_addr, m_begin_addr + static_cast(m_slab_size) }; } - -} \ No newline at end of file + +} diff --git a/src/dbzero/core/memory/SlabAllocator.hpp b/src/dbzero/core/memory/SlabAllocator.hpp index 0c9e26af..611e083a 100644 --- a/src/dbzero/core/memory/SlabAllocator.hpp +++ b/src/dbzero/core/memory/SlabAllocator.hpp @@ -18,7 +18,7 @@ namespace db0 { -DB0_PACKED_BEGIN +DB0_PACKED_BEGIN struct DB0_PACKED_ATTR o_slab_header: public db0::o_fixed { const std::uint32_t m_version = 1; @@ -32,25 +32,25 @@ DB0_PACKED_BEGIN std::uint32_t m_stripe_set_ptr = 0; std::uint32_t m_alloc_counter_ptr = 0; std::array m_reserved = {0, 0}; - + o_slab_header() = default; - - o_slab_header(std::uint32_t size, std::uint32_t alloc_set_ptr, std::uint32_t blank_set_ptr, + + o_slab_header(std::uint32_t size, std::uint32_t alloc_set_ptr, std::uint32_t blank_set_ptr, std::uint32_t aligned_blank_set_ptr, std::uint32_t stripe_set_ptr, std::uint32_t alloc_counter_ptr) : m_size(size) , m_alloc_set_ptr(alloc_set_ptr) , m_blank_set_ptr(blank_set_ptr) , m_aligned_blank_set_ptr(aligned_blank_set_ptr) - , m_stripe_set_ptr(stripe_set_ptr) + , m_stripe_set_ptr(stripe_set_ptr) , m_alloc_counter_ptr(alloc_counter_ptr) { } }; DB0_PACKED_END - + /** * The SlabAllocator takes a fixed size address range (e.g. 64MB) - * and organizes the space with the use of BitSetAllocator/BitSpace + CRDT_Allocator + * and organizes the space with the use of BitSetAllocator/BitSpace + CRDT_Allocator * providing a clean Allocator interface */ class SlabAllocator: public Allocator @@ -66,62 +66,65 @@ DB0_PACKED_END */ SlabAllocator(std::shared_ptr prefix, Address begin_addr, std::uint32_t size, std::size_t page_size, std::optional remaining_capacity = {}, std::optional lost_capacity = {}); - + virtual ~SlabAllocator(); - + std::optional
tryAlloc(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0) override; - + void free(Address) override; std::size_t getAllocSize(Address) const override; std::size_t getAllocSize(Address, unsigned char realm_id) const override; - + bool isAllocated(Address, std::size_t *size_of_result = nullptr) const override; bool isAllocated(Address, unsigned char realm_id, std::size_t *size_of_result = nullptr) const override; - + + AllocationInfo findAllocation(Address) const override; + AllocationInfo findAllocation(Address, unsigned char realm_id) const override; + void commit() const override; void detach() const override; - + bool inRange(Address) const override; - + /** * Initialize a new allocator over a specific slab - * + * * @return the created slab's capacity in bytes */ static std::size_t formatSlab(std::shared_ptr prefix, Address begin_addr, std::uint32_t size, std::size_t page_size); - + const std::size_t getSlabSize() const; - + /** * Get number of bytes occupied by the administrative data structures * this number varies depending on the number and size of allocated objects */ const std::size_t getAdminSpaceSize(bool include_margin) const; - + /** * Calculate the initial administrative space size given specific criteria */ static std::size_t calculateAdminSpaceSize(std::size_t page_size); - + /** * Get maximum theoretical size of a single allocation possible on this type of slab when unoccupied */ std::size_t getMaxAllocSize() const; - + /** * Get estimated remaining capacity (if initialized with remaining capacity, otherwise throws) */ std::size_t getRemainingCapacity() const; - + // Get allocator's (possibly irrecoverably) lost capacity std::size_t getLostCapacity() const; - + const Prefix &getPrefix() const; - + bool empty() const; /** @@ -136,7 +139,7 @@ DB0_PACKED_END */ static Address getFirstAddress(); - // SlabAllocator specific address conversions + // SlabAllocator specific address conversions inline Address makeAbsolute(std::uint32_t relative) const { return m_begin_addr + static_cast(relative); } @@ -144,13 +147,13 @@ DB0_PACKED_END inline std::uint32_t makeRelative(Address absolute) const { return absolute - m_begin_addr; } - + bool tryMakeAddressUnique(Address, std::uint16_t &instance_id); UniqueAddress tryMakeAddressUnique(Address); - + // Get address range of the entire slab (begin, end), not the actually allocated space std::pair > getRange(std::uint32_t slot_num = 0) const override; - + private: using AllocSetT = db0::CRDT_Allocator::AllocSetT; using BlankSetT = db0::CRDT_Allocator::BlankSetT; @@ -158,19 +161,19 @@ DB0_PACKED_END using StripeSetT = db0::CRDT_Allocator::StripeSetT; using CompT = typename AlignedBlankSetT::CompT; using LimitedVectorT = LimitedVector; - + // the number of administrative area pages - // we determined this number to reflect the number of underlying collections + // we determined this number to reflect the number of underlying collections static constexpr std::uint32_t ADMIN_SPAN() { return 5; } - + // this calculation is based on the assumption that each CRDT collection // requires 2 DP margin to allow for its growth static constexpr std::uint32_t ADMIN_MARGIN() { return ADMIN_SPAN() * 2; } - + std::shared_ptr m_prefix; const Address m_begin_addr; const std::size_t m_page_size; @@ -189,8 +192,8 @@ DB0_PACKED_END const std::optional m_initial_remaining_capacity; const std::optional m_initial_lost_capacity; std::size_t m_initial_admin_size; - + static Address headerAddr(Address begin_addr, std::uint32_t size); }; - + } diff --git a/src/dbzero/core/memory/SlabManager.cpp b/src/dbzero/core/memory/SlabManager.cpp index b4dc5778..d1281754 100644 --- a/src/dbzero/core/memory/SlabManager.cpp +++ b/src/dbzero/core/memory/SlabManager.cpp @@ -10,7 +10,7 @@ namespace db0 SlabManager::SlabManager(std::shared_ptr prefix, MetaAllocator::SlabTreeT &slab_defs, MetaAllocator::CapacityTreeT &capacity_items, SlabRecycler *recycler, std::uint32_t slab_size, std::uint32_t page_size, - std::function address_func, std::function slab_id_func, + std::function address_func, std::function slab_id_func, unsigned char realm_id, bool deferred_free) : m_prefix(prefix) , m_realm_id(realm_id) @@ -25,15 +25,15 @@ namespace db0 , m_deferred_free(deferred_free) { } - + bool SlabManager::ActiveSlab::contains(std::uint32_t slab_id) const { return (((*this)[0] && *(*this)[0] == slab_id) || ((*this)[1] && *(*this)[1] == slab_id)); } - + bool SlabManager::ActiveSlab::contains(std::shared_ptr slab) const { return ((*this)[0] == slab || (*this)[1] == slab); } - + std::shared_ptr SlabManager::ActiveSlab::find(std::uint32_t slab_id) const { if ((*this)[0] && *(*this)[0] == slab_id) { @@ -43,7 +43,7 @@ namespace db0 } return {}; } - + void SlabManager::ActiveSlab::erase(std::shared_ptr slab) { if ((*this)[0] == slab) { @@ -55,13 +55,13 @@ namespace db0 THROWF(db0::InternalException) << "Slab not found in active slabs." << THROWF_END; } } - + std::shared_ptr SlabManager::tryGetActiveSlab(unsigned char locality) { assert(locality < m_active_slab.size()); return m_active_slab[locality]; } - + void SlabManager::resetActiveSlab(unsigned char locality) { assert(locality < m_active_slab.size()); @@ -77,7 +77,7 @@ namespace db0 m_active_slab[1] = {}; } } - + std::shared_ptr SlabManager::findFirst(std::size_t size, unsigned char locality) { // NOTE: before accessing capacity items we must synchronize any updates @@ -90,7 +90,7 @@ namespace db0 // no existing slab has sufficient capacity return {}; } - + if (m_active_slab.contains(it->m_slab_id)) { // do not include active slab in find operation ++it; @@ -98,16 +98,16 @@ namespace db0 } auto slab = openSlab(m_slab_address_func(it->m_slab_id)); // make the slab active - m_active_slab[locality] = slab; + m_active_slab[locality] = slab; return slab; } } - + std::shared_ptr SlabManager::findNext(std::shared_ptr last_result, std::size_t size, unsigned char locality) { saveDirtySlabs(); - auto min_capacity = std::max(size, SlabAllocatorConfig::MIN_OP_CAPACITY(m_slab_size)); + auto min_capacity = std::max(size, SlabAllocatorConfig::MIN_OP_CAPACITY(m_slab_size)); auto last_key = last_result->m_cap_item; for (;;) { // this is to find the next item in order @@ -116,10 +116,10 @@ namespace db0 if (!it.first || it.first->m_remaining_capacity < min_capacity) { return {}; } - + if (m_active_slab.contains(it.first->m_slab_id)) { last_key = *(it.first); - // do not include active slab in find operation + // do not include active slab in find operation continue; } auto slab = openSlab(m_slab_address_func(it.first->m_slab_id)); @@ -128,7 +128,7 @@ namespace db0 return slab; } } - + std::pair, std::uint32_t> SlabManager::createNewSlab() { if (!m_next_slab_id) { @@ -146,22 +146,22 @@ namespace db0 // if atomic operation is in progress, add to the volatile slabs m_atomic_stack.back().m_volatile_slabs.push_back(address); } - + return { slab, slab_id }; } - + std::shared_ptr SlabManager::addNewSlab(unsigned char locality) { auto [slab, slab_id] = createNewSlab(); auto address = m_slab_address_func(slab_id); - CapacityItem cap_item { - static_cast(slab->getRemainingCapacity()), + CapacityItem cap_item { + static_cast(slab->getRemainingCapacity()), static_cast(slab->getLostCapacity()), slab_id }; // register with slab defs m_slab_defs.emplace(slab_id, - static_cast(cap_item.m_remaining_capacity), + static_cast(cap_item.m_remaining_capacity), static_cast(cap_item.m_lost_capacity) ); // register with capacity items @@ -169,12 +169,12 @@ namespace db0 // add to cache auto cache_item = std::make_shared(slab, cap_item); m_slabs.emplace(address, cache_item); - + // append with the recycler if (m_recycler_ptr) { m_recycler_ptr->append(cache_item); } - + // make the newly added slab active m_active_slab[locality] = cache_item; return m_active_slab[locality]; @@ -199,15 +199,15 @@ namespace db0 } return slab_def_ptr.first->m_remaining_capacity; } - + void SlabManager::close() { m_active_slab = {}; m_reserved_slabs.clear(); saveDirtySlabs(); m_slabs.clear(); - } - + } + std::shared_ptr SlabManager::tryFind(std::uint32_t slab_id) const { if (slab_id < nextSlabId()) { @@ -273,15 +273,15 @@ namespace db0 } m_slabs.erase(it); } - + void SlabManager::erase(std::shared_ptr slab) { erase(slab, true); } - + bool SlabManager::empty() const { return nextSlabId() == m_realm_id; } - + std::shared_ptr SlabManager::reserveNewSlab() { auto [slab, slab_id] = createNewSlab(); @@ -289,20 +289,20 @@ namespace db0 CapacityItem cap_item { 0, 0, slab_id }; // register with slab defs m_slab_defs.emplace( - slab_id, - static_cast(cap_item.m_remaining_capacity), + slab_id, + static_cast(cap_item.m_remaining_capacity), static_cast(cap_item.m_lost_capacity) ); // register with capacity items m_capacity_items.insert(cap_item); return slab; } - + std::shared_ptr SlabManager::openExistingSlab(const SlabDef &slab_def) { if (slab_def.m_slab_id >= nextSlabId()) { THROWF(db0::InputException) << "Slab " << slab_def.m_slab_id << " does not exist"; - } + } auto address = m_slab_address_func(slab_def.m_slab_id); // look up with the cache first auto it = m_slabs.find(address); @@ -315,11 +315,11 @@ namespace db0 // pull through cache return openSlab(slab_def)->m_slab; } - + std::shared_ptr SlabManager::openReservedSlab(Address address) const { return openReservedSlab(address, m_slab_id_func(address)); } - + std::shared_ptr SlabManager::openReservedSlab(Address address, std::uint32_t slab_id) const { assert(m_slab_id_func(address) == slab_id); @@ -341,7 +341,7 @@ namespace db0 if (!slab_def_ptr.first) { THROWF(db0::InternalException) << "Slab definition not found: " << slab_id; } - + // pull through cache auto result = openSlab(*slab_def_ptr.first)->m_slab; // and add for non-expiry cache @@ -352,7 +352,7 @@ namespace db0 Address SlabManager::getFirstAddress() const { return m_slab_address_func(m_realm_id) + SlabAllocator::getFirstAddress(); } - + void SlabManager::commit() const { saveDirtySlabs(); @@ -363,7 +363,7 @@ namespace db0 } } } - + void SlabManager::detach() const { // detach all cached slabs @@ -377,7 +377,7 @@ namespace db0 // invalidate cached variable m_next_slab_id = {}; } - + std::uint32_t SlabManager::nextSlabId() const { if (!m_next_slab_id) { @@ -394,9 +394,9 @@ namespace db0 saveDirtySlabs(); m_atomic_stack.emplace_back(); } - + void SlabManager::endAtomic() - { + { assert(!m_atomic_stack.empty()); auto atomicFrame = std::move(m_atomic_stack.back()); m_atomic_stack.pop_back(); @@ -453,7 +453,7 @@ namespace db0 m_next_slab_id = {}; } } - + void SlabManager::saveItem(SlabItem &item) const { // if the remaining capacity has hanged, reflect this with backend @@ -461,18 +461,18 @@ namespace db0 auto slab_id = item.m_cap_item.m_slab_id; auto remaining_capacity = item->getRemainingCapacity(); auto lost_capacity = item->getLostCapacity(); - + auto it = m_capacity_items.find_equal(item.m_cap_item); assert(!it.isEnd()); - // re-register under a modified key + // re-register under a modified key m_capacity_items.erase(it); m_capacity_items.emplace( remaining_capacity, lost_capacity, slab_id ); - + // and update with the slab defs - auto slab_def_ptr = m_slab_defs.find_equal(slab_id); + auto slab_def_ptr = m_slab_defs.find_equal(slab_id); m_slab_defs.modify(slab_def_ptr)->m_remaining_capacity = remaining_capacity; m_slab_defs.modify(slab_def_ptr)->m_lost_capacity = lost_capacity; @@ -490,7 +490,7 @@ namespace db0 } m_dirty_slabs.clear(); } - + std::shared_ptr SlabManager::tryOpenSlab(Address address) const { auto it = m_slabs.find(address); @@ -501,15 +501,15 @@ namespace db0 } m_slabs.erase(it); } - + auto slab_id = m_slab_id_func(address); // retrieve slab definition auto slab_def_ptr = m_slab_defs.find_equal(slab_id); if (!slab_def_ptr.first) { return {}; } - - return openSlab(*slab_def_ptr.first); + + return openSlab(*slab_def_ptr.first); } std::shared_ptr SlabManager::openSlab(Address address) const @@ -531,7 +531,7 @@ namespace db0 // add to cache (it's safe to reference item from the unordered_map) auto cache_item = std::make_shared(slab, cap_item); m_slabs.emplace(addr, cache_item); - + // append with the recycler if (m_recycler_ptr) { m_recycler_ptr->append(cache_item); @@ -539,7 +539,7 @@ namespace db0 return cache_item; } - + void SlabManager::erase(std::shared_ptr slab, bool cleanup) { assert(slab); @@ -603,7 +603,7 @@ namespace db0 return m_realm_id; } } - + std::optional
SlabManager::tryAlloc(std::size_t size, std::uint32_t slot_num, bool aligned, bool unique, std::uint16_t &instance_id, unsigned char locality) { @@ -619,11 +619,11 @@ namespace db0 auto addr = (*slab)->tryAlloc(size, 0, aligned); if (!addr) { // NOTE: since the last allocation failed, don't use this slab as "active" - resetActiveSlab(locality); + resetActiveSlab(locality); break; } - - if (!unique || ((*slab)->tryMakeAddressUnique(*addr, instance_id))) { + + if (!unique || ((*slab)->tryMakeAddressUnique(*addr, instance_id))) { // Modified slabs are tracked per atomic frame so rollback can evict // only stale allocator instances from that frame. if (!slab->m_is_dirty) { @@ -631,13 +631,13 @@ namespace db0 } return addr; } - + // unable to make the address unique, schedule for deferred free and try again // NOTE: the allocation is lost deferredFree(*addr); } if (size > ((*slab)->getMaxAllocSize())) { - THROWF(db0::InternalException) + THROWF(db0::InternalException) << "Requested allocation size " << size << " is larger than the slab size " << (*slab)->getMaxAllocSize(); } if (is_new) { @@ -661,7 +661,7 @@ namespace db0 } } } - + void SlabManager::free(Address address) { if (m_deferred_free) { @@ -670,7 +670,7 @@ namespace db0 _free(address); } } - + void SlabManager::free(Address address, std::uint32_t slab_id) { assert(m_deferred_free_ops.find(address) == m_deferred_free_ops.end()); @@ -684,10 +684,10 @@ namespace db0 void SlabManager::_free(Address address) { _free(address, m_slab_id_func(address)); } - + void SlabManager::_free(Address address, std::uint32_t slab_id) { - assert(m_slab_id_func(address) == slab_id); + assert(m_slab_id_func(address) == slab_id); auto slab = find(slab_id); assert(slab); (*slab)->free(address); @@ -705,21 +705,21 @@ namespace db0 std::size_t SlabManager::getAllocSize(Address address) const { return getAllocSize(address, m_slab_id_func(address)); } - + std::size_t SlabManager::getAllocSize(Address address, std::uint32_t slab_id) const - { + { if (m_deferred_free_ops.find(address) != m_deferred_free_ops.end()) { THROWF(db0::BadAddressException) << "Address " << address << " not found (pending deferred free)"; } - + assert(m_slab_id_func(address) == slab_id); return (*find(slab_id))->getAllocSize(address); } - + bool SlabManager::isAllocated(Address address, std::size_t *size_of_result) const { return isAllocated(address, m_slab_id_func(address), size_of_result); } - + bool SlabManager::isAllocated(Address address, std::uint32_t slab_id, std::size_t *size_of_result) const { if (m_deferred_free_ops.find(address) != m_deferred_free_ops.end()) { @@ -732,7 +732,25 @@ namespace db0 } return ((*slab)->isAllocated(address, size_of_result)); } - + + Allocator::AllocationInfo SlabManager::findAllocation(Address address) const { + return findAllocation(address, m_slab_id_func(address)); + } + + Allocator::AllocationInfo SlabManager::findAllocation(Address address, std::uint32_t slab_id) const + { + assert(m_slab_id_func(address) == slab_id); + auto slab = tryFind(slab_id); + if (!slab) { + THROWF(db0::BadAddressException) << "Invalid address: " << address; + } + auto result = (*slab)->findAllocation(address); + if (m_deferred_free_ops.find(result.address) != m_deferred_free_ops.end()) { + THROWF(db0::BadAddressException) << "Invalid address: " << address; + } + return result; + } + void SlabManager::forAllSlabs(std::function f) const { auto it = m_slab_defs.cbegin(); @@ -741,9 +759,9 @@ namespace db0 f(*slab, it->m_slab_id); } } - + void SlabManager::deferredFree(Address address) - { + { if (!m_atomic_stack.empty()) { m_atomic_stack.back().m_deferred_free_ops.push_back(address); } else { diff --git a/src/dbzero/core/memory/SlabManager.hpp b/src/dbzero/core/memory/SlabManager.hpp index dfd6e5ac..28bbe607 100644 --- a/src/dbzero/core/memory/SlabManager.hpp +++ b/src/dbzero/core/memory/SlabManager.hpp @@ -20,7 +20,7 @@ namespace db0 { - + /** * SlabManager allows efficient access to a working set of slabs * either for read-only or read-write operations @@ -36,39 +36,42 @@ namespace db0 static constexpr std::size_t NUM_REALMS = MetaAllocator::NUM_REALMS; using SlabTreeT = MetaAllocator::SlabTreeT; using CapacityTreeT = MetaAllocator::CapacityTreeT; - + SlabManager(std::shared_ptr prefix, SlabTreeT &slab_defs, CapacityTreeT &capacity_items, SlabRecycler *recycler, std::uint32_t slab_size, std::uint32_t page_size, - std::function address_func, std::function slab_id_func, + std::function address_func, std::function slab_id_func, unsigned char realm_id, bool deferred_free); - - std::optional
tryAlloc(std::size_t size, std::uint32_t slot_num, bool aligned, bool unique, + + std::optional
tryAlloc(std::size_t size, std::uint32_t slot_num, bool aligned, bool unique, std::uint16_t &instance_id, unsigned char locality); - + void free(Address address); // @param slab_id must match the one calcuated from the address void free(Address address, std::uint32_t slab_id); - + std::size_t getAllocSize(Address address) const; std::size_t getAllocSize(Address address, std::uint32_t slab_id) const; - + bool isAllocated(Address address, std::size_t *size_of_result) const; bool isAllocated(Address address, std::uint32_t slab_id, std::size_t *size_of_result) const; - + + Allocator::AllocationInfo findAllocation(Address address) const; + Allocator::AllocationInfo findAllocation(Address address, std::uint32_t slab_id) const; + unsigned int getSlabCount() const { return (nextSlabId() - m_realm_id) / NUM_REALMS; } - + // NOTE: reserved slabs are not updated in the CapacityItems tree // since they're registered with capacity = 0 (to avoid using them in regular allocations) std::shared_ptr reserveNewSlab(); - + // Open an existing reserved slab std::shared_ptr openReservedSlab(Address) const; std::shared_ptr openReservedSlab(Address, std::uint32_t slab_id) const; - + std::uint32_t getRemainingCapacity(std::uint32_t slab_id) const; - + std::size_t getDeferredFreeCount() const; Address getFirstAddress() const; @@ -76,72 +79,72 @@ namespace db0 bool empty() const; void commit() const; - + void detach() const; void beginAtomic(); void endAtomic(); void cancelAtomic(); - + void close(); - + void forAllSlabs(std::function f) const; - + void flush() const; - + private: - + // NOTE: only localities 0 and 1 are currently supported struct ActiveSlab: public std::array, 2> { bool contains(std::uint32_t slab_id) const; bool contains(std::shared_ptr) const; - + std::shared_ptr find(std::uint32_t slab_id) const; void erase(std::shared_ptr); }; - + /** * Retrieves the active slab or returns nullptr if no active slab available */ - std::shared_ptr tryGetActiveSlab(unsigned char locality); + std::shared_ptr tryGetActiveSlab(unsigned char locality); void resetActiveSlab(unsigned char locality); void resetActiveSlab(std::shared_ptr); /** * Retrieve the 1st slab to allocate a block of at least min_capacity - * this is only a 'hint' and if the allocation is not possible, the next slab should be attempted + * this is only a 'hint' and if the allocation is not possible, the next slab should be attempted */ std::shared_ptr findFirst(std::size_t size, unsigned char locality); // Continue after findFirst - std::shared_ptr findNext(std::shared_ptr last_result, std::size_t size, + std::shared_ptr findNext(std::shared_ptr last_result, std::size_t size, unsigned char locality); - + /** * Create a new, unregistered slab instance */ std::pair, std::uint32_t> createNewSlab(); - + // Create a new, registered slab instance std::shared_ptr addNewSlab(unsigned char locality); - + // Find existing slab by ID std::shared_ptr tryFind(std::uint32_t slab_id) const; std::shared_ptr find(std::uint32_t slab_id) const; void markDirty(std::shared_ptr); void invalidateCachedSlab(std::uint64_t); - + /** * Erase if 'slab' is the last slab */ void erase(std::shared_ptr); - + std::shared_ptr openExistingSlab(const SlabDef &); - + std::uint32_t nextSlabId() const; - + std::shared_ptr m_prefix; const unsigned char m_realm_id; SlabTreeT &m_slab_defs; @@ -175,20 +178,20 @@ namespace db0 mutable std::unordered_set
m_deferred_free_ops; // the list of modified slabs (need backend refresh) mutable std::vector > m_dirty_slabs; - + // Reflect item changes with the backend (if modified) void saveItem(SlabItem &item) const; // Save all dirty slabs to the backend void saveDirtySlabs() const; - + std::shared_ptr tryOpenSlab(Address address) const; std::shared_ptr openSlab(Address address) const; - + // open slab by definition and add to cache std::shared_ptr openSlab(const SlabDef &def) const; - + void erase(std::shared_ptr, bool cleanup); - + std::uint32_t fetchNextSlabId() const; void deferredFree(Address); @@ -197,5 +200,5 @@ namespace db0 void _free(Address); void _free(Address, std::uint32_t slab_id); }; - + } diff --git a/src/dbzero/core/memory/SlotAllocator.cpp b/src/dbzero/core/memory/SlotAllocator.cpp index 61a56721..c97824c7 100644 --- a/src/dbzero/core/memory/SlotAllocator.cpp +++ b/src/dbzero/core/memory/SlotAllocator.cpp @@ -31,13 +31,13 @@ namespace db0 { SlabAllocator &m_allocator; std::vector
m_pending_free; - + ScopedAllocBuf(SlabAllocator &allocator) : m_allocator(allocator) { } - ~ScopedAllocBuf() + ~ScopedAllocBuf() { for (auto addr: m_pending_free) { m_allocator.free(addr); @@ -48,32 +48,32 @@ namespace db0 m_pending_free.push_back(addr); } }; - - std::optional
SlotAllocator::tryAlloc(std::size_t size, std::uint32_t slot_num, + + std::optional
SlotAllocator::tryAlloc(std::size_t size, std::uint32_t slot_num, bool aligned, unsigned char realm_id, unsigned char locality) { if (!slot_num) { return m_allocator_ptr->tryAlloc(size, 0, aligned, realm_id, locality); } - + return select(slot_num).tryAlloc(size, 0, aligned, realm_id, locality); } - - std::optional SlotAllocator::tryAllocUnique(std::size_t size, std::uint32_t slot_num, + + std::optional SlotAllocator::tryAllocUnique(std::size_t size, std::uint32_t slot_num, bool aligned, unsigned char realm_id, unsigned char locality) { if (!slot_num) { return m_allocator_ptr->tryAllocUnique(size, 0, aligned, realm_id, locality); } - + return select(slot_num).tryAllocUnique(size, 0, aligned, realm_id, locality); } - + void SlotAllocator::free(Address address) { // can free from the general allocator m_allocator_ptr->free(address); } - + std::size_t SlotAllocator::getAllocSize(Address address) const { return m_allocator_ptr->getAllocSize(address); } @@ -85,11 +85,37 @@ namespace db0 bool SlotAllocator::isAllocated(Address address, std::size_t *size_of_result) const { return m_allocator_ptr->isAllocated(address, size_of_result); } - + bool SlotAllocator::isAllocated(Address address, unsigned char realm_id, std::size_t *size_of_result) const { return m_allocator_ptr->isAllocated(address, realm_id, size_of_result); } + Allocator::AllocationInfo SlotAllocator::findAllocation(Address address) const + { + return m_allocator_ptr->findAllocation(address); + } + + Allocator::AllocationInfo SlotAllocator::findAllocation(Address address, unsigned char realm_id) const + { + return m_allocator_ptr->findAllocation(address, realm_id); + } + + Allocator::AllocationInfo SlotAllocator::findAllocation(Address address, std::uint32_t slot_num) const + { + if (slot_num == 0) { + return findAllocation(address); + } + return getSlot(slot_num).findAllocation(address); + } + + Allocator::AllocationInfo SlotAllocator::findAllocation(Address address, std::uint32_t slot_num, unsigned char realm_id) const + { + if (slot_num == 0) { + return findAllocation(address, realm_id); + } + return getSlot(slot_num).findAllocation(address, realm_id); + } + void SlotAllocator::commit() const { m_allocator_ptr->commit(); @@ -99,7 +125,7 @@ namespace db0 } } } - + void SlotAllocator::detach() const { m_allocator_ptr->detach(); @@ -109,7 +135,7 @@ namespace db0 } } } - + Allocator &SlotAllocator::select(std::uint32_t slot_num) { if (slot_num == 0) { @@ -118,7 +144,7 @@ namespace db0 assert(slot_num < m_slots.size() && m_slots[slot_num]); return *m_slots[slot_num]; } - + SlabAllocator &SlotAllocator::getSlot(std::uint32_t slot_num) const { if (!slot_num || slot_num >= m_slots.size() || !m_slots[slot_num]) { @@ -126,7 +152,7 @@ namespace db0 } return *m_slots[slot_num]; } - + bool SlotAllocator::inRange(Address address) const { return m_allocator_ptr->inRange(address); } @@ -138,5 +164,5 @@ namespace db0 } return getSlot(slot_num).getRange(0); } - + } diff --git a/src/dbzero/core/memory/SlotAllocator.hpp b/src/dbzero/core/memory/SlotAllocator.hpp index 10b2b765..2c9a242c 100644 --- a/src/dbzero/core/memory/SlotAllocator.hpp +++ b/src/dbzero/core/memory/SlotAllocator.hpp @@ -11,7 +11,7 @@ namespace db0 { - + class SlabAllocator; /** @@ -24,14 +24,14 @@ namespace db0 // initialize slot-specific allocator void setSlot(std::uint32_t slot_num, std::shared_ptr slot_allocator); - - std::optional
tryAlloc(std::size_t size, std::uint32_t slot_num = 0, + + std::optional
tryAlloc(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0) override; - + // Unique allocations are not supported because of the limited slot's address space - std::optional tryAllocUnique(std::size_t size, std::uint32_t slot_num = 0, + std::optional tryAllocUnique(std::size_t size, std::uint32_t slot_num = 0, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0) override; - + void free(Address) override; std::size_t getAllocSize(Address) const override; @@ -39,19 +39,24 @@ namespace db0 bool isAllocated(Address, std::size_t *size_of_result = nullptr) const override; bool isAllocated(Address, unsigned char realm_id, std::size_t *size_of_result = nullptr) const override; - + + AllocationInfo findAllocation(Address) const override; + AllocationInfo findAllocation(Address, unsigned char realm_id) const override; + AllocationInfo findAllocation(Address, std::uint32_t slot_num) const; + AllocationInfo findAllocation(Address, std::uint32_t slot_num, unsigned char realm_id) const; + void commit() const override; void detach() const override; - + bool inRange(Address) const override; std::shared_ptr getAllocator() const { return m_allocator; } SlabAllocator &getSlot(std::uint32_t slot_num) const; - + std::pair > getRange(std::uint32_t slot_num = 0) const override; - + private: std::shared_ptr m_allocator; Allocator *m_allocator_ptr; @@ -60,4 +65,4 @@ namespace db0 Allocator &select(std::uint32_t slot_num); }; -} \ No newline at end of file +} diff --git a/tests/unit_tests/AlgoAllocatorTest.cpp b/tests/unit_tests/AlgoAllocatorTest.cpp index f0ae5b05..1ca0f93f 100644 --- a/tests/unit_tests/AlgoAllocatorTest.cpp +++ b/tests/unit_tests/AlgoAllocatorTest.cpp @@ -19,7 +19,7 @@ namespace tests public: virtual void SetUp() override { // set up the address pool functions - m_pool_f = [](unsigned int i) -> Address { + m_pool_f = [](unsigned int i) -> Address { return Address::fromOffset(i * (PAGE_SIZE + SLAB_SIZE * PAGE_SIZE)); }; @@ -27,13 +27,13 @@ namespace tests if (address % (PAGE_SIZE + SLAB_SIZE * PAGE_SIZE) != 0) { THROWF(db0::InternalException) << "AlgoAllocatorTests: invalid address " << address; } - return address / (PAGE_SIZE + SLAB_SIZE * PAGE_SIZE); + return address / (PAGE_SIZE + SLAB_SIZE * PAGE_SIZE); }; } - virtual void TearDown() override { + virtual void TearDown() override { } - + protected: static constexpr unsigned int PAGE_SIZE = 4096; // as number of pages @@ -41,24 +41,44 @@ namespace tests db0::AlgoAllocator::AddressPoolF m_pool_f; db0::AlgoAllocator::ReverseAddressPoolF m_reverse_pool_f; + + db0::AlgoAllocator makeAllocator() const { + return db0::AlgoAllocator(m_pool_f, m_reverse_pool_f, PAGE_SIZE); + } + + Address addressAt(unsigned int index) const { + return Address::fromOffset(index * (PAGE_SIZE + SLAB_SIZE * PAGE_SIZE)); + } + + void allocMany(db0::AlgoAllocator &allocator, int count) const { + for (int i = 0;i < count;++i) { + allocator.alloc(PAGE_SIZE); + } + } + + void assertFindsAllocation(db0::AlgoAllocator &allocator, Address query, Address expectedAddress) const { + auto result = allocator.findAllocation(query); + ASSERT_EQ(result.address, expectedAddress); + ASSERT_EQ(result.size, PAGE_SIZE); + } }; - + TEST_F( AlgoAllocatorTests , testAlgoAllocatorAllocatesMonotonicAddresses ) - { + { db0::AlgoAllocator cut(m_pool_f, m_reverse_pool_f, PAGE_SIZE); std::uint64_t last_address = 0; for (int i = 0;i < 100;++i) { - auto address = cut.alloc(PAGE_SIZE); + auto address = cut.alloc(PAGE_SIZE); ASSERT_TRUE(address >= last_address); last_address = address; } } - + TEST_F( AlgoAllocatorTests , testAlgoAllocatorThrowsOnInvalidFree ) { db0::AlgoAllocator cut(m_pool_f, m_reverse_pool_f, PAGE_SIZE); for (int i = 0;i < 10;++i) { - cut.alloc(PAGE_SIZE); + cut.alloc(PAGE_SIZE); } // unaligned address is OK (inner address) ASSERT_NO_THROW(cut.free(Address::fromOffset(7 * (PAGE_SIZE + SLAB_SIZE * PAGE_SIZE) + 13))); @@ -70,7 +90,7 @@ namespace tests { db0::AlgoAllocator cut(m_pool_f, m_reverse_pool_f, PAGE_SIZE); for (int i = 0;i < 10;++i) { - cut.alloc(PAGE_SIZE); + cut.alloc(PAGE_SIZE); } ASSERT_EQ(PAGE_SIZE, cut.getAllocSize(Address::fromOffset(0))); // inner address @@ -78,5 +98,62 @@ namespace tests // address out of range ASSERT_ANY_THROW(cut.getAllocSize(Address::fromOffset(11 * (PAGE_SIZE + SLAB_SIZE * PAGE_SIZE)))); } - + + TEST_F( AlgoAllocatorTests , testAlgoAllocatorCanFindAllocationByInnerAddress ) + { + auto cut = makeAllocator(); + allocMany(cut, 10); + + auto address = addressAt(7); + assertFindsAllocation(cut, address + static_cast(13), address); + ASSERT_THROW(cut.findAllocation(addressAt(11)), db0::BadAddressException); + ASSERT_THROW(cut.findAllocation(address + static_cast(PAGE_SIZE)), db0::BadAddressException); + } + + TEST_F( AlgoAllocatorTests , testAlgoAllocatorFindAllocationExactAddress ) + { + auto cut = makeAllocator(); + auto address = cut.alloc(PAGE_SIZE); + assertFindsAllocation(cut, address, address); + } + + TEST_F( AlgoAllocatorTests , testAlgoAllocatorFindAllocationLastByte ) + { + auto cut = makeAllocator(); + allocMany(cut, 2); + auto address = addressAt(1); + assertFindsAllocation(cut, address + static_cast(PAGE_SIZE - 1), address); + } + + TEST_F( AlgoAllocatorTests , testAlgoAllocatorFindAllocationRejectsGapBetweenAddresses ) + { + auto cut = makeAllocator(); + auto address = cut.alloc(PAGE_SIZE); + ASSERT_THROW(cut.findAllocation(address + static_cast(PAGE_SIZE)), db0::BadAddressException); + } + + TEST_F( AlgoAllocatorTests , testAlgoAllocatorFindAllocationRejectsNextUnallocatedIndex ) + { + auto cut = makeAllocator(); + allocMany(cut, 3); + ASSERT_THROW(cut.findAllocation(addressAt(3)), db0::BadAddressException); + } + + TEST_F( AlgoAllocatorTests , testAlgoAllocatorFindAllocationRejectsFreedTail ) + { + auto cut = makeAllocator(); + allocMany(cut, 3); + cut.free(addressAt(2)); + ASSERT_THROW(cut.findAllocation(addressAt(2)), db0::BadAddressException); + assertFindsAllocation(cut, addressAt(1), addressAt(1)); + } + + TEST_F( AlgoAllocatorTests , testAlgoAllocatorFindAllocationRejectsAddressAfterReset ) + { + auto cut = makeAllocator(); + auto address = cut.alloc(PAGE_SIZE); + cut.reset(); + ASSERT_THROW(cut.findAllocation(address), db0::BadAddressException); + } + } diff --git a/tests/unit_tests/BitsetAllocatorTest.cpp b/tests/unit_tests/BitsetAllocatorTest.cpp index 6e2228ba..00f3300d 100644 --- a/tests/unit_tests/BitsetAllocatorTest.cpp +++ b/tests/unit_tests/BitsetAllocatorTest.cpp @@ -14,37 +14,51 @@ namespace tests using Address = db0::Address; - class BitsetAllocatorTests: public testing::Test + class BitsetAllocatorTests: public testing::Test { public: - virtual void SetUp() override { + virtual void SetUp() override { } virtual void TearDown() override { - + } protected: db0::TestWorkspace m_workspace; + static constexpr std::size_t PAGE_SIZE = 4096; + + using AllocatorT = db0::BitsetAllocator >; + + AllocatorT makeAllocator(Address base_addr, int direction = 1) { + auto memspace = m_workspace.getMemspace("my-test-prefix_1"); + return AllocatorT(db0::VFixedBitset<123>(memspace), base_addr, PAGE_SIZE, direction); + } + + void assertFindsAllocation(AllocatorT &allocator, Address query, Address expectedAddress) { + auto result = allocator.findAllocation(query); + ASSERT_EQ(result.address, expectedAddress); + ASSERT_EQ(result.size, PAGE_SIZE); + } }; - + TEST_F( BitsetAllocatorTests , testAllocAssignsValidAddresses ) - { + { std::size_t page_size = 4096; auto memspace = m_workspace.getMemspace("my-test-prefix_1"); - + auto base_addr = Address::fromOffset(0); db0::BitsetAllocator > cut(db0::VFixedBitset<123>(memspace), base_addr, page_size, 1); auto ptr1 = cut.alloc(page_size); auto ptr2 = cut.alloc(page_size); ASSERT_NE(ptr1, ptr2); } - + TEST_F( BitsetAllocatorTests , testAllocThenFree ) - { + { std::size_t page_size = 4096; auto memspace = m_workspace.getMemspace("my-test-prefix_1"); - + auto base_addr = Address::fromOffset(0); db0::BitsetAllocator > cut(db0::VFixedBitset<123>(memspace), base_addr, page_size, 1); auto ptr1 = cut.alloc(page_size); @@ -56,25 +70,30 @@ namespace tests ASSERT_ANY_THROW(cut.getAllocSize(ptr2)); // allocates the same range again auto ptr3 = cut.alloc(page_size); - ASSERT_EQ(ptr2, ptr3); + ASSERT_EQ(ptr2, ptr3); // get alloc size works for allocated ranges ASSERT_EQ(page_size, cut.getAllocSize(ptr1)); ASSERT_EQ(page_size, cut.getAllocSize(ptr3)); } - + TEST_F( BitsetAllocatorTests , testBitsetAllocatorCanHandleSubAddresses ) { // this functionality allows allocating some range and then requesting a subrange of it std::size_t page_size = 4096; auto memspace = m_workspace.getMemspace("my-test-prefix_1"); - + auto base_addr = Address::fromOffset(0); db0::BitsetAllocator > cut(db0::VFixedBitset<123>(memspace), base_addr, page_size, 1); auto ptr1 = cut.alloc(page_size); std::size_t offset = 128; ASSERT_EQ(cut.getAllocSize(ptr1 + offset), page_size - offset); + auto result = cut.findAllocation(ptr1 + offset); + ASSERT_EQ(result.address, ptr1); + ASSERT_EQ(result.size, page_size); + ASSERT_THROW(cut.findAllocation(ptr1 + page_size), db0::BadAddressException); + ASSERT_THROW(cut.findAllocation(Address::fromOffset(page_size * 123)), db0::BadAddressException); } - + TEST_F( BitsetAllocatorTests , testBitsetAllocatorCanAllocateInNegativeDirection ) { // this functionality allows allocating some range and then requesting a subrange of it @@ -87,6 +106,56 @@ namespace tests ASSERT_EQ(ptr1.getOffset(), page_size * 1024 - page_size); auto ptr2 = cut.alloc(page_size); ASSERT_TRUE(ptr2 < ptr1); + auto result = cut.findAllocation(ptr1 + static_cast(123)); + ASSERT_EQ(result.address, ptr1); + ASSERT_EQ(result.size, page_size); + } + + TEST_F( BitsetAllocatorTests , testBitsetAllocatorFindAllocationExactAddress ) + { + auto cut = makeAllocator(Address::fromOffset(0)); + auto address = cut.alloc(PAGE_SIZE); + assertFindsAllocation(cut, address, address); } - -} + + TEST_F( BitsetAllocatorTests , testBitsetAllocatorFindAllocationLastByte ) + { + auto cut = makeAllocator(Address::fromOffset(0)); + auto address = cut.alloc(PAGE_SIZE); + assertFindsAllocation(cut, address + static_cast(PAGE_SIZE - 1), address); + } + + TEST_F( BitsetAllocatorTests , testBitsetAllocatorFindAllocationRejectsFreedBlock ) + { + auto cut = makeAllocator(Address::fromOffset(0)); + auto first = cut.alloc(PAGE_SIZE); + auto second = cut.alloc(PAGE_SIZE); + cut.free(second); + ASSERT_THROW(cut.findAllocation(second), db0::BadAddressException); + assertFindsAllocation(cut, first, first); + } + + TEST_F( BitsetAllocatorTests , testBitsetAllocatorFindAllocationRejectsAddressBeyondBitset ) + { + auto cut = makeAllocator(Address::fromOffset(0)); + cut.alloc(PAGE_SIZE); + ASSERT_THROW(cut.findAllocation(Address::fromOffset(PAGE_SIZE * 123)), db0::BadAddressException); + } + + TEST_F( BitsetAllocatorTests , testBitsetAllocatorFindAllocationRejectsAddressBeforeForwardBase ) + { + auto cut = makeAllocator(Address::fromOffset(PAGE_SIZE)); + cut.alloc(PAGE_SIZE); + ASSERT_THROW(cut.findAllocation(Address::fromOffset(0)), db0::BadAddressException); + } + + TEST_F( BitsetAllocatorTests , testBitsetAllocatorFindAllocationRejectsReverseBaseBoundary ) + { + auto base_addr = Address::fromOffset(PAGE_SIZE * 1024); + auto cut = makeAllocator(base_addr, -1); + auto address = cut.alloc(PAGE_SIZE); + assertFindsAllocation(cut, address + static_cast(PAGE_SIZE - 1), address); + ASSERT_THROW(cut.findAllocation(base_addr), db0::BadAddressException); + } + +} diff --git a/tests/unit_tests/CRDT_AllocatorTests.cpp b/tests/unit_tests/CRDT_AllocatorTests.cpp index bd80e131..67d78f26 100644 --- a/tests/unit_tests/CRDT_AllocatorTests.cpp +++ b/tests/unit_tests/CRDT_AllocatorTests.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -36,7 +37,7 @@ namespace tests m_blanks.reset(); m_aligned_blanks.reset(); m_stripes.reset(); - m_bitspace.clear(); + m_bitspace.clear(); } protected: @@ -56,7 +57,7 @@ namespace tests std::unique_ptr m_aligned_blanks; std::unique_ptr m_stripes; - void init(std::size_t max_addr, const std::vector> &blanks = {}, + void init(std::size_t max_addr, const std::vector> &blanks = {}, std::optional min_aligned_alloc_size = {}) { using CompT = typename AlignedBlankSetT::CompT; @@ -79,18 +80,30 @@ namespace tests } } } + + db0::CRDT_Allocator makeAllocator(std::uint32_t max_address = MAX_ADDRESS) { + return db0::CRDT_Allocator(*m_allocs, *m_blanks, *m_aligned_blanks, *m_stripes, max_address, page_size); + } + + void assertFindsAllocation(db0::CRDT_Allocator &allocator, std::uint64_t query, + std::uint64_t expected_address, std::size_t expected_size) const + { + auto result = allocator.findAllocation(query); + ASSERT_EQ(result.first, expected_address); + ASSERT_EQ(result.second, expected_size); + } }; - + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorCanAllocFromBlanks ) { db0::CRDT_Allocator cut(*m_allocs, *m_blanks, *m_aligned_blanks, *m_stripes, MAX_ADDRESS, page_size); cut.alloc(8); ASSERT_EQ(m_allocs->size(), 1); - ASSERT_EQ(m_blanks->size(), 1); + ASSERT_EQ(m_blanks->size(), 1); ASSERT_EQ(m_stripes->size(), 1); } - + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorCanAllocFromStripes ) { db0::CRDT_Allocator cut(*m_allocs, *m_blanks, *m_aligned_blanks, *m_stripes, MAX_ADDRESS, page_size); @@ -100,7 +113,7 @@ namespace tests cut.alloc(8); ASSERT_EQ(m_allocs->size(), 2); - ASSERT_EQ(m_blanks->size(), 1); + ASSERT_EQ(m_blanks->size(), 1); ASSERT_EQ(m_stripes->size(), 1); } @@ -129,7 +142,7 @@ namespace tests cut.alloc(11); ASSERT_EQ(m_allocs->size(), 4); - ASSERT_EQ(m_blanks->size(), 1); + ASSERT_EQ(m_blanks->size(), 1); ASSERT_EQ(m_stripes->size(), 2); // the subsequent allocations done from existing stripes @@ -150,7 +163,7 @@ namespace tests } ASSERT_TRUE(fill_map.all()); } - + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorGetAllocSize ) { db0::CRDT_Allocator cut(*m_allocs, *m_blanks, *m_aligned_blanks, *m_stripes, MAX_ADDRESS, page_size); @@ -166,20 +179,86 @@ namespace tests ASSERT_EQ(cut.getAllocSize(addresses[i]), sizes[i]); } } - + + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorCanFindAllocationByInnerAddress ) + { + auto cut = makeAllocator(); + auto address = cut.alloc(33); + + assertFindsAllocation(cut, address + 17, address, 33u); + } + + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorFindAllocationExactAddress ) + { + auto cut = makeAllocator(); + auto address = cut.alloc(33); + assertFindsAllocation(cut, address, address, 33u); + } + + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorFindAllocationLastByte ) + { + auto cut = makeAllocator(); + auto address = cut.alloc(33); + assertFindsAllocation(cut, address + 32, address, 33u); + } + + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorFindAllocationRejectsEndBoundary ) + { + auto cut = makeAllocator(); + auto address = cut.alloc(33); + ASSERT_THROW(cut.findAllocation(address + 33), db0::BadAddressException); + } + + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorFindAllocationRejectsAddressOverUInt32 ) + { + auto cut = makeAllocator(); + ASSERT_THROW(cut.findAllocation(std::numeric_limits::max() + 1ULL), db0::BadAddressException); + } + + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorFindAllocationRejectsEmptyAllocatorAddress ) + { + auto cut = makeAllocator(); + ASSERT_THROW(cut.findAllocation(0), db0::BadAddressException); + } + + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorFindAllocationSkipsFreedFillMapUnit ) + { + auto cut = makeAllocator(); + std::vector addresses; + for (int i = 0; i < 10; ++i) { + addresses.push_back(cut.alloc(8)); + } + + cut.free(addresses[5]); + + ASSERT_THROW(cut.findAllocation(addresses[5]), db0::BadAddressException); + ASSERT_THROW(cut.findAllocation(addresses[5] + 7), db0::BadAddressException); + + assertFindsAllocation(cut, addresses[4] + 7, addresses[4], 8u); + assertFindsAllocation(cut, addresses[6], addresses[6], 8u); + } + + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorFindAllocationRejectsFullyFreedAllocation ) + { + auto cut = makeAllocator(); + auto address = cut.alloc(33); + cut.free(address); + ASSERT_THROW(cut.findAllocation(address), db0::BadAddressException); + } + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorFreeFromAllocs ) { db0::CRDT_Allocator cut(*m_allocs, *m_blanks, *m_aligned_blanks, *m_stripes, 64 * page_size, page_size); std::vector sizes = { 16, 16, 16, 1, 2, 4 }; std::vector addresses; - + for (auto size : sizes) { addresses.push_back(cut.alloc(size)); } - + ASSERT_EQ(m_blanks->size(), 1); ASSERT_EQ(m_stripes->size(), 4); - + cut.free(addresses[0]); cut.free(addresses[1]); cut.free(addresses[2]); @@ -190,7 +269,7 @@ namespace tests cut.free(addresses[4]); ASSERT_EQ(m_stripes->size(), 2); ASSERT_EQ(m_blanks->size(), 3); - + cut.free(addresses[3]); ASSERT_EQ(m_stripes->size(), 1); ASSERT_EQ(m_blanks->size(), 2); @@ -212,12 +291,12 @@ namespace tests if (last_alloc_count && alloc_count > *last_alloc_count) { if (!stripe_sizes.empty() && stripe_sizes.back() == current_stripe_size) { break; - } + } stripe_sizes.push_back(current_stripe_size); current_stripe_size = 0; } ++current_stripe_size; - last_alloc_count = alloc_count; + last_alloc_count = alloc_count; } // validate stripe sizes ASSERT_EQ(stripe_sizes.size(), 4); @@ -225,7 +304,7 @@ namespace tests ASSERT_TRUE(stripe_sizes[i] > stripe_sizes[i - 1]); } } - + TEST_F( CRDT_AllocatorTests , DISABLED_testCRDT_AllocatorCanReclaimSpaceFromStripes ) { db0::CRDT_Allocator cut(*m_allocs, *m_blanks, *m_aligned_blanks, *m_stripes, MAX_ADDRESS, page_size); @@ -255,13 +334,13 @@ namespace tests } } ASSERT_EQ(m_allocs->size(), *last_alloc_count); - + // now try a different size alloc, the space should be reclaimed from stripes ASSERT_NO_THROW(cut.alloc(11)); ASSERT_NO_THROW(cut.alloc(15)); ASSERT_NO_THROW(cut.alloc(31)); } - + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorFromStripeCanBeConstrainedWithDynamicBounds ) { std::uint32_t admin_span = 200; @@ -272,7 +351,7 @@ namespace tests std::uint32_t b0 = (b1 >= admin_span) ? b1 - admin_span : 0; return { b0, b1, b2 }; }; - + db0::CRDT_Allocator cut(*m_allocs, *m_blanks, *m_aligned_blanks, *m_stripes, MAX_ADDRESS, page_size); cut.setDynamicBound(bounds_fn); @@ -294,11 +373,11 @@ namespace tests { srand(5916412u); db0::CRDT_Allocator cut(*m_allocs, *m_blanks, *m_aligned_blanks, *m_stripes, MAX_ADDRESS, page_size); - + std::vector alloc_sizes; std::vector addresses; auto count = 100; - { + { // make random allocations for (int i = 0; i < count; ++i) { auto alloc_size = rand() % 50 + 1; @@ -307,9 +386,9 @@ namespace tests alloc_sizes.push_back(alloc_size); addresses.push_back(*ptr); } - } + } } - + // free selected addresses in random order for (int i = 0; i < count / 5; ++i) { auto index = rand() % addresses.size(); @@ -317,9 +396,9 @@ namespace tests cut.free(addresses[index]); alloc_sizes[index] = 0; } - } + } } - + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorAllocSpeed ) { auto max_addr = 64 * 1024 * 1024; @@ -338,13 +417,13 @@ namespace tests std::cout << "Total bytes: " << total_bytes << std::endl; std::cout << "MB / sec : " << (total_bytes / 1024.0 / 1024.0) * 1000.0 / elapsed.count() << std::endl; } - + TEST_F( CRDT_AllocatorTests , testCRDT_AllocatorFirstAllocatedAddress ) { db0::CRDT_Allocator cut(*m_allocs, *m_blanks, *m_aligned_blanks, *m_stripes, MAX_ADDRESS, page_size); ASSERT_EQ(cut.alloc(8), db0::CRDT_Allocator::getFirstAddress()); } - + TEST_F( CRDT_AllocatorTests , testCRDT_PageAlignedAllocs ) { auto capacity = page_size * 3 * 100; @@ -360,7 +439,7 @@ namespace tests ASSERT_EQ(addr % page_size, 0); } } - + TEST_F( CRDT_AllocatorTests , testCRDT_AlignedAllocsFromSmallBlanks ) { auto capacity = page_size * 256; @@ -376,7 +455,7 @@ namespace tests for (auto alloc_size: { 8192, 2000, 100, 4 }) { auto addr = cut.alloc(alloc_size, true); // make sure is page aligned - ASSERT_EQ(addr % page_size, 0); + ASSERT_EQ(addr % page_size, 0); } } @@ -400,5 +479,5 @@ namespace tests auto blank_ptr = m_blanks->upper_equal_bound(Blank(2304, 0)); ASSERT_EQ((*blank_ptr.first).m_size, 5617); } - + } diff --git a/tests/unit_tests/DRAMSpaceTest.cpp b/tests/unit_tests/DRAMSpaceTest.cpp index d6311a49..7cae4428 100644 --- a/tests/unit_tests/DRAMSpaceTest.cpp +++ b/tests/unit_tests/DRAMSpaceTest.cpp @@ -18,20 +18,26 @@ namespace tests { using namespace db0; - - class DRAMSpaceTest: public testing::Test + + class DRAMSpaceTest: public testing::Test { public: - virtual void SetUp() override { + virtual void SetUp() override { } - virtual void TearDown() override { + virtual void TearDown() override { } protected: - const std::size_t m_page_size = 4096; + const std::size_t m_page_size = 4096; + + void assertFindsAllocation(DRAM_Allocator &allocator, Address query, Address expected_address) const { + auto result = allocator.findAllocation(query); + ASSERT_EQ(result.address, expected_address); + ASSERT_EQ(result.size, m_page_size); + } }; - + TEST_F( DRAMSpaceTest, testDRAMSpaceCanAlloc ) { auto cut = DRAMSpace::create(m_page_size); @@ -60,7 +66,7 @@ namespace tests TEST_F( DRAMSpaceTest, testDRAMSpaceCanReuseAddress ) { - auto cut = DRAMSpace::create(m_page_size); + auto cut = DRAMSpace::create(m_page_size); cut.alloc(m_page_size); auto addr_2 = cut.alloc(m_page_size); cut.free(addr_2); @@ -76,13 +82,13 @@ namespace tests for (std::uint64_t i = 0; 100 < 3; ++i) { sgb_tree.insert(i); } - + std::uint64_t i = 0; for (auto it = sgb_tree.cbegin(); !it.is_end(); ++it, ++i) { - ASSERT_EQ(*it, i); + ASSERT_EQ(*it, i); } } - + TEST_F( DRAMSpaceTest, testDRAMSpaceInsertSpeed ) { // use 16kb page @@ -98,7 +104,7 @@ namespace tests for (int i = 0; i < item_count; ++i) { values.push_back(rand()); } - + // measure speed { auto start = std::chrono::high_resolution_clock::now(); @@ -110,7 +116,7 @@ namespace tests std::cout << "SGB_Tree inserted " << item_count << " items in " << elapsed.count() << " ms" << std::endl; } - + // the same test with std::set std::set std_set; { @@ -124,7 +130,7 @@ namespace tests std::cout << "std::set inserted " << item_count << " items in " << elapsed.count() << " ms" << std::endl; } } - + TEST_F( DRAMSpaceTest, testDRAMAllocatorCanBeCreatedWithAllocs ) { std::unordered_set allocs { 1, 2, 3, 9, 15 }; @@ -134,14 +140,66 @@ namespace tests ASSERT_NO_THROW(cut.free(Address::fromOffset(addr))); } } - + + TEST_F( DRAMSpaceTest, testDRAMAllocatorCanFindAllocationByInnerAddress ) + { + DRAM_Allocator cut(m_page_size); + auto address = cut.alloc(m_page_size); + assertFindsAllocation(cut, address + static_cast(31), address); + } + + TEST_F( DRAMSpaceTest, testDRAMAllocatorFindAllocationExactAddress ) + { + DRAM_Allocator cut(m_page_size); + auto address = cut.alloc(m_page_size); + assertFindsAllocation(cut, address, address); + } + + TEST_F( DRAMSpaceTest, testDRAMAllocatorFindAllocationLastByte ) + { + DRAM_Allocator cut(m_page_size); + auto address = cut.alloc(m_page_size); + assertFindsAllocation(cut, address + static_cast(m_page_size - 1), address); + } + + TEST_F( DRAMSpaceTest, testDRAMAllocatorFindAllocationRejectsEndBoundary ) + { + DRAM_Allocator cut(m_page_size); + auto address = cut.alloc(m_page_size); + ASSERT_THROW(cut.findAllocation(address + static_cast(m_page_size)), db0::BadAddressException); + } + + TEST_F( DRAMSpaceTest, testDRAMAllocatorFindAllocationRejectsReservedZeroAddress ) + { + DRAM_Allocator cut(m_page_size); + cut.alloc(m_page_size); + ASSERT_THROW(cut.findAllocation(Address::fromOffset(0)), db0::BadAddressException); + } + + TEST_F( DRAMSpaceTest, testDRAMAllocatorFindAllocationRejectsFreedPage ) + { + DRAM_Allocator cut(m_page_size); + auto address = cut.alloc(m_page_size); + cut.free(address); + ASSERT_THROW(cut.findAllocation(address + static_cast(31)), db0::BadAddressException); + } + + TEST_F( DRAMSpaceTest, testDRAMAllocatorFindAllocationWorksWithExistingAllocs ) + { + std::unordered_set allocs { m_page_size, 3 * m_page_size }; + DRAM_Allocator cut(allocs, m_page_size); + + assertFindsAllocation(cut, Address::fromOffset(3 * m_page_size + 17), Address::fromOffset(3 * m_page_size)); + ASSERT_THROW(cut.findAllocation(Address::fromOffset(2 * m_page_size)), db0::BadAddressException); + } + TEST_F( DRAMSpaceTest, testVBVectorCanBePutOnDRAMSpace ) { // use 4KiB, 16KiB page sizes std::vector page_sizes { 4u << 10, 16u << 10 }; - for (auto page_size: page_sizes) { + for (auto page_size: page_sizes) { auto cut = DRAMSpace::create(page_size); - + // Using std::uint32_t as capacity type to handle large page size using BVectorT = db0::v_bvector; // NOTE: must be created as fixed-block to DRAM space requirements @@ -152,5 +210,5 @@ namespace tests ASSERT_EQ(b_vector.size(), 10000u); } } - -} \ No newline at end of file + +} diff --git a/tests/unit_tests/MetaAllocatorTest.cpp b/tests/unit_tests/MetaAllocatorTest.cpp index 6b8c2371..e15365df 100644 --- a/tests/unit_tests/MetaAllocatorTest.cpp +++ b/tests/unit_tests/MetaAllocatorTest.cpp @@ -18,7 +18,7 @@ namespace tests using namespace db0; using SlabRecycler = db0::Recycler; - + // a proxy class to expose protected members for testing class MetaAllocatorProxy: public MetaAllocator { @@ -41,7 +41,7 @@ namespace tests : m_recycler(2u << 30, m_dirty_meter) { } - + void SetUp() override { using StorageT = db0::Storage0; @@ -50,26 +50,34 @@ namespace tests m_dirty_meter = 0; m_recycler.clear(); } - - void TearDown() override + + void TearDown() override { m_prefix->close(); m_prefix = nullptr; m_dirty_meter = 0; m_recycler.clear(); } - + protected: // in bytes static constexpr std::size_t PAGE_SIZE = 4096; static constexpr std::size_t SLAB_SIZE = 4u << 20; static constexpr std::size_t SMALL_SLAB_SIZE = 64 * 4096; - + std::atomic m_dirty_meter = 0; CacheRecycler m_recycler; std::shared_ptr m_prefix; + + void assertFindsAllocation(MetaAllocator &allocator, Address query, Address expected_address, + std::size_t expected_size) const + { + auto result = allocator.findAllocation(query); + ASSERT_EQ(result.address, expected_address); + ASSERT_EQ(result.size, expected_size); + } }; - + TEST_F( MetaAllocatorTests , testAddressPoolFunction ) { auto page_size = 4096; @@ -84,14 +92,14 @@ namespace tests addr += page_size; expected_addresses.push_back(addr); addr += page_size; - expected_addresses.push_back(addr); + expected_addresses.push_back(addr); } addr += slab_size * slab_count; } for (unsigned int i = 0; i < expected_addresses.size(); ++i) { ASSERT_EQ(Address::fromOffset(expected_addresses[i]), f(i)); - } + } } TEST_F( MetaAllocatorTests , testReverseAddressPoolFunction ) @@ -108,33 +116,33 @@ namespace tests addr += page_size; expected_addresses.push_back(addr); addr += page_size; - expected_addresses.push_back(addr); + expected_addresses.push_back(addr); } addr += slab_size * slab_count; } - + for (unsigned int i = 0; i < expected_addresses.size(); ++i) { ASSERT_EQ(rf(Address::fromOffset(expected_addresses[i])), i); - } + } } - + TEST_F( MetaAllocatorTests , testMetaAllocatorCanBeInitialized ) - { + { // prepare prefix before first use MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SLAB_SIZE); - MetaAllocator cut(m_prefix); + MetaAllocator cut(m_prefix); } TEST_F( MetaAllocatorTests , testMetaAllocatorCanAllocateFromNewSlab ) - { + { // prepare prefix before first use MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SLAB_SIZE); MetaAllocator cut(m_prefix); - + std::vector alloc_sizes = { 100, 200, 300, 400, 500, 600, 700, 800, 900 }; std::uint64_t last_address = 0; for (auto alloc_size: alloc_sizes) { - auto ptr = cut.alloc(alloc_size); + auto ptr = cut.alloc(alloc_size); ASSERT_TRUE(ptr.getOffset() > last_address); last_address = ptr; } @@ -145,26 +153,26 @@ namespace tests { // prepare prefix before first use MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SLAB_SIZE); - + { // first assign from a new slab MetaAllocatorProxy cut(m_prefix); - std::vector alloc_sizes = { 100, 200, 300, 400, 500, 600, 700, 800, 900 }; + std::vector alloc_sizes = { 100, 200, 300, 400, 500, 600, 700, 800, 900 }; for (auto alloc_size: alloc_sizes) { auto ptr = cut.alloc(alloc_size); ASSERT_EQ(cut.getSlabId(ptr), 0); } cut.close(); } - + // open again and try to allocate MetaAllocatorProxy cut(m_prefix); auto ptr = cut.alloc(100); // the allocation should be in the same slab - ASSERT_EQ(cut.getSlabId(ptr), 0); - cut.close(); + ASSERT_EQ(cut.getSlabId(ptr), 0); + cut.close(); } - + TEST_F( MetaAllocatorTests , testMetaAllocatorCanAllocateFromMultipleExistingSlabs ) { MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SMALL_SLAB_SIZE); @@ -185,7 +193,7 @@ namespace tests ASSERT_TRUE(cut.getSlabId(ptr) > 0); cut.close(); } - + TEST_F( MetaAllocatorTests , testMetaAllocatorRemainingCapacityIsTrackedPerSlab ) { MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SMALL_SLAB_SIZE); @@ -219,13 +227,13 @@ namespace tests // make allocations until the 2 slabs are occupied MetaAllocator cut(m_prefix, &recycler); while (cut.getSlabCount() < 2) { - cut.alloc(100); + cut.alloc(100); } cut.close(); } ASSERT_EQ(recycler.size(), 0); } - + TEST_F( MetaAllocatorTests , testMetaAllocatorRemainingCapacityIsPersistedOnClose ) { MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SMALL_SLAB_SIZE); @@ -243,12 +251,12 @@ namespace tests } cut.close(); } - + MetaAllocator cut(m_prefix, &recycler); ASSERT_TRUE(cut.getRemainingCapacity(slab_ids[0]) < 100); ASSERT_TRUE(cut.getRemainingCapacity(slab_ids[1]) > 100); } - + TEST_F( MetaAllocatorTests , testMetaAllocatorCanGetAllocSize ) { srand(191231u); @@ -274,7 +282,106 @@ namespace tests ASSERT_EQ(cut.getAllocSize(addresses[i]), alloc_sizes[i]); } } - + + TEST_F( MetaAllocatorTests , testMetaAllocatorCanFindAllocationByInnerAddressAfterReopen ) + { + srand(191231u); + MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SMALL_SLAB_SIZE); + SlabRecycler recycler; + std::vector alloc_sizes; + std::vector
addresses; + auto count = 1000; + { + MetaAllocator cut(m_prefix, &recycler); + for (int i = 0; i < count; ++i) { + auto alloc_size = rand() % 1000 + 32; + alloc_sizes.push_back(alloc_size); + addresses.push_back(cut.alloc(alloc_size)); + } + cut.close(); + } + + MetaAllocator cut(m_prefix, &recycler); + for (int i = 0; i < count; ++i) { + assertFindsAllocation(cut, addresses[i] + static_cast(alloc_sizes[i] - 1), + addresses[i], alloc_sizes[i]); + } + } + + TEST_F( MetaAllocatorTests , testMetaAllocatorFindAllocationExactAddress ) + { + MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SMALL_SLAB_SIZE); + SlabRecycler recycler; + MetaAllocator cut(m_prefix, &recycler); + + auto address = cut.alloc(128); + assertFindsAllocation(cut, address, address, 128); + cut.close(); + } + + TEST_F( MetaAllocatorTests , testMetaAllocatorFindAllocationLastByte ) + { + MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SMALL_SLAB_SIZE); + SlabRecycler recycler; + MetaAllocator cut(m_prefix, &recycler); + + auto address = cut.alloc(128); + assertFindsAllocation(cut, address + static_cast(127), address, 128); + cut.close(); + } + + TEST_F( MetaAllocatorTests , testMetaAllocatorFindAllocationRejectsEndBoundary ) + { + MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SMALL_SLAB_SIZE); + SlabRecycler recycler; + MetaAllocator cut(m_prefix, &recycler); + + auto address = cut.alloc(128); + ASSERT_THROW(cut.findAllocation(address + static_cast(128)), db0::BadAddressException); + cut.close(); + } + + TEST_F( MetaAllocatorTests , testMetaAllocatorFindAllocationValidatesRealm ) + { + MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SMALL_SLAB_SIZE); + SlabRecycler recycler; + MetaAllocator cut(m_prefix, &recycler); + + auto address = cut.alloc(128, 0, false, 1); + assertFindsAllocation(cut, address + static_cast(12), address, 128); + cut.findAllocation(address + static_cast(12), 1); + ASSERT_THROW(cut.findAllocation(address + static_cast(12), 0), db0::BadAddressException); + cut.close(); + } + + TEST_F( MetaAllocatorTests , testMetaAllocatorFindAllocationRejectsFlushedFree ) + { + MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SMALL_SLAB_SIZE); + SlabRecycler recycler; + MetaAllocator cut(m_prefix, &recycler); + + auto address = cut.alloc(128); + cut.free(address); + cut.flush(); + ASSERT_THROW(cut.findAllocation(address + static_cast(12)), db0::BadAddressException); + cut.close(); + } + + TEST_F( MetaAllocatorTests , testMetaAllocatorFindAllocationHidesDeferredFree ) + { + MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SMALL_SLAB_SIZE); + SlabRecycler recycler; + MetaAllocator cut(m_prefix, &recycler); + + auto address = cut.alloc(128); + cut.findAllocation(address + static_cast(12)); + + cut.free(address); + ASSERT_THROW(cut.findAllocation(address), db0::BadAddressException); + ASSERT_THROW(cut.findAllocation(address + static_cast(12)), db0::BadAddressException); + cut.close(); + } + TEST_F( MetaAllocatorTests , testMetaAllocatorCanFree ) { srand(191231u); @@ -302,9 +409,9 @@ namespace tests cut.free(addresses[index]); alloc_sizes[index] = 0; } - } + } } - + TEST_F( MetaAllocatorTests , testMetaAllocatorCanReleaseEmptySlab ) { srand(191231u); @@ -324,7 +431,7 @@ namespace tests ASSERT_EQ(cut.getSlabCount(), addr_map.size()); cut.close(); } - + // Remove from the highest slabs first auto slab_count = addr_map.size(); MetaAllocator cut(m_prefix, &recycler, false); @@ -336,7 +443,7 @@ namespace tests ASSERT_EQ(cut.getSlabCount(), slab_count); } } - + TEST_F( MetaAllocatorTests , testMetaAllocatorAllocSpeed ) { /* @@ -344,7 +451,7 @@ namespace tests for comparison the regular malloc operation did 13.7 allocs / sec */ MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, 64 * 1024 * 1024); - SlabRecycler recycler; + SlabRecycler recycler; MetaAllocator cut(m_prefix, &recycler); // measure speed auto start = std::chrono::high_resolution_clock::now(); @@ -371,7 +478,7 @@ namespace tests MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SMALL_SLAB_SIZE); SlabRecycler recycler; MetaAllocator cut(m_prefix, &recycler); - auto private_slab = cut.reserveNewSlab(); + auto private_slab = cut.reserveNewSlab(); std::pair range { private_slab->getAddress(), private_slab->getAddress() + static_cast(private_slab->size()) }; @@ -389,18 +496,18 @@ namespace tests } cut.close(); } - + TEST_F( MetaAllocatorTests , testMetaAllocatorFirstAllocatedAddress ) - { + { MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SMALL_SLAB_SIZE); SlabRecycler recycler; MetaAllocator cut(m_prefix, &recycler); ASSERT_EQ(cut.alloc(8), cut.getFirstAddress()); cut.close(); } - + TEST_F( MetaAllocatorTests , testMetaAllocatorLocalityAwareAllocation ) - { + { MetaAllocator::formatPrefix(m_prefix, PAGE_SIZE, SMALL_SLAB_SIZE); SlabRecycler recycler; MetaAllocatorProxy cut(m_prefix, &recycler); @@ -416,5 +523,5 @@ namespace tests ASSERT_NE(cut.getSlabId(addr_0), cut.getSlabId(addr_2)); cut.close(); } - -} \ No newline at end of file + +} diff --git a/tests/unit_tests/SlabAllocatorTests.cpp b/tests/unit_tests/SlabAllocatorTests.cpp index 4e6ca961..9cc1650a 100644 --- a/tests/unit_tests/SlabAllocatorTests.cpp +++ b/tests/unit_tests/SlabAllocatorTests.cpp @@ -7,7 +7,10 @@ #include #include #include +#include #include +#include +#include using namespace std; @@ -21,12 +24,12 @@ namespace tests { public: - SlabAllocatorTests() - : m_memspace(m_workspace.getMemspace("test_slab_allocator")) + SlabAllocatorTests() + : m_memspace(m_workspace.getMemspace("test_slab_allocator")) { } - virtual void SetUp() override { + virtual void SetUp() override { } virtual void TearDown() override { @@ -37,19 +40,33 @@ namespace tests db0::Memspace m_memspace; static constexpr std::size_t page_size = 4096; static constexpr std::size_t slab_size = 64 * 1024 * 1024; + + db0::SlabAllocator makeSlabAllocator(Address begin_addr, std::size_t size = slab_size) const + { + db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, size, page_size); + return db0::SlabAllocator(m_memspace.getPrefixPtr(), begin_addr, size, page_size); + } + + void assertFindsAllocation(db0::SlabAllocator &allocator, Address query, Address expected_address, + std::size_t expected_size) const + { + auto result = allocator.findAllocation(query); + ASSERT_EQ(result.address, expected_address); + ASSERT_EQ(result.size, expected_size); + } }; - + TEST_F( SlabAllocatorTests , testNewlyFormattedSlabAllocatorCanBeOpened ) - { - // initialize a new slab under the address = 0 + { + // initialize a new slab under the address = 0 auto begin_addr = Address::fromOffset(0); db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, slab_size, page_size); db0::SlabAllocator cut(m_memspace.getPrefixPtr(), begin_addr, slab_size, page_size); } TEST_F( SlabAllocatorTests , testSlabAllocationsAreTakenFromSlabFront ) - { - // initialize a new slab under the address = 0 + { + // initialize a new slab under the address = 0 auto begin_addr = Address::fromOffset(0); db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, slab_size, page_size); db0::SlabAllocator cut(m_memspace.getPrefixPtr(), begin_addr, slab_size, page_size); @@ -58,10 +75,10 @@ namespace tests auto addr_2 = cut.alloc(71); ASSERT_TRUE (addr_2 > addr_1); } - + TEST_F( SlabAllocatorTests , testSlabAllocatorGetAllocSize ) - { - // initialize a new slab under the address = 0 + { + // initialize a new slab under the address = 0 auto begin_addr = Address::fromOffset(0); db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, slab_size, page_size); db0::SlabAllocator cut(m_memspace.getPrefixPtr(), begin_addr, slab_size, page_size); @@ -78,35 +95,120 @@ namespace tests ASSERT_EQ(sizes[i], cut.getAllocSize(addresses[i])); } } - + + TEST_F( SlabAllocatorTests , testSlabAllocatorCanFindAllocationByInnerAddress ) + { + auto begin_addr = Address::fromOffset(0); + auto cut = makeSlabAllocator(begin_addr); + + auto address = cut.alloc(71); + assertFindsAllocation(cut, address + static_cast(17), address, 71); + ASSERT_THROW(cut.findAllocation(address + static_cast(71)), db0::BadAddressException); + ASSERT_THROW(cut.findAllocation(begin_addr + static_cast(slab_size)), db0::BadAddressException); + } + + TEST_F( SlabAllocatorTests , testSlabAllocatorFindAllocationExactAddress ) + { + auto cut = makeSlabAllocator(Address::fromOffset(0)); + auto address = cut.alloc(71); + + assertFindsAllocation(cut, address, address, 71); + } + + TEST_F( SlabAllocatorTests , testSlabAllocatorFindAllocationLastByte ) + { + auto cut = makeSlabAllocator(Address::fromOffset(0)); + auto address = cut.alloc(71); + + assertFindsAllocation(cut, address + static_cast(70), address, 71); + } + + TEST_F( SlabAllocatorTests , testSlabAllocatorFindAllocationRejectsAddressBeforeSlab ) + { + auto begin_addr = Address::fromOffset(8 * 1024 * 1024); + auto cut = makeSlabAllocator(begin_addr); + + cut.alloc(71); + ASSERT_THROW(cut.findAllocation(begin_addr - static_cast(1)), db0::BadAddressException); + } + + TEST_F( SlabAllocatorTests , testSlabAllocatorFindAllocationRejectsFreedAllocation ) + { + auto cut = makeSlabAllocator(Address::fromOffset(0)); + auto address = cut.alloc(71); + + cut.free(address); + ASSERT_THROW(cut.findAllocation(address + static_cast(17)), db0::BadAddressException); + } + + TEST_F( SlabAllocatorTests , testSlotAllocatorCanFindGeneralAndSlotAllocations ) + { + auto main_allocator = std::make_shared(); + db0::SlotAllocator cut(main_allocator); + auto generalAddress = cut.alloc(64); + + auto slotBegin = Address::fromOffset(8 * 1024 * 1024); + db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), slotBegin, slab_size, page_size); + auto slotAllocator = std::make_shared(m_memspace.getPrefixPtr(), slotBegin, slab_size, page_size); + cut.setSlot(1, slotAllocator); + auto slotAddress = cut.alloc(80, 1); + + auto general = cut.findAllocation(generalAddress + static_cast(13)); + ASSERT_EQ(general.address, generalAddress); + ASSERT_EQ(general.size, 64u); + + ASSERT_THROW(cut.findAllocation(slotAddress + static_cast(19)), db0::BadAddressException); + + auto slot = cut.findAllocation(slotAddress + static_cast(19), static_cast(1)); + ASSERT_EQ(slot.address, slotAddress); + ASSERT_EQ(slot.size, 80u); + ASSERT_THROW(cut.findAllocation(slotAddress + static_cast(80), static_cast(1)), db0::BadAddressException); + ASSERT_THROW(cut.findAllocation(Address::fromOffset(32 * 1024 * 1024)), db0::BadAddressException); + } + + TEST_F( SlabAllocatorTests , testOneShotAllocatorCanFindAllocationByInnerAddress ) + { + auto address = Address::fromOffset(4096); + db0::OneShotAllocator cut(address, 256); + ASSERT_EQ(cut.alloc(256), address); + + auto result = cut.findAllocation(address + static_cast(255)); + ASSERT_EQ(result.address, address); + ASSERT_EQ(result.size, 256u); + ASSERT_THROW(cut.findAllocation(address + static_cast(256)), db0::BadAddressException); + + cut.free(address); + ASSERT_THROW(cut.findAllocation(address), db0::BadAddressException); + } + TEST_F( SlabAllocatorTests , testCalculateAdminSpaceSize ) - { + { auto calculated_size = db0::SlabAllocator::calculateAdminSpaceSize(page_size); // construct acutal SlabAllocator with identical parameters auto begin_addr = Address::fromOffset(0); db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, page_size * 64, page_size); db0::SlabAllocator cut(m_memspace.getPrefixPtr(), begin_addr, page_size * 64, page_size); - + ASSERT_EQ(cut.getAdminSpaceSize(false), calculated_size); } - + TEST_F( SlabAllocatorTests , testSlabAllocatorCannotBeCreatedIfSizeTooSmall ) - { - // throws because size of the administrative space exceeds the size of the slab + { + // throws because size of the administrative space exceeds the size of the slab auto begin_addr = Address::fromOffset(0); ASSERT_ANY_THROW( db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, page_size * 3, page_size); ); } - + TEST_F( SlabAllocatorTests , testSlabAllocatorCanDynamicallyManageAvailableSize ) - { + { // this test if to check if the available data space is adjusted according to changing administrative space // create allocator over the 4-pages slab auto begin_addr = Address::fromOffset(0); db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, page_size * 64, page_size); db0::SlabAllocator cut(m_memspace.getPrefixPtr(), begin_addr, page_size * 64, page_size); - + std::vector
addresses; // make allocations until the entire slab is occupied std::uint64_t max_addr = 0; @@ -125,10 +227,10 @@ namespace tests total_size += size; ++size; } - + // make sure the max address is not conflicting with the admin space ASSERT_TRUE(max_addr + max_size + cut.getAdminSpaceSize(false) <= cut.getSlabSize()); - + size = 1; for (auto &addr: addresses) { ASSERT_EQ(size, cut.getAllocSize(addr)); @@ -155,25 +257,25 @@ namespace tests // measure the administrative overhead auto admin_overhead = cut.getAdminSpaceSize(true) / (double)size_; - ASSERT_TRUE(admin_overhead < 0.1); + ASSERT_TRUE(admin_overhead < 0.1); } - + TEST_F( SlabAllocatorTests , testSlabAllocatorCanFillAvailableCapacity ) { // 256kb slab auto size_ = 256 * 1024; auto begin_addr = Address::fromOffset(0); - auto init_capacity = db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, size_, page_size); + auto init_capacity = db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, size_, page_size); db0::SlabAllocator cut(m_memspace.getPrefixPtr(), begin_addr, size_, page_size, init_capacity); - + std::size_t total_allocated = 0; // perform 100b allocations until full - for (;;) { + for (;;) { auto addr = cut.tryAlloc(100); if (!addr) break; - total_allocated += 100; + total_allocated += 100; } ASSERT_TRUE(cut.getRemainingCapacity() < 100); @@ -187,7 +289,7 @@ namespace tests auto begin_addr = Address::fromOffset(0); db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, size_, page_size); db0::SlabAllocator cut(m_memspace.getPrefixPtr(), begin_addr, size_, page_size); - + ASSERT_TRUE(cut.empty()); // perform random allocations until full std::vector
addresses; @@ -205,7 +307,7 @@ namespace tests } TEST_F( SlabAllocatorTests , testSlabAllocatorAllocSpeed ) - { + { auto size_ = 64 * 1024 * 1024; auto begin_addr = Address::fromOffset(0); db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, size_, page_size); @@ -224,9 +326,9 @@ namespace tests std::cout << "Total bytes: " << total_bytes << std::endl; std::cout << "MB / sec : " << (total_bytes / 1024.0 / 1024.0) * 1000.0 / elapsed.count() << std::endl; } - + TEST_F( SlabAllocatorTests , testSlabAllocatorCanMakeAddressUnique ) - { + { auto size_ = 64 * 1024 * 1024; auto begin_addr = Address::fromOffset(0); db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, size_, page_size); @@ -236,17 +338,17 @@ namespace tests auto addr1 = cut.tryMakeAddressUnique(addr); ASSERT_TRUE(addr1.isValid()); auto addr2 = cut.tryMakeAddressUnique(addr); - ASSERT_TRUE(addr2.isValid()); + ASSERT_TRUE(addr2.isValid()); ASSERT_NE(addr1, addr2); } - + TEST_F( SlabAllocatorTests , testSlabAllocatorDeallocAfterReachingCapacity ) { auto size_ = 64 << 20; auto begin_addr = Address::fromOffset(0); - auto init_capacity = db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, size_, page_size); + auto init_capacity = db0::SlabAllocator::formatSlab(m_memspace.getPrefixPtr(), begin_addr, size_, page_size); db0::SlabAllocator cut(m_memspace.getPrefixPtr(), begin_addr, size_, page_size, init_capacity, 0); - + std::vector
addresses; // perform allocations until full for (;;) { @@ -264,8 +366,8 @@ namespace tests for (auto &addr: addresses) { cut.free(addr); } - + ASSERT_TRUE(cut.getRemainingCapacity() + cut.getLostCapacity() >= init_capacity); } - -} \ No newline at end of file + +} diff --git a/tests/utils/EmbeddedAllocator.cpp b/tests/utils/EmbeddedAllocator.cpp index d79dd1ca..4937ad5b 100644 --- a/tests/utils/EmbeddedAllocator.cpp +++ b/tests/utils/EmbeddedAllocator.cpp @@ -8,7 +8,7 @@ namespace db0 { - std::optional
EmbeddedAllocator::tryAlloc(std::size_t size, std::uint32_t slot_num, + std::optional
EmbeddedAllocator::tryAlloc(std::size_t size, std::uint32_t slot_num, bool aligned, unsigned char, unsigned char) { auto new_address = Address::fromOffset(4096 * ++m_count); @@ -18,14 +18,14 @@ namespace db0 } return new_address; } - + void EmbeddedAllocator::free(Address address) { auto it = m_allocations.find(address); if (it == m_allocations.end()) { THROWF(db0::InternalException) << "address not found: " << address; } - m_allocations.erase(it); + m_allocations.erase(it); } std::size_t EmbeddedAllocator::getAllocSize(Address address) const @@ -36,7 +36,7 @@ namespace db0 } return it->second; } - + bool EmbeddedAllocator::isAllocated(Address address, std::size_t *size_of_result) const { auto it = m_allocations.find(address); @@ -46,9 +46,21 @@ namespace db0 if (size_of_result) { *size_of_result = it->second; } - return true; + return true; } - + + Allocator::AllocationInfo EmbeddedAllocator::findAllocation(Address address) const + { + for (auto &alloc: m_allocations) { + auto begin = alloc.first.getOffset(); + auto end = begin + alloc.second; + if (address.getOffset() >= begin && address.getOffset() < end) { + return AllocationInfo { alloc.first, alloc.second }; + } + } + THROWF(db0::BadAddressException) << "Invalid address: " << address << THROWF_END; + } + void EmbeddedAllocator::commit() const { // nothing to do } @@ -56,9 +68,9 @@ namespace db0 void EmbeddedAllocator::detach() const { // nothing to do } - + void EmbeddedAllocator::setAllocCallback(AllocCallbackT callback) { this->m_alloc_callback = callback; } - -} \ No newline at end of file + +} diff --git a/tests/utils/EmbeddedAllocator.hpp b/tests/utils/EmbeddedAllocator.hpp index 01949381..0e756d99 100644 --- a/tests/utils/EmbeddedAllocator.hpp +++ b/tests/utils/EmbeddedAllocator.hpp @@ -20,16 +20,18 @@ namespace db0 public: using AllocCallbackT = std::function)>; EmbeddedAllocator() = default; - - std::optional
tryAlloc(std::size_t size, std::uint32_t, + + std::optional
tryAlloc(std::size_t size, std::uint32_t, bool aligned = false, unsigned char realm_id = 0, unsigned char locality = 0) override; - + void free(Address) override; std::size_t getAllocSize(Address) const override; bool isAllocated(Address, std::size_t *size_of_result = nullptr) const override; + AllocationInfo findAllocation(Address) const override; + void commit() const override; void detach() const override; @@ -42,5 +44,5 @@ namespace db0 std::unordered_map m_allocations; AllocCallbackT m_alloc_callback; }; - -} \ No newline at end of file + +}