diff --git a/README.md b/README.md index 0871e5b..f453ca1 100644 --- a/README.md +++ b/README.md @@ -1366,7 +1366,7 @@ cd build && ctest --output-on-failure | `msgpack_unit` | C | 29 | Core SQLite extension unit tests | | `msgpack_sql` | SQL | — | SQL integration tests via CLI | | `msgpack_spec_p1`–`p10` | C | 10 suites | Per-section msgpack spec compliance | -| `msgpack_blob_unit` | C++ | 320 | Standalone C++ API (no SQLite dependency) | +| `msgpack_blob_unit` | C++ | 632 | Standalone C++ API (no SQLite dependency) | | `msgpack_interop` | C++ | 197 | C++ ↔ SQLite interoperability | | `fuzz_corpus` | C | 100+ | Fuzz corpus against SQL extension | | `fuzz_blob_corpus` | C++ | 100+ | Fuzz corpus against C++ API | diff --git a/cpp/README.md b/cpp/README.md index 2841ccf..16ca7a3 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -317,13 +317,47 @@ Blob blob = b.build(); | Method | Description | |---|---| -| `buf_data() → const uint8_t*` | Pointer to bytes accumulated so far | +| `buf_data() → const uint8_t*` | Zero-copy pointer to bytes accumulated so far | | `buf_size() → size_t` | Number of bytes accumulated so far | +| `capacity() → size_t` | Bytes currently allocated (≥ `buf_size()`) | + +The `buf_data()`/`buf_size()` view stays valid until the next encode call, +`reset()`, or `build()` — consume it before mutating the Builder again. + +### Buffer reuse (tight loops) + +`build()` *moves* the internal buffer into the returned `Blob`, so a Builder that +is built and discarded each iteration allocates a fresh buffer every time. To +encode many blobs in a hot loop **without re-allocating**, rewind one Builder +with `reset()` and read the bytes through `buf_data()`/`buf_size()` instead: + +| Method | Description | +|---|---| +| `reset() → Builder&` | Rewind to empty, **keeping** the heap allocation (capacity) | +| `reserve(size_t) → Builder&` | Pre-grow the buffer so even the first iterations don't malloc | +| `capacity() → size_t` | Inspect the retained allocation size | + +`reset()` is built on `std::vector::clear()`, which keeps capacity, so after the +buffer warms up to its peak size the loop performs **zero allocations**. Use +`build()` only when you want to hand ownership of the bytes to a `Blob` (which +ends reuse). + +```cpp +Builder b; +b.reserve(64); // optional: skip warm-up growth +for (const auto& item : items) { + b.reset(); // rewind, keep the allocation + b.map_header(2) + .string("id").integer(item.id) + .string("v").real(item.value); + sink(b.buf_data(), b.buf_size()); // consume the zero-copy view +} +``` ### Finalize ```cpp -Blob build(); // consume builder, return Blob +Blob build(); // consume builder (move out), return Blob static Blob quote(const Value&); // one-shot: Value → Blob ``` @@ -447,6 +481,28 @@ std::string json = blob.to_json(); // json == {"name":"Alice","scores":[95,87,91]} ``` +### Reuse one buffer across a tight loop + +When streaming many small blobs (e.g. to a socket, file, or DB), reuse a single +Builder so the encode loop allocates only while the buffer is warming up: + +```cpp +Builder b; +b.reserve(128); // size once for the largest expected row + +for (const Reading& r : readings) { + b.reset(); // rewind; capacity is retained + b.map_header(3) + .string("sensor").string(r.sensor) + .string("temp").real32(r.temp) + .string("ts").value(Value::timestamp(r.epoch)); + + // Emit the freshly encoded msgpack without copying or allocating. + write(fd, b.buf_data(), b.buf_size()); +} +// After the first few iterations b.capacity() stops growing → zero mallocs. +``` + --- ## Build integration @@ -467,7 +523,7 @@ additionally builds the C++ ↔ SQLite `msgpack_interop` test). It produces: - `libmsgpack_blob_static.a` — static library -- `msgpack_blob_unit` — unit test executable (320 tests, standalone C++ API) +- `msgpack_blob_unit` — unit test executable (632 tests, standalone C++ API) - `blob_vectors_gen` — generator for the shared cross-language test vectors ([`../tests/vectors/blob_vectors.json`](../tests/vectors/blob_vectors.json)) - `fuzz_blob_corpus_runner` — corpus-based fuzz runner (100+ corpus files) diff --git a/cpp/include/msgpack_blob.hpp b/cpp/include/msgpack_blob.hpp index b71dab1..d37c8c5 100644 --- a/cpp/include/msgpack_blob.hpp +++ b/cpp/include/msgpack_blob.hpp @@ -242,10 +242,34 @@ class Builder { Builder& timestamp(int64_t sec); Builder& timestamp(int64_t sec, uint32_t nsec); + /* Buffer reuse — encode many blobs with one Builder, no re-malloc. + ** + ** reset() rewinds the Builder to empty but KEEPS the heap allocation, so a + ** single Builder can encode blob after blob inside a hot loop without + ** reallocating. Pair it with the zero-copy buf_data()/buf_size() accessors + ** to emit each encoded blob without ever copying or allocating: + ** + ** Builder b; + ** b.reserve(64); // optional: pre-size to skip warm-up + ** for (auto& item : items) { // grows on heap + ** b.reset(); // rewind, keep capacity + ** b.map_header(2) + ** .string("id").integer(item.id) + ** .string("v").real(item.value); + ** sink(b.buf_data(), b.buf_size()); // consume the bytes + ** } + ** + ** build() instead moves the buffer out (handing ownership to the returned + ** Blob) and therefore ends reuse — prefer reset() in tight loops. */ + Builder& reset() noexcept; + Builder& reserve(size_t bytes); + size_t capacity() const noexcept; + /* Finalize */ Blob build(); - /* Internal buffer access (used by mutation internals) */ + /* Raw buffer view — the bytes encoded so far (zero-copy; valid until the + ** next mutating call, reset(), or build()). Also used by mutation internals. */ const uint8_t* buf_data() const noexcept; size_t buf_size() const noexcept; @@ -256,7 +280,6 @@ class Builder { std::vector buf_; void append(const uint8_t* data, size_t n); void append1(uint8_t b); - uint8_t* reserve(size_t n); }; /* ── Iterator ─────────────────────────────────────────────────────────── */ diff --git a/cpp/src/msgpack_blob_encode.cpp b/cpp/src/msgpack_blob_encode.cpp index b72e816..b7fd735 100644 --- a/cpp/src/msgpack_blob_encode.cpp +++ b/cpp/src/msgpack_blob_encode.cpp @@ -70,11 +70,6 @@ void Builder::append(const uint8_t* data, size_t n) { buf_.insert(buf_.end(), data, data + n); } void Builder::append1(uint8_t b) { buf_.push_back(b); } -uint8_t* Builder::reserve(size_t n) { - size_t old = buf_.size(); - buf_.resize(old + n); - return buf_.data() + old; -} Builder& Builder::nil() { append1(MP_NIL); return *this; } @@ -319,6 +314,13 @@ Builder& Builder::timestamp(int64_t sec, uint32_t nsec) { const uint8_t* Builder::buf_data() const noexcept { return buf_.data(); } size_t Builder::buf_size() const noexcept { return buf_.size(); } +/* Buffer reuse: std::vector::clear() drops the elements but keeps the +** allocated storage, so a Builder can be rewound and reused without a fresh +** malloc. reserve() pre-grows that storage; capacity() reports it. */ +Builder& Builder::reset() noexcept { buf_.clear(); return *this; } +Builder& Builder::reserve(size_t bytes) { buf_.reserve(bytes); return *this; } +size_t Builder::capacity() const noexcept { return buf_.capacity(); } + Builder& Builder::value(const Value& v) { switch (v.type()) { case Type::Nil: return nil(); diff --git a/cpp/tests/test_msgpack_blob.cpp b/cpp/tests/test_msgpack_blob.cpp index 860b1eb..793c503 100644 --- a/cpp/tests/test_msgpack_blob.cpp +++ b/cpp/tests/test_msgpack_blob.cpp @@ -1396,6 +1396,84 @@ static void test_value_as_float() { CHECK(std::fabs(v2.as_float() - 2.25f) < 1e-6f, "as_float from double"); } +/* ── Builder buffer reuse ─────────────────────────────────────────── */ + +static void test_builder_buffer_reuse() { + using namespace msgpack; + + /* reset() clears the contents but keeps the allocated capacity */ + { + Builder b; + b.map_header(2).string("name").string("Alice").string("age").integer(30); + size_t produced = b.buf_size(); + CHECK(produced > 0, "builder has bytes before reset"); + size_t cap_before = b.capacity(); + b.reset(); + CHECK_EQ_INT(b.buf_size(), 0u, "reset rewinds size to 0"); + CHECK(b.capacity() == cap_before, "reset keeps capacity"); + CHECK(b.capacity() >= produced, "retained capacity covers prior blob"); + } + + /* reserve() raises capacity; reset() preserves it */ + { + Builder b; + b.reserve(256); + CHECK(b.capacity() >= 256, "reserve raises capacity"); + size_t cap = b.capacity(); + b.map_header(1).string("k").integer(1); + b.reset(); + CHECK(b.capacity() == cap, "capacity unchanged after encode+reset within reserve"); + } + + /* reset() returns *this so it chains into a fresh encode */ + { + Builder b; + b.integer(999); /* stale content */ + Blob blob = b.reset().boolean(true).build(); + CHECK(blob.valid(), "chained reset()+encode valid"); + CHECK(blob.type() == Type::True, "reset() discarded stale content"); + CHECK_EQ_INT(blob.size(), 1u, "only the post-reset byte remains"); + } + + /* Tight loop: one Builder encodes many varying blobs while reusing the + ** SAME heap allocation — proven by a stable data pointer and capacity. */ + { + Builder b; + b.reserve(64); + const uint8_t* base = b.buf_data(); + size_t cap = b.capacity(); + bool realloced = false; + + for (int i = 0; i < 100; ++i) { + b.reset(); + b.map_header(2) + .string("id").integer(i) + .string("sq").integer(static_cast(i) * i); + + if (b.buf_data() != base || b.capacity() != cap) realloced = true; + + /* Zero-copy view over the builder's live buffer */ + Blob view(b.buf_data(), b.buf_size()); + CHECK(view.valid(), "reused-buffer blob valid"); + CHECK(view.extract("$.id").as_int64() == i, "reused-buffer id correct"); + CHECK(view.extract("$.sq").as_int64() == static_cast(i) * i, + "reused-buffer sq correct"); + } + CHECK(!realloced, "buffer not reallocated across loop iterations"); + } + + /* build() still hands off ownership (ends reuse) and yields a valid Blob */ + { + Builder b; + b.reserve(32); + b.reset(); + b.array_header(3).integer(1).integer(2).integer(3); + Blob blob = b.build(); + CHECK(blob.valid(), "build() after reuse setup is valid"); + CHECK(blob.array_length() == 3, "build() produced expected array"); + } +} + /* ── main ─────────────────────────────────────────────────────────── */ int main() { @@ -1429,6 +1507,7 @@ int main() { test_iterator_empty_containers(); test_unsigned_integer_value_and_builder(); test_value_as_float(); + test_builder_buffer_reuse(); std::printf("\n%d passed, %d failed\n", g_pass, g_fail); return g_fail ? 1 : 0;