diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5718f37..82ecd95 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: run: | sudo apt-get update -q sudo apt-get install -y --no-install-recommends \ - cmake ninja-build gcc + cmake ninja-build gcc g++ - name: Configure run: | @@ -34,9 +34,129 @@ jobs: - name: Test run: ctest --test-dir build --output-on-failure --parallel 4 + - name: Verify shared test vectors are up to date + run: ./build/cpp/blob_vectors_gen | diff - tests/vectors/blob_vectors.json + - name: Upload test results on failure if: failure() uses: actions/upload-artifact@v4 with: name: ctest-output path: build/Testing/ + + cpp-standalone: + name: C++ Blob library (standalone cpp/) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update -q + sudo apt-get install -y --no-install-recommends cmake ninja-build g++ + + - name: Configure, build & test the cpp/ package on its own + run: | + cmake -B build cpp -G Ninja -DMSGPACK_BUILD_TESTS=ON + cmake --build build --parallel + ctest --test-dir build --output-on-failure + + python: + name: Python port + runs-on: ubuntu-latest + defaults: + run: + working-directory: python + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Run tests + run: python -m unittest discover -s tests -v + + js: + name: TypeScript / JS port + runs-on: ubuntu-latest + defaults: + run: + working-directory: js + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '24' + + - name: Install dev dependencies + run: npm ci + + - name: Type-check + run: npm run typecheck + + - name: Build (emit dist/) + run: npm run build + + - name: Run tests + run: npm test + + rust: + name: Rust port + runs-on: ubuntu-latest + defaults: + run: + working-directory: rust + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + + - name: Format check + run: cargo fmt --check + + - name: Clippy + run: cargo clippy --all-targets -- -D warnings + + - name: Run tests + run: cargo test + + go: + name: Go port + runs-on: ubuntu-latest + defaults: + run: + working-directory: go + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 'stable' + + - name: Check formatting + run: | + unformatted="$(gofmt -l .)" + if [ -n "$unformatted" ]; then + echo "These files are not gofmt-clean:" + echo "$unformatted" + exit 1 + fi + + - name: Vet + run: go vet ./... + + - name: Run tests + run: go test ./... -v diff --git a/.gitignore b/.gitignore index 267dae9..c6070c6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,24 @@ build/ *.so *.dSYM/ .DS_Store + +# Node / TypeScript (js/) +node_modules/ +js/dist/ +*.tsbuildinfo + +# Python (python/) +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.mypy_cache/ +python/dist/ +python/build/ + +# Rust (rust/) +rust/target/ +Cargo.lock + +# Go (go/) +go/vendor/ diff --git a/CMakeLists.txt b/CMakeLists.txt index b78e4c2..59ff563 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,7 @@ project(sqlite_msgpack VERSION 1.5.0 LANGUAGES C CXX) # ── Options ────────────────────────────────────────────────────────────────── option(BUILD_SHARED_LIBS "Build msgpack as a loadable extension (.so/.dylib/.dll)" ON) option(MSGPACK_BUILD_TESTS "Build and register CTest tests" ON) +option(MSGPACK_BUILD_FUZZ "Build libFuzzer-based fuzz harness" OFF) # ── Compiler flags ──────────────────────────────────────────────────────────── # Suppress MSVC warnings about POSIX names and use of standard C library functions @@ -80,12 +81,11 @@ target_include_directories(msgpack_static PRIVATE include) target_compile_definitions(msgpack_static PRIVATE SQLITE_CORE) # ── Standalone C++ MsgPack Blob library (no SQLite dependency) ───────────────── -add_library(msgpack_blob_static STATIC src/msgpack_blob.cpp) -target_include_directories(msgpack_blob_static PUBLIC include) -set_target_properties(msgpack_blob_static PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) -if(NOT MSVC) - target_compile_options(msgpack_blob_static PRIVATE -Wall -Wextra) -endif() +# Lives in its own self-contained package under cpp/ (the C++ sibling of the +# python/, js/, rust/ and go/ ports). add_subdirectory makes the +# `msgpack_blob_static` target and its blob unit/corpus/vector-gen tests +# available to this top-level build; the interop test below reuses the target. +add_subdirectory(cpp) # ── SQLite3 CLI shell ───────────────────────────────────────────────────────── add_executable(sqlite3_cli src/shell.c src/sqlite3.c) @@ -145,15 +145,11 @@ if(MSGPACK_BUILD_TESTS) add_msgpack_test(msgpack_spec_p9 test_spec_p9_typed_primitives.c) add_msgpack_test(msgpack_spec_p10 test_spec_p10_timestamp.c) - # C++ Blob API unit tests (standalone, no SQLite dependency) - add_executable(msgpack_blob_unit tests/test_msgpack_blob.cpp) - target_include_directories(msgpack_blob_unit PRIVATE include) - target_link_libraries(msgpack_blob_unit PRIVATE msgpack_blob_static) - set_target_properties(msgpack_blob_unit PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) - add_test(NAME msgpack_blob_unit COMMAND msgpack_blob_unit) - set_tests_properties(msgpack_blob_unit PROPERTIES PASS_REGULAR_EXPRESSION "0 failed") + # C++ Blob API unit tests, the cross-language vector generator and the C++ + # corpus runner live in cpp/CMakeLists.txt (built via add_subdirectory). - # C++ ↔ SQLite interop integration tests + # C++ ↔ SQLite interop integration tests (bridge the SQLite extension and the + # standalone Blob library, so this one stays at the top level). add_executable(msgpack_interop tests/test_interop.cpp) target_include_directories(msgpack_interop PRIVATE include) target_compile_definitions(msgpack_interop PRIVATE SQLITE_CORE) @@ -168,7 +164,7 @@ if(MSGPACK_BUILD_TESTS) endif() # ── Fuzz testing (requires Clang with libFuzzer support) ────────────────────── -option(MSGPACK_BUILD_FUZZ "Build libFuzzer-based fuzz harness" OFF) +# (MSGPACK_BUILD_FUZZ is declared in the options block near the top.) if(MSGPACK_BUILD_FUZZ) # Apply sanitizer flags to all targets so coverage instrumentation @@ -181,13 +177,7 @@ if(MSGPACK_BUILD_FUZZ) target_compile_options(fuzz_msgpack PRIVATE -fsanitize=fuzzer,address -g) target_link_options(fuzz_msgpack PRIVATE -fsanitize=fuzzer,address) - # C++ API fuzz harness - add_executable(fuzz_msgpack_blob - tests/fuzz_msgpack_blob.cpp src/msgpack_blob.cpp) - target_include_directories(fuzz_msgpack_blob PRIVATE include) - set_target_properties(fuzz_msgpack_blob PROPERTIES CXX_STANDARD 17) - target_compile_options(fuzz_msgpack_blob PRIVATE -fsanitize=fuzzer,address -g) - target_link_options(fuzz_msgpack_blob PRIVATE -fsanitize=fuzzer,address) + # The C++ Blob API fuzz harness (fuzz_msgpack_blob) is defined in cpp/. endif() # ── Fuzz corpus runner (no libFuzzer needed — runs corpus files directly) ───── @@ -205,17 +195,7 @@ if(MSGPACK_BUILD_TESTS) COMMAND fuzz_corpus_runner ${CMAKE_SOURCE_DIR}/tests/fuzz_corpus) set_tests_properties(fuzz_corpus PROPERTIES PASS_REGULAR_EXPRESSION "0 failed") - # C++ API corpus runner (no libFuzzer needed) - add_executable(fuzz_blob_corpus_runner - tests/fuzz_blob_corpus_runner.cpp tests/fuzz_msgpack_blob.cpp src/msgpack_blob.cpp) - target_include_directories(fuzz_blob_corpus_runner PRIVATE include) - set_target_properties(fuzz_blob_corpus_runner PROPERTIES CXX_STANDARD 17) - if(NOT MSVC) - target_compile_options(fuzz_blob_corpus_runner PRIVATE -w) - endif() - add_test(NAME fuzz_blob_corpus - COMMAND fuzz_blob_corpus_runner ${CMAKE_SOURCE_DIR}/tests/fuzz_corpus) - set_tests_properties(fuzz_blob_corpus PROPERTIES PASS_REGULAR_EXPRESSION "0 failed") + # The C++ API corpus runner (fuzz_blob_corpus_runner) is defined in cpp/. endif() # ── Install headers ─────────────────────────────────────────────────────────── diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c6ce8dd..36c05ea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,7 @@ your fix. - **C code** (`src/msgpack.c`): follows the existing SQLite-adjacent style — 2-space indent, `camelCase` locals, `snake_case` functions. -- **C++ code** (`src/msgpack_blob.cpp`, `include/msgpack_blob.hpp`): `snake_case` +- **C++ code** (`cpp/src/msgpack_blob_*.cpp`, `cpp/include/msgpack_blob.hpp`): `snake_case` for methods and variables, `PascalCase` for types/enums, 4-space indent. - Compiler warnings are the primary lint mechanism. The C++ library compiles cleanly with `-Wall -Wextra -Wpedantic -Wshadow -Wconversion -Wsign-conversion`. @@ -79,21 +79,25 @@ your fix. ``` include/ - msgpack_blob.hpp C++ public header sqlite3.h SQLite amalgamation header src/ msgpack.c SQLite extension implementation - msgpack_blob.cpp Standalone C++ library sqlite3.c SQLite amalgamation +cpp/ Standalone C++ Blob library (self-contained package) + include/msgpack_blob.hpp C++ public header + src/msgpack_blob_detail.hpp Shared private internals + src/msgpack_blob_{decode,encode,json,mutate,iterate}.cpp Library modules + tests/test_msgpack_blob.cpp C++ API unit tests + tests/fuzz_msgpack_blob.cpp libFuzzer harness (C++ API) + tests/gen_blob_vectors.cpp Cross-language test-vector generator + README.md C++ API reference +python/ js/ rust/ go/ Native ports of the Blob API (see each README) tests/ test_msgpack.c C unit tests - test_msgpack_blob.cpp C++ API unit tests test_interop.cpp C++ ↔ SQLite integration tests fuzz_msgpack.c libFuzzer harness (SQL extension) - fuzz_msgpack_blob.cpp libFuzzer harness (C++ API) - fuzz_corpus/ Seed corpus files -docs/ - cpp-api.md C++ API reference + fuzz_corpus/ Seed corpus files (shared) + vectors/blob_vectors.json Shared cross-language test vectors ``` ## Submitting changes @@ -102,7 +106,7 @@ docs/ 2. Make your changes — keep commits focused and well-described. 3. Ensure all tests pass: `cd build && ctest --output-on-failure` 4. If you add new functionality, add corresponding tests. -5. If you change the public API, update `docs/cpp-api.md` and/or `README.md`. +5. If you change the public API, update `cpp/README.md` and/or `README.md`. 6. Open a pull request with a clear description of what and why. ## Byte-identical encoding diff --git a/README.md b/README.md index 1fb8d1c..0871e5b 100644 --- a/README.md +++ b/README.md @@ -14,13 +14,22 @@ who has used `json_extract`, `json_set`, or `json_each`. MessagePack values are stored as ordinary SQLite `BLOB` columns. All functions are deterministic and side-effect free (copy-on-write mutation). -A standalone **[C++ Blob API](docs/cpp-api.md)** is also provided for use outside +A standalone **[C++ Blob API](cpp/README.md)** is also provided for use outside SQLite — it supports all msgpack primitive types (including fixed-width integers, -float32/64, ext, timestamp, and binary) and produces byte-identical blobs. Full +float32/64, ext, timestamp, and binary) and produces byte-identical blobs. It +lives in its own [`cpp/`](cpp/) package alongside the other language ports. Full [interop tests](tests/test_interop.cpp) verify round-trip compatibility between -the SQL and C++ APIs, and a [fuzz harness](tests/fuzz_msgpack_blob.cpp) exercises +the SQL and C++ APIs, and a [fuzz harness](cpp/tests/fuzz_msgpack_blob.cpp) exercises every public entry point with arbitrary byte sequences. +Native re-implementations of the Blob API are also available for +**[Python](python/)**, **[TypeScript / JavaScript](js/)**, **[Rust](rust/)** and +**[Go](go/)**. They expose the same `Blob` / `Builder` / `Value` / `Iterator` +API and produce byte-identical output, so blobs are fully interchangeable +between SQL, C++, Python, JS, Rust and Go. All ports are validated against a +[shared set of test vectors](tests/vectors/blob_vectors.json) generated from the +C++ reference implementation. + --- ## Table of contents @@ -45,8 +54,9 @@ every public entry point with arbitrary byte sequences. 8. [MessagePack spec compliance](#messagepack-spec-compliance) 9. [Performance benchmarks](#performance-benchmarks) 10. [Serialised-size comparison](#serialised-size-comparison) -11. [C++ Blob API](docs/cpp-api.md) -12. [Testing](#testing) +11. [C++ Blob API](cpp/README.md) +12. [Python & JS/TS libraries](#language-libraries) +13. [Testing](#testing) --- @@ -1286,6 +1296,59 @@ is more compact. --- +## Language libraries + +Besides the SQLite extension, the Blob API ships as self-contained packages — +the reference **[C++ library](cpp/README.md)** plus native, zero-dependency +re-implementations in other languages. Each one exposes the same `Blob` / +`Builder` / `Value` / `Iterator` API and produces **byte-identical** msgpack +output, so blobs round-trip freely between SQL, C++, Python, JavaScript, Rust and +Go. + +| Library | Location | Runtime | Tests | +|---|---|---|---| +| C++ (reference) | [`cpp/`](cpp/) | C++17, no deps | `cmake -B build cpp && ctest --test-dir build` | +| Python | [`python/`](python/) | Python ≥ 3.7, stdlib only | `python -m unittest discover -s tests` | +| TypeScript / JS | [`js/`](js/) | Node ≥ 18 (ESM), no deps | `npm test` | +| Rust | [`rust/`](rust/) | Rust ≥ 1.70, no deps | `cargo test` | +| Go | [`go/`](go/) | Go ≥ 1.21, stdlib only | `go test ./...` | + +```python +# Python +from msgpack_blob import Blob, Value +blob = Blob.from_json('{"name":"Alice"}') +blob.set("$.age", Value.integer(30)).to_json() # '{"name":"Alice","age":30}' +``` + +```ts +// TypeScript +import { Blob, Value } from "msgpack-blob"; +const blob = Blob.fromJson('{"name":"Alice"}'); +blob.set("$.age", Value.integer(30)).toJson(); // '{"name":"Alice","age":30}' +``` + +```rust +// Rust +use msgpack_blob::{Blob, Value}; +let blob = Blob::from_json(r#"{"name":"Alice"}"#); +blob.set("$.age", &Value::integer(30)).to_json(); // {"name":"Alice","age":30} +``` + +```go +// Go +import mb "github.com/khanaffan/sqlite-msgpack/go" +blob := mb.FromJSON(`{"name":"Alice"}`) +blob.Set("$.age", mb.Int(30)).ToJSON() // {"name":"Alice","age":30} +``` + +All ports are verified against a [shared vector file](tests/vectors/blob_vectors.json) +generated from the C++ reference implementation +([`cpp/tests/gen_blob_vectors.cpp`](cpp/tests/gen_blob_vectors.cpp), CTest target +`blob_vectors_gen`), guaranteeing cross-language byte-identity for encoding, +JSON conversion, mutation, extraction and iteration. + +--- + ## Testing The project includes a comprehensive test suite covering both the SQL extension and the C++ API. @@ -1311,7 +1374,7 @@ cd build && ctest --output-on-failure ### Fuzz testing Both the SQL extension and C++ API have dedicated libFuzzer harnesses -(`tests/fuzz_msgpack.c` and `tests/fuzz_msgpack_blob.cpp`). Without libFuzzer, +(`tests/fuzz_msgpack.c` and `cpp/tests/fuzz_msgpack_blob.cpp`). Without libFuzzer, the corpus runners (`fuzz_corpus_runner` and `fuzz_blob_corpus_runner`) exercise the same code paths using 100+ seed files (including adversarially deep nesting, truncated length prefixes, and reserved-byte inputs) as part of the @@ -1323,5 +1386,5 @@ cmake -B build-fuzz -DMSGPACK_BUILD_FUZZ=ON \ -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ cmake --build build-fuzz ./build-fuzz/fuzz_msgpack tests/fuzz_corpus -max_total_time=300 -./build-fuzz/fuzz_msgpack_blob tests/fuzz_corpus -max_total_time=300 +./build-fuzz/cpp/fuzz_msgpack_blob tests/fuzz_corpus -max_total_time=300 ``` diff --git a/SECURITY.md b/SECURITY.md index 621ce63..b7bfe38 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -32,7 +32,7 @@ within **7 days** for confirmed vulnerabilities. The following are in scope for security reports: - **Buffer overflows / out-of-bounds reads** in the SQLite extension (`src/msgpack.c`) - or the C++ library (`src/msgpack_blob.cpp`) when processing untrusted msgpack blobs + or the C++ library (`cpp/src/msgpack_blob_*.cpp`) when processing untrusted msgpack blobs - **Memory leaks** triggered by crafted input - **Denial of service** via pathological input (excessive CPU or memory consumption) - **Integer overflows** leading to incorrect behaviour or memory corruption diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt new file mode 100644 index 0000000..b9cf1ab --- /dev/null +++ b/cpp/CMakeLists.txt @@ -0,0 +1,83 @@ +cmake_minimum_required(VERSION 3.16) +project(msgpack_blob VERSION 1.5.0 LANGUAGES CXX) + +# ── Standalone C++ MessagePack Blob library (no SQLite dependency) ───────────── +# A self-contained package — the C++ sibling of the python/, js/, rust/ and go/ +# ports. Split into logical translation units (decode, encode, json, mutate, +# iterate) that share private internals via src/msgpack_blob_detail.hpp. +# +# Can be built standalone: +# cmake -B build cpp && cmake --build build && ctest --test-dir build +# or pulled into the top-level build via add_subdirectory(cpp), which is how the +# repository root reuses the `msgpack_blob_static` target for its C++ ↔ SQLite +# interop tests. + +option(MSGPACK_BUILD_TESTS "Build and register CTest tests" ON) +option(MSGPACK_BUILD_FUZZ "Build the libFuzzer-based fuzz harness" OFF) + +# Raw sources, exported to the parent scope so an instrumented fuzz build can +# recompile them with sanitizer flags. +set(MSGPACK_BLOB_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/src/msgpack_blob_decode.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/msgpack_blob_encode.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/msgpack_blob_json.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/msgpack_blob_mutate.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/msgpack_blob_iterate.cpp +) +if(NOT CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(MSGPACK_BLOB_SOURCES ${MSGPACK_BLOB_SOURCES} PARENT_SCOPE) +endif() + +add_library(msgpack_blob_static STATIC ${MSGPACK_BLOB_SOURCES}) +target_include_directories(msgpack_blob_static PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) +set_target_properties(msgpack_blob_static PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) +if(NOT MSVC) + target_compile_options(msgpack_blob_static PRIVATE -Wall -Wextra) +endif() + +install(TARGETS msgpack_blob_static ARCHIVE DESTINATION lib) +install(FILES include/msgpack_blob.hpp DESTINATION include) + +# ── Tests ───────────────────────────────────────────────────────────────────── +if(MSGPACK_BUILD_TESTS) + enable_testing() + + # C++ Blob API unit tests (standalone, no SQLite dependency) + add_executable(msgpack_blob_unit tests/test_msgpack_blob.cpp) + target_link_libraries(msgpack_blob_unit PRIVATE msgpack_blob_static) + set_target_properties(msgpack_blob_unit PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) + add_test(NAME msgpack_blob_unit COMMAND msgpack_blob_unit) + set_tests_properties(msgpack_blob_unit PROPERTIES PASS_REGULAR_EXPRESSION "0 failed") + + # Cross-language test-vector generator (reference impl = C++ Blob library). + # Run it to refresh tests/vectors/blob_vectors.json consumed by every port: + # cmake --build build --target blob_vectors_gen + # ./build/blob_vectors_gen > tests/vectors/blob_vectors.json + add_executable(blob_vectors_gen tests/gen_blob_vectors.cpp) + target_link_libraries(blob_vectors_gen PRIVATE msgpack_blob_static) + set_target_properties(blob_vectors_gen PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) + + # C++ API corpus runner (no libFuzzer needed — runs the shared corpus files). + add_executable(fuzz_blob_corpus_runner + tests/fuzz_blob_corpus_runner.cpp tests/fuzz_msgpack_blob.cpp) + target_link_libraries(fuzz_blob_corpus_runner PRIVATE msgpack_blob_static) + set_target_properties(fuzz_blob_corpus_runner PROPERTIES CXX_STANDARD 17) + if(NOT MSVC) + target_compile_options(fuzz_blob_corpus_runner PRIVATE -w) + endif() + add_test(NAME fuzz_blob_corpus + COMMAND fuzz_blob_corpus_runner ${CMAKE_CURRENT_SOURCE_DIR}/../tests/fuzz_corpus) + set_tests_properties(fuzz_blob_corpus PROPERTIES PASS_REGULAR_EXPRESSION "0 failed") +endif() + +# ── Fuzz testing (requires Clang with libFuzzer support) ────────────────────── +if(MSGPACK_BUILD_FUZZ) + # Compile the library sources directly so sanitizer/coverage instrumentation + # reaches the Blob code, not just the harness. + add_executable(fuzz_msgpack_blob + tests/fuzz_msgpack_blob.cpp ${MSGPACK_BLOB_SOURCES}) + target_include_directories(fuzz_msgpack_blob PRIVATE include) + set_target_properties(fuzz_msgpack_blob PROPERTIES CXX_STANDARD 17) + target_compile_options(fuzz_msgpack_blob PRIVATE -fsanitize=fuzzer,address -g) + target_link_options(fuzz_msgpack_blob PRIVATE -fsanitize=fuzzer,address) +endif() diff --git a/docs/cpp-api.md b/cpp/README.md similarity index 91% rename from docs/cpp-api.md rename to cpp/README.md index 1560ae0..2841ccf 100644 --- a/docs/cpp-api.md +++ b/cpp/README.md @@ -451,37 +451,50 @@ std::string json = blob.to_json(); ## Build integration -The library is built as part of the CMake project: +This directory is a self-contained CMake package — the C++ sibling of the +[`python/`](../python/), [`js/`](../js/), [`rust/`](../rust/) and [`go/`](../go/) +ports. Build and test it standalone: -```cmake -cmake -B build -DMSGPACK_BUILD_TESTS=ON +```bash +cmake -B build cpp -DMSGPACK_BUILD_TESTS=ON cmake --build build +ctest --test-dir build --output-on-failure ``` -This produces: +…or let the repository-root build pull it in automatically via +`add_subdirectory(cpp)` (the default top-level `cmake -B build` does this and +additionally builds the C++ ↔ SQLite `msgpack_interop` test). + +It produces: - `libmsgpack_blob_static.a` — static library -- `msgpack_blob_unit` — unit test executable (289 tests, standalone C++ API) -- `msgpack_interop` — integration test executable (197 tests, C++ ↔ SQLite interop) -- `fuzz_blob_corpus_runner` — corpus-based fuzz runner (83+ corpus files) +- `msgpack_blob_unit` — unit test executable (320 tests, standalone C++ API) +- `blob_vectors_gen` — generator for the shared cross-language test vectors + ([`../tests/vectors/blob_vectors.json`](../tests/vectors/blob_vectors.json)) +- `fuzz_blob_corpus_runner` — corpus-based fuzz runner (100+ corpus files) +- `msgpack_interop` — C++ ↔ SQLite interop test (197 tests; built by the root project only) -Link against `msgpack_blob_static` and add `include/` to your include path. +Link against `msgpack_blob_static`; its `PUBLIC` include directory +(`cpp/include`) propagates automatically, so `#include "msgpack_blob.hpp"` just +works. ### Testing ```bash -# Run all tests (unit + integration + fuzz corpus) -cd build && ctest --output-on-failure +# Standalone cpp/ package build: +cmake -B build cpp -DMSGPACK_BUILD_TESTS=ON && cmake --build build +ctest --test-dir build --output-on-failure -# Run only C++ API unit tests +# Run only the C++ API unit tests ./build/msgpack_blob_unit -# Run C++ ↔ SQLite interop tests -./build/msgpack_interop - -# Run C++ fuzz corpus +# Run the C++ fuzz corpus (corpus ships at the repository root) ./build/fuzz_blob_corpus_runner tests/fuzz_corpus ``` +When built from the repository root instead, the blob binaries land under +`build/cpp/` (e.g. `./build/cpp/msgpack_blob_unit`) and the interop test runs as +`./build/msgpack_interop`. + ### Fuzz testing The C++ API has a dedicated libFuzzer harness (`tests/fuzz_msgpack_blob.cpp`) that @@ -499,7 +512,7 @@ exercises every public entry point with arbitrary byte sequences: To run with libFuzzer (requires Clang with libFuzzer support): ```bash -cmake -B build-fuzz -DMSGPACK_BUILD_FUZZ=ON \ +cmake -B build-fuzz cpp -DMSGPACK_BUILD_FUZZ=ON \ -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ cmake --build build-fuzz --target fuzz_msgpack_blob ./build-fuzz/fuzz_msgpack_blob tests/fuzz_corpus -max_total_time=300 diff --git a/include/msgpack_blob.hpp b/cpp/include/msgpack_blob.hpp similarity index 100% rename from include/msgpack_blob.hpp rename to cpp/include/msgpack_blob.hpp diff --git a/cpp/src/msgpack_blob_decode.cpp b/cpp/src/msgpack_blob_decode.cpp new file mode 100644 index 0000000..95c91c3 --- /dev/null +++ b/cpp/src/msgpack_blob_decode.cpp @@ -0,0 +1,798 @@ +/* +** msgpack_blob_decode.cpp — decoding & inspection +** +** Part of the standalone C++ MsgPack Blob library (see msgpack_blob.hpp). +** Contains: element skipping, validation, type inspection, path resolution, +** element decoding, the Value type, and the read-only Blob accessors. +*/ + +#include "msgpack_blob_detail.hpp" + +#include +#include +#include +#include + +namespace msgpack { +namespace detail { + +/* ── skip_one — skip one complete msgpack element ─────────────────── */ + +static uint32_t skip_one_d(const uint8_t* a, uint32_t n, uint32_t i, int depth); + +uint32_t skip_one(const uint8_t* a, uint32_t n, uint32_t i) { + return skip_one_d(a, n, i, 0); +} +static uint32_t skip_one_d(const uint8_t* a, uint32_t n, uint32_t i, int depth) { + if (depth > kMaxDepth) return 0; + if (i >= n) return 0; + uint8_t b = a[i++]; + + if (b <= 0x7f) return i; /* positive fixint */ + if (b >= 0xe0) return i; /* negative fixint */ + + switch (b) { + case MP_NIL: case MP_FALSE: case MP_TRUE: + return i; + case MP_FLOAT32: + return (i + 4 <= n) ? i + 4 : 0; + case MP_FLOAT64: case MP_INT64: case MP_UINT64: + return (i + 8 <= n) ? i + 8 : 0; + case MP_UINT8: case MP_INT8: + return (i + 1 <= n) ? i + 1 : 0; + case MP_UINT16: case MP_INT16: + return (i + 2 <= n) ? i + 2 : 0; + case MP_UINT32: case MP_INT32: + return (i + 4 <= n) ? i + 4 : 0; + case MP_BIN8: { + if (i + 1 > n) return 0; + uint32_t sz = a[i]; i++; + return (sz <= n - i) ? i + sz : 0; + } + case MP_BIN16: { + if (i + 2 > n) return 0; + uint32_t sz = read16(a + i); i += 2; + return (sz <= n - i) ? i + sz : 0; + } + case MP_BIN32: { + if (i + 4 > n) return 0; + uint32_t sz = read32(a + i); i += 4; + return (sz <= n - i) ? i + sz : 0; + } + case MP_STR8: { + if (i + 1 > n) return 0; + uint32_t sz = a[i]; i++; + return (sz <= n - i) ? i + sz : 0; + } + case MP_STR16: { + if (i + 2 > n) return 0; + uint32_t sz = read16(a + i); i += 2; + return (sz <= n - i) ? i + sz : 0; + } + case MP_STR32: { + if (i + 4 > n) return 0; + uint32_t sz = read32(a + i); i += 4; + return (sz <= n - i) ? i + sz : 0; + } + case MP_FIXEXT1: return (i + 2 <= n) ? i + 2 : 0; + case MP_FIXEXT2: return (i + 3 <= n) ? i + 3 : 0; + case MP_FIXEXT4: return (i + 5 <= n) ? i + 5 : 0; + case MP_FIXEXT8: return (i + 9 <= n) ? i + 9 : 0; + case MP_FIXEXT16: return (i + 17 <= n) ? i + 17 : 0; + case MP_EXT8: { + if (i + 2 > n) return 0; + uint32_t sz = a[i]; i += 2; + return (sz <= n - i) ? i + sz : 0; + } + case MP_EXT16: { + if (i + 3 > n) return 0; + uint32_t sz = read16(a + i); i += 3; + return (sz <= n - i) ? i + sz : 0; + } + case MP_EXT32: { + if (i + 5 > n) return 0; + uint32_t sz = read32(a + i); i += 5; + return (sz <= n - i) ? i + sz : 0; + } + default: break; + } + + /* fixstr */ + if (b >= 0xa0 && b <= 0xbf) { + uint32_t sz = b & 0x1f; + return (sz <= n - i) ? i + sz : 0; + } + + /* fixarray */ + if (b >= 0x90 && b <= 0x9f) { + uint32_t count = b & 0x0f; + for (uint32_t j = 0; j < count; j++) { + i = skip_one_d(a, n, i, depth + 1); + if (!i) return 0; + } + return i; + } + + /* fixmap */ + if (b >= 0x80 && b <= 0x8f) { + uint32_t count = b & 0x0f; + for (uint32_t j = 0; j < count; j++) { + i = skip_one_d(a, n, i, depth + 1); if (!i) return 0; + i = skip_one_d(a, n, i, depth + 1); if (!i) return 0; + } + return i; + } + + /* array16/32 */ + if (b == MP_ARRAY16 || b == MP_ARRAY32) { + uint32_t count; + if (b == MP_ARRAY16) { + if (i + 2 > n) return 0; + count = read16(a + i); i += 2; + } else { + if (i + 4 > n) return 0; + count = read32(a + i); i += 4; + } + for (uint32_t j = 0; j < count; j++) { + i = skip_one_d(a, n, i, depth + 1); + if (!i) return 0; + } + return i; + } + + /* map16/32 */ + if (b == MP_MAP16 || b == MP_MAP32) { + uint32_t count; + if (b == MP_MAP16) { + if (i + 2 > n) return 0; + count = read16(a + i); i += 2; + } else { + if (i + 4 > n) return 0; + count = read32(a + i); i += 4; + } + for (uint32_t j = 0; j < count; j++) { + i = skip_one_d(a, n, i, depth + 1); if (!i) return 0; + i = skip_one_d(a, n, i, depth + 1); if (!i) return 0; + } + return i; + } + + return 0; +} + +/* ── is_valid ─────────────────────────────────────────────────────── */ + +static bool is_valid(const uint8_t* a, uint32_t n) { + if (n == 0) return false; + uint32_t end = skip_one(a, n, 0); + return end == n; +} + +/* ── error_position_of — byte offset of first error ───────────────── */ + +static size_t error_position_of(const uint8_t* a, uint32_t n) { + if (n == 0) return 0; + uint32_t end = skip_one(a, n, 0); + if (end == n) return 0; + /* Walk byte by byte to find where it goes wrong */ + for (uint32_t i = 0; i < n;) { + uint32_t next = skip_one(a, n, i); + if (!next) return i; + i = next; + } + return 0; +} + +/* ── is_timestamp_ext — check if element at offset is a timestamp ext ── */ + +static bool is_timestamp_ext(const uint8_t* a, uint32_t n, uint32_t i) { + if (i >= n) return false; + uint8_t b = a[i]; + if (b == MP_FIXEXT4 && i + 6 <= n && a[i+1] == MP_TIMESTAMP_TYPE) return true; + if (b == MP_FIXEXT8 && i + 10 <= n && a[i+1] == MP_TIMESTAMP_TYPE) return true; + if (b == MP_EXT8 && i + 3 <= n && a[i+1] == 12 && a[i+2] == MP_TIMESTAMP_TYPE) return true; + return false; +} + +static bool decode_timestamp(const uint8_t* a, uint32_t n, uint32_t i, + int64_t* pSec, uint32_t* pNsec) { + if (i >= n) return false; + uint8_t b = a[i]; + if (b == MP_FIXEXT4 && i + 6 <= n && a[i+1] == MP_TIMESTAMP_TYPE) { + *pSec = static_cast(read32(a + i + 2)); + *pNsec = 0; + return true; + } + if (b == MP_FIXEXT8 && i + 10 <= n && a[i+1] == MP_TIMESTAMP_TYPE) { + uint64_t v = read64(a + i + 2); + *pNsec = static_cast(v >> 34); + *pSec = static_cast(v & 0x3FFFFFFFFULL); + return true; + } + if (b == MP_EXT8 && i + 15 <= n && a[i+1] == 12 && a[i+2] == MP_TIMESTAMP_TYPE) { + *pNsec = read32(a + i + 3); + *pSec = static_cast(read64(a + i + 7)); + return true; + } + return false; +} + +/* ── get_type — return Type for element at offset ─────────────────── */ + +Type get_type(const uint8_t* a, uint32_t n, uint32_t i) { + if (i >= n) return Type::Nil; + uint8_t b = a[i]; + if (b == MP_NIL) return Type::Nil; + if (b == MP_TRUE) return Type::True; + if (b == MP_FALSE) return Type::False; + if (b <= 0x7f || b >= 0xe0) return Type::Integer; + if (b >= 0xa0 && b <= 0xbf) return Type::String; + if (b >= 0x90 && b <= 0x9f) return Type::Array; + if (b >= 0x80 && b <= 0x8f) return Type::Map; + switch (b) { + case MP_UINT8: case MP_UINT16: case MP_UINT32: case MP_UINT64: + case MP_INT8: case MP_INT16: case MP_INT32: case MP_INT64: + return Type::Integer; + case MP_FLOAT32: + return Type::Float32; + case MP_FLOAT64: + return Type::Real; + case MP_STR8: case MP_STR16: case MP_STR32: + return Type::String; + case MP_BIN8: case MP_BIN16: case MP_BIN32: + return Type::Binary; + case MP_ARRAY16: case MP_ARRAY32: + return Type::Array; + case MP_MAP16: case MP_MAP32: + return Type::Map; + case MP_EXT8: case MP_EXT16: case MP_EXT32: + case MP_FIXEXT1: case MP_FIXEXT2: case MP_FIXEXT4: + case MP_FIXEXT8: case MP_FIXEXT16: + if (is_timestamp_ext(a, n, i)) return Type::Timestamp; + return Type::Ext; + default: + return Type::Nil; + } +} + +static const char* get_type_str_at(const uint8_t* a, uint32_t n, uint32_t i) { + switch (get_type(a, n, i)) { + case Type::Nil: return "null"; + case Type::True: return "true"; + case Type::False: return "false"; + case Type::Integer: return "integer"; + case Type::Real: return "real"; + case Type::Float32: return "float32"; + case Type::String: return "text"; + case Type::Binary: return "binary"; + case Type::Array: return "array"; + case Type::Map: return "map"; + case Type::Ext: return "ext"; + case Type::Timestamp: return "timestamp"; + } + return "null"; +} + +/* ── get_container_count ──────────────────────────────────────────── */ + +int64_t get_container_count(const uint8_t* a, uint32_t n, uint32_t i) { + if (i >= n) return -1; + uint8_t b = a[i]; + if (b >= 0x90 && b <= 0x9f) return b & 0x0f; + if (b >= 0x80 && b <= 0x8f) return b & 0x0f; + if (b == MP_ARRAY16 && i + 3 <= n) return read16(a + i + 1); + if (b == MP_ARRAY32 && i + 5 <= n) return read32(a + i + 1); + if (b == MP_MAP16 && i + 3 <= n) return read16(a + i + 1); + if (b == MP_MAP32 && i + 5 <= n) return read32(a + i + 1); + return -1; +} + +/* ── path_step — parse one step of $.path[0].key syntax ───────────── */ + +int path_step( + const char* zPath, int* pi, + const char** pKey, int* nKey, + int64_t* pIdx +) { + int i = *pi; + if (zPath[i] == '\0') return 0; + if (zPath[i] == '.') { + int start; + i++; + start = i; + while (zPath[i] && zPath[i] != '.' && zPath[i] != '[') i++; + *pKey = zPath + start; + *nKey = i - start; + *pi = i; + return 'k'; + } + if (zPath[i] == '[') { + int64_t idx = 0; + int hasDigit = 0; + i++; + while (zPath[i] >= '0' && zPath[i] <= '9') { + idx = idx * 10 + (zPath[i] - '0'); + i++; + hasDigit = 1; + } + if (!hasDigit || zPath[i] != ']') return -1; + i++; + *pIdx = idx; + *pi = i; + return 'i'; + } + return -1; +} + +/* ── lookup — resolve path to byte range ──────────────────────────── */ + +int lookup( + const uint8_t* a, uint32_t n, uint32_t iRoot, + const char* zPath, + uint32_t* piStart, uint32_t* piEnd +) { + int pi; + uint32_t iCur = iRoot; + if (!zPath || zPath[0] != '$') return RC_ERROR; + pi = 1; + + for (;;) { + const char* zKey = nullptr; + int nKey = 0; + int64_t idx = 0; + int step = path_step(zPath, &pi, &zKey, &nKey, &idx); + + if (step == 0) { + uint32_t iNext = skip_one(a, n, iCur); + *piStart = iCur; + *piEnd = iNext ? iNext : n; + return (iNext || iCur == n) ? RC_OK : RC_ERROR; + } + if (step < 0) return RC_ERROR; + if (iCur >= n) return RC_NOTFOUND; + + if (step == 'i') { + uint8_t b = a[iCur]; + uint32_t count, elemOff; + if (b >= 0x90 && b <= 0x9f) { + count = b & 0x0f; elemOff = iCur + 1; + } else if (b == MP_ARRAY16) { + if (iCur + 3 > n) return RC_ERROR; + count = read16(a + iCur + 1); elemOff = iCur + 3; + } else if (b == MP_ARRAY32) { + if (iCur + 5 > n) return RC_ERROR; + count = read32(a + iCur + 1); elemOff = iCur + 5; + } else { + return RC_NOTFOUND; + } + if (idx < 0 || static_cast(idx) >= count) return RC_NOTFOUND; + iCur = elemOff; + for (int64_t j = 0; j < idx; j++) { + iCur = skip_one(a, n, iCur); + if (!iCur) return RC_ERROR; + } + } else { + uint8_t b = a[iCur]; + uint32_t count, elemOff; + bool found = false; + if (b >= 0x80 && b <= 0x8f) { + count = b & 0x0f; elemOff = iCur + 1; + } else if (b == MP_MAP16) { + if (iCur + 3 > n) return RC_ERROR; + count = read16(a + iCur + 1); elemOff = iCur + 3; + } else if (b == MP_MAP32) { + if (iCur + 5 > n) return RC_ERROR; + count = read32(a + iCur + 1); elemOff = iCur + 5; + } else { + return RC_NOTFOUND; + } + iCur = elemOff; + for (uint32_t j = 0; j < count && !found; j++) { + if (iCur >= n) return RC_ERROR; + uint8_t kb = a[iCur]; + const char* kStr = nullptr; + uint32_t kLen = 0; + if (kb >= 0xa0 && kb <= 0xbf) { + kLen = kb & 0x1f; kStr = reinterpret_cast(a + iCur + 1); + } else if (kb == MP_STR8 && iCur + 2 <= n) { + kLen = a[iCur + 1]; kStr = reinterpret_cast(a + iCur + 2); + } else if (kb == MP_STR16 && iCur + 3 <= n) { + kLen = read16(a + iCur + 1); kStr = reinterpret_cast(a + iCur + 3); + } else if (kb == MP_STR32 && iCur + 5 <= n) { + kLen = read32(a + iCur + 1); kStr = reinterpret_cast(a + iCur + 5); + } + uint32_t valOff = skip_one(a, n, iCur); + if (!valOff) return RC_ERROR; + if (kStr && static_cast(kLen) == nKey && + std::memcmp(kStr, zKey, static_cast(nKey)) == 0) { + iCur = valOff; + found = true; + } else { + iCur = skip_one(a, n, valOff); + if (!iCur) return RC_ERROR; + } + } + if (!found) return RC_NOTFOUND; + } + } +} + +/* ── decode_element — decode element at offset into Value ─────────── */ + +Value decode_element(const uint8_t* a, uint32_t n, uint32_t iStart, uint32_t iEnd) { + if (iStart >= n || iStart >= iEnd) return Value::nil(); + uint8_t b = a[iStart]; + + if (b == MP_NIL) return Value::nil(); + if (b == MP_FALSE) return Value::boolean(false); + if (b == MP_TRUE) return Value::boolean(true); + if (b <= 0x7f) return Value::integer(static_cast(b)); + if (b >= 0xe0) return Value::integer(static_cast(static_cast(b))); + + switch (b) { + case MP_UINT8: + if (iStart + 2 <= n) return Value::integer(static_cast(a[iStart + 1])); + break; + case MP_UINT16: + if (iStart + 3 <= n) return Value::integer(static_cast(read16(a + iStart + 1))); + break; + case MP_UINT32: + if (iStart + 5 <= n) return Value::integer(static_cast(read32(a + iStart + 1))); + break; + case MP_UINT64: + if (iStart + 9 <= n) { + uint64_t v = read64(a + iStart + 1); + return Value::unsigned_integer(v); + } + break; + case MP_INT8: + if (iStart + 2 <= n) return Value::integer(static_cast(static_cast(a[iStart + 1]))); + break; + case MP_INT16: + if (iStart + 3 <= n) return Value::integer(static_cast(static_cast(read16(a + iStart + 1)))); + break; + case MP_INT32: + if (iStart + 5 <= n) return Value::integer(static_cast(static_cast(read32(a + iStart + 1)))); + break; + case MP_INT64: + if (iStart + 9 <= n) return Value::integer(static_cast(read64(a + iStart + 1))); + break; + case MP_FLOAT32: + if (iStart + 5 <= n) { + uint32_t bits = read32(a + iStart + 1); + float f; + std::memcpy(&f, &bits, 4); + return Value::real32(f); + } + break; + case MP_FLOAT64: + if (iStart + 9 <= n) { + uint64_t bits = read64(a + iStart + 1); + double d; + std::memcpy(&d, &bits, 8); + return Value::real(d); + } + break; + default: break; + } + + /* str → String */ + uint32_t sLen = 0, sOff = 0; + if (b >= 0xa0 && b <= 0xbf) { + sLen = b & 0x1f; sOff = iStart + 1; + } else if (b == MP_STR8 && iStart + 2 <= n) { + sLen = a[iStart + 1]; sOff = iStart + 2; + } else if (b == MP_STR16 && iStart + 3 <= n) { + sLen = read16(a + iStart + 1); sOff = iStart + 3; + } else if (b == MP_STR32 && iStart + 5 <= n) { + sLen = read32(a + iStart + 1); sOff = iStart + 5; + } + if (sOff) { + if (sLen > n - sOff) sLen = n - sOff; + return Value::string(std::string_view(reinterpret_cast(a + sOff), sLen)); + } + + /* bin → Binary (payload only, no header) */ + { + uint32_t bLen = 0, bOff = 0; + if (b == MP_BIN8 && iStart + 2 <= n) { + bLen = a[iStart + 1]; bOff = iStart + 2; + } else if (b == MP_BIN16 && iStart + 3 <= n) { + bLen = read16(a + iStart + 1); bOff = iStart + 3; + } else if (b == MP_BIN32 && iStart + 5 <= n) { + bLen = read32(a + iStart + 1); bOff = iStart + 5; + } + if (bOff) { + if (bLen > n - bOff) bLen = n - bOff; + return Value::binary(a + bOff, bLen); + } + } + + /* timestamp ext → Timestamp value */ + { + int64_t tsec; uint32_t tnsec; + if (decode_timestamp(a, n, iStart, &tsec, &tnsec)) { + return Value::timestamp(tsec, tnsec); + } + } + + /* ext → Ext (type code + payload, no header) */ + { + int8_t tc = 0; + uint32_t elen = 0, eOff = 0; + switch (b) { + case MP_FIXEXT1: if (iStart+3<=n) { tc=static_cast(a[iStart+1]); elen=1; eOff=iStart+2; } break; + case MP_FIXEXT2: if (iStart+4<=n) { tc=static_cast(a[iStart+1]); elen=2; eOff=iStart+2; } break; + case MP_FIXEXT4: if (iStart+6<=n) { tc=static_cast(a[iStart+1]); elen=4; eOff=iStart+2; } break; + case MP_FIXEXT8: if (iStart+10<=n){ tc=static_cast(a[iStart+1]); elen=8; eOff=iStart+2; } break; + case MP_FIXEXT16: if (iStart+18<=n){ tc=static_cast(a[iStart+1]); elen=16; eOff=iStart+2; } break; + case MP_EXT8: + if (iStart+3<=n) { elen=a[iStart+1]; tc=static_cast(a[iStart+2]); eOff=iStart+3; } break; + case MP_EXT16: + if (iStart+4<=n) { elen=read16(a+iStart+1); tc=static_cast(a[iStart+3]); eOff=iStart+4; } break; + case MP_EXT32: + if (iStart+6<=n) { elen=read32(a+iStart+1); tc=static_cast(a[iStart+5]); eOff=iStart+6; } break; + default: break; + } + if (eOff) { + if (elen > n - eOff) elen = n - eOff; + return Value::ext(tc, a + eOff, elen); + } + } + + /* containers → raw binary blob (includes header) */ + return Value::binary(a + iStart, iEnd - iStart); +} + +} /* namespace detail */ + +using namespace detail; + +/* ══════════════════════════════════════════════════════════════════════ +** Public API: type_str, Value, and the read-only Blob accessors +** ══════════════════════════════════════════════════════════════════════ */ + +const char* type_str(Type t) noexcept { + switch (t) { + case Type::Nil: return "null"; + case Type::True: return "true"; + case Type::False: return "false"; + case Type::Integer: return "integer"; + case Type::Real: return "real"; + case Type::Float32: return "float32"; + case Type::String: return "text"; + case Type::Binary: return "binary"; + case Type::Array: return "array"; + case Type::Map: return "map"; + case Type::Ext: return "ext"; + case Type::Timestamp: return "timestamp"; + } + return "null"; +} + +/* ── Value ────────────────────────────────────────────────────────── */ + +Value::Value() noexcept : type_(Type::Nil), i64_(0) {} + +Type Value::type() const noexcept { return type_; } +bool Value::is_nil() const noexcept { return type_ == Type::Nil; } + +bool Value::as_bool() const noexcept { + return type_ == Type::True; +} + +int64_t Value::as_int64() const noexcept { + if (type_ == Type::Integer) return i64_; + if (type_ == Type::Real) return static_cast(f64_); + if (type_ == Type::Float32) return static_cast(f32_); + if (type_ == Type::Timestamp) return i64_; + if (type_ == Type::True) return 1; + return 0; +} + +uint64_t Value::as_uint64() const noexcept { + if (type_ == Type::Integer) return u64_; + return 0; +} + +double Value::as_double() const noexcept { + if (type_ == Type::Real) return f64_; + if (type_ == Type::Float32) return static_cast(f32_); + if (type_ == Type::Integer) return static_cast(i64_); + return 0.0; +} + +float Value::as_float() const noexcept { + if (type_ == Type::Float32) return f32_; + if (type_ == Type::Real) return static_cast(f64_); + return 0.0f; +} + +int8_t Value::ext_type() const noexcept { return ext_type_; } + +int64_t Value::timestamp_seconds() const noexcept { + if (type_ == Type::Timestamp) return i64_; + return 0; +} + +uint32_t Value::timestamp_nanoseconds() const noexcept { + if (type_ == Type::Timestamp) return ts_nsec_; + return 0; +} + +IntWidth Value::int_width() const noexcept { return int_width_; } + +std::string_view Value::as_string() const noexcept { + if (type_ == Type::String) return str_; + return {}; +} + +const uint8_t* Value::blob_data() const noexcept { + return !owned_blob_.empty() ? owned_blob_.data() : blob_ptr_; +} +size_t Value::blob_size() const noexcept { return blob_len_; } + +Value Value::nil() { + Value v; v.type_ = Type::Nil; return v; +} + +Value Value::boolean(bool b) { + Value v; v.type_ = b ? Type::True : Type::False; return v; +} + +Value Value::integer(int64_t x) { + Value v; v.type_ = Type::Integer; v.i64_ = x; return v; +} + +Value Value::unsigned_integer(uint64_t x) { + Value v; v.type_ = Type::Integer; v.u64_ = x; + /* Values that don't fit in int64 must be encoded as unsigned to round-trip. */ + if (x > static_cast(INT64_MAX)) { + v.int_width_ = IntWidth::Uint64; + } + return v; +} + +Value Value::real(double d) { + Value v; v.type_ = Type::Real; v.f64_ = d; return v; +} + +Value Value::real32(float f) { + Value v; v.type_ = Type::Float32; v.f32_ = f; return v; +} + +Value Value::string(std::string_view s) { + Value v; v.type_ = Type::String; v.str_ = std::string(s); return v; +} + +Value Value::binary(const uint8_t* data, size_t len) { + Value v; + v.type_ = Type::Binary; + if (len > 0 && data) { + v.owned_blob_.assign(data, data + len); + v.blob_ptr_ = v.owned_blob_.data(); + } else { + v.blob_ptr_ = nullptr; + } + v.blob_len_ = len; + return v; +} + +Value Value::ext(int8_t type_code, const uint8_t* data, size_t len) { + Value v; + v.type_ = Type::Ext; + v.ext_type_ = type_code; + v.owned_blob_.assign(data, data + len); + v.blob_ptr_ = v.owned_blob_.data(); + v.blob_len_ = len; + return v; +} + +Value Value::timestamp(int64_t seconds) { + Value v; v.type_ = Type::Timestamp; v.i64_ = seconds; v.ts_nsec_ = 0; return v; +} + +Value Value::timestamp(int64_t seconds, uint32_t nanoseconds) { + Value v; v.type_ = Type::Timestamp; v.i64_ = seconds; v.ts_nsec_ = nanoseconds; return v; +} + +Value Value::int8(int8_t x) { + Value v; v.type_ = Type::Integer; v.i64_ = x; v.int_width_ = IntWidth::Int8; return v; +} + +Value Value::int16(int16_t x) { + Value v; v.type_ = Type::Integer; v.i64_ = x; v.int_width_ = IntWidth::Int16; return v; +} + +Value Value::int32(int32_t x) { + Value v; v.type_ = Type::Integer; v.i64_ = x; v.int_width_ = IntWidth::Int32; return v; +} + +Value Value::int64(int64_t x) { + Value v; v.type_ = Type::Integer; v.i64_ = x; v.int_width_ = IntWidth::Int64; return v; +} + +Value Value::uint8(uint8_t x) { + Value v; v.type_ = Type::Integer; v.u64_ = x; v.int_width_ = IntWidth::Uint8; return v; +} + +Value Value::uint16(uint16_t x) { + Value v; v.type_ = Type::Integer; v.u64_ = x; v.int_width_ = IntWidth::Uint16; return v; +} + +Value Value::uint32(uint32_t x) { + Value v; v.type_ = Type::Integer; v.u64_ = x; v.int_width_ = IntWidth::Uint32; return v; +} + +Value Value::uint64(uint64_t x) { + Value v; v.type_ = Type::Integer; v.u64_ = x; v.int_width_ = IntWidth::Uint64; return v; +} + +/* ── Blob — construction & read-only accessors ────────────────────── */ + +Blob::Blob() = default; + +Blob::Blob(const uint8_t* data, size_t size) + : data_(data, data + size) {} + +Blob::Blob(std::vector data) + : data_(std::move(data)) {} + +const uint8_t* Blob::data() const noexcept { return data_.data(); } +size_t Blob::size() const noexcept { return data_.size(); } +bool Blob::empty() const noexcept { return data_.empty(); } + +bool Blob::valid() const { + if (data_.empty()) return false; + return is_valid(data_.data(), static_cast(data_.size())); +} + +size_t Blob::error_position() const { + if (data_.empty()) return 0; + return error_position_of(data_.data(), static_cast(data_.size())); +} + +Type Blob::type() const { + if (data_.empty()) return Type::Nil; + return get_type(data_.data(), static_cast(data_.size()), 0); +} + +Type Blob::type(const char* path) const { + uint32_t iStart, iEnd; + int rc = lookup(data_.data(), static_cast(data_.size()), 0, path, &iStart, &iEnd); + if (rc != RC_OK) return Type::Nil; + return get_type(data_.data(), static_cast(data_.size()), iStart); +} + +const char* Blob::type_str() const { + if (data_.empty()) return "null"; + return get_type_str_at(data_.data(), static_cast(data_.size()), 0); +} + +const char* Blob::type_str(const char* path) const { + uint32_t iStart, iEnd; + int rc = lookup(data_.data(), static_cast(data_.size()), 0, path, &iStart, &iEnd); + if (rc != RC_OK) return "null"; + return get_type_str_at(data_.data(), static_cast(data_.size()), iStart); +} + +Value Blob::extract(const char* path) const { + uint32_t iStart, iEnd; + int rc = lookup(data_.data(), static_cast(data_.size()), 0, path, &iStart, &iEnd); + if (rc != RC_OK) return Value::nil(); + return decode_element(data_.data(), static_cast(data_.size()), iStart, iEnd); +} + +int64_t Blob::array_length() const { + if (data_.empty()) return -1; + return get_container_count(data_.data(), static_cast(data_.size()), 0); +} + +int64_t Blob::array_length(const char* path) const { + uint32_t iStart, iEnd; + int rc = lookup(data_.data(), static_cast(data_.size()), 0, path, &iStart, &iEnd); + if (rc != RC_OK) return -1; + return get_container_count(data_.data(), static_cast(data_.size()), iStart); +} + +} /* namespace msgpack */ diff --git a/cpp/src/msgpack_blob_detail.hpp b/cpp/src/msgpack_blob_detail.hpp new file mode 100644 index 0000000..a2150bc --- /dev/null +++ b/cpp/src/msgpack_blob_detail.hpp @@ -0,0 +1,151 @@ +/* +** msgpack_blob_detail.hpp — internal shared declarations for the +** standalone C++ MsgPack Blob library. +** +** This header is PRIVATE to the implementation files under src/. It is not +** installed and is not part of the public API (include/msgpack_blob.hpp). +** It exposes the format constants, byte-order helpers, the growable output +** buffer, and the internal codec routines shared between the decode, encode, +** json, mutate, and iterate translation units. +*/ + +#ifndef MSGPACK_BLOB_DETAIL_HPP +#define MSGPACK_BLOB_DETAIL_HPP + +#include "msgpack_blob.hpp" + +#include +#include +#include + +namespace msgpack { +namespace detail { + +/* ── MessagePack format constants (same as msgpack.c) ─────────────── */ + +static constexpr uint8_t MP_NIL = 0xc0; +static constexpr uint8_t MP_FALSE = 0xc2; +static constexpr uint8_t MP_TRUE = 0xc3; +static constexpr uint8_t MP_BIN8 = 0xc4; +static constexpr uint8_t MP_BIN16 = 0xc5; +static constexpr uint8_t MP_BIN32 = 0xc6; +static constexpr uint8_t MP_EXT8 = 0xc7; +static constexpr uint8_t MP_EXT16 = 0xc8; +static constexpr uint8_t MP_EXT32 = 0xc9; +static constexpr uint8_t MP_FLOAT32 = 0xca; +static constexpr uint8_t MP_FLOAT64 = 0xcb; +static constexpr uint8_t MP_UINT8 = 0xcc; +static constexpr uint8_t MP_UINT16 = 0xcd; +static constexpr uint8_t MP_UINT32 = 0xce; +static constexpr uint8_t MP_UINT64 = 0xcf; +static constexpr uint8_t MP_INT8 = 0xd0; +static constexpr uint8_t MP_INT16 = 0xd1; +static constexpr uint8_t MP_INT32 = 0xd2; +static constexpr uint8_t MP_INT64 = 0xd3; +static constexpr uint8_t MP_FIXEXT1 = 0xd4; +static constexpr uint8_t MP_FIXEXT2 = 0xd5; +static constexpr uint8_t MP_FIXEXT4 = 0xd6; +static constexpr uint8_t MP_FIXEXT8 = 0xd7; +static constexpr uint8_t MP_FIXEXT16 = 0xd8; +static constexpr uint8_t MP_STR8 = 0xd9; +static constexpr uint8_t MP_STR16 = 0xda; +static constexpr uint8_t MP_STR32 = 0xdb; +static constexpr uint8_t MP_ARRAY16 = 0xdc; +static constexpr uint8_t MP_ARRAY32 = 0xdd; +static constexpr uint8_t MP_MAP16 = 0xde; +static constexpr uint8_t MP_MAP32 = 0xdf; + +static constexpr uint8_t MP_FIXMAP_MASK = 0x80; +static constexpr uint8_t MP_FIXARRAY_MASK = 0x90; +static constexpr uint8_t MP_FIXSTR_MASK = 0xa0; + +/* Ext type code reserved for the msgpack timestamp extension. */ +static constexpr uint8_t MP_TIMESTAMP_TYPE = 0xFF; + +/* Edit modes */ +static constexpr int EDIT_SET = 0; +static constexpr int EDIT_INSERT = 1; +static constexpr int EDIT_REPLACE = 2; +static constexpr int EDIT_REMOVE = 3; +static constexpr int EDIT_ARRAY_INS = 4; + +/* Result codes (internal, not exposed) */ +static constexpr int RC_OK = 0; +static constexpr int RC_ERROR = 1; +static constexpr int RC_NOTFOUND = 2; + +/* ── Big-endian byte-order helpers ────────────────────────────────── */ + +static inline uint16_t read16(const uint8_t* p) { + return static_cast((static_cast(p[0]) << 8) | p[1]); +} +static inline uint32_t read32(const uint8_t* p) { + return (static_cast(p[0]) << 24) | + (static_cast(p[1]) << 16) | + (static_cast(p[2]) << 8) | p[3]; +} +static inline uint64_t read64(const uint8_t* p) { + return (static_cast(read32(p)) << 32) | read32(p + 4); +} +static inline void write16(uint8_t* p, uint16_t v) { + p[0] = static_cast(v >> 8); + p[1] = static_cast(v); +} +static inline void write32(uint8_t* p, uint32_t v) { + p[0] = static_cast(v >> 24); + p[1] = static_cast(v >> 16); + p[2] = static_cast(v >> 8); + p[3] = static_cast(v); +} +static inline void write64(uint8_t* p, uint64_t v) { + write32(p, static_cast(v >> 32)); + write32(p + 4, static_cast(v)); +} + +/* ── Buf — growable output buffer ─────────────────────────────────── */ + +class Buf { +public: + std::vector data; + + void append(const uint8_t* p, size_t n) { + data.insert(data.end(), p, p + n); + } + void append1(uint8_t b) { + data.push_back(b); + } + uint8_t* reserve(size_t n) { + size_t old = data.size(); + data.resize(old + n); + return data.data() + old; + } + void clear() { data.clear(); } + size_t size() const { return data.size(); } + const uint8_t* ptr() const { return data.data(); } +}; + +/* ── Cross-module internal routines ─────────────────────────────────── +** Each routine is defined in exactly one translation unit but may be called +** from several. Single-TU helpers stay file-local in their .cpp. +*/ + +/* decode module */ +uint32_t skip_one(const uint8_t* a, uint32_t n, uint32_t i); +Type get_type(const uint8_t* a, uint32_t n, uint32_t i); +int64_t get_container_count(const uint8_t* a, uint32_t n, uint32_t i); +int path_step(const char* zPath, int* pi, + const char** pKey, int* nKey, int64_t* pIdx); +int lookup(const uint8_t* a, uint32_t n, uint32_t iRoot, + const char* zPath, uint32_t* piStart, uint32_t* piEnd); +Value decode_element(const uint8_t* a, uint32_t n, + uint32_t iStart, uint32_t iEnd); + +/* encode module */ +void encode_array_header(Buf& buf, uint32_t count); +void encode_map_header(Buf& buf, uint32_t count); +void encode_string(Buf& buf, const char* s, uint32_t len); + +} /* namespace detail */ +} /* namespace msgpack */ + +#endif /* MSGPACK_BLOB_DETAIL_HPP */ diff --git a/cpp/src/msgpack_blob_encode.cpp b/cpp/src/msgpack_blob_encode.cpp new file mode 100644 index 0000000..b72e816 --- /dev/null +++ b/cpp/src/msgpack_blob_encode.cpp @@ -0,0 +1,362 @@ +/* +** msgpack_blob_encode.cpp — encoding +** +** Part of the standalone C++ MsgPack Blob library (see msgpack_blob.hpp). +** Contains: the low-level header encoders shared with the json/mutate +** modules, and the streaming Builder used to construct blobs. +*/ + +#include "msgpack_blob_detail.hpp" + +#include +#include + +namespace msgpack { +namespace detail { + +/* ── encode helpers ───────────────────────────────────────────────── */ + +void encode_array_header(Buf& buf, uint32_t count) { + if (count <= 15) { + buf.append1(static_cast(MP_FIXARRAY_MASK | count)); + } else if (count <= 0xffff) { + uint8_t h[3]; h[0] = MP_ARRAY16; write16(h + 1, static_cast(count)); + buf.append(h, 3); + } else { + uint8_t h[5]; h[0] = MP_ARRAY32; write32(h + 1, count); + buf.append(h, 5); + } +} + +void encode_map_header(Buf& buf, uint32_t count) { + if (count <= 15) { + buf.append1(static_cast(MP_FIXMAP_MASK | count)); + } else if (count <= 0xffff) { + uint8_t h[3]; h[0] = MP_MAP16; write16(h + 1, static_cast(count)); + buf.append(h, 3); + } else { + uint8_t h[5]; h[0] = MP_MAP32; write32(h + 1, count); + buf.append(h, 5); + } +} + +void encode_string(Buf& buf, const char* s, uint32_t len) { + if (len <= 31) { + buf.append1(static_cast(MP_FIXSTR_MASK | len)); + } else if (len <= 0xff) { + uint8_t h[2] = {MP_STR8, static_cast(len)}; + buf.append(h, 2); + } else if (len <= 0xffff) { + uint8_t h[3]; h[0] = MP_STR16; write16(h + 1, static_cast(len)); + buf.append(h, 3); + } else { + uint8_t h[5]; h[0] = MP_STR32; write32(h + 1, len); + buf.append(h, 5); + } + buf.append(reinterpret_cast(s), len); +} + +} /* namespace detail */ + +using namespace detail; + +/* ══════════════════════════════════════════════════════════════════════ +** Public API: Builder +** ══════════════════════════════════════════════════════════════════════ */ + +Builder::Builder() = default; + +void Builder::append(const uint8_t* data, size_t n) { + buf_.insert(buf_.end(), data, data + n); +} +void Builder::append1(uint8_t b) { buf_.push_back(b); } +uint8_t* Builder::reserve(size_t n) { + size_t old = buf_.size(); + buf_.resize(old + n); + return buf_.data() + old; +} + +Builder& Builder::nil() { append1(MP_NIL); return *this; } + +Builder& Builder::boolean(bool v) { + append1(v ? MP_TRUE : MP_FALSE); + return *this; +} + +Builder& Builder::integer(int64_t x) { + if (x >= 0) { + if (x <= 0x7f) { + append1(static_cast(x)); + } else if (x <= 0xff) { + uint8_t b[2] = {MP_UINT8, static_cast(x)}; + append(b, 2); + } else if (x <= 0xffff) { + uint8_t b[3]; b[0] = MP_UINT16; write16(b + 1, static_cast(x)); + append(b, 3); + } else if (x <= static_cast(0xffffffff)) { + uint8_t b[5]; b[0] = MP_UINT32; write32(b + 1, static_cast(x)); + append(b, 5); + } else { + uint8_t b[9]; b[0] = MP_UINT64; write64(b + 1, static_cast(x)); + append(b, 9); + } + } else { + if (x >= -32) { + append1(static_cast(x)); + } else if (x >= -128) { + uint8_t b[2] = {MP_INT8, static_cast(x)}; + append(b, 2); + } else if (x >= -32768) { + uint8_t b[3]; b[0] = MP_INT16; write16(b + 1, static_cast(x)); + append(b, 3); + } else if (x >= static_cast(-2147483648LL)) { + uint8_t b[5]; b[0] = MP_INT32; write32(b + 1, static_cast(x)); + append(b, 5); + } else { + uint8_t b[9]; b[0] = MP_INT64; write64(b + 1, static_cast(x)); + append(b, 9); + } + } + return *this; +} + +Builder& Builder::unsigned_integer(uint64_t x) { + if (x <= 0x7f) { + append1(static_cast(x)); + } else if (x <= 0xff) { + uint8_t b[2] = {MP_UINT8, static_cast(x)}; + append(b, 2); + } else if (x <= 0xffff) { + uint8_t b[3]; b[0] = MP_UINT16; write16(b + 1, static_cast(x)); + append(b, 3); + } else if (x <= 0xffffffff) { + uint8_t b[5]; b[0] = MP_UINT32; write32(b + 1, static_cast(x)); + append(b, 5); + } else { + uint8_t b[9]; b[0] = MP_UINT64; write64(b + 1, x); + append(b, 9); + } + return *this; +} + +Builder& Builder::real(double d) { + uint8_t b[9]; uint64_t bits; + b[0] = MP_FLOAT64; + std::memcpy(&bits, &d, 8); + write64(b + 1, bits); + append(b, 9); + return *this; +} + +Builder& Builder::real32(float f) { + uint8_t b[5]; uint32_t bits; + b[0] = MP_FLOAT32; + std::memcpy(&bits, &f, 4); + write32(b + 1, bits); + append(b, 5); + return *this; +} + +Builder& Builder::string(std::string_view s) { + auto len = static_cast(s.size()); + if (len <= 31) { + append1(static_cast(MP_FIXSTR_MASK | len)); + } else if (len <= 0xff) { + uint8_t h[2] = {MP_STR8, static_cast(len)}; + append(h, 2); + } else if (len <= 0xffff) { + uint8_t h[3]; h[0] = MP_STR16; write16(h + 1, static_cast(len)); + append(h, 3); + } else { + uint8_t h[5]; h[0] = MP_STR32; write32(h + 1, len); + append(h, 5); + } + append(reinterpret_cast(s.data()), len); + return *this; +} + +Builder& Builder::binary(const uint8_t* data, size_t len) { + auto n = static_cast(len); + if (n <= 0xff) { + uint8_t h[2] = {MP_BIN8, static_cast(n)}; + append(h, 2); + } else if (n <= 0xffff) { + uint8_t h[3]; h[0] = MP_BIN16; write16(h + 1, static_cast(n)); + append(h, 3); + } else { + uint8_t h[5]; h[0] = MP_BIN32; write32(h + 1, n); + append(h, 5); + } + if (data) append(data, n); + return *this; +} + +Builder& Builder::ext(int8_t type_code, const uint8_t* data, size_t len) { + auto n = static_cast(len); + switch (n) { + case 1: append1(MP_FIXEXT1); break; + case 2: append1(MP_FIXEXT2); break; + case 4: append1(MP_FIXEXT4); break; + case 8: append1(MP_FIXEXT8); break; + case 16: append1(MP_FIXEXT16); break; + default: + if (n <= 0xff) { + uint8_t h[2] = {MP_EXT8, static_cast(n)}; + append(h, 2); + } else if (n <= 0xffff) { + uint8_t h[3]; h[0] = MP_EXT16; write16(h + 1, static_cast(n)); + append(h, 3); + } else { + uint8_t h[5]; h[0] = MP_EXT32; write32(h + 1, n); + append(h, 5); + } + break; + } + append1(static_cast(type_code)); + if (data) append(data, n); + return *this; +} + +Builder& Builder::int8(int8_t x) { + uint8_t b[2] = {MP_INT8, static_cast(x)}; + append(b, 2); return *this; +} + +Builder& Builder::int16(int16_t x) { + uint8_t b[3]; b[0] = MP_INT16; write16(b + 1, static_cast(x)); + append(b, 3); return *this; +} + +Builder& Builder::int32(int32_t x) { + uint8_t b[5]; b[0] = MP_INT32; write32(b + 1, static_cast(x)); + append(b, 5); return *this; +} + +Builder& Builder::int64(int64_t x) { + uint8_t b[9]; b[0] = MP_INT64; write64(b + 1, static_cast(x)); + append(b, 9); return *this; +} + +Builder& Builder::uint8(uint8_t x) { + uint8_t b[2] = {MP_UINT8, x}; + append(b, 2); return *this; +} + +Builder& Builder::uint16(uint16_t x) { + uint8_t b[3]; b[0] = MP_UINT16; write16(b + 1, x); + append(b, 3); return *this; +} + +Builder& Builder::uint32(uint32_t x) { + uint8_t b[5]; b[0] = MP_UINT32; write32(b + 1, x); + append(b, 5); return *this; +} + +Builder& Builder::uint64(uint64_t x) { + uint8_t b[9]; b[0] = MP_UINT64; write64(b + 1, x); + append(b, 9); return *this; +} + +Builder& Builder::array_header(uint32_t count) { + if (count <= 15) { + append1(static_cast(MP_FIXARRAY_MASK | count)); + } else if (count <= 0xffff) { + uint8_t h[3]; h[0] = MP_ARRAY16; write16(h + 1, static_cast(count)); + append(h, 3); + } else { + uint8_t h[5]; h[0] = MP_ARRAY32; write32(h + 1, count); + append(h, 5); + } + return *this; +} + +Builder& Builder::map_header(uint32_t count) { + if (count <= 15) { + append1(static_cast(MP_FIXMAP_MASK | count)); + } else if (count <= 0xffff) { + uint8_t h[3]; h[0] = MP_MAP16; write16(h + 1, static_cast(count)); + append(h, 3); + } else { + uint8_t h[5]; h[0] = MP_MAP32; write32(h + 1, count); + append(h, 5); + } + return *this; +} + +Builder& Builder::raw(const uint8_t* data, size_t len) { + append(data, len); + return *this; +} + +Builder& Builder::raw(const Blob& blob) { + append(blob.data(), blob.size()); + return *this; +} + +Builder& Builder::timestamp(int64_t sec) { + return timestamp(sec, 0); +} + +Builder& Builder::timestamp(int64_t sec, uint32_t nsec) { + if (nsec == 0 && sec >= 0 && sec <= static_cast(0xFFFFFFFFLL)) { + uint8_t b[6]; b[0] = MP_FIXEXT4; b[1] = 0xFF; + write32(b + 2, static_cast(sec)); + append(b, 6); + } else if (sec >= 0 && sec <= static_cast(0x3FFFFFFFFLL)) { + uint8_t b[10]; b[0] = MP_FIXEXT8; b[1] = 0xFF; + uint64_t v64 = (static_cast(nsec) << 34) | static_cast(sec); + write64(b + 2, v64); + append(b, 10); + } else { + uint8_t b[15]; b[0] = MP_EXT8; b[1] = 12; b[2] = 0xFF; + write32(b + 3, nsec); + write64(b + 7, static_cast(sec)); + append(b, 15); + } + return *this; +} + +const uint8_t* Builder::buf_data() const noexcept { return buf_.data(); } +size_t Builder::buf_size() const noexcept { return buf_.size(); } + +Builder& Builder::value(const Value& v) { + switch (v.type()) { + case Type::Nil: return nil(); + case Type::True: return boolean(true); + case Type::False: return boolean(false); + case Type::Integer: { + IntWidth w = v.int_width(); + switch (w) { + case IntWidth::Int8: return int8(static_cast(v.as_int64())); + case IntWidth::Int16: return int16(static_cast(v.as_int64())); + case IntWidth::Int32: return int32(static_cast(v.as_int64())); + case IntWidth::Int64: return int64(v.as_int64()); + case IntWidth::Uint8: return uint8(static_cast(v.as_uint64())); + case IntWidth::Uint16: return uint16(static_cast(v.as_uint64())); + case IntWidth::Uint32: return uint32(static_cast(v.as_uint64())); + case IntWidth::Uint64: return uint64(v.as_uint64()); + case IntWidth::Auto: break; + } + return integer(v.as_int64()); + } + case Type::Real: return real(v.as_double()); + case Type::Float32: return real32(v.as_float()); + case Type::String: return string(v.as_string()); + case Type::Binary: return binary(v.blob_data(), v.blob_size()); + case Type::Ext: return ext(v.ext_type(), v.blob_data(), v.blob_size()); + case Type::Timestamp: return timestamp(v.timestamp_seconds(), v.timestamp_nanoseconds()); + default: return nil(); + } +} + +Blob Builder::build() { + return Blob(std::move(buf_)); +} + +Blob Builder::quote(const Value& v) { + Builder b; + b.value(v); + return b.build(); +} + +} /* namespace msgpack */ diff --git a/cpp/src/msgpack_blob_iterate.cpp b/cpp/src/msgpack_blob_iterate.cpp new file mode 100644 index 0000000..e269253 --- /dev/null +++ b/cpp/src/msgpack_blob_iterate.cpp @@ -0,0 +1,202 @@ +/* +** msgpack_blob_iterate.cpp — container iteration +** +** Part of the standalone C++ MsgPack Blob library (see msgpack_blob.hpp). +** Contains: flat (each) and recursive (tree) traversal of container children +** and the public Iterator cursor. +*/ + +#include "msgpack_blob_detail.hpp" + +#include +#include + +namespace msgpack { +namespace detail { + +/* ── Iteration internals ──────────────────────────────────────────── */ + +static void each_iter( + const uint8_t* a, uint32_t n, uint32_t iCont, + const std::string& zBase, std::vector& rows +) { + if (iCont >= n) return; + uint8_t b = a[iCont]; + bool isArr = false, isMap = false; + uint32_t count = 0, dataOff = 0; + + if (b >= 0x90 && b <= 0x9f) { isArr = true; count = b & 0x0f; dataOff = iCont + 1; } + else if (b == MP_ARRAY16 && iCont + 3 <= n) { isArr = true; count = read16(a + iCont + 1); dataOff = iCont + 3; } + else if (b == MP_ARRAY32 && iCont + 5 <= n) { isArr = true; count = read32(a + iCont + 1); dataOff = iCont + 5; } + else if (b >= 0x80 && b <= 0x8f) { isMap = true; count = b & 0x0f; dataOff = iCont + 1; } + else if (b == MP_MAP16 && iCont + 3 <= n) { isMap = true; count = read16(a + iCont + 1); dataOff = iCont + 3; } + else if (b == MP_MAP32 && iCont + 5 <= n) { isMap = true; count = read32(a + iCont + 1); dataOff = iCont + 5; } + + if (!isArr && !isMap) return; + + /* Sanity: reject counts that exceed remaining data capacity */ + uint32_t remaining = (dataOff <= n) ? (n - dataOff) : 0; + uint32_t minBytesPerElem = isMap ? 2u : 1u; + if (count > remaining / minBytesPerElem + 1) return; + + uint32_t cur = dataOff; + + for (uint32_t j = 0; j < count; j++) { + if (cur >= n) break; + if (isArr) { + uint32_t cEnd = skip_one(a, n, cur); + if (!cEnd) break; + EachRow row; + row.index = static_cast(j); + row.fullkey = zBase + "[" + std::to_string(j) + "]"; + row.path = zBase; + row.id = cur; + row.type = get_type(a, n, cur); + row.value = decode_element(a, n, cur, cEnd); + rows.push_back(std::move(row)); + cur = cEnd; + } else { + uint8_t kb = a[cur]; + const char* zKey = nullptr; uint32_t nKey = 0; + if (kb >= 0xa0 && kb <= 0xbf) { nKey = kb & 0x1f; zKey = reinterpret_cast(a + cur + 1); } + else if (kb == MP_STR8 && cur + 2 <= n) { nKey = a[cur + 1]; zKey = reinterpret_cast(a + cur + 2); } + else if (kb == MP_STR16 && cur + 3 <= n) { nKey = read16(a + cur + 1); zKey = reinterpret_cast(a + cur + 3); } + else if (kb == MP_STR32 && cur + 5 <= n) { nKey = read32(a + cur + 1); zKey = reinterpret_cast(a + cur + 5); } + uint32_t vOff = skip_one(a, n, cur); + if (!vOff) break; + uint32_t pEnd = skip_one(a, n, vOff); + if (!pEnd) break; + + EachRow row; + row.key = zKey ? std::string(zKey, nKey) : "?"; + row.index = static_cast(j); + row.fullkey = zBase + "." + row.key; + row.path = zBase; + row.id = vOff; + row.type = get_type(a, n, vOff); + row.value = decode_element(a, n, vOff, pEnd); + rows.push_back(std::move(row)); + cur = pEnd; + } + } +} + +static void tree_walk( + const uint8_t* a, uint32_t n, uint32_t iOff, + const std::string& zFull, const std::string& zParPath, + int depth, std::vector& rows +) { + if (depth > kMaxDepth || iOff >= n) return; + uint32_t iEnd = skip_one(a, n, iOff); + if (!iEnd) return; + + /* Yield this element */ + { + EachRow row; + row.fullkey = zFull; + row.path = zParPath; + row.id = iOff; + row.type = get_type(a, n, iOff); + row.value = decode_element(a, n, iOff, iEnd); + rows.push_back(std::move(row)); + } + + uint8_t b = a[iOff]; + bool isArr = false, isMap = false; + uint32_t count = 0, dataOff = 0; + + if (b >= 0x90 && b <= 0x9f) { isArr = true; count = b & 0x0f; dataOff = iOff + 1; } + else if (b == MP_ARRAY16 && iOff + 3 <= n) { isArr = true; count = read16(a + iOff + 1); dataOff = iOff + 3; } + else if (b == MP_ARRAY32 && iOff + 5 <= n) { isArr = true; count = read32(a + iOff + 1); dataOff = iOff + 5; } + else if (b >= 0x80 && b <= 0x8f) { isMap = true; count = b & 0x0f; dataOff = iOff + 1; } + else if (b == MP_MAP16 && iOff + 3 <= n) { isMap = true; count = read16(a + iOff + 1); dataOff = iOff + 3; } + else if (b == MP_MAP32 && iOff + 5 <= n) { isMap = true; count = read32(a + iOff + 1); dataOff = iOff + 5; } + + if (!isArr && !isMap) return; + + /* Sanity: reject counts that exceed remaining data capacity */ + uint32_t tRemaining = (dataOff <= n) ? (n - dataOff) : 0; + uint32_t tMinBytes = isMap ? 2u : 1u; + if (count > tRemaining / tMinBytes + 1) return; + + uint32_t cur = dataOff; + + for (uint32_t j = 0; j < count; j++) { + if (cur >= n) break; + if (isArr) { + uint32_t cEnd = skip_one(a, n, cur); if (!cEnd) break; + std::string childFull = zFull + "[" + std::to_string(j) + "]"; + tree_walk(a, n, cur, childFull, zFull, depth + 1, rows); + cur = cEnd; + } else { + uint8_t kb = a[cur]; + const char* zKey = nullptr; uint32_t nKey = 0; + if (kb >= 0xa0 && kb <= 0xbf) { nKey = kb & 0x1f; zKey = reinterpret_cast(a + cur + 1); } + else if (kb == MP_STR8 && cur + 2 <= n) { nKey = a[cur + 1]; zKey = reinterpret_cast(a + cur + 2); } + else if (kb == MP_STR16 && cur + 3 <= n) { nKey = read16(a + cur + 1); zKey = reinterpret_cast(a + cur + 3); } + else if (kb == MP_STR32 && cur + 5 <= n) { nKey = read32(a + cur + 1); zKey = reinterpret_cast(a + cur + 5); } + uint32_t vOff = skip_one(a, n, cur); if (!vOff) break; + uint32_t pEnd = skip_one(a, n, vOff); if (!pEnd) break; + std::string keyStr = zKey ? std::string(zKey, nKey) : "?"; + std::string childFull = zFull + "." + keyStr; + tree_walk(a, n, vOff, childFull, zFull, depth + 1, rows); + cur = pEnd; + } + } +} + +} /* namespace detail */ + +using namespace detail; + +/* ══════════════════════════════════════════════════════════════════════ +** Public API: Iterator +** ══════════════════════════════════════════════════════════════════════ */ + +Iterator::Iterator(const Blob& blob, const char* path, bool recursive) + : blob_(blob), base_path_(path ? path : "$"), + recursive_(recursive), cursor_(-1), populated_(false) {} + +void Iterator::populate() { + if (populated_) return; + populated_ = true; + rows_.clear(); + + const uint8_t* a = blob_.data(); + auto n = static_cast(blob_.size()); + if (!a || n == 0) return; + + uint32_t iRoot = 0; + std::string zBase = base_path_; + + if (base_path_ != "$") { + uint32_t iStart, iEnd; + if (lookup(a, n, 0, base_path_.c_str(), &iStart, &iEnd) == RC_OK) { + iRoot = iStart; + } else { + return; + } + } + + if (recursive_) { + tree_walk(a, n, iRoot, zBase, zBase, 0, rows_); + } else { + each_iter(a, n, iRoot, zBase, rows_); + } +} + +bool Iterator::next() { + populate(); + cursor_++; + return cursor_ < static_cast(rows_.size()); +} + +const EachRow& Iterator::current() const { + return rows_[static_cast(cursor_)]; +} + +void Iterator::reset() { + cursor_ = -1; +} + +} /* namespace msgpack */ diff --git a/cpp/src/msgpack_blob_json.cpp b/cpp/src/msgpack_blob_json.cpp new file mode 100644 index 0000000..acbab74 --- /dev/null +++ b/cpp/src/msgpack_blob_json.cpp @@ -0,0 +1,426 @@ +/* +** msgpack_blob_json.cpp — JSON conversion +** +** Part of the standalone C++ MsgPack Blob library (see msgpack_blob.hpp). +** Contains: msgpack → JSON serialisation (compact and pretty) and a +** JSON → msgpack parser, plus the Blob JSON entry points. +*/ + +#include "msgpack_blob_detail.hpp" + +#include +#include +#include +#include +#include + +namespace msgpack { +namespace detail { + +/* ── JSON output ──────────────────────────────────────────────────── */ + +static void json_escape_str(Buf& out, const uint8_t* s, uint32_t len) { + out.append1('"'); + uint32_t start = 0; + for (uint32_t j = 0; j < len; j++) { + uint8_t c = s[j]; + if (c >= 0x20 && c != '"' && c != '\\') continue; + if (j > start) out.append(s + start, j - start); + if (c == '"') { uint8_t b[2] = {'\\', '"'}; out.append(b, 2); } + else if (c == '\\') { uint8_t b[2] = {'\\', '\\'}; out.append(b, 2); } + else if (c == '\n') { uint8_t b[2] = {'\\', 'n'}; out.append(b, 2); } + else if (c == '\r') { uint8_t b[2] = {'\\', 'r'}; out.append(b, 2); } + else if (c == '\t') { uint8_t b[2] = {'\\', 't'}; out.append(b, 2); } + else { + char esc[8]; std::snprintf(esc, 8, "\\u%04x", static_cast(c)); + out.append(reinterpret_cast(esc), 6); + } + start = j + 1; + } + if (len > start) out.append(s + start, len - start); + out.append1('"'); +} + +static void json_newline(Buf& out, int depth, int indentW) { + static const char spaces[] = + " "; + int nSpaces = depth * indentW; + out.append1('\n'); + while (nSpaces > 0) { + int chunk = nSpaces > static_cast(sizeof(spaces) - 1) + ? static_cast(sizeof(spaces) - 1) : nSpaces; + out.append(reinterpret_cast(spaces), static_cast(chunk)); + nSpaces -= chunk; + } +} + +static void to_json_at( + Buf& out, const uint8_t* a, uint32_t n, uint32_t i, + bool pretty, int depth, int indentW +) { + char s[64]; + if (i >= n || depth > kMaxDepth) { + out.append(reinterpret_cast("null"), 4); return; + } + uint8_t b = a[i]; + + if (b == MP_NIL) { out.append(reinterpret_cast("null"), 4); return; } + if (b == MP_FALSE) { out.append(reinterpret_cast("false"), 5); return; } + if (b == MP_TRUE) { out.append(reinterpret_cast("true"), 4); return; } + if (b <= 0x7f) { + int len = std::snprintf(s, sizeof(s), "%d", static_cast(b)); + out.append(reinterpret_cast(s), static_cast(len)); return; + } + if (b >= 0xe0) { + int len = std::snprintf(s, sizeof(s), "%d", static_cast(static_cast(b))); + out.append(reinterpret_cast(s), static_cast(len)); return; + } + + switch (b) { + case MP_UINT8: if (i+2>n) break; { int l=std::snprintf(s,sizeof(s),"%u",static_cast(a[i+1])); out.append(reinterpret_cast(s),static_cast(l)); return; } + case MP_UINT16: if (i+3>n) break; { int l=std::snprintf(s,sizeof(s),"%u",static_cast(read16(a+i+1))); out.append(reinterpret_cast(s),static_cast(l)); return; } + case MP_UINT32: if (i+5>n) break; { int l=std::snprintf(s,sizeof(s),"%u",static_cast(read32(a+i+1))); out.append(reinterpret_cast(s),static_cast(l)); return; } + case MP_UINT64: if (i+9>n) break; { int l=std::snprintf(s,sizeof(s),"%llu",static_cast(read64(a+i+1))); out.append(reinterpret_cast(s),static_cast(l)); return; } + case MP_INT8: if (i+2>n) break; { int l=std::snprintf(s,sizeof(s),"%d",static_cast(static_cast(a[i+1]))); out.append(reinterpret_cast(s),static_cast(l)); return; } + case MP_INT16: if (i+3>n) break; { int l=std::snprintf(s,sizeof(s),"%d",static_cast(static_cast(read16(a+i+1)))); out.append(reinterpret_cast(s),static_cast(l)); return; } + case MP_INT32: if (i+5>n) break; { int l=std::snprintf(s,sizeof(s),"%d",static_cast(static_cast(read32(a+i+1)))); out.append(reinterpret_cast(s),static_cast(l)); return; } + case MP_INT64: if (i+9>n) break; { int l=std::snprintf(s,sizeof(s),"%lld",static_cast(read64(a+i+1))); out.append(reinterpret_cast(s),static_cast(l)); return; } + case MP_FLOAT32: { + if (i+5>n) break; + uint32_t bits = read32(a+i+1); float f; std::memcpy(&f, &bits, 4); + if (!std::isfinite(static_cast(f))) { out.append(reinterpret_cast("null"),4); return; } + int l=std::snprintf(s,sizeof(s),"%.7g",static_cast(f)); + out.append(reinterpret_cast(s),static_cast(l)); return; + } + case MP_FLOAT64: { + if (i+9>n) break; + uint64_t bits = read64(a+i+1); double d; std::memcpy(&d, &bits, 8); + if (!std::isfinite(d)) { out.append(reinterpret_cast("null"),4); return; } + int l = std::snprintf(s,sizeof(s),"%.17g",d); + if (!std::strchr(s,'.') && !std::strchr(s,'e') && !std::strchr(s,'E')) + l = std::snprintf(s,sizeof(s),"%.1f",d); + out.append(reinterpret_cast(s),static_cast(l)); return; + } + default: break; + } + + /* str */ + { + uint32_t sLen = 0, sOff = 0; + if (b >= 0xa0 && b <= 0xbf) { sLen = b & 0x1f; sOff = i + 1; } + else if (b == MP_STR8 && i + 2 <= n) { sLen = a[i+1]; sOff = i + 2; } + else if (b == MP_STR16 && i + 3 <= n) { sLen = read16(a+i+1); sOff = i + 3; } + else if (b == MP_STR32 && i + 5 <= n) { sLen = read32(a+i+1); sOff = i + 5; } + if (sOff) { + if (sLen > n - sOff) sLen = n - sOff; + json_escape_str(out, a + sOff, sLen); + return; + } + } + + /* bin → hex string */ + { + uint32_t bLen = 0, bOff = 0; + if (b == MP_BIN8 && i + 2 <= n) { bLen = a[i+1]; bOff = i + 2; } + else if (b == MP_BIN16 && i + 3 <= n) { bLen = read16(a+i+1); bOff = i + 3; } + else if (b == MP_BIN32 && i + 5 <= n) { bLen = read32(a+i+1); bOff = i + 5; } + if (bOff) { + static const char hex[] = "0123456789abcdef"; + if (bLen > n - bOff) bLen = n - bOff; + out.append1('"'); + for (uint32_t j = 0; j < bLen; j++) { + uint8_t by = a[bOff + j]; + out.append1(static_cast(hex[by >> 4])); + out.append1(static_cast(hex[by & 0xf])); + } + out.append1('"'); + return; + } + } + + /* array */ + { + bool isArr = false; uint32_t count = 0, dataOff = 0; + if (b >= 0x90 && b <= 0x9f) { isArr = true; count = b & 0x0f; dataOff = i + 1; } + else if (b == MP_ARRAY16 && i + 3 <= n) { isArr = true; count = read16(a+i+1); dataOff = i + 3; } + else if (b == MP_ARRAY32 && i + 5 <= n) { isArr = true; count = read32(a+i+1); dataOff = i + 5; } + if (isArr) { + uint32_t cur = dataOff; + out.append1('['); + for (uint32_t j = 0; j < count; j++) { + if (cur >= n) break; + uint32_t next = skip_one(a, n, cur); + if (j > 0) out.append1(','); + if (pretty) json_newline(out, depth + 1, indentW); + to_json_at(out, a, n, cur, pretty, depth + 1, indentW); + cur = next ? next : n; + } + if (pretty && count > 0) json_newline(out, depth, indentW); + out.append1(']'); + return; + } + } + + /* map */ + { + bool isMap = false; uint32_t count = 0, dataOff = 0; + if (b >= 0x80 && b <= 0x8f) { isMap = true; count = b & 0x0f; dataOff = i + 1; } + else if (b == MP_MAP16 && i + 3 <= n) { isMap = true; count = read16(a+i+1); dataOff = i + 3; } + else if (b == MP_MAP32 && i + 5 <= n) { isMap = true; count = read32(a+i+1); dataOff = i + 5; } + if (isMap) { + uint32_t cur = dataOff; + out.append1('{'); + for (uint32_t j = 0; j < count; j++) { + if (cur >= n) break; + uint32_t valOff = skip_one(a, n, cur); + uint32_t pairEnd = valOff ? skip_one(a, n, valOff) : 0; + if (j > 0) out.append1(','); + if (pretty) json_newline(out, depth + 1, indentW); + to_json_at(out, a, n, cur, pretty, depth + 1, indentW); + out.append1(':'); + if (pretty) out.append1(' '); + to_json_at(out, a, n, valOff ? valOff : n, pretty, depth + 1, indentW); + cur = pairEnd ? pairEnd : n; + } + if (pretty && count > 0) json_newline(out, depth, indentW); + out.append1('}'); + return; + } + } + + /* ext / unknown → null */ + out.append(reinterpret_cast("null"), 4); +} + +/* ── JSON parser → msgpack ────────────────────────────────────────── */ + +struct JsonParser { + const char* z; + int n, i; +}; + +static void jp_skip_ws(JsonParser& p) { + while (p.i < p.n && (p.z[p.i] == ' ' || p.z[p.i] == '\t' || + p.z[p.i] == '\n' || p.z[p.i] == '\r')) p.i++; +} + +static int jp_hex4(const char* z) { + int v = 0; + for (int j = 0; j < 4; j++) { + char c = z[j]; int h; + if (c >= '0' && c <= '9') h = c - '0'; + else if (c >= 'a' && c <= 'f') h = c - 'a' + 10; + else if (c >= 'A' && c <= 'F') h = c - 'A' + 10; + else return -1; + v = (v << 4) | h; + } + return v; +} + +static int jp_codepoint_to_utf8(uint32_t cp, uint8_t* buf) { + if (cp < 0x80) { buf[0] = static_cast(cp); return 1; } + if (cp < 0x800) { buf[0] = static_cast(0xc0 | (cp >> 6)); buf[1] = static_cast(0x80 | (cp & 0x3f)); return 2; } + if (cp < 0x10000) { buf[0] = static_cast(0xe0 | (cp >> 12)); buf[1] = static_cast(0x80 | ((cp >> 6) & 0x3f)); buf[2] = static_cast(0x80 | (cp & 0x3f)); return 3; } + buf[0] = static_cast(0xf0 | (cp >> 18)); buf[1] = static_cast(0x80 | ((cp >> 12) & 0x3f)); + buf[2] = static_cast(0x80 | ((cp >> 6) & 0x3f)); buf[3] = static_cast(0x80 | (cp & 0x3f)); return 4; +} + +static int jp_parse_value(JsonParser& p, Buf& out); + +static int jp_parse_string(JsonParser& p, Buf& out) { + Buf sb; + p.i++; /* skip '"' */ + while (p.i < p.n) { + auto c = static_cast(p.z[p.i]); + if (c == '"') { p.i++; break; } + if (c == '\\') { + p.i++; + if (p.i >= p.n) return RC_ERROR; + char esc = p.z[p.i++]; + switch (esc) { + case '"': sb.append1('"'); break; + case '\\': sb.append1('\\'); break; + case '/': sb.append1('/'); break; + case 'n': sb.append1('\n'); break; + case 'r': sb.append1('\r'); break; + case 't': sb.append1('\t'); break; + case 'b': sb.append1('\b'); break; + case 'f': sb.append1('\f'); break; + case 'u': { + if (p.i + 4 > p.n) return RC_ERROR; + int cp = jp_hex4(p.z + p.i); p.i += 4; + if (cp < 0) return RC_ERROR; + if (cp >= 0xD800 && cp <= 0xDBFF && p.i + 6 <= p.n && + p.z[p.i] == '\\' && p.z[p.i + 1] == 'u') { + int lo = jp_hex4(p.z + p.i + 2); + if (lo >= 0xDC00 && lo <= 0xDFFF) { + p.i += 6; + cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); + } + } + uint8_t utf[4]; + int ulen = jp_codepoint_to_utf8(static_cast(cp), utf); + sb.append(utf, static_cast(ulen)); + break; + } + default: sb.append1(static_cast(esc)); break; + } + } else { + sb.append1(c); p.i++; + } + } + encode_string(out, reinterpret_cast(sb.ptr()), + static_cast(sb.size())); + return RC_OK; +} + +static int jp_parse_number(JsonParser& p, Buf& out) { + int start = p.i; + bool isFloat = false; + if (p.i < p.n && p.z[p.i] == '-') p.i++; + while (p.i < p.n && p.z[p.i] >= '0' && p.z[p.i] <= '9') p.i++; + if (p.i < p.n && p.z[p.i] == '.') { + isFloat = true; p.i++; + while (p.i < p.n && p.z[p.i] >= '0' && p.z[p.i] <= '9') p.i++; + } + if (p.i < p.n && (p.z[p.i] == 'e' || p.z[p.i] == 'E')) { + isFloat = true; p.i++; + if (p.i < p.n && (p.z[p.i] == '+' || p.z[p.i] == '-')) p.i++; + while (p.i < p.n && p.z[p.i] >= '0' && p.z[p.i] <= '9') p.i++; + } + int len = p.i - start; + if (len <= 0 || len >= 64) return RC_ERROR; + char buf[64]; + std::memcpy(buf, p.z + start, static_cast(len)); buf[len] = '\0'; + + if (isFloat) { + double d = std::strtod(buf, nullptr); + uint8_t b[9]; uint64_t bits; b[0] = MP_FLOAT64; + std::memcpy(&bits, &d, 8); write64(b + 1, bits); + out.append(b, 9); + } else { + int64_t v = static_cast(std::strtoll(buf, nullptr, 10)); + if (v >= 0) { + if (v <= 0x7f) out.append1(static_cast(v)); + else if (v <= 0xff) { uint8_t b[2] = {MP_UINT8, static_cast(v)}; out.append(b, 2); } + else if (v <= 0xffff) { uint8_t b[3]; b[0] = MP_UINT16; write16(b+1, static_cast(v)); out.append(b, 3); } + else if (v <= static_cast(0xffffffff)) { uint8_t b[5]; b[0] = MP_UINT32; write32(b+1, static_cast(v)); out.append(b, 5); } + else { uint8_t b[9]; b[0] = MP_UINT64; write64(b+1, static_cast(v)); out.append(b, 9); } + } else { + if (v >= -32) out.append1(static_cast(v)); + else if (v >= -128) { uint8_t b[2] = {MP_INT8, static_cast(v)}; out.append(b, 2); } + else if (v >= -32768) { uint8_t b[3]; b[0] = MP_INT16; write16(b+1, static_cast(v)); out.append(b, 3); } + else if (v >= static_cast(-2147483648LL)) { uint8_t b[5]; b[0] = MP_INT32; write32(b+1, static_cast(v)); out.append(b, 5); } + else { uint8_t b[9]; b[0] = MP_INT64; write64(b+1, static_cast(v)); out.append(b, 9); } + } + } + return RC_OK; +} + +static int jp_parse_array(JsonParser& p, Buf& out) { + Buf tmp; uint32_t count = 0; + p.i++; /* skip '[' */ + jp_skip_ws(p); + while (p.i < p.n && p.z[p.i] != ']') { + if (count > 0) { + jp_skip_ws(p); + if (p.i >= p.n || p.z[p.i] != ',') return RC_ERROR; + p.i++; + } + jp_skip_ws(p); + if (jp_parse_value(p, tmp) != RC_OK) return RC_ERROR; + count++; + jp_skip_ws(p); + } + if (p.i >= p.n) return RC_ERROR; + p.i++; /* skip ']' */ + encode_array_header(out, count); + out.append(tmp.ptr(), tmp.size()); + return RC_OK; +} + +static int jp_parse_object(JsonParser& p, Buf& out) { + Buf tmp; uint32_t count = 0; + p.i++; /* skip '{' */ + jp_skip_ws(p); + while (p.i < p.n && p.z[p.i] != '}') { + if (count > 0) { + jp_skip_ws(p); + if (p.i >= p.n || p.z[p.i] != ',') return RC_ERROR; + p.i++; + } + jp_skip_ws(p); + if (p.i >= p.n || p.z[p.i] != '"') return RC_ERROR; + if (jp_parse_string(p, tmp) != RC_OK) return RC_ERROR; + jp_skip_ws(p); + if (p.i >= p.n || p.z[p.i] != ':') return RC_ERROR; + p.i++; + jp_skip_ws(p); + if (jp_parse_value(p, tmp) != RC_OK) return RC_ERROR; + count++; + jp_skip_ws(p); + } + if (p.i >= p.n) return RC_ERROR; + p.i++; /* skip '}' */ + encode_map_header(out, count); + out.append(tmp.ptr(), tmp.size()); + return RC_OK; +} + +static int jp_parse_value(JsonParser& p, Buf& out) { + jp_skip_ws(p); + if (p.i >= p.n) return RC_ERROR; + char c = p.z[p.i]; + if (c == 'n' && p.i + 4 <= p.n && std::memcmp(p.z + p.i, "null", 4) == 0) { + p.i += 4; out.append1(MP_NIL); return RC_OK; + } + if (c == 't' && p.i + 4 <= p.n && std::memcmp(p.z + p.i, "true", 4) == 0) { + p.i += 4; out.append1(MP_TRUE); return RC_OK; + } + if (c == 'f' && p.i + 5 <= p.n && std::memcmp(p.z + p.i, "false", 5) == 0) { + p.i += 5; out.append1(MP_FALSE); return RC_OK; + } + if (c == '"') return jp_parse_string(p, out); + if (c == '[') return jp_parse_array(p, out); + if (c == '{') return jp_parse_object(p, out); + if (c == '-' || (c >= '0' && c <= '9')) return jp_parse_number(p, out); + return RC_ERROR; +} + +} /* namespace detail */ + +using namespace detail; + +/* ══════════════════════════════════════════════════════════════════════ +** Public API: Blob JSON conversion +** ══════════════════════════════════════════════════════════════════════ */ + +std::string Blob::to_json() const { + if (data_.empty()) return "null"; + Buf out; + to_json_at(out, data_.data(), static_cast(data_.size()), 0, false, 0, 0); + return std::string(reinterpret_cast(out.ptr()), out.size()); +} + +std::string Blob::to_json_pretty(int indent) const { + if (data_.empty()) return "null"; + if (indent < 0) indent = 0; + if (indent > 8) indent = 8; + Buf out; + to_json_at(out, data_.data(), static_cast(data_.size()), 0, true, 0, indent); + return std::string(reinterpret_cast(out.ptr()), out.size()); +} + +Blob Blob::from_json(const char* json) { + if (!json) return Blob(); + JsonParser p{json, static_cast(std::strlen(json)), 0}; + Buf out; + if (jp_parse_value(p, out) != RC_OK) return Blob(); + return Blob(std::move(out.data)); +} + +Blob Blob::from_json(const std::string& json) { + return from_json(json.c_str()); +} + +} /* namespace msgpack */ diff --git a/cpp/src/msgpack_blob_mutate.cpp b/cpp/src/msgpack_blob_mutate.cpp new file mode 100644 index 0000000..3a99c04 --- /dev/null +++ b/cpp/src/msgpack_blob_mutate.cpp @@ -0,0 +1,459 @@ +/* +** msgpack_blob_mutate.cpp — copy-on-write mutation +** +** Part of the standalone C++ MsgPack Blob library (see msgpack_blob.hpp). +** Contains: path-targeted edits (set/insert/replace/remove/array_insert), +** RFC 7386 merge patch, and the public Blob mutation methods. Every +** operation produces a brand-new blob; the source is never modified. +*/ + +#include "msgpack_blob_detail.hpp" + +#include +#include + +namespace msgpack { +namespace detail { + +/* ── Mutation internals ───────────────────────────────────────────── */ + +static int edit_step(Buf& out, const uint8_t* a, uint32_t n, uint32_t iCur, + const char* zPath, int pi, + const uint8_t* newBin, uint32_t nNew, + int mode, int* pSkip); + +static int edit_map( + Buf& out, const uint8_t* a, uint32_t n, uint32_t iCur, + const char* zKey, int nKey, + const char* zPath, int pi, + const uint8_t* newBin, uint32_t nNew, int mode +) { + if (iCur >= n) return RC_ERROR; + uint8_t b = a[iCur]; + uint32_t count, dataOff; + + if (b >= 0x80 && b <= 0x8f) { count = b & 0x0f; dataOff = iCur + 1; } + else if (b == MP_MAP16) { + if (iCur + 3 > n) return RC_ERROR; + count = read16(a + iCur + 1); dataOff = iCur + 3; + } else if (b == MP_MAP32) { + if (iCur + 5 > n) return RC_ERROR; + count = read32(a + iCur + 1); dataOff = iCur + 5; + } else { + if (mode == EDIT_REPLACE || mode == EDIT_REMOVE) { + uint32_t iEnd = skip_one(a, n, iCur); + if (iEnd) out.append(a + iCur, iEnd - iCur); + return RC_OK; + } + return RC_ERROR; + } + + uint32_t newCount = count; + Buf tmp; + uint32_t cur2 = dataOff; + bool foundKey = false; + int rc = RC_OK; + + for (uint32_t j = 0; j < count; j++) { + if (cur2 >= n) return RC_ERROR; + uint8_t kb = a[cur2]; + const char* kStr = nullptr; uint32_t kLen = 0; + if (kb >= 0xa0 && kb <= 0xbf) { + kLen = kb & 0x1f; kStr = reinterpret_cast(a + cur2 + 1); + } else if (kb == MP_STR8 && cur2 + 2 <= n) { + kLen = a[cur2 + 1]; kStr = reinterpret_cast(a + cur2 + 2); + } else if (kb == MP_STR16 && cur2 + 3 <= n) { + kLen = read16(a + cur2 + 1); kStr = reinterpret_cast(a + cur2 + 3); + } else if (kb == MP_STR32 && cur2 + 5 <= n) { + kLen = read32(a + cur2 + 1); kStr = reinterpret_cast(a + cur2 + 5); + } + + uint32_t valOff = skip_one(a, n, cur2); + if (!valOff) return RC_ERROR; + uint32_t pairEnd = skip_one(a, n, valOff); + if (!pairEnd) return RC_ERROR; + + bool isMatch = (kStr && static_cast(kLen) == nKey && + std::memcmp(kStr, zKey, static_cast(nKey)) == 0); + + if (isMatch) { + foundKey = true; + if (mode == EDIT_INSERT) { + tmp.append(a + cur2, pairEnd - cur2); + } else { + Buf vbuf; int skip = 0; + rc = edit_step(vbuf, a, n, valOff, zPath, pi, newBin, nNew, mode, &skip); + if (rc != RC_OK) return rc; + if (skip) { + newCount--; + } else { + tmp.append(a + cur2, valOff - cur2); + tmp.append(vbuf.ptr(), vbuf.size()); + } + } + } else { + tmp.append(a + cur2, pairEnd - cur2); + } + cur2 = pairEnd; + } + + if (!foundKey) { + if (mode == EDIT_SET || mode == EDIT_INSERT) { + int pi2 = pi; const char* zk2; int nk2; int64_t idx2; + if (path_step(zPath, &pi2, &zk2, &nk2, &idx2) != 0) { + uint32_t iEnd = skip_one(a, n, iCur); + if (iEnd) out.append(a + iCur, iEnd - iCur); + return RC_OK; + } + encode_string(tmp, zKey, static_cast(nKey)); + tmp.append(newBin, nNew); + newCount++; + } else { + uint32_t iEnd = skip_one(a, n, iCur); + if (iEnd) out.append(a + iCur, iEnd - iCur); + return RC_OK; + } + } + + encode_map_header(out, newCount); + out.append(tmp.ptr(), tmp.size()); + return RC_OK; +} + +static int edit_array( + Buf& out, const uint8_t* a, uint32_t n, uint32_t iCur, + int64_t stepIdx, + const char* zPath, int pi, + const uint8_t* newBin, uint32_t nNew, int mode +) { + if (iCur >= n) return RC_ERROR; + uint8_t b = a[iCur]; + uint32_t count, dataOff; + + if (b >= 0x90 && b <= 0x9f) { count = b & 0x0f; dataOff = iCur + 1; } + else if (b == MP_ARRAY16) { + if (iCur + 3 > n) return RC_ERROR; + count = read16(a + iCur + 1); dataOff = iCur + 3; + } else if (b == MP_ARRAY32) { + if (iCur + 5 > n) return RC_ERROR; + count = read32(a + iCur + 1); dataOff = iCur + 5; + } else { + if (mode == EDIT_REPLACE || mode == EDIT_REMOVE) { + uint32_t iEnd = skip_one(a, n, iCur); + if (iEnd) out.append(a + iCur, iEnd - iCur); + return RC_OK; + } + return RC_ERROR; + } + + uint32_t newCount = count; + Buf tmp; + uint32_t cur2 = dataOff; + bool foundIt = false; + int rc = RC_OK; + + for (uint32_t j = 0; j < count; j++) { + uint32_t eEnd = skip_one(a, n, cur2); + if (!eEnd) return RC_ERROR; + + if (static_cast(j) == stepIdx) { + foundIt = true; + if (mode == EDIT_ARRAY_INS) { + tmp.append(newBin, nNew); + tmp.append(a + cur2, eEnd - cur2); + newCount++; + } else if (mode == EDIT_INSERT) { + tmp.append(a + cur2, eEnd - cur2); + } else { + Buf ebuf; int skip = 0; + rc = edit_step(ebuf, a, n, cur2, zPath, pi, newBin, nNew, mode, &skip); + if (rc != RC_OK) return rc; + if (skip) { + newCount--; + } else { + tmp.append(ebuf.ptr(), ebuf.size()); + } + } + } else { + tmp.append(a + cur2, eEnd - cur2); + } + cur2 = eEnd; + } + + if (!foundIt) { + if (mode == EDIT_ARRAY_INS) { + tmp.append(newBin, nNew); + newCount++; + } else if ((mode == EDIT_SET || mode == EDIT_INSERT) && + static_cast(stepIdx) == count) { + tmp.append(newBin, nNew); + newCount++; + } else if (mode == EDIT_REPLACE || mode == EDIT_REMOVE) { + uint32_t iEnd = skip_one(a, n, iCur); + if (iEnd) out.append(a + iCur, iEnd - iCur); + return RC_OK; + } else { + return RC_NOTFOUND; + } + } + + encode_array_header(out, newCount); + out.append(tmp.ptr(), tmp.size()); + return RC_OK; +} + +static int edit_step( + Buf& out, const uint8_t* a, uint32_t n, uint32_t iCur, + const char* zPath, int pi, + const uint8_t* newBin, uint32_t nNew, + int mode, int* pSkip +) { + const char* zKey = nullptr; int nKey = 0; int64_t stepIdx = 0; + int step = path_step(zPath, &pi, &zKey, &nKey, &stepIdx); + if (pSkip) *pSkip = 0; + + if (step == 0) { + if (mode == EDIT_REMOVE) { + if (pSkip) *pSkip = 1; + return RC_OK; + } + if (mode == EDIT_ARRAY_INS) return RC_ERROR; + if (mode == EDIT_INSERT) { + uint32_t iEnd = skip_one(a, n, iCur); + if (iEnd) out.append(a + iCur, iEnd - iCur); + return RC_OK; + } + out.append(newBin, nNew); + return RC_OK; + } + if (step < 0) return RC_ERROR; + + if (step == 'k') { + return edit_map(out, a, n, iCur, zKey, nKey, zPath, pi, newBin, nNew, mode); + } else { + return edit_array(out, a, n, iCur, stepIdx, zPath, pi, newBin, nNew, mode); + } +} + +static int apply_edit( + Buf& out, + const uint8_t* a, uint32_t n, + const char* zPath, + const uint8_t* newBin, uint32_t nNew, + int mode +) { + if (!zPath || zPath[0] != '$') return RC_ERROR; + return edit_step(out, a, n, 0, zPath, 1, newBin, nNew, mode, nullptr); +} + +/* ── merge_patch (RFC 7386) ───────────────────────────────────────── */ + +static int merge_patch( + Buf& out, + const uint8_t* a, uint32_t n, uint32_t ia, + const uint8_t* p, uint32_t np, uint32_t ip, + int depth +) { + if (ip >= np) return RC_ERROR; + if (depth > kMaxDepth) return RC_ERROR; + uint8_t pb = p[ip]; + + if (pb == MP_NIL) { out.append1(MP_NIL); return RC_OK; } + + bool pIsMap = (pb >= 0x80 && pb <= 0x8f) || pb == MP_MAP16 || pb == MP_MAP32; + if (!pIsMap) { + uint32_t pEnd = skip_one(p, np, ip); + if (pEnd) out.append(p + ip, pEnd - ip); + return RC_OK; + } + + uint8_t ab = (ia < n) ? a[ia] : 0; + bool aIsMap = (ab >= 0x80 && ab <= 0x8f) || ab == MP_MAP16 || ab == MP_MAP32; + + uint32_t pCount, pDataOff; + if (pb >= 0x80 && pb <= 0x8f) { pCount = pb & 0x0f; pDataOff = ip + 1; } + else if (pb == MP_MAP16) { + if (ip + 3 > np) return RC_ERROR; + pCount = read16(p + ip + 1); pDataOff = ip + 3; + } else { + if (ip + 5 > np) return RC_ERROR; + pCount = read32(p + ip + 1); pDataOff = ip + 5; + } + + uint32_t aCount = 0, aDataOff = 0; + if (aIsMap) { + if (ab >= 0x80 && ab <= 0x8f) { aCount = ab & 0x0f; aDataOff = ia + 1; } + else if (ab == MP_MAP16) { + if (ia + 3 > n) { aIsMap = false; } + else { aCount = read16(a + ia + 1); aDataOff = ia + 3; } + } else { + if (ia + 5 > n) { aIsMap = false; } + else { aCount = read32(a + ia + 1); aDataOff = ia + 5; } + } + } + + /* Pre-scan patch keys */ + struct PatchEntry { + const char* zKey; uint32_t nKey; + uint32_t keyOff, valOff, pairEnd; + bool matched; + }; + /* Sanity: each map pair needs at least 2 bytes; reject implausible counts */ + if (pCount > (np - pDataOff) / 2 + 1) return RC_ERROR; + std::vector pIdx(pCount); + { + uint32_t pc2 = pDataOff; + for (uint32_t k = 0; k < pCount; k++) { + if (pc2 >= np) return RC_ERROR; + uint8_t pkb = p[pc2]; + pIdx[k] = {nullptr, 0, pc2, 0, 0, false}; + if (pkb >= 0xa0 && pkb <= 0xbf) { + pIdx[k].nKey = pkb & 0x1f; + pIdx[k].zKey = reinterpret_cast(p + pc2 + 1); + } else if (pkb == MP_STR8 && pc2 + 2 <= np) { + pIdx[k].nKey = p[pc2 + 1]; + pIdx[k].zKey = reinterpret_cast(p + pc2 + 2); + } else if (pkb == MP_STR16 && pc2 + 3 <= np) { + pIdx[k].nKey = read16(p + pc2 + 1); + pIdx[k].zKey = reinterpret_cast(p + pc2 + 3); + } else if (pkb == MP_STR32 && pc2 + 5 <= np) { + pIdx[k].nKey = read32(p + pc2 + 1); + pIdx[k].zKey = reinterpret_cast(p + pc2 + 5); + } + pIdx[k].valOff = skip_one(p, np, pc2); + if (!pIdx[k].valOff) return RC_ERROR; + pIdx[k].pairEnd = skip_one(p, np, pIdx[k].valOff); + if (!pIdx[k].pairEnd) return RC_ERROR; + pc2 = pIdx[k].pairEnd; + } + } + + Buf tmp; + uint32_t newCount = 0; + + /* Phase 1: iterate target pairs */ + if (aIsMap) { + uint32_t ac = aDataOff; + for (uint32_t j = 0; j < aCount; j++) { + if (ac >= n) return RC_ERROR; + uint8_t kb = a[ac]; + const char* kStr = nullptr; uint32_t kLen = 0; + if (kb >= 0xa0 && kb <= 0xbf) { + kLen = kb & 0x1f; kStr = reinterpret_cast(a + ac + 1); + } else if (kb == MP_STR8 && ac + 2 <= n) { + kLen = a[ac + 1]; kStr = reinterpret_cast(a + ac + 2); + } else if (kb == MP_STR16 && ac + 3 <= n) { + kLen = read16(a + ac + 1); kStr = reinterpret_cast(a + ac + 3); + } else if (kb == MP_STR32 && ac + 5 <= n) { + kLen = read32(a + ac + 1); kStr = reinterpret_cast(a + ac + 5); + } + + uint32_t aValOff = skip_one(a, n, ac); + if (!aValOff) return RC_ERROR; + uint32_t aPairEnd = skip_one(a, n, aValOff); + if (!aPairEnd) return RC_ERROR; + + bool foundInPatch = false, patchIsNil = false; + uint32_t pMatchVal = 0; + for (uint32_t k = 0; k < pCount; k++) { + if (pIdx[k].zKey && kStr && pIdx[k].nKey == kLen && + std::memcmp(pIdx[k].zKey, kStr, kLen) == 0) { + foundInPatch = true; + pMatchVal = pIdx[k].valOff; + patchIsNil = (pIdx[k].valOff < np && p[pIdx[k].valOff] == MP_NIL); + pIdx[k].matched = true; + break; + } + } + + if (foundInPatch && patchIsNil) { + /* Drop this pair */ + } else if (foundInPatch) { + Buf mb; + int mrc = merge_patch(mb, a, n, aValOff, p, np, pMatchVal, depth + 1); + if (mrc == RC_OK) { + tmp.append(a + ac, aValOff - ac); + tmp.append(mb.ptr(), mb.size()); + newCount++; + } + } else { + tmp.append(a + ac, aPairEnd - ac); + newCount++; + } + ac = aPairEnd; + } + } + + /* Phase 2: add unmatched patch pairs */ + for (uint32_t k = 0; k < pCount; k++) { + if (!pIdx[k].matched && pIdx[k].valOff < np && p[pIdx[k].valOff] != MP_NIL) { + tmp.append(p + pIdx[k].keyOff, pIdx[k].pairEnd - pIdx[k].keyOff); + newCount++; + } + } + + encode_map_header(out, newCount); + out.append(tmp.ptr(), tmp.size()); + return RC_OK; +} + +} /* namespace detail */ + +using namespace detail; + +/* ══════════════════════════════════════════════════════════════════════ +** Public API: Blob mutation (copy-on-write) +** ══════════════════════════════════════════════════════════════════════ */ + +static Blob apply_mutation(const Blob& blob, const char* path, const Value& val, int mode) { + Builder enc; + enc.value(val); + Buf out; + int rc = apply_edit(out, blob.data(), static_cast(blob.size()), + path, enc.buf_data(), static_cast(enc.buf_size()), mode); + if (rc != RC_OK) return blob; + return Blob(std::move(out.data)); +} + +Blob Blob::set(const char* path, const Value& val) const { + return apply_mutation(*this, path, val, EDIT_SET); +} + +Blob Blob::set(const char* path, const Blob& sub) const { + Buf out; + int rc = apply_edit(out, data_.data(), static_cast(data_.size()), + path, sub.data(), static_cast(sub.size()), EDIT_SET); + if (rc != RC_OK) return *this; + return Blob(std::move(out.data)); +} + +Blob Blob::insert(const char* path, const Value& val) const { + return apply_mutation(*this, path, val, EDIT_INSERT); +} + +Blob Blob::replace(const char* path, const Value& val) const { + return apply_mutation(*this, path, val, EDIT_REPLACE); +} + +Blob Blob::remove(const char* path) const { + Buf out; + int rc = apply_edit(out, data_.data(), static_cast(data_.size()), + path, nullptr, 0, EDIT_REMOVE); + if (rc != RC_OK) return *this; + return Blob(std::move(out.data)); +} + +Blob Blob::array_insert(const char* path, const Value& val) const { + return apply_mutation(*this, path, val, EDIT_ARRAY_INS); +} + +Blob Blob::patch(const Blob& mp) const { + Buf out; + int rc = merge_patch(out, + data_.data(), static_cast(data_.size()), 0, + mp.data(), static_cast(mp.size()), 0, 0); + if (rc != RC_OK) return *this; + return Blob(std::move(out.data)); +} + +} /* namespace msgpack */ diff --git a/tests/fuzz_blob_corpus_runner.cpp b/cpp/tests/fuzz_blob_corpus_runner.cpp similarity index 100% rename from tests/fuzz_blob_corpus_runner.cpp rename to cpp/tests/fuzz_blob_corpus_runner.cpp diff --git a/tests/fuzz_msgpack_blob.cpp b/cpp/tests/fuzz_msgpack_blob.cpp similarity index 100% rename from tests/fuzz_msgpack_blob.cpp rename to cpp/tests/fuzz_msgpack_blob.cpp diff --git a/cpp/tests/gen_blob_vectors.cpp b/cpp/tests/gen_blob_vectors.cpp new file mode 100644 index 0000000..3546bb6 --- /dev/null +++ b/cpp/tests/gen_blob_vectors.cpp @@ -0,0 +1,374 @@ +/* +** gen_blob_vectors.cpp — generate cross-language test vectors +** +** Uses the standalone C++ MsgPack Blob library (the reference implementation) +** to emit a language-neutral JSON file of test vectors. The Python and JS/TS +** ports replay these vectors to prove they produce byte-identical output. +** +** Build: linked against msgpack_blob_static (see CMakeLists target +** `blob_vectors_gen`). +** Run: ./blob_vectors_gen > tests/vectors/blob_vectors.json +** +** A "ValueSpec" is a small JSON object the ports interpret to build a Value: +** {"k":"nil"} {"k":"bool","v":true} +** {"k":"int","v":"42"} {"k":"uint","v":"...."} (compact) +** {"k":"int8","v":"-5"} ... {"k":"uint64","v":"...."} (fixed width) +** {"k":"real","v":1.5} {"k":"real32","v":1.5} +** {"k":"str","v":"hi"} {"k":"binary","hex":"deadbeef"} +** {"k":"ext","type":42,"hex":"0102"} {"k":"timestamp","sec":"..","nsec":0} +** {"k":"blob","json":"[1,2,3]"} (sub-blob via from_json) +** Integers are decimal strings so 64-bit values survive JSON without loss. +*/ + +#include "msgpack_blob.hpp" + +#include +#include +#include +#include +#include + +using namespace msgpack; + +/* ── tiny JSON writer ─────────────────────────────────────────────── */ + +static std::string jesc(const std::string& s) { + std::string o; + o.reserve(s.size() + 2); + for (unsigned char c : s) { + switch (c) { + case '"': o += "\\\""; break; + case '\\': o += "\\\\"; break; + case '\n': o += "\\n"; break; + case '\r': o += "\\r"; break; + case '\t': o += "\\t"; break; + case '\b': o += "\\b"; break; + case '\f': o += "\\f"; break; + default: + if (c < 0x20) { char b[8]; std::snprintf(b, 8, "\\u%04x", c); o += b; } + else o += static_cast(c); + } + } + return o; +} +static std::string q(const std::string& s) { return "\"" + jesc(s) + "\""; } + +static std::string to_hex(const Blob& b) { + static const char* hx = "0123456789abcdef"; + std::string o; + const uint8_t* d = b.data(); + for (size_t i = 0; i < b.size(); i++) { o += hx[d[i] >> 4]; o += hx[d[i] & 0xf]; } + return o; +} +static std::vector from_hex(const std::string& h) { + std::vector o; + auto nib = [](char c) -> int { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return 0; + }; + for (size_t i = 0; i + 1 < h.size(); i += 2) + o.push_back(static_cast((nib(h[i]) << 4) | nib(h[i + 1]))); + return o; +} + +/* ── object accumulation ──────────────────────────────────────────── */ + +struct Obj { + std::vector kv; + Obj& s(const std::string& k, const std::string& v) { kv.push_back(q(k) + ":" + q(v)); return *this; } + Obj& raw(const std::string& k, const std::string& v) { kv.push_back(q(k) + ":" + v); return *this; } + Obj& i(const std::string& k, long long v) { kv.push_back(q(k) + ":" + std::to_string(v)); return *this; } + Obj& b(const std::string& k, bool v) { kv.push_back(q(k) + std::string(":") + (v ? "true" : "false")); return *this; } + std::string str() const { + std::string o = "{"; + for (size_t j = 0; j < kv.size(); j++) { if (j) o += ","; o += kv[j]; } + return o + "}"; + } +}; + +struct Section { + std::string name; + std::vector items; + explicit Section(std::string n) : name(std::move(n)) {} + void add(const std::string& o) { items.push_back(o); } + std::string str() const { + std::string o = q(name) + ":["; + for (size_t j = 0; j < items.size(); j++) { if (j) o += ","; o += "\n " + items[j]; } + o += "]"; + return o; + } +}; + +/* ── ValueSpec helpers: build a Value and its spec JSON together ───── */ + +struct Spec { Value v; std::string json; }; + +static Spec sp_nil() { return {Value::nil(), "{\"k\":\"nil\"}"}; } +static Spec sp_bool(bool x) { return {Value::boolean(x), std::string("{\"k\":\"bool\",\"v\":") + (x?"true":"false") + "}"}; } +static Spec sp_int(int64_t x) { return {Value::integer(x), "{\"k\":\"int\",\"v\":\"" + std::to_string(x) + "\"}"}; } +static Spec sp_uint(uint64_t x) { return {Value::unsigned_integer(x), "{\"k\":\"uint\",\"v\":\"" + std::to_string(x) + "\"}"}; } +static Spec sp_i8(int8_t x) { return {Value::int8(x), "{\"k\":\"int8\",\"v\":\"" + std::to_string((int)x) + "\"}"}; } +static Spec sp_i16(int16_t x) { return {Value::int16(x), "{\"k\":\"int16\",\"v\":\"" + std::to_string((int)x) + "\"}"}; } +static Spec sp_i32(int32_t x) { return {Value::int32(x), "{\"k\":\"int32\",\"v\":\"" + std::to_string((long)x) + "\"}"}; } +static Spec sp_i64(int64_t x) { return {Value::int64(x), "{\"k\":\"int64\",\"v\":\"" + std::to_string(x) + "\"}"}; } +static Spec sp_u8(uint8_t x) { return {Value::uint8(x), "{\"k\":\"uint8\",\"v\":\"" + std::to_string((unsigned)x) + "\"}"}; } +static Spec sp_u16(uint16_t x) { return {Value::uint16(x), "{\"k\":\"uint16\",\"v\":\"" + std::to_string((unsigned)x) + "\"}"}; } +static Spec sp_u32(uint32_t x) { return {Value::uint32(x), "{\"k\":\"uint32\",\"v\":\"" + std::to_string((unsigned long)x) + "\"}"}; } +static Spec sp_u64(uint64_t x) { return {Value::uint64(x), "{\"k\":\"uint64\",\"v\":\"" + std::to_string(x) + "\"}"}; } +static Spec sp_real(double x, const std::string& lit) { return {Value::real(x), "{\"k\":\"real\",\"v\":" + lit + "}"}; } +static Spec sp_real32(float x, const std::string& lit){ return {Value::real32(x), "{\"k\":\"real32\",\"v\":" + lit + "}"}; } +static Spec sp_str(const std::string& s) { return {Value::string(s), "{\"k\":\"str\",\"v\":" + q(s) + "}"}; } +static Spec sp_bin(const std::string& hex) { + auto d = from_hex(hex); + return {Value::binary(d.data(), d.size()), "{\"k\":\"binary\",\"hex\":" + q(hex) + "}"}; +} +static Spec sp_ext(int8_t t, const std::string& hex) { + auto d = from_hex(hex); + return {Value::ext(t, d.data(), d.size()), + "{\"k\":\"ext\",\"type\":" + std::to_string((int)t) + ",\"hex\":" + q(hex) + "}"}; +} +static Spec sp_ts(int64_t sec, uint32_t nsec) { + return {Value::timestamp(sec, nsec), + "{\"k\":\"timestamp\",\"sec\":\"" + std::to_string(sec) + "\",\"nsec\":" + std::to_string((long long)nsec) + "}"}; +} + +int main() { + Section from_json("from_json"), to_json("to_json"), to_json_pretty("to_json_pretty"); + Section typed("typed"), mutate("mutate"), extract("extract"), + array_length("array_length"), iterate("iterate"); + + /* ── from_json: JSON input → expected msgpack hex ─────────────── */ + const char* json_inputs[] = { + "null", "true", "false", + "0", "1", "127", "128", "255", "256", "65535", "65536", + "4294967295", "4294967296", "9007199254740991", + "-1", "-32", "-33", "-128", "-129", "-32768", "-32769", + "-2147483648", "-2147483649", + "1.5", "0.1", "-0.0", "3.0", "1e10", "1.25e-3", "95.5", + "1e-7", "1e-4", "1e-5", "1e20", "1e21", "1e22", "9.999999e-8", + "0.30000000000000004", "123456789012345680000", + "\"\"", "\"hello\"", "\"a string longer than thirty-one chars!!\"", + "\"unicode: \\u00e9\\u4e2d\\ud83d\\ude00\"", + "[]", "{}", "[1,2,3]", "{\"a\":1,\"b\":2}", + "{\"name\":\"Alice\",\"age\":30,\"scores\":[95,87,91]}", + "[[1,2],[3,4],{\"x\":[true,null,false]}]", + "{\"nested\":{\"deep\":{\"value\":42}}}", + }; + for (auto* j : json_inputs) { + Blob b = Blob::from_json(j); + from_json.add(Obj().s("json", j).s("hex", to_hex(b)).str()); + /* round-trip → canonical JSON */ + to_json.add(Obj().s("hex", to_hex(b)).s("json", b.to_json()).str()); + } + + /* ── to_json: special hex inputs (bin, ext, float32, mixed) ───── */ + { + const char* hexes[] = { + "ca3f800000", /* float32 1.0 */ + "ca40490fdb", /* float32 pi-ish */ + "cb3ff0000000000000", /* float64 1.0 → 1.0 */ + "cb3fb999999999999a", /* float64 0.1 */ + "c403abcdef", /* bin8 → hex string */ + "d6ff0102", /* fixext... actually timestamp? d6 ff = fixext4 type -1 */ + "d40102", /* fixext1 type 1 → null */ + "92c2c3", /* [false,true] */ + "81a16382", /* truncated-ish; map {"c": fixmap2...} */ + }; + for (auto* h : hexes) { + auto d = from_hex(h); + Blob b(d.data(), d.size()); + to_json.add(Obj().s("hex", h).s("json", b.to_json()).str()); + } + } + + /* ── to_json_pretty ───────────────────────────────────────────── */ + { + struct PJ { const char* json; int indent; }; + PJ cases[] = { + {"{\"a\":1,\"b\":[2,3]}", 2}, + {"[1,[2,[3]]]", 4}, + {"{}", 2}, {"[]", 2}, + {"{\"x\":{\"y\":1}}", 0}, + }; + for (auto& c : cases) { + Blob b = Blob::from_json(c.json); + to_json_pretty.add(Obj().s("hex", to_hex(b)).i("indent", c.indent) + .s("json", b.to_json_pretty(c.indent)).str()); + } + } + + /* ── typed: Value → Builder::quote → hex ──────────────────────── */ + std::vector specs; + specs.push_back(sp_nil()); + specs.push_back(sp_bool(true)); specs.push_back(sp_bool(false)); + specs.push_back(sp_int(0)); specs.push_back(sp_int(127)); specs.push_back(sp_int(128)); + specs.push_back(sp_int(255)); specs.push_back(sp_int(256)); specs.push_back(sp_int(65536)); + specs.push_back(sp_int(4294967296LL)); + specs.push_back(sp_int(-1)); specs.push_back(sp_int(-32)); specs.push_back(sp_int(-33)); + specs.push_back(sp_int(-128)); specs.push_back(sp_int(-129)); specs.push_back(sp_int(-32769)); + specs.push_back(sp_uint(0)); specs.push_back(sp_uint(255)); + specs.push_back(sp_uint(18446744073709551615ULL)); + specs.push_back(sp_i8(-5)); specs.push_back(sp_i16(500)); specs.push_back(sp_i32(-70000)); + specs.push_back(sp_i64(-5)); specs.push_back(sp_u8(200)); specs.push_back(sp_u16(60000)); + specs.push_back(sp_u32(4000000000u)); specs.push_back(sp_u64(42)); + specs.push_back(sp_real(1.5, "1.5")); specs.push_back(sp_real(0.1, "0.1")); + specs.push_back(sp_real(3.0, "3.0")); + specs.push_back(sp_real32(1.5f, "1.5")); specs.push_back(sp_real32(0.5f, "0.5")); + specs.push_back(sp_str("")); specs.push_back(sp_str("hello")); + specs.push_back(sp_str(std::string(40, 'x'))); + specs.push_back(sp_bin("deadbeef")); specs.push_back(sp_bin("")); + specs.push_back(sp_ext(42, "0102")); specs.push_back(sp_ext(1, "00")); + specs.push_back(sp_ext(7, "0102030405")); /* len5 → ext8 */ + specs.push_back(sp_ts(0, 0)); specs.push_back(sp_ts(1700000000, 0)); + specs.push_back(sp_ts(1700000000, 500000000)); + specs.push_back(sp_ts(17000000000LL, 0)); /* needs fixext8 */ + specs.push_back(sp_ts(-1, 0)); /* needs ext8 */ + for (auto& s : specs) { + Blob b = Builder::quote(s.v); + typed.add(Obj().raw("spec", s.json).s("hex", to_hex(b)).str()); + } + + /* ── mutate: base + op + path (+ spec/patch) → hex ────────────── */ + auto add_mut = [&](const std::string& base, const std::string& op, + const std::string& path, const Spec& s) { + Blob b = Blob::from_json(base.c_str()); + Blob r; + if (op == "set") r = b.set(path.c_str(), s.v); + else if (op == "insert") r = b.insert(path.c_str(), s.v); + else if (op == "replace") r = b.replace(path.c_str(), s.v); + else if (op == "array_insert") r = b.array_insert(path.c_str(), s.v); + mutate.add(Obj().s("base", base).s("op", op).s("path", path) + .raw("spec", s.json).s("hex", to_hex(r)).str()); + }; + add_mut("{\"a\":1}", "set", "$.b", sp_int(2)); + add_mut("{\"a\":1}", "set", "$.a", sp_int(99)); + add_mut("{\"a\":1}", "set", "$.a", sp_i16(1000)); + add_mut("{\"x\":0}", "set", "$.created", sp_ts(1700000000, 0)); + add_mut("{\"a\":1}", "insert", "$.b", sp_str("new")); + add_mut("{\"a\":1}", "insert", "$.a", sp_int(5)); /* exists → no-op */ + add_mut("{\"a\":1,\"b\":2}", "replace", "$.a", sp_real(2.5, "2.5")); + add_mut("{\"a\":1}", "replace", "$.zzz", sp_int(9)); /* missing → no-op */ + add_mut("[1,2,3]", "set", "$[1]", sp_int(20)); + add_mut("[1,2,3]", "set", "$[3]", sp_int(4)); /* append */ + add_mut("[1,2,3]", "array_insert", "$[1]", sp_int(99)); + add_mut("[1,2,3]", "array_insert", "$[0]", sp_str("head")); + /* remove */ + { + struct RM { const char* base; const char* path; }; + RM rms[] = { + {"{\"a\":1,\"b\":2}", "$.a"}, + {"{\"a\":1,\"b\":2}", "$.b"}, + {"[1,2,3]", "$[1]"}, + {"{\"a\":{\"b\":1,\"c\":2}}", "$.a.b"}, + }; + for (auto& rm : rms) { + Blob b = Blob::from_json(rm.base); + Blob r = b.remove(rm.path); + mutate.add(Obj().s("base", rm.base).s("op", "remove").s("path", rm.path) + .s("hex", to_hex(r)).str()); + } + } + /* set_blob (sub-blob) */ + { + Blob base = Blob::from_json("{\"a\":1}"); + Blob sub = Blob::from_json("[1,2,3]"); + Blob r = base.set("$.b", sub); + mutate.add(Obj().s("base", "{\"a\":1}").s("op", "set_blob").s("path", "$.b") + .raw("spec", "{\"k\":\"blob\",\"json\":\"[1,2,3]\"}").s("hex", to_hex(r)).str()); + } + /* patch (RFC 7386) */ + { + struct PT { const char* base; const char* patch; }; + PT pts[] = { + {"{\"a\":1,\"b\":2}", "{\"a\":9}"}, + {"{\"a\":1,\"b\":2}", "{\"b\":null}"}, + {"{\"a\":1}", "{\"c\":3}"}, + {"{\"a\":{\"x\":1,\"y\":2}}", "{\"a\":{\"y\":null,\"z\":3}}"}, + {"{\"a\":1}", "{\"a\":{\"nested\":true}}"}, + }; + for (auto& pt : pts) { + Blob base = Blob::from_json(pt.base); + Blob patch = Blob::from_json(pt.patch); + Blob r = base.patch(patch); + mutate.add(Obj().s("base", pt.base).s("op", "patch") + .raw("patch", "\"" + jesc(pt.patch) + "\"") + .s("hex", to_hex(r)).str()); + } + } + + /* ── extract: base + path → type + value-as-json ──────────────── */ + auto add_ext = [&](const std::string& base, const std::string& path) { + Blob b = Blob::from_json(base.c_str()); + const char* ty = b.type_str(path.c_str()); + /* represent the extracted scalar through a one-element round trip */ + Value v = b.extract(path.c_str()); + std::string vj = Builder::quote(v).empty() ? "null" : Builder::quote(v).to_json(); + extract.add(Obj().s("base", base).s("path", path).s("type", ty).s("vjson", vj).str()); + }; + add_ext("{\"name\":\"Alice\",\"age\":30}", "$.name"); + add_ext("{\"name\":\"Alice\",\"age\":30}", "$.age"); + add_ext("{\"a\":[10,20,30]}", "$.a[1]"); + add_ext("{\"a\":[10,20,30]}", "$.a"); + add_ext("{\"a\":1}", "$.missing"); + add_ext("{\"f\":1.5}", "$.f"); + add_ext("{\"b\":true,\"n\":null}", "$.b"); + add_ext("{\"b\":true,\"n\":null}", "$.n"); + add_ext("[{\"x\":1}]", "$[0].x"); + + /* ── array_length ─────────────────────────────────────────────── */ + auto add_len = [&](const std::string& base, const std::string& path) { + Blob b = Blob::from_json(base.c_str()); + int64_t n = (path == "$") ? b.array_length() : b.array_length(path.c_str()); + array_length.add(Obj().s("base", base).s("path", path).i("len", n).str()); + }; + add_len("[1,2,3]", "$"); + add_len("{\"a\":[1,2,3,4]}", "$.a"); + add_len("{\"a\":1}", "$"); /* not a container → -1 */ + add_len("{\"m\":{\"x\":1,\"y\":2}}", "$.m"); /* map count */ + add_len("[]", "$"); + + /* ── iterate: each + tree → rows ──────────────────────────────── */ + auto add_iter = [&](const std::string& base, const std::string& path, bool recursive) { + Blob b = Blob::from_json(base.c_str()); + Iterator it(b, path.c_str(), recursive); + std::string rows = "["; + bool first = true; + while (it.next()) { + const EachRow& r = it.current(); + if (!first) rows += ","; + first = false; + /* key/index are only meaningful for flat (each) iteration; tree + ** rows identify themselves by fullkey/path/id instead. */ + Obj o; + if (!recursive) o.s("key", r.key).i("index", r.index); + o.s("fullkey", r.fullkey).s("path", r.path) + .i("id", (long long)r.id).s("type", type_str(r.type)); + rows += o.str(); + } + rows += "]"; + iterate.add(Obj().s("base", base).s("path", path).b("recursive", recursive) + .raw("rows", rows).str()); + }; + add_iter("{\"a\":1,\"b\":2,\"c\":3}", "$", false); + add_iter("[10,20,30]", "$", false); + add_iter("{\"x\":{\"y\":[1,2,3]}}", "$", true); + add_iter("{\"users\":[{\"name\":\"A\"},{\"name\":\"B\"}]}", "$", true); + add_iter("{\"users\":[{\"name\":\"A\"},{\"name\":\"B\"}]}", "$.users", false); + add_iter("[1,[2,3],4]", "$", true); + + /* ── emit document ────────────────────────────────────────────── */ + std::ostringstream out; + out << "{\n " + << from_json.str() << ",\n " + << to_json.str() << ",\n " + << to_json_pretty.str() << ",\n " + << typed.str() << ",\n " + << mutate.str() << ",\n " + << extract.str() << ",\n " + << array_length.str() << ",\n " + << iterate.str() << "\n}\n"; + std::fputs(out.str().c_str(), stdout); + return 0; +} diff --git a/tests/test_msgpack_blob.cpp b/cpp/tests/test_msgpack_blob.cpp similarity index 100% rename from tests/test_msgpack_blob.cpp rename to cpp/tests/test_msgpack_blob.cpp diff --git a/go/README.md b/go/README.md new file mode 100644 index 0000000..9a9e530 --- /dev/null +++ b/go/README.md @@ -0,0 +1,89 @@ +# msgpackblob (Go) + +A **pure-Go**, zero-dependency port of the standalone [C++ MessagePack Blob +API](../cpp/README.md) from [sqlite-msgpack](../README.md). It creates, +queries, mutates and iterates [MessagePack](https://msgpack.org/) binary blobs +and produces **byte-identical** output to the C++ library and the +`sqlite-msgpack` SQLite extension, so blobs are fully interchangeable across all +of them. + +- Standard library only (no third-party dependencies) +- Same `Blob` / `Builder` / `Value` / `Iterator` API as the C++ library +- All msgpack primitive types: fixed-width ints, float32/64, ext, timestamp, binary +- Native `int64` / `uint64`; Go strings naturally preserve non-UTF-8 bytes +- JSON conversion modelled on SQLite's JSON1 extension + +## Install + +```bash +go get github.com/khanaffan/sqlite-msgpack/go +``` + +```go +import mb "github.com/khanaffan/sqlite-msgpack/go" +``` + +## Quick start + +```go +package main + +import ( + "fmt" + + mb "github.com/khanaffan/sqlite-msgpack/go" +) + +func main() { + // Build from JSON + blob := mb.FromJSON(`{"name":"Alice","scores":[95,87,91]}`) + fmt.Println(blob.Extract("$.name").AsString()) // Alice + fmt.Println(blob.ArrayLengthAt("$.scores")) // 3 + fmt.Println(blob.ToJSON()) // {"name":"Alice","scores":[95,87,91]} + + // Mutate (copy-on-write — original is unchanged) + updated := blob.Set("$.age", mb.Int(30)) + fmt.Println(updated.ToJSON()) // {"name":"Alice","scores":[95,87,91],"age":30} + + // Build with the streaming Builder + b := mb.NewBuilder(). + MapHeader(2). + String("temp").Real32(23.5). + String("ts").TimestampNs(1700000000, 500000000). + Build() + _ = b + + // Iterate (flat "each" or recursive "tree") + for _, row := range mb.NewIterator(blob, "$", true).Rows() { + fmt.Println(row.Fullkey, row.Type) + } +} +``` + +## API overview + +| Type | Purpose | +|---|---| +| `Value` | A decoded scalar / sub-blob. Constructors are package functions: `mb.Int`, `mb.Uint`, `mb.Real32`, `mb.Str` / `mb.StrBytes`, `mb.Bin`, `mb.Ext`, `mb.Timestamp` / `mb.TimestampNs`, fixed-width `mb.Int8`…`mb.Uint64`. | +| `Blob` | Owning byte buffer. `FromJSON`, `ToJSON` / `ToJSONBytes`, `ToJSONPretty`, `Extract`, `TypeAt`, `ArrayLength`, `Valid`, and copy-on-write `Set` / `Insert` / `Replace` / `Remove` / `ArrayInsert` / `Patch`. | +| `Builder` | Streaming encoder. Chainable `Nil`/`Boolean`/`Integer`/`Real`/`String`/`Binary`/`Ext`/`Timestamp`/`ArrayHeader`/`MapHeader`/`Value`, plus fixed-width integer methods. `Build()` → `Blob`. | +| `Iterator` | Cursor over container children (`each` / `tree`). Use the `Next()`/`Current()` cursor or `Rows()`. | +| `Type`, `IntWidth`, `TypeStr` | Type enum (with `String()`), integer-width hint, and label helper. | + +Methods come in path-aware pairs where the C++ API overloads: `TypeAt(path)` / +`TypeStrAt(path)` / `ArrayLengthAt(path)` versus the root-level `Type()` / +`TypeStr()` / `ArrayLength()`. Non-UTF-8 string payloads are preserved +byte-exactly via `ToJSONBytes()` and `mb.StrBytes` / `Value.AsBytes`. Paths use +the same `$`-rooted syntax as the SQLite extension: `$`, `$.key`, `$[0]`, +`$.users[0].email`. + +## Tests + +```bash +cd go +go test ./... +``` + +`vectors_test.go` replays +[`tests/vectors/blob_vectors.json`](../tests/vectors/blob_vectors.json) — vectors +generated from the C++ reference implementation — to prove byte-identical output. diff --git a/go/api_test.go b/go/api_test.go new file mode 100644 index 0000000..925c96c --- /dev/null +++ b/go/api_test.go @@ -0,0 +1,265 @@ +package msgpackblob_test + +// API behaviour and round-trip tests for the Go port. + +import ( + "bytes" + "testing" + + mb "github.com/khanaffan/sqlite-msgpack/go" +) + +func TestBuilderMatchesFromJSON(t *testing.T) { + built := mb.NewBuilder(). + MapHeader(3). + String("name").String("Alice"). + String("age").Integer(30). + String("scores").ArrayHeader(3). + Real(95.5).Real(87.5).Real(91.0). + Build() + ref := mb.FromJSON(`{"name":"Alice","age":30,"scores":[95.5,87.5,91.0]}`) + if built.Hex() != ref.Hex() { + t.Errorf("got %s want %s", built.Hex(), ref.Hex()) + } +} + +func TestQuoteRoundtripsType(t *testing.T) { + values := []mb.Value{ + mb.Nil(), + mb.Bool(true), + mb.Int(-12345), + mb.Real(3.25), + mb.Real32(1.5), + mb.Str("hello"), + mb.Bin([]byte{0xde, 0xad}), + mb.Ext(7, []byte{1, 2}), + mb.TimestampNs(1700000000, 123456789), + } + for _, v := range values { + blob := mb.Quote(v) + if !blob.Valid() { + t.Errorf("quote(%v) invalid", v.Type()) + } + if blob.Extract("$").Type() != v.Type() { + t.Errorf("roundtrip type: got %v want %v", blob.Extract("$").Type(), v.Type()) + } + } +} + +func TestJSONBytesStable(t *testing.T) { + cases := []string{ + "null", "true", "false", "0", "-1", "127", "128", "65536", + "1.5", "0.1", "1e10", `"hi"`, "[]", "{}", "[1,2,3]", + `{"a":1,"b":[2,3],"c":{"d":true}}`, + `{"u":"caf\u00e9","emoji":"\ud83d\ude00"}`, + } + for _, c := range cases { + once := mb.FromJSON(c) + twice := mb.FromJSON(once.ToJSON()) + if once.Hex() != twice.Hex() { + t.Errorf("%s: %s != %s", c, once.Hex(), twice.Hex()) + } + } +} + +func TestIntegers64Bit(t *testing.T) { + blob := mb.Quote(mb.Uint64(^uint64(0))) + if blob.Hex() != "cfffffffffffffffff" { + t.Errorf("uint64 max hex: %s", blob.Hex()) + } + if blob.Extract("$").AsUint64() != ^uint64(0) { + t.Errorf("uint64 max roundtrip") + } + if blob.ToJSON() != "18446744073709551615" { + t.Errorf("uint64 json: %s", blob.ToJSON()) + } + const minI64 = -9223372036854775808 + neg := mb.Quote(mb.Int64(minI64)) + if neg.Extract("$").AsInt64() != minI64 { + t.Errorf("int64 min roundtrip") + } +} + +func TestExtraction(t *testing.T) { + blob := mb.FromJSON(`{"name":"Alice","age":30,"tall":true,"pets":["cat","dog"],"addr":{"city":"NYC"}}`) + if blob.Extract("$.name").AsString() != "Alice" { + t.Error("name") + } + if blob.Extract("$.age").AsInt64() != 30 { + t.Error("age") + } + if !blob.Extract("$.tall").AsBool() { + t.Error("tall") + } + if blob.Extract("$.pets[1]").AsString() != "dog" { + t.Error("pets[1]") + } + if blob.Extract("$.addr.city").AsString() != "NYC" { + t.Error("addr.city") + } + if !blob.Extract("$.nope").IsNil() { + t.Error("missing should be nil") + } + if blob.TypeStrAt("$.pets") != "array" { + t.Error("pets type") + } + if blob.ArrayLengthAt("$.pets") != 2 { + t.Error("pets length") + } + if blob.ArrayLengthAt("$.name") != -1 { + t.Error("name length should be -1") + } +} + +func TestBinaryExtTimestamp(t *testing.T) { + bin := mb.NewBuilder().Binary([]byte{1, 2, 3, 4}).Build() + if bin.Extract("$").Type() != mb.TypeBinary { + t.Error("binary type") + } + if !bytes.Equal(bin.Extract("$").BlobData(), []byte{1, 2, 3, 4}) { + t.Error("binary data") + } + if bin.ToJSON() != `"01020304"` { + t.Errorf("binary json: %s", bin.ToJSON()) + } + + ext := mb.NewBuilder().Ext(42, []byte{0xaa, 0xbb}).Build() + if ext.Extract("$").ExtType() != 42 { + t.Error("ext type") + } + + ts := mb.NewBuilder().TimestampNs(1700000000, 500000000).Build() + if ts.Extract("$").TimestampSeconds() != 1700000000 { + t.Error("ts seconds") + } + if ts.Extract("$").TimestampNanoseconds() != 500000000 { + t.Error("ts nanos") + } +} + +func TestCopyOnWriteMutation(t *testing.T) { + orig := mb.FromJSON(`{"a":1}`) + updated := orig.Set("$.b", mb.Int(2)) + if orig.ToJSON() != `{"a":1}` { + t.Errorf("orig mutated: %s", orig.ToJSON()) + } + if updated.ToJSON() != `{"a":1,"b":2}` { + t.Errorf("set: %s", updated.ToJSON()) + } + + b := mb.FromJSON(`{"a":1,"b":2,"c":3}`) + if b.Remove("$.b").ToJSON() != `{"a":1,"c":3}` { + t.Errorf("remove: %s", b.Remove("$.b").ToJSON()) + } + patched := b.Patch(mb.FromJSON(`{"b":null,"d":4}`)) + if patched.ToJSON() != `{"a":1,"c":3,"d":4}` { + t.Errorf("patch: %s", patched.ToJSON()) + } + + arr := mb.FromJSON("[1,2,3]") + if arr.ArrayInsert("$[1]", mb.Int(9)).ToJSON() != "[1,9,2,3]" { + t.Errorf("array_insert: %s", arr.ArrayInsert("$[1]", mb.Int(9)).ToJSON()) + } + if arr.Set("$[3]", mb.Int(4)).ToJSON() != "[1,2,3,4]" { + t.Errorf("append: %s", arr.Set("$[3]", mb.Int(4)).ToJSON()) + } +} + +func TestIterator(t *testing.T) { + m := mb.FromJSON(`{"a":1,"b":2,"c":3}`) + var keys []string + for _, r := range mb.NewIterator(m, "$", false).Rows() { + keys = append(keys, r.Key) + } + if len(keys) != 3 || keys[0] != "a" || keys[2] != "c" { + t.Errorf("each keys: %v", keys) + } + + nested := mb.FromJSON(`{"x":{"y":[1,2]}}`) + var fullkeys []string + for _, r := range mb.NewIterator(nested, "$", true).Rows() { + fullkeys = append(fullkeys, r.Fullkey) + } + want := []string{"$", "$.x", "$.x.y", "$.x.y[0]", "$.x.y[1]"} + if len(fullkeys) != len(want) { + t.Fatalf("tree fullkeys: %v", fullkeys) + } + for i := range want { + if fullkeys[i] != want[i] { + t.Errorf("tree[%d]: %s != %s", i, fullkeys[i], want[i]) + } + } + + it := mb.NewIterator(mb.FromJSON("[1,2]"), "$", false) + var seen []int64 + for it.Next() { + seen = append(seen, it.Current().Value.AsInt64()) + } + if len(seen) != 2 || seen[0] != 1 || seen[1] != 2 { + t.Errorf("cursor: %v", seen) + } +} + +func TestValidity(t *testing.T) { + if !mb.FromJSON("[1,2,3]").Valid() { + t.Error("valid array") + } + if mb.NewBlob(nil).Valid() { + t.Error("empty should be invalid") + } + if mb.NewBlob([]byte{0x91}).Valid() { + t.Error("truncated array should be invalid") + } +} + +func TestTypeStr(t *testing.T) { + if mb.TypeStr(mb.TypeNil) != "null" || mb.TypeStr(mb.TypeString) != "text" || + mb.TypeStr(mb.TypeFloat32) != "float32" || mb.TypeStr(mb.TypeTimestamp) != "timestamp" { + t.Error("type labels") + } +} + +func TestTruncatedMapKeyNoPanic(t *testing.T) { + // One-entry map whose str8 key declares length 10 but supplies only 2 bytes. + // The C++ reference bails via skipOne; the port must not panic. + blob := mb.NewBlob([]byte{0x81, 0xd9, 0x0a, 0x61, 0x62}) + if !bytes.Equal(blob.Set("$.x", mb.Int(1)).Data(), blob.Data()) { + t.Error("set should return original on truncated key") + } + if !bytes.Equal(blob.Remove("$.x").Data(), blob.Data()) { + t.Error("remove should return original on truncated key") + } + if !bytes.Equal(blob.Patch(mb.FromJSON(`{"a":1}`)).Data(), blob.Data()) { + t.Error("patch should return original on truncated key") + } + if len(mb.NewIterator(blob, "$", false).Rows()) != 0 { + t.Error("flat iterate should yield 0 rows on truncated key") + } + _ = mb.NewIterator(blob, "$", true).Rows() + // Truncated fixstr key as well. + fix := mb.NewBlob([]byte{0x81, 0xa5, 0x68, 0x69}) + if !bytes.Equal(fix.Set("$.x", mb.Int(1)).Data(), fix.Data()) { + t.Error("set should return original on truncated fixstr key") + } + if len(mb.NewIterator(fix, "$", false).Rows()) != 0 { + t.Error("flat iterate should yield 0 rows on truncated fixstr key") + } +} + +func TestNonUTF8Preserved(t *testing.T) { + // {"k": <0xff 0x80 0xfe 0xc0>} — a str with non-UTF-8 payload, as a foreign + // encoder (C++/SQLite) may produce. C++ passes raw bytes through verbatim. + blob := mb.NewBlob([]byte{0x81, 0xa1, 0x6b, 0xa4, 0xff, 0x80, 0xfe, 0xc0}) + want := []byte{0x7b, 0x22, 0x6b, 0x22, 0x3a, 0x22, 0xff, 0x80, 0xfe, 0xc0, 0x22, 0x7d} + if !bytes.Equal(blob.ToJSONBytes(), want) { + t.Errorf("to_json bytes: %x want %x", blob.ToJSONBytes(), want) + } + v := blob.Extract("$.k") + if !bytes.Equal(v.AsBytes(), []byte{0xff, 0x80, 0xfe, 0xc0}) { + t.Errorf("as_bytes: %x", v.AsBytes()) + } + rebuilt := mb.NewBuilder().StringBytes(v.AsBytes()).Build() + if rebuilt.Hex() != "a4ff80fec0" { + t.Errorf("rebuilt: %s", rebuilt.Hex()) + } +} diff --git a/go/blob.go b/go/blob.go new file mode 100644 index 0000000..486a092 --- /dev/null +++ b/go/blob.go @@ -0,0 +1,172 @@ +package msgpackblob + +import "encoding/hex" + +// Blob is a MessagePack BLOB supporting read, mutation (copy-on-write) and JSON. +// The zero value is an empty (invalid) blob. +type Blob struct { + data []byte +} + +// NewBlob constructs a Blob from raw bytes (copies). +func NewBlob(data []byte) Blob { + return Blob{data: append([]byte(nil), data...)} +} + +// blobFromVec takes ownership of data without copying. +func blobFromVec(data []byte) Blob { + return Blob{data: data} +} + +// ── raw access ──────────────────────────────────────────────────────── +func (b Blob) Data() []byte { return b.data } +func (b Blob) Size() int { return len(b.data) } +func (b Blob) Empty() bool { return len(b.data) == 0 } +func (b Blob) Hex() string { return hex.EncodeToString(b.data) } + +// ── validation ──────────────────────────────────────────────────────── +func (b Blob) Valid() bool { return isValid(b.data, len(b.data)) } +func (b Blob) ErrorPosition() int { return errorPosition(b.data, len(b.data)) } + +// ── type inspection ─────────────────────────────────────────────────── + +// Type returns the type of the root element. +func (b Blob) Type() Type { + if len(b.data) == 0 { + return TypeNil + } + return getType(b.data, len(b.data), 0) +} + +// TypeAt returns the type of the element at path. +func (b Blob) TypeAt(path string) Type { + n := len(b.data) + rc, istart, _ := lookup(b.data, n, 0, path) + if rc != rcOK { + return TypeNil + } + return getType(b.data, n, istart) +} + +// TypeStr returns the label of the root element type. +func (b Blob) TypeStr() string { return b.Type().String() } + +// TypeStrAt returns the label of the element type at path. +func (b Blob) TypeStrAt(path string) string { return b.TypeAt(path).String() } + +// ── extraction ──────────────────────────────────────────────────────── + +// Extract decodes the element at path (nil Value if not found). +func (b Blob) Extract(path string) Value { + n := len(b.data) + rc, istart, iend := lookup(b.data, n, 0, path) + if rc != rcOK { + return Nil() + } + return decodeElement(b.data, n, istart, iend) +} + +// ArrayLength returns the element count of the root container, or -1. +func (b Blob) ArrayLength() int64 { + if len(b.data) == 0 { + return -1 + } + return getContainerCount(b.data, len(b.data), 0) +} + +// ArrayLengthAt returns the element count of the container at path, or -1. +func (b Blob) ArrayLengthAt(path string) int64 { + n := len(b.data) + rc, istart, _ := lookup(b.data, n, 0, path) + if rc != rcOK { + return -1 + } + return getContainerCount(b.data, n, istart) +} + +// ── mutation (copy-on-write) ────────────────────────────────────────── +func (b Blob) apply(path string, value Value, mode int) Blob { + var nb []byte + encodeValue(&nb, value) + rc, out := applyEdit(b.data, len(b.data), path, nb, mode) + if rc == rcOK { + return blobFromVec(out) + } + return b +} + +// Set assigns value at path (creating it where appropriate). +func (b Blob) Set(path string, value Value) Blob { return b.apply(path, value, editSet) } + +// SetBlob assigns an existing sub-blob at path (embedded verbatim). +func (b Blob) SetBlob(path string, sub Blob) Blob { + rc, out := applyEdit(b.data, len(b.data), path, sub.data, editSet) + if rc == rcOK { + return blobFromVec(out) + } + return b +} + +// Insert adds value at path only if it does not already exist. +func (b Blob) Insert(path string, value Value) Blob { return b.apply(path, value, editInsert) } + +// Replace overwrites value at path only if it already exists. +func (b Blob) Replace(path string, value Value) Blob { return b.apply(path, value, editReplace) } + +// ArrayInsert inserts value before the element at the array index path. +func (b Blob) ArrayInsert(path string, value Value) Blob { return b.apply(path, value, editArrayIns) } + +// Remove deletes the element at path. +func (b Blob) Remove(path string) Blob { + rc, out := applyEdit(b.data, len(b.data), path, nil, editRemove) + if rc == rcOK { + return blobFromVec(out) + } + return b +} + +// Patch applies an RFC 7386 merge patch. +func (b Blob) Patch(merge Blob) Blob { + rc, out := mergePatch(b.data, len(b.data), 0, merge.data, len(merge.data), 0) + if rc == rcOK { + return blobFromVec(out) + } + return b +} + +// ── JSON conversion ─────────────────────────────────────────────────── + +// ToJSONBytes returns the byte-exact JSON serialisation (may contain non-UTF-8 +// string bytes, identical to the C++ library output). +func (b Blob) ToJSONBytes() []byte { + if len(b.data) == 0 { + return []byte("null") + } + return toJSONBytes(b.data, len(b.data), false, 0) +} + +// ToJSON returns the JSON serialisation as a string. +func (b Blob) ToJSON() string { return string(b.ToJSONBytes()) } + +// ToJSONPrettyBytes returns the indented JSON serialisation as bytes. +func (b Blob) ToJSONPrettyBytes(indent int) []byte { + if len(b.data) == 0 { + return []byte("null") + } + if indent < 0 { + indent = 0 + } + if indent > 8 { + indent = 8 + } + return toJSONBytes(b.data, len(b.data), true, indent) +} + +// ToJSONPretty returns the indented JSON serialisation as a string. +func (b Blob) ToJSONPretty(indent int) string { return string(b.ToJSONPrettyBytes(indent)) } + +// FromJSONBytes parses JSON bytes into a msgpack blob. +func FromJSONBytes(json []byte) Blob { return blobFromVec(fromJSON(json)) } + +// FromJSON parses a JSON string into a msgpack blob. +func FromJSON(json string) Blob { return blobFromVec(fromJSON([]byte(json))) } diff --git a/go/builder.go b/go/builder.go new file mode 100644 index 0000000..21fbf4a --- /dev/null +++ b/go/builder.go @@ -0,0 +1,65 @@ +package msgpackblob + +// Builder is a streaming encoder that produces a Blob. Methods return the +// Builder so calls can be chained; finalise with Build. +type Builder struct { + buf []byte +} + +// NewBuilder returns a new, empty Builder. +func NewBuilder() *Builder { return &Builder{} } + +// ── scalars ─────────────────────────────────────────────────────────── +func (b *Builder) Nil() *Builder { encNil(&b.buf); return b } +func (b *Builder) Boolean(v bool) *Builder { encBool(&b.buf, v); return b } +func (b *Builder) Integer(x int64) *Builder { encInteger(&b.buf, x); return b } +func (b *Builder) UnsignedInteger(x uint64) *Builder { encUnsigned(&b.buf, x); return b } +func (b *Builder) Real(d float64) *Builder { encReal(&b.buf, d); return b } +func (b *Builder) Real32(f float32) *Builder { encReal32(&b.buf, f); return b } +func (b *Builder) String(s string) *Builder { encString(&b.buf, []byte(s)); return b } +func (b *Builder) StringBytes(s []byte) *Builder { encString(&b.buf, s); return b } +func (b *Builder) Binary(data []byte) *Builder { encBinary(&b.buf, data); return b } + +// Ext appends an ext element. +func (b *Builder) Ext(typeCode int8, data []byte) *Builder { encExt(&b.buf, typeCode, data); return b } + +// ── fixed-width integers ────────────────────────────────────────────── +func (b *Builder) Int8(x int8) *Builder { encInt8(&b.buf, int64(x)); return b } +func (b *Builder) Int16(x int16) *Builder { encInt16(&b.buf, int64(x)); return b } +func (b *Builder) Int32(x int32) *Builder { encInt32(&b.buf, int64(x)); return b } +func (b *Builder) Int64(x int64) *Builder { encInt64(&b.buf, x); return b } +func (b *Builder) Uint8(x uint8) *Builder { encUint8(&b.buf, uint64(x)); return b } +func (b *Builder) Uint16(x uint16) *Builder { encUint16(&b.buf, uint64(x)); return b } +func (b *Builder) Uint32(x uint32) *Builder { encUint32(&b.buf, uint64(x)); return b } +func (b *Builder) Uint64(x uint64) *Builder { encUint64(&b.buf, x); return b } + +// ── containers ──────────────────────────────────────────────────────── +func (b *Builder) ArrayHeader(count uint32) *Builder { encArrayHeader(&b.buf, count); return b } +func (b *Builder) MapHeader(count uint32) *Builder { encMapHeader(&b.buf, count); return b } + +// ── embedding & timestamp ───────────────────────────────────────────── +func (b *Builder) Raw(data []byte) *Builder { b.buf = append(b.buf, data...); return b } +func (b *Builder) RawBlob(blob Blob) *Builder { b.buf = append(b.buf, blob.data...); return b } +func (b *Builder) Value(v Value) *Builder { encodeValue(&b.buf, v); return b } +func (b *Builder) Timestamp(sec int64) *Builder { encTimestamp(&b.buf, sec, 0); return b } + +// TimestampNs appends a timestamp with nanoseconds. +func (b *Builder) TimestampNs(sec int64, nsec uint32) *Builder { + encTimestamp(&b.buf, sec, nsec) + return b +} + +// ── finalize ────────────────────────────────────────────────────────── + +// Build consumes the accumulated bytes and returns a Blob. +func (b *Builder) Build() Blob { return blobFromVec(b.buf) } + +// Len returns the number of bytes accumulated so far. +func (b *Builder) Len() int { return len(b.buf) } + +// Quote encodes a single Value into a Blob. +func Quote(v Value) Blob { + var b Builder + b.Value(v) + return b.Build() +} diff --git a/go/decode.go b/go/decode.go new file mode 100644 index 0000000..ca909a7 --- /dev/null +++ b/go/decode.go @@ -0,0 +1,466 @@ +package msgpackblob + +import "math" + +const ( + rcOK = 0 + rcError = 1 + rcNotFound = 2 +) + +func isValid(a []byte, n int) bool { + if n == 0 { + return false + } + return skipOne(a, n, 0) == n +} + +func errorPosition(a []byte, n int) int { + if n == 0 { + return 0 + } + if skipOne(a, n, 0) == n { + return 0 + } + for i := 0; i < n; { + nxt := skipOne(a, n, i) + if nxt == 0 { + return i + } + i = nxt + } + return 0 +} + +func isTimestampExt(a []byte, n, i int) bool { + if i >= n { + return false + } + b := a[i] + if b == mpFixext4 && i+6 <= n && a[i+1] == mpTimestampType { + return true + } + if b == mpFixext8 && i+10 <= n && a[i+1] == mpTimestampType { + return true + } + if b == mpExt8 && i+3 <= n && a[i+1] == 12 && a[i+2] == mpTimestampType { + return true + } + return false +} + +func decodeTimestamp(a []byte, n, i int) (int64, uint32, bool) { + if i >= n { + return 0, 0, false + } + b := a[i] + if b == mpFixext4 && i+6 <= n && a[i+1] == mpTimestampType { + return int64(read32(a, i+2)), 0, true + } + if b == mpFixext8 && i+10 <= n && a[i+1] == mpTimestampType { + v := read64(a, i+2) + return int64(v & 0x3FFFFFFFF), uint32(v >> 34), true + } + if b == mpExt8 && i+15 <= n && a[i+1] == 12 && a[i+2] == mpTimestampType { + nsec := read32(a, i+3) + sec := int64(read64(a, i+7)) + return sec, nsec, true + } + return 0, 0, false +} + +func getType(a []byte, n, i int) Type { + if i >= n { + return TypeNil + } + b := a[i] + if b == mpNil { + return TypeNil + } + if b == mpTrue { + return TypeTrue + } + if b == mpFalse { + return TypeFalse + } + if b <= 0x7f || b >= 0xe0 { + return TypeInteger + } + if b >= 0xa0 && b <= 0xbf { + return TypeString + } + if b >= 0x90 && b <= 0x9f { + return TypeArray + } + if b >= 0x80 && b <= 0x8f { + return TypeMap + } + switch b { + case mpUint8, mpUint16, mpUint32, mpUint64, mpInt8, mpInt16, mpInt32, mpInt64: + return TypeInteger + case mpFloat32: + return TypeFloat32 + case mpFloat64: + return TypeReal + case mpStr8, mpStr16, mpStr32: + return TypeString + case mpBin8, mpBin16, mpBin32: + return TypeBinary + case mpArray16, mpArray32: + return TypeArray + case mpMap16, mpMap32: + return TypeMap + case mpExt8, mpExt16, mpExt32, mpFixext1, mpFixext2, mpFixext4, mpFixext8, mpFixext16: + if isTimestampExt(a, n, i) { + return TypeTimestamp + } + return TypeExt + } + return TypeNil +} + +func getContainerCount(a []byte, n, i int) int64 { + if i >= n { + return -1 + } + b := a[i] + if b >= 0x90 && b <= 0x9f { + return int64(b & 0x0f) + } + if b >= 0x80 && b <= 0x8f { + return int64(b & 0x0f) + } + if b == mpArray16 && i+3 <= n { + return int64(read16(a, i+1)) + } + if b == mpArray32 && i+5 <= n { + return int64(read32(a, i+1)) + } + if b == mpMap16 && i+3 <= n { + return int64(read16(a, i+1)) + } + if b == mpMap32 && i+5 <= n { + return int64(read32(a, i+1)) + } + return -1 +} + +// stepKind classifies one parsed path step. +type stepKind int + +const ( + stepEnd stepKind = iota + stepErr + stepKey + stepIndex +) + +func pathStep(zpath string, pi int) (stepKind, int, string, int64) { + i := pi + if i >= len(zpath) { + return stepEnd, i, "", 0 + } + c := zpath[i] + if c == '.' { + i++ + start := i + for i < len(zpath) && zpath[i] != '.' && zpath[i] != '[' { + i++ + } + return stepKey, i, zpath[start:i], 0 + } + if c == '[' { + var idx int64 + hasDigit := false + i++ + for i < len(zpath) && zpath[i] >= '0' && zpath[i] <= '9' { + idx = idx*10 + int64(zpath[i]-'0') + i++ + hasDigit = true + } + if !hasDigit || i >= len(zpath) || zpath[i] != ']' { + return stepErr, i, "", 0 + } + i++ + return stepIndex, i, "", idx + } + return stepErr, i, "", 0 +} + +// keyAt returns the key bytes for a map key at i, or nil. +func keyAt(a []byte, n, i int) ([]byte, bool) { + kb := a[i] + var klen, koff int + switch { + case kb >= 0xa0 && kb <= 0xbf: + klen, koff = int(kb&0x1f), i+1 + case kb == mpStr8 && i+2 <= n: + klen, koff = int(a[i+1]), i+2 + case kb == mpStr16 && i+3 <= n: + klen, koff = int(read16(a, i+1)), i+3 + case kb == mpStr32 && i+5 <= n: + klen, koff = int(read32(a, i+1)), i+5 + default: + return nil, false + } + if klen > n-koff { + return nil, false + } + return a[koff : koff+klen], true +} + +func bytesEqual(x, y []byte) bool { + if len(x) != len(y) { + return false + } + for i := range x { + if x[i] != y[i] { + return false + } + } + return true +} + +// lookup resolves zpath to a byte range; returns (rc, iStart, iEnd). +func lookup(a []byte, n, iroot int, zpath string) (int, int, int) { + if len(zpath) == 0 || zpath[0] != '$' { + return rcError, 0, 0 + } + icur := iroot + pi := 1 + + for { + kind, npi, key, idx := pathStep(zpath, pi) + pi = npi + switch kind { + case stepEnd: + inext := skipOne(a, n, icur) + iend := inext + if iend == 0 { + iend = n + } + if inext != 0 || icur == n { + return rcOK, icur, iend + } + return rcError, icur, iend + case stepErr: + return rcError, 0, 0 + case stepIndex: + if icur >= n { + return rcNotFound, 0, 0 + } + b := a[icur] + var count int64 + var elemOff int + if b >= 0x90 && b <= 0x9f { + count, elemOff = int64(b&0x0f), icur+1 + } else if b == mpArray16 { + if icur+3 > n { + return rcError, 0, 0 + } + count, elemOff = int64(read16(a, icur+1)), icur+3 + } else if b == mpArray32 { + if icur+5 > n { + return rcError, 0, 0 + } + count, elemOff = int64(read32(a, icur+1)), icur+5 + } else { + return rcNotFound, 0, 0 + } + if idx < 0 || idx >= count { + return rcNotFound, 0, 0 + } + icur = elemOff + for j := int64(0); j < idx; j++ { + icur = skipOne(a, n, icur) + if icur == 0 { + return rcError, 0, 0 + } + } + case stepKey: + if icur >= n { + return rcNotFound, 0, 0 + } + b := a[icur] + var count, elemOff int + if b >= 0x80 && b <= 0x8f { + count, elemOff = int(b&0x0f), icur+1 + } else if b == mpMap16 { + if icur+3 > n { + return rcError, 0, 0 + } + count, elemOff = int(read16(a, icur+1)), icur+3 + } else if b == mpMap32 { + if icur+5 > n { + return rcError, 0, 0 + } + count, elemOff = int(read32(a, icur+1)), icur+5 + } else { + return rcNotFound, 0, 0 + } + keyBytes := []byte(key) + icur = elemOff + found := false + for j := 0; j < count && !found; j++ { + if icur >= n { + return rcError, 0, 0 + } + kstr, ok := keyAt(a, n, icur) + valOff := skipOne(a, n, icur) + if valOff == 0 { + return rcError, 0, 0 + } + if ok && bytesEqual(kstr, keyBytes) { + icur = valOff + found = true + } else { + icur = skipOne(a, n, valOff) + if icur == 0 { + return rcError, 0, 0 + } + } + } + if !found { + return rcNotFound, 0, 0 + } + } + } +} + +func decodeElement(a []byte, n, istart, iend int) Value { + if istart >= n || istart >= iend { + return Nil() + } + b := a[istart] + + if b == mpNil { + return Nil() + } + if b == mpFalse { + return Bool(false) + } + if b == mpTrue { + return Bool(true) + } + if b <= 0x7f { + return Int(int64(b)) + } + if b >= 0xe0 { + return Int(int64(int8(b))) + } + + switch b { + case mpUint8: + if istart+2 <= n { + return Int(int64(a[istart+1])) + } + case mpUint16: + if istart+3 <= n { + return Int(int64(read16(a, istart+1))) + } + case mpUint32: + if istart+5 <= n { + return Int(int64(read32(a, istart+1))) + } + case mpUint64: + if istart+9 <= n { + return Uint(read64(a, istart+1)) + } + case mpInt8: + if istart+2 <= n { + return Int(int64(int8(a[istart+1]))) + } + case mpInt16: + if istart+3 <= n { + return Int(int64(int16(read16(a, istart+1)))) + } + case mpInt32: + if istart+5 <= n { + return Int(int64(int32(read32(a, istart+1)))) + } + case mpInt64: + if istart+9 <= n { + return Int(int64(read64(a, istart+1))) + } + case mpFloat32: + if istart+5 <= n { + return Real32(math.Float32frombits(read32(a, istart+1))) + } + case mpFloat64: + if istart+9 <= n { + return Real(math.Float64frombits(read64(a, istart+1))) + } + } + + // str + var slen, soff int + switch { + case b >= 0xa0 && b <= 0xbf: + slen, soff = int(b&0x1f), istart+1 + case b == mpStr8 && istart+2 <= n: + slen, soff = int(a[istart+1]), istart+2 + case b == mpStr16 && istart+3 <= n: + slen, soff = int(read16(a, istart+1)), istart+3 + case b == mpStr32 && istart+5 <= n: + slen, soff = int(read32(a, istart+1)), istart+5 + } + if soff != 0 { + if slen > n-soff { + slen = n - soff + } + return StrBytes(a[soff : soff+slen]) + } + + // bin + var blen, boff int + switch { + case b == mpBin8 && istart+2 <= n: + blen, boff = int(a[istart+1]), istart+2 + case b == mpBin16 && istart+3 <= n: + blen, boff = int(read16(a, istart+1)), istart+3 + case b == mpBin32 && istart+5 <= n: + blen, boff = int(read32(a, istart+1)), istart+5 + } + if boff != 0 { + if blen > n-boff { + blen = n - boff + } + return Bin(a[boff : boff+blen]) + } + + // timestamp + if sec, nsec, ok := decodeTimestamp(a, n, istart); ok { + return TimestampNs(sec, nsec) + } + + // ext + var tc int8 + var elen, eoff int + switch { + case b == mpFixext1 && istart+3 <= n: + tc, elen, eoff = int8(a[istart+1]), 1, istart+2 + case b == mpFixext2 && istart+4 <= n: + tc, elen, eoff = int8(a[istart+1]), 2, istart+2 + case b == mpFixext4 && istart+6 <= n: + tc, elen, eoff = int8(a[istart+1]), 4, istart+2 + case b == mpFixext8 && istart+10 <= n: + tc, elen, eoff = int8(a[istart+1]), 8, istart+2 + case b == mpFixext16 && istart+18 <= n: + tc, elen, eoff = int8(a[istart+1]), 16, istart+2 + case b == mpExt8 && istart+3 <= n: + tc, elen, eoff = int8(a[istart+2]), int(a[istart+1]), istart+3 + case b == mpExt16 && istart+4 <= n: + tc, elen, eoff = int8(a[istart+3]), int(read16(a, istart+1)), istart+4 + case b == mpExt32 && istart+6 <= n: + tc, elen, eoff = int8(a[istart+5]), int(read32(a, istart+1)), istart+6 + } + if eoff != 0 { + if elen > n-eoff { + elen = n - eoff + } + return Ext(tc, a[eoff:eoff+elen]) + } + + // containers → raw binary blob (includes header) + return Bin(a[istart:iend]) +} diff --git a/go/encode.go b/go/encode.go new file mode 100644 index 0000000..76d49b3 --- /dev/null +++ b/go/encode.go @@ -0,0 +1,233 @@ +package msgpackblob + +import "math" + +func encNil(out *[]byte) { *out = append(*out, mpNil) } + +func encBool(out *[]byte, v bool) { + if v { + *out = append(*out, mpTrue) + } else { + *out = append(*out, mpFalse) + } +} + +func encInteger(out *[]byte, x int64) { + if x >= 0 { + switch { + case x <= 0x7f: + *out = append(*out, byte(x)) + case x <= 0xff: + *out = append(*out, mpUint8, byte(x)) + case x <= 0xffff: + *out = append(*out, mpUint16) + put16(out, uint16(x)) + case x <= 0xffffffff: + *out = append(*out, mpUint32) + put32(out, uint32(x)) + default: + *out = append(*out, mpUint64) + put64(out, uint64(x)) + } + } else { + switch { + case x >= -32: + *out = append(*out, byte(x)) + case x >= -128: + *out = append(*out, mpInt8, byte(x)) + case x >= -32768: + *out = append(*out, mpInt16) + put16(out, uint16(x)) + case x >= -2147483648: + *out = append(*out, mpInt32) + put32(out, uint32(x)) + default: + *out = append(*out, mpInt64) + put64(out, uint64(x)) + } + } +} + +func encUnsigned(out *[]byte, x uint64) { + switch { + case x <= 0x7f: + *out = append(*out, byte(x)) + case x <= 0xff: + *out = append(*out, mpUint8, byte(x)) + case x <= 0xffff: + *out = append(*out, mpUint16) + put16(out, uint16(x)) + case x <= 0xffffffff: + *out = append(*out, mpUint32) + put32(out, uint32(x)) + default: + *out = append(*out, mpUint64) + put64(out, x) + } +} + +func encReal(out *[]byte, d float64) { + *out = append(*out, mpFloat64) + put64(out, math.Float64bits(d)) +} + +func encReal32(out *[]byte, f float32) { + *out = append(*out, mpFloat32) + put32(out, math.Float32bits(f)) +} + +func encString(out *[]byte, s []byte) { + n := len(s) + switch { + case n <= 31: + *out = append(*out, byte(mpFixstrMask|n)) + case n <= 0xff: + *out = append(*out, mpStr8, byte(n)) + case n <= 0xffff: + *out = append(*out, mpStr16) + put16(out, uint16(n)) + default: + *out = append(*out, mpStr32) + put32(out, uint32(n)) + } + *out = append(*out, s...) +} + +func encBinary(out *[]byte, data []byte) { + n := len(data) + switch { + case n <= 0xff: + *out = append(*out, mpBin8, byte(n)) + case n <= 0xffff: + *out = append(*out, mpBin16) + put16(out, uint16(n)) + default: + *out = append(*out, mpBin32) + put32(out, uint32(n)) + } + *out = append(*out, data...) +} + +func encExt(out *[]byte, typeCode int8, data []byte) { + n := len(data) + switch n { + case 1: + *out = append(*out, mpFixext1) + case 2: + *out = append(*out, mpFixext2) + case 4: + *out = append(*out, mpFixext4) + case 8: + *out = append(*out, mpFixext8) + case 16: + *out = append(*out, mpFixext16) + default: + switch { + case n <= 0xff: + *out = append(*out, mpExt8, byte(n)) + case n <= 0xffff: + *out = append(*out, mpExt16) + put16(out, uint16(n)) + default: + *out = append(*out, mpExt32) + put32(out, uint32(n)) + } + } + *out = append(*out, byte(typeCode)) + *out = append(*out, data...) +} + +func encInt8(out *[]byte, x int64) { *out = append(*out, mpInt8, byte(x)) } +func encInt16(out *[]byte, x int64) { *out = append(*out, mpInt16); put16(out, uint16(x)) } +func encInt32(out *[]byte, x int64) { *out = append(*out, mpInt32); put32(out, uint32(x)) } +func encInt64(out *[]byte, x int64) { *out = append(*out, mpInt64); put64(out, uint64(x)) } +func encUint8(out *[]byte, x uint64) { *out = append(*out, mpUint8, byte(x)) } +func encUint16(out *[]byte, x uint64) { *out = append(*out, mpUint16); put16(out, uint16(x)) } +func encUint32(out *[]byte, x uint64) { *out = append(*out, mpUint32); put32(out, uint32(x)) } +func encUint64(out *[]byte, x uint64) { *out = append(*out, mpUint64); put64(out, x) } + +func encArrayHeader(out *[]byte, count uint32) { + switch { + case count <= 15: + *out = append(*out, byte(mpFixarrayMask|count)) + case count <= 0xffff: + *out = append(*out, mpArray16) + put16(out, uint16(count)) + default: + *out = append(*out, mpArray32) + put32(out, count) + } +} + +func encMapHeader(out *[]byte, count uint32) { + switch { + case count <= 15: + *out = append(*out, byte(mpFixmapMask|count)) + case count <= 0xffff: + *out = append(*out, mpMap16) + put16(out, uint16(count)) + default: + *out = append(*out, mpMap32) + put32(out, count) + } +} + +func encTimestamp(out *[]byte, sec int64, nsec uint32) { + if nsec == 0 && sec >= 0 && sec <= 0xffffffff { + *out = append(*out, mpFixext4, 0xff) + put32(out, uint32(sec)) + } else if sec >= 0 && sec <= 0x3FFFFFFFF { + *out = append(*out, mpFixext8, 0xff) + put64(out, (uint64(nsec)<<34)|uint64(sec)) + } else { + *out = append(*out, mpExt8, 12, 0xff) + put32(out, nsec) + put64(out, uint64(sec)) + } +} + +func encodeValue(out *[]byte, v Value) { + switch v.ty { + case TypeNil: + encNil(out) + case TypeTrue: + encBool(out, true) + case TypeFalse: + encBool(out, false) + case TypeInteger: + switch v.intWidth { + case WidthInt8: + encInt8(out, v.AsInt64()) + case WidthInt16: + encInt16(out, v.AsInt64()) + case WidthInt32: + encInt32(out, v.AsInt64()) + case WidthInt64: + encInt64(out, v.AsInt64()) + case WidthUint8: + encUint8(out, v.AsUint64()) + case WidthUint16: + encUint16(out, v.AsUint64()) + case WidthUint32: + encUint32(out, v.AsUint64()) + case WidthUint64: + encUint64(out, v.AsUint64()) + default: + encInteger(out, v.AsInt64()) + } + case TypeReal: + encReal(out, v.AsFloat64()) + case TypeFloat32: + encReal32(out, v.AsFloat32()) + case TypeString: + encString(out, v.AsBytes()) + case TypeBinary: + encBinary(out, v.BlobData()) + case TypeExt: + encExt(out, v.ExtType(), v.BlobData()) + case TypeTimestamp: + encTimestamp(out, v.TimestampSeconds(), v.TimestampNanoseconds()) + default: + encNil(out) + } +} diff --git a/go/format.go b/go/format.go new file mode 100644 index 0000000..d34368f --- /dev/null +++ b/go/format.go @@ -0,0 +1,317 @@ +// Package msgpackblob is a pure-Go MessagePack Blob library. +// +// It is a zero-dependency port of the standalone C++ "msgpack" Blob API from +// sqlite-msgpack. It creates, queries, mutates and iterates MessagePack binary +// blobs and produces byte-identical output to the C++ library and the +// sqlite-msgpack SQLite extension, so blobs are fully interchangeable. +package msgpackblob + +// MaxDepth is the maximum container nesting depth (matches the C++ library and +// the SQLite extension). +const MaxDepth = 200 + +// MaxOutput is the maximum output buffer size (64 MiB). +const MaxOutput = 64 * 1024 * 1024 + +// Version of the library. +const Version = "1.5.0" + +// MessagePack format bytes. +const ( + mpNil = 0xc0 + mpFalse = 0xc2 + mpTrue = 0xc3 + mpBin8 = 0xc4 + mpBin16 = 0xc5 + mpBin32 = 0xc6 + mpExt8 = 0xc7 + mpExt16 = 0xc8 + mpExt32 = 0xc9 + mpFloat32 = 0xca + mpFloat64 = 0xcb + mpUint8 = 0xcc + mpUint16 = 0xcd + mpUint32 = 0xce + mpUint64 = 0xcf + mpInt8 = 0xd0 + mpInt16 = 0xd1 + mpInt32 = 0xd2 + mpInt64 = 0xd3 + mpFixext1 = 0xd4 + mpFixext2 = 0xd5 + mpFixext4 = 0xd6 + mpFixext8 = 0xd7 + mpFixext16 = 0xd8 + mpStr8 = 0xd9 + mpStr16 = 0xda + mpStr32 = 0xdb + mpArray16 = 0xdc + mpArray32 = 0xdd + mpMap16 = 0xde + mpMap32 = 0xdf + + mpFixmapMask = 0x80 + mpFixarrayMask = 0x90 + mpFixstrMask = 0xa0 + + mpTimestampType = 0xff +) + +// ── big-endian read helpers ───────────────────────────────────────── +func read16(a []byte, i int) uint32 { + return uint32(a[i])<<8 | uint32(a[i+1]) +} + +func read32(a []byte, i int) uint32 { + return uint32(a[i])<<24 | uint32(a[i+1])<<16 | uint32(a[i+2])<<8 | uint32(a[i+3]) +} + +func read64(a []byte, i int) uint64 { + return uint64(read32(a, i))<<32 | uint64(read32(a, i+4)) +} + +// ── big-endian write helpers ──────────────────────────────────────── +func put16(out *[]byte, v uint16) { + *out = append(*out, byte(v>>8), byte(v)) +} + +func put32(out *[]byte, v uint32) { + *out = append(*out, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) +} + +func put64(out *[]byte, v uint64) { + *out = append(*out, byte(v>>56), byte(v>>48), byte(v>>40), byte(v>>32), + byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) +} + +// skipOne returns the offset just past one complete element starting at i, or 0 +// on malformed / truncated input. +func skipOne(a []byte, n, i int) int { + return skipOneD(a, n, i, 0) +} + +func skipOneD(a []byte, n, i, depth int) int { + if depth > MaxDepth { + return 0 + } + if i >= n { + return 0 + } + b := a[i] + i++ + + if b <= 0x7f { + return i + } + if b >= 0xe0 { + return i + } + + switch b { + case mpNil, mpFalse, mpTrue: + return i + case mpFloat32: + if i+4 <= n { + return i + 4 + } + return 0 + case mpFloat64, mpInt64, mpUint64: + if i+8 <= n { + return i + 8 + } + return 0 + case mpUint8, mpInt8: + if i+1 <= n { + return i + 1 + } + return 0 + case mpUint16, mpInt16: + if i+2 <= n { + return i + 2 + } + return 0 + case mpUint32, mpInt32: + if i+4 <= n { + return i + 4 + } + return 0 + case mpBin8, mpStr8: + if i+1 > n { + return 0 + } + sz := int(a[i]) + i++ + if sz <= n-i { + return i + sz + } + return 0 + case mpBin16, mpStr16: + if i+2 > n { + return 0 + } + sz := int(read16(a, i)) + i += 2 + if sz <= n-i { + return i + sz + } + return 0 + case mpBin32, mpStr32: + if i+4 > n { + return 0 + } + sz := int(read32(a, i)) + i += 4 + if sz <= n-i { + return i + sz + } + return 0 + case mpFixext1: + if i+2 <= n { + return i + 2 + } + return 0 + case mpFixext2: + if i+3 <= n { + return i + 3 + } + return 0 + case mpFixext4: + if i+5 <= n { + return i + 5 + } + return 0 + case mpFixext8: + if i+9 <= n { + return i + 9 + } + return 0 + case mpFixext16: + if i+17 <= n { + return i + 17 + } + return 0 + case mpExt8: + if i+2 > n { + return 0 + } + sz := int(a[i]) + i += 2 + if sz <= n-i { + return i + sz + } + return 0 + case mpExt16: + if i+3 > n { + return 0 + } + sz := int(read16(a, i)) + i += 3 + if sz <= n-i { + return i + sz + } + return 0 + case mpExt32: + if i+5 > n { + return 0 + } + sz := int(read32(a, i)) + i += 5 + if sz <= n-i { + return i + sz + } + return 0 + } + + // fixstr + if b >= 0xa0 && b <= 0xbf { + sz := int(b & 0x1f) + if sz <= n-i { + return i + sz + } + return 0 + } + + // fixarray + if b >= 0x90 && b <= 0x9f { + count := int(b & 0x0f) + for j := 0; j < count; j++ { + i = skipOneD(a, n, i, depth+1) + if i == 0 { + return 0 + } + } + return i + } + + // fixmap + if b >= 0x80 && b <= 0x8f { + count := int(b & 0x0f) + for j := 0; j < count; j++ { + i = skipOneD(a, n, i, depth+1) + if i == 0 { + return 0 + } + i = skipOneD(a, n, i, depth+1) + if i == 0 { + return 0 + } + } + return i + } + + // array16/32 + if b == mpArray16 || b == mpArray32 { + var count int + if b == mpArray16 { + if i+2 > n { + return 0 + } + count = int(read16(a, i)) + i += 2 + } else { + if i+4 > n { + return 0 + } + count = int(read32(a, i)) + i += 4 + } + for j := 0; j < count; j++ { + i = skipOneD(a, n, i, depth+1) + if i == 0 { + return 0 + } + } + return i + } + + // map16/32 + if b == mpMap16 || b == mpMap32 { + var count int + if b == mpMap16 { + if i+2 > n { + return 0 + } + count = int(read16(a, i)) + i += 2 + } else { + if i+4 > n { + return 0 + } + count = int(read32(a, i)) + i += 4 + } + for j := 0; j < count; j++ { + i = skipOneD(a, n, i, depth+1) + if i == 0 { + return 0 + } + i = skipOneD(a, n, i, depth+1) + if i == 0 { + return 0 + } + } + return i + } + + return 0 +} diff --git a/go/go.mod b/go/go.mod new file mode 100644 index 0000000..a7e3f56 --- /dev/null +++ b/go/go.mod @@ -0,0 +1,3 @@ +module github.com/khanaffan/sqlite-msgpack/go + +go 1.21 diff --git a/go/iterate.go b/go/iterate.go new file mode 100644 index 0000000..45c04dc --- /dev/null +++ b/go/iterate.go @@ -0,0 +1,193 @@ +package msgpackblob + +import "strconv" + +// EachRow is a single row yielded by an Iterator. +type EachRow struct { + Key string // map key ("" for arrays / tree rows) + Index int64 // array/pair index (flat iteration only) + Fullkey string // e.g. "$.users[0].name" + Path string // parent path + ID int // byte offset in the blob + Type Type // element type + Value Value // element value +} + +func keyStr(a []byte, n, i int) (string, bool) { + kb := a[i] + var klen, koff int + switch { + case kb >= 0xa0 && kb <= 0xbf: + klen, koff = int(kb&0x1f), i+1 + case kb == mpStr8 && i+2 <= n: + klen, koff = int(a[i+1]), i+2 + case kb == mpStr16 && i+3 <= n: + klen, koff = int(read16(a, i+1)), i+3 + case kb == mpStr32 && i+5 <= n: + klen, koff = int(read32(a, i+1)), i+5 + default: + return "", false + } + // Bounds guard (matches decode keyAt): reject a key length that runs past the + // end of the blob. eachIter then defers to skipOne, which fails and stops + // iteration — matching the C++ reference (no out-of-bounds read). + if klen > n-koff { + return "", false + } + return string(a[koff : koff+klen]), true +} + +func containerInfo(a []byte, n, i int) (isArr, isMap bool, count, dataOff int) { + b := a[i] + switch { + case b >= 0x90 && b <= 0x9f: + return true, false, int(b & 0x0f), i + 1 + case b == mpArray16 && i+3 <= n: + return true, false, int(read16(a, i+1)), i + 3 + case b == mpArray32 && i+5 <= n: + return true, false, int(read32(a, i+1)), i + 5 + case b >= 0x80 && b <= 0x8f: + return false, true, int(b & 0x0f), i + 1 + case b == mpMap16 && i+3 <= n: + return false, true, int(read16(a, i+1)), i + 3 + case b == mpMap32 && i+5 <= n: + return false, true, int(read32(a, i+1)), i + 5 + } + return false, false, 0, 0 +} + +func eachIter(a []byte, n, icont int, zbase string) []EachRow { + var rows []EachRow + if icont >= n { + return rows + } + isArr, isMap, count, dataOff := containerInfo(a, n, icont) + if !isArr && !isMap { + return rows + } + + remaining := 0 + if dataOff <= n { + remaining = n - dataOff + } + minBytes := 1 + if isMap { + minBytes = 2 + } + if count > remaining/minBytes+1 { + return rows + } + + cur := dataOff + for j := 0; j < count; j++ { + if cur >= n { + break + } + if isArr { + cEnd := skipOne(a, n, cur) + if cEnd == 0 { + break + } + rows = append(rows, EachRow{ + Index: int64(j), + Fullkey: zbase + "[" + strconv.Itoa(j) + "]", + Path: zbase, + ID: cur, + Type: getType(a, n, cur), + Value: decodeElement(a, n, cur, cEnd), + }) + cur = cEnd + } else { + ks, ok := keyStr(a, n, cur) + vOff := skipOne(a, n, cur) + if vOff == 0 { + break + } + pEnd := skipOne(a, n, vOff) + if pEnd == 0 { + break + } + key := ks + if !ok { + key = "?" + } + rows = append(rows, EachRow{ + Key: key, + Index: int64(j), + Fullkey: zbase + "." + key, + Path: zbase, + ID: vOff, + Type: getType(a, n, vOff), + Value: decodeElement(a, n, vOff, pEnd), + }) + cur = pEnd + } + } + return rows +} + +func treeWalk(a []byte, n, ioff int, zfull, zparPath string, depth int, rows *[]EachRow) { + if depth > MaxDepth || ioff >= n { + return + } + iend := skipOne(a, n, ioff) + if iend == 0 { + return + } + + *rows = append(*rows, EachRow{ + Fullkey: zfull, + Path: zparPath, + ID: ioff, + Type: getType(a, n, ioff), + Value: decodeElement(a, n, ioff, iend), + }) + + isArr, isMap, count, dataOff := containerInfo(a, n, ioff) + if !isArr && !isMap { + return + } + + remaining := 0 + if dataOff <= n { + remaining = n - dataOff + } + minBytes := 1 + if isMap { + minBytes = 2 + } + if count > remaining/minBytes+1 { + return + } + + cur := dataOff + for j := 0; j < count; j++ { + if cur >= n { + break + } + if isArr { + cEnd := skipOne(a, n, cur) + if cEnd == 0 { + break + } + treeWalk(a, n, cur, zfull+"["+strconv.Itoa(j)+"]", zfull, depth+1, rows) + cur = cEnd + } else { + ks, ok := keyStr(a, n, cur) + vOff := skipOne(a, n, cur) + if vOff == 0 { + break + } + pEnd := skipOne(a, n, vOff) + if pEnd == 0 { + break + } + key := ks + if !ok { + key = "?" + } + treeWalk(a, n, vOff, zfull+"."+key, zfull, depth+1, rows) + cur = pEnd + } + } +} diff --git a/go/iterator.go b/go/iterator.go new file mode 100644 index 0000000..8ae15b0 --- /dev/null +++ b/go/iterator.go @@ -0,0 +1,73 @@ +package msgpackblob + +// Iterator is a cursor over a container's children, supporting flat (each) and +// recursive (tree) modes — mirroring the SQLite extension's msgpack_each and +// msgpack_tree table-valued functions. +type Iterator struct { + blob Blob + base string + recursive bool + rows []EachRow + cursor int + populated bool +} + +// NewIterator creates an iterator over the container at path. When recursive is +// true it walks the whole subtree (tree); otherwise it yields direct children +// (each). +func NewIterator(blob Blob, path string, recursive bool) *Iterator { + if path == "" { + path = "$" + } + return &Iterator{blob: blob, base: path, recursive: recursive, cursor: -1} +} + +func (it *Iterator) populate() { + if it.populated { + return + } + it.populated = true + it.rows = nil + + a := it.blob.data + n := len(a) + if n == 0 { + return + } + + iroot := 0 + if it.base != "$" { + rc, istart, _ := lookup(a, n, 0, it.base) + if rc != rcOK { + return + } + iroot = istart + } + + if it.recursive { + treeWalk(a, n, iroot, it.base, it.base, 0, &it.rows) + } else { + it.rows = eachIter(a, n, iroot, it.base) + } +} + +// Next advances the cursor; returns false past the last row. +func (it *Iterator) Next() bool { + it.populate() + it.cursor++ + return it.cursor < len(it.rows) +} + +// Current returns the row at the cursor (valid after Next returns true). +func (it *Iterator) Current() EachRow { + return it.rows[it.cursor] +} + +// Reset rewinds the cursor to before the first row. +func (it *Iterator) Reset() { it.cursor = -1 } + +// Rows returns all rows as a slice. +func (it *Iterator) Rows() []EachRow { + it.populate() + return it.rows +} diff --git a/go/json.go b/go/json.go new file mode 100644 index 0000000..d98e6f4 --- /dev/null +++ b/go/json.go @@ -0,0 +1,580 @@ +package msgpackblob + +import ( + "math" + "strconv" + "strings" +) + +const hexDigits = "0123456789abcdef" + +// ── float formatting (C printf "%.

g") ──────────────────────────── +// Go's strconv 'g' formatting with an explicit precision matches C's "%g" +// digit-for-digit (verified against the reference across 35k+ values). +func fmtDouble(d float64) string { + s := strconv.FormatFloat(d, 'g', 17, 64) + if !strings.ContainsAny(s, ".eE") { + // C re-formats integer-valued doubles with "%.1f". + s = strconv.FormatFloat(d, 'f', 1, 64) + } + return s +} + +func fmtFloat32(f float32) string { + return strconv.FormatFloat(float64(f), 'g', 7, 64) +} + +// ── JSON output ───────────────────────────────────────────────────── +func escapeStr(out *[]byte, s []byte) { + *out = append(*out, '"') + start := 0 + for j := 0; j < len(s); j++ { + c := s[j] + if c >= 0x20 && c != '"' && c != '\\' { + continue + } + if j > start { + *out = append(*out, s[start:j]...) + } + switch c { + case '"': + *out = append(*out, '\\', '"') + case '\\': + *out = append(*out, '\\', '\\') + case '\n': + *out = append(*out, '\\', 'n') + case '\r': + *out = append(*out, '\\', 'r') + case '\t': + *out = append(*out, '\\', 't') + default: + *out = append(*out, '\\', 'u', '0', '0', + hexDigits[c>>4], hexDigits[c&0xf]) + } + start = j + 1 + } + if len(s) > start { + *out = append(*out, s[start:]...) + } + *out = append(*out, '"') +} + +func newline(out *[]byte, depth, indentW int) { + *out = append(*out, '\n') + for k := 0; k < depth*indentW; k++ { + *out = append(*out, ' ') + } +} + +func toJSONAt(out *[]byte, a []byte, n, i int, pretty bool, depth, indentW int) { + if i >= n || depth > MaxDepth { + *out = append(*out, "null"...) + return + } + b := a[i] + + if b == mpNil { + *out = append(*out, "null"...) + return + } + if b == mpFalse { + *out = append(*out, "false"...) + return + } + if b == mpTrue { + *out = append(*out, "true"...) + return + } + if b <= 0x7f { + *out = strconv.AppendInt(*out, int64(b), 10) + return + } + if b >= 0xe0 { + *out = strconv.AppendInt(*out, int64(int8(b)), 10) + return + } + + switch b { + case mpUint8: + if i+2 <= n { + *out = strconv.AppendUint(*out, uint64(a[i+1]), 10) + return + } + case mpUint16: + if i+3 <= n { + *out = strconv.AppendUint(*out, uint64(read16(a, i+1)), 10) + return + } + case mpUint32: + if i+5 <= n { + *out = strconv.AppendUint(*out, uint64(read32(a, i+1)), 10) + return + } + case mpUint64: + if i+9 <= n { + *out = strconv.AppendUint(*out, read64(a, i+1), 10) + return + } + case mpInt8: + if i+2 <= n { + *out = strconv.AppendInt(*out, int64(int8(a[i+1])), 10) + return + } + case mpInt16: + if i+3 <= n { + *out = strconv.AppendInt(*out, int64(int16(read16(a, i+1))), 10) + return + } + case mpInt32: + if i+5 <= n { + *out = strconv.AppendInt(*out, int64(int32(read32(a, i+1))), 10) + return + } + case mpInt64: + if i+9 <= n { + *out = strconv.AppendInt(*out, int64(read64(a, i+1)), 10) + return + } + case mpFloat32: + if i+5 <= n { + f := math.Float32frombits(read32(a, i+1)) + if math.IsInf(float64(f), 0) || math.IsNaN(float64(f)) { + *out = append(*out, "null"...) + return + } + *out = append(*out, fmtFloat32(f)...) + return + } + case mpFloat64: + if i+9 <= n { + d := math.Float64frombits(read64(a, i+1)) + if math.IsInf(d, 0) || math.IsNaN(d) { + *out = append(*out, "null"...) + return + } + *out = append(*out, fmtDouble(d)...) + return + } + } + + // str + var slen, soff int + switch { + case b >= 0xa0 && b <= 0xbf: + slen, soff = int(b&0x1f), i+1 + case b == mpStr8 && i+2 <= n: + slen, soff = int(a[i+1]), i+2 + case b == mpStr16 && i+3 <= n: + slen, soff = int(read16(a, i+1)), i+3 + case b == mpStr32 && i+5 <= n: + slen, soff = int(read32(a, i+1)), i+5 + } + if soff != 0 { + if slen > n-soff { + slen = n - soff + } + escapeStr(out, a[soff:soff+slen]) + return + } + + // bin → hex string + var blen, boff int + switch { + case b == mpBin8 && i+2 <= n: + blen, boff = int(a[i+1]), i+2 + case b == mpBin16 && i+3 <= n: + blen, boff = int(read16(a, i+1)), i+3 + case b == mpBin32 && i+5 <= n: + blen, boff = int(read32(a, i+1)), i+5 + } + if boff != 0 { + if blen > n-boff { + blen = n - boff + } + *out = append(*out, '"') + for j := 0; j < blen; j++ { + by := a[boff+j] + *out = append(*out, hexDigits[by>>4], hexDigits[by&0xf]) + } + *out = append(*out, '"') + return + } + + // array + isArr := false + count := 0 + dataOff := 0 + switch { + case b >= 0x90 && b <= 0x9f: + isArr, count, dataOff = true, int(b&0x0f), i+1 + case b == mpArray16 && i+3 <= n: + isArr, count, dataOff = true, int(read16(a, i+1)), i+3 + case b == mpArray32 && i+5 <= n: + isArr, count, dataOff = true, int(read32(a, i+1)), i+5 + } + if isArr { + cur := dataOff + *out = append(*out, '[') + for j := 0; j < count; j++ { + if cur >= n { + break + } + nxt := skipOne(a, n, cur) + if j > 0 { + *out = append(*out, ',') + } + if pretty { + newline(out, depth+1, indentW) + } + toJSONAt(out, a, n, cur, pretty, depth+1, indentW) + if nxt != 0 { + cur = nxt + } else { + cur = n + } + } + if pretty && count > 0 { + newline(out, depth, indentW) + } + *out = append(*out, ']') + return + } + + // map + isMap := false + count = 0 + dataOff = 0 + switch { + case b >= 0x80 && b <= 0x8f: + isMap, count, dataOff = true, int(b&0x0f), i+1 + case b == mpMap16 && i+3 <= n: + isMap, count, dataOff = true, int(read16(a, i+1)), i+3 + case b == mpMap32 && i+5 <= n: + isMap, count, dataOff = true, int(read32(a, i+1)), i+5 + } + if isMap { + cur := dataOff + *out = append(*out, '{') + for j := 0; j < count; j++ { + if cur >= n { + break + } + valOff := skipOne(a, n, cur) + pairEnd := 0 + if valOff != 0 { + pairEnd = skipOne(a, n, valOff) + } + if j > 0 { + *out = append(*out, ',') + } + if pretty { + newline(out, depth+1, indentW) + } + toJSONAt(out, a, n, cur, pretty, depth+1, indentW) + *out = append(*out, ':') + if pretty { + *out = append(*out, ' ') + } + vo := valOff + if vo == 0 { + vo = n + } + toJSONAt(out, a, n, vo, pretty, depth+1, indentW) + if pairEnd != 0 { + cur = pairEnd + } else { + cur = n + } + } + if pretty && count > 0 { + newline(out, depth, indentW) + } + *out = append(*out, '}') + return + } + + // ext / unknown → null + *out = append(*out, "null"...) +} + +func toJSONBytes(a []byte, n int, pretty bool, indent int) []byte { + out := make([]byte, 0, n*2) + toJSONAt(&out, a, n, 0, pretty, 0, indent) + return out +} + +// ── JSON parser → msgpack ─────────────────────────────────────────── +type jsonParser struct { + z []byte + n int + i int +} + +func (p *jsonParser) skipWS() { + for p.i < p.n { + switch p.z[p.i] { + case ' ', '\t', '\n', '\r': + p.i++ + default: + return + } + } +} + +func hex4(z []byte, off int) int { + v := 0 + for j := 0; j < 4; j++ { + c := z[off+j] + var h int + switch { + case c >= '0' && c <= '9': + h = int(c - '0') + case c >= 'a' && c <= 'f': + h = int(c-'a') + 10 + case c >= 'A' && c <= 'F': + h = int(c-'A') + 10 + default: + return -1 + } + v = (v << 4) | h + } + return v +} + +func cpToUTF8(out *[]byte, cp int) { + switch { + case cp < 0x80: + *out = append(*out, byte(cp)) + case cp < 0x800: + *out = append(*out, byte(0xc0|cp>>6), byte(0x80|cp&0x3f)) + case cp < 0x10000: + *out = append(*out, byte(0xe0|cp>>12), byte(0x80|(cp>>6)&0x3f), byte(0x80|cp&0x3f)) + default: + *out = append(*out, byte(0xf0|cp>>18), byte(0x80|(cp>>12)&0x3f), + byte(0x80|(cp>>6)&0x3f), byte(0x80|cp&0x3f)) + } +} + +func (p *jsonParser) parseString(out *[]byte) int { + var sb []byte + p.i++ // skip " + for p.i < p.n { + c := p.z[p.i] + if c == '"' { + p.i++ + break + } + if c == '\\' { + p.i++ + if p.i >= p.n { + return rcError + } + esc := p.z[p.i] + p.i++ + switch esc { + case '"': + sb = append(sb, '"') + case '\\': + sb = append(sb, '\\') + case '/': + sb = append(sb, '/') + case 'n': + sb = append(sb, '\n') + case 'r': + sb = append(sb, '\r') + case 't': + sb = append(sb, '\t') + case 'b': + sb = append(sb, 0x08) + case 'f': + sb = append(sb, 0x0c) + case 'u': + if p.i+4 > p.n { + return rcError + } + cp := hex4(p.z, p.i) + p.i += 4 + if cp < 0 { + return rcError + } + if cp >= 0xD800 && cp <= 0xDBFF && p.i+6 <= p.n && + p.z[p.i] == '\\' && p.z[p.i+1] == 'u' { + lo := hex4(p.z, p.i+2) + if lo >= 0xDC00 && lo <= 0xDFFF { + p.i += 6 + cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00) + } + } + cpToUTF8(&sb, cp) + default: + sb = append(sb, esc) + } + } else { + sb = append(sb, c) + p.i++ + } + } + encString(out, sb) + return rcOK +} + +func (p *jsonParser) parseNumber(out *[]byte) int { + start := p.i + isFloat := false + if p.i < p.n && p.z[p.i] == '-' { + p.i++ + } + for p.i < p.n && p.z[p.i] >= '0' && p.z[p.i] <= '9' { + p.i++ + } + if p.i < p.n && p.z[p.i] == '.' { + isFloat = true + p.i++ + for p.i < p.n && p.z[p.i] >= '0' && p.z[p.i] <= '9' { + p.i++ + } + } + if p.i < p.n && (p.z[p.i] == 'e' || p.z[p.i] == 'E') { + isFloat = true + p.i++ + if p.i < p.n && (p.z[p.i] == '+' || p.z[p.i] == '-') { + p.i++ + } + for p.i < p.n && p.z[p.i] >= '0' && p.z[p.i] <= '9' { + p.i++ + } + } + length := p.i - start + if length <= 0 || length >= 64 { + return rcError + } + text := string(p.z[start:p.i]) + + if isFloat { + d, _ := strconv.ParseFloat(text, 64) + encReal(out, d) + } else { + // strconv.ParseInt clamps to MaxInt64/MinInt64 on overflow, matching + // C strtoll saturation. + v, _ := strconv.ParseInt(text, 10, 64) + if v >= 0 { + encUnsigned(out, uint64(v)) + } else { + encInteger(out, v) + } + } + return rcOK +} + +func (p *jsonParser) parseArray(out *[]byte) int { + var tmp []byte + var count uint32 + p.i++ // skip [ + p.skipWS() + for p.i < p.n && p.z[p.i] != ']' { + if count > 0 { + p.skipWS() + if p.i >= p.n || p.z[p.i] != ',' { + return rcError + } + p.i++ + } + p.skipWS() + if p.parseValue(&tmp) != rcOK { + return rcError + } + count++ + p.skipWS() + } + if p.i >= p.n { + return rcError + } + p.i++ // skip ] + encArrayHeader(out, count) + *out = append(*out, tmp...) + return rcOK +} + +func (p *jsonParser) parseObject(out *[]byte) int { + var tmp []byte + var count uint32 + p.i++ // skip { + p.skipWS() + for p.i < p.n && p.z[p.i] != '}' { + if count > 0 { + p.skipWS() + if p.i >= p.n || p.z[p.i] != ',' { + return rcError + } + p.i++ + } + p.skipWS() + if p.i >= p.n || p.z[p.i] != '"' { + return rcError + } + if p.parseString(&tmp) != rcOK { + return rcError + } + p.skipWS() + if p.i >= p.n || p.z[p.i] != ':' { + return rcError + } + p.i++ + p.skipWS() + if p.parseValue(&tmp) != rcOK { + return rcError + } + count++ + p.skipWS() + } + if p.i >= p.n { + return rcError + } + p.i++ // skip } + encMapHeader(out, count) + *out = append(*out, tmp...) + return rcOK +} + +func (p *jsonParser) parseValue(out *[]byte) int { + p.skipWS() + if p.i >= p.n { + return rcError + } + c := p.z[p.i] + if c == 'n' && p.i+4 <= p.n && string(p.z[p.i:p.i+4]) == "null" { + p.i += 4 + *out = append(*out, mpNil) + return rcOK + } + if c == 't' && p.i+4 <= p.n && string(p.z[p.i:p.i+4]) == "true" { + p.i += 4 + *out = append(*out, mpTrue) + return rcOK + } + if c == 'f' && p.i+5 <= p.n && string(p.z[p.i:p.i+5]) == "false" { + p.i += 5 + *out = append(*out, mpFalse) + return rcOK + } + switch { + case c == '"': + return p.parseString(out) + case c == '[': + return p.parseArray(out) + case c == '{': + return p.parseObject(out) + case c == '-' || (c >= '0' && c <= '9'): + return p.parseNumber(out) + } + return rcError +} + +func fromJSON(json []byte) []byte { + p := jsonParser{z: json, n: len(json)} + var out []byte + if p.parseValue(&out) != rcOK { + return nil + } + return out +} diff --git a/go/mutate.go b/go/mutate.go new file mode 100644 index 0000000..dc6b559 --- /dev/null +++ b/go/mutate.go @@ -0,0 +1,422 @@ +package msgpackblob + +const ( + editSet = 0 + editInsert = 1 + editReplace = 2 + editRemove = 3 + editArrayIns = 4 +) + +func mapKey(a []byte, n, i int) ([]byte, bool) { + kb := a[i] + var klen, koff int + switch { + case kb >= 0xa0 && kb <= 0xbf: + klen, koff = int(kb&0x1f), i+1 + case kb == mpStr8 && i+2 <= n: + klen, koff = int(a[i+1]), i+2 + case kb == mpStr16 && i+3 <= n: + klen, koff = int(read16(a, i+1)), i+3 + case kb == mpStr32 && i+5 <= n: + klen, koff = int(read32(a, i+1)), i+5 + default: + return nil, false + } + // Bounds guard (matches decode keyAt): a truncated key length must not slice + // past the end of the blob — the C++ reference defers to skipOne, which fails + // the pair and aborts the edit gracefully. + if klen > n-koff { + return nil, false + } + return a[koff : koff+klen], true +} + +func editMap(out *[]byte, a []byte, n, icur int, zkey []byte, zpath string, pi int, newBin []byte, mode int) int { + if icur >= n { + return rcError + } + b := a[icur] + var count, dataOff int + if b >= 0x80 && b <= 0x8f { + count, dataOff = int(b&0x0f), icur+1 + } else if b == mpMap16 { + if icur+3 > n { + return rcError + } + count, dataOff = int(read16(a, icur+1)), icur+3 + } else if b == mpMap32 { + if icur+5 > n { + return rcError + } + count, dataOff = int(read32(a, icur+1)), icur+5 + } else { + if mode == editReplace || mode == editRemove { + iend := skipOne(a, n, icur) + if iend != 0 { + *out = append(*out, a[icur:iend]...) + } + return rcOK + } + return rcError + } + + newCount := uint32(count) + var tmp []byte + cur2 := dataOff + foundKey := false + + for j := 0; j < count; j++ { + if cur2 >= n { + return rcError + } + kstr, kok := mapKey(a, n, cur2) + valOff := skipOne(a, n, cur2) + if valOff == 0 { + return rcError + } + pairEnd := skipOne(a, n, valOff) + if pairEnd == 0 { + return rcError + } + + isMatch := kok && bytesEqual(kstr, zkey) + + if isMatch { + foundKey = true + if mode == editInsert { + tmp = append(tmp, a[cur2:pairEnd]...) + } else { + var vbuf []byte + rc, skip := editStep(&vbuf, a, n, valOff, zpath, pi, newBin, mode) + if rc != rcOK { + return rc + } + if skip { + newCount-- + } else { + tmp = append(tmp, a[cur2:valOff]...) + tmp = append(tmp, vbuf...) + } + } + } else { + tmp = append(tmp, a[cur2:pairEnd]...) + } + cur2 = pairEnd + } + + if !foundKey { + if mode == editSet || mode == editInsert { + kind, _, _, _ := pathStep(zpath, pi) + if kind != stepEnd { + iend := skipOne(a, n, icur) + if iend != 0 { + *out = append(*out, a[icur:iend]...) + } + return rcOK + } + encString(&tmp, zkey) + tmp = append(tmp, newBin...) + newCount++ + } else { + iend := skipOne(a, n, icur) + if iend != 0 { + *out = append(*out, a[icur:iend]...) + } + return rcOK + } + } + + encMapHeader(out, newCount) + *out = append(*out, tmp...) + return rcOK +} + +func editArray(out *[]byte, a []byte, n, icur int, stepIdx int64, zpath string, pi int, newBin []byte, mode int) int { + if icur >= n { + return rcError + } + b := a[icur] + var count, dataOff int + if b >= 0x90 && b <= 0x9f { + count, dataOff = int(b&0x0f), icur+1 + } else if b == mpArray16 { + if icur+3 > n { + return rcError + } + count, dataOff = int(read16(a, icur+1)), icur+3 + } else if b == mpArray32 { + if icur+5 > n { + return rcError + } + count, dataOff = int(read32(a, icur+1)), icur+5 + } else { + if mode == editReplace || mode == editRemove { + iend := skipOne(a, n, icur) + if iend != 0 { + *out = append(*out, a[icur:iend]...) + } + return rcOK + } + return rcError + } + + newCount := uint32(count) + var tmp []byte + cur2 := dataOff + foundIt := false + + for j := 0; j < count; j++ { + eEnd := skipOne(a, n, cur2) + if eEnd == 0 { + return rcError + } + + if int64(j) == stepIdx { + foundIt = true + if mode == editArrayIns { + tmp = append(tmp, newBin...) + tmp = append(tmp, a[cur2:eEnd]...) + newCount++ + } else if mode == editInsert { + tmp = append(tmp, a[cur2:eEnd]...) + } else { + var ebuf []byte + rc, skip := editStep(&ebuf, a, n, cur2, zpath, pi, newBin, mode) + if rc != rcOK { + return rc + } + if skip { + newCount-- + } else { + tmp = append(tmp, ebuf...) + } + } + } else { + tmp = append(tmp, a[cur2:eEnd]...) + } + cur2 = eEnd + } + + if !foundIt { + if mode == editArrayIns { + tmp = append(tmp, newBin...) + newCount++ + } else if (mode == editSet || mode == editInsert) && stepIdx == int64(count) { + tmp = append(tmp, newBin...) + newCount++ + } else if mode == editReplace || mode == editRemove { + iend := skipOne(a, n, icur) + if iend != 0 { + *out = append(*out, a[icur:iend]...) + } + return rcOK + } else { + return rcNotFound + } + } + + encArrayHeader(out, newCount) + *out = append(*out, tmp...) + return rcOK +} + +// editStep returns (rc, skip). +func editStep(out *[]byte, a []byte, n, icur int, zpath string, pi int, newBin []byte, mode int) (int, bool) { + kind, npi, key, stepIdx := pathStep(zpath, pi) + + switch kind { + case stepEnd: + if mode == editRemove { + return rcOK, true + } + if mode == editArrayIns { + return rcError, false + } + if mode == editInsert { + iend := skipOne(a, n, icur) + if iend != 0 { + *out = append(*out, a[icur:iend]...) + } + return rcOK, false + } + *out = append(*out, newBin...) + return rcOK, false + case stepErr: + return rcError, false + case stepKey: + return editMap(out, a, n, icur, []byte(key), zpath, npi, newBin, mode), false + default: // stepIndex + return editArray(out, a, n, icur, stepIdx, zpath, npi, newBin, mode), false + } +} + +// applyEdit applies a path-targeted edit; returns (rc, outBytes). +func applyEdit(a []byte, n int, zpath string, newBin []byte, mode int) (int, []byte) { + if len(zpath) == 0 || zpath[0] != '$' { + return rcError, nil + } + var out []byte + rc, _ := editStep(&out, a, n, 0, zpath, 1, newBin, mode) + return rc, out +} + +// ── merge patch (RFC 7386) ────────────────────────────────────────── + +// mergePatch applies an RFC 7386 merge patch; returns (rc, outBytes). +func mergePatch(a []byte, n, ia int, p []byte, np, ip int) (int, []byte) { + var out []byte + rc := mergePatchInto(&out, a, n, ia, p, np, ip, 0) + return rc, out +} + +type patchEntry struct { + keyOff, valOff, pairEnd int + matched bool +} + +func mergePatchInto(out *[]byte, a []byte, n, ia int, p []byte, np, ip, depth int) int { + if ip >= np { + return rcError + } + if depth > MaxDepth { + return rcError + } + pb := p[ip] + + if pb == mpNil { + *out = append(*out, mpNil) + return rcOK + } + + pIsMap := (pb >= 0x80 && pb <= 0x8f) || pb == mpMap16 || pb == mpMap32 + if !pIsMap { + pEnd := skipOne(p, np, ip) + if pEnd != 0 { + *out = append(*out, p[ip:pEnd]...) + } + return rcOK + } + + var ab byte + if ia < n { + ab = a[ia] + } + aIsMap := (ab >= 0x80 && ab <= 0x8f) || ab == mpMap16 || ab == mpMap32 + + var pCount, pDataOff int + if pb >= 0x80 && pb <= 0x8f { + pCount, pDataOff = int(pb&0x0f), ip+1 + } else if pb == mpMap16 { + if ip+3 > np { + return rcError + } + pCount, pDataOff = int(read16(p, ip+1)), ip+3 + } else { + if ip+5 > np { + return rcError + } + pCount, pDataOff = int(read32(p, ip+1)), ip+5 + } + + aCount, aDataOff := 0, 0 + if aIsMap { + if ab >= 0x80 && ab <= 0x8f { + aCount, aDataOff = int(ab&0x0f), ia+1 + } else if ab == mpMap16 { + if ia+3 > n { + aIsMap = false + } else { + aCount, aDataOff = int(read16(a, ia+1)), ia+3 + } + } else if ia+5 > n { + aIsMap = false + } else { + aCount, aDataOff = int(read32(a, ia+1)), ia+5 + } + } + + // Pre-scan patch keys. + if pCount > (np-pDataOff)/2+1 { + return rcError + } + pIdx := make([]patchEntry, 0, pCount) + pc2 := pDataOff + for k := 0; k < pCount; k++ { + if pc2 >= np { + return rcError + } + valOff := skipOne(p, np, pc2) + if valOff == 0 { + return rcError + } + pairEnd := skipOne(p, np, valOff) + if pairEnd == 0 { + return rcError + } + pIdx = append(pIdx, patchEntry{keyOff: pc2, valOff: valOff, pairEnd: pairEnd}) + pc2 = pairEnd + } + + var tmp []byte + var newCount uint32 + + if aIsMap { + ac := aDataOff + for j := 0; j < aCount; j++ { + if ac >= n { + return rcError + } + kstr, kok := mapKey(a, n, ac) + aValOff := skipOne(a, n, ac) + if aValOff == 0 { + return rcError + } + aPairEnd := skipOne(a, n, aValOff) + if aPairEnd == 0 { + return rcError + } + + foundInPatch := false + patchIsNil := false + pMatchVal := 0 + for k := range pIdx { + pkey, pok := mapKey(p, np, pIdx[k].keyOff) + if pok && kok && bytesEqual(pkey, kstr) { + foundInPatch = true + pMatchVal = pIdx[k].valOff + patchIsNil = pIdx[k].valOff < np && p[pIdx[k].valOff] == mpNil + pIdx[k].matched = true + break + } + } + + if foundInPatch && patchIsNil { + // drop + } else if foundInPatch { + var mb []byte + mrc := mergePatchInto(&mb, a, n, aValOff, p, np, pMatchVal, depth+1) + if mrc == rcOK { + tmp = append(tmp, a[ac:aValOff]...) + tmp = append(tmp, mb...) + newCount++ + } + } else { + tmp = append(tmp, a[ac:aPairEnd]...) + newCount++ + } + ac = aPairEnd + } + } + + for k := range pIdx { + if !pIdx[k].matched && pIdx[k].valOff < np && p[pIdx[k].valOff] != mpNil { + tmp = append(tmp, p[pIdx[k].keyOff:pIdx[k].pairEnd]...) + newCount++ + } + } + + encMapHeader(out, newCount) + *out = append(*out, tmp...) + return rcOK +} diff --git a/go/value.go b/go/value.go new file mode 100644 index 0000000..c6f3159 --- /dev/null +++ b/go/value.go @@ -0,0 +1,234 @@ +package msgpackblob + +// Type is the semantic type of a MessagePack element. +type Type int + +// Type constants. +const ( + TypeNil Type = iota + TypeTrue + TypeFalse + TypeInteger + TypeReal + TypeFloat32 + TypeString + TypeBinary + TypeArray + TypeMap + TypeExt + TypeTimestamp +) + +// String returns the human-readable label (matches the C++ type_str). +func (t Type) String() string { + switch t { + case TypeNil: + return "null" + case TypeTrue: + return "true" + case TypeFalse: + return "false" + case TypeInteger: + return "integer" + case TypeReal: + return "real" + case TypeFloat32: + return "float32" + case TypeString: + return "text" + case TypeBinary: + return "binary" + case TypeArray: + return "array" + case TypeMap: + return "map" + case TypeExt: + return "ext" + case TypeTimestamp: + return "timestamp" + } + return "null" +} + +// TypeStr returns the label for t (`"text"`, `"integer"`, …). +func TypeStr(t Type) string { + return t.String() +} + +// IntWidth is an integer encoding-width hint (forces a specific wire format). +type IntWidth int + +// IntWidth constants. +const ( + WidthAuto IntWidth = iota + WidthInt8 + WidthInt16 + WidthInt32 + WidthInt64 + WidthUint8 + WidthUint16 + WidthUint32 + WidthUint64 +) + +// Value is a decoded scalar or sub-blob value. Integer payloads are stored as +// raw 64-bit bits so the full signed/unsigned range round-trips exactly. +type Value struct { + ty Type + bits uint64 // integer bits / timestamp seconds + float float64 + bytes []byte // string / binary / ext payload (no header) + extType int8 + tsNsec uint32 + intWidth IntWidth +} + +// ── accessors ───────────────────────────────────────────────────────── +func (v Value) Type() Type { return v.ty } +func (v Value) IsNil() bool { return v.ty == TypeNil } +func (v Value) AsBool() bool { return v.ty == TypeTrue } + +func (v Value) AsInt64() int64 { + switch v.ty { + case TypeInteger, TypeTimestamp: + return int64(v.bits) + case TypeReal, TypeFloat32: + return int64(v.float) + case TypeTrue: + return 1 + } + return 0 +} + +func (v Value) AsUint64() uint64 { + if v.ty == TypeInteger { + return v.bits + } + return 0 +} + +func (v Value) AsFloat64() float64 { + switch v.ty { + case TypeReal, TypeFloat32: + return v.float + case TypeInteger: + return float64(v.AsInt64()) + } + return 0 +} + +func (v Value) AsFloat32() float32 { + switch v.ty { + case TypeFloat32, TypeReal: + return float32(v.float) + } + return 0 +} + +// AsString returns the string payload as a Go string (byte-preserving — Go +// strings hold arbitrary bytes). +func (v Value) AsString() string { + if v.ty == TypeString { + return string(v.bytes) + } + return "" +} + +// AsBytes returns the raw string payload bytes. +func (v Value) AsBytes() []byte { + if v.ty == TypeString { + return v.bytes + } + return nil +} + +// BlobData returns the Binary/Ext payload (no header), or raw bytes for +// container values. +func (v Value) BlobData() []byte { return v.bytes } +func (v Value) BlobSize() int { return len(v.bytes) } +func (v Value) ExtType() int8 { return v.extType } + +func (v Value) TimestampSeconds() int64 { + if v.ty == TypeTimestamp { + return int64(v.bits) + } + return 0 +} + +func (v Value) TimestampNanoseconds() uint32 { + if v.ty == TypeTimestamp { + return v.tsNsec + } + return 0 +} + +func (v Value) IntWidth() IntWidth { return v.intWidth } + +// ── constructors ────────────────────────────────────────────────────── + +// Nil returns a nil Value. +func Nil() Value { return Value{ty: TypeNil} } + +// Bool returns a boolean Value. +func Bool(b bool) Value { + if b { + return Value{ty: TypeTrue} + } + return Value{ty: TypeFalse} +} + +// Int returns a compact-encoded integer Value. +func Int(x int64) Value { return Value{ty: TypeInteger, bits: uint64(x)} } + +// Uint returns a compact-encoded unsigned integer Value. +func Uint(x uint64) Value { + v := Value{ty: TypeInteger, bits: x} + if x > uint64(^uint64(0)>>1) { + v.intWidth = WidthUint64 + } + return v +} + +// Real returns a float64 Value. +func Real(d float64) Value { return Value{ty: TypeReal, float: d} } + +// Real32 returns a float32 Value. +func Real32(f float32) Value { return Value{ty: TypeFloat32, float: float64(f)} } + +// Str returns a string Value. +func Str(s string) Value { return Value{ty: TypeString, bytes: []byte(s)} } + +// StrBytes returns a string Value from raw bytes (may be non-UTF-8). +func StrBytes(b []byte) Value { return Value{ty: TypeString, bytes: append([]byte(nil), b...)} } + +// Bin returns a binary Value. +func Bin(data []byte) Value { return Value{ty: TypeBinary, bytes: append([]byte(nil), data...)} } + +// Ext returns an ext Value. +func Ext(typeCode int8, data []byte) Value { + return Value{ty: TypeExt, extType: typeCode, bytes: append([]byte(nil), data...)} +} + +// Timestamp returns a timestamp Value (nanoseconds = 0). +func Timestamp(seconds int64) Value { + return Value{ty: TypeTimestamp, bits: uint64(seconds)} +} + +// TimestampNs returns a timestamp Value with nanoseconds. +func TimestampNs(seconds int64, nanoseconds uint32) Value { + return Value{ty: TypeTimestamp, bits: uint64(seconds), tsNsec: nanoseconds} +} + +func fixedInt(width IntWidth, bits uint64) Value { + return Value{ty: TypeInteger, bits: bits, intWidth: width} +} + +// Fixed-width integer constructors (force a specific wire encoding). +func Int8(x int8) Value { return fixedInt(WidthInt8, uint64(int64(x))) } +func Int16(x int16) Value { return fixedInt(WidthInt16, uint64(int64(x))) } +func Int32(x int32) Value { return fixedInt(WidthInt32, uint64(int64(x))) } +func Int64(x int64) Value { return fixedInt(WidthInt64, uint64(x)) } +func Uint8(x uint8) Value { return fixedInt(WidthUint8, uint64(x)) } +func Uint16(x uint16) Value { return fixedInt(WidthUint16, uint64(x)) } +func Uint32(x uint32) Value { return fixedInt(WidthUint32, uint64(x)) } +func Uint64(x uint64) Value { return fixedInt(WidthUint64, x) } diff --git a/go/vectors_test.go b/go/vectors_test.go new file mode 100644 index 0000000..5492164 --- /dev/null +++ b/go/vectors_test.go @@ -0,0 +1,272 @@ +package msgpackblob_test + +// Replay the shared cross-language vectors (tests/vectors/blob_vectors.json). +// Generated from the C++ reference implementation, so passing them proves the +// Go port is byte-identical. + +import ( + "encoding/hex" + "encoding/json" + "os" + "strconv" + "testing" + + mb "github.com/khanaffan/sqlite-msgpack/go" +) + +type specT struct { + K string `json:"k"` + V json.RawMessage `json:"v"` + Hex string `json:"hex"` + Type int `json:"type"` + Sec string `json:"sec"` + Nsec int64 `json:"nsec"` + JSON string `json:"json"` +} + +type iterRow struct { + Key *string `json:"key"` + Index *int64 `json:"index"` + Fullkey string `json:"fullkey"` + Path string `json:"path"` + ID int `json:"id"` + Type string `json:"type"` +} + +type vectorsT struct { + FromJSON []struct { + JSON string `json:"json"` + Hex string `json:"hex"` + } `json:"from_json"` + ToJSON []struct { + Hex string `json:"hex"` + JSON string `json:"json"` + } `json:"to_json"` + ToJSONPretty []struct { + Hex string `json:"hex"` + Indent int `json:"indent"` + JSON string `json:"json"` + } `json:"to_json_pretty"` + Typed []struct { + Spec specT `json:"spec"` + Hex string `json:"hex"` + } `json:"typed"` + Mutate []struct { + Base string `json:"base"` + Op string `json:"op"` + Path string `json:"path"` + Spec specT `json:"spec"` + Patch string `json:"patch"` + Hex string `json:"hex"` + } `json:"mutate"` + Extract []struct { + Base string `json:"base"` + Path string `json:"path"` + Type string `json:"type"` + Vjson string `json:"vjson"` + } `json:"extract"` + ArrayLength []struct { + Base string `json:"base"` + Path string `json:"path"` + Len int64 `json:"len"` + } `json:"array_length"` + Iterate []struct { + Base string `json:"base"` + Path string `json:"path"` + Recursive bool `json:"recursive"` + Rows []iterRow `json:"rows"` + } `json:"iterate"` +} + +func loadVectors(t *testing.T) vectorsT { + t.Helper() + raw, err := os.ReadFile("../tests/vectors/blob_vectors.json") + if err != nil { + t.Fatalf("read vectors: %v", err) + } + var v vectorsT + if err := json.Unmarshal(raw, &v); err != nil { + t.Fatalf("parse vectors: %v", err) + } + return v +} + +func mustHex(s string) []byte { + b, _ := hex.DecodeString(s) + return b +} + +func parseI64(s string) int64 { + v, err := strconv.ParseInt(s, 10, 64) + if err != nil { + panic(err) + } + return v +} + +func parseU64(s string) uint64 { + v, err := strconv.ParseUint(s, 10, 64) + if err != nil { + panic(err) + } + return v +} + +func buildValue(spec specT) mb.Value { + str := func() string { + var s string + if err := json.Unmarshal(spec.V, &s); err != nil { + panic(err) + } + return s + } + switch spec.K { + case "nil": + return mb.Nil() + case "bool": + var b bool + _ = json.Unmarshal(spec.V, &b) + return mb.Bool(b) + case "int": + return mb.Int(parseI64(str())) + case "uint": + return mb.Uint(parseU64(str())) + case "int8": + return mb.Int8(int8(parseI64(str()))) + case "int16": + return mb.Int16(int16(parseI64(str()))) + case "int32": + return mb.Int32(int32(parseI64(str()))) + case "int64": + return mb.Int64(parseI64(str())) + case "uint8": + return mb.Uint8(uint8(parseU64(str()))) + case "uint16": + return mb.Uint16(uint16(parseU64(str()))) + case "uint32": + return mb.Uint32(uint32(parseU64(str()))) + case "uint64": + return mb.Uint64(parseU64(str())) + case "real": + var f float64 + _ = json.Unmarshal(spec.V, &f) + return mb.Real(f) + case "real32": + var f float64 + _ = json.Unmarshal(spec.V, &f) + return mb.Real32(float32(f)) + case "str": + return mb.Str(str()) + case "binary": + return mb.Bin(mustHex(spec.Hex)) + case "ext": + return mb.Ext(int8(spec.Type), mustHex(spec.Hex)) + case "timestamp": + return mb.TimestampNs(parseI64(spec.Sec), uint32(spec.Nsec)) + } + panic("unknown spec kind " + spec.K) +} + +func TestFromJSON(t *testing.T) { + for _, c := range loadVectors(t).FromJSON { + if got := mb.FromJSON(c.JSON).Hex(); got != c.Hex { + t.Errorf("from_json %q: got %s want %s", c.JSON, got, c.Hex) + } + } +} + +func TestToJSON(t *testing.T) { + for _, c := range loadVectors(t).ToJSON { + if got := mb.NewBlob(mustHex(c.Hex)).ToJSON(); got != c.JSON { + t.Errorf("to_json %s: got %q want %q", c.Hex, got, c.JSON) + } + } +} + +func TestToJSONPretty(t *testing.T) { + for _, c := range loadVectors(t).ToJSONPretty { + if got := mb.NewBlob(mustHex(c.Hex)).ToJSONPretty(c.Indent); got != c.JSON { + t.Errorf("to_json_pretty %s: got %q want %q", c.Hex, got, c.JSON) + } + } +} + +func TestTyped(t *testing.T) { + for _, c := range loadVectors(t).Typed { + if got := mb.Quote(buildValue(c.Spec)).Hex(); got != c.Hex { + t.Errorf("typed %+v: got %s want %s", c.Spec, got, c.Hex) + } + } +} + +func TestMutate(t *testing.T) { + for _, c := range loadVectors(t).Mutate { + base := mb.FromJSON(c.Base) + var r mb.Blob + switch c.Op { + case "set": + r = base.Set(c.Path, buildValue(c.Spec)) + case "insert": + r = base.Insert(c.Path, buildValue(c.Spec)) + case "replace": + r = base.Replace(c.Path, buildValue(c.Spec)) + case "array_insert": + r = base.ArrayInsert(c.Path, buildValue(c.Spec)) + case "remove": + r = base.Remove(c.Path) + case "set_blob": + r = base.SetBlob(c.Path, mb.FromJSON(c.Spec.JSON)) + case "patch": + r = base.Patch(mb.FromJSON(c.Patch)) + default: + t.Fatalf("unknown op %s", c.Op) + } + if got := r.Hex(); got != c.Hex { + t.Errorf("mutate %s %s %s: got %s want %s", c.Op, c.Base, c.Path, got, c.Hex) + } + } +} + +func TestExtract(t *testing.T) { + for _, c := range loadVectors(t).Extract { + blob := mb.FromJSON(c.Base) + if got := blob.TypeStrAt(c.Path); got != c.Type { + t.Errorf("extract type %s %s: got %s want %s", c.Base, c.Path, got, c.Type) + } + if got := mb.Quote(blob.Extract(c.Path)).ToJSON(); got != c.Vjson { + t.Errorf("extract value %s %s: got %q want %q", c.Base, c.Path, got, c.Vjson) + } + } +} + +func TestArrayLength(t *testing.T) { + for _, c := range loadVectors(t).ArrayLength { + var got int64 + if c.Path == "$" { + got = mb.FromJSON(c.Base).ArrayLength() + } else { + got = mb.FromJSON(c.Base).ArrayLengthAt(c.Path) + } + if got != c.Len { + t.Errorf("array_length %s %s: got %d want %d", c.Base, c.Path, got, c.Len) + } + } +} + +func TestIterate(t *testing.T) { + for _, c := range loadVectors(t).Iterate { + rows := mb.NewIterator(mb.FromJSON(c.Base), c.Path, c.Recursive).Rows() + if len(rows) != len(c.Rows) { + t.Fatalf("iterate %s %s: got %d rows want %d", c.Base, c.Path, len(rows), len(c.Rows)) + } + for i, exp := range c.Rows { + got := rows[i] + if got.Fullkey != exp.Fullkey || got.Path != exp.Path || got.ID != exp.ID || got.Type.String() != exp.Type { + t.Errorf("iterate row %d: got %+v want %+v", i, got, exp) + } + if exp.Key != nil && (got.Key != *exp.Key || got.Index != *exp.Index) { + t.Errorf("iterate row %d key/index: got %q/%d want %q/%d", i, got.Key, got.Index, *exp.Key, *exp.Index) + } + } + } +} diff --git a/js/README.md b/js/README.md new file mode 100644 index 0000000..8db8e50 --- /dev/null +++ b/js/README.md @@ -0,0 +1,80 @@ +# msgpack-blob (TypeScript / JavaScript) + +A **pure-TypeScript** port of the standalone [C++ MessagePack Blob API](../cpp/README.md) +from [sqlite-msgpack](../README.md). It creates, queries, mutates and iterates +[MessagePack](https://msgpack.org/) binary blobs and produces **byte-identical** +output to the C++ library and the `sqlite-msgpack` SQLite extension, so blobs are +fully interchangeable across all three. + +- Zero runtime dependencies +- Same `Blob` / `Builder` / `Value` / `Iterator` API as the C++ library +- All msgpack primitive types: fixed-width ints, float32/64, ext, timestamp, binary +- Full signed/unsigned **64-bit integers** via `bigint` +- JSON conversion modelled on SQLite's JSON1 extension +- Ships ESM + `.d.ts` type declarations + +## Install + +```bash +cd js +npm install # dev only (TypeScript, for building/type-checking) +npm run build # emit dist/ (ESM + .d.ts) +``` + +## Quick start + +```ts +import { Blob, Builder, Value, Iterator } from "msgpack-blob"; + +// Build from JSON +const blob = Blob.fromJson('{"name":"Alice","scores":[95,87,91]}'); +blob.extract("$.name").asString(); // 'Alice' +blob.arrayLength("$.scores"); // 3 +blob.toJson(); // '{"name":"Alice","scores":[95,87,91]}' + +// Mutate (copy-on-write — original is unchanged) +const updated = blob.set("$.age", Value.integer(30)); +updated.toJson(); // '{"name":"Alice","scores":[95,87,91],"age":30}' + +// Build with the streaming Builder +const b = new Builder() + .mapHeader(2) + .string("temp").real32(23.5) + .string("ts").timestamp(1700000000, 500000000) + .build(); + +// Iterate (flat "each" or recursive "tree") +for (const row of new Iterator(blob, "$", true)) { + console.log(row.fullkey, row.type); +} + +// 64-bit integers use bigint +Builder.quote(Value.uint64(18446744073709551615n)).hex(); // 'cfffffffffffffffff' +``` + +## API overview + +| Class | Purpose | +|---|---| +| `Value` | A decoded scalar / sub-blob. Factories: `Value.integer`, `Value.real32`, `Value.string`, `Value.binary`, `Value.ext`, `Value.timestamp`, fixed-width `Value.int8`…`Value.uint64`. Integer accessors return `bigint`. | +| `Blob` | Owning byte buffer. `fromJson`, `toJson`, `toJsonPretty`, `extract`, `type`, `arrayLength`, `valid`, and copy-on-write `set` / `insert` / `replace` / `remove` / `arrayInsert` / `patch`. | +| `Builder` | Streaming encoder. Chainable `nil`/`boolean`/`integer`/`real`/`string`/`binary`/`ext`/`timestamp`/`arrayHeader`/`mapHeader`/`value`, plus fixed-width integer methods. `build()` → `Blob`. | +| `Iterator` | Cursor over container children (`each` / `tree`). Use `for…of` or the C++-style `next()`/`current()` cursor. | +| `Type`, `IntWidth`, `typeStr` | Type labels, integer-width hint, and label helper. | + +Integer factories and `Builder.integer` accept `number | bigint`; values are +stored as `bigint` so the full 64-bit range round-trips exactly. Paths use the +same `$`-rooted syntax as the SQLite extension: `$`, `$.key`, `$[0]`, +`$.users[0].email`. + +## Tests + +```bash +cd js +npm test # runs test/*.test.ts via Node's built-in test runner +``` + +Node ≥ 22 runs the TypeScript test files directly (type stripping); no build step +is required for testing. `test/vectors.test.ts` replays +[`tests/vectors/blob_vectors.json`](../tests/vectors/blob_vectors.json) — vectors +generated from the C++ reference implementation — to prove byte-identical output. diff --git a/js/package-lock.json b/js/package-lock.json new file mode 100644 index 0000000..fbf0154 --- /dev/null +++ b/js/package-lock.json @@ -0,0 +1,33 @@ +{ + "name": "msgpack-blob", + "version": "1.5.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "msgpack-blob", + "version": "1.5.0", + "license": "MIT", + "devDependencies": { + "typescript": "^5.7.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + } + } +} diff --git a/js/package.json b/js/package.json new file mode 100644 index 0000000..263ce81 --- /dev/null +++ b/js/package.json @@ -0,0 +1,35 @@ +{ + "name": "msgpack-blob", + "version": "1.5.0", + "description": "Pure-TypeScript MessagePack Blob API — byte-identical to the sqlite-msgpack C++ library", + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "files": ["dist", "src", "README.md"], + "scripts": { + "build": "tsc -p tsconfig.json", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "node --test \"test/*.test.ts\"", + "test:dist": "npm run build && node --test \"test/*.test.ts\"" + }, + "keywords": ["messagepack", "msgpack", "blob", "sqlite", "serialization"], + "license": "MIT", + "homepage": "https://github.com/khanaffan/sqlite-msgpack", + "repository": { + "type": "git", + "url": "https://github.com/khanaffan/sqlite-msgpack.git", + "directory": "js" + }, + "engines": { + "node": ">=18" + }, + "devDependencies": { + "typescript": "^5.7.0" + } +} diff --git a/js/src/blob.ts b/js/src/blob.ts new file mode 100644 index 0000000..ae5ac5d --- /dev/null +++ b/js/src/blob.ts @@ -0,0 +1,123 @@ +/* Blob — an owning byte buffer wrapping a msgpack-encoded value. */ + +import { Buf } from "./format.ts"; +import * as D from "./decode.ts"; +import * as E from "./encode.ts"; +import * as J from "./json.ts"; +import * as M from "./mutate.ts"; +import { Type, Value } from "./value.ts"; + +export class Blob { + _data: Uint8Array; + + constructor(data?: Uint8Array | number[] | null) { + if (data === null || data === undefined) this._data = new Uint8Array(0); + else if (data instanceof Uint8Array) this._data = data; + else this._data = Uint8Array.from(data); + } + + // ── raw access ──────────────────────────────────────────────────── + data(): Uint8Array { + return this._data; + } + size(): number { + return this._data.length; + } + empty(): boolean { + return this._data.length === 0; + } + hex(): string { + let s = ""; + for (let i = 0; i < this._data.length; i++) s += this._data[i].toString(16).padStart(2, "0"); + return s; + } + equals(other: Blob): boolean { + if (this._data.length !== other._data.length) return false; + for (let i = 0; i < this._data.length; i++) if (this._data[i] !== other._data[i]) return false; + return true; + } + + // ── validation ──────────────────────────────────────────────────── + valid(): boolean { + return D.isValid(this._data, this._data.length); + } + errorPosition(): number { + return D.errorPosition(this._data, this._data.length); + } + + // ── type inspection ─────────────────────────────────────────────── + type(path?: string): Type { + const n = this._data.length; + if (path === undefined) return n === 0 ? Type.Nil : D.getType(this._data, n, 0); + const r = D.lookup(this._data, n, 0, path); + if (r.rc !== D.RC_OK) return Type.Nil; + return D.getType(this._data, n, r.iStart); + } + typeStr(path?: string): string { + return this.type(path); + } + + // ── extraction ──────────────────────────────────────────────────── + extract(path: string): Value { + const n = this._data.length; + const r = D.lookup(this._data, n, 0, path); + if (r.rc !== D.RC_OK) return Value.nil(); + return D.decodeElement(this._data, n, r.iStart, r.iEnd); + } + + arrayLength(path?: string): number { + const n = this._data.length; + if (path === undefined) return n === 0 ? -1 : D.getContainerCount(this._data, n, 0); + const r = D.lookup(this._data, n, 0, path); + if (r.rc !== D.RC_OK) return -1; + return D.getContainerCount(this._data, n, r.iStart); + } + + // ── mutation (copy-on-write) ────────────────────────────────────── + _apply(path: string, value: Value, mode: number): Blob { + const nb = new Buf(); + E.encodeValue(nb, value); + const r = M.applyEdit(this._data, this._data.length, path, nb.toBytes(), mode); + return r.rc === M.RC_OK ? new Blob(r.out) : this; + } + + set(path: string, value: Value | Blob): Blob { + if (value instanceof Blob) { + const r = M.applyEdit(this._data, this._data.length, path, value._data, M.EDIT_SET); + return r.rc === M.RC_OK ? new Blob(r.out) : this; + } + return this._apply(path, value, M.EDIT_SET); + } + insert(path: string, value: Value): Blob { + return this._apply(path, value, M.EDIT_INSERT); + } + replace(path: string, value: Value): Blob { + return this._apply(path, value, M.EDIT_REPLACE); + } + arrayInsert(path: string, value: Value): Blob { + return this._apply(path, value, M.EDIT_ARRAY_INS); + } + remove(path: string): Blob { + const r = M.applyEdit(this._data, this._data.length, path, new Uint8Array(0), M.EDIT_REMOVE); + return r.rc === M.RC_OK ? new Blob(r.out) : this; + } + patch(mergePatch: Blob): Blob { + const r = M.mergePatch(this._data, this._data.length, 0, mergePatch._data, mergePatch._data.length, 0, 0); + return r.rc === M.RC_OK ? new Blob(r.out) : this; + } + + // ── JSON conversion ─────────────────────────────────────────────── + toJson(): string { + if (this._data.length === 0) return "null"; + return J.toJson(this._data, this._data.length, false, 0); + } + toJsonPretty(indent: number = 2): string { + if (this._data.length === 0) return "null"; + indent = Math.max(0, Math.min(8, indent)); + return J.toJson(this._data, this._data.length, true, indent); + } + + static fromJson(json: string | Uint8Array | null | undefined): Blob { + return new Blob(J.fromJson(json)); + } +} diff --git a/js/src/builder.ts b/js/src/builder.ts new file mode 100644 index 0000000..96d830e --- /dev/null +++ b/js/src/builder.ts @@ -0,0 +1,124 @@ +/* Builder — a streaming encoder that produces a Blob. */ + +import { Buf } from "./format.ts"; +import { utf8Encode } from "./format.ts"; +import * as E from "./encode.ts"; +import { Blob } from "./blob.ts"; +import { Value } from "./value.ts"; + +export class Builder { + _buf: Buf; + + constructor() { + this._buf = new Buf(); + } + + // ── scalars ─────────────────────────────────────────────────────── + nil(): Builder { + E.encNil(this._buf); + return this; + } + boolean(v: boolean): Builder { + E.encBool(this._buf, v); + return this; + } + integer(x: number | bigint): Builder { + E.encInteger(this._buf, BigInt(x)); + return this; + } + unsignedInteger(x: number | bigint): Builder { + E.encUnsigned(this._buf, BigInt(x)); + return this; + } + real(d: number): Builder { + E.encReal(this._buf, d); + return this; + } + real32(f: number): Builder { + E.encReal32(this._buf, f); + return this; + } + string(s: string | Uint8Array): Builder { + E.encString(this._buf, typeof s === "string" ? utf8Encode(s) : s); + return this; + } + binary(data: Uint8Array): Builder { + E.encBinary(this._buf, data); + return this; + } + ext(typeCode: number, data: Uint8Array): Builder { + E.encExt(this._buf, typeCode, data); + return this; + } + + // ── fixed-width integers ────────────────────────────────────────── + int8(x: number | bigint): Builder { + E.encInt8(this._buf, BigInt(x)); + return this; + } + int16(x: number | bigint): Builder { + E.encInt16(this._buf, BigInt(x)); + return this; + } + int32(x: number | bigint): Builder { + E.encInt32(this._buf, BigInt(x)); + return this; + } + int64(x: number | bigint): Builder { + E.encInt64(this._buf, BigInt(x)); + return this; + } + uint8(x: number | bigint): Builder { + E.encUint8(this._buf, BigInt(x)); + return this; + } + uint16(x: number | bigint): Builder { + E.encUint16(this._buf, BigInt(x)); + return this; + } + uint32(x: number | bigint): Builder { + E.encUint32(this._buf, BigInt(x)); + return this; + } + uint64(x: number | bigint): Builder { + E.encUint64(this._buf, BigInt(x)); + return this; + } + + // ── containers ──────────────────────────────────────────────────── + arrayHeader(count: number): Builder { + E.encArrayHeader(this._buf, count); + return this; + } + mapHeader(count: number): Builder { + E.encMapHeader(this._buf, count); + return this; + } + + // ── embedding & timestamp ───────────────────────────────────────── + raw(data: Uint8Array | Blob): Builder { + this._buf.pushBytes(data instanceof Blob ? data.data() : data); + return this; + } + value(v: Value): Builder { + E.encodeValue(this._buf, v); + return this; + } + timestamp(sec: number | bigint, nsec: number = 0): Builder { + E.encTimestamp(this._buf, BigInt(sec), nsec); + return this; + } + + // ── finalize ────────────────────────────────────────────────────── + build(): Blob { + return new Blob(this._buf.toBytes()); + } + + get length(): number { + return this._buf.length; + } + + static quote(v: Value): Blob { + return new Builder().value(v).build(); + } +} diff --git a/js/src/decode.ts b/js/src/decode.ts new file mode 100644 index 0000000..b83be98 --- /dev/null +++ b/js/src/decode.ts @@ -0,0 +1,415 @@ +/* Internal: decoding & inspection (mirrors msgpack_blob_decode.cpp). */ + +import * as F from "./format.ts"; +import { Type, Value } from "./value.ts"; + +export const RC_OK = 0; +export const RC_ERROR = 1; +export const RC_NOTFOUND = 2; + +export interface LookupResult { + rc: number; + iStart: number; + iEnd: number; +} + +export interface PathStep { + kind: number | string; // 0 | -1 | 'k' | 'i' + pi: number; + key: string; + idx: number; +} + +export function isValid(a: Uint8Array, n: number): boolean { + if (n === 0) return false; + return F.skipOne(a, n, 0) === n; +} + +export function errorPosition(a: Uint8Array, n: number): number { + if (n === 0) return 0; + if (F.skipOne(a, n, 0) === n) return 0; + let i = 0; + while (i < n) { + const nxt = F.skipOne(a, n, i); + if (!nxt) return i; + i = nxt; + } + return 0; +} + +function isTimestampExt(a: Uint8Array, n: number, i: number): boolean { + if (i >= n) return false; + const b = a[i]; + if (b === F.MP_FIXEXT4 && i + 6 <= n && a[i + 1] === F.MP_TIMESTAMP_TYPE) return true; + if (b === F.MP_FIXEXT8 && i + 10 <= n && a[i + 1] === F.MP_TIMESTAMP_TYPE) return true; + if (b === F.MP_EXT8 && i + 3 <= n && a[i + 1] === 12 && a[i + 2] === F.MP_TIMESTAMP_TYPE) return true; + return false; +} + +function decodeTimestamp(a: Uint8Array, n: number, i: number): [bigint, number] | null { + if (i >= n) return null; + const b = a[i]; + if (b === F.MP_FIXEXT4 && i + 6 <= n && a[i + 1] === F.MP_TIMESTAMP_TYPE) { + return [BigInt(F.read32(a, i + 2)), 0]; + } + if (b === F.MP_FIXEXT8 && i + 10 <= n && a[i + 1] === F.MP_TIMESTAMP_TYPE) { + const v = F.read64(a, i + 2); + return [v & 0x3ffffffffn, Number(v >> 34n)]; + } + if (b === F.MP_EXT8 && i + 15 <= n && a[i + 1] === 12 && a[i + 2] === F.MP_TIMESTAMP_TYPE) { + const nsec = F.read32(a, i + 3); + const sec = BigInt.asIntN(64, F.read64(a, i + 7)); + return [sec, nsec]; + } + return null; +} + +export function getType(a: Uint8Array, n: number, i: number): Type { + if (i >= n) return Type.Nil; + const b = a[i]; + if (b === F.MP_NIL) return Type.Nil; + if (b === F.MP_TRUE) return Type.True; + if (b === F.MP_FALSE) return Type.False; + if (b <= 0x7f || b >= 0xe0) return Type.Integer; + if (b >= 0xa0 && b <= 0xbf) return Type.String; + if (b >= 0x90 && b <= 0x9f) return Type.Array; + if (b >= 0x80 && b <= 0x8f) return Type.Map; + switch (b) { + case F.MP_UINT8: + case F.MP_UINT16: + case F.MP_UINT32: + case F.MP_UINT64: + case F.MP_INT8: + case F.MP_INT16: + case F.MP_INT32: + case F.MP_INT64: + return Type.Integer; + case F.MP_FLOAT32: + return Type.Float32; + case F.MP_FLOAT64: + return Type.Real; + case F.MP_STR8: + case F.MP_STR16: + case F.MP_STR32: + return Type.String; + case F.MP_BIN8: + case F.MP_BIN16: + case F.MP_BIN32: + return Type.Binary; + case F.MP_ARRAY16: + case F.MP_ARRAY32: + return Type.Array; + case F.MP_MAP16: + case F.MP_MAP32: + return Type.Map; + case F.MP_EXT8: + case F.MP_EXT16: + case F.MP_EXT32: + case F.MP_FIXEXT1: + case F.MP_FIXEXT2: + case F.MP_FIXEXT4: + case F.MP_FIXEXT8: + case F.MP_FIXEXT16: + return isTimestampExt(a, n, i) ? Type.Timestamp : Type.Ext; + default: + return Type.Nil; + } +} + +export function getTypeStr(a: Uint8Array, n: number, i: number): string { + return getType(a, n, i); +} + +export function getContainerCount(a: Uint8Array, n: number, i: number): number { + if (i >= n) return -1; + const b = a[i]; + if (b >= 0x90 && b <= 0x9f) return b & 0x0f; + if (b >= 0x80 && b <= 0x8f) return b & 0x0f; + if (b === F.MP_ARRAY16 && i + 3 <= n) return F.read16(a, i + 1); + if (b === F.MP_ARRAY32 && i + 5 <= n) return F.read32(a, i + 1); + if (b === F.MP_MAP16 && i + 3 <= n) return F.read16(a, i + 1); + if (b === F.MP_MAP32 && i + 5 <= n) return F.read32(a, i + 1); + return -1; +} + +export function pathStep(zpath: string, pi: number): PathStep { + let i = pi; + if (i >= zpath.length) return { kind: 0, pi: i, key: "", idx: 0 }; + const c = zpath[i]; + if (c === ".") { + i += 1; + const start = i; + while (i < zpath.length && zpath[i] !== "." && zpath[i] !== "[") i += 1; + return { kind: "k", pi: i, key: zpath.slice(start, i), idx: 0 }; + } + if (c === "[") { + let idx = 0; + let hasDigit = false; + i += 1; + while (i < zpath.length && zpath[i] >= "0" && zpath[i] <= "9") { + idx = idx * 10 + (zpath.charCodeAt(i) - 48); + i += 1; + hasDigit = true; + } + if (!hasDigit || i >= zpath.length || zpath[i] !== "]") return { kind: -1, pi: i, key: "", idx: 0 }; + i += 1; + return { kind: "i", pi: i, key: "", idx }; + } + return { kind: -1, pi: i, key: "", idx: 0 }; +} + +// Returns the key bytes for a map key at offset i, or null. +function keyAt(a: Uint8Array, n: number, i: number): Uint8Array | null { + const kb = a[i]; + let klen: number; + let koff: number; + if (kb >= 0xa0 && kb <= 0xbf) { + klen = kb & 0x1f; + koff = i + 1; + } else if (kb === F.MP_STR8 && i + 2 <= n) { + klen = a[i + 1]; + koff = i + 2; + } else if (kb === F.MP_STR16 && i + 3 <= n) { + klen = F.read16(a, i + 1); + koff = i + 3; + } else if (kb === F.MP_STR32 && i + 5 <= n) { + klen = F.read32(a, i + 1); + koff = i + 5; + } else { + return null; + } + if (klen > n - koff) return null; + return a.subarray(koff, koff + klen); +} + +function bytesEqual(x: Uint8Array, y: Uint8Array): boolean { + if (x.length !== y.length) return false; + for (let i = 0; i < x.length; i++) if (x[i] !== y[i]) return false; + return true; +} + +export function lookup(a: Uint8Array, n: number, iroot: number, zpath: string): LookupResult { + if (!zpath || zpath[0] !== "$") return { rc: RC_ERROR, iStart: 0, iEnd: 0 }; + let icur = iroot; + let pi = 1; + + for (;;) { + const st = pathStep(zpath, pi); + pi = st.pi; + + if (st.kind === 0) { + const inext = F.skipOne(a, n, icur); + const iEnd = inext ? inext : n; + return { rc: inext || icur === n ? RC_OK : RC_ERROR, iStart: icur, iEnd }; + } + if (st.kind === -1) return { rc: RC_ERROR, iStart: 0, iEnd: 0 }; + if (icur >= n) return { rc: RC_NOTFOUND, iStart: 0, iEnd: 0 }; + + if (st.kind === "i") { + const b = a[icur]; + let count: number; + let elemOff: number; + if (b >= 0x90 && b <= 0x9f) { + count = b & 0x0f; + elemOff = icur + 1; + } else if (b === F.MP_ARRAY16) { + if (icur + 3 > n) return { rc: RC_ERROR, iStart: 0, iEnd: 0 }; + count = F.read16(a, icur + 1); + elemOff = icur + 3; + } else if (b === F.MP_ARRAY32) { + if (icur + 5 > n) return { rc: RC_ERROR, iStart: 0, iEnd: 0 }; + count = F.read32(a, icur + 1); + elemOff = icur + 5; + } else { + return { rc: RC_NOTFOUND, iStart: 0, iEnd: 0 }; + } + if (st.idx < 0 || st.idx >= count) return { rc: RC_NOTFOUND, iStart: 0, iEnd: 0 }; + icur = elemOff; + for (let j = 0; j < st.idx; j++) { + icur = F.skipOne(a, n, icur); + if (!icur) return { rc: RC_ERROR, iStart: 0, iEnd: 0 }; + } + } else { + const b = a[icur]; + let count: number; + let elemOff: number; + if (b >= 0x80 && b <= 0x8f) { + count = b & 0x0f; + elemOff = icur + 1; + } else if (b === F.MP_MAP16) { + if (icur + 3 > n) return { rc: RC_ERROR, iStart: 0, iEnd: 0 }; + count = F.read16(a, icur + 1); + elemOff = icur + 3; + } else if (b === F.MP_MAP32) { + if (icur + 5 > n) return { rc: RC_ERROR, iStart: 0, iEnd: 0 }; + count = F.read32(a, icur + 1); + elemOff = icur + 5; + } else { + return { rc: RC_NOTFOUND, iStart: 0, iEnd: 0 }; + } + const keyBytes = F.utf8Encode(st.key); + icur = elemOff; + let found = false; + let j = 0; + while (j < count && !found) { + if (icur >= n) return { rc: RC_ERROR, iStart: 0, iEnd: 0 }; + const kstr = keyAt(a, n, icur); + const valOff = F.skipOne(a, n, icur); + if (!valOff) return { rc: RC_ERROR, iStart: 0, iEnd: 0 }; + if (kstr !== null && bytesEqual(kstr, keyBytes)) { + icur = valOff; + found = true; + } else { + icur = F.skipOne(a, n, valOff); + if (!icur) return { rc: RC_ERROR, iStart: 0, iEnd: 0 }; + } + j += 1; + } + if (!found) return { rc: RC_NOTFOUND, iStart: 0, iEnd: 0 }; + } + } +} + +export function decodeElement(a: Uint8Array, n: number, iStart: number, iEnd: number): Value { + if (iStart >= n || iStart >= iEnd) return Value.nil(); + const b = a[iStart]; + + if (b === F.MP_NIL) return Value.nil(); + if (b === F.MP_FALSE) return Value.boolean(false); + if (b === F.MP_TRUE) return Value.boolean(true); + if (b <= 0x7f) return Value.integer(b); + if (b >= 0xe0) return Value.integer(b - 256); + + switch (b) { + case F.MP_UINT8: + if (iStart + 2 <= n) return Value.integer(a[iStart + 1]); + break; + case F.MP_UINT16: + if (iStart + 3 <= n) return Value.integer(F.read16(a, iStart + 1)); + break; + case F.MP_UINT32: + if (iStart + 5 <= n) return Value.integer(F.read32(a, iStart + 1)); + break; + case F.MP_UINT64: + if (iStart + 9 <= n) return Value.unsignedInteger(F.read64(a, iStart + 1)); + break; + case F.MP_INT8: + if (iStart + 2 <= n) { + const v = a[iStart + 1]; + return Value.integer(v >= 128 ? v - 256 : v); + } + break; + case F.MP_INT16: + if (iStart + 3 <= n) { + const v = F.read16(a, iStart + 1); + return Value.integer(v >= 0x8000 ? v - 0x10000 : v); + } + break; + case F.MP_INT32: + if (iStart + 5 <= n) { + const v = F.read32(a, iStart + 1); + return Value.integer(v >= 0x80000000 ? v - 0x100000000 : v); + } + break; + case F.MP_INT64: + if (iStart + 9 <= n) return Value.integer(BigInt.asIntN(64, F.read64(a, iStart + 1))); + break; + case F.MP_FLOAT32: + if (iStart + 5 <= n) return Value.real32(F.readF32(a, iStart + 1)); + break; + case F.MP_FLOAT64: + if (iStart + 9 <= n) return Value.real(F.readF64(a, iStart + 1)); + break; + default: + break; + } + + // str + let soff = 0; + let slen = 0; + if (b >= 0xa0 && b <= 0xbf) { + slen = b & 0x1f; + soff = iStart + 1; + } else if (b === F.MP_STR8 && iStart + 2 <= n) { + slen = a[iStart + 1]; + soff = iStart + 2; + } else if (b === F.MP_STR16 && iStart + 3 <= n) { + slen = F.read16(a, iStart + 1); + soff = iStart + 3; + } else if (b === F.MP_STR32 && iStart + 5 <= n) { + slen = F.read32(a, iStart + 1); + soff = iStart + 5; + } + if (soff) { + if (slen > n - soff) slen = n - soff; + return Value.string(a.slice(soff, soff + slen)); + } + + // bin + let boff = 0; + let blen = 0; + if (b === F.MP_BIN8 && iStart + 2 <= n) { + blen = a[iStart + 1]; + boff = iStart + 2; + } else if (b === F.MP_BIN16 && iStart + 3 <= n) { + blen = F.read16(a, iStart + 1); + boff = iStart + 3; + } else if (b === F.MP_BIN32 && iStart + 5 <= n) { + blen = F.read32(a, iStart + 1); + boff = iStart + 5; + } + if (boff) { + if (blen > n - boff) blen = n - boff; + return Value.binary(a.slice(boff, boff + blen)); + } + + // timestamp + const ts = decodeTimestamp(a, n, iStart); + if (ts !== null) return Value.timestamp(ts[0], ts[1]); + + // ext + let tc = 0; + let elen = 0; + let eoff = 0; + if (b === F.MP_FIXEXT1 && iStart + 3 <= n) { + tc = a[iStart + 1]; + elen = 1; + eoff = iStart + 2; + } else if (b === F.MP_FIXEXT2 && iStart + 4 <= n) { + tc = a[iStart + 1]; + elen = 2; + eoff = iStart + 2; + } else if (b === F.MP_FIXEXT4 && iStart + 6 <= n) { + tc = a[iStart + 1]; + elen = 4; + eoff = iStart + 2; + } else if (b === F.MP_FIXEXT8 && iStart + 10 <= n) { + tc = a[iStart + 1]; + elen = 8; + eoff = iStart + 2; + } else if (b === F.MP_FIXEXT16 && iStart + 18 <= n) { + tc = a[iStart + 1]; + elen = 16; + eoff = iStart + 2; + } else if (b === F.MP_EXT8 && iStart + 3 <= n) { + elen = a[iStart + 1]; + tc = a[iStart + 2]; + eoff = iStart + 3; + } else if (b === F.MP_EXT16 && iStart + 4 <= n) { + elen = F.read16(a, iStart + 1); + tc = a[iStart + 3]; + eoff = iStart + 4; + } else if (b === F.MP_EXT32 && iStart + 6 <= n) { + elen = F.read32(a, iStart + 1); + tc = a[iStart + 5]; + eoff = iStart + 6; + } + if (eoff) { + if (elen > n - eoff) elen = n - eoff; + const tcSigned = tc >= 128 ? tc - 256 : tc; + return Value.ext(tcSigned, a.slice(eoff, eoff + elen)); + } + + // containers → raw binary blob (includes header) + return Value.binary(a.slice(iStart, iEnd)); +} diff --git a/js/src/encode.ts b/js/src/encode.ts new file mode 100644 index 0000000..232647a --- /dev/null +++ b/js/src/encode.ts @@ -0,0 +1,225 @@ +/* Internal: encoding primitives (mirrors msgpack_blob_encode.cpp). */ + +import * as F from "./format.ts"; +import { Buf } from "./format.ts"; +import { IntWidth, Type, Value } from "./value.ts"; + +export function encNil(out: Buf): void { + out.push(F.MP_NIL); +} + +export function encBool(out: Buf, v: boolean): void { + out.push(v ? F.MP_TRUE : F.MP_FALSE); +} + +export function encInteger(out: Buf, x: bigint): void { + if (x >= 0n) { + if (x <= 0x7fn) out.push(Number(x)); + else if (x <= 0xffn) { + out.push(F.MP_UINT8); + out.push(Number(x)); + } else if (x <= 0xffffn) { + out.push(F.MP_UINT16); + out.pushU16(Number(x)); + } else if (x <= 0xffffffffn) { + out.push(F.MP_UINT32); + out.pushU32(Number(x)); + } else { + out.push(F.MP_UINT64); + out.pushU64(x); + } + } else { + if (x >= -32n) out.push(Number(x & 0xffn)); + else if (x >= -128n) { + out.push(F.MP_INT8); + out.push(Number(x & 0xffn)); + } else if (x >= -32768n) { + out.push(F.MP_INT16); + out.pushU16(Number(x & 0xffffn)); + } else if (x >= -2147483648n) { + out.push(F.MP_INT32); + out.pushU32(Number(x & 0xffffffffn)); + } else { + out.push(F.MP_INT64); + out.pushU64(x); + } + } +} + +export function encUnsigned(out: Buf, x: bigint): void { + x = BigInt.asUintN(64, x); + if (x <= 0x7fn) out.push(Number(x)); + else if (x <= 0xffn) { + out.push(F.MP_UINT8); + out.push(Number(x)); + } else if (x <= 0xffffn) { + out.push(F.MP_UINT16); + out.pushU16(Number(x)); + } else if (x <= 0xffffffffn) { + out.push(F.MP_UINT32); + out.pushU32(Number(x)); + } else { + out.push(F.MP_UINT64); + out.pushU64(x); + } +} + +export function encReal(out: Buf, d: number): void { + out.push(F.MP_FLOAT64); + out.pushF64(d); +} + +export function encReal32(out: Buf, f: number): void { + out.push(F.MP_FLOAT32); + out.pushF32(f); +} + +export function encString(out: Buf, s: Uint8Array): void { + const n = s.length; + if (n <= 31) out.push(F.MP_FIXSTR_MASK | n); + else if (n <= 0xff) { + out.push(F.MP_STR8); + out.push(n); + } else if (n <= 0xffff) { + out.push(F.MP_STR16); + out.pushU16(n); + } else { + out.push(F.MP_STR32); + out.pushU32(n); + } + out.pushBytes(s); +} + +export function encBinary(out: Buf, data: Uint8Array): void { + const n = data.length; + if (n <= 0xff) { + out.push(F.MP_BIN8); + out.push(n); + } else if (n <= 0xffff) { + out.push(F.MP_BIN16); + out.pushU16(n); + } else { + out.push(F.MP_BIN32); + out.pushU32(n); + } + out.pushBytes(data); +} + +export function encExt(out: Buf, typeCode: number, data: Uint8Array): void { + const n = data.length; + if (n === 1) out.push(F.MP_FIXEXT1); + else if (n === 2) out.push(F.MP_FIXEXT2); + else if (n === 4) out.push(F.MP_FIXEXT4); + else if (n === 8) out.push(F.MP_FIXEXT8); + else if (n === 16) out.push(F.MP_FIXEXT16); + else if (n <= 0xff) { + out.push(F.MP_EXT8); + out.push(n); + } else if (n <= 0xffff) { + out.push(F.MP_EXT16); + out.pushU16(n); + } else { + out.push(F.MP_EXT32); + out.pushU32(n); + } + out.push(typeCode & 0xff); + out.pushBytes(data); +} + +export function encInt8(out: Buf, x: bigint): void { + out.push(F.MP_INT8); + out.push(Number(BigInt.asUintN(8, x))); +} +export function encInt16(out: Buf, x: bigint): void { + out.push(F.MP_INT16); + out.pushU16(Number(BigInt.asUintN(16, x))); +} +export function encInt32(out: Buf, x: bigint): void { + out.push(F.MP_INT32); + out.pushU32(Number(BigInt.asUintN(32, x))); +} +export function encInt64(out: Buf, x: bigint): void { + out.push(F.MP_INT64); + out.pushU64(x); +} +export function encUint8(out: Buf, x: bigint): void { + out.push(F.MP_UINT8); + out.push(Number(BigInt.asUintN(8, x))); +} +export function encUint16(out: Buf, x: bigint): void { + out.push(F.MP_UINT16); + out.pushU16(Number(BigInt.asUintN(16, x))); +} +export function encUint32(out: Buf, x: bigint): void { + out.push(F.MP_UINT32); + out.pushU32(Number(BigInt.asUintN(32, x))); +} +export function encUint64(out: Buf, x: bigint): void { + out.push(F.MP_UINT64); + out.pushU64(x); +} + +export function encArrayHeader(out: Buf, count: number): void { + if (count <= 15) out.push(F.MP_FIXARRAY_MASK | count); + else if (count <= 0xffff) { + out.push(F.MP_ARRAY16); + out.pushU16(count); + } else { + out.push(F.MP_ARRAY32); + out.pushU32(count); + } +} + +export function encMapHeader(out: Buf, count: number): void { + if (count <= 15) out.push(F.MP_FIXMAP_MASK | count); + else if (count <= 0xffff) { + out.push(F.MP_MAP16); + out.pushU16(count); + } else { + out.push(F.MP_MAP32); + out.pushU32(count); + } +} + +export function encTimestamp(out: Buf, sec: bigint, nsec: number): void { + if (nsec === 0 && sec >= 0n && sec <= 0xffffffffn) { + out.push(F.MP_FIXEXT4); + out.push(0xff); + out.pushU32(Number(sec)); + } else if (sec >= 0n && sec <= 0x3ffffffffn) { + out.push(F.MP_FIXEXT8); + out.push(0xff); + out.pushU64((BigInt(nsec) << 34n) | sec); + } else { + out.push(F.MP_EXT8); + out.push(12); + out.push(0xff); + out.pushU32(nsec >>> 0); + out.pushU64(sec); + } +} + +export function encodeValue(out: Buf, v: Value): void { + const t = v.type(); + if (t === Type.Nil) encNil(out); + else if (t === Type.True) encBool(out, true); + else if (t === Type.False) encBool(out, false); + else if (t === Type.Integer) { + const w = v.intWidth(); + if (w === IntWidth.Int8) encInt8(out, v.asInt64()); + else if (w === IntWidth.Int16) encInt16(out, v.asInt64()); + else if (w === IntWidth.Int32) encInt32(out, v.asInt64()); + else if (w === IntWidth.Int64) encInt64(out, v.asInt64()); + else if (w === IntWidth.Uint8) encUint8(out, v.asUint64()); + else if (w === IntWidth.Uint16) encUint16(out, v.asUint64()); + else if (w === IntWidth.Uint32) encUint32(out, v.asUint64()); + else if (w === IntWidth.Uint64) encUint64(out, v.asUint64()); + else encInteger(out, v.asInt64()); + } else if (t === Type.Real) encReal(out, v.asDouble()); + else if (t === Type.Float32) encReal32(out, v.asFloat()); + else if (t === Type.String) encString(out, v.asBytes()); + else if (t === Type.Binary) encBinary(out, v.blobData()); + else if (t === Type.Ext) encExt(out, v.extType(), v.blobData()); + else if (t === Type.Timestamp) encTimestamp(out, v.timestampSeconds(), v.timestampNanoseconds()); + else encNil(out); +} diff --git a/js/src/format.ts b/js/src/format.ts new file mode 100644 index 0000000..986c750 --- /dev/null +++ b/js/src/format.ts @@ -0,0 +1,368 @@ +/* + * Internal: MessagePack format constants, byte-order helpers, the growable + * output buffer, and skip_one. Private to the implementation; mirrors + * cpp/src/msgpack_blob_detail.hpp and the skip routine from the decode module. + */ + +export const MAX_DEPTH = 200; +export const MAX_OUTPUT = 64 * 1024 * 1024; + +export const MP_NIL = 0xc0; +export const MP_FALSE = 0xc2; +export const MP_TRUE = 0xc3; +export const MP_BIN8 = 0xc4; +export const MP_BIN16 = 0xc5; +export const MP_BIN32 = 0xc6; +export const MP_EXT8 = 0xc7; +export const MP_EXT16 = 0xc8; +export const MP_EXT32 = 0xc9; +export const MP_FLOAT32 = 0xca; +export const MP_FLOAT64 = 0xcb; +export const MP_UINT8 = 0xcc; +export const MP_UINT16 = 0xcd; +export const MP_UINT32 = 0xce; +export const MP_UINT64 = 0xcf; +export const MP_INT8 = 0xd0; +export const MP_INT16 = 0xd1; +export const MP_INT32 = 0xd2; +export const MP_INT64 = 0xd3; +export const MP_FIXEXT1 = 0xd4; +export const MP_FIXEXT2 = 0xd5; +export const MP_FIXEXT4 = 0xd6; +export const MP_FIXEXT8 = 0xd7; +export const MP_FIXEXT16 = 0xd8; +export const MP_STR8 = 0xd9; +export const MP_STR16 = 0xda; +export const MP_STR32 = 0xdb; +export const MP_ARRAY16 = 0xdc; +export const MP_ARRAY32 = 0xdd; +export const MP_MAP16 = 0xde; +export const MP_MAP32 = 0xdf; + +export const MP_FIXMAP_MASK = 0x80; +export const MP_FIXARRAY_MASK = 0x90; +export const MP_FIXSTR_MASK = 0xa0; + +export const MP_TIMESTAMP_TYPE = 0xff; + +export const U64_MASK = 0xffffffffffffffffn; + +// ── big-endian read helpers ───────────────────────────────────────── +export function read16(a: Uint8Array, i: number): number { + return (a[i] << 8) | a[i + 1]; +} +export function read32(a: Uint8Array, i: number): number { + return ((a[i] << 24) | (a[i + 1] << 16) | (a[i + 2] << 8) | a[i + 3]) >>> 0; +} +export function read64(a: Uint8Array, i: number): bigint { + let v = 0n; + for (let k = 0; k < 8; k++) v = (v << 8n) | BigInt(a[i + k]); + return v; +} + +const _scratch = new DataView(new ArrayBuffer(8)); +export function readF32(a: Uint8Array, i: number): number { + for (let k = 0; k < 4; k++) _scratch.setUint8(k, a[i + k]); + return _scratch.getFloat32(0, false); +} +export function readF64(a: Uint8Array, i: number): number { + for (let k = 0; k < 8; k++) _scratch.setUint8(k, a[i + k]); + return _scratch.getFloat64(0, false); +} + +// ── growable output buffer ────────────────────────────────────────── +export class Buf { + bytes: number[]; + constructor() { + this.bytes = []; + } + get length(): number { + return this.bytes.length; + } + push(b: number): void { + this.bytes.push(b & 0xff); + } + pushU16(v: number): void { + this.bytes.push((v >>> 8) & 0xff, v & 0xff); + } + pushU32(v: number): void { + this.bytes.push((v >>> 24) & 0xff, (v >>> 16) & 0xff, (v >>> 8) & 0xff, v & 0xff); + } + pushU64(v: bigint): void { + const b = BigInt.asUintN(64, v); + for (let s = 56n; s >= 0n; s -= 8n) this.bytes.push(Number((b >> s) & 0xffn)); + } + pushF32(f: number): void { + _scratch.setFloat32(0, f, false); + for (let k = 0; k < 4; k++) this.bytes.push(_scratch.getUint8(k)); + } + pushF64(d: number): void { + _scratch.setFloat64(0, d, false); + for (let k = 0; k < 8; k++) this.bytes.push(_scratch.getUint8(k)); + } + pushBytes(src: Uint8Array | number[]): void { + for (let k = 0; k < src.length; k++) this.bytes.push(src[k] & 0xff); + } + pushSlice(a: Uint8Array, start: number, end: number): void { + for (let i = start; i < end; i++) this.bytes.push(a[i]); + } + toBytes(): Uint8Array { + return Uint8Array.from(this.bytes); + } +} + +// ── skip_one — offset just past one element, or 0 on malformed input ─ +export function skipOne(a: Uint8Array, n: number, i: number): number { + return skipOneD(a, n, i, 0); +} + +function skipOneD(a: Uint8Array, n: number, i: number, depth: number): number { + if (depth > MAX_DEPTH) return 0; + if (i >= n) return 0; + const b = a[i]; + i += 1; + + if (b <= 0x7f) return i; + if (b >= 0xe0) return i; + + switch (b) { + case MP_NIL: + case MP_FALSE: + case MP_TRUE: + return i; + case MP_FLOAT32: + return i + 4 <= n ? i + 4 : 0; + case MP_FLOAT64: + case MP_INT64: + case MP_UINT64: + return i + 8 <= n ? i + 8 : 0; + case MP_UINT8: + case MP_INT8: + return i + 1 <= n ? i + 1 : 0; + case MP_UINT16: + case MP_INT16: + return i + 2 <= n ? i + 2 : 0; + case MP_UINT32: + case MP_INT32: + return i + 4 <= n ? i + 4 : 0; + case MP_BIN8: + case MP_STR8: { + if (i + 1 > n) return 0; + const sz = a[i]; + i += 1; + return sz <= n - i ? i + sz : 0; + } + case MP_BIN16: + case MP_STR16: { + if (i + 2 > n) return 0; + const sz = read16(a, i); + i += 2; + return sz <= n - i ? i + sz : 0; + } + case MP_BIN32: + case MP_STR32: { + if (i + 4 > n) return 0; + const sz = read32(a, i); + i += 4; + return sz <= n - i ? i + sz : 0; + } + case MP_FIXEXT1: + return i + 2 <= n ? i + 2 : 0; + case MP_FIXEXT2: + return i + 3 <= n ? i + 3 : 0; + case MP_FIXEXT4: + return i + 5 <= n ? i + 5 : 0; + case MP_FIXEXT8: + return i + 9 <= n ? i + 9 : 0; + case MP_FIXEXT16: + return i + 17 <= n ? i + 17 : 0; + case MP_EXT8: { + if (i + 2 > n) return 0; + const sz = a[i]; + i += 2; + return sz <= n - i ? i + sz : 0; + } + case MP_EXT16: { + if (i + 3 > n) return 0; + const sz = read16(a, i); + i += 3; + return sz <= n - i ? i + sz : 0; + } + case MP_EXT32: { + if (i + 5 > n) return 0; + const sz = read32(a, i); + i += 5; + return sz <= n - i ? i + sz : 0; + } + default: + break; + } + + if (b >= 0xa0 && b <= 0xbf) { + const sz = b & 0x1f; + return sz <= n - i ? i + sz : 0; + } + + if (b >= 0x90 && b <= 0x9f) { + const count = b & 0x0f; + for (let j = 0; j < count; j++) { + i = skipOneD(a, n, i, depth + 1); + if (!i) return 0; + } + return i; + } + + if (b >= 0x80 && b <= 0x8f) { + const count = b & 0x0f; + for (let j = 0; j < count; j++) { + i = skipOneD(a, n, i, depth + 1); + if (!i) return 0; + i = skipOneD(a, n, i, depth + 1); + if (!i) return 0; + } + return i; + } + + if (b === MP_ARRAY16 || b === MP_ARRAY32) { + let count: number; + if (b === MP_ARRAY16) { + if (i + 2 > n) return 0; + count = read16(a, i); + i += 2; + } else { + if (i + 4 > n) return 0; + count = read32(a, i); + i += 4; + } + for (let j = 0; j < count; j++) { + i = skipOneD(a, n, i, depth + 1); + if (!i) return 0; + } + return i; + } + + if (b === MP_MAP16 || b === MP_MAP32) { + let count: number; + if (b === MP_MAP16) { + if (i + 2 > n) return 0; + count = read16(a, i); + i += 2; + } else { + if (i + 4 > n) return 0; + count = read32(a, i); + i += 4; + } + for (let j = 0; j < count; j++) { + i = skipOneD(a, n, i, depth + 1); + if (!i) return 0; + i = skipOneD(a, n, i, depth + 1); + if (!i) return 0; + } + return i; + } + + return 0; +} + +// ── UTF-8 helpers ─────────────────────────────────────────────────── +// Byte-preserving (lossless) UTF-8 codec matching Python's +// `bytes.decode("utf-8", "surrogateescape")` / `str.encode(...)`. The C++ +// reference passes raw `str` bytes through verbatim, so a plain replacing +// TextDecoder (which substitutes U+FFFD) would diverge on non-UTF-8 input. +// Invalid bytes round-trip through lone low surrogates U+DC80..U+DCFF. +export function utf8Decode(bytes: Uint8Array): string { + let out = ""; + let i = 0; + const n = bytes.length; + while (i < n) { + const b0 = bytes[i]; + if (b0 < 0x80) { + out += String.fromCharCode(b0); + i++; + continue; + } + let len: number; + let cp: number; + if (b0 >= 0xc2 && b0 <= 0xdf) { + len = 2; + cp = b0 & 0x1f; + } else if (b0 >= 0xe0 && b0 <= 0xef) { + len = 3; + cp = b0 & 0x0f; + } else if (b0 >= 0xf0 && b0 <= 0xf4) { + len = 4; + cp = b0 & 0x07; + } else { + out += String.fromCharCode(0xdc00 + b0); + i++; + continue; + } + if (i + len > n) { + out += String.fromCharCode(0xdc00 + b0); + i++; + continue; + } + const b1 = bytes[i + 1]; + let valid: boolean; + if (len === 2) valid = b1 >= 0x80 && b1 <= 0xbf; + else if (len === 3) { + if (b0 === 0xe0) valid = b1 >= 0xa0 && b1 <= 0xbf; + else if (b0 === 0xed) valid = b1 >= 0x80 && b1 <= 0x9f; // exclude surrogates + else valid = b1 >= 0x80 && b1 <= 0xbf; + } else { + if (b0 === 0xf0) valid = b1 >= 0x90 && b1 <= 0xbf; + else if (b0 === 0xf4) valid = b1 >= 0x80 && b1 <= 0x8f; + else valid = b1 >= 0x80 && b1 <= 0xbf; + } + if (!valid) { + out += String.fromCharCode(0xdc00 + b0); + i++; + continue; + } + cp = (cp << 6) | (b1 & 0x3f); + let ok = true; + for (let k = 2; k < len; k++) { + const bk = bytes[i + k]; + if (bk < 0x80 || bk > 0xbf) { + ok = false; + break; + } + cp = (cp << 6) | (bk & 0x3f); + } + if (!ok) { + out += String.fromCharCode(0xdc00 + b0); + i++; + continue; + } + if (cp <= 0xffff) out += String.fromCharCode(cp); + else { + cp -= 0x10000; + out += String.fromCharCode(0xd800 + (cp >> 10), 0xdc00 + (cp & 0x3ff)); + } + i += len; + } + return out; +} + +export function utf8Encode(s: string): Uint8Array { + const out: number[] = []; + for (let i = 0; i < s.length; i++) { + let cp = s.charCodeAt(i); + if (cp >= 0xdc80 && cp <= 0xdcff) { + out.push(cp & 0xff); // surrogateescape byte + continue; + } + if (cp >= 0xd800 && cp <= 0xdbff && i + 1 < s.length) { + const lo = s.charCodeAt(i + 1); + if (lo >= 0xdc00 && lo <= 0xdfff) { + cp = 0x10000 + ((cp - 0xd800) << 10) + (lo - 0xdc00); + i++; + } + } + if (cp < 0x80) out.push(cp); + else if (cp < 0x800) out.push(0xc0 | (cp >> 6), 0x80 | (cp & 0x3f)); + else if (cp < 0x10000) out.push(0xe0 | (cp >> 12), 0x80 | ((cp >> 6) & 0x3f), 0x80 | (cp & 0x3f)); + else out.push(0xf0 | (cp >> 18), 0x80 | ((cp >> 12) & 0x3f), 0x80 | ((cp >> 6) & 0x3f), 0x80 | (cp & 0x3f)); + } + return Uint8Array.from(out); +} diff --git a/js/src/index.ts b/js/src/index.ts new file mode 100644 index 0000000..3fc6714 --- /dev/null +++ b/js/src/index.ts @@ -0,0 +1,28 @@ +/* + * msgpack-blob — a pure-TypeScript MessagePack Blob library. + * + * A zero-dependency port of the standalone C++ `msgpack` Blob API. It creates, + * queries, mutates and iterates MessagePack binary blobs and produces + * byte-identical output to the C++ library and the `sqlite-msgpack` extension, + * so blobs are fully interchangeable across all three. + * + * @example + * import { Blob, Builder, Value, Iterator } from "msgpack-blob"; + * + * const blob = Blob.fromJson('{"name":"Alice","scores":[95,87,91]}'); + * blob.extract("$.name").asString(); // 'Alice' + * blob.toJson(); // '{"name":"Alice","scores":[95,87,91]}' + * + * const updated = blob.set("$.age", Value.integer(30)); + * for (const row of new Iterator(blob, "$", true)) { + * console.log(row.fullkey, row.type); + * } + */ + +export { Type, IntWidth, Value, typeStr } from "./value.ts"; +export { Blob } from "./blob.ts"; +export { Builder } from "./builder.ts"; +export { Iterator, EachRow } from "./iterator.ts"; +export { MAX_DEPTH, MAX_OUTPUT } from "./format.ts"; + +export const VERSION = "1.5.0"; diff --git a/js/src/iterate.ts b/js/src/iterate.ts new file mode 100644 index 0000000..6d663e7 --- /dev/null +++ b/js/src/iterate.ts @@ -0,0 +1,144 @@ +/* Internal: container iteration (mirrors msgpack_blob_iterate.cpp). */ + +import * as F from "./format.ts"; +import { decodeElement, getType } from "./decode.ts"; +import { Type, Value } from "./value.ts"; + +export class EachRow { + fullkey: string = "$"; + path: string = "$"; + id: number = 0; + type: Type = Type.Nil; + value: Value = Value.nil(); + key: string = ""; // map key ("" for arrays / tree rows) + index: number = 0; // array/pair index (each mode only) +} + +function keyStr(a: Uint8Array, n: number, i: number): string | null { + const kb = a[i]; + let klen: number; + let koff: number; + if (kb >= 0xa0 && kb <= 0xbf) { + klen = kb & 0x1f; + koff = i + 1; + } else if (kb === F.MP_STR8 && i + 2 <= n) { + klen = a[i + 1]; + koff = i + 2; + } else if (kb === F.MP_STR16 && i + 3 <= n) { + klen = F.read16(a, i + 1); + koff = i + 3; + } else if (kb === F.MP_STR32 && i + 5 <= n) { + klen = F.read32(a, i + 1); + koff = i + 5; + } else { + return null; + } + return F.utf8Decode(a.subarray(koff, koff + klen)); +} + +interface Container { + isArr: boolean; + isMap: boolean; + count: number; + dataOff: number; +} + +function container(a: Uint8Array, n: number, i: number): Container { + const b = a[i]; + if (b >= 0x90 && b <= 0x9f) return { isArr: true, isMap: false, count: b & 0x0f, dataOff: i + 1 }; + if (b === F.MP_ARRAY16 && i + 3 <= n) return { isArr: true, isMap: false, count: F.read16(a, i + 1), dataOff: i + 3 }; + if (b === F.MP_ARRAY32 && i + 5 <= n) return { isArr: true, isMap: false, count: F.read32(a, i + 1), dataOff: i + 5 }; + if (b >= 0x80 && b <= 0x8f) return { isArr: false, isMap: true, count: b & 0x0f, dataOff: i + 1 }; + if (b === F.MP_MAP16 && i + 3 <= n) return { isArr: false, isMap: true, count: F.read16(a, i + 1), dataOff: i + 3 }; + if (b === F.MP_MAP32 && i + 5 <= n) return { isArr: false, isMap: true, count: F.read32(a, i + 1), dataOff: i + 5 }; + return { isArr: false, isMap: false, count: 0, dataOff: 0 }; +} + +export function eachIter(a: Uint8Array, n: number, icont: number, zbase: string): EachRow[] { + const rows: EachRow[] = []; + if (icont >= n) return rows; + const c = container(a, n, icont); + if (!c.isArr && !c.isMap) return rows; + + const remaining = c.dataOff <= n ? n - c.dataOff : 0; + const minBytes = c.isMap ? 2 : 1; + if (c.count > Math.floor(remaining / minBytes) + 1) return rows; + + let cur = c.dataOff; + for (let j = 0; j < c.count; j++) { + if (cur >= n) break; + if (c.isArr) { + const cEnd = F.skipOne(a, n, cur); + if (!cEnd) break; + const row = new EachRow(); + row.fullkey = `${zbase}[${j}]`; + row.path = zbase; + row.id = cur; + row.type = getType(a, n, cur); + row.value = decodeElement(a, n, cur, cEnd); + row.key = ""; + row.index = j; + rows.push(row); + cur = cEnd; + } else { + const ks = keyStr(a, n, cur); + const vOff = F.skipOne(a, n, cur); + if (!vOff) break; + const pEnd = F.skipOne(a, n, vOff); + if (!pEnd) break; + const key = ks !== null ? ks : "?"; + const row = new EachRow(); + row.fullkey = `${zbase}.${key}`; + row.path = zbase; + row.id = vOff; + row.type = getType(a, n, vOff); + row.value = decodeElement(a, n, vOff, pEnd); + row.key = key; + row.index = j; + rows.push(row); + cur = pEnd; + } + } + return rows; +} + +export function treeWalk(a: Uint8Array, n: number, ioff: number, zfull: string, zparPath: string, depth: number, rows: EachRow[]): void { + if (depth > F.MAX_DEPTH || ioff >= n) return; + const iend = F.skipOne(a, n, ioff); + if (!iend) return; + + const row = new EachRow(); + row.fullkey = zfull; + row.path = zparPath; + row.id = ioff; + row.type = getType(a, n, ioff); + row.value = decodeElement(a, n, ioff, iend); + rows.push(row); + + const c = container(a, n, ioff); + if (!c.isArr && !c.isMap) return; + + const remaining = c.dataOff <= n ? n - c.dataOff : 0; + const minBytes = c.isMap ? 2 : 1; + if (c.count > Math.floor(remaining / minBytes) + 1) return; + + let cur = c.dataOff; + for (let j = 0; j < c.count; j++) { + if (cur >= n) break; + if (c.isArr) { + const cEnd = F.skipOne(a, n, cur); + if (!cEnd) break; + treeWalk(a, n, cur, `${zfull}[${j}]`, zfull, depth + 1, rows); + cur = cEnd; + } else { + const ks = keyStr(a, n, cur); + const vOff = F.skipOne(a, n, cur); + if (!vOff) break; + const pEnd = F.skipOne(a, n, vOff); + if (!pEnd) break; + const key = ks !== null ? ks : "?"; + treeWalk(a, n, vOff, `${zfull}.${key}`, zfull, depth + 1, rows); + cur = pEnd; + } + } +} diff --git a/js/src/iterator.ts b/js/src/iterator.ts new file mode 100644 index 0000000..23360f4 --- /dev/null +++ b/js/src/iterator.ts @@ -0,0 +1,67 @@ +/* Iterator — a cursor over container children (flat each / recursive tree). */ + +import * as D from "./decode.ts"; +import { EachRow, eachIter, treeWalk } from "./iterate.ts"; +import { Blob } from "./blob.ts"; + +export { EachRow }; + +export class Iterator { + _blob: Blob; + _base: string; + _recursive: boolean; + _rows: EachRow[]; + _cursor: number; + _populated: boolean; + + constructor(blob: Blob, path: string = "$", recursive: boolean = false) { + this._blob = blob; + this._base = path ? path : "$"; + this._recursive = recursive; + this._rows = []; + this._cursor = -1; + this._populated = false; + } + + _populate(): void { + if (this._populated) return; + this._populated = true; + this._rows = []; + const a = this._blob.data(); + const n = a.length; + if (n === 0) return; + + let iroot = 0; + if (this._base !== "$") { + const r = D.lookup(a, n, 0, this._base); + if (r.rc !== D.RC_OK) return; + iroot = r.iStart; + } + + if (this._recursive) treeWalk(a, n, iroot, this._base, this._base, 0, this._rows); + else this._rows = eachIter(a, n, iroot, this._base); + } + + // ── C++-style cursor protocol ───────────────────────────────────── + next(): boolean { + this._populate(); + this._cursor += 1; + return this._cursor < this._rows.length; + } + current(): EachRow { + return this._rows[this._cursor]; + } + reset(): void { + this._cursor = -1; + } + + // ── iterable protocol ───────────────────────────────────────────── + rows(): EachRow[] { + this._populate(); + return this._rows.slice(); + } + [Symbol.iterator](): IterableIterator { + this._populate(); + return this._rows.slice()[Symbol.iterator](); + } +} diff --git a/js/src/json.ts b/js/src/json.ts new file mode 100644 index 0000000..50bc698 --- /dev/null +++ b/js/src/json.ts @@ -0,0 +1,607 @@ +/* + * Internal: JSON conversion (mirrors msgpack_blob_json.cpp). + * + * to_json builds a byte buffer exactly like the C++ implementation so float + * formatting and string escaping stay byte-identical. The float formatter + * reproduces C printf "%.

g" with round-half-to-even using exact BigInt + * arithmetic. + */ + +import * as F from "./format.ts"; +import { Buf } from "./format.ts"; +import * as E from "./encode.ts"; + +export const RC_OK = 0; +export const RC_ERROR = 1; + +const I64_MIN = -(1n << 63n); +const I64_MAX = (1n << 63n) - 1n; +const HEX = "0123456789abcdef"; + +// ── C printf "%.

g" with round-half-to-even ─────────────────────── +function decompose(x: number): [bigint, number] { + const dv = new DataView(new ArrayBuffer(8)); + dv.setFloat64(0, x, false); + const hi = dv.getUint32(0, false) >>> 0; + const lo = dv.getUint32(4, false) >>> 0; + const expBits = (hi >>> 20) & 0x7ff; + const mantHi = hi & 0xfffff; + let mant = (BigInt(mantHi) << 32n) | BigInt(lo); + let exp: number; + if (expBits === 0) { + exp = -1074; + } else { + mant |= 1n << 52n; + exp = expBits - 1075; + } + return [mant, exp]; +} + +// Sign of (m*2^e - 10^k), computed exactly with BigInt. +function cmpPow10(m: bigint, e: number, k: number): number { + let lhs = m; + let rhs = 1n; + if (e >= 0) lhs <<= BigInt(e); + else rhs <<= BigInt(-e); + if (k >= 0) rhs *= 10n ** BigInt(k); + else lhs *= 10n ** BigInt(-k); + return lhs < rhs ? -1 : lhs > rhs ? 1 : 0; +} + +function roundToDigits(m: bigint, e: number, p: number): [string, number] { + // Exact decimal exponent: the integer x with 10^x <= value < 10^(x+1). + let x = Math.floor((Math.log(Number(m)) + e * Math.LN2) / Math.LN10); + if (!Number.isFinite(x)) x = 0; + while (cmpPow10(m, e, x + 1) >= 0) x += 1; + while (cmpPow10(m, e, x) < 0) x -= 1; + + // Round value / 10^(x+1-p) to the nearest integer, ties to even. + const s = x + 1 - p; + let num = m; + let den = 1n; + if (e >= 0) num <<= BigInt(e); + else den <<= BigInt(-e); + if (s >= 0) den *= 10n ** BigInt(s); + else num *= 10n ** BigInt(-s); + let q = num / den; + const r = num % den; + const twice = r * 2n; + if (twice > den || (twice === den && (q & 1n) === 1n)) q += 1n; + + let digits = q.toString(); + if (digits.length > p) { + // Rounding carried across a power of ten (e.g. 9.99e0 -> 1.0e1). + x += 1; + digits = "1" + "0".repeat(p - 1); + } + return [digits, x]; +} + +export function cFormatG(value: number, p: number): string { + if (p <= 0) p = 1; + if (value === 0) return Object.is(value, -0) ? "-0" : "0"; + const neg = value < 0; + const [m, e] = decompose(Math.abs(value)); + const [digits, x] = roundToDigits(m, e, p); + let out: string; + if (x >= -4 && x < p) { + if (x >= 0) { + const intLen = x + 1; + const frac = digits.slice(intLen); + out = digits.slice(0, intLen) + (frac ? "." + frac : ""); + } else { + out = "0." + "0".repeat(-x - 1) + digits; + } + if (out.indexOf(".") >= 0) out = out.replace(/0+$/, "").replace(/\.$/, ""); + } else { + let mant = digits[0] + (digits.length > 1 ? "." + digits.slice(1) : ""); + if (mant.indexOf(".") >= 0) mant = mant.replace(/0+$/, "").replace(/\.$/, ""); + let ea = Math.abs(x).toString(); + if (ea.length < 2) ea = "0" + ea; + out = mant + "e" + (x < 0 ? "-" : "+") + ea; + } + return neg ? "-" + out : out; +} + +function fmtDouble(d: number): string { + let s = cFormatG(d, 17); + if (!/[.eE]/.test(s)) { + s = (d < 0 || Object.is(d, -0) ? "-" : "") + Math.abs(d).toFixed(1); + } + return s; +} + +function fmtFloat32(f: number): string { + return cFormatG(f, 7); +} + +// ── JSON output ───────────────────────────────────────────────────── +function pushAscii(out: Buf, s: string): void { + for (let i = 0; i < s.length; i++) out.push(s.charCodeAt(i)); +} + +function escapeStr(out: Buf, s: Uint8Array): void { + out.push(0x22); + let start = 0; + const n = s.length; + for (let j = 0; j < n; j++) { + const c = s[j]; + if (c >= 0x20 && c !== 0x22 && c !== 0x5c) continue; + if (j > start) out.pushSlice(s, start, j); + if (c === 0x22) { + out.push(0x5c); + out.push(0x22); + } else if (c === 0x5c) { + out.push(0x5c); + out.push(0x5c); + } else if (c === 0x0a) { + out.push(0x5c); + out.push(0x6e); + } else if (c === 0x0d) { + out.push(0x5c); + out.push(0x72); + } else if (c === 0x09) { + out.push(0x5c); + out.push(0x74); + } else { + pushAscii(out, "\\u" + c.toString(16).padStart(4, "0")); + } + start = j + 1; + } + if (n > start) out.pushSlice(s, start, n); + out.push(0x22); +} + +function newline(out: Buf, depth: number, indentW: number): void { + out.push(0x0a); + const sp = depth * indentW; + for (let k = 0; k < sp; k++) out.push(0x20); +} + +function toJsonAt(out: Buf, a: Uint8Array, n: number, i: number, pretty: boolean, depth: number, indentW: number): void { + if (i >= n || depth > F.MAX_DEPTH) { + pushAscii(out, "null"); + return; + } + const b = a[i]; + + if (b === F.MP_NIL) { + pushAscii(out, "null"); + return; + } + if (b === F.MP_FALSE) { + pushAscii(out, "false"); + return; + } + if (b === F.MP_TRUE) { + pushAscii(out, "true"); + return; + } + if (b <= 0x7f) { + pushAscii(out, String(b)); + return; + } + if (b >= 0xe0) { + pushAscii(out, String(b - 256)); + return; + } + + switch (b) { + case F.MP_UINT8: + if (i + 2 <= n) { + pushAscii(out, String(a[i + 1])); + return; + } + break; + case F.MP_UINT16: + if (i + 3 <= n) { + pushAscii(out, String(F.read16(a, i + 1))); + return; + } + break; + case F.MP_UINT32: + if (i + 5 <= n) { + pushAscii(out, String(F.read32(a, i + 1))); + return; + } + break; + case F.MP_UINT64: + if (i + 9 <= n) { + pushAscii(out, F.read64(a, i + 1).toString()); + return; + } + break; + case F.MP_INT8: + if (i + 2 <= n) { + const v = a[i + 1]; + pushAscii(out, String(v >= 128 ? v - 256 : v)); + return; + } + break; + case F.MP_INT16: + if (i + 3 <= n) { + const v = F.read16(a, i + 1); + pushAscii(out, String(v >= 0x8000 ? v - 0x10000 : v)); + return; + } + break; + case F.MP_INT32: + if (i + 5 <= n) { + const v = F.read32(a, i + 1); + pushAscii(out, String(v >= 0x80000000 ? v - 0x100000000 : v)); + return; + } + break; + case F.MP_INT64: + if (i + 9 <= n) { + pushAscii(out, BigInt.asIntN(64, F.read64(a, i + 1)).toString()); + return; + } + break; + case F.MP_FLOAT32: + if (i + 5 <= n) { + const f = F.readF32(a, i + 1); + if (!Number.isFinite(f)) { + pushAscii(out, "null"); + return; + } + pushAscii(out, fmtFloat32(f)); + return; + } + break; + case F.MP_FLOAT64: + if (i + 9 <= n) { + const d = F.readF64(a, i + 1); + if (!Number.isFinite(d)) { + pushAscii(out, "null"); + return; + } + pushAscii(out, fmtDouble(d)); + return; + } + break; + default: + break; + } + + // str + let soff = 0; + let slen = 0; + if (b >= 0xa0 && b <= 0xbf) { + slen = b & 0x1f; + soff = i + 1; + } else if (b === F.MP_STR8 && i + 2 <= n) { + slen = a[i + 1]; + soff = i + 2; + } else if (b === F.MP_STR16 && i + 3 <= n) { + slen = F.read16(a, i + 1); + soff = i + 3; + } else if (b === F.MP_STR32 && i + 5 <= n) { + slen = F.read32(a, i + 1); + soff = i + 5; + } + if (soff) { + if (slen > n - soff) slen = n - soff; + escapeStr(out, a.subarray(soff, soff + slen)); + return; + } + + // bin → hex string + let boff = 0; + let blen = 0; + if (b === F.MP_BIN8 && i + 2 <= n) { + blen = a[i + 1]; + boff = i + 2; + } else if (b === F.MP_BIN16 && i + 3 <= n) { + blen = F.read16(a, i + 1); + boff = i + 3; + } else if (b === F.MP_BIN32 && i + 5 <= n) { + blen = F.read32(a, i + 1); + boff = i + 5; + } + if (boff) { + if (blen > n - boff) blen = n - boff; + out.push(0x22); + for (let j = 0; j < blen; j++) { + const by = a[boff + j]; + out.push(HEX.charCodeAt(by >> 4)); + out.push(HEX.charCodeAt(by & 0xf)); + } + out.push(0x22); + return; + } + + // array + let isArr = false; + let count = 0; + let dataOff = 0; + if (b >= 0x90 && b <= 0x9f) { + isArr = true; + count = b & 0x0f; + dataOff = i + 1; + } else if (b === F.MP_ARRAY16 && i + 3 <= n) { + isArr = true; + count = F.read16(a, i + 1); + dataOff = i + 3; + } else if (b === F.MP_ARRAY32 && i + 5 <= n) { + isArr = true; + count = F.read32(a, i + 1); + dataOff = i + 5; + } + if (isArr) { + let cur = dataOff; + out.push(0x5b); + for (let j = 0; j < count; j++) { + if (cur >= n) break; + const nxt = F.skipOne(a, n, cur); + if (j > 0) out.push(0x2c); + if (pretty) newline(out, depth + 1, indentW); + toJsonAt(out, a, n, cur, pretty, depth + 1, indentW); + cur = nxt ? nxt : n; + } + if (pretty && count > 0) newline(out, depth, indentW); + out.push(0x5d); + return; + } + + // map + let isMap = false; + count = 0; + dataOff = 0; + if (b >= 0x80 && b <= 0x8f) { + isMap = true; + count = b & 0x0f; + dataOff = i + 1; + } else if (b === F.MP_MAP16 && i + 3 <= n) { + isMap = true; + count = F.read16(a, i + 1); + dataOff = i + 3; + } else if (b === F.MP_MAP32 && i + 5 <= n) { + isMap = true; + count = F.read32(a, i + 1); + dataOff = i + 5; + } + if (isMap) { + let cur = dataOff; + out.push(0x7b); + for (let j = 0; j < count; j++) { + if (cur >= n) break; + const valOff = F.skipOne(a, n, cur); + const pairEnd = valOff ? F.skipOne(a, n, valOff) : 0; + if (j > 0) out.push(0x2c); + if (pretty) newline(out, depth + 1, indentW); + toJsonAt(out, a, n, cur, pretty, depth + 1, indentW); + out.push(0x3a); + if (pretty) out.push(0x20); + toJsonAt(out, a, n, valOff ? valOff : n, pretty, depth + 1, indentW); + cur = pairEnd ? pairEnd : n; + } + if (pretty && count > 0) newline(out, depth, indentW); + out.push(0x7d); + return; + } + + // ext / unknown → null + pushAscii(out, "null"); +} + +export function toJson(a: Uint8Array, n: number, pretty: boolean, indent: number): string { + const out = new Buf(); + toJsonAt(out, a, n, 0, pretty, 0, indent); + return F.utf8Decode(out.toBytes()); +} + +// ── JSON parser → msgpack ─────────────────────────────────────────── +class P { + z: Uint8Array; + n: number; + i: number; + constructor(z: Uint8Array) { + this.z = z; + this.n = z.length; + this.i = 0; + } +} + +function skipWs(p: P): void { + while (p.i < p.n && (p.z[p.i] === 0x20 || p.z[p.i] === 0x09 || p.z[p.i] === 0x0a || p.z[p.i] === 0x0d)) p.i++; +} + +function hex4(z: Uint8Array, off: number): number { + let v = 0; + for (let j = 0; j < 4; j++) { + const c = z[off + j]; + let h: number; + if (c >= 0x30 && c <= 0x39) h = c - 0x30; + else if (c >= 0x61 && c <= 0x66) h = c - 0x61 + 10; + else if (c >= 0x41 && c <= 0x46) h = c - 0x41 + 10; + else return -1; + v = (v << 4) | h; + } + return v; +} + +function cpToUtf8(out: Buf, cp: number): void { + if (cp < 0x80) out.push(cp); + else if (cp < 0x800) { + out.push(0xc0 | (cp >> 6)); + out.push(0x80 | (cp & 0x3f)); + } else if (cp < 0x10000) { + out.push(0xe0 | (cp >> 12)); + out.push(0x80 | ((cp >> 6) & 0x3f)); + out.push(0x80 | (cp & 0x3f)); + } else { + out.push(0xf0 | (cp >> 18)); + out.push(0x80 | ((cp >> 12) & 0x3f)); + out.push(0x80 | ((cp >> 6) & 0x3f)); + out.push(0x80 | (cp & 0x3f)); + } +} + +function parseString(p: P, out: Buf): number { + const sb = new Buf(); + p.i++; + while (p.i < p.n) { + const c = p.z[p.i]; + if (c === 0x22) { + p.i++; + break; + } + if (c === 0x5c) { + p.i++; + if (p.i >= p.n) return RC_ERROR; + const esc = p.z[p.i++]; + if (esc === 0x22) sb.push(0x22); + else if (esc === 0x5c) sb.push(0x5c); + else if (esc === 0x2f) sb.push(0x2f); + else if (esc === 0x6e) sb.push(0x0a); + else if (esc === 0x72) sb.push(0x0d); + else if (esc === 0x74) sb.push(0x09); + else if (esc === 0x62) sb.push(0x08); + else if (esc === 0x66) sb.push(0x0c); + else if (esc === 0x75) { + if (p.i + 4 > p.n) return RC_ERROR; + let cp = hex4(p.z, p.i); + p.i += 4; + if (cp < 0) return RC_ERROR; + if (cp >= 0xd800 && cp <= 0xdbff && p.i + 6 <= p.n && p.z[p.i] === 0x5c && p.z[p.i + 1] === 0x75) { + const lo = hex4(p.z, p.i + 2); + if (lo >= 0xdc00 && lo <= 0xdfff) { + p.i += 6; + cp = 0x10000 + ((cp - 0xd800) << 10) + (lo - 0xdc00); + } + } + cpToUtf8(sb, cp); + } else sb.push(esc); + } else { + sb.push(c); + p.i++; + } + } + E.encString(out, sb.toBytes()); + return RC_OK; +} + +function parseNumber(p: P, out: Buf): number { + const start = p.i; + let isFloat = false; + if (p.i < p.n && p.z[p.i] === 0x2d) p.i++; + while (p.i < p.n && p.z[p.i] >= 0x30 && p.z[p.i] <= 0x39) p.i++; + if (p.i < p.n && p.z[p.i] === 0x2e) { + isFloat = true; + p.i++; + while (p.i < p.n && p.z[p.i] >= 0x30 && p.z[p.i] <= 0x39) p.i++; + } + if (p.i < p.n && (p.z[p.i] === 0x65 || p.z[p.i] === 0x45)) { + isFloat = true; + p.i++; + if (p.i < p.n && (p.z[p.i] === 0x2b || p.z[p.i] === 0x2d)) p.i++; + while (p.i < p.n && p.z[p.i] >= 0x30 && p.z[p.i] <= 0x39) p.i++; + } + const length = p.i - start; + if (length <= 0 || length >= 64) return RC_ERROR; + const text = F.utf8Decode(p.z.subarray(start, p.i)); + + if (isFloat) { + E.encReal(out, Number(text)); + } else { + let v = BigInt(text); + if (v > I64_MAX) v = I64_MAX; + else if (v < I64_MIN) v = I64_MIN; + if (v >= 0n) E.encUnsigned(out, v); + else E.encInteger(out, v); + } + return RC_OK; +} + +function parseArray(p: P, out: Buf): number { + const tmp = new Buf(); + let count = 0; + p.i++; + skipWs(p); + while (p.i < p.n && p.z[p.i] !== 0x5d) { + if (count > 0) { + skipWs(p); + if (p.i >= p.n || p.z[p.i] !== 0x2c) return RC_ERROR; + p.i++; + } + skipWs(p); + if (parseValue(p, tmp) !== RC_OK) return RC_ERROR; + count++; + skipWs(p); + } + if (p.i >= p.n) return RC_ERROR; + p.i++; + E.encArrayHeader(out, count); + out.pushBytes(tmp.toBytes()); + return RC_OK; +} + +function parseObject(p: P, out: Buf): number { + const tmp = new Buf(); + let count = 0; + p.i++; + skipWs(p); + while (p.i < p.n && p.z[p.i] !== 0x7d) { + if (count > 0) { + skipWs(p); + if (p.i >= p.n || p.z[p.i] !== 0x2c) return RC_ERROR; + p.i++; + } + skipWs(p); + if (p.i >= p.n || p.z[p.i] !== 0x22) return RC_ERROR; + if (parseString(p, tmp) !== RC_OK) return RC_ERROR; + skipWs(p); + if (p.i >= p.n || p.z[p.i] !== 0x3a) return RC_ERROR; + p.i++; + skipWs(p); + if (parseValue(p, tmp) !== RC_OK) return RC_ERROR; + count++; + skipWs(p); + } + if (p.i >= p.n) return RC_ERROR; + p.i++; + E.encMapHeader(out, count); + out.pushBytes(tmp.toBytes()); + return RC_OK; +} + +function matches(z: Uint8Array, i: number, word: string): boolean { + for (let k = 0; k < word.length; k++) if (z[i + k] !== word.charCodeAt(k)) return false; + return true; +} + +function parseValue(p: P, out: Buf): number { + skipWs(p); + if (p.i >= p.n) return RC_ERROR; + const c = p.z[p.i]; + if (c === 0x6e && p.i + 4 <= p.n && matches(p.z, p.i, "null")) { + p.i += 4; + out.push(F.MP_NIL); + return RC_OK; + } + if (c === 0x74 && p.i + 4 <= p.n && matches(p.z, p.i, "true")) { + p.i += 4; + out.push(F.MP_TRUE); + return RC_OK; + } + if (c === 0x66 && p.i + 5 <= p.n && matches(p.z, p.i, "false")) { + p.i += 5; + out.push(F.MP_FALSE); + return RC_OK; + } + if (c === 0x22) return parseString(p, out); + if (c === 0x5b) return parseArray(p, out); + if (c === 0x7b) return parseObject(p, out); + if (c === 0x2d || (c >= 0x30 && c <= 0x39)) return parseNumber(p, out); + return RC_ERROR; +} + +export function fromJson(json: string | Uint8Array | null | undefined): Uint8Array { + if (json === null || json === undefined) return new Uint8Array(0); + const z = typeof json === "string" ? F.utf8Encode(json) : json; + const p = new P(z); + const out = new Buf(); + if (parseValue(p, out) !== RC_OK) return new Uint8Array(0); + return out.toBytes(); +} diff --git a/js/src/mutate.ts b/js/src/mutate.ts new file mode 100644 index 0000000..3128f23 --- /dev/null +++ b/js/src/mutate.ts @@ -0,0 +1,401 @@ +/* Internal: copy-on-write mutation (mirrors msgpack_blob_mutate.cpp). */ + +import * as F from "./format.ts"; +import { Buf } from "./format.ts"; +import * as E from "./encode.ts"; +import { pathStep } from "./decode.ts"; + +export const RC_OK = 0; +export const RC_ERROR = 1; +export const RC_NOTFOUND = 2; + +export const EDIT_SET = 0; +export const EDIT_INSERT = 1; +export const EDIT_REPLACE = 2; +export const EDIT_REMOVE = 3; +export const EDIT_ARRAY_INS = 4; + +function mapKey(a: Uint8Array, n: number, i: number): Uint8Array | null { + const kb = a[i]; + let klen: number; + let koff: number; + if (kb >= 0xa0 && kb <= 0xbf) { + klen = kb & 0x1f; + koff = i + 1; + } else if (kb === F.MP_STR8 && i + 2 <= n) { + klen = a[i + 1]; + koff = i + 2; + } else if (kb === F.MP_STR16 && i + 3 <= n) { + klen = F.read16(a, i + 1); + koff = i + 3; + } else if (kb === F.MP_STR32 && i + 5 <= n) { + klen = F.read32(a, i + 1); + koff = i + 5; + } else { + return null; + } + return a.subarray(koff, koff + klen); +} + +function bytesEqual(x: Uint8Array, y: Uint8Array): boolean { + if (x.length !== y.length) return false; + for (let i = 0; i < x.length; i++) if (x[i] !== y[i]) return false; + return true; +} + +interface StepResult { + rc: number; + skip: boolean; +} + +function editMap( + out: Buf, + a: Uint8Array, + n: number, + icur: number, + zkey: Uint8Array, + zpath: string, + pi: number, + newBin: Uint8Array, + mode: number, +): number { + if (icur >= n) return RC_ERROR; + const b = a[icur]; + let count: number; + let dataOff: number; + if (b >= 0x80 && b <= 0x8f) { + count = b & 0x0f; + dataOff = icur + 1; + } else if (b === F.MP_MAP16) { + if (icur + 3 > n) return RC_ERROR; + count = F.read16(a, icur + 1); + dataOff = icur + 3; + } else if (b === F.MP_MAP32) { + if (icur + 5 > n) return RC_ERROR; + count = F.read32(a, icur + 1); + dataOff = icur + 5; + } else { + if (mode === EDIT_REPLACE || mode === EDIT_REMOVE) { + const iend = F.skipOne(a, n, icur); + if (iend) out.pushSlice(a, icur, iend); + return RC_OK; + } + return RC_ERROR; + } + + let newCount = count; + const tmp = new Buf(); + let cur2 = dataOff; + let foundKey = false; + + for (let j = 0; j < count; j++) { + if (cur2 >= n) return RC_ERROR; + const kstr = mapKey(a, n, cur2); + const valOff = F.skipOne(a, n, cur2); + if (!valOff) return RC_ERROR; + const pairEnd = F.skipOne(a, n, valOff); + if (!pairEnd) return RC_ERROR; + + const isMatch = kstr !== null && bytesEqual(kstr, zkey); + + if (isMatch) { + foundKey = true; + if (mode === EDIT_INSERT) { + tmp.pushSlice(a, cur2, pairEnd); + } else { + const vbuf = new Buf(); + const res = editStep(vbuf, a, n, valOff, zpath, pi, newBin, mode); + if (res.rc !== RC_OK) return res.rc; + if (res.skip) { + newCount--; + } else { + tmp.pushSlice(a, cur2, valOff); + tmp.pushBytes(vbuf.toBytes()); + } + } + } else { + tmp.pushSlice(a, cur2, pairEnd); + } + cur2 = pairEnd; + } + + if (!foundKey) { + if (mode === EDIT_SET || mode === EDIT_INSERT) { + if (pathStep(zpath, pi).kind !== 0) { + const iend = F.skipOne(a, n, icur); + if (iend) out.pushSlice(a, icur, iend); + return RC_OK; + } + E.encString(tmp, zkey); + tmp.pushBytes(newBin); + newCount++; + } else { + const iend = F.skipOne(a, n, icur); + if (iend) out.pushSlice(a, icur, iend); + return RC_OK; + } + } + + E.encMapHeader(out, newCount); + out.pushBytes(tmp.toBytes()); + return RC_OK; +} + +function editArray( + out: Buf, + a: Uint8Array, + n: number, + icur: number, + stepIdx: number, + zpath: string, + pi: number, + newBin: Uint8Array, + mode: number, +): number { + if (icur >= n) return RC_ERROR; + const b = a[icur]; + let count: number; + let dataOff: number; + if (b >= 0x90 && b <= 0x9f) { + count = b & 0x0f; + dataOff = icur + 1; + } else if (b === F.MP_ARRAY16) { + if (icur + 3 > n) return RC_ERROR; + count = F.read16(a, icur + 1); + dataOff = icur + 3; + } else if (b === F.MP_ARRAY32) { + if (icur + 5 > n) return RC_ERROR; + count = F.read32(a, icur + 1); + dataOff = icur + 5; + } else { + if (mode === EDIT_REPLACE || mode === EDIT_REMOVE) { + const iend = F.skipOne(a, n, icur); + if (iend) out.pushSlice(a, icur, iend); + return RC_OK; + } + return RC_ERROR; + } + + let newCount = count; + const tmp = new Buf(); + let cur2 = dataOff; + let foundIt = false; + + for (let j = 0; j < count; j++) { + const eEnd = F.skipOne(a, n, cur2); + if (!eEnd) return RC_ERROR; + + if (j === stepIdx) { + foundIt = true; + if (mode === EDIT_ARRAY_INS) { + tmp.pushBytes(newBin); + tmp.pushSlice(a, cur2, eEnd); + newCount++; + } else if (mode === EDIT_INSERT) { + tmp.pushSlice(a, cur2, eEnd); + } else { + const ebuf = new Buf(); + const res = editStep(ebuf, a, n, cur2, zpath, pi, newBin, mode); + if (res.rc !== RC_OK) return res.rc; + if (res.skip) newCount--; + else tmp.pushBytes(ebuf.toBytes()); + } + } else { + tmp.pushSlice(a, cur2, eEnd); + } + cur2 = eEnd; + } + + if (!foundIt) { + if (mode === EDIT_ARRAY_INS) { + tmp.pushBytes(newBin); + newCount++; + } else if ((mode === EDIT_SET || mode === EDIT_INSERT) && stepIdx === count) { + tmp.pushBytes(newBin); + newCount++; + } else if (mode === EDIT_REPLACE || mode === EDIT_REMOVE) { + const iend = F.skipOne(a, n, icur); + if (iend) out.pushSlice(a, icur, iend); + return RC_OK; + } else { + return RC_NOTFOUND; + } + } + + E.encArrayHeader(out, newCount); + out.pushBytes(tmp.toBytes()); + return RC_OK; +} + +function editStep( + out: Buf, + a: Uint8Array, + n: number, + icur: number, + zpath: string, + pi: number, + newBin: Uint8Array, + mode: number, +): StepResult { + const st = pathStep(zpath, pi); + + if (st.kind === 0) { + if (mode === EDIT_REMOVE) return { rc: RC_OK, skip: true }; + if (mode === EDIT_ARRAY_INS) return { rc: RC_ERROR, skip: false }; + if (mode === EDIT_INSERT) { + const iend = F.skipOne(a, n, icur); + if (iend) out.pushSlice(a, icur, iend); + return { rc: RC_OK, skip: false }; + } + out.pushBytes(newBin); + return { rc: RC_OK, skip: false }; + } + if (st.kind === -1) return { rc: RC_ERROR, skip: false }; + + if (st.kind === "k") { + const zkey = F.utf8Encode(st.key); + return { rc: editMap(out, a, n, icur, zkey, zpath, st.pi, newBin, mode), skip: false }; + } + return { rc: editArray(out, a, n, icur, st.idx, zpath, st.pi, newBin, mode), skip: false }; +} + +export function applyEdit(a: Uint8Array, n: number, zpath: string, newBin: Uint8Array, mode: number): { rc: number; out: Uint8Array } { + if (!zpath || zpath[0] !== "$") return { rc: RC_ERROR, out: new Uint8Array(0) }; + const out = new Buf(); + const res = editStep(out, a, n, 0, zpath, 1, newBin, mode); + return { rc: res.rc, out: out.toBytes() }; +} + +// ── merge_patch (RFC 7386) ────────────────────────────────────────── +export function mergePatch(a: Uint8Array, n: number, ia: number, p: Uint8Array, np: number, ip: number, depth: number): { rc: number; out: Uint8Array } { + const out = new Buf(); + const rc = mergePatchInto(out, a, n, ia, p, np, ip, depth); + return { rc, out: out.toBytes() }; +} + +function mergePatchInto(out: Buf, a: Uint8Array, n: number, ia: number, p: Uint8Array, np: number, ip: number, depth: number): number { + if (ip >= np) return RC_ERROR; + if (depth > F.MAX_DEPTH) return RC_ERROR; + const pb = p[ip]; + + if (pb === F.MP_NIL) { + out.push(F.MP_NIL); + return RC_OK; + } + + const pIsMap = (pb >= 0x80 && pb <= 0x8f) || pb === F.MP_MAP16 || pb === F.MP_MAP32; + if (!pIsMap) { + const pEnd = F.skipOne(p, np, ip); + if (pEnd) out.pushSlice(p, ip, pEnd); + return RC_OK; + } + + const ab = ia < n ? a[ia] : 0; + let aIsMap = (ab >= 0x80 && ab <= 0x8f) || ab === F.MP_MAP16 || ab === F.MP_MAP32; + + let pCount: number; + let pDataOff: number; + if (pb >= 0x80 && pb <= 0x8f) { + pCount = pb & 0x0f; + pDataOff = ip + 1; + } else if (pb === F.MP_MAP16) { + if (ip + 3 > np) return RC_ERROR; + pCount = F.read16(p, ip + 1); + pDataOff = ip + 3; + } else { + if (ip + 5 > np) return RC_ERROR; + pCount = F.read32(p, ip + 1); + pDataOff = ip + 5; + } + + let aCount = 0; + let aDataOff = 0; + if (aIsMap) { + if (ab >= 0x80 && ab <= 0x8f) { + aCount = ab & 0x0f; + aDataOff = ia + 1; + } else if (ab === F.MP_MAP16) { + if (ia + 3 > n) aIsMap = false; + else { + aCount = F.read16(a, ia + 1); + aDataOff = ia + 3; + } + } else { + if (ia + 5 > n) aIsMap = false; + else { + aCount = F.read32(a, ia + 1); + aDataOff = ia + 5; + } + } + } + + // Pre-scan patch keys: [key|null, keyOff, valOff, pairEnd, matched] + if (pCount > Math.floor((np - pDataOff) / 2) + 1) return RC_ERROR; + const pIdx: Array<[Uint8Array | null, number, number, number, boolean]> = []; + let pc2 = pDataOff; + for (let k = 0; k < pCount; k++) { + if (pc2 >= np) return RC_ERROR; + const key = mapKey(p, np, pc2); + const valOff = F.skipOne(p, np, pc2); + if (!valOff) return RC_ERROR; + const pairEnd = F.skipOne(p, np, valOff); + if (!pairEnd) return RC_ERROR; + pIdx.push([key, pc2, valOff, pairEnd, false]); + pc2 = pairEnd; + } + + const tmp = new Buf(); + let newCount = 0; + + if (aIsMap) { + let ac = aDataOff; + for (let j = 0; j < aCount; j++) { + if (ac >= n) return RC_ERROR; + const kstr = mapKey(a, n, ac); + const aValOff = F.skipOne(a, n, ac); + if (!aValOff) return RC_ERROR; + const aPairEnd = F.skipOne(a, n, aValOff); + if (!aPairEnd) return RC_ERROR; + + let foundInPatch = false; + let patchIsNil = false; + let pMatchVal = 0; + for (const entry of pIdx) { + if (entry[0] !== null && kstr !== null && bytesEqual(entry[0], kstr)) { + foundInPatch = true; + pMatchVal = entry[2]; + patchIsNil = entry[2] < np && p[entry[2]] === F.MP_NIL; + entry[4] = true; + break; + } + } + + if (foundInPatch && patchIsNil) { + // drop + } else if (foundInPatch) { + const mb = new Buf(); + const mrc = mergePatchInto(mb, a, n, aValOff, p, np, pMatchVal, depth + 1); + if (mrc === RC_OK) { + tmp.pushSlice(a, ac, aValOff); + tmp.pushBytes(mb.toBytes()); + newCount++; + } + } else { + tmp.pushSlice(a, ac, aPairEnd); + newCount++; + } + ac = aPairEnd; + } + } + + for (const entry of pIdx) { + if (!entry[4] && entry[2] < np && p[entry[2]] !== F.MP_NIL) { + tmp.pushSlice(p, entry[1], entry[3]); + newCount++; + } + } + + E.encMapHeader(out, newCount); + out.pushBytes(tmp.toBytes()); + return RC_OK; +} diff --git a/js/src/value.ts b/js/src/value.ts new file mode 100644 index 0000000..02412e1 --- /dev/null +++ b/js/src/value.ts @@ -0,0 +1,219 @@ +/* + * Value — a decoded scalar or sub-blob MessagePack value. + * + * Mirrors msgpack::Value from the C++ Blob library. Integer values use bigint + * so the full signed/unsigned 64-bit range round-trips exactly. + */ + +import { utf8Decode, utf8Encode } from "./format.ts"; + +export const Type = { + Nil: "null", + True: "true", + False: "false", + Integer: "integer", + Real: "real", + Float32: "float32", + String: "text", + Binary: "binary", + Array: "array", + Map: "map", + Ext: "ext", + Timestamp: "timestamp", +} as const; +export type Type = (typeof Type)[keyof typeof Type]; + +export const IntWidth = { + Auto: 0, + Int8: 1, + Int16: 2, + Int32: 3, + Int64: 4, + Uint8: 5, + Uint16: 6, + Uint32: 7, + Uint64: 8, +} as const; +export type IntWidth = (typeof IntWidth)[keyof typeof IntWidth]; + +export function typeStr(t: Type): string { + return t; +} + +const I64_MAX = (1n << 63n) - 1n; + +export class Value { + _type: Type; + _int: bigint; + _float: number; + _str: Uint8Array; + _blob: Uint8Array; + _extType: number; + _tsSec: bigint; + _tsNsec: number; + _intWidth: IntWidth; + + constructor() { + this._type = Type.Nil; + this._int = 0n; + this._float = 0; + this._str = new Uint8Array(0); + this._blob = new Uint8Array(0); + this._extType = 0; + this._tsSec = 0n; + this._tsNsec = 0; + this._intWidth = IntWidth.Auto; + } + + // ── accessors ───────────────────────────────────────────────────── + type(): Type { + return this._type; + } + isNil(): boolean { + return this._type === Type.Nil; + } + asBool(): boolean { + return this._type === Type.True; + } + asInt64(): bigint { + if (this._type === Type.Integer) return BigInt.asIntN(64, this._int); + if (this._type === Type.Real || this._type === Type.Float32) { + return BigInt(Math.trunc(this._float)); + } + if (this._type === Type.Timestamp) return this._tsSec; + if (this._type === Type.True) return 1n; + return 0n; + } + asUint64(): bigint { + if (this._type === Type.Integer) return BigInt.asUintN(64, this._int); + return 0n; + } + asDouble(): number { + if (this._type === Type.Real || this._type === Type.Float32) return this._float; + if (this._type === Type.Integer) return Number(this.asInt64()); + return 0; + } + asFloat(): number { + if (this._type === Type.Float32) return this._float; + if (this._type === Type.Real) return Math.fround(this._float); + return 0; + } + asString(): string { + if (this._type === Type.String) return utf8Decode(this._str); + return ""; + } + asBytes(): Uint8Array { + return this._type === Type.String ? this._str : new Uint8Array(0); + } + blobData(): Uint8Array { + return this._blob; + } + blobSize(): number { + return this._blob.length; + } + extType(): number { + return this._extType; + } + timestampSeconds(): bigint { + return this._type === Type.Timestamp ? this._tsSec : 0n; + } + timestampNanoseconds(): number { + return this._type === Type.Timestamp ? this._tsNsec : 0; + } + intWidth(): IntWidth { + return this._intWidth; + } + + // ── static constructors ─────────────────────────────────────────── + static nil(): Value { + return new Value(); + } + static boolean(b: boolean): Value { + const v = new Value(); + v._type = b ? Type.True : Type.False; + return v; + } + static integer(x: number | bigint): Value { + const v = new Value(); + v._type = Type.Integer; + v._int = BigInt(x); + return v; + } + static unsignedInteger(x: number | bigint): Value { + const v = new Value(); + v._type = Type.Integer; + v._int = BigInt.asUintN(64, BigInt(x)); + if (v._int > I64_MAX) v._intWidth = IntWidth.Uint64; + return v; + } + static real(d: number): Value { + const v = new Value(); + v._type = Type.Real; + v._float = d; + return v; + } + static real32(f: number): Value { + const v = new Value(); + v._type = Type.Float32; + v._float = Math.fround(f); + return v; + } + static string(s: string | Uint8Array): Value { + const v = new Value(); + v._type = Type.String; + v._str = typeof s === "string" ? utf8Encode(s) : s; + return v; + } + static binary(data: Uint8Array): Value { + const v = new Value(); + v._type = Type.Binary; + v._blob = data; + return v; + } + static ext(typeCode: number, data: Uint8Array): Value { + const v = new Value(); + v._type = Type.Ext; + v._extType = typeCode | 0; + v._blob = data; + return v; + } + static timestamp(seconds: number | bigint, nanoseconds: number = 0): Value { + const v = new Value(); + v._type = Type.Timestamp; + v._tsSec = BigInt(seconds); + v._tsNsec = nanoseconds | 0; + return v; + } + + static _fixed(width: IntWidth, x: number | bigint): Value { + const v = new Value(); + v._type = Type.Integer; + v._int = BigInt(x); + v._intWidth = width; + return v; + } + static int8(x: number | bigint): Value { + return Value._fixed(IntWidth.Int8, x); + } + static int16(x: number | bigint): Value { + return Value._fixed(IntWidth.Int16, x); + } + static int32(x: number | bigint): Value { + return Value._fixed(IntWidth.Int32, x); + } + static int64(x: number | bigint): Value { + return Value._fixed(IntWidth.Int64, x); + } + static uint8(x: number | bigint): Value { + return Value._fixed(IntWidth.Uint8, x); + } + static uint16(x: number | bigint): Value { + return Value._fixed(IntWidth.Uint16, x); + } + static uint32(x: number | bigint): Value { + return Value._fixed(IntWidth.Uint32, x); + } + static uint64(x: number | bigint): Value { + return Value._fixed(IntWidth.Uint64, x); + } +} diff --git a/js/test/api.test.ts b/js/test/api.test.ts new file mode 100644 index 0000000..8ea2e27 --- /dev/null +++ b/js/test/api.test.ts @@ -0,0 +1,153 @@ +/* API behaviour and round-trip tests for the TypeScript port. */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { Blob, Builder, Iterator, Type, Value, typeStr } from "../src/index.ts"; +import { utf8Encode } from "../src/format.ts"; + +test("Builder matches fromJson", () => { + const built = new Builder() + .mapHeader(3) + .string("name").string("Alice") + .string("age").integer(30) + .string("scores").arrayHeader(3) + .real(95.5).real(87.5).real(91.0) + .build(); + const ref = Blob.fromJson('{"name":"Alice","age":30,"scores":[95.5,87.5,91.0]}'); + assert.equal(built.hex(), ref.hex()); +}); + +test("quote round-trips type", () => { + const values = [ + Value.nil(), + Value.boolean(true), + Value.integer(-12345), + Value.real(3.25), + Value.real32(1.5), + Value.string("hello"), + Value.binary(Uint8Array.from([0xde, 0xad])), + Value.ext(7, Uint8Array.from([1, 2])), + Value.timestamp(1700000000n, 123456789), + ]; + for (const v of values) { + const blob = Builder.quote(v); + assert.ok(blob.valid()); + assert.equal(blob.extract("$").type(), v.type()); + } +}); + +const ROUND_TRIP = [ + "null", "true", "false", "0", "-1", "127", "128", "65536", + "1.5", "0.1", "1e10", '"hi"', "[]", "{}", "[1,2,3]", + '{"a":1,"b":[2,3],"c":{"d":true}}', + '{"u":"caf\\u00e9","emoji":"\\ud83d\\ude00"}', +]; + +test("json bytes stable across round-trip", () => { + for (const c of ROUND_TRIP) { + const once = Blob.fromJson(c); + const twice = Blob.fromJson(once.toJson()); + assert.equal(once.hex(), twice.hex(), c); + } +}); + +test("64-bit integers round-trip via bigint", () => { + const big = 18446744073709551615n; + const blob = Builder.quote(Value.uint64(big)); + assert.equal(blob.hex(), "cfffffffffffffffff"); + assert.equal(blob.extract("$").asUint64(), big); + assert.equal(blob.toJson(), "18446744073709551615"); + + const negMax = -9223372036854775808n; + const b2 = Builder.quote(Value.int64(negMax)); + assert.equal(b2.extract("$").asInt64(), negMax); +}); + +test("extraction", () => { + const blob = Blob.fromJson('{"name":"Alice","age":30,"tall":true,"pets":["cat","dog"],"addr":{"city":"NYC"}}'); + assert.equal(blob.extract("$.name").asString(), "Alice"); + assert.equal(blob.extract("$.age").asInt64(), 30n); + assert.equal(blob.extract("$.tall").asBool(), true); + assert.equal(blob.extract("$.pets[1]").asString(), "dog"); + assert.equal(blob.extract("$.addr.city").asString(), "NYC"); + assert.ok(blob.extract("$.nope").isNil()); + assert.equal(blob.typeStr("$.pets"), "array"); + assert.equal(blob.arrayLength("$.pets"), 2); + assert.equal(blob.arrayLength("$.name"), -1); +}); + +test("binary, ext and timestamp", () => { + const bin = new Builder().binary(Uint8Array.from([1, 2, 3, 4])).build(); + assert.equal(bin.extract("$").type(), Type.Binary); + assert.deepEqual([...bin.extract("$").blobData()], [1, 2, 3, 4]); + assert.equal(bin.toJson(), '"01020304"'); + + const ext = new Builder().ext(42, Uint8Array.from([0xaa, 0xbb])).build(); + assert.equal(ext.extract("$").extType(), 42); + + const ts = new Builder().timestamp(1700000000, 500000000).build(); + assert.equal(ts.extract("$").timestampSeconds(), 1700000000n); + assert.equal(ts.extract("$").timestampNanoseconds(), 500000000); +}); + +test("copy-on-write mutation", () => { + const orig = Blob.fromJson('{"a":1}'); + const updated = orig.set("$.b", Value.integer(2)); + assert.equal(orig.toJson(), '{"a":1}'); + assert.equal(updated.toJson(), '{"a":1,"b":2}'); + + const b = Blob.fromJson('{"a":1,"b":2,"c":3}'); + assert.equal(b.remove("$.b").toJson(), '{"a":1,"c":3}'); + assert.equal(b.patch(Blob.fromJson('{"b":null,"d":4}')).toJson(), '{"a":1,"c":3,"d":4}'); + + const arr = Blob.fromJson("[1,2,3]"); + assert.equal(arr.arrayInsert("$[1]", Value.integer(9)).toJson(), "[1,9,2,3]"); + assert.equal(arr.set("$[3]", Value.integer(4)).toJson(), "[1,2,3,4]"); +}); + +test("iterator each and tree", () => { + const map = Blob.fromJson('{"a":1,"b":2,"c":3}'); + assert.deepEqual(new Iterator(map).rows().map((r) => r.key), ["a", "b", "c"]); + assert.deepEqual([...new Iterator(map)].map((r) => Number(r.value.asInt64())), [1, 2, 3]); + + const nested = Blob.fromJson('{"x":{"y":[1,2]}}'); + assert.deepEqual( + [...new Iterator(nested, "$", true)].map((r) => r.fullkey), + ["$", "$.x", "$.x.y", "$.x.y[0]", "$.x.y[1]"], + ); + + const it = new Iterator(Blob.fromJson("[1,2]")); + const seen: number[] = []; + while (it.next()) seen.push(Number(it.current().value.asInt64())); + assert.deepEqual(seen, [1, 2]); +}); + +test("validity", () => { + assert.ok(Blob.fromJson("[1,2,3]").valid()); + assert.equal(new Blob().valid(), false); + assert.equal(new Blob(Uint8Array.from([0x91])).valid(), false); +}); + +test("typeStr labels", () => { + assert.equal(typeStr(Type.Nil), "null"); + assert.equal(typeStr(Type.String), "text"); + assert.equal(typeStr(Type.Float32), "float32"); + assert.equal(typeStr(Type.Timestamp), "timestamp"); +}); + +test("non-UTF-8 str bytes preserved (byte-identical to C++)", () => { + // {"k": <0xff 0x80 0xfe 0xc0>} — a str with non-UTF-8 payload, as a foreign + // encoder (C++/SQLite) may produce. C++ passes raw bytes through verbatim. + const blob = new Blob(Uint8Array.from([0x81, 0xa1, 0x6b, 0xa4, 0xff, 0x80, 0xfe, 0xc0])); + // Re-encoding the JSON text reproduces the C++ raw output bytes exactly. + assert.deepEqual( + [...utf8Encode(blob.toJson())], + [0x7b, 0x22, 0x6b, 0x22, 0x3a, 0x22, 0xff, 0x80, 0xfe, 0xc0, 0x22, 0x7d], + ); + const v = blob.extract("$.k"); + assert.deepEqual([...utf8Encode(v.asString())], [0xff, 0x80, 0xfe, 0xc0]); + // Value.string round-trips a surrogateescape string back to identical bytes. + const rebuilt = new Builder().string(v.asString()).build(); + assert.deepEqual([...rebuilt.data()], [0xa4, 0xff, 0x80, 0xfe, 0xc0]); +}); diff --git a/js/test/vectors.test.ts b/js/test/vectors.test.ts new file mode 100644 index 0000000..b57dec3 --- /dev/null +++ b/js/test/vectors.test.ts @@ -0,0 +1,157 @@ +/* + * Replay the shared cross-language vectors (tests/vectors/blob_vectors.json). + * Generated from the C++ reference implementation, so passing them proves the + * TypeScript port is byte-identical. + */ + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; + +import { Blob, Builder, Iterator, Value } from "../src/index.ts"; + +const vectors = JSON.parse( + readFileSync(new URL("../../tests/vectors/blob_vectors.json", import.meta.url), "utf8"), +); + +function hexToBytes(h: string): Uint8Array { + const out = new Uint8Array(h.length / 2); + for (let i = 0; i < out.length; i++) out[i] = parseInt(h.substr(i * 2, 2), 16); + return out; +} + +function buildValue(spec: any): Value { + switch (spec.k) { + case "nil": + return Value.nil(); + case "bool": + return Value.boolean(Boolean(spec.v)); + case "int": + return Value.integer(BigInt(spec.v)); + case "uint": + return Value.unsignedInteger(BigInt(spec.v)); + case "int8": + return Value.int8(BigInt(spec.v)); + case "int16": + return Value.int16(BigInt(spec.v)); + case "int32": + return Value.int32(BigInt(spec.v)); + case "int64": + return Value.int64(BigInt(spec.v)); + case "uint8": + return Value.uint8(BigInt(spec.v)); + case "uint16": + return Value.uint16(BigInt(spec.v)); + case "uint32": + return Value.uint32(BigInt(spec.v)); + case "uint64": + return Value.uint64(BigInt(spec.v)); + case "real": + return Value.real(spec.v); + case "real32": + return Value.real32(spec.v); + case "str": + return Value.string(spec.v); + case "binary": + return Value.binary(hexToBytes(spec.hex)); + case "ext": + return Value.ext(spec.type, hexToBytes(spec.hex)); + case "timestamp": + return Value.timestamp(BigInt(spec.sec), spec.nsec); + default: + throw new Error("unknown spec kind: " + spec.k); + } +} + +test("from_json → byte-identical msgpack", () => { + for (const v of vectors.from_json) { + assert.equal(Blob.fromJson(v.json).hex(), v.hex, v.json); + } +}); + +test("to_json", () => { + for (const v of vectors.to_json) { + assert.equal(new Blob(hexToBytes(v.hex)).toJson(), v.json, v.hex); + } +}); + +test("to_json_pretty", () => { + for (const v of vectors.to_json_pretty) { + assert.equal(new Blob(hexToBytes(v.hex)).toJsonPretty(v.indent), v.json, v.hex); + } +}); + +test("typed Value → quote", () => { + for (const v of vectors.typed) { + assert.equal(Builder.quote(buildValue(v.spec)).hex(), v.hex, JSON.stringify(v.spec)); + } +}); + +test("mutate", () => { + for (const v of vectors.mutate) { + const base = Blob.fromJson(v.base); + let r: Blob; + switch (v.op) { + case "set": + r = base.set(v.path, buildValue(v.spec)); + break; + case "insert": + r = base.insert(v.path, buildValue(v.spec)); + break; + case "replace": + r = base.replace(v.path, buildValue(v.spec)); + break; + case "array_insert": + r = base.arrayInsert(v.path, buildValue(v.spec)); + break; + case "remove": + r = base.remove(v.path); + break; + case "set_blob": + r = base.set(v.path, Blob.fromJson(v.spec.json)); + break; + case "patch": + r = base.patch(Blob.fromJson(v.patch)); + break; + default: + throw new Error("unknown op " + v.op); + } + assert.equal(r.hex(), v.hex, `${v.op} ${v.base} ${v.path ?? ""}`); + } +}); + +test("extract type + value", () => { + for (const v of vectors.extract) { + const blob = Blob.fromJson(v.base); + assert.equal(blob.typeStr(v.path), v.type, `${v.base} ${v.path}`); + assert.equal(Builder.quote(blob.extract(v.path)).toJson(), v.vjson, `${v.base} ${v.path}`); + } +}); + +test("array_length", () => { + for (const v of vectors.array_length) { + const blob = Blob.fromJson(v.base); + const got = v.path === "$" ? blob.arrayLength() : blob.arrayLength(v.path); + assert.equal(got, v.len, `${v.base} ${v.path}`); + } +}); + +test("iterate each + tree", () => { + for (const v of vectors.iterate) { + const blob = Blob.fromJson(v.base); + const rows = new Iterator(blob, v.path, v.recursive).rows(); + assert.equal(rows.length, v.rows.length, `${v.base} ${v.path}`); + for (let i = 0; i < rows.length; i++) { + const got = rows[i]; + const exp = v.rows[i]; + assert.equal(got.fullkey, exp.fullkey); + assert.equal(got.path, exp.path); + assert.equal(got.id, exp.id); + assert.equal(got.type, exp.type); + if ("key" in exp) { + assert.equal(got.key, exp.key); + assert.equal(got.index, exp.index); + } + } + } +}); diff --git a/js/tsconfig.json b/js/tsconfig.json new file mode 100644 index 0000000..e49f0e3 --- /dev/null +++ b/js/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2020"], + "declaration": true, + "outDir": "dist", + "rootDir": "src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "allowImportingTsExtensions": true, + "rewriteRelativeImportExtensions": true, + "verbatimModuleSyntax": false + }, + "include": ["src/**/*.ts"] +} diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..e228412 --- /dev/null +++ b/python/README.md @@ -0,0 +1,70 @@ +# msgpack-blob (Python) + +A **pure-Python** port of the standalone [C++ MessagePack Blob API](../cpp/README.md) +from [sqlite-msgpack](../README.md). It creates, queries, mutates and iterates +[MessagePack](https://msgpack.org/) binary blobs and produces **byte-identical** +output to the C++ library and the `sqlite-msgpack` SQLite extension, so blobs are +fully interchangeable across all three. + +- Zero dependencies (standard library only) +- Same `Blob` / `Builder` / `Value` / `Iterator` API as the C++ library +- All msgpack primitive types: fixed-width ints, float32/64, ext, timestamp, binary +- JSON conversion modelled on SQLite's JSON1 extension + +## Install + +```bash +cd python +pip install -e . +``` + +## Quick start + +```python +from msgpack_blob import Blob, Builder, Value, Iterator + +# Build from JSON +blob = Blob.from_json('{"name":"Alice","scores":[95,87,91]}') +blob.extract("$.name").as_string() # 'Alice' +blob.array_length("$.scores") # 3 +blob.to_json() # '{"name":"Alice","scores":[95,87,91]}' + +# Mutate (copy-on-write — original is unchanged) +updated = blob.set("$.age", Value.integer(30)) +updated.to_json() # '{"name":"Alice","scores":[95,87,91],"age":30}' + +# Build with the streaming Builder +b = (Builder() + .map_header(2) + .string("temp").real32(23.5) + .string("ts").timestamp(1700000000, 500000000) + .build()) + +# Iterate (flat "each" or recursive "tree") +for row in Iterator(blob, "$", recursive=True): + print(row.fullkey, row.type.value) +``` + +## API overview + +| Class | Purpose | +|---|---| +| `Value` | A decoded scalar / sub-blob. Factories: `Value.integer`, `Value.real32`, `Value.string`, `Value.binary`, `Value.ext`, `Value.timestamp`, fixed-width `Value.int8`…`Value.uint64`. | +| `Blob` | Owning byte buffer. `from_json`, `to_json`, `to_json_pretty`, `extract`, `type`, `array_length`, `valid`, and copy-on-write `set` / `insert` / `replace` / `remove` / `array_insert` / `patch`. | +| `Builder` | Streaming encoder. Chainable `nil`/`boolean`/`integer`/`real`/`string`/`binary`/`ext`/`timestamp`/`array_header`/`map_header`/`value`, plus fixed-width integer methods. `build()` → `Blob`. | +| `Iterator` | Cursor over container children (`each` / `tree`). Pythonic iteration (`for row in ...`) or C++-style `next()`/`current()`. | +| `Type`, `IntWidth`, `type_str` | Type enum, integer-width hint enum, and label helper. | + +Paths use the same `$`-rooted syntax as the SQLite extension: `$`, `$.key`, +`$[0]`, `$.users[0].email`. + +## Tests + +```bash +cd python +python -m unittest discover -s tests +``` + +The suite includes `test_vectors.py`, which replays +[`tests/vectors/blob_vectors.json`](../tests/vectors/blob_vectors.json) — vectors +generated from the C++ reference implementation — to prove byte-identical output. diff --git a/python/msgpack_blob/__init__.py b/python/msgpack_blob/__init__.py new file mode 100644 index 0000000..cae9a0b --- /dev/null +++ b/python/msgpack_blob/__init__.py @@ -0,0 +1,43 @@ +"""msgpack_blob — a pure-Python MessagePack Blob library. + +A zero-dependency port of the standalone C++ ``msgpack`` Blob API. It creates, +queries, mutates and iterates MessagePack binary blobs and produces +byte-identical output to the C++ library and the ``sqlite-msgpack`` extension, +so blobs are fully interchangeable across all three. + +Quick start:: + + from msgpack_blob import Blob, Builder, Value, Iterator + + blob = Blob.from_json('{"name":"Alice","scores":[95,87,91]}') + blob.extract("$.name").as_string() # 'Alice' + blob.to_json() # '{"name":"Alice","scores":[95,87,91]}' + + b2 = blob.set("$.age", Value.integer(30)) + for row in Iterator(blob, "$", recursive=True): + print(row.fullkey, row.type.value) +""" + +from __future__ import annotations + +from ._format import MAX_DEPTH, MAX_OUTPUT +from .blob import Blob +from .builder import Builder +from .iterator import EachRow, Iterator +from .value import IntWidth, Type, Value, type_str + +__version__ = "1.5.0" + +__all__ = [ + "Blob", + "Builder", + "Value", + "Iterator", + "EachRow", + "Type", + "IntWidth", + "type_str", + "MAX_DEPTH", + "MAX_OUTPUT", + "__version__", +] diff --git a/python/msgpack_blob/_decode.py b/python/msgpack_blob/_decode.py new file mode 100644 index 0000000..6bb6031 --- /dev/null +++ b/python/msgpack_blob/_decode.py @@ -0,0 +1,404 @@ +"""Internal: decoding & inspection (mirrors msgpack_blob_decode.cpp).""" + +from __future__ import annotations + +import struct +from typing import Optional, Tuple + +from . import _format as F +from .value import Type, Value + +__all__ = [ + "RC_OK", + "RC_ERROR", + "RC_NOTFOUND", + "is_valid", + "error_position", + "get_type", + "get_type_str", + "get_container_count", + "path_step", + "lookup", + "decode_element", +] + +RC_OK = 0 +RC_ERROR = 1 +RC_NOTFOUND = 2 + +_I64_MAX = (1 << 63) - 1 + + +def is_valid(a, n: int) -> bool: + if n == 0: + return False + return F.skip_one(a, n, 0) == n + + +def error_position(a, n: int) -> int: + if n == 0: + return 0 + if F.skip_one(a, n, 0) == n: + return 0 + i = 0 + while i < n: + nxt = F.skip_one(a, n, i) + if not nxt: + return i + i = nxt + return 0 + + +def _is_timestamp_ext(a, n: int, i: int) -> bool: + if i >= n: + return False + b = a[i] + if b == F.MP_FIXEXT4 and i + 6 <= n and a[i + 1] == F.MP_TIMESTAMP_TYPE: + return True + if b == F.MP_FIXEXT8 and i + 10 <= n and a[i + 1] == F.MP_TIMESTAMP_TYPE: + return True + if b == F.MP_EXT8 and i + 3 <= n and a[i + 1] == 12 and a[i + 2] == F.MP_TIMESTAMP_TYPE: + return True + return False + + +def _decode_timestamp(a, n: int, i: int) -> Optional[Tuple[int, int]]: + if i >= n: + return None + b = a[i] + if b == F.MP_FIXEXT4 and i + 6 <= n and a[i + 1] == F.MP_TIMESTAMP_TYPE: + return (F.read32(a, i + 2), 0) + if b == F.MP_FIXEXT8 and i + 10 <= n and a[i + 1] == F.MP_TIMESTAMP_TYPE: + v = F.read64(a, i + 2) + return (v & 0x3FFFFFFFF, v >> 34) + if b == F.MP_EXT8 and i + 15 <= n and a[i + 1] == 12 and a[i + 2] == F.MP_TIMESTAMP_TYPE: + nsec = F.read32(a, i + 3) + sec = F.read64(a, i + 7) + if sec >= (1 << 63): + sec -= 1 << 64 + return (sec, nsec) + return None + + +def get_type(a, n: int, i: int) -> Type: + if i >= n: + return Type.NIL + b = a[i] + if b == F.MP_NIL: + return Type.NIL + if b == F.MP_TRUE: + return Type.TRUE + if b == F.MP_FALSE: + return Type.FALSE + if b <= 0x7F or b >= 0xE0: + return Type.INTEGER + if 0xA0 <= b <= 0xBF: + return Type.STRING + if 0x90 <= b <= 0x9F: + return Type.ARRAY + if 0x80 <= b <= 0x8F: + return Type.MAP + if b in (F.MP_UINT8, F.MP_UINT16, F.MP_UINT32, F.MP_UINT64, + F.MP_INT8, F.MP_INT16, F.MP_INT32, F.MP_INT64): + return Type.INTEGER + if b == F.MP_FLOAT32: + return Type.FLOAT32 + if b == F.MP_FLOAT64: + return Type.REAL + if b in (F.MP_STR8, F.MP_STR16, F.MP_STR32): + return Type.STRING + if b in (F.MP_BIN8, F.MP_BIN16, F.MP_BIN32): + return Type.BINARY + if b in (F.MP_ARRAY16, F.MP_ARRAY32): + return Type.ARRAY + if b in (F.MP_MAP16, F.MP_MAP32): + return Type.MAP + if b in (F.MP_EXT8, F.MP_EXT16, F.MP_EXT32, F.MP_FIXEXT1, F.MP_FIXEXT2, + F.MP_FIXEXT4, F.MP_FIXEXT8, F.MP_FIXEXT16): + return Type.TIMESTAMP if _is_timestamp_ext(a, n, i) else Type.EXT + return Type.NIL + + +def get_type_str(a, n: int, i: int) -> str: + return get_type(a, n, i).value + + +def get_container_count(a, n: int, i: int) -> int: + if i >= n: + return -1 + b = a[i] + if 0x90 <= b <= 0x9F: + return b & 0x0F + if 0x80 <= b <= 0x8F: + return b & 0x0F + if b == F.MP_ARRAY16 and i + 3 <= n: + return F.read16(a, i + 1) + if b == F.MP_ARRAY32 and i + 5 <= n: + return F.read32(a, i + 1) + if b == F.MP_MAP16 and i + 3 <= n: + return F.read16(a, i + 1) + if b == F.MP_MAP32 and i + 5 <= n: + return F.read32(a, i + 1) + return -1 + + +def path_step(zpath: str, pi: int): + """Parse one step of $.key[idx] syntax. + + Returns ``(kind, new_pi, key, idx)`` where *kind* is ``0`` (end), + ``-1`` (error), ``'k'`` (key) or ``'i'`` (index). + """ + i = pi + if i >= len(zpath): + return (0, i, None, 0) + c = zpath[i] + if c == ".": + i += 1 + start = i + while i < len(zpath) and zpath[i] not in ".[": + i += 1 + return ("k", i, zpath[start:i], 0) + if c == "[": + idx = 0 + has_digit = False + i += 1 + while i < len(zpath) and "0" <= zpath[i] <= "9": + idx = idx * 10 + (ord(zpath[i]) - 48) + i += 1 + has_digit = True + if not has_digit or i >= len(zpath) or zpath[i] != "]": + return (-1, i, None, 0) + i += 1 + return ("i", i, None, idx) + return (-1, i, None, 0) + + +def _key_at(a, n: int, i: int): + """Return (key_bytes, value_offset) for a map key at *i*, or (None, off).""" + kb = a[i] + if 0xA0 <= kb <= 0xBF: + klen = kb & 0x1F + koff = i + 1 + elif kb == F.MP_STR8 and i + 2 <= n: + klen = a[i + 1] + koff = i + 2 + elif kb == F.MP_STR16 and i + 3 <= n: + klen = F.read16(a, i + 1) + koff = i + 3 + elif kb == F.MP_STR32 and i + 5 <= n: + klen = F.read32(a, i + 1) + koff = i + 5 + else: + return None + if klen > n - koff: + return None + return bytes(a[koff:koff + klen]) + + +def lookup(a, n: int, iroot: int, zpath: str): + """Resolve *zpath* to a byte range. Returns ``(rc, iStart, iEnd)``.""" + if not zpath or zpath[0] != "$": + return (RC_ERROR, 0, 0) + icur = iroot + pi = 1 + + while True: + kind, pi, key, idx = path_step(zpath, pi) + + if kind == 0: + inext = F.skip_one(a, n, icur) + istart = icur + iend = inext if inext else n + return ((RC_OK if (inext or icur == n) else RC_ERROR), istart, iend) + if kind == -1: + return (RC_ERROR, 0, 0) + if icur >= n: + return (RC_NOTFOUND, 0, 0) + + if kind == "i": + b = a[icur] + if 0x90 <= b <= 0x9F: + count = b & 0x0F + elem_off = icur + 1 + elif b == F.MP_ARRAY16: + if icur + 3 > n: + return (RC_ERROR, 0, 0) + count = F.read16(a, icur + 1) + elem_off = icur + 3 + elif b == F.MP_ARRAY32: + if icur + 5 > n: + return (RC_ERROR, 0, 0) + count = F.read32(a, icur + 1) + elem_off = icur + 5 + else: + return (RC_NOTFOUND, 0, 0) + if idx < 0 or idx >= count: + return (RC_NOTFOUND, 0, 0) + icur = elem_off + for _ in range(idx): + icur = F.skip_one(a, n, icur) + if not icur: + return (RC_ERROR, 0, 0) + else: + b = a[icur] + if 0x80 <= b <= 0x8F: + count = b & 0x0F + elem_off = icur + 1 + elif b == F.MP_MAP16: + if icur + 3 > n: + return (RC_ERROR, 0, 0) + count = F.read16(a, icur + 1) + elem_off = icur + 3 + elif b == F.MP_MAP32: + if icur + 5 > n: + return (RC_ERROR, 0, 0) + count = F.read32(a, icur + 1) + elem_off = icur + 5 + else: + return (RC_NOTFOUND, 0, 0) + key_bytes = key.encode("utf-8", "surrogateescape") + icur = elem_off + found = False + j = 0 + while j < count and not found: + if icur >= n: + return (RC_ERROR, 0, 0) + kstr = _key_at(a, n, icur) + val_off = F.skip_one(a, n, icur) + if not val_off: + return (RC_ERROR, 0, 0) + if kstr is not None and kstr == key_bytes: + icur = val_off + found = True + else: + icur = F.skip_one(a, n, val_off) + if not icur: + return (RC_ERROR, 0, 0) + j += 1 + if not found: + return (RC_NOTFOUND, 0, 0) + + +def decode_element(a, n: int, istart: int, iend: int) -> Value: + if istart >= n or istart >= iend: + return Value.nil() + b = a[istart] + + if b == F.MP_NIL: + return Value.nil() + if b == F.MP_FALSE: + return Value.boolean(False) + if b == F.MP_TRUE: + return Value.boolean(True) + if b <= 0x7F: + return Value.integer(b) + if b >= 0xE0: + return Value.integer(b - 256) + + if b == F.MP_UINT8: + if istart + 2 <= n: + return Value.integer(a[istart + 1]) + elif b == F.MP_UINT16: + if istart + 3 <= n: + return Value.integer(F.read16(a, istart + 1)) + elif b == F.MP_UINT32: + if istart + 5 <= n: + return Value.integer(F.read32(a, istart + 1)) + elif b == F.MP_UINT64: + if istart + 9 <= n: + return Value.unsigned_integer(F.read64(a, istart + 1)) + elif b == F.MP_INT8: + if istart + 2 <= n: + v = a[istart + 1] + return Value.integer(v - 256 if v >= 128 else v) + elif b == F.MP_INT16: + if istart + 3 <= n: + v = F.read16(a, istart + 1) + return Value.integer(v - (1 << 16) if v >= (1 << 15) else v) + elif b == F.MP_INT32: + if istart + 5 <= n: + v = F.read32(a, istart + 1) + return Value.integer(v - (1 << 32) if v >= (1 << 31) else v) + elif b == F.MP_INT64: + if istart + 9 <= n: + v = F.read64(a, istart + 1) + return Value.integer(v - (1 << 64) if v >= (1 << 63) else v) + elif b == F.MP_FLOAT32: + if istart + 5 <= n: + f = struct.unpack(">f", bytes(a[istart + 1:istart + 5]))[0] + return Value.real32(f) + elif b == F.MP_FLOAT64: + if istart + 9 <= n: + d = struct.unpack(">d", bytes(a[istart + 1:istart + 9]))[0] + return Value.real(d) + + # str → String + soff = 0 + slen = 0 + if 0xA0 <= b <= 0xBF: + slen = b & 0x1F + soff = istart + 1 + elif b == F.MP_STR8 and istart + 2 <= n: + slen = a[istart + 1] + soff = istart + 2 + elif b == F.MP_STR16 and istart + 3 <= n: + slen = F.read16(a, istart + 1) + soff = istart + 3 + elif b == F.MP_STR32 and istart + 5 <= n: + slen = F.read32(a, istart + 1) + soff = istart + 5 + if soff: + if slen > n - soff: + slen = n - soff + return Value.string(bytes(a[soff:soff + slen])) + + # bin → Binary (payload only) + boff = 0 + blen = 0 + if b == F.MP_BIN8 and istart + 2 <= n: + blen = a[istart + 1] + boff = istart + 2 + elif b == F.MP_BIN16 and istart + 3 <= n: + blen = F.read16(a, istart + 1) + boff = istart + 3 + elif b == F.MP_BIN32 and istart + 5 <= n: + blen = F.read32(a, istart + 1) + boff = istart + 5 + if boff: + if blen > n - boff: + blen = n - boff + return Value.binary(bytes(a[boff:boff + blen])) + + # timestamp ext + ts = _decode_timestamp(a, n, istart) + if ts is not None: + return Value.timestamp(ts[0], ts[1]) + + # ext → Ext (type code + payload) + tc = 0 + elen = 0 + eoff = 0 + if b == F.MP_FIXEXT1 and istart + 3 <= n: + tc = a[istart + 1]; elen = 1; eoff = istart + 2 + elif b == F.MP_FIXEXT2 and istart + 4 <= n: + tc = a[istart + 1]; elen = 2; eoff = istart + 2 + elif b == F.MP_FIXEXT4 and istart + 6 <= n: + tc = a[istart + 1]; elen = 4; eoff = istart + 2 + elif b == F.MP_FIXEXT8 and istart + 10 <= n: + tc = a[istart + 1]; elen = 8; eoff = istart + 2 + elif b == F.MP_FIXEXT16 and istart + 18 <= n: + tc = a[istart + 1]; elen = 16; eoff = istart + 2 + elif b == F.MP_EXT8 and istart + 3 <= n: + elen = a[istart + 1]; tc = a[istart + 2]; eoff = istart + 3 + elif b == F.MP_EXT16 and istart + 4 <= n: + elen = F.read16(a, istart + 1); tc = a[istart + 3]; eoff = istart + 4 + elif b == F.MP_EXT32 and istart + 6 <= n: + elen = F.read32(a, istart + 1); tc = a[istart + 5]; eoff = istart + 6 + if eoff: + if elen > n - eoff: + elen = n - eoff + tc_signed = tc - 256 if tc >= 128 else tc + return Value.ext(tc_signed, bytes(a[eoff:eoff + elen])) + + # containers → raw binary blob (includes header) + return Value.binary(bytes(a[istart:iend])) diff --git a/python/msgpack_blob/_encode.py b/python/msgpack_blob/_encode.py new file mode 100644 index 0000000..688a1c6 --- /dev/null +++ b/python/msgpack_blob/_encode.py @@ -0,0 +1,239 @@ +"""Internal: encoding primitives (mirrors msgpack_blob_encode.cpp). + +All functions append to a ``bytearray`` and are shared by the Builder, the JSON +parser and the mutation engine. +""" + +from __future__ import annotations + +import struct + +from . import _format as F +from .value import IntWidth, Type, Value + +__all__ = [ + "enc_nil", + "enc_bool", + "enc_integer", + "enc_unsigned", + "enc_real", + "enc_real32", + "enc_string", + "enc_binary", + "enc_ext", + "enc_int8", "enc_int16", "enc_int32", "enc_int64", + "enc_uint8", "enc_uint16", "enc_uint32", "enc_uint64", + "enc_array_header", + "enc_map_header", + "enc_timestamp", + "encode_value", +] + +_U64 = 0xFFFFFFFFFFFFFFFF + + +def enc_nil(out: bytearray) -> None: + out.append(F.MP_NIL) + + +def enc_bool(out: bytearray, v: bool) -> None: + out.append(F.MP_TRUE if v else F.MP_FALSE) + + +def enc_integer(out: bytearray, x: int) -> None: + """Compact signed-integer encoding (fixint → int64/uint64).""" + if x >= 0: + if x <= 0x7F: + out.append(x) + elif x <= 0xFF: + out += bytes((F.MP_UINT8, x)) + elif x <= 0xFFFF: + out += bytes((F.MP_UINT16,)) + F.w16(x) + elif x <= 0xFFFFFFFF: + out += bytes((F.MP_UINT32,)) + F.w32(x) + else: + out += bytes((F.MP_UINT64,)) + F.w64(x) + else: + if x >= -32: + out.append(x & 0xFF) + elif x >= -128: + out += bytes((F.MP_INT8, x & 0xFF)) + elif x >= -32768: + out += bytes((F.MP_INT16,)) + F.w16(x) + elif x >= -2147483648: + out += bytes((F.MP_INT32,)) + F.w32(x) + else: + out += bytes((F.MP_INT64,)) + F.w64(x) + + +def enc_unsigned(out: bytearray, x: int) -> None: + """Compact unsigned-integer encoding.""" + x &= _U64 + if x <= 0x7F: + out.append(x) + elif x <= 0xFF: + out += bytes((F.MP_UINT8, x)) + elif x <= 0xFFFF: + out += bytes((F.MP_UINT16,)) + F.w16(x) + elif x <= 0xFFFFFFFF: + out += bytes((F.MP_UINT32,)) + F.w32(x) + else: + out += bytes((F.MP_UINT64,)) + F.w64(x) + + +def enc_real(out: bytearray, d: float) -> None: + out += bytes((F.MP_FLOAT64,)) + struct.pack(">d", d) + + +def enc_real32(out: bytearray, f: float) -> None: + out += bytes((F.MP_FLOAT32,)) + struct.pack(">f", f) + + +def enc_string(out: bytearray, s: bytes) -> None: + n = len(s) + if n <= 31: + out.append(F.MP_FIXSTR_MASK | n) + elif n <= 0xFF: + out += bytes((F.MP_STR8, n)) + elif n <= 0xFFFF: + out += bytes((F.MP_STR16,)) + F.w16(n) + else: + out += bytes((F.MP_STR32,)) + F.w32(n) + out += s + + +def enc_binary(out: bytearray, data: bytes) -> None: + n = len(data) + if n <= 0xFF: + out += bytes((F.MP_BIN8, n)) + elif n <= 0xFFFF: + out += bytes((F.MP_BIN16,)) + F.w16(n) + else: + out += bytes((F.MP_BIN32,)) + F.w32(n) + out += data + + +def enc_ext(out: bytearray, type_code: int, data: bytes) -> None: + n = len(data) + if n == 1: + out.append(F.MP_FIXEXT1) + elif n == 2: + out.append(F.MP_FIXEXT2) + elif n == 4: + out.append(F.MP_FIXEXT4) + elif n == 8: + out.append(F.MP_FIXEXT8) + elif n == 16: + out.append(F.MP_FIXEXT16) + elif n <= 0xFF: + out += bytes((F.MP_EXT8, n)) + elif n <= 0xFFFF: + out += bytes((F.MP_EXT16,)) + F.w16(n) + else: + out += bytes((F.MP_EXT32,)) + F.w32(n) + out.append(type_code & 0xFF) + out += data + + +def enc_int8(out: bytearray, x: int) -> None: + out += bytes((F.MP_INT8, x & 0xFF)) + + +def enc_int16(out: bytearray, x: int) -> None: + out += bytes((F.MP_INT16,)) + F.w16(x) + + +def enc_int32(out: bytearray, x: int) -> None: + out += bytes((F.MP_INT32,)) + F.w32(x) + + +def enc_int64(out: bytearray, x: int) -> None: + out += bytes((F.MP_INT64,)) + F.w64(x) + + +def enc_uint8(out: bytearray, x: int) -> None: + out += bytes((F.MP_UINT8, x & 0xFF)) + + +def enc_uint16(out: bytearray, x: int) -> None: + out += bytes((F.MP_UINT16,)) + F.w16(x) + + +def enc_uint32(out: bytearray, x: int) -> None: + out += bytes((F.MP_UINT32,)) + F.w32(x) + + +def enc_uint64(out: bytearray, x: int) -> None: + out += bytes((F.MP_UINT64,)) + F.w64(x) + + +def enc_array_header(out: bytearray, count: int) -> None: + if count <= 15: + out.append(F.MP_FIXARRAY_MASK | count) + elif count <= 0xFFFF: + out += bytes((F.MP_ARRAY16,)) + F.w16(count) + else: + out += bytes((F.MP_ARRAY32,)) + F.w32(count) + + +def enc_map_header(out: bytearray, count: int) -> None: + if count <= 15: + out.append(F.MP_FIXMAP_MASK | count) + elif count <= 0xFFFF: + out += bytes((F.MP_MAP16,)) + F.w16(count) + else: + out += bytes((F.MP_MAP32,)) + F.w32(count) + + +def enc_timestamp(out: bytearray, sec: int, nsec: int = 0) -> None: + if nsec == 0 and 0 <= sec <= 0xFFFFFFFF: + out += bytes((F.MP_FIXEXT4, 0xFF)) + F.w32(sec) + elif 0 <= sec <= 0x3FFFFFFFF: + out += bytes((F.MP_FIXEXT8, 0xFF)) + F.w64((nsec << 34) | sec) + else: + out += bytes((F.MP_EXT8, 12, 0xFF)) + F.w32(nsec) + F.w64(sec) + + +def encode_value(out: bytearray, v: Value) -> None: + """Encode a Value, honouring its integer-width hint.""" + t = v.type() + if t is Type.NIL: + enc_nil(out) + elif t is Type.TRUE: + enc_bool(out, True) + elif t is Type.FALSE: + enc_bool(out, False) + elif t is Type.INTEGER: + w = v.int_width() + if w is IntWidth.INT8: + enc_int8(out, v.as_int64()) + elif w is IntWidth.INT16: + enc_int16(out, v.as_int64()) + elif w is IntWidth.INT32: + enc_int32(out, v.as_int64()) + elif w is IntWidth.INT64: + enc_int64(out, v.as_int64()) + elif w is IntWidth.UINT8: + enc_uint8(out, v.as_uint64()) + elif w is IntWidth.UINT16: + enc_uint16(out, v.as_uint64()) + elif w is IntWidth.UINT32: + enc_uint32(out, v.as_uint64()) + elif w is IntWidth.UINT64: + enc_uint64(out, v.as_uint64()) + else: + enc_integer(out, v.as_int64()) + elif t is Type.REAL: + enc_real(out, v.as_double()) + elif t is Type.FLOAT32: + enc_real32(out, v.as_float()) + elif t is Type.STRING: + enc_string(out, v.as_bytes()) + elif t is Type.BINARY: + enc_binary(out, v.blob_data()) + elif t is Type.EXT: + enc_ext(out, v.ext_type(), v.blob_data()) + elif t is Type.TIMESTAMP: + enc_timestamp(out, v.timestamp_seconds(), v.timestamp_nanoseconds()) + else: + enc_nil(out) diff --git a/python/msgpack_blob/_format.py b/python/msgpack_blob/_format.py new file mode 100644 index 0000000..180bef7 --- /dev/null +++ b/python/msgpack_blob/_format.py @@ -0,0 +1,237 @@ +"""Internal: MessagePack format constants, byte-order helpers and skip_one. + +Private to the implementation; not part of the public API. Mirrors the shared +internals from ``cpp/src/msgpack_blob_detail.hpp`` and the skip routine from the +decode module. +""" + +from __future__ import annotations + +__all__ = [ + "MAX_DEPTH", + "MAX_OUTPUT", + "skip_one", + "read16", + "read32", + "read64", + "w16", + "w32", + "w64", +] + +# Limits (match the SQLite extension and C++ library) +MAX_DEPTH = 200 +MAX_OUTPUT = 64 * 1024 * 1024 + +# MessagePack format bytes +MP_NIL = 0xC0 +MP_FALSE = 0xC2 +MP_TRUE = 0xC3 +MP_BIN8 = 0xC4 +MP_BIN16 = 0xC5 +MP_BIN32 = 0xC6 +MP_EXT8 = 0xC7 +MP_EXT16 = 0xC8 +MP_EXT32 = 0xC9 +MP_FLOAT32 = 0xCA +MP_FLOAT64 = 0xCB +MP_UINT8 = 0xCC +MP_UINT16 = 0xCD +MP_UINT32 = 0xCE +MP_UINT64 = 0xCF +MP_INT8 = 0xD0 +MP_INT16 = 0xD1 +MP_INT32 = 0xD2 +MP_INT64 = 0xD3 +MP_FIXEXT1 = 0xD4 +MP_FIXEXT2 = 0xD5 +MP_FIXEXT4 = 0xD6 +MP_FIXEXT8 = 0xD7 +MP_FIXEXT16 = 0xD8 +MP_STR8 = 0xD9 +MP_STR16 = 0xDA +MP_STR32 = 0xDB +MP_ARRAY16 = 0xDC +MP_ARRAY32 = 0xDD +MP_MAP16 = 0xDE +MP_MAP32 = 0xDF + +MP_FIXMAP_MASK = 0x80 +MP_FIXARRAY_MASK = 0x90 +MP_FIXSTR_MASK = 0xA0 + +MP_TIMESTAMP_TYPE = 0xFF + + +# ── big-endian read helpers ─────────────────────────────────────────── +def read16(a, i: int) -> int: + return (a[i] << 8) | a[i + 1] + + +def read32(a, i: int) -> int: + return (a[i] << 24) | (a[i + 1] << 16) | (a[i + 2] << 8) | a[i + 3] + + +def read64(a, i: int) -> int: + return (read32(a, i) << 32) | read32(a, i + 4) + + +# ── big-endian write helpers (return bytes) ─────────────────────────── +def w16(v: int) -> bytes: + return (v & 0xFFFF).to_bytes(2, "big") + + +def w32(v: int) -> bytes: + return (v & 0xFFFFFFFF).to_bytes(4, "big") + + +def w64(v: int) -> bytes: + return (v & 0xFFFFFFFFFFFFFFFF).to_bytes(8, "big") + + +# ── skip_one — return the offset just past one complete element ─────── +def skip_one(a, n: int, i: int) -> int: + """Return offset just past the element at *i*, or 0 on malformed input.""" + return _skip_one_d(a, n, i, 0) + + +def _skip_one_d(a, n: int, i: int, depth: int) -> int: + if depth > MAX_DEPTH: + return 0 + if i >= n: + return 0 + b = a[i] + i += 1 + + if b <= 0x7F: # positive fixint + return i + if b >= 0xE0: # negative fixint + return i + + if b in (MP_NIL, MP_FALSE, MP_TRUE): + return i + if b == MP_FLOAT32: + return i + 4 if i + 4 <= n else 0 + if b in (MP_FLOAT64, MP_INT64, MP_UINT64): + return i + 8 if i + 8 <= n else 0 + if b in (MP_UINT8, MP_INT8): + return i + 1 if i + 1 <= n else 0 + if b in (MP_UINT16, MP_INT16): + return i + 2 if i + 2 <= n else 0 + if b in (MP_UINT32, MP_INT32): + return i + 4 if i + 4 <= n else 0 + + if b == MP_BIN8 or b == MP_STR8: + if i + 1 > n: + return 0 + sz = a[i] + i += 1 + return i + sz if sz <= n - i else 0 + if b == MP_BIN16 or b == MP_STR16: + if i + 2 > n: + return 0 + sz = read16(a, i) + i += 2 + return i + sz if sz <= n - i else 0 + if b == MP_BIN32 or b == MP_STR32: + if i + 4 > n: + return 0 + sz = read32(a, i) + i += 4 + return i + sz if sz <= n - i else 0 + + if b == MP_FIXEXT1: + return i + 2 if i + 2 <= n else 0 + if b == MP_FIXEXT2: + return i + 3 if i + 3 <= n else 0 + if b == MP_FIXEXT4: + return i + 5 if i + 5 <= n else 0 + if b == MP_FIXEXT8: + return i + 9 if i + 9 <= n else 0 + if b == MP_FIXEXT16: + return i + 17 if i + 17 <= n else 0 + if b == MP_EXT8: + if i + 2 > n: + return 0 + sz = a[i] + i += 2 + return i + sz if sz <= n - i else 0 + if b == MP_EXT16: + if i + 3 > n: + return 0 + sz = read16(a, i) + i += 3 + return i + sz if sz <= n - i else 0 + if b == MP_EXT32: + if i + 5 > n: + return 0 + sz = read32(a, i) + i += 5 + return i + sz if sz <= n - i else 0 + + # fixstr + if 0xA0 <= b <= 0xBF: + sz = b & 0x1F + return i + sz if sz <= n - i else 0 + + # fixarray + if 0x90 <= b <= 0x9F: + count = b & 0x0F + for _ in range(count): + i = _skip_one_d(a, n, i, depth + 1) + if not i: + return 0 + return i + + # fixmap + if 0x80 <= b <= 0x8F: + count = b & 0x0F + for _ in range(count): + i = _skip_one_d(a, n, i, depth + 1) + if not i: + return 0 + i = _skip_one_d(a, n, i, depth + 1) + if not i: + return 0 + return i + + # array16/32 + if b in (MP_ARRAY16, MP_ARRAY32): + if b == MP_ARRAY16: + if i + 2 > n: + return 0 + count = read16(a, i) + i += 2 + else: + if i + 4 > n: + return 0 + count = read32(a, i) + i += 4 + for _ in range(count): + i = _skip_one_d(a, n, i, depth + 1) + if not i: + return 0 + return i + + # map16/32 + if b in (MP_MAP16, MP_MAP32): + if b == MP_MAP16: + if i + 2 > n: + return 0 + count = read16(a, i) + i += 2 + else: + if i + 4 > n: + return 0 + count = read32(a, i) + i += 4 + for _ in range(count): + i = _skip_one_d(a, n, i, depth + 1) + if not i: + return 0 + i = _skip_one_d(a, n, i, depth + 1) + if not i: + return 0 + return i + + return 0 diff --git a/python/msgpack_blob/_iterate.py b/python/msgpack_blob/_iterate.py new file mode 100644 index 0000000..b191313 --- /dev/null +++ b/python/msgpack_blob/_iterate.py @@ -0,0 +1,163 @@ +"""Internal: container iteration (mirrors msgpack_blob_iterate.cpp).""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import List + +from . import _format as F +from ._decode import decode_element, get_type +from .value import Type, Value + +__all__ = ["EachRow", "each_iter", "tree_walk"] + + +@dataclass +class EachRow: + """A single row yielded by :class:`~msgpack_blob.iterator.Iterator`.""" + + fullkey: str = "$" + path: str = "$" + id: int = 0 + type: Type = Type.NIL + value: Value = field(default_factory=Value) + key: str = "" # map key ("" for arrays / tree rows) + index: int = 0 # array index or pair index (each mode only) + + +def _key_str(a, n: int, i: int): + kb = a[i] + if 0xA0 <= kb <= 0xBF: + klen = kb & 0x1F + koff = i + 1 + elif kb == F.MP_STR8 and i + 2 <= n: + klen = a[i + 1] + koff = i + 2 + elif kb == F.MP_STR16 and i + 3 <= n: + klen = F.read16(a, i + 1) + koff = i + 3 + elif kb == F.MP_STR32 and i + 5 <= n: + klen = F.read32(a, i + 1) + koff = i + 5 + else: + return None + return bytes(a[koff:koff + klen]).decode("utf-8", "surrogateescape") + + +def _container(a, n: int, i: int): + """Return (is_arr, is_map, count, data_off) for the container at *i*.""" + b = a[i] + if 0x90 <= b <= 0x9F: + return (True, False, b & 0x0F, i + 1) + if b == F.MP_ARRAY16 and i + 3 <= n: + return (True, False, F.read16(a, i + 1), i + 3) + if b == F.MP_ARRAY32 and i + 5 <= n: + return (True, False, F.read32(a, i + 1), i + 5) + if 0x80 <= b <= 0x8F: + return (False, True, b & 0x0F, i + 1) + if b == F.MP_MAP16 and i + 3 <= n: + return (False, True, F.read16(a, i + 1), i + 3) + if b == F.MP_MAP32 and i + 5 <= n: + return (False, True, F.read32(a, i + 1), i + 5) + return (False, False, 0, 0) + + +def each_iter(a, n: int, icont: int, zbase: str) -> List[EachRow]: + rows: List[EachRow] = [] + if icont >= n: + return rows + is_arr, is_map, count, data_off = _container(a, n, icont) + if not is_arr and not is_map: + return rows + + remaining = (n - data_off) if data_off <= n else 0 + min_bytes = 2 if is_map else 1 + if count > remaining // min_bytes + 1: + return rows + + cur = data_off + for j in range(count): + if cur >= n: + break + if is_arr: + c_end = F.skip_one(a, n, cur) + if not c_end: + break + rows.append(EachRow( + fullkey=f"{zbase}[{j}]", + path=zbase, + id=cur, + type=get_type(a, n, cur), + value=decode_element(a, n, cur, c_end), + key="", + index=j, + )) + cur = c_end + else: + ks = _key_str(a, n, cur) + v_off = F.skip_one(a, n, cur) + if not v_off: + break + p_end = F.skip_one(a, n, v_off) + if not p_end: + break + key = ks if ks is not None else "?" + rows.append(EachRow( + fullkey=f"{zbase}.{key}", + path=zbase, + id=v_off, + type=get_type(a, n, v_off), + value=decode_element(a, n, v_off, p_end), + key=key, + index=j, + )) + cur = p_end + return rows + + +def tree_walk(a, n: int, ioff: int, zfull: str, zpar_path: str, + depth: int, rows: List[EachRow]) -> None: + if depth > F.MAX_DEPTH or ioff >= n: + return + iend = F.skip_one(a, n, ioff) + if not iend: + return + + rows.append(EachRow( + fullkey=zfull, + path=zpar_path, + id=ioff, + type=get_type(a, n, ioff), + value=decode_element(a, n, ioff, iend), + )) + + is_arr, is_map, count, data_off = _container(a, n, ioff) + if not is_arr and not is_map: + return + + remaining = (n - data_off) if data_off <= n else 0 + min_bytes = 2 if is_map else 1 + if count > remaining // min_bytes + 1: + return + + cur = data_off + for j in range(count): + if cur >= n: + break + if is_arr: + c_end = F.skip_one(a, n, cur) + if not c_end: + break + tree_walk(a, n, cur, f"{zfull}[{j}]", zfull, depth + 1, rows) + cur = c_end + else: + ks = _key_str(a, n, cur) + v_off = F.skip_one(a, n, cur) + if not v_off: + break + p_end = F.skip_one(a, n, v_off) + if not p_end: + break + key = ks if ks is not None else "?" + tree_walk(a, n, v_off, f"{zfull}.{key}", zfull, depth + 1, rows) + cur = p_end diff --git a/python/msgpack_blob/_json.py b/python/msgpack_blob/_json.py new file mode 100644 index 0000000..305c0fa --- /dev/null +++ b/python/msgpack_blob/_json.py @@ -0,0 +1,482 @@ +"""Internal: JSON conversion (mirrors msgpack_blob_json.cpp). + +``to_json`` builds a ``bytearray`` exactly like the C++ implementation so float +formatting and string escaping stay byte-identical; ``from_json`` parses UTF-8 +bytes into msgpack. +""" + +from __future__ import annotations + +import math +import struct + +from . import _format as F +from . import _encode as E + +__all__ = ["to_json", "from_json"] + +RC_OK = 0 +RC_ERROR = 1 + +_I64_MIN = -(1 << 63) +_I64_MAX = (1 << 63) - 1 +_HEX = b"0123456789abcdef" + + +# ── float formatting (mimics C printf %.

g) ───────────────────────── +def _fmt_g(d: float, precision: int) -> str: + return "%.*g" % (precision, d) + + +def _fmt_double(d: float) -> str: + s = _fmt_g(d, 17) + if "." not in s and "e" not in s and "E" not in s: + s = "%.1f" % d + return s + + +def _fmt_float32(f: float) -> str: + return _fmt_g(f, 7) + + +# ── JSON output ─────────────────────────────────────────────────────── +def _escape_str(out: bytearray, s: bytes) -> None: + out.append(0x22) # " + start = 0 + n = len(s) + for j in range(n): + c = s[j] + if c >= 0x20 and c != 0x22 and c != 0x5C: + continue + if j > start: + out += s[start:j] + if c == 0x22: + out += b'\\"' + elif c == 0x5C: + out += b"\\\\" + elif c == 0x0A: + out += b"\\n" + elif c == 0x0D: + out += b"\\r" + elif c == 0x09: + out += b"\\t" + else: + out += ("\\u%04x" % c).encode("ascii") + start = j + 1 + if n > start: + out += s[start:n] + out.append(0x22) + + +def _newline(out: bytearray, depth: int, indent_w: int) -> None: + out.append(0x0A) + out += b" " * (depth * indent_w) + + +def _to_json_at(out: bytearray, a, n: int, i: int, pretty: bool, depth: int, indent_w: int) -> None: + if i >= n or depth > F.MAX_DEPTH: + out += b"null" + return + b = a[i] + + if b == F.MP_NIL: + out += b"null"; return + if b == F.MP_FALSE: + out += b"false"; return + if b == F.MP_TRUE: + out += b"true"; return + if b <= 0x7F: + out += str(b).encode("ascii"); return + if b >= 0xE0: + out += str(b - 256).encode("ascii"); return + + if b == F.MP_UINT8: + if i + 2 <= n: + out += str(a[i + 1]).encode("ascii"); return + elif b == F.MP_UINT16: + if i + 3 <= n: + out += str(F.read16(a, i + 1)).encode("ascii"); return + elif b == F.MP_UINT32: + if i + 5 <= n: + out += str(F.read32(a, i + 1)).encode("ascii"); return + elif b == F.MP_UINT64: + if i + 9 <= n: + out += str(F.read64(a, i + 1)).encode("ascii"); return + elif b == F.MP_INT8: + if i + 2 <= n: + v = a[i + 1] + out += str(v - 256 if v >= 128 else v).encode("ascii"); return + elif b == F.MP_INT16: + if i + 3 <= n: + v = F.read16(a, i + 1) + out += str(v - (1 << 16) if v >= (1 << 15) else v).encode("ascii"); return + elif b == F.MP_INT32: + if i + 5 <= n: + v = F.read32(a, i + 1) + out += str(v - (1 << 32) if v >= (1 << 31) else v).encode("ascii"); return + elif b == F.MP_INT64: + if i + 9 <= n: + v = F.read64(a, i + 1) + out += str(v - (1 << 64) if v >= (1 << 63) else v).encode("ascii"); return + elif b == F.MP_FLOAT32: + if i + 5 <= n: + f = struct.unpack(">f", bytes(a[i + 1:i + 5]))[0] + if not math.isfinite(f): + out += b"null"; return + out += _fmt_float32(f).encode("ascii"); return + elif b == F.MP_FLOAT64: + if i + 9 <= n: + d = struct.unpack(">d", bytes(a[i + 1:i + 9]))[0] + if not math.isfinite(d): + out += b"null"; return + out += _fmt_double(d).encode("ascii"); return + + # str + soff = 0 + slen = 0 + if 0xA0 <= b <= 0xBF: + slen = b & 0x1F; soff = i + 1 + elif b == F.MP_STR8 and i + 2 <= n: + slen = a[i + 1]; soff = i + 2 + elif b == F.MP_STR16 and i + 3 <= n: + slen = F.read16(a, i + 1); soff = i + 3 + elif b == F.MP_STR32 and i + 5 <= n: + slen = F.read32(a, i + 1); soff = i + 5 + if soff: + if slen > n - soff: + slen = n - soff + _escape_str(out, bytes(a[soff:soff + slen])) + return + + # bin → hex string + boff = 0 + blen = 0 + if b == F.MP_BIN8 and i + 2 <= n: + blen = a[i + 1]; boff = i + 2 + elif b == F.MP_BIN16 and i + 3 <= n: + blen = F.read16(a, i + 1); boff = i + 3 + elif b == F.MP_BIN32 and i + 5 <= n: + blen = F.read32(a, i + 1); boff = i + 5 + if boff: + if blen > n - boff: + blen = n - boff + out.append(0x22) + for j in range(blen): + by = a[boff + j] + out.append(_HEX[by >> 4]) + out.append(_HEX[by & 0xF]) + out.append(0x22) + return + + # array + is_arr = False + count = 0 + data_off = 0 + if 0x90 <= b <= 0x9F: + is_arr = True; count = b & 0x0F; data_off = i + 1 + elif b == F.MP_ARRAY16 and i + 3 <= n: + is_arr = True; count = F.read16(a, i + 1); data_off = i + 3 + elif b == F.MP_ARRAY32 and i + 5 <= n: + is_arr = True; count = F.read32(a, i + 1); data_off = i + 5 + if is_arr: + cur = data_off + out.append(0x5B) # [ + for j in range(count): + if cur >= n: + break + nxt = F.skip_one(a, n, cur) + if j > 0: + out.append(0x2C) + if pretty: + _newline(out, depth + 1, indent_w) + _to_json_at(out, a, n, cur, pretty, depth + 1, indent_w) + cur = nxt if nxt else n + if pretty and count > 0: + _newline(out, depth, indent_w) + out.append(0x5D) # ] + return + + # map + is_map = False + count = 0 + data_off = 0 + if 0x80 <= b <= 0x8F: + is_map = True; count = b & 0x0F; data_off = i + 1 + elif b == F.MP_MAP16 and i + 3 <= n: + is_map = True; count = F.read16(a, i + 1); data_off = i + 3 + elif b == F.MP_MAP32 and i + 5 <= n: + is_map = True; count = F.read32(a, i + 1); data_off = i + 5 + if is_map: + cur = data_off + out.append(0x7B) # { + for j in range(count): + if cur >= n: + break + val_off = F.skip_one(a, n, cur) + pair_end = F.skip_one(a, n, val_off) if val_off else 0 + if j > 0: + out.append(0x2C) + if pretty: + _newline(out, depth + 1, indent_w) + _to_json_at(out, a, n, cur, pretty, depth + 1, indent_w) + out.append(0x3A) # : + if pretty: + out.append(0x20) + _to_json_at(out, a, n, val_off if val_off else n, pretty, depth + 1, indent_w) + cur = pair_end if pair_end else n + if pretty and count > 0: + _newline(out, depth, indent_w) + out.append(0x7D) # } + return + + # ext / unknown → null + out += b"null" + + +def to_json(a, n: int, pretty: bool = False, indent: int = 0) -> str: + out = bytearray() + _to_json_at(out, a, n, 0, pretty, 0, indent) + return bytes(out).decode("utf-8", "surrogateescape") + + +# ── JSON parser → msgpack ───────────────────────────────────────────── +class _P: + __slots__ = ("z", "n", "i") + + def __init__(self, z: bytes) -> None: + self.z = z + self.n = len(z) + self.i = 0 + + +def _skip_ws(p: "_P") -> None: + while p.i < p.n and p.z[p.i] in (0x20, 0x09, 0x0A, 0x0D): + p.i += 1 + + +def _hex4(z: bytes, off: int) -> int: + v = 0 + for j in range(4): + c = z[off + j] + if 0x30 <= c <= 0x39: + h = c - 0x30 + elif 0x61 <= c <= 0x66: + h = c - 0x61 + 10 + elif 0x41 <= c <= 0x46: + h = c - 0x41 + 10 + else: + return -1 + v = (v << 4) | h + return v + + +def _cp_to_utf8(cp: int) -> bytes: + if cp < 0x80: + return bytes((cp,)) + if cp < 0x800: + return bytes((0xC0 | (cp >> 6), 0x80 | (cp & 0x3F))) + if cp < 0x10000: + return bytes((0xE0 | (cp >> 12), 0x80 | ((cp >> 6) & 0x3F), 0x80 | (cp & 0x3F))) + return bytes(( + 0xF0 | (cp >> 18), + 0x80 | ((cp >> 12) & 0x3F), + 0x80 | ((cp >> 6) & 0x3F), + 0x80 | (cp & 0x3F), + )) + + +def _parse_string(p: "_P", out: bytearray) -> int: + sb = bytearray() + p.i += 1 # skip " + while p.i < p.n: + c = p.z[p.i] + if c == 0x22: + p.i += 1 + break + if c == 0x5C: + p.i += 1 + if p.i >= p.n: + return RC_ERROR + esc = p.z[p.i] + p.i += 1 + if esc == 0x22: + sb.append(0x22) + elif esc == 0x5C: + sb.append(0x5C) + elif esc == 0x2F: + sb.append(0x2F) + elif esc == 0x6E: + sb.append(0x0A) + elif esc == 0x72: + sb.append(0x0D) + elif esc == 0x74: + sb.append(0x09) + elif esc == 0x62: + sb.append(0x08) + elif esc == 0x66: + sb.append(0x0C) + elif esc == 0x75: + if p.i + 4 > p.n: + return RC_ERROR + cp = _hex4(p.z, p.i) + p.i += 4 + if cp < 0: + return RC_ERROR + if 0xD800 <= cp <= 0xDBFF and p.i + 6 <= p.n and \ + p.z[p.i] == 0x5C and p.z[p.i + 1] == 0x75: + lo = _hex4(p.z, p.i + 2) + if 0xDC00 <= lo <= 0xDFFF: + p.i += 6 + cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00) + sb += _cp_to_utf8(cp) + else: + sb.append(esc) + else: + sb.append(c) + p.i += 1 + E.enc_string(out, bytes(sb)) + return RC_OK + + +def _parse_number(p: "_P", out: bytearray) -> int: + start = p.i + is_float = False + if p.i < p.n and p.z[p.i] == 0x2D: + p.i += 1 + while p.i < p.n and 0x30 <= p.z[p.i] <= 0x39: + p.i += 1 + if p.i < p.n and p.z[p.i] == 0x2E: + is_float = True + p.i += 1 + while p.i < p.n and 0x30 <= p.z[p.i] <= 0x39: + p.i += 1 + if p.i < p.n and p.z[p.i] in (0x65, 0x45): + is_float = True + p.i += 1 + if p.i < p.n and p.z[p.i] in (0x2B, 0x2D): + p.i += 1 + while p.i < p.n and 0x30 <= p.z[p.i] <= 0x39: + p.i += 1 + length = p.i - start + if length <= 0 or length >= 64: + return RC_ERROR + text = bytes(p.z[start:p.i]).decode("ascii") + + if is_float: + E.enc_real(out, float(text)) + else: + v = int(text) + if v > _I64_MAX: + v = _I64_MAX + elif v < _I64_MIN: + v = _I64_MIN + enc_signed_compact(out, v) + return RC_OK + + +def enc_signed_compact(out: bytearray, v: int) -> None: + """Replicates the from_json integer path (uses uint encodings for >=0).""" + if v >= 0: + E.enc_unsigned(out, v) + else: + E.enc_integer(out, v) + + +def _parse_array(p: "_P", out: bytearray) -> int: + tmp = bytearray() + count = 0 + p.i += 1 # skip [ + _skip_ws(p) + while p.i < p.n and p.z[p.i] != 0x5D: + if count > 0: + _skip_ws(p) + if p.i >= p.n or p.z[p.i] != 0x2C: + return RC_ERROR + p.i += 1 + _skip_ws(p) + if _parse_value(p, tmp) != RC_OK: + return RC_ERROR + count += 1 + _skip_ws(p) + if p.i >= p.n: + return RC_ERROR + p.i += 1 # skip ] + E.enc_array_header(out, count) + out += tmp + return RC_OK + + +def _parse_object(p: "_P", out: bytearray) -> int: + tmp = bytearray() + count = 0 + p.i += 1 # skip { + _skip_ws(p) + while p.i < p.n and p.z[p.i] != 0x7D: + if count > 0: + _skip_ws(p) + if p.i >= p.n or p.z[p.i] != 0x2C: + return RC_ERROR + p.i += 1 + _skip_ws(p) + if p.i >= p.n or p.z[p.i] != 0x22: + return RC_ERROR + if _parse_string(p, tmp) != RC_OK: + return RC_ERROR + _skip_ws(p) + if p.i >= p.n or p.z[p.i] != 0x3A: + return RC_ERROR + p.i += 1 + _skip_ws(p) + if _parse_value(p, tmp) != RC_OK: + return RC_ERROR + count += 1 + _skip_ws(p) + if p.i >= p.n: + return RC_ERROR + p.i += 1 # skip } + E.enc_map_header(out, count) + out += tmp + return RC_OK + + +def _parse_value(p: "_P", out: bytearray) -> int: + _skip_ws(p) + if p.i >= p.n: + return RC_ERROR + c = p.z[p.i] + if c == 0x6E and p.i + 4 <= p.n and p.z[p.i:p.i + 4] == b"null": + p.i += 4 + out.append(F.MP_NIL) + return RC_OK + if c == 0x74 and p.i + 4 <= p.n and p.z[p.i:p.i + 4] == b"true": + p.i += 4 + out.append(F.MP_TRUE) + return RC_OK + if c == 0x66 and p.i + 5 <= p.n and p.z[p.i:p.i + 5] == b"false": + p.i += 5 + out.append(F.MP_FALSE) + return RC_OK + if c == 0x22: + return _parse_string(p, out) + if c == 0x5B: + return _parse_array(p, out) + if c == 0x7B: + return _parse_object(p, out) + if c == 0x2D or 0x30 <= c <= 0x39: + return _parse_number(p, out) + return RC_ERROR + + +def from_json(json) -> bytes: + """Parse JSON (str or bytes) into msgpack; returns b'' on failure.""" + if json is None: + return b"" + if isinstance(json, str): + z = json.encode("utf-8", "surrogateescape") + else: + z = bytes(json) + p = _P(z) + out = bytearray() + if _parse_value(p, out) != RC_OK: + return b"" + return bytes(out) diff --git a/python/msgpack_blob/_mutate.py b/python/msgpack_blob/_mutate.py new file mode 100644 index 0000000..6514f32 --- /dev/null +++ b/python/msgpack_blob/_mutate.py @@ -0,0 +1,379 @@ +"""Internal: copy-on-write mutation (mirrors msgpack_blob_mutate.cpp).""" + +from __future__ import annotations + +from . import _format as F +from . import _encode as E +from ._decode import path_step + +__all__ = [ + "EDIT_SET", + "EDIT_INSERT", + "EDIT_REPLACE", + "EDIT_REMOVE", + "EDIT_ARRAY_INS", + "apply_edit", + "merge_patch", +] + +RC_OK = 0 +RC_ERROR = 1 +RC_NOTFOUND = 2 + +EDIT_SET = 0 +EDIT_INSERT = 1 +EDIT_REPLACE = 2 +EDIT_REMOVE = 3 +EDIT_ARRAY_INS = 4 + + +def _map_key(a, n: int, i: int): + kb = a[i] + if 0xA0 <= kb <= 0xBF: + klen = kb & 0x1F + koff = i + 1 + elif kb == F.MP_STR8 and i + 2 <= n: + klen = a[i + 1] + koff = i + 2 + elif kb == F.MP_STR16 and i + 3 <= n: + klen = F.read16(a, i + 1) + koff = i + 3 + elif kb == F.MP_STR32 and i + 5 <= n: + klen = F.read32(a, i + 1) + koff = i + 5 + else: + return None + return bytes(a[koff:koff + klen]) + + +def _edit_map(out, a, n, icur, zkey, zpath, pi, new_bin, mode): + b = a[icur] if icur < n else 0 + if icur >= n: + return RC_ERROR + if 0x80 <= b <= 0x8F: + count = b & 0x0F + data_off = icur + 1 + elif b == F.MP_MAP16: + if icur + 3 > n: + return RC_ERROR + count = F.read16(a, icur + 1) + data_off = icur + 3 + elif b == F.MP_MAP32: + if icur + 5 > n: + return RC_ERROR + count = F.read32(a, icur + 1) + data_off = icur + 5 + else: + if mode in (EDIT_REPLACE, EDIT_REMOVE): + iend = F.skip_one(a, n, icur) + if iend: + out += a[icur:iend] + return RC_OK + return RC_ERROR + + new_count = count + tmp = bytearray() + cur2 = data_off + found_key = False + + for _ in range(count): + if cur2 >= n: + return RC_ERROR + kstr = _map_key(a, n, cur2) + val_off = F.skip_one(a, n, cur2) + if not val_off: + return RC_ERROR + pair_end = F.skip_one(a, n, val_off) + if not pair_end: + return RC_ERROR + + is_match = kstr is not None and kstr == zkey + + if is_match: + found_key = True + if mode == EDIT_INSERT: + tmp += a[cur2:pair_end] + else: + vbuf = bytearray() + rc, skip = _edit_step(vbuf, a, n, val_off, zpath, pi, new_bin, mode) + if rc != RC_OK: + return rc + if skip: + new_count -= 1 + else: + tmp += a[cur2:val_off] + tmp += vbuf + else: + tmp += a[cur2:pair_end] + cur2 = pair_end + + if not found_key: + if mode in (EDIT_SET, EDIT_INSERT): + kind = path_step(zpath, pi)[0] + if kind != 0: + iend = F.skip_one(a, n, icur) + if iend: + out += a[icur:iend] + return RC_OK + E.enc_string(tmp, zkey) + tmp += new_bin + new_count += 1 + else: + iend = F.skip_one(a, n, icur) + if iend: + out += a[icur:iend] + return RC_OK + + E.enc_map_header(out, new_count) + out += tmp + return RC_OK + + +def _edit_array(out, a, n, icur, step_idx, zpath, pi, new_bin, mode): + if icur >= n: + return RC_ERROR + b = a[icur] + if 0x90 <= b <= 0x9F: + count = b & 0x0F + data_off = icur + 1 + elif b == F.MP_ARRAY16: + if icur + 3 > n: + return RC_ERROR + count = F.read16(a, icur + 1) + data_off = icur + 3 + elif b == F.MP_ARRAY32: + if icur + 5 > n: + return RC_ERROR + count = F.read32(a, icur + 1) + data_off = icur + 5 + else: + if mode in (EDIT_REPLACE, EDIT_REMOVE): + iend = F.skip_one(a, n, icur) + if iend: + out += a[icur:iend] + return RC_OK + return RC_ERROR + + new_count = count + tmp = bytearray() + cur2 = data_off + found_it = False + + for j in range(count): + e_end = F.skip_one(a, n, cur2) + if not e_end: + return RC_ERROR + + if j == step_idx: + found_it = True + if mode == EDIT_ARRAY_INS: + tmp += new_bin + tmp += a[cur2:e_end] + new_count += 1 + elif mode == EDIT_INSERT: + tmp += a[cur2:e_end] + else: + ebuf = bytearray() + rc, skip = _edit_step(ebuf, a, n, cur2, zpath, pi, new_bin, mode) + if rc != RC_OK: + return rc + if skip: + new_count -= 1 + else: + tmp += ebuf + else: + tmp += a[cur2:e_end] + cur2 = e_end + + if not found_it: + if mode == EDIT_ARRAY_INS: + tmp += new_bin + new_count += 1 + elif mode in (EDIT_SET, EDIT_INSERT) and step_idx == count: + tmp += new_bin + new_count += 1 + elif mode in (EDIT_REPLACE, EDIT_REMOVE): + iend = F.skip_one(a, n, icur) + if iend: + out += a[icur:iend] + return RC_OK + else: + return RC_NOTFOUND + + E.enc_array_header(out, new_count) + out += tmp + return RC_OK + + +# path_step needs the whole path string; thread it through the recursion to +# keep the helpers reentrant. + + +def _edit_step(out, a, n, icur, zpath, pi, new_bin, mode): + """Returns (rc, skip).""" + kind, pi2, key, step_idx = path_step(zpath, pi) + + if kind == 0: + if mode == EDIT_REMOVE: + return (RC_OK, True) + if mode == EDIT_ARRAY_INS: + return (RC_ERROR, False) + if mode == EDIT_INSERT: + iend = F.skip_one(a, n, icur) + if iend: + out += a[icur:iend] + return (RC_OK, False) + out += new_bin + return (RC_OK, False) + if kind == -1: + return (RC_ERROR, False) + + if kind == "k": + zkey = key.encode("utf-8", "surrogateescape") + rc = _edit_map(out, a, n, icur, zkey, zpath, pi2, new_bin, mode) + return (rc, False) + else: + rc = _edit_array(out, a, n, icur, step_idx, zpath, pi2, new_bin, mode) + return (rc, False) + + +def apply_edit(a, n, zpath, new_bin, mode): + """Returns (rc, out_bytes).""" + if not zpath or zpath[0] != "$": + return (RC_ERROR, b"") + out = bytearray() + rc, _skip = _edit_step(out, a, n, 0, zpath, 1, new_bin, mode) + return (rc, bytes(out)) + + +# ── merge_patch (RFC 7386) ──────────────────────────────────────────── +def merge_patch(a, n, ia, p, np, ip, depth): + """Returns (rc, out_bytes).""" + out = bytearray() + rc = _merge_patch(out, a, n, ia, p, np, ip, depth) + return (rc, bytes(out)) + + +def _merge_patch(out, a, n, ia, p, np, ip, depth): + if ip >= np: + return RC_ERROR + if depth > F.MAX_DEPTH: + return RC_ERROR + pb = p[ip] + + if pb == F.MP_NIL: + out.append(F.MP_NIL) + return RC_OK + + p_is_map = (0x80 <= pb <= 0x8F) or pb in (F.MP_MAP16, F.MP_MAP32) + if not p_is_map: + p_end = F.skip_one(p, np, ip) + if p_end: + out += p[ip:p_end] + return RC_OK + + ab = a[ia] if ia < n else 0 + a_is_map = (0x80 <= ab <= 0x8F) or ab in (F.MP_MAP16, F.MP_MAP32) + + if 0x80 <= pb <= 0x8F: + p_count = pb & 0x0F + p_data_off = ip + 1 + elif pb == F.MP_MAP16: + if ip + 3 > np: + return RC_ERROR + p_count = F.read16(p, ip + 1) + p_data_off = ip + 3 + else: + if ip + 5 > np: + return RC_ERROR + p_count = F.read32(p, ip + 1) + p_data_off = ip + 5 + + a_count = 0 + a_data_off = 0 + if a_is_map: + if 0x80 <= ab <= 0x8F: + a_count = ab & 0x0F + a_data_off = ia + 1 + elif ab == F.MP_MAP16: + if ia + 3 > n: + a_is_map = False + else: + a_count = F.read16(a, ia + 1) + a_data_off = ia + 3 + else: + if ia + 5 > n: + a_is_map = False + else: + a_count = F.read32(a, ia + 1) + a_data_off = ia + 5 + + # Pre-scan patch keys + if p_count > (np - p_data_off) // 2 + 1: + return RC_ERROR + p_idx = [] # each: [key_bytes_or_None, key_off, val_off, pair_end, matched] + pc2 = p_data_off + for _ in range(p_count): + if pc2 >= np: + return RC_ERROR + key = _map_key(p, np, pc2) + val_off = F.skip_one(p, np, pc2) + if not val_off: + return RC_ERROR + pair_end = F.skip_one(p, np, val_off) + if not pair_end: + return RC_ERROR + p_idx.append([key, pc2, val_off, pair_end, False]) + pc2 = pair_end + + tmp = bytearray() + new_count = 0 + + # Phase 1: iterate target pairs + if a_is_map: + ac = a_data_off + for _ in range(a_count): + if ac >= n: + return RC_ERROR + kstr = _map_key(a, n, ac) + a_val_off = F.skip_one(a, n, ac) + if not a_val_off: + return RC_ERROR + a_pair_end = F.skip_one(a, n, a_val_off) + if not a_pair_end: + return RC_ERROR + + found_in_patch = False + patch_is_nil = False + p_match_val = 0 + for entry in p_idx: + if entry[0] is not None and kstr is not None and entry[0] == kstr: + found_in_patch = True + p_match_val = entry[2] + patch_is_nil = entry[2] < np and p[entry[2]] == F.MP_NIL + entry[4] = True + break + + if found_in_patch and patch_is_nil: + pass # drop + elif found_in_patch: + mb = bytearray() + mrc = _merge_patch(mb, a, n, a_val_off, p, np, p_match_val, depth + 1) + if mrc == RC_OK: + tmp += a[ac:a_val_off] + tmp += mb + new_count += 1 + else: + tmp += a[ac:a_pair_end] + new_count += 1 + ac = a_pair_end + + # Phase 2: add unmatched patch pairs + for entry in p_idx: + if not entry[4] and entry[2] < np and p[entry[2]] != F.MP_NIL: + tmp += p[entry[1]:entry[3]] + new_count += 1 + + E.enc_map_header(out, new_count) + out += tmp + return RC_OK diff --git a/python/msgpack_blob/blob.py b/python/msgpack_blob/blob.py new file mode 100644 index 0000000..3c90719 --- /dev/null +++ b/python/msgpack_blob/blob.py @@ -0,0 +1,137 @@ +"""Blob — an owning byte buffer wrapping a msgpack-encoded value.""" + +from __future__ import annotations + +from typing import Optional, Union + +from . import _decode as D +from . import _encode as E +from . import _json as J +from . import _mutate as M +from .value import Type, Value + +__all__ = ["Blob"] + + +class Blob: + """A msgpack BLOB supporting read, mutation (copy-on-write) and JSON.""" + + __slots__ = ("_data",) + + def __init__(self, data: Union[bytes, bytearray, memoryview, None] = b"") -> None: + self._data = bytes(data) if data is not None else b"" + + # ── raw access ──────────────────────────────────────────────────── + def data(self) -> bytes: + return self._data + + def size(self) -> int: + return len(self._data) + + def empty(self) -> bool: + return len(self._data) == 0 + + def hex(self) -> str: + return self._data.hex() + + def __len__(self) -> int: + return len(self._data) + + def __bytes__(self) -> bytes: + return self._data + + def __eq__(self, other) -> bool: + if isinstance(other, Blob): + return self._data == other._data + return NotImplemented + + def __hash__(self) -> int: + return hash(self._data) + + def __repr__(self) -> str: # pragma: no cover - debug aid + return f"Blob({self._data.hex()})" + + # ── validation ──────────────────────────────────────────────────── + def valid(self) -> bool: + return D.is_valid(self._data, len(self._data)) + + def error_position(self) -> int: + return D.error_position(self._data, len(self._data)) + + # ── type inspection ─────────────────────────────────────────────── + def type(self, path: Optional[str] = None) -> Type: + n = len(self._data) + if path is None: + return Type.NIL if n == 0 else D.get_type(self._data, n, 0) + rc, istart, _ = D.lookup(self._data, n, 0, path) + if rc != D.RC_OK: + return Type.NIL + return D.get_type(self._data, n, istart) + + def type_str(self, path: Optional[str] = None) -> str: + return self.type(path).value + + # ── extraction ──────────────────────────────────────────────────── + def extract(self, path: str) -> Value: + n = len(self._data) + rc, istart, iend = D.lookup(self._data, n, 0, path) + if rc != D.RC_OK: + return Value.nil() + return D.decode_element(self._data, n, istart, iend) + + def array_length(self, path: Optional[str] = None) -> int: + n = len(self._data) + if path is None: + return -1 if n == 0 else D.get_container_count(self._data, n, 0) + rc, istart, _ = D.lookup(self._data, n, 0, path) + if rc != D.RC_OK: + return -1 + return D.get_container_count(self._data, n, istart) + + # ── mutation (copy-on-write) ────────────────────────────────────── + def _apply(self, path: str, value: Value, mode: int) -> "Blob": + new_bin = bytearray() + E.encode_value(new_bin, value) + rc, out = M.apply_edit(self._data, len(self._data), path, bytes(new_bin), mode) + return Blob(out) if rc == M.RC_OK else self + + def set(self, path: str, value: Union[Value, "Blob"]) -> "Blob": + if isinstance(value, Blob): + rc, out = M.apply_edit(self._data, len(self._data), path, + value._data, M.EDIT_SET) + return Blob(out) if rc == M.RC_OK else self + return self._apply(path, value, M.EDIT_SET) + + def insert(self, path: str, value: Value) -> "Blob": + return self._apply(path, value, M.EDIT_INSERT) + + def replace(self, path: str, value: Value) -> "Blob": + return self._apply(path, value, M.EDIT_REPLACE) + + def array_insert(self, path: str, value: Value) -> "Blob": + return self._apply(path, value, M.EDIT_ARRAY_INS) + + def remove(self, path: str) -> "Blob": + rc, out = M.apply_edit(self._data, len(self._data), path, b"", M.EDIT_REMOVE) + return Blob(out) if rc == M.RC_OK else self + + def patch(self, merge_patch: "Blob") -> "Blob": + rc, out = M.merge_patch(self._data, len(self._data), 0, + merge_patch._data, len(merge_patch._data), 0, 0) + return Blob(out) if rc == M.RC_OK else self + + # ── JSON conversion ─────────────────────────────────────────────── + def to_json(self) -> str: + if not self._data: + return "null" + return J.to_json(self._data, len(self._data), False, 0) + + def to_json_pretty(self, indent: int = 2) -> str: + if not self._data: + return "null" + indent = max(0, min(8, indent)) + return J.to_json(self._data, len(self._data), True, indent) + + @staticmethod + def from_json(json: Union[str, bytes, None]) -> "Blob": + return Blob(J.from_json(json)) diff --git a/python/msgpack_blob/builder.py b/python/msgpack_blob/builder.py new file mode 100644 index 0000000..a4c892c --- /dev/null +++ b/python/msgpack_blob/builder.py @@ -0,0 +1,128 @@ +"""Builder — a streaming encoder that produces a :class:`Blob`.""" + +from __future__ import annotations + +from typing import Union + +from . import _encode as E +from .blob import Blob +from .value import Value + +__all__ = ["Builder"] + + +class Builder: + """Append msgpack elements in order, then finalise with :meth:`build`.""" + + __slots__ = ("_buf",) + + def __init__(self) -> None: + self._buf = bytearray() + + # ── scalars ─────────────────────────────────────────────────────── + def nil(self) -> "Builder": + E.enc_nil(self._buf) + return self + + def boolean(self, v: bool) -> "Builder": + E.enc_bool(self._buf, v) + return self + + def integer(self, x: int) -> "Builder": + E.enc_integer(self._buf, int(x)) + return self + + def unsigned_integer(self, x: int) -> "Builder": + E.enc_unsigned(self._buf, int(x)) + return self + + def real(self, d: float) -> "Builder": + E.enc_real(self._buf, float(d)) + return self + + def real32(self, f: float) -> "Builder": + E.enc_real32(self._buf, float(f)) + return self + + def string(self, s: Union[str, bytes]) -> "Builder": + E.enc_string(self._buf, s.encode("utf-8", "surrogateescape") if isinstance(s, str) else bytes(s)) + return self + + def binary(self, data: Union[bytes, bytearray, memoryview]) -> "Builder": + E.enc_binary(self._buf, bytes(data)) + return self + + def ext(self, type_code: int, data: Union[bytes, bytearray, memoryview]) -> "Builder": + E.enc_ext(self._buf, int(type_code), bytes(data)) + return self + + # ── fixed-width integers ────────────────────────────────────────── + def int8(self, x: int) -> "Builder": + E.enc_int8(self._buf, int(x)) + return self + + def int16(self, x: int) -> "Builder": + E.enc_int16(self._buf, int(x)) + return self + + def int32(self, x: int) -> "Builder": + E.enc_int32(self._buf, int(x)) + return self + + def int64(self, x: int) -> "Builder": + E.enc_int64(self._buf, int(x)) + return self + + def uint8(self, x: int) -> "Builder": + E.enc_uint8(self._buf, int(x)) + return self + + def uint16(self, x: int) -> "Builder": + E.enc_uint16(self._buf, int(x)) + return self + + def uint32(self, x: int) -> "Builder": + E.enc_uint32(self._buf, int(x)) + return self + + def uint64(self, x: int) -> "Builder": + E.enc_uint64(self._buf, int(x)) + return self + + # ── containers ──────────────────────────────────────────────────── + def array_header(self, count: int) -> "Builder": + E.enc_array_header(self._buf, int(count)) + return self + + def map_header(self, count: int) -> "Builder": + E.enc_map_header(self._buf, int(count)) + return self + + # ── embedding & timestamp ───────────────────────────────────────── + def raw(self, data: Union[bytes, bytearray, memoryview, Blob]) -> "Builder": + if isinstance(data, Blob): + self._buf += data.data() + else: + self._buf += bytes(data) + return self + + def value(self, v: Value) -> "Builder": + E.encode_value(self._buf, v) + return self + + def timestamp(self, sec: int, nsec: int = 0) -> "Builder": + E.enc_timestamp(self._buf, int(sec), int(nsec)) + return self + + # ── finalize ────────────────────────────────────────────────────── + def build(self) -> Blob: + return Blob(bytes(self._buf)) + + def __len__(self) -> int: + return len(self._buf) + + @staticmethod + def quote(v: Value) -> Blob: + b = Builder() + b.value(v) + return b.build() diff --git a/python/msgpack_blob/iterator.py b/python/msgpack_blob/iterator.py new file mode 100644 index 0000000..c773cfb --- /dev/null +++ b/python/msgpack_blob/iterator.py @@ -0,0 +1,76 @@ +"""Iterator — a cursor over container children (flat ``each`` / recursive ``tree``).""" + +from __future__ import annotations + +from typing import Iterator as _PyIterator +from typing import List, Optional + +from . import _decode as D +from . import _iterate as I +from ._iterate import EachRow +from .blob import Blob + +__all__ = ["Iterator", "EachRow"] + + +class Iterator: + """Iterate over a container's children. + + Supports flat (``each``) and recursive (``tree``) modes, mirroring the + SQLite extension's ``msgpack_each`` / ``msgpack_tree`` table-valued + functions. Use it as a Python iterator or via the C++-style + :meth:`next` / :meth:`current` cursor protocol. + """ + + __slots__ = ("_blob", "_base", "_recursive", "_rows", "_cursor", "_populated") + + def __init__(self, blob: Blob, path: str = "$", recursive: bool = False) -> None: + self._blob = blob + self._base = path if path else "$" + self._recursive = recursive + self._rows: List[EachRow] = [] + self._cursor = -1 + self._populated = False + + def _populate(self) -> None: + if self._populated: + return + self._populated = True + self._rows = [] + a = self._blob.data() + n = len(a) + if n == 0: + return + + iroot = 0 + if self._base != "$": + rc, istart, _ = D.lookup(a, n, 0, self._base) + if rc != D.RC_OK: + return + iroot = istart + + if self._recursive: + I.tree_walk(a, n, iroot, self._base, self._base, 0, self._rows) + else: + self._rows = I.each_iter(a, n, iroot, self._base) + + # ── C++-style cursor protocol ───────────────────────────────────── + def next(self) -> bool: + self._populate() + self._cursor += 1 + return self._cursor < len(self._rows) + + def current(self) -> EachRow: + return self._rows[self._cursor] + + def reset(self) -> None: + self._cursor = -1 + + # ── Pythonic protocols ──────────────────────────────────────────── + def rows(self) -> List[EachRow]: + self._populate() + return list(self._rows) + + def __iter__(self) -> _PyIterator[EachRow]: + self._populate() + return iter(list(self._rows)) diff --git a/python/msgpack_blob/py.typed b/python/msgpack_blob/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/python/msgpack_blob/value.py b/python/msgpack_blob/value.py new file mode 100644 index 0000000..0d9350c --- /dev/null +++ b/python/msgpack_blob/value.py @@ -0,0 +1,285 @@ +"""Value — a decoded scalar or sub-blob MessagePack value. + +Mirrors ``msgpack::Value`` from the C++ Blob library. Values are cheap to copy +and can be used to both read from and write to blobs. +""" + +from __future__ import annotations + +import struct +from enum import Enum +from typing import Optional, Union + +__all__ = ["Type", "IntWidth", "Value", "type_str"] + +_U64 = 0xFFFFFFFFFFFFFFFF +_I64_MIN = -(1 << 63) +_I64_MAX = (1 << 63) - 1 + + +class Type(Enum): + """Semantic type of a MessagePack element.""" + + NIL = "null" + TRUE = "true" + FALSE = "false" + INTEGER = "integer" + REAL = "real" + FLOAT32 = "float32" + STRING = "text" + BINARY = "binary" + ARRAY = "array" + MAP = "map" + EXT = "ext" + TIMESTAMP = "timestamp" + + +class IntWidth(Enum): + """Integer encoding-width hint (forces a specific wire format).""" + + AUTO = 0 + INT8 = 1 + INT16 = 2 + INT32 = 3 + INT64 = 4 + UINT8 = 5 + UINT16 = 6 + UINT32 = 7 + UINT64 = 8 + + +def type_str(t: Type) -> str: + """Return the human-readable label for *t* (``"text"``, ``"integer"`` ...).""" + return t.value + + +def _f32(x: float) -> float: + """Round a Python float to float32 precision (returned as a float).""" + return struct.unpack(">f", struct.pack(">f", x))[0] + + +class Value: + """A decoded scalar or sub-blob value.""" + + __slots__ = ( + "_type", + "_int", + "_float", + "_str", + "_blob", + "_ext_type", + "_ts_nsec", + "_int_width", + ) + + def __init__(self) -> None: + self._type: Type = Type.NIL + self._int: int = 0 + self._float: float = 0.0 + self._str: bytes = b"" + self._blob: bytes = b"" + self._ext_type: int = 0 + self._ts_nsec: int = 0 + self._int_width: IntWidth = IntWidth.AUTO + + # ── accessors ───────────────────────────────────────────────────── + def type(self) -> Type: + return self._type + + def is_nil(self) -> bool: + return self._type is Type.NIL + + def as_bool(self) -> bool: + return self._type is Type.TRUE + + def as_int64(self) -> int: + """Signed 64-bit view (Integer, Real, Float32, Timestamp, True).""" + if self._type is Type.INTEGER: + v = self._int & _U64 + return v - (1 << 64) if v > _I64_MAX else v + if self._type in (Type.REAL, Type.FLOAT32): + return int(self._float) # truncate toward zero + if self._type is Type.TIMESTAMP: + return self._int + if self._type is Type.TRUE: + return 1 + return 0 + + def as_uint64(self) -> int: + """Raw unsigned 64-bit bits (Integer only).""" + if self._type is Type.INTEGER: + return self._int & _U64 + return 0 + + def as_double(self) -> float: + if self._type is Type.REAL: + return self._float + if self._type is Type.FLOAT32: + return self._float + if self._type is Type.INTEGER: + return float(self.as_int64()) + return 0.0 + + def as_float(self) -> float: + if self._type is Type.FLOAT32: + return self._float + if self._type is Type.REAL: + return _f32(self._float) + return 0.0 + + def as_string(self) -> str: + """String payload decoded as UTF-8 (lossless round-trip via surrogateescape).""" + if self._type is Type.STRING: + return self._str.decode("utf-8", "surrogateescape") + return "" + + def as_bytes(self) -> bytes: + """Raw String payload bytes (UTF-8).""" + if self._type is Type.STRING: + return self._str + return b"" + + def blob_data(self) -> bytes: + """Binary/Ext payload (no header), or raw bytes for container values.""" + return self._blob + + def blob_size(self) -> int: + return len(self._blob) + + def ext_type(self) -> int: + return self._ext_type + + def timestamp_seconds(self) -> int: + return self._int if self._type is Type.TIMESTAMP else 0 + + def timestamp_nanoseconds(self) -> int: + return self._ts_nsec if self._type is Type.TIMESTAMP else 0 + + def int_width(self) -> IntWidth: + return self._int_width + + # ── static constructors ─────────────────────────────────────────── + @staticmethod + def nil() -> "Value": + return Value() + + @staticmethod + def boolean(b: bool) -> "Value": + v = Value() + v._type = Type.TRUE if b else Type.FALSE + return v + + @staticmethod + def integer(x: int) -> "Value": + v = Value() + v._type = Type.INTEGER + v._int = int(x) + return v + + @staticmethod + def unsigned_integer(x: int) -> "Value": + v = Value() + v._type = Type.INTEGER + v._int = int(x) & _U64 + if v._int > _I64_MAX: + v._int_width = IntWidth.UINT64 + return v + + @staticmethod + def real(d: float) -> "Value": + v = Value() + v._type = Type.REAL + v._float = float(d) + return v + + @staticmethod + def real32(f: float) -> "Value": + v = Value() + v._type = Type.FLOAT32 + v._float = _f32(float(f)) + return v + + @staticmethod + def string(s: Union[str, bytes]) -> "Value": + v = Value() + v._type = Type.STRING + v._str = s.encode("utf-8", "surrogateescape") if isinstance(s, str) else bytes(s) + return v + + @staticmethod + def binary(data: Union[bytes, bytearray, memoryview]) -> "Value": + v = Value() + v._type = Type.BINARY + v._blob = bytes(data) + return v + + @staticmethod + def ext(type_code: int, data: Union[bytes, bytearray, memoryview]) -> "Value": + v = Value() + v._type = Type.EXT + v._ext_type = int(type_code) + v._blob = bytes(data) + return v + + @staticmethod + def timestamp(seconds: int, nanoseconds: int = 0) -> "Value": + v = Value() + v._type = Type.TIMESTAMP + v._int = int(seconds) + v._ts_nsec = int(nanoseconds) + return v + + @staticmethod + def _fixed(width: IntWidth, x: int) -> "Value": + v = Value() + v._type = Type.INTEGER + v._int = int(x) & _U64 + v._int_width = width + return v + + @staticmethod + def int8(x: int) -> "Value": + return Value._fixed(IntWidth.INT8, x) + + @staticmethod + def int16(x: int) -> "Value": + return Value._fixed(IntWidth.INT16, x) + + @staticmethod + def int32(x: int) -> "Value": + return Value._fixed(IntWidth.INT32, x) + + @staticmethod + def int64(x: int) -> "Value": + return Value._fixed(IntWidth.INT64, x) + + @staticmethod + def uint8(x: int) -> "Value": + return Value._fixed(IntWidth.UINT8, x) + + @staticmethod + def uint16(x: int) -> "Value": + return Value._fixed(IntWidth.UINT16, x) + + @staticmethod + def uint32(x: int) -> "Value": + return Value._fixed(IntWidth.UINT32, x) + + @staticmethod + def uint64(x: int) -> "Value": + return Value._fixed(IntWidth.UINT64, x) + + # ── debugging ───────────────────────────────────────────────────── + def __repr__(self) -> str: # pragma: no cover - debug aid + t = self._type + if t is Type.INTEGER: + return f"Value(integer={self.as_int64()}, width={self._int_width.name})" + if t in (Type.REAL, Type.FLOAT32): + return f"Value({t.value}={self._float})" + if t is Type.STRING: + return f"Value(text={self.as_string()!r})" + if t in (Type.BINARY, Type.EXT): + return f"Value({t.value}, {self._blob.hex()})" + if t is Type.TIMESTAMP: + return f"Value(timestamp={self._int}.{self._ts_nsec:09d})" + return f"Value({t.value})" diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..b4f05c5 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "msgpack-blob" +version = "1.5.0" +description = "Pure-Python MessagePack Blob API — byte-identical to the sqlite-msgpack C++ library" +readme = "README.md" +requires-python = ">=3.7" +license = { text = "MIT" } +authors = [{ name = "sqlite-msgpack contributors" }] +keywords = ["messagepack", "msgpack", "blob", "sqlite", "serialization"] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Topic :: Software Development :: Libraries", +] + +[project.urls] +Homepage = "https://github.com/khanaffan/sqlite-msgpack" +Source = "https://github.com/khanaffan/sqlite-msgpack" + +[tool.setuptools] +packages = ["msgpack_blob"] + +[tool.setuptools.package-data] +msgpack_blob = ["py.typed"] diff --git a/python/tests/test_api.py b/python/tests/test_api.py new file mode 100644 index 0000000..5e6f136 --- /dev/null +++ b/python/tests/test_api.py @@ -0,0 +1,188 @@ +"""API behaviour and round-trip tests for the Python port (no C++ needed).""" + +import unittest + +from msgpack_blob import Blob, Builder, EachRow, Iterator, Type, Value, type_str + + +class TestBuilder(unittest.TestCase): + def test_builder_matches_from_json(self): + built = ( + Builder() + .map_header(3) + .string("name").string("Alice") + .string("age").integer(30) + .string("scores").array_header(3) + .real(95.5).real(87.5).real(91.0) + .build() + ) + ref = Blob.from_json('{"name":"Alice","age":30,"scores":[95.5,87.5,91.0]}') + self.assertEqual(built.hex(), ref.hex()) + + def test_quote_roundtrip(self): + for v in [ + Value.nil(), Value.boolean(True), Value.integer(-12345), + Value.real(3.25), Value.real32(1.5), Value.string("hello"), + Value.binary(b"\xde\xad"), Value.ext(7, b"\x01\x02"), + Value.timestamp(1700000000, 123456789), + ]: + blob = Builder.quote(v) + self.assertTrue(blob.valid()) + self.assertEqual(blob.extract("$").type(), v.type()) + + +class TestRoundTrip(unittest.TestCase): + CASES = [ + "null", "true", "false", "0", "-1", "127", "128", "65536", + "1.5", "0.1", "1e10", '"hi"', "[]", "{}", "[1,2,3]", + '{"a":1,"b":[2,3],"c":{"d":true}}', + '{"u":"caf\\u00e9","emoji":"\\ud83d\\ude00"}', + ] + + def test_json_bytes_stable(self): + for case in self.CASES: + once = Blob.from_json(case) + twice = Blob.from_json(once.to_json()) + self.assertEqual(once.hex(), twice.hex(), case) + + def test_to_json_idempotent(self): + for case in self.CASES: + blob = Blob.from_json(case) + self.assertEqual(blob.to_json(), Blob.from_json(blob.to_json()).to_json()) + + +class TestExtraction(unittest.TestCase): + def setUp(self): + self.blob = Blob.from_json( + '{"name":"Alice","age":30,"tall":true,"pets":["cat","dog"],' + '"addr":{"city":"NYC"}}' + ) + + def test_scalar_paths(self): + self.assertEqual(self.blob.extract("$.name").as_string(), "Alice") + self.assertEqual(self.blob.extract("$.age").as_int64(), 30) + self.assertTrue(self.blob.extract("$.tall").as_bool()) + self.assertEqual(self.blob.extract("$.pets[1]").as_string(), "dog") + self.assertEqual(self.blob.extract("$.addr.city").as_string(), "NYC") + + def test_missing(self): + self.assertTrue(self.blob.extract("$.nope").is_nil()) + self.assertEqual(self.blob.type_str("$.nope"), "null") + + def test_types(self): + self.assertEqual(self.blob.type_str("$.name"), "text") + self.assertEqual(self.blob.type_str("$.age"), "integer") + self.assertEqual(self.blob.type_str("$.pets"), "array") + self.assertEqual(self.blob.type_str("$.addr"), "map") + + def test_array_length(self): + self.assertEqual(self.blob.array_length("$.pets"), 2) + self.assertEqual(self.blob.array_length("$.name"), -1) + + +class TestBinaryExt(unittest.TestCase): + def test_binary(self): + blob = Builder().binary(b"\x01\x02\x03\x04").build() + v = blob.extract("$") + self.assertEqual(v.type(), Type.BINARY) + self.assertEqual(v.blob_data(), b"\x01\x02\x03\x04") + self.assertEqual(blob.to_json(), '"01020304"') + + def test_ext(self): + blob = Builder().ext(42, b"\xaa\xbb").build() + v = blob.extract("$") + self.assertEqual(v.type(), Type.EXT) + self.assertEqual(v.ext_type(), 42) + self.assertEqual(v.blob_data(), b"\xaa\xbb") + + def test_timestamp(self): + blob = Builder().timestamp(1700000000, 500000000).build() + v = blob.extract("$") + self.assertEqual(v.type(), Type.TIMESTAMP) + self.assertEqual(v.timestamp_seconds(), 1700000000) + self.assertEqual(v.timestamp_nanoseconds(), 500000000) + + +class TestMutation(unittest.TestCase): + def test_copy_on_write(self): + orig = Blob.from_json('{"a":1}') + new = orig.set("$.b", Value.integer(2)) + self.assertEqual(orig.to_json(), '{"a":1}') + self.assertEqual(new.to_json(), '{"a":1,"b":2}') + + def test_remove_and_patch(self): + b = Blob.from_json('{"a":1,"b":2,"c":3}') + self.assertEqual(b.remove("$.b").to_json(), '{"a":1,"c":3}') + patched = b.patch(Blob.from_json('{"b":null,"d":4}')) + self.assertEqual(patched.to_json(), '{"a":1,"c":3,"d":4}') + + def test_array_ops(self): + b = Blob.from_json("[1,2,3]") + self.assertEqual(b.array_insert("$[1]", Value.integer(9)).to_json(), "[1,9,2,3]") + self.assertEqual(b.set("$[3]", Value.integer(4)).to_json(), "[1,2,3,4]") + + +class TestIterator(unittest.TestCase): + def test_each_map(self): + b = Blob.from_json('{"a":1,"b":2,"c":3}') + rows = Iterator(b).rows() + self.assertEqual([r.key for r in rows], ["a", "b", "c"]) + self.assertEqual([r.value.as_int64() for r in rows], [1, 2, 3]) + self.assertEqual([r.index for r in rows], [0, 1, 2]) + + def test_each_array(self): + b = Blob.from_json("[10,20,30]") + rows = list(Iterator(b)) + self.assertEqual([r.fullkey for r in rows], ["$[0]", "$[1]", "$[2]"]) + + def test_tree(self): + b = Blob.from_json('{"x":{"y":[1,2]}}') + keys = [r.fullkey for r in Iterator(b, "$", recursive=True)] + self.assertEqual(keys, ["$", "$.x", "$.x.y", "$.x.y[0]", "$.x.y[1]"]) + + def test_cursor_protocol(self): + b = Blob.from_json("[1,2]") + it = Iterator(b) + seen = [] + while it.next(): + seen.append(it.current().value.as_int64()) + self.assertEqual(seen, [1, 2]) + it.reset() + self.assertTrue(it.next()) + + +class TestValidity(unittest.TestCase): + def test_valid(self): + self.assertTrue(Blob.from_json("[1,2,3]").valid()) + self.assertFalse(Blob(b"").valid()) + self.assertFalse(Blob(b"\x91").valid()) # array claims 1 elem, none present + + def test_error_position(self): + self.assertEqual(Blob(b"\x01").error_position(), 0) # valid + self.assertNotEqual(Blob(b"\x91").error_position(), -1) + + +class TestTypeStr(unittest.TestCase): + def test_labels(self): + self.assertEqual(type_str(Type.NIL), "null") + self.assertEqual(type_str(Type.STRING), "text") + self.assertEqual(type_str(Type.FLOAT32), "float32") + self.assertEqual(type_str(Type.TIMESTAMP), "timestamp") + + +class TestNonUtf8(unittest.TestCase): + def test_non_utf8_preserved(self): + # {"k": <0xff 0x80 0xfe 0xc0>} — non-UTF-8 str payload from a foreign encoder. + blob = Blob(bytes([0x81, 0xA1, 0x6B, 0xA4, 0xFF, 0x80, 0xFE, 0xC0])) + self.assertEqual( + blob.to_json().encode("utf-8", "surrogateescape").hex(), + "7b226b223a22ff80fec0227d", + ) + v = blob.extract("$.k") + self.assertEqual(v.as_bytes(), bytes([0xFF, 0x80, 0xFE, 0xC0])) + rebuilt = Builder().string(v.as_string()).build() + self.assertEqual(rebuilt.hex(), "a4ff80fec0") + + +if __name__ == "__main__": + unittest.main() diff --git a/python/tests/test_vectors.py b/python/tests/test_vectors.py new file mode 100644 index 0000000..9887731 --- /dev/null +++ b/python/tests/test_vectors.py @@ -0,0 +1,156 @@ +"""Replay the shared cross-language vectors (tests/vectors/blob_vectors.json). + +These vectors are generated from the C++ reference implementation, so passing +them proves the Python port is byte-identical. +""" + +import json +import os +import unittest + +from msgpack_blob import Blob, Builder, Iterator, Value + +_VECTORS_PATH = os.path.join( + os.path.dirname(__file__), "..", "..", "tests", "vectors", "blob_vectors.json" +) + + +def load_vectors(): + with open(_VECTORS_PATH, "r", encoding="utf-8") as fh: + return json.load(fh) + + +def build_value(spec): + """Construct a Value from a ValueSpec dict (see cpp/tests/gen_blob_vectors.cpp).""" + k = spec["k"] + if k == "nil": + return Value.nil() + if k == "bool": + return Value.boolean(bool(spec["v"])) + if k == "int": + return Value.integer(int(spec["v"])) + if k == "uint": + return Value.unsigned_integer(int(spec["v"])) + if k == "int8": + return Value.int8(int(spec["v"])) + if k == "int16": + return Value.int16(int(spec["v"])) + if k == "int32": + return Value.int32(int(spec["v"])) + if k == "int64": + return Value.int64(int(spec["v"])) + if k == "uint8": + return Value.uint8(int(spec["v"])) + if k == "uint16": + return Value.uint16(int(spec["v"])) + if k == "uint32": + return Value.uint32(int(spec["v"])) + if k == "uint64": + return Value.uint64(int(spec["v"])) + if k == "real": + return Value.real(float(spec["v"])) + if k == "real32": + return Value.real32(float(spec["v"])) + if k == "str": + return Value.string(spec["v"]) + if k == "binary": + return Value.binary(bytes.fromhex(spec["hex"])) + if k == "ext": + return Value.ext(int(spec["type"]), bytes.fromhex(spec["hex"])) + if k == "timestamp": + return Value.timestamp(int(spec["sec"]), int(spec["nsec"])) + raise ValueError(f"unknown spec kind: {k}") + + +class TestFromJson(unittest.TestCase): + def test_from_json(self): + for v in load_vectors()["from_json"]: + with self.subTest(json=v["json"]): + self.assertEqual(Blob.from_json(v["json"]).hex(), v["hex"]) + + +class TestToJson(unittest.TestCase): + def test_to_json(self): + for v in load_vectors()["to_json"]: + with self.subTest(hex=v["hex"]): + blob = Blob(bytes.fromhex(v["hex"])) + self.assertEqual(blob.to_json(), v["json"]) + + def test_to_json_pretty(self): + for v in load_vectors()["to_json_pretty"]: + with self.subTest(hex=v["hex"], indent=v["indent"]): + blob = Blob(bytes.fromhex(v["hex"])) + self.assertEqual(blob.to_json_pretty(v["indent"]), v["json"]) + + +class TestTyped(unittest.TestCase): + def test_typed(self): + for v in load_vectors()["typed"]: + with self.subTest(spec=v["spec"]): + blob = Builder.quote(build_value(v["spec"])) + self.assertEqual(blob.hex(), v["hex"]) + + +class TestMutate(unittest.TestCase): + def test_mutate(self): + for v in load_vectors()["mutate"]: + with self.subTest(base=v["base"], op=v["op"], path=v.get("path")): + base = Blob.from_json(v["base"]) + op = v["op"] + if op == "set": + r = base.set(v["path"], build_value(v["spec"])) + elif op == "insert": + r = base.insert(v["path"], build_value(v["spec"])) + elif op == "replace": + r = base.replace(v["path"], build_value(v["spec"])) + elif op == "array_insert": + r = base.array_insert(v["path"], build_value(v["spec"])) + elif op == "remove": + r = base.remove(v["path"]) + elif op == "set_blob": + r = base.set(v["path"], Blob.from_json(v["spec"]["json"])) + elif op == "patch": + r = base.patch(Blob.from_json(v["patch"])) + else: + self.fail(f"unknown op {op}") + self.assertEqual(r.hex(), v["hex"]) + + +class TestExtract(unittest.TestCase): + def test_extract(self): + for v in load_vectors()["extract"]: + with self.subTest(base=v["base"], path=v["path"]): + blob = Blob.from_json(v["base"]) + self.assertEqual(blob.type_str(v["path"]), v["type"]) + value = blob.extract(v["path"]) + self.assertEqual(Builder.quote(value).to_json(), v["vjson"]) + + +class TestArrayLength(unittest.TestCase): + def test_array_length(self): + for v in load_vectors()["array_length"]: + with self.subTest(base=v["base"], path=v["path"]): + blob = Blob.from_json(v["base"]) + got = blob.array_length() if v["path"] == "$" else blob.array_length(v["path"]) + self.assertEqual(got, v["len"]) + + +class TestIterate(unittest.TestCase): + def test_iterate(self): + for v in load_vectors()["iterate"]: + with self.subTest(base=v["base"], path=v["path"], recursive=v["recursive"]): + blob = Blob.from_json(v["base"]) + rows = Iterator(blob, v["path"], v["recursive"]).rows() + self.assertEqual(len(rows), len(v["rows"])) + for got, exp in zip(rows, v["rows"]): + self.assertEqual(got.fullkey, exp["fullkey"]) + self.assertEqual(got.path, exp["path"]) + self.assertEqual(got.id, exp["id"]) + self.assertEqual(got.type.value, exp["type"]) + if "key" in exp: + self.assertEqual(got.key, exp["key"]) + self.assertEqual(got.index, exp["index"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000..f944baa --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "msgpack_blob" +version = "1.5.0" +edition = "2021" +rust-version = "1.70" +description = "Pure-Rust MessagePack Blob API — byte-identical to the sqlite-msgpack C++ library" +license = "MIT" +repository = "https://github.com/khanaffan/sqlite-msgpack" +keywords = ["messagepack", "msgpack", "blob", "sqlite", "serialization"] +categories = ["encoding", "no-std"] + +[lib] +name = "msgpack_blob" +path = "src/lib.rs" + +# Zero dependencies (std only). The test suite uses a tiny std-only JSON reader +# to replay the shared cross-language vectors. diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 0000000..9897145 --- /dev/null +++ b/rust/README.md @@ -0,0 +1,79 @@ +# msgpack_blob (Rust) + +A **pure-Rust**, zero-dependency port of the standalone [C++ MessagePack Blob +API](../cpp/README.md) from [sqlite-msgpack](../README.md). It creates, +queries, mutates and iterates [MessagePack](https://msgpack.org/) binary blobs +and produces **byte-identical** output to the C++ library and the +`sqlite-msgpack` SQLite extension, so blobs are fully interchangeable across all +of them. + +- Zero dependencies (no `serde`, no `num-bigint` — std only) +- Same `Blob` / `Builder` / `Value` / `Iterator` API as the C++ library +- All msgpack primitive types: fixed-width ints, float32/64, ext, timestamp, binary +- Native `i64` / `u64`, byte-preserving strings (`string_bytes` / `as_bytes`) +- JSON conversion modelled on SQLite's JSON1 extension + +## Add to your project + +```toml +[dependencies] +msgpack_blob = { path = "rust" } # or a published version +``` + +## Quick start + +```rust +use msgpack_blob::{Blob, Builder, Value, Iterator}; + +// Build from JSON +let blob = Blob::from_json(r#"{"name":"Alice","scores":[95,87,91]}"#); +assert_eq!(blob.extract("$.name").as_string(), "Alice"); +assert_eq!(blob.array_length_at("$.scores"), 3); +assert_eq!(blob.to_json(), r#"{"name":"Alice","scores":[95,87,91]}"#); + +// Mutate (copy-on-write — original is unchanged) +let updated = blob.set("$.age", &Value::integer(30)); +assert_eq!(updated.to_json(), r#"{"name":"Alice","scores":[95,87,91],"age":30}"#); + +// Build with the streaming Builder +let b = Builder::new() + .map_header(2) + .string("temp").real32(23.5) + .string("ts").timestamp_ns(1_700_000_000, 500_000_000) + .build(); + +// Iterate (flat "each" or recursive "tree") +for row in Iterator::new(&blob, "$", true) { + println!("{} {}", row.fullkey, msgpack_blob::type_str(row.ty)); +} +``` + +## API overview + +| Type | Purpose | +|---|---| +| `Value` | A decoded scalar / sub-blob. Constructors: `Value::integer`, `Value::real32`, `Value::string` / `Value::string_bytes`, `Value::binary`, `Value::ext`, `Value::timestamp` / `Value::timestamp_ns`, fixed-width `Value::int8`…`Value::uint64`. | +| `Blob` | Owning byte buffer. `from_json`, `to_json` / `to_json_bytes`, `to_json_pretty`, `extract`, `type_at`, `array_length`, `valid`, and copy-on-write `set` / `insert` / `replace` / `remove` / `array_insert` / `patch`. | +| `Builder` | Streaming encoder. Chainable `nil`/`boolean`/`integer`/`real`/`string`/`binary`/`ext`/`timestamp`/`array_header`/`map_header`/`value`, plus fixed-width integer methods. `build()` → `Blob`. | +| `Iterator` | Cursor over container children (`each` / `tree`). Use the `next()`/`current()` cursor, `for row in iter`, or `.rows()`. | +| `Type`, `IntWidth`, `type_str` | Type enum, integer-width hint, and label helper. | + +Methods come in path-aware pairs where the C++ API overloads: `type_at(path)` / +`type_str_at(path)` / `array_length_at(path)` versus the root-level +`root_type()` / `type_str()` / `array_length()`. Non-UTF-8 string payloads are +preserved byte-exactly via `to_json_bytes()` and `Value::string_bytes` / +`Value::as_bytes`. Paths use the same `$`-rooted syntax as the SQLite extension: +`$`, `$.key`, `$[0]`, `$.users[0].email`. + +## Tests + +```bash +cd rust +cargo test +``` + +The suite includes `tests/vectors.rs`, which replays +[`tests/vectors/blob_vectors.json`](../tests/vectors/blob_vectors.json) — vectors +generated from the C++ reference implementation — to prove byte-identical output. +A tiny std-only JSON reader (`tests/common/mod.rs`) keeps the crate itself +dependency-free. diff --git a/rust/src/blob.rs b/rust/src/blob.rs new file mode 100644 index 0000000..533a714 --- /dev/null +++ b/rust/src/blob.rs @@ -0,0 +1,205 @@ +//! `Blob` — an owning byte buffer wrapping a msgpack-encoded value. + +use crate::decode as d; +use crate::encode as e; +use crate::json as j; +use crate::mutate as m; +use crate::value::{Type, Value}; + +/// A MessagePack BLOB supporting read, mutation (copy-on-write) and JSON. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Blob { + data: Vec, +} + +impl Blob { + /// Construct from raw bytes (copies). + pub fn new(data: &[u8]) -> Blob { + Blob { + data: data.to_vec(), + } + } + /// Construct by taking ownership of a byte vector. + pub fn from_vec(data: Vec) -> Blob { + Blob { data } + } + + // ── raw access ──────────────────────────────────────────────────── + pub fn data(&self) -> &[u8] { + &self.data + } + pub fn size(&self) -> usize { + self.data.len() + } + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } + pub fn hex(&self) -> String { + let mut s = String::with_capacity(self.data.len() * 2); + for &b in &self.data { + s.push_str(&format!("{:02x}", b)); + } + s + } + + // ── validation ──────────────────────────────────────────────────── + pub fn valid(&self) -> bool { + d::is_valid(&self.data, self.data.len()) + } + pub fn error_position(&self) -> usize { + d::error_position(&self.data, self.data.len()) + } + + // ── type inspection ─────────────────────────────────────────────── + /// Type of the root element. + pub fn root_type(&self) -> Type { + if self.data.is_empty() { + Type::Nil + } else { + d::get_type(&self.data, self.data.len(), 0) + } + } + /// Type of the element at `path`. + pub fn type_at(&self, path: &str) -> Type { + let n = self.data.len(); + let (rc, istart, _) = d::lookup(&self.data, n, 0, path); + if rc != d::RC_OK { + Type::Nil + } else { + d::get_type(&self.data, n, istart) + } + } + pub fn type_str(&self) -> &'static str { + crate::value::type_str(self.root_type()) + } + pub fn type_str_at(&self, path: &str) -> &'static str { + crate::value::type_str(self.type_at(path)) + } + + // ── extraction ──────────────────────────────────────────────────── + pub fn extract(&self, path: &str) -> Value { + let n = self.data.len(); + let (rc, istart, iend) = d::lookup(&self.data, n, 0, path); + if rc != d::RC_OK { + Value::nil() + } else { + d::decode_element(&self.data, n, istart, iend) + } + } + + /// Element count of the root container, or `-1`. + pub fn array_length(&self) -> i64 { + if self.data.is_empty() { + -1 + } else { + d::get_container_count(&self.data, self.data.len(), 0) + } + } + /// Element count of the container at `path`, or `-1`. + pub fn array_length_at(&self, path: &str) -> i64 { + let n = self.data.len(); + let (rc, istart, _) = d::lookup(&self.data, n, 0, path); + if rc != d::RC_OK { + -1 + } else { + d::get_container_count(&self.data, n, istart) + } + } + + // ── mutation (copy-on-write) ────────────────────────────────────── + fn apply(&self, path: &str, value: &Value, mode: i32) -> Blob { + let mut nb = Vec::new(); + e::encode_value(&mut nb, value); + let (rc, out) = m::apply_edit(&self.data, self.data.len(), path, &nb, mode); + if rc == m::RC_OK { + Blob::from_vec(out) + } else { + self.clone() + } + } + + pub fn set(&self, path: &str, value: &Value) -> Blob { + self.apply(path, value, m::EDIT_SET) + } + /// Set `path` to an existing sub-blob (embedded verbatim). + pub fn set_blob(&self, path: &str, sub: &Blob) -> Blob { + let (rc, out) = m::apply_edit(&self.data, self.data.len(), path, &sub.data, m::EDIT_SET); + if rc == m::RC_OK { + Blob::from_vec(out) + } else { + self.clone() + } + } + pub fn insert(&self, path: &str, value: &Value) -> Blob { + self.apply(path, value, m::EDIT_INSERT) + } + pub fn replace(&self, path: &str, value: &Value) -> Blob { + self.apply(path, value, m::EDIT_REPLACE) + } + pub fn array_insert(&self, path: &str, value: &Value) -> Blob { + self.apply(path, value, m::EDIT_ARRAY_INS) + } + pub fn remove(&self, path: &str) -> Blob { + let (rc, out) = m::apply_edit(&self.data, self.data.len(), path, &[], m::EDIT_REMOVE); + if rc == m::RC_OK { + Blob::from_vec(out) + } else { + self.clone() + } + } + pub fn patch(&self, merge_patch: &Blob) -> Blob { + let (rc, out) = m::merge_patch( + &self.data, + self.data.len(), + 0, + &merge_patch.data, + merge_patch.data.len(), + 0, + ); + if rc == m::RC_OK { + Blob::from_vec(out) + } else { + self.clone() + } + } + + // ── JSON conversion ─────────────────────────────────────────────── + /// Byte-exact JSON serialisation (may contain non-UTF-8 string bytes, + /// identical to the C++ library output). + pub fn to_json_bytes(&self) -> Vec { + if self.data.is_empty() { + return b"null".to_vec(); + } + j::to_json_bytes(&self.data, self.data.len(), false, 0) + } + /// JSON serialisation as a `String` (exact for valid UTF-8 input). + pub fn to_json(&self) -> String { + bytes_to_string(self.to_json_bytes()) + } + pub fn to_json_pretty_bytes(&self, indent: i32) -> Vec { + if self.data.is_empty() { + return b"null".to_vec(); + } + let indent = indent.clamp(0, 8); + j::to_json_bytes(&self.data, self.data.len(), true, indent) + } + pub fn to_json_pretty(&self, indent: i32) -> String { + bytes_to_string(self.to_json_pretty_bytes(indent)) + } + + /// Parse JSON bytes into a msgpack blob. + pub fn from_json_bytes(json: &[u8]) -> Blob { + Blob::from_vec(j::from_json(json)) + } + /// Parse a JSON string into a msgpack blob. + pub fn from_json(json: &str) -> Blob { + Blob::from_vec(j::from_json(json.as_bytes())) + } +} + +fn bytes_to_string(b: Vec) -> String { + match String::from_utf8(b) { + Ok(s) => s, + Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(), + } +} diff --git a/rust/src/builder.rs b/rust/src/builder.rs new file mode 100644 index 0000000..efeedbd --- /dev/null +++ b/rust/src/builder.rs @@ -0,0 +1,145 @@ +//! `Builder` — a streaming encoder that produces a [`Blob`]. + +use crate::blob::Blob; +use crate::encode as e; +use crate::value::Value; + +/// Append msgpack elements in order, then finalise with [`Builder::build`]. +#[derive(Default)] +pub struct Builder { + buf: Vec, +} + +impl Builder { + pub fn new() -> Builder { + Builder { buf: Vec::new() } + } + + // ── scalars ─────────────────────────────────────────────────────── + pub fn nil(&mut self) -> &mut Self { + e::enc_nil(&mut self.buf); + self + } + pub fn boolean(&mut self, v: bool) -> &mut Self { + e::enc_bool(&mut self.buf, v); + self + } + pub fn integer(&mut self, x: i64) -> &mut Self { + e::enc_integer(&mut self.buf, x); + self + } + pub fn unsigned_integer(&mut self, x: u64) -> &mut Self { + e::enc_unsigned(&mut self.buf, x); + self + } + pub fn real(&mut self, d: f64) -> &mut Self { + e::enc_real(&mut self.buf, d); + self + } + pub fn real32(&mut self, val: f32) -> &mut Self { + e::enc_real32(&mut self.buf, val); + self + } + pub fn string(&mut self, s: &str) -> &mut Self { + e::enc_string(&mut self.buf, s.as_bytes()); + self + } + pub fn string_bytes(&mut self, s: &[u8]) -> &mut Self { + e::enc_string(&mut self.buf, s); + self + } + pub fn binary(&mut self, data: &[u8]) -> &mut Self { + e::enc_binary(&mut self.buf, data); + self + } + pub fn ext(&mut self, type_code: i8, data: &[u8]) -> &mut Self { + e::enc_ext(&mut self.buf, type_code, data); + self + } + + // ── fixed-width integers ────────────────────────────────────────── + pub fn int8(&mut self, x: i8) -> &mut Self { + e::enc_int8(&mut self.buf, x as i64); + self + } + pub fn int16(&mut self, x: i16) -> &mut Self { + e::enc_int16(&mut self.buf, x as i64); + self + } + pub fn int32(&mut self, x: i32) -> &mut Self { + e::enc_int32(&mut self.buf, x as i64); + self + } + pub fn int64(&mut self, x: i64) -> &mut Self { + e::enc_int64(&mut self.buf, x); + self + } + pub fn uint8(&mut self, x: u8) -> &mut Self { + e::enc_uint8(&mut self.buf, x as u64); + self + } + pub fn uint16(&mut self, x: u16) -> &mut Self { + e::enc_uint16(&mut self.buf, x as u64); + self + } + pub fn uint32(&mut self, x: u32) -> &mut Self { + e::enc_uint32(&mut self.buf, x as u64); + self + } + pub fn uint64(&mut self, x: u64) -> &mut Self { + e::enc_uint64(&mut self.buf, x); + self + } + + // ── containers ──────────────────────────────────────────────────── + pub fn array_header(&mut self, count: u32) -> &mut Self { + e::enc_array_header(&mut self.buf, count); + self + } + pub fn map_header(&mut self, count: u32) -> &mut Self { + e::enc_map_header(&mut self.buf, count); + self + } + + // ── embedding & timestamp ───────────────────────────────────────── + pub fn raw(&mut self, data: &[u8]) -> &mut Self { + self.buf.extend_from_slice(data); + self + } + pub fn raw_blob(&mut self, blob: &Blob) -> &mut Self { + self.buf.extend_from_slice(blob.data()); + self + } + pub fn value(&mut self, v: &Value) -> &mut Self { + e::encode_value(&mut self.buf, v); + self + } + pub fn timestamp(&mut self, sec: i64) -> &mut Self { + e::enc_timestamp(&mut self.buf, sec, 0); + self + } + pub fn timestamp_ns(&mut self, sec: i64, nsec: u32) -> &mut Self { + e::enc_timestamp(&mut self.buf, sec, nsec); + self + } + + // ── finalize ────────────────────────────────────────────────────── + /// Finalise into a [`Blob`]. Takes `&self` so it can terminate a method + /// chain (`Builder::new().integer(1).build()`). + pub fn build(&self) -> Blob { + Blob::from_vec(self.buf.clone()) + } + pub fn len(&self) -> usize { + self.buf.len() + } + pub fn is_empty(&self) -> bool { + self.buf.is_empty() + } + + /// One-shot: encode a single [`Value`] into a [`Blob`]. + pub fn quote(v: &Value) -> Blob { + let mut b = Builder::new(); + b.value(v); + b.build() + } +} diff --git a/rust/src/decode.rs b/rust/src/decode.rs new file mode 100644 index 0000000..d01077e --- /dev/null +++ b/rust/src/decode.rs @@ -0,0 +1,476 @@ +//! Internal: decoding & inspection (mirrors `msgpack_blob_decode.cpp`). + +use crate::format as f; +use crate::value::{Type, Value}; + +pub const RC_OK: i32 = 0; +pub const RC_ERROR: i32 = 1; +pub const RC_NOTFOUND: i32 = 2; + +pub fn is_valid(a: &[u8], n: usize) -> bool { + if n == 0 { + return false; + } + f::skip_one(a, n, 0) == n +} + +pub fn error_position(a: &[u8], n: usize) -> usize { + if n == 0 { + return 0; + } + if f::skip_one(a, n, 0) == n { + return 0; + } + let mut i = 0; + while i < n { + let nxt = f::skip_one(a, n, i); + if nxt == 0 { + return i; + } + i = nxt; + } + 0 +} + +fn is_timestamp_ext(a: &[u8], n: usize, i: usize) -> bool { + if i >= n { + return false; + } + let b = a[i]; + if b == f::MP_FIXEXT4 && i + 6 <= n && a[i + 1] == f::MP_TIMESTAMP_TYPE { + return true; + } + if b == f::MP_FIXEXT8 && i + 10 <= n && a[i + 1] == f::MP_TIMESTAMP_TYPE { + return true; + } + if b == f::MP_EXT8 && i + 3 <= n && a[i + 1] == 12 && a[i + 2] == f::MP_TIMESTAMP_TYPE { + return true; + } + false +} + +fn decode_timestamp(a: &[u8], n: usize, i: usize) -> Option<(i64, u32)> { + if i >= n { + return None; + } + let b = a[i]; + if b == f::MP_FIXEXT4 && i + 6 <= n && a[i + 1] == f::MP_TIMESTAMP_TYPE { + return Some((f::read32(a, i + 2) as i64, 0)); + } + if b == f::MP_FIXEXT8 && i + 10 <= n && a[i + 1] == f::MP_TIMESTAMP_TYPE { + let v = f::read64(a, i + 2); + return Some(((v & 0x3_FFFF_FFFF) as i64, (v >> 34) as u32)); + } + if b == f::MP_EXT8 && i + 15 <= n && a[i + 1] == 12 && a[i + 2] == f::MP_TIMESTAMP_TYPE { + let nsec = f::read32(a, i + 3); + let sec = f::read64(a, i + 7) as i64; + return Some((sec, nsec)); + } + None +} + +pub fn get_type(a: &[u8], n: usize, i: usize) -> Type { + if i >= n { + return Type::Nil; + } + let b = a[i]; + if b == f::MP_NIL { + return Type::Nil; + } + if b == f::MP_TRUE { + return Type::True; + } + if b == f::MP_FALSE { + return Type::False; + } + if b <= 0x7f || b >= 0xe0 { + return Type::Integer; + } + if (0xa0..=0xbf).contains(&b) { + return Type::String; + } + if (0x90..=0x9f).contains(&b) { + return Type::Array; + } + if (0x80..=0x8f).contains(&b) { + return Type::Map; + } + match b { + f::MP_UINT8 + | f::MP_UINT16 + | f::MP_UINT32 + | f::MP_UINT64 + | f::MP_INT8 + | f::MP_INT16 + | f::MP_INT32 + | f::MP_INT64 => Type::Integer, + f::MP_FLOAT32 => Type::Float32, + f::MP_FLOAT64 => Type::Real, + f::MP_STR8 | f::MP_STR16 | f::MP_STR32 => Type::String, + f::MP_BIN8 | f::MP_BIN16 | f::MP_BIN32 => Type::Binary, + f::MP_ARRAY16 | f::MP_ARRAY32 => Type::Array, + f::MP_MAP16 | f::MP_MAP32 => Type::Map, + f::MP_EXT8 + | f::MP_EXT16 + | f::MP_EXT32 + | f::MP_FIXEXT1 + | f::MP_FIXEXT2 + | f::MP_FIXEXT4 + | f::MP_FIXEXT8 + | f::MP_FIXEXT16 => { + if is_timestamp_ext(a, n, i) { + Type::Timestamp + } else { + Type::Ext + } + } + _ => Type::Nil, + } +} + +pub fn get_container_count(a: &[u8], n: usize, i: usize) -> i64 { + if i >= n { + return -1; + } + let b = a[i]; + if (0x90..=0x9f).contains(&b) { + return (b & 0x0f) as i64; + } + if (0x80..=0x8f).contains(&b) { + return (b & 0x0f) as i64; + } + if b == f::MP_ARRAY16 && i + 3 <= n { + return f::read16(a, i + 1) as i64; + } + if b == f::MP_ARRAY32 && i + 5 <= n { + return f::read32(a, i + 1) as i64; + } + if b == f::MP_MAP16 && i + 3 <= n { + return f::read16(a, i + 1) as i64; + } + if b == f::MP_MAP32 && i + 5 <= n { + return f::read32(a, i + 1) as i64; + } + -1 +} + +/// One parsed step of `$.key[idx]` syntax. +pub enum Step<'a> { + End, + Error, + Key(&'a str), + Index(i64), +} + +/// Parse one step starting at byte index `pi` in `zpath`; returns the step and +/// the next index. +pub fn path_step(zpath: &[u8], pi: usize) -> (Step<'_>, usize) { + let mut i = pi; + if i >= zpath.len() { + return (Step::End, i); + } + let c = zpath[i]; + if c == b'.' { + i += 1; + let start = i; + while i < zpath.len() && zpath[i] != b'.' && zpath[i] != b'[' { + i += 1; + } + // Path key bytes are ASCII/UTF-8 from the caller's &str. + let key = std::str::from_utf8(&zpath[start..i]).unwrap_or(""); + return (Step::Key(key), i); + } + if c == b'[' { + let mut idx: i64 = 0; + let mut has_digit = false; + i += 1; + while i < zpath.len() && zpath[i].is_ascii_digit() { + idx = idx * 10 + (zpath[i] - b'0') as i64; + i += 1; + has_digit = true; + } + if !has_digit || i >= zpath.len() || zpath[i] != b']' { + return (Step::Error, i); + } + i += 1; + return (Step::Index(idx), i); + } + (Step::Error, i) +} + +/// Key bytes of a map key at `i`, or `None` if not a string key. +fn key_at(a: &[u8], n: usize, i: usize) -> Option<&[u8]> { + let kb = a[i]; + let (klen, koff) = if (0xa0..=0xbf).contains(&kb) { + ((kb & 0x1f) as usize, i + 1) + } else if kb == f::MP_STR8 && i + 2 <= n { + (a[i + 1] as usize, i + 2) + } else if kb == f::MP_STR16 && i + 3 <= n { + (f::read16(a, i + 1) as usize, i + 3) + } else if kb == f::MP_STR32 && i + 5 <= n { + (f::read32(a, i + 1) as usize, i + 5) + } else { + return None; + }; + if klen > n - koff { + return None; + } + Some(&a[koff..koff + klen]) +} + +/// Resolve `zpath` to a byte range. Returns `(rc, i_start, i_end)`. +pub fn lookup(a: &[u8], n: usize, iroot: usize, zpath: &str) -> (i32, usize, usize) { + let zb = zpath.as_bytes(); + if zb.is_empty() || zb[0] != b'$' { + return (RC_ERROR, 0, 0); + } + let mut icur = iroot; + let mut pi = 1; + + loop { + let (step, npi) = path_step(zb, pi); + pi = npi; + match step { + Step::End => { + let inext = f::skip_one(a, n, icur); + let iend = if inext != 0 { inext } else { n }; + let rc = if inext != 0 || icur == n { + RC_OK + } else { + RC_ERROR + }; + return (rc, icur, iend); + } + Step::Error => return (RC_ERROR, 0, 0), + Step::Index(idx) => { + if icur >= n { + return (RC_NOTFOUND, 0, 0); + } + let b = a[icur]; + let (count, elem_off) = if (0x90..=0x9f).contains(&b) { + ((b & 0x0f) as i64, icur + 1) + } else if b == f::MP_ARRAY16 { + if icur + 3 > n { + return (RC_ERROR, 0, 0); + } + (f::read16(a, icur + 1) as i64, icur + 3) + } else if b == f::MP_ARRAY32 { + if icur + 5 > n { + return (RC_ERROR, 0, 0); + } + (f::read32(a, icur + 1) as i64, icur + 5) + } else { + return (RC_NOTFOUND, 0, 0); + }; + if idx < 0 || idx >= count { + return (RC_NOTFOUND, 0, 0); + } + icur = elem_off; + for _ in 0..idx { + icur = f::skip_one(a, n, icur); + if icur == 0 { + return (RC_ERROR, 0, 0); + } + } + } + Step::Key(key) => { + if icur >= n { + return (RC_NOTFOUND, 0, 0); + } + let b = a[icur]; + let (count, elem_off) = if (0x80..=0x8f).contains(&b) { + ((b & 0x0f) as usize, icur + 1) + } else if b == f::MP_MAP16 { + if icur + 3 > n { + return (RC_ERROR, 0, 0); + } + (f::read16(a, icur + 1) as usize, icur + 3) + } else if b == f::MP_MAP32 { + if icur + 5 > n { + return (RC_ERROR, 0, 0); + } + (f::read32(a, icur + 1) as usize, icur + 5) + } else { + return (RC_NOTFOUND, 0, 0); + }; + let key_bytes = key.as_bytes(); + icur = elem_off; + let mut found = false; + let mut j = 0; + while j < count && !found { + if icur >= n { + return (RC_ERROR, 0, 0); + } + let kstr = key_at(a, n, icur); + let val_off = f::skip_one(a, n, icur); + if val_off == 0 { + return (RC_ERROR, 0, 0); + } + if kstr == Some(key_bytes) { + icur = val_off; + found = true; + } else { + icur = f::skip_one(a, n, val_off); + if icur == 0 { + return (RC_ERROR, 0, 0); + } + } + j += 1; + } + if !found { + return (RC_NOTFOUND, 0, 0); + } + } + } + } +} + +pub fn decode_element(a: &[u8], n: usize, istart: usize, iend: usize) -> Value { + if istart >= n || istart >= iend { + return Value::nil(); + } + let b = a[istart]; + + if b == f::MP_NIL { + return Value::nil(); + } + if b == f::MP_FALSE { + return Value::boolean(false); + } + if b == f::MP_TRUE { + return Value::boolean(true); + } + if b <= 0x7f { + return Value::integer(b as i64); + } + if b >= 0xe0 { + return Value::integer(b as i8 as i64); + } + + match b { + f::MP_UINT8 => { + if istart + 2 <= n { + return Value::integer(a[istart + 1] as i64); + } + } + f::MP_UINT16 => { + if istart + 3 <= n { + return Value::integer(f::read16(a, istart + 1) as i64); + } + } + f::MP_UINT32 => { + if istart + 5 <= n { + return Value::integer(f::read32(a, istart + 1) as i64); + } + } + f::MP_UINT64 => { + if istart + 9 <= n { + return Value::unsigned_integer(f::read64(a, istart + 1)); + } + } + f::MP_INT8 => { + if istart + 2 <= n { + return Value::integer(a[istart + 1] as i8 as i64); + } + } + f::MP_INT16 => { + if istart + 3 <= n { + return Value::integer(f::read16(a, istart + 1) as u16 as i16 as i64); + } + } + f::MP_INT32 => { + if istart + 5 <= n { + return Value::integer(f::read32(a, istart + 1) as i32 as i64); + } + } + f::MP_INT64 => { + if istart + 9 <= n { + return Value::integer(f::read64(a, istart + 1) as i64); + } + } + f::MP_FLOAT32 => { + if istart + 5 <= n { + let bits = f::read32(a, istart + 1); + return Value::real32(f32::from_bits(bits)); + } + } + f::MP_FLOAT64 => { + if istart + 9 <= n { + let bits = f::read64(a, istart + 1); + return Value::real(f64::from_bits(bits)); + } + } + _ => {} + } + + // str + let (mut slen, soff) = if (0xa0..=0xbf).contains(&b) { + ((b & 0x1f) as usize, istart + 1) + } else if b == f::MP_STR8 && istart + 2 <= n { + (a[istart + 1] as usize, istart + 2) + } else if b == f::MP_STR16 && istart + 3 <= n { + (f::read16(a, istart + 1) as usize, istart + 3) + } else if b == f::MP_STR32 && istart + 5 <= n { + (f::read32(a, istart + 1) as usize, istart + 5) + } else { + (0, 0) + }; + if soff != 0 { + if slen > n - soff { + slen = n - soff; + } + return Value::string_bytes(&a[soff..soff + slen]); + } + + // bin + let (mut blen, boff) = if b == f::MP_BIN8 && istart + 2 <= n { + (a[istart + 1] as usize, istart + 2) + } else if b == f::MP_BIN16 && istart + 3 <= n { + (f::read16(a, istart + 1) as usize, istart + 3) + } else if b == f::MP_BIN32 && istart + 5 <= n { + (f::read32(a, istart + 1) as usize, istart + 5) + } else { + (0, 0) + }; + if boff != 0 { + if blen > n - boff { + blen = n - boff; + } + return Value::binary(&a[boff..boff + blen]); + } + + // timestamp + if let Some((sec, nsec)) = decode_timestamp(a, n, istart) { + return Value::timestamp_ns(sec, nsec); + } + + // ext + let (tc, mut elen, eoff): (i8, usize, usize) = match b { + f::MP_FIXEXT1 if istart + 3 <= n => (a[istart + 1] as i8, 1, istart + 2), + f::MP_FIXEXT2 if istart + 4 <= n => (a[istart + 1] as i8, 2, istart + 2), + f::MP_FIXEXT4 if istart + 6 <= n => (a[istart + 1] as i8, 4, istart + 2), + f::MP_FIXEXT8 if istart + 10 <= n => (a[istart + 1] as i8, 8, istart + 2), + f::MP_FIXEXT16 if istart + 18 <= n => (a[istart + 1] as i8, 16, istart + 2), + f::MP_EXT8 if istart + 3 <= n => (a[istart + 2] as i8, a[istart + 1] as usize, istart + 3), + f::MP_EXT16 if istart + 4 <= n => ( + a[istart + 3] as i8, + f::read16(a, istart + 1) as usize, + istart + 4, + ), + f::MP_EXT32 if istart + 6 <= n => ( + a[istart + 5] as i8, + f::read32(a, istart + 1) as usize, + istart + 6, + ), + _ => (0, 0, 0), + }; + if eoff != 0 { + if elen > n - eoff { + elen = n - eoff; + } + return Value::ext(tc, &a[eoff..eoff + elen]); + } + + // containers → raw binary blob (includes header) + Value::binary(&a[istart..iend]) +} diff --git a/rust/src/encode.rs b/rust/src/encode.rs new file mode 100644 index 0000000..a7ac2ea --- /dev/null +++ b/rust/src/encode.rs @@ -0,0 +1,234 @@ +//! Internal: encoding primitives (mirrors `msgpack_blob_encode.cpp`). + +use crate::format as f; +use crate::value::{IntWidth, Type, Value}; + +pub fn enc_nil(out: &mut Vec) { + out.push(f::MP_NIL); +} + +pub fn enc_bool(out: &mut Vec, v: bool) { + out.push(if v { f::MP_TRUE } else { f::MP_FALSE }); +} + +pub fn enc_integer(out: &mut Vec, x: i64) { + if x >= 0 { + if x <= 0x7f { + out.push(x as u8); + } else if x <= 0xff { + out.push(f::MP_UINT8); + out.push(x as u8); + } else if x <= 0xffff { + out.push(f::MP_UINT16); + f::push16(out, x as u16); + } else if x <= 0xffff_ffff { + out.push(f::MP_UINT32); + f::push32(out, x as u32); + } else { + out.push(f::MP_UINT64); + f::push64(out, x as u64); + } + } else if x >= -32 { + out.push(x as u8); + } else if x >= -128 { + out.push(f::MP_INT8); + out.push(x as u8); + } else if x >= -32768 { + out.push(f::MP_INT16); + f::push16(out, x as u16); + } else if x >= -2147483648 { + out.push(f::MP_INT32); + f::push32(out, x as u32); + } else { + out.push(f::MP_INT64); + f::push64(out, x as u64); + } +} + +pub fn enc_unsigned(out: &mut Vec, x: u64) { + if x <= 0x7f { + out.push(x as u8); + } else if x <= 0xff { + out.push(f::MP_UINT8); + out.push(x as u8); + } else if x <= 0xffff { + out.push(f::MP_UINT16); + f::push16(out, x as u16); + } else if x <= 0xffff_ffff { + out.push(f::MP_UINT32); + f::push32(out, x as u32); + } else { + out.push(f::MP_UINT64); + f::push64(out, x); + } +} + +pub fn enc_real(out: &mut Vec, d: f64) { + out.push(f::MP_FLOAT64); + f::push64(out, d.to_bits()); +} + +pub fn enc_real32(out: &mut Vec, val: f32) { + out.push(f::MP_FLOAT32); + f::push32(out, val.to_bits()); +} + +pub fn enc_string(out: &mut Vec, s: &[u8]) { + let n = s.len(); + if n <= 31 { + out.push(f::MP_FIXSTR_MASK | n as u8); + } else if n <= 0xff { + out.push(f::MP_STR8); + out.push(n as u8); + } else if n <= 0xffff { + out.push(f::MP_STR16); + f::push16(out, n as u16); + } else { + out.push(f::MP_STR32); + f::push32(out, n as u32); + } + out.extend_from_slice(s); +} + +pub fn enc_binary(out: &mut Vec, data: &[u8]) { + let n = data.len(); + if n <= 0xff { + out.push(f::MP_BIN8); + out.push(n as u8); + } else if n <= 0xffff { + out.push(f::MP_BIN16); + f::push16(out, n as u16); + } else { + out.push(f::MP_BIN32); + f::push32(out, n as u32); + } + out.extend_from_slice(data); +} + +pub fn enc_ext(out: &mut Vec, type_code: i8, data: &[u8]) { + let n = data.len(); + match n { + 1 => out.push(f::MP_FIXEXT1), + 2 => out.push(f::MP_FIXEXT2), + 4 => out.push(f::MP_FIXEXT4), + 8 => out.push(f::MP_FIXEXT8), + 16 => out.push(f::MP_FIXEXT16), + _ => { + if n <= 0xff { + out.push(f::MP_EXT8); + out.push(n as u8); + } else if n <= 0xffff { + out.push(f::MP_EXT16); + f::push16(out, n as u16); + } else { + out.push(f::MP_EXT32); + f::push32(out, n as u32); + } + } + } + out.push(type_code as u8); + out.extend_from_slice(data); +} + +pub fn enc_int8(out: &mut Vec, x: i64) { + out.push(f::MP_INT8); + out.push(x as u8); +} +pub fn enc_int16(out: &mut Vec, x: i64) { + out.push(f::MP_INT16); + f::push16(out, x as u16); +} +pub fn enc_int32(out: &mut Vec, x: i64) { + out.push(f::MP_INT32); + f::push32(out, x as u32); +} +pub fn enc_int64(out: &mut Vec, x: i64) { + out.push(f::MP_INT64); + f::push64(out, x as u64); +} +pub fn enc_uint8(out: &mut Vec, x: u64) { + out.push(f::MP_UINT8); + out.push(x as u8); +} +pub fn enc_uint16(out: &mut Vec, x: u64) { + out.push(f::MP_UINT16); + f::push16(out, x as u16); +} +pub fn enc_uint32(out: &mut Vec, x: u64) { + out.push(f::MP_UINT32); + f::push32(out, x as u32); +} +pub fn enc_uint64(out: &mut Vec, x: u64) { + out.push(f::MP_UINT64); + f::push64(out, x); +} + +pub fn enc_array_header(out: &mut Vec, count: u32) { + if count <= 15 { + out.push(f::MP_FIXARRAY_MASK | count as u8); + } else if count <= 0xffff { + out.push(f::MP_ARRAY16); + f::push16(out, count as u16); + } else { + out.push(f::MP_ARRAY32); + f::push32(out, count); + } +} + +pub fn enc_map_header(out: &mut Vec, count: u32) { + if count <= 15 { + out.push(f::MP_FIXMAP_MASK | count as u8); + } else if count <= 0xffff { + out.push(f::MP_MAP16); + f::push16(out, count as u16); + } else { + out.push(f::MP_MAP32); + f::push32(out, count); + } +} + +pub fn enc_timestamp(out: &mut Vec, sec: i64, nsec: u32) { + if nsec == 0 && (0..=0xffff_ffff).contains(&sec) { + out.push(f::MP_FIXEXT4); + out.push(0xff); + f::push32(out, sec as u32); + } else if (0..=0x3_FFFF_FFFF).contains(&sec) { + out.push(f::MP_FIXEXT8); + out.push(0xff); + f::push64(out, ((nsec as u64) << 34) | sec as u64); + } else { + out.push(f::MP_EXT8); + out.push(12); + out.push(0xff); + f::push32(out, nsec); + f::push64(out, sec as u64); + } +} + +pub fn encode_value(out: &mut Vec, v: &Value) { + match v.get_type() { + Type::Nil => enc_nil(out), + Type::True => enc_bool(out, true), + Type::False => enc_bool(out, false), + Type::Integer => match v.int_width() { + IntWidth::Int8 => enc_int8(out, v.as_i64()), + IntWidth::Int16 => enc_int16(out, v.as_i64()), + IntWidth::Int32 => enc_int32(out, v.as_i64()), + IntWidth::Int64 => enc_int64(out, v.as_i64()), + IntWidth::Uint8 => enc_uint8(out, v.as_u64()), + IntWidth::Uint16 => enc_uint16(out, v.as_u64()), + IntWidth::Uint32 => enc_uint32(out, v.as_u64()), + IntWidth::Uint64 => enc_uint64(out, v.as_u64()), + IntWidth::Auto => enc_integer(out, v.as_i64()), + }, + Type::Real => enc_real(out, v.as_f64()), + Type::Float32 => enc_real32(out, v.as_f32()), + Type::String => enc_string(out, v.as_bytes()), + Type::Binary => enc_binary(out, v.blob_data()), + Type::Ext => enc_ext(out, v.ext_type(), v.blob_data()), + Type::Timestamp => enc_timestamp(out, v.timestamp_seconds(), v.timestamp_nanoseconds()), + // Array / Map are not directly encodable as a scalar Value (the C++ + // reference falls through to nil here). + Type::Array | Type::Map => enc_nil(out), + } +} diff --git a/rust/src/format.rs b/rust/src/format.rs new file mode 100644 index 0000000..0d7a59f --- /dev/null +++ b/rust/src/format.rs @@ -0,0 +1,252 @@ +//! Internal: MessagePack format constants, byte-order helpers and `skip_one`. +//! +//! Private to the crate; mirrors `cpp/src/msgpack_blob_detail.hpp` and the skip +//! routine from the C++ decode module. + +pub const MAX_DEPTH: i32 = 200; +pub const MAX_OUTPUT: usize = 64 * 1024 * 1024; + +pub const MP_NIL: u8 = 0xc0; +pub const MP_FALSE: u8 = 0xc2; +pub const MP_TRUE: u8 = 0xc3; +pub const MP_BIN8: u8 = 0xc4; +pub const MP_BIN16: u8 = 0xc5; +pub const MP_BIN32: u8 = 0xc6; +pub const MP_EXT8: u8 = 0xc7; +pub const MP_EXT16: u8 = 0xc8; +pub const MP_EXT32: u8 = 0xc9; +pub const MP_FLOAT32: u8 = 0xca; +pub const MP_FLOAT64: u8 = 0xcb; +pub const MP_UINT8: u8 = 0xcc; +pub const MP_UINT16: u8 = 0xcd; +pub const MP_UINT32: u8 = 0xce; +pub const MP_UINT64: u8 = 0xcf; +pub const MP_INT8: u8 = 0xd0; +pub const MP_INT16: u8 = 0xd1; +pub const MP_INT32: u8 = 0xd2; +pub const MP_INT64: u8 = 0xd3; +pub const MP_FIXEXT1: u8 = 0xd4; +pub const MP_FIXEXT2: u8 = 0xd5; +pub const MP_FIXEXT4: u8 = 0xd6; +pub const MP_FIXEXT8: u8 = 0xd7; +pub const MP_FIXEXT16: u8 = 0xd8; +pub const MP_STR8: u8 = 0xd9; +pub const MP_STR16: u8 = 0xda; +pub const MP_STR32: u8 = 0xdb; +pub const MP_ARRAY16: u8 = 0xdc; +pub const MP_ARRAY32: u8 = 0xdd; +pub const MP_MAP16: u8 = 0xde; +pub const MP_MAP32: u8 = 0xdf; + +pub const MP_FIXMAP_MASK: u8 = 0x80; +pub const MP_FIXARRAY_MASK: u8 = 0x90; +pub const MP_FIXSTR_MASK: u8 = 0xa0; + +pub const MP_TIMESTAMP_TYPE: u8 = 0xff; + +// ── big-endian read helpers ───────────────────────────────────────── +#[inline] +pub fn read16(a: &[u8], i: usize) -> u32 { + ((a[i] as u32) << 8) | a[i + 1] as u32 +} +#[inline] +pub fn read32(a: &[u8], i: usize) -> u32 { + ((a[i] as u32) << 24) | ((a[i + 1] as u32) << 16) | ((a[i + 2] as u32) << 8) | a[i + 3] as u32 +} +#[inline] +pub fn read64(a: &[u8], i: usize) -> u64 { + ((read32(a, i) as u64) << 32) | read32(a, i + 4) as u64 +} + +// ── big-endian write helpers ──────────────────────────────────────── +#[inline] +pub fn push16(out: &mut Vec, v: u16) { + out.extend_from_slice(&v.to_be_bytes()); +} +#[inline] +pub fn push32(out: &mut Vec, v: u32) { + out.extend_from_slice(&v.to_be_bytes()); +} +#[inline] +pub fn push64(out: &mut Vec, v: u64) { + out.extend_from_slice(&v.to_be_bytes()); +} + +/// Return the offset just past one complete element starting at `i`, +/// or 0 on malformed / truncated input. +pub fn skip_one(a: &[u8], n: usize, i: usize) -> usize { + skip_one_d(a, n, i, 0) +} + +fn skip_one_d(a: &[u8], n: usize, mut i: usize, depth: i32) -> usize { + if depth > MAX_DEPTH { + return 0; + } + if i >= n { + return 0; + } + let b = a[i]; + i += 1; + + if b <= 0x7f { + return i; + } + if b >= 0xe0 { + return i; + } + + match b { + MP_NIL | MP_FALSE | MP_TRUE => return i, + MP_FLOAT32 => return if i + 4 <= n { i + 4 } else { 0 }, + MP_FLOAT64 | MP_INT64 | MP_UINT64 => return if i + 8 <= n { i + 8 } else { 0 }, + MP_UINT8 | MP_INT8 => return if i + 1 <= n { i + 1 } else { 0 }, + MP_UINT16 | MP_INT16 => return if i + 2 <= n { i + 2 } else { 0 }, + MP_UINT32 | MP_INT32 => return if i + 4 <= n { i + 4 } else { 0 }, + MP_BIN8 | MP_STR8 => { + if i + 1 > n { + return 0; + } + let sz = a[i] as usize; + i += 1; + return if sz <= n - i { i + sz } else { 0 }; + } + MP_BIN16 | MP_STR16 => { + if i + 2 > n { + return 0; + } + let sz = read16(a, i) as usize; + i += 2; + return if sz <= n - i { i + sz } else { 0 }; + } + MP_BIN32 | MP_STR32 => { + if i + 4 > n { + return 0; + } + let sz = read32(a, i) as usize; + i += 4; + return if sz <= n - i { i + sz } else { 0 }; + } + MP_FIXEXT1 => return if i + 2 <= n { i + 2 } else { 0 }, + MP_FIXEXT2 => return if i + 3 <= n { i + 3 } else { 0 }, + MP_FIXEXT4 => return if i + 5 <= n { i + 5 } else { 0 }, + MP_FIXEXT8 => return if i + 9 <= n { i + 9 } else { 0 }, + MP_FIXEXT16 => return if i + 17 <= n { i + 17 } else { 0 }, + MP_EXT8 => { + if i + 2 > n { + return 0; + } + let sz = a[i] as usize; + i += 2; + return if sz <= n - i { i + sz } else { 0 }; + } + MP_EXT16 => { + if i + 3 > n { + return 0; + } + let sz = read16(a, i) as usize; + i += 3; + return if sz <= n - i { i + sz } else { 0 }; + } + MP_EXT32 => { + if i + 5 > n { + return 0; + } + let sz = read32(a, i) as usize; + i += 5; + return if sz <= n - i { i + sz } else { 0 }; + } + _ => {} + } + + // fixstr + if (0xa0..=0xbf).contains(&b) { + let sz = (b & 0x1f) as usize; + return if sz <= n - i { i + sz } else { 0 }; + } + + // fixarray + if (0x90..=0x9f).contains(&b) { + let count = (b & 0x0f) as usize; + for _ in 0..count { + i = skip_one_d(a, n, i, depth + 1); + if i == 0 { + return 0; + } + } + return i; + } + + // fixmap + if (0x80..=0x8f).contains(&b) { + let count = (b & 0x0f) as usize; + for _ in 0..count { + i = skip_one_d(a, n, i, depth + 1); + if i == 0 { + return 0; + } + i = skip_one_d(a, n, i, depth + 1); + if i == 0 { + return 0; + } + } + return i; + } + + // array16/32 + if b == MP_ARRAY16 || b == MP_ARRAY32 { + let count = if b == MP_ARRAY16 { + if i + 2 > n { + return 0; + } + let c = read16(a, i) as usize; + i += 2; + c + } else { + if i + 4 > n { + return 0; + } + let c = read32(a, i) as usize; + i += 4; + c + }; + for _ in 0..count { + i = skip_one_d(a, n, i, depth + 1); + if i == 0 { + return 0; + } + } + return i; + } + + // map16/32 + if b == MP_MAP16 || b == MP_MAP32 { + let count = if b == MP_MAP16 { + if i + 2 > n { + return 0; + } + let c = read16(a, i) as usize; + i += 2; + c + } else { + if i + 4 > n { + return 0; + } + let c = read32(a, i) as usize; + i += 4; + c + }; + for _ in 0..count { + i = skip_one_d(a, n, i, depth + 1); + if i == 0 { + return 0; + } + i = skip_one_d(a, n, i, depth + 1); + if i == 0 { + return 0; + } + } + return i; + } + + 0 +} diff --git a/rust/src/iterate.rs b/rust/src/iterate.rs new file mode 100644 index 0000000..57c4d1e --- /dev/null +++ b/rust/src/iterate.rs @@ -0,0 +1,256 @@ +//! Internal: container iteration (mirrors `msgpack_blob_iterate.cpp`). + +use crate::decode::{decode_element, get_type}; +use crate::format as f; +use crate::value::{Type, Value}; + +/// A single row yielded by [`crate::Iterator`]. +#[derive(Clone, Debug)] +pub struct EachRow { + /// Map key (`""` for arrays / tree rows). + pub key: String, + /// Array index or pair index (meaningful for flat iteration only). + pub index: i64, + /// Full path, e.g. `"$.users[0].name"`. + pub fullkey: String, + /// Parent path. + pub path: String, + /// Byte offset of the element in the blob. + pub id: usize, + /// Element type. + pub ty: Type, + /// Element value. + pub value: Value, +} + +fn key_str(a: &[u8], n: usize, i: usize) -> Option { + let kb = a[i]; + let (klen, koff) = if (0xa0..=0xbf).contains(&kb) { + ((kb & 0x1f) as usize, i + 1) + } else if kb == f::MP_STR8 && i + 2 <= n { + (a[i + 1] as usize, i + 2) + } else if kb == f::MP_STR16 && i + 3 <= n { + (f::read16(a, i + 1) as usize, i + 3) + } else if kb == f::MP_STR32 && i + 5 <= n { + (f::read32(a, i + 1) as usize, i + 5) + } else { + return None; + }; + // Bounds guard (matches decode::key_at): reject a key length that runs past + // the end of the blob. each_iter then defers to skip_one, which fails and + // stops iteration — matching the C++ reference (no out-of-bounds read). + if klen > n - koff { + return None; + } + Some(String::from_utf8_lossy(&a[koff..koff + klen]).into_owned()) +} + +struct Container { + is_arr: bool, + is_map: bool, + count: usize, + data_off: usize, +} + +fn container(a: &[u8], n: usize, i: usize) -> Container { + let b = a[i]; + if (0x90..=0x9f).contains(&b) { + return Container { + is_arr: true, + is_map: false, + count: (b & 0x0f) as usize, + data_off: i + 1, + }; + } + if b == f::MP_ARRAY16 && i + 3 <= n { + return Container { + is_arr: true, + is_map: false, + count: f::read16(a, i + 1) as usize, + data_off: i + 3, + }; + } + if b == f::MP_ARRAY32 && i + 5 <= n { + return Container { + is_arr: true, + is_map: false, + count: f::read32(a, i + 1) as usize, + data_off: i + 5, + }; + } + if (0x80..=0x8f).contains(&b) { + return Container { + is_arr: false, + is_map: true, + count: (b & 0x0f) as usize, + data_off: i + 1, + }; + } + if b == f::MP_MAP16 && i + 3 <= n { + return Container { + is_arr: false, + is_map: true, + count: f::read16(a, i + 1) as usize, + data_off: i + 3, + }; + } + if b == f::MP_MAP32 && i + 5 <= n { + return Container { + is_arr: false, + is_map: true, + count: f::read32(a, i + 1) as usize, + data_off: i + 5, + }; + } + Container { + is_arr: false, + is_map: false, + count: 0, + data_off: 0, + } +} + +pub fn each_iter(a: &[u8], n: usize, icont: usize, zbase: &str) -> Vec { + let mut rows = Vec::new(); + if icont >= n { + return rows; + } + let c = container(a, n, icont); + if !c.is_arr && !c.is_map { + return rows; + } + + let remaining = if c.data_off <= n { n - c.data_off } else { 0 }; + let min_bytes = if c.is_map { 2 } else { 1 }; + if c.count > remaining / min_bytes + 1 { + return rows; + } + + let mut cur = c.data_off; + for j in 0..c.count { + if cur >= n { + break; + } + if c.is_arr { + let c_end = f::skip_one(a, n, cur); + if c_end == 0 { + break; + } + rows.push(EachRow { + key: String::new(), + index: j as i64, + fullkey: format!("{}[{}]", zbase, j), + path: zbase.to_string(), + id: cur, + ty: get_type(a, n, cur), + value: decode_element(a, n, cur, c_end), + }); + cur = c_end; + } else { + let ks = key_str(a, n, cur); + let v_off = f::skip_one(a, n, cur); + if v_off == 0 { + break; + } + let p_end = f::skip_one(a, n, v_off); + if p_end == 0 { + break; + } + let key = ks.unwrap_or_else(|| "?".to_string()); + rows.push(EachRow { + fullkey: format!("{}.{}", zbase, key), + key, + index: j as i64, + path: zbase.to_string(), + id: v_off, + ty: get_type(a, n, v_off), + value: decode_element(a, n, v_off, p_end), + }); + cur = p_end; + } + } + rows +} + +pub fn tree_walk( + a: &[u8], + n: usize, + ioff: usize, + zfull: &str, + zpar_path: &str, + depth: i32, + rows: &mut Vec, +) { + if depth > f::MAX_DEPTH || ioff >= n { + return; + } + let iend = f::skip_one(a, n, ioff); + if iend == 0 { + return; + } + + rows.push(EachRow { + key: String::new(), + index: 0, + fullkey: zfull.to_string(), + path: zpar_path.to_string(), + id: ioff, + ty: get_type(a, n, ioff), + value: decode_element(a, n, ioff, iend), + }); + + let c = container(a, n, ioff); + if !c.is_arr && !c.is_map { + return; + } + + let remaining = if c.data_off <= n { n - c.data_off } else { 0 }; + let min_bytes = if c.is_map { 2 } else { 1 }; + if c.count > remaining / min_bytes + 1 { + return; + } + + let mut cur = c.data_off; + for j in 0..c.count { + if cur >= n { + break; + } + if c.is_arr { + let c_end = f::skip_one(a, n, cur); + if c_end == 0 { + break; + } + tree_walk( + a, + n, + cur, + &format!("{}[{}]", zfull, j), + zfull, + depth + 1, + rows, + ); + cur = c_end; + } else { + let ks = key_str(a, n, cur); + let v_off = f::skip_one(a, n, cur); + if v_off == 0 { + break; + } + let p_end = f::skip_one(a, n, v_off); + if p_end == 0 { + break; + } + let key = ks.unwrap_or_else(|| "?".to_string()); + tree_walk( + a, + n, + v_off, + &format!("{}.{}", zfull, key), + zfull, + depth + 1, + rows, + ); + cur = p_end; + } + } +} diff --git a/rust/src/iterator.rs b/rust/src/iterator.rs new file mode 100644 index 0000000..86cead9 --- /dev/null +++ b/rust/src/iterator.rs @@ -0,0 +1,95 @@ +//! `Iterator` — a cursor over container children (flat `each` / recursive `tree`). + +use crate::blob::Blob; +use crate::decode as d; +use crate::iterate::{each_iter, tree_walk, EachRow}; + +/// Iterate over a container's children. +/// +/// Supports flat (`each`) and recursive (`tree`) modes, mirroring the SQLite +/// extension's `msgpack_each` / `msgpack_tree` table-valued functions. Use the +/// C++-style [`Iterator::next`] / [`Iterator::current`] cursor, collect +/// [`Iterator::rows`], or drive it as a standard Rust iterator. +pub struct Iterator<'a> { + blob: &'a Blob, + base: String, + recursive: bool, + rows: Vec, + cursor: i64, + populated: bool, +} + +impl<'a> Iterator<'a> { + pub fn new(blob: &'a Blob, path: &str, recursive: bool) -> Iterator<'a> { + Iterator { + blob, + base: if path.is_empty() { + "$".to_string() + } else { + path.to_string() + }, + recursive, + rows: Vec::new(), + cursor: -1, + populated: false, + } + } + + fn populate(&mut self) { + if self.populated { + return; + } + self.populated = true; + self.rows.clear(); + let a = self.blob.data(); + let n = a.len(); + if n == 0 { + return; + } + + let mut iroot = 0; + if self.base != "$" { + let (rc, istart, _) = d::lookup(a, n, 0, &self.base); + if rc != d::RC_OK { + return; + } + iroot = istart; + } + + if self.recursive { + let base = self.base.clone(); + tree_walk(a, n, iroot, &base, &base, 0, &mut self.rows); + } else { + self.rows = each_iter(a, n, iroot, &self.base); + } + } + + // ── C++-style cursor protocol ───────────────────────────────────── + #[allow(clippy::should_implement_trait)] + pub fn next(&mut self) -> bool { + self.populate(); + self.cursor += 1; + (self.cursor as usize) < self.rows.len() + } + pub fn current(&self) -> &EachRow { + &self.rows[self.cursor as usize] + } + pub fn reset(&mut self) { + self.cursor = -1; + } + + /// Collect all rows. + pub fn rows(mut self) -> Vec { + self.populate(); + self.rows + } +} + +impl<'a> IntoIterator for Iterator<'a> { + type Item = EachRow; + type IntoIter = std::vec::IntoIter; + fn into_iter(mut self) -> Self::IntoIter { + self.populate(); + self.rows.into_iter() + } +} diff --git a/rust/src/json.rs b/rust/src/json.rs new file mode 100644 index 0000000..f60e818 --- /dev/null +++ b/rust/src/json.rs @@ -0,0 +1,670 @@ +//! Internal: JSON conversion (mirrors `msgpack_blob_json.cpp`). +//! +//! `to_json` builds a byte buffer exactly like the C++ implementation so float +//! formatting and string escaping stay byte-identical. The float formatter +//! reproduces C `printf("%.

g")` by delegating the round-half-to-even digit +//! generation to Rust's `{:.*e}` formatter (verified to match C digit-for-digit) +//! and then re-applying C's `%g` layout rules. + +use crate::encode as e; +use crate::format as f; + +pub const RC_OK: i32 = 0; +pub const RC_ERROR: i32 = 1; + +const HEX: &[u8; 16] = b"0123456789abcdef"; + +// ── C printf "%.

g" ─────────────────────────────────────────────── +pub fn c_format_g(value: f64, p_in: i32) -> String { + let p = if p_in <= 0 { 1 } else { p_in } as usize; + if value == 0.0 { + return if value.is_sign_negative() { + "-0".to_string() + } else { + "0".to_string() + }; + } + let neg = value < 0.0; + let a = value.abs(); + + // `{:.*e}` with (p-1) fractional digits yields p significant digits, rounded + // half-to-even — identical digits/exponent to C's `%.*e`. + let sci = format!("{:.*e}", p - 1, a); + let (mant, exp_str) = sci.split_once('e').expect("scientific format"); + let x: i32 = exp_str.parse().expect("exponent"); + let digits: String = mant.chars().filter(|c| *c != '.').collect(); + + let mut out = if x >= -4 && x < p as i32 { + // fixed-point notation + let mut s = if x >= 0 { + let int_len = (x + 1) as usize; + let int_part = &digits[..int_len.min(digits.len())]; + let frac_part = &digits[int_len.min(digits.len())..]; + let mut t = String::from(int_part); + if !frac_part.is_empty() { + t.push('.'); + t.push_str(frac_part); + } + t + } else { + let mut t = String::from("0."); + for _ in 0..(-x - 1) { + t.push('0'); + } + t.push_str(&digits); + t + }; + if s.contains('.') { + while s.ends_with('0') { + s.pop(); + } + if s.ends_with('.') { + s.pop(); + } + } + s + } else { + // scientific notation + let mut m = String::new(); + m.push(digits.as_bytes()[0] as char); + if digits.len() > 1 { + m.push('.'); + m.push_str(&digits[1..]); + } + if m.contains('.') { + while m.ends_with('0') { + m.pop(); + } + if m.ends_with('.') { + m.pop(); + } + } + let mut ea = x.unsigned_abs().to_string(); + if ea.len() < 2 { + ea.insert(0, '0'); + } + format!("{}e{}{}", m, if x < 0 { "-" } else { "+" }, ea) + }; + + if neg { + out.insert(0, '-'); + } + out +} + +fn fmt_double(d: f64) -> String { + let s = c_format_g(d, 17); + if !s.contains('.') && !s.contains('e') && !s.contains('E') { + // C re-formats integer-valued doubles with "%.1f". + format!("{:.1}", d) + } else { + s + } +} + +fn fmt_float32(val: f32) -> String { + c_format_g(val as f64, 7) +} + +// ── JSON output ───────────────────────────────────────────────────── +fn escape_str(out: &mut Vec, s: &[u8]) { + out.push(b'"'); + let mut start = 0; + let n = s.len(); + for j in 0..n { + let c = s[j]; + if c >= 0x20 && c != b'"' && c != b'\\' { + continue; + } + if j > start { + out.extend_from_slice(&s[start..j]); + } + match c { + b'"' => out.extend_from_slice(b"\\\""), + b'\\' => out.extend_from_slice(b"\\\\"), + b'\n' => out.extend_from_slice(b"\\n"), + b'\r' => out.extend_from_slice(b"\\r"), + b'\t' => out.extend_from_slice(b"\\t"), + _ => out.extend_from_slice(format!("\\u{:04x}", c).as_bytes()), + } + start = j + 1; + } + if n > start { + out.extend_from_slice(&s[start..n]); + } + out.push(b'"'); +} + +fn newline(out: &mut Vec, depth: i32, indent_w: i32) { + out.push(b'\n'); + for _ in 0..(depth * indent_w) { + out.push(b' '); + } +} + +#[allow(clippy::too_many_arguments)] +fn to_json_at( + out: &mut Vec, + a: &[u8], + n: usize, + i: usize, + pretty: bool, + depth: i32, + indent_w: i32, +) { + if i >= n || depth > f::MAX_DEPTH { + out.extend_from_slice(b"null"); + return; + } + let b = a[i]; + + if b == f::MP_NIL { + out.extend_from_slice(b"null"); + return; + } + if b == f::MP_FALSE { + out.extend_from_slice(b"false"); + return; + } + if b == f::MP_TRUE { + out.extend_from_slice(b"true"); + return; + } + if b <= 0x7f { + out.extend_from_slice(b.to_string().as_bytes()); + return; + } + if b >= 0xe0 { + out.extend_from_slice((b as i8).to_string().as_bytes()); + return; + } + + match b { + f::MP_UINT8 => { + if i + 2 <= n { + out.extend_from_slice(a[i + 1].to_string().as_bytes()); + return; + } + } + f::MP_UINT16 => { + if i + 3 <= n { + out.extend_from_slice(f::read16(a, i + 1).to_string().as_bytes()); + return; + } + } + f::MP_UINT32 => { + if i + 5 <= n { + out.extend_from_slice(f::read32(a, i + 1).to_string().as_bytes()); + return; + } + } + f::MP_UINT64 => { + if i + 9 <= n { + out.extend_from_slice(f::read64(a, i + 1).to_string().as_bytes()); + return; + } + } + f::MP_INT8 => { + if i + 2 <= n { + out.extend_from_slice((a[i + 1] as i8).to_string().as_bytes()); + return; + } + } + f::MP_INT16 => { + if i + 3 <= n { + out.extend_from_slice((f::read16(a, i + 1) as u16 as i16).to_string().as_bytes()); + return; + } + } + f::MP_INT32 => { + if i + 5 <= n { + out.extend_from_slice((f::read32(a, i + 1) as i32).to_string().as_bytes()); + return; + } + } + f::MP_INT64 => { + if i + 9 <= n { + out.extend_from_slice((f::read64(a, i + 1) as i64).to_string().as_bytes()); + return; + } + } + f::MP_FLOAT32 => { + if i + 5 <= n { + let val = f32::from_bits(f::read32(a, i + 1)); + if !val.is_finite() { + out.extend_from_slice(b"null"); + return; + } + out.extend_from_slice(fmt_float32(val).as_bytes()); + return; + } + } + f::MP_FLOAT64 => { + if i + 9 <= n { + let d = f64::from_bits(f::read64(a, i + 1)); + if !d.is_finite() { + out.extend_from_slice(b"null"); + return; + } + out.extend_from_slice(fmt_double(d).as_bytes()); + return; + } + } + _ => {} + } + + // str + let (mut slen, soff) = if (0xa0..=0xbf).contains(&b) { + ((b & 0x1f) as usize, i + 1) + } else if b == f::MP_STR8 && i + 2 <= n { + (a[i + 1] as usize, i + 2) + } else if b == f::MP_STR16 && i + 3 <= n { + (f::read16(a, i + 1) as usize, i + 3) + } else if b == f::MP_STR32 && i + 5 <= n { + (f::read32(a, i + 1) as usize, i + 5) + } else { + (0, 0) + }; + if soff != 0 { + if slen > n - soff { + slen = n - soff; + } + escape_str(out, &a[soff..soff + slen]); + return; + } + + // bin → hex string + let (mut blen, boff) = if b == f::MP_BIN8 && i + 2 <= n { + (a[i + 1] as usize, i + 2) + } else if b == f::MP_BIN16 && i + 3 <= n { + (f::read16(a, i + 1) as usize, i + 3) + } else if b == f::MP_BIN32 && i + 5 <= n { + (f::read32(a, i + 1) as usize, i + 5) + } else { + (0, 0) + }; + if boff != 0 { + if blen > n - boff { + blen = n - boff; + } + out.push(b'"'); + for j in 0..blen { + let by = a[boff + j]; + out.push(HEX[(by >> 4) as usize]); + out.push(HEX[(by & 0xf) as usize]); + } + out.push(b'"'); + return; + } + + // array + let (is_arr, count, data_off) = if (0x90..=0x9f).contains(&b) { + (true, (b & 0x0f) as usize, i + 1) + } else if b == f::MP_ARRAY16 && i + 3 <= n { + (true, f::read16(a, i + 1) as usize, i + 3) + } else if b == f::MP_ARRAY32 && i + 5 <= n { + (true, f::read32(a, i + 1) as usize, i + 5) + } else { + (false, 0, 0) + }; + if is_arr { + let mut cur = data_off; + out.push(b'['); + for j in 0..count { + if cur >= n { + break; + } + let nxt = f::skip_one(a, n, cur); + if j > 0 { + out.push(b','); + } + if pretty { + newline(out, depth + 1, indent_w); + } + to_json_at(out, a, n, cur, pretty, depth + 1, indent_w); + cur = if nxt != 0 { nxt } else { n }; + } + if pretty && count > 0 { + newline(out, depth, indent_w); + } + out.push(b']'); + return; + } + + // map + let (is_map, count, data_off) = if (0x80..=0x8f).contains(&b) { + (true, (b & 0x0f) as usize, i + 1) + } else if b == f::MP_MAP16 && i + 3 <= n { + (true, f::read16(a, i + 1) as usize, i + 3) + } else if b == f::MP_MAP32 && i + 5 <= n { + (true, f::read32(a, i + 1) as usize, i + 5) + } else { + (false, 0, 0) + }; + if is_map { + let mut cur = data_off; + out.push(b'{'); + for j in 0..count { + if cur >= n { + break; + } + let val_off = f::skip_one(a, n, cur); + let pair_end = if val_off != 0 { + f::skip_one(a, n, val_off) + } else { + 0 + }; + if j > 0 { + out.push(b','); + } + if pretty { + newline(out, depth + 1, indent_w); + } + to_json_at(out, a, n, cur, pretty, depth + 1, indent_w); + out.push(b':'); + if pretty { + out.push(b' '); + } + to_json_at( + out, + a, + n, + if val_off != 0 { val_off } else { n }, + pretty, + depth + 1, + indent_w, + ); + cur = if pair_end != 0 { pair_end } else { n }; + } + if pretty && count > 0 { + newline(out, depth, indent_w); + } + out.push(b'}'); + return; + } + + // ext / unknown → null + out.extend_from_slice(b"null"); +} + +pub fn to_json_bytes(a: &[u8], n: usize, pretty: bool, indent: i32) -> Vec { + let mut out = Vec::new(); + to_json_at(&mut out, a, n, 0, pretty, 0, indent); + out +} + +// ── JSON parser → msgpack ─────────────────────────────────────────── +struct P<'a> { + z: &'a [u8], + n: usize, + i: usize, +} + +fn skip_ws(p: &mut P) { + while p.i < p.n && matches!(p.z[p.i], 0x20 | 0x09 | 0x0a | 0x0d) { + p.i += 1; + } +} + +fn hex4(z: &[u8], off: usize) -> i32 { + let mut v: i32 = 0; + for j in 0..4 { + let c = z[off + j]; + let h = match c { + b'0'..=b'9' => (c - b'0') as i32, + b'a'..=b'f' => (c - b'a' + 10) as i32, + b'A'..=b'F' => (c - b'A' + 10) as i32, + _ => return -1, + }; + v = (v << 4) | h; + } + v +} + +fn cp_to_utf8(out: &mut Vec, cp: u32) { + if cp < 0x80 { + out.push(cp as u8); + } else if cp < 0x800 { + out.push(0xc0 | (cp >> 6) as u8); + out.push(0x80 | (cp & 0x3f) as u8); + } else if cp < 0x10000 { + out.push(0xe0 | (cp >> 12) as u8); + out.push(0x80 | ((cp >> 6) & 0x3f) as u8); + out.push(0x80 | (cp & 0x3f) as u8); + } else { + out.push(0xf0 | (cp >> 18) as u8); + out.push(0x80 | ((cp >> 12) & 0x3f) as u8); + out.push(0x80 | ((cp >> 6) & 0x3f) as u8); + out.push(0x80 | (cp & 0x3f) as u8); + } +} + +fn parse_string(p: &mut P, out: &mut Vec) -> i32 { + let mut sb = Vec::new(); + p.i += 1; // skip " + while p.i < p.n { + let c = p.z[p.i]; + if c == b'"' { + p.i += 1; + break; + } + if c == b'\\' { + p.i += 1; + if p.i >= p.n { + return RC_ERROR; + } + let esc = p.z[p.i]; + p.i += 1; + match esc { + b'"' => sb.push(b'"'), + b'\\' => sb.push(b'\\'), + b'/' => sb.push(b'/'), + b'n' => sb.push(b'\n'), + b'r' => sb.push(b'\r'), + b't' => sb.push(b'\t'), + b'b' => sb.push(0x08), + b'f' => sb.push(0x0c), + b'u' => { + if p.i + 4 > p.n { + return RC_ERROR; + } + let mut cp = hex4(p.z, p.i); + p.i += 4; + if cp < 0 { + return RC_ERROR; + } + if (0xd800..=0xdbff).contains(&cp) + && p.i + 6 <= p.n + && p.z[p.i] == b'\\' + && p.z[p.i + 1] == b'u' + { + let lo = hex4(p.z, p.i + 2); + if (0xdc00..=0xdfff).contains(&lo) { + p.i += 6; + cp = 0x10000 + ((cp - 0xd800) << 10) + (lo - 0xdc00); + } + } + cp_to_utf8(&mut sb, cp as u32); + } + _ => sb.push(esc), + } + } else { + sb.push(c); + p.i += 1; + } + } + e::enc_string(out, &sb); + RC_OK +} + +fn parse_number(p: &mut P, out: &mut Vec) -> i32 { + let start = p.i; + let mut is_float = false; + if p.i < p.n && p.z[p.i] == b'-' { + p.i += 1; + } + while p.i < p.n && p.z[p.i].is_ascii_digit() { + p.i += 1; + } + if p.i < p.n && p.z[p.i] == b'.' { + is_float = true; + p.i += 1; + while p.i < p.n && p.z[p.i].is_ascii_digit() { + p.i += 1; + } + } + if p.i < p.n && (p.z[p.i] == b'e' || p.z[p.i] == b'E') { + is_float = true; + p.i += 1; + if p.i < p.n && (p.z[p.i] == b'+' || p.z[p.i] == b'-') { + p.i += 1; + } + while p.i < p.n && p.z[p.i].is_ascii_digit() { + p.i += 1; + } + } + let len = p.i - start; + if len == 0 || len >= 64 { + return RC_ERROR; + } + let text = std::str::from_utf8(&p.z[start..p.i]).unwrap_or(""); + + if is_float { + let d: f64 = text.parse().unwrap_or(0.0); + e::enc_real(out, d); + } else { + // strtoll-style saturation to the i64 range + let v: i64 = text.parse().unwrap_or_else(|_| { + if text.starts_with('-') { + i64::MIN + } else { + i64::MAX + } + }); + if v >= 0 { + e::enc_unsigned(out, v as u64); + } else { + e::enc_integer(out, v); + } + } + RC_OK +} + +fn parse_array(p: &mut P, out: &mut Vec) -> i32 { + let mut tmp = Vec::new(); + let mut count: u32 = 0; + p.i += 1; // skip [ + skip_ws(p); + while p.i < p.n && p.z[p.i] != b']' { + if count > 0 { + skip_ws(p); + if p.i >= p.n || p.z[p.i] != b',' { + return RC_ERROR; + } + p.i += 1; + } + skip_ws(p); + if parse_value(p, &mut tmp) != RC_OK { + return RC_ERROR; + } + count += 1; + skip_ws(p); + } + if p.i >= p.n { + return RC_ERROR; + } + p.i += 1; // skip ] + e::enc_array_header(out, count); + out.extend_from_slice(&tmp); + RC_OK +} + +fn parse_object(p: &mut P, out: &mut Vec) -> i32 { + let mut tmp = Vec::new(); + let mut count: u32 = 0; + p.i += 1; // skip { + skip_ws(p); + while p.i < p.n && p.z[p.i] != b'}' { + if count > 0 { + skip_ws(p); + if p.i >= p.n || p.z[p.i] != b',' { + return RC_ERROR; + } + p.i += 1; + } + skip_ws(p); + if p.i >= p.n || p.z[p.i] != b'"' { + return RC_ERROR; + } + if parse_string(p, &mut tmp) != RC_OK { + return RC_ERROR; + } + skip_ws(p); + if p.i >= p.n || p.z[p.i] != b':' { + return RC_ERROR; + } + p.i += 1; + skip_ws(p); + if parse_value(p, &mut tmp) != RC_OK { + return RC_ERROR; + } + count += 1; + skip_ws(p); + } + if p.i >= p.n { + return RC_ERROR; + } + p.i += 1; // skip } + e::enc_map_header(out, count); + out.extend_from_slice(&tmp); + RC_OK +} + +fn parse_value(p: &mut P, out: &mut Vec) -> i32 { + skip_ws(p); + if p.i >= p.n { + return RC_ERROR; + } + let c = p.z[p.i]; + if c == b'n' && p.i + 4 <= p.n && &p.z[p.i..p.i + 4] == b"null" { + p.i += 4; + out.push(f::MP_NIL); + return RC_OK; + } + if c == b't' && p.i + 4 <= p.n && &p.z[p.i..p.i + 4] == b"true" { + p.i += 4; + out.push(f::MP_TRUE); + return RC_OK; + } + if c == b'f' && p.i + 5 <= p.n && &p.z[p.i..p.i + 5] == b"false" { + p.i += 5; + out.push(f::MP_FALSE); + return RC_OK; + } + if c == b'"' { + return parse_string(p, out); + } + if c == b'[' { + return parse_array(p, out); + } + if c == b'{' { + return parse_object(p, out); + } + if c == b'-' || c.is_ascii_digit() { + return parse_number(p, out); + } + RC_ERROR +} + +pub fn from_json(json: &[u8]) -> Vec { + let mut p = P { + z: json, + n: json.len(), + i: 0, + }; + let mut out = Vec::new(); + if parse_value(&mut p, &mut out) != RC_OK { + return Vec::new(); + } + out +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 0000000..7a806d3 --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,64 @@ +//! A pure-Rust MessagePack **Blob** library. +//! +//! A zero-dependency port of the standalone C++ `msgpack` Blob API from +//! [sqlite-msgpack](https://github.com/khanaffan/sqlite-msgpack). It creates, +//! queries, mutates and iterates MessagePack binary blobs and produces +//! **byte-identical** output to the C++ library and the `sqlite-msgpack` +//! extension, so blobs are fully interchangeable across all of them. +//! +//! # Example +//! ``` +//! use msgpack_blob::{Blob, Builder, Value, Iterator}; +//! +//! let blob = Blob::from_json(r#"{"name":"Alice","scores":[95,87,91]}"#); +//! assert_eq!(blob.extract("$.name").as_string(), "Alice"); +//! assert_eq!(blob.to_json(), r#"{"name":"Alice","scores":[95,87,91]}"#); +//! +//! let updated = blob.set("$.age", &Value::integer(30)); +//! assert_eq!(updated.to_json(), r#"{"name":"Alice","scores":[95,87,91],"age":30}"#); +//! +//! for row in Iterator::new(&blob, "$", true) { +//! let _ = (&row.fullkey, row.ty); +//! } +//! ``` + +// The internal modules are deliberately a close, line-for-line port of the C++ +// reference (cpp/src/msgpack_blob_*.cpp) to guarantee byte-identical behaviour. +// Several Clippy lints flag idioms that intentionally mirror the C source +// (verbatim `i + 1 <= n` bounds checks, faithful multi-argument recursive +// helpers, ternary-style `if`s); silencing them keeps the port a faithful copy. +#![allow( + clippy::too_many_arguments, + clippy::collapsible_match, + clippy::collapsible_else_if, + clippy::if_same_then_else, + clippy::int_plus_one, + clippy::implicit_saturating_sub, + clippy::manual_range_contains +)] + +mod decode; +mod encode; +mod format; +mod json; +mod mutate; + +mod blob; +mod builder; +mod iterate; +mod iterator; +mod value; + +pub use blob::Blob; +pub use builder::Builder; +pub use iterate::EachRow; +pub use iterator::Iterator; +pub use value::{type_str, IntWidth, Type, Value}; + +/// Maximum container nesting depth (matches the C++ library / SQLite extension). +pub const MAX_DEPTH: i32 = format::MAX_DEPTH; +/// Maximum output buffer size (64 MiB). +pub const MAX_OUTPUT: usize = format::MAX_OUTPUT; + +/// Crate version. +pub const VERSION: &str = "1.5.0"; diff --git a/rust/src/mutate.rs b/rust/src/mutate.rs new file mode 100644 index 0000000..5c691fd --- /dev/null +++ b/rust/src/mutate.rs @@ -0,0 +1,488 @@ +//! Internal: copy-on-write mutation (mirrors `msgpack_blob_mutate.cpp`). + +use crate::decode::{path_step, Step}; +use crate::encode as e; +use crate::format as f; + +pub const RC_OK: i32 = 0; +pub const RC_ERROR: i32 = 1; +pub const RC_NOTFOUND: i32 = 2; + +pub const EDIT_SET: i32 = 0; +pub const EDIT_INSERT: i32 = 1; +pub const EDIT_REPLACE: i32 = 2; +pub const EDIT_REMOVE: i32 = 3; +pub const EDIT_ARRAY_INS: i32 = 4; + +fn map_key(a: &[u8], n: usize, i: usize) -> Option<&[u8]> { + let kb = a[i]; + let (klen, koff) = if (0xa0..=0xbf).contains(&kb) { + ((kb & 0x1f) as usize, i + 1) + } else if kb == f::MP_STR8 && i + 2 <= n { + (a[i + 1] as usize, i + 2) + } else if kb == f::MP_STR16 && i + 3 <= n { + (f::read16(a, i + 1) as usize, i + 3) + } else if kb == f::MP_STR32 && i + 5 <= n { + (f::read32(a, i + 1) as usize, i + 5) + } else { + return None; + }; + // Bounds guard (matches decode::key_at): a truncated key length must not + // slice past the end of the blob — the C++ reference defers to skip_one, + // which fails the pair and aborts the edit gracefully. + if klen > n - koff { + return None; + } + Some(&a[koff..koff + klen]) +} + +struct StepResult { + rc: i32, + skip: bool, +} + +fn edit_map( + out: &mut Vec, + a: &[u8], + n: usize, + icur: usize, + zkey: &[u8], + zpath: &[u8], + pi: usize, + new_bin: &[u8], + mode: i32, +) -> i32 { + if icur >= n { + return RC_ERROR; + } + let b = a[icur]; + let (count, data_off) = if (0x80..=0x8f).contains(&b) { + ((b & 0x0f) as usize, icur + 1) + } else if b == f::MP_MAP16 { + if icur + 3 > n { + return RC_ERROR; + } + (f::read16(a, icur + 1) as usize, icur + 3) + } else if b == f::MP_MAP32 { + if icur + 5 > n { + return RC_ERROR; + } + (f::read32(a, icur + 1) as usize, icur + 5) + } else { + if mode == EDIT_REPLACE || mode == EDIT_REMOVE { + let iend = f::skip_one(a, n, icur); + if iend != 0 { + out.extend_from_slice(&a[icur..iend]); + } + return RC_OK; + } + return RC_ERROR; + }; + + let mut new_count = count as u32; + let mut tmp = Vec::new(); + let mut cur2 = data_off; + let mut found_key = false; + + for _ in 0..count { + if cur2 >= n { + return RC_ERROR; + } + let kstr = map_key(a, n, cur2); + let val_off = f::skip_one(a, n, cur2); + if val_off == 0 { + return RC_ERROR; + } + let pair_end = f::skip_one(a, n, val_off); + if pair_end == 0 { + return RC_ERROR; + } + + let is_match = kstr == Some(zkey); + + if is_match { + found_key = true; + if mode == EDIT_INSERT { + tmp.extend_from_slice(&a[cur2..pair_end]); + } else { + let mut vbuf = Vec::new(); + let res = edit_step(&mut vbuf, a, n, val_off, zpath, pi, new_bin, mode); + if res.rc != RC_OK { + return res.rc; + } + if res.skip { + new_count -= 1; + } else { + tmp.extend_from_slice(&a[cur2..val_off]); + tmp.extend_from_slice(&vbuf); + } + } + } else { + tmp.extend_from_slice(&a[cur2..pair_end]); + } + cur2 = pair_end; + } + + if !found_key { + if mode == EDIT_SET || mode == EDIT_INSERT { + if !matches!(path_step(zpath, pi).0, Step::End) { + let iend = f::skip_one(a, n, icur); + if iend != 0 { + out.extend_from_slice(&a[icur..iend]); + } + return RC_OK; + } + e::enc_string(&mut tmp, zkey); + tmp.extend_from_slice(new_bin); + new_count += 1; + } else { + let iend = f::skip_one(a, n, icur); + if iend != 0 { + out.extend_from_slice(&a[icur..iend]); + } + return RC_OK; + } + } + + e::enc_map_header(out, new_count); + out.extend_from_slice(&tmp); + RC_OK +} + +fn edit_array( + out: &mut Vec, + a: &[u8], + n: usize, + icur: usize, + step_idx: i64, + zpath: &[u8], + pi: usize, + new_bin: &[u8], + mode: i32, +) -> i32 { + if icur >= n { + return RC_ERROR; + } + let b = a[icur]; + let (count, data_off) = if (0x90..=0x9f).contains(&b) { + ((b & 0x0f) as usize, icur + 1) + } else if b == f::MP_ARRAY16 { + if icur + 3 > n { + return RC_ERROR; + } + (f::read16(a, icur + 1) as usize, icur + 3) + } else if b == f::MP_ARRAY32 { + if icur + 5 > n { + return RC_ERROR; + } + (f::read32(a, icur + 1) as usize, icur + 5) + } else { + if mode == EDIT_REPLACE || mode == EDIT_REMOVE { + let iend = f::skip_one(a, n, icur); + if iend != 0 { + out.extend_from_slice(&a[icur..iend]); + } + return RC_OK; + } + return RC_ERROR; + }; + + let mut new_count = count as u32; + let mut tmp = Vec::new(); + let mut cur2 = data_off; + let mut found_it = false; + + for j in 0..count { + let e_end = f::skip_one(a, n, cur2); + if e_end == 0 { + return RC_ERROR; + } + + if j as i64 == step_idx { + found_it = true; + if mode == EDIT_ARRAY_INS { + tmp.extend_from_slice(new_bin); + tmp.extend_from_slice(&a[cur2..e_end]); + new_count += 1; + } else if mode == EDIT_INSERT { + tmp.extend_from_slice(&a[cur2..e_end]); + } else { + let mut ebuf = Vec::new(); + let res = edit_step(&mut ebuf, a, n, cur2, zpath, pi, new_bin, mode); + if res.rc != RC_OK { + return res.rc; + } + if res.skip { + new_count -= 1; + } else { + tmp.extend_from_slice(&ebuf); + } + } + } else { + tmp.extend_from_slice(&a[cur2..e_end]); + } + cur2 = e_end; + } + + if !found_it { + if mode == EDIT_ARRAY_INS { + tmp.extend_from_slice(new_bin); + new_count += 1; + } else if (mode == EDIT_SET || mode == EDIT_INSERT) && step_idx == count as i64 { + tmp.extend_from_slice(new_bin); + new_count += 1; + } else if mode == EDIT_REPLACE || mode == EDIT_REMOVE { + let iend = f::skip_one(a, n, icur); + if iend != 0 { + out.extend_from_slice(&a[icur..iend]); + } + return RC_OK; + } else { + return RC_NOTFOUND; + } + } + + e::enc_array_header(out, new_count); + out.extend_from_slice(&tmp); + RC_OK +} + +fn edit_step( + out: &mut Vec, + a: &[u8], + n: usize, + icur: usize, + zpath: &[u8], + pi: usize, + new_bin: &[u8], + mode: i32, +) -> StepResult { + let (step, npi) = path_step(zpath, pi); + + match step { + Step::End => { + if mode == EDIT_REMOVE { + return StepResult { + rc: RC_OK, + skip: true, + }; + } + if mode == EDIT_ARRAY_INS { + return StepResult { + rc: RC_ERROR, + skip: false, + }; + } + if mode == EDIT_INSERT { + let iend = f::skip_one(a, n, icur); + if iend != 0 { + out.extend_from_slice(&a[icur..iend]); + } + return StepResult { + rc: RC_OK, + skip: false, + }; + } + out.extend_from_slice(new_bin); + StepResult { + rc: RC_OK, + skip: false, + } + } + Step::Error => StepResult { + rc: RC_ERROR, + skip: false, + }, + Step::Key(key) => { + let rc = edit_map(out, a, n, icur, key.as_bytes(), zpath, npi, new_bin, mode); + StepResult { rc, skip: false } + } + Step::Index(idx) => { + let rc = edit_array(out, a, n, icur, idx, zpath, npi, new_bin, mode); + StepResult { rc, skip: false } + } + } +} + +/// Apply a path-targeted edit; returns `(rc, out_bytes)`. +pub fn apply_edit(a: &[u8], n: usize, zpath: &str, new_bin: &[u8], mode: i32) -> (i32, Vec) { + let zb = zpath.as_bytes(); + if zb.is_empty() || zb[0] != b'$' { + return (RC_ERROR, Vec::new()); + } + let mut out = Vec::new(); + let res = edit_step(&mut out, a, n, 0, zb, 1, new_bin, mode); + (res.rc, out) +} + +// ── merge_patch (RFC 7386) ────────────────────────────────────────── +/// Apply an RFC 7386 merge patch; returns `(rc, out_bytes)`. +pub fn merge_patch( + a: &[u8], + n: usize, + ia: usize, + p: &[u8], + np: usize, + ip: usize, +) -> (i32, Vec) { + let mut out = Vec::new(); + let rc = merge_patch_into(&mut out, a, n, ia, p, np, ip, 0); + (rc, out) +} + +#[allow(clippy::too_many_arguments)] +fn merge_patch_into( + out: &mut Vec, + a: &[u8], + n: usize, + ia: usize, + p: &[u8], + np: usize, + ip: usize, + depth: i32, +) -> i32 { + if ip >= np { + return RC_ERROR; + } + if depth > f::MAX_DEPTH { + return RC_ERROR; + } + let pb = p[ip]; + + if pb == f::MP_NIL { + out.push(f::MP_NIL); + return RC_OK; + } + + let p_is_map = (0x80..=0x8f).contains(&pb) || pb == f::MP_MAP16 || pb == f::MP_MAP32; + if !p_is_map { + let p_end = f::skip_one(p, np, ip); + if p_end != 0 { + out.extend_from_slice(&p[ip..p_end]); + } + return RC_OK; + } + + let ab = if ia < n { a[ia] } else { 0 }; + let mut a_is_map = (0x80..=0x8f).contains(&ab) || ab == f::MP_MAP16 || ab == f::MP_MAP32; + + let (p_count, p_data_off) = if (0x80..=0x8f).contains(&pb) { + ((pb & 0x0f) as usize, ip + 1) + } else if pb == f::MP_MAP16 { + if ip + 3 > np { + return RC_ERROR; + } + (f::read16(p, ip + 1) as usize, ip + 3) + } else { + if ip + 5 > np { + return RC_ERROR; + } + (f::read32(p, ip + 1) as usize, ip + 5) + }; + + let mut a_count = 0usize; + let mut a_data_off = 0usize; + if a_is_map { + if (0x80..=0x8f).contains(&ab) { + a_count = (ab & 0x0f) as usize; + a_data_off = ia + 1; + } else if ab == f::MP_MAP16 { + if ia + 3 > n { + a_is_map = false; + } else { + a_count = f::read16(a, ia + 1) as usize; + a_data_off = ia + 3; + } + } else if ia + 5 > n { + a_is_map = false; + } else { + a_count = f::read32(a, ia + 1) as usize; + a_data_off = ia + 5; + } + } + + // Pre-scan patch keys. + if p_count > (np - p_data_off) / 2 + 1 { + return RC_ERROR; + } + // (key_off, val_off, pair_end, matched); key bytes resolved on demand. + let mut p_idx: Vec<(usize, usize, usize, bool)> = Vec::with_capacity(p_count); + let mut pc2 = p_data_off; + for _ in 0..p_count { + if pc2 >= np { + return RC_ERROR; + } + let val_off = f::skip_one(p, np, pc2); + if val_off == 0 { + return RC_ERROR; + } + let pair_end = f::skip_one(p, np, val_off); + if pair_end == 0 { + return RC_ERROR; + } + p_idx.push((pc2, val_off, pair_end, false)); + pc2 = pair_end; + } + + let mut tmp = Vec::new(); + let mut new_count: u32 = 0; + + if a_is_map { + let mut ac = a_data_off; + for _ in 0..a_count { + if ac >= n { + return RC_ERROR; + } + let kstr = map_key(a, n, ac); + let a_val_off = f::skip_one(a, n, ac); + if a_val_off == 0 { + return RC_ERROR; + } + let a_pair_end = f::skip_one(a, n, a_val_off); + if a_pair_end == 0 { + return RC_ERROR; + } + + let mut found_in_patch = false; + let mut patch_is_nil = false; + let mut p_match_val = 0usize; + for entry in p_idx.iter_mut() { + let pkey = map_key(p, np, entry.0); + if pkey.is_some() && kstr.is_some() && pkey == kstr { + found_in_patch = true; + p_match_val = entry.1; + patch_is_nil = entry.1 < np && p[entry.1] == f::MP_NIL; + entry.3 = true; + break; + } + } + + if found_in_patch && patch_is_nil { + // drop + } else if found_in_patch { + let mut mb = Vec::new(); + let mrc = merge_patch_into(&mut mb, a, n, a_val_off, p, np, p_match_val, depth + 1); + if mrc == RC_OK { + tmp.extend_from_slice(&a[ac..a_val_off]); + tmp.extend_from_slice(&mb); + new_count += 1; + } + } else { + tmp.extend_from_slice(&a[ac..a_pair_end]); + new_count += 1; + } + ac = a_pair_end; + } + } + + for entry in p_idx.iter() { + if !entry.3 && entry.1 < np && p[entry.1] != f::MP_NIL { + tmp.extend_from_slice(&p[entry.0..entry.2]); + new_count += 1; + } + } + + e::enc_map_header(out, new_count); + out.extend_from_slice(&tmp); + RC_OK +} diff --git a/rust/src/value.rs b/rust/src/value.rs new file mode 100644 index 0000000..9c17708 --- /dev/null +++ b/rust/src/value.rs @@ -0,0 +1,282 @@ +//! `Value` — a decoded scalar or sub-blob MessagePack value. +//! +//! Mirrors `msgpack::Value` from the C++ Blob library. Integer values are kept +//! as raw 64-bit bits so the full signed/unsigned range round-trips exactly. + +/// Semantic type of a MessagePack element. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Type { + Nil, + True, + False, + Integer, + Real, + Float32, + String, + Binary, + Array, + Map, + Ext, + Timestamp, +} + +/// Integer encoding-width hint (forces a specific wire format). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum IntWidth { + Auto, + Int8, + Int16, + Int32, + Int64, + Uint8, + Uint16, + Uint32, + Uint64, +} + +/// Human-readable label for a [`Type`] (`"text"`, `"integer"`, …). +pub fn type_str(t: Type) -> &'static str { + match t { + Type::Nil => "null", + Type::True => "true", + Type::False => "false", + Type::Integer => "integer", + Type::Real => "real", + Type::Float32 => "float32", + Type::String => "text", + Type::Binary => "binary", + Type::Array => "array", + Type::Map => "map", + Type::Ext => "ext", + Type::Timestamp => "timestamp", + } +} + +/// A decoded scalar or sub-blob value. +#[derive(Clone, Debug)] +pub struct Value { + pub(crate) ty: Type, + pub(crate) bits: u64, // integer bits (signed or unsigned) / timestamp seconds + pub(crate) float: f64, // real / float32 payload + pub(crate) bytes: Vec, // string / binary / ext payload (no header) + pub(crate) ext_type: i8, + pub(crate) ts_nsec: u32, + pub(crate) int_width: IntWidth, +} + +impl Default for Value { + fn default() -> Self { + Value { + ty: Type::Nil, + bits: 0, + float: 0.0, + bytes: Vec::new(), + ext_type: 0, + ts_nsec: 0, + int_width: IntWidth::Auto, + } + } +} + +impl Value { + // ── accessors ───────────────────────────────────────────────────── + pub fn get_type(&self) -> Type { + self.ty + } + pub fn is_nil(&self) -> bool { + self.ty == Type::Nil + } + pub fn as_bool(&self) -> bool { + self.ty == Type::True + } + pub fn as_i64(&self) -> i64 { + match self.ty { + Type::Integer => self.bits as i64, + Type::Real | Type::Float32 => self.float as i64, + Type::Timestamp => self.bits as i64, + Type::True => 1, + _ => 0, + } + } + pub fn as_u64(&self) -> u64 { + if self.ty == Type::Integer { + self.bits + } else { + 0 + } + } + pub fn as_f64(&self) -> f64 { + match self.ty { + Type::Real | Type::Float32 => self.float, + Type::Integer => self.as_i64() as f64, + _ => 0.0, + } + } + pub fn as_f32(&self) -> f32 { + match self.ty { + Type::Float32 => self.float as f32, + Type::Real => self.float as f32, + _ => 0.0, + } + } + /// String payload bytes (verbatim; may not be valid UTF-8). + pub fn as_bytes(&self) -> &[u8] { + if self.ty == Type::String { + &self.bytes + } else { + &[] + } + } + /// String payload decoded lossily as UTF-8 (exact for valid UTF-8). + pub fn as_string(&self) -> String { + if self.ty == Type::String { + String::from_utf8_lossy(&self.bytes).into_owned() + } else { + String::new() + } + } + /// Binary / Ext payload (no header), or raw bytes for container values. + pub fn blob_data(&self) -> &[u8] { + &self.bytes + } + pub fn blob_size(&self) -> usize { + self.bytes.len() + } + pub fn ext_type(&self) -> i8 { + self.ext_type + } + pub fn timestamp_seconds(&self) -> i64 { + if self.ty == Type::Timestamp { + self.bits as i64 + } else { + 0 + } + } + pub fn timestamp_nanoseconds(&self) -> u32 { + if self.ty == Type::Timestamp { + self.ts_nsec + } else { + 0 + } + } + pub fn int_width(&self) -> IntWidth { + self.int_width + } + + // ── static constructors ─────────────────────────────────────────── + pub fn nil() -> Value { + Value::default() + } + pub fn boolean(b: bool) -> Value { + Value { + ty: if b { Type::True } else { Type::False }, + ..Default::default() + } + } + pub fn integer(x: i64) -> Value { + Value { + ty: Type::Integer, + bits: x as u64, + ..Default::default() + } + } + pub fn unsigned_integer(x: u64) -> Value { + let mut v = Value { + ty: Type::Integer, + bits: x, + ..Default::default() + }; + if x > i64::MAX as u64 { + v.int_width = IntWidth::Uint64; + } + v + } + pub fn real(d: f64) -> Value { + Value { + ty: Type::Real, + float: d, + ..Default::default() + } + } + pub fn real32(f: f32) -> Value { + Value { + ty: Type::Float32, + float: f as f64, + ..Default::default() + } + } + pub fn string(s: &str) -> Value { + Value::string_bytes(s.as_bytes()) + } + pub fn string_bytes(b: &[u8]) -> Value { + Value { + ty: Type::String, + bytes: b.to_vec(), + ..Default::default() + } + } + pub fn binary(data: &[u8]) -> Value { + Value { + ty: Type::Binary, + bytes: data.to_vec(), + ..Default::default() + } + } + pub fn ext(type_code: i8, data: &[u8]) -> Value { + Value { + ty: Type::Ext, + ext_type: type_code, + bytes: data.to_vec(), + ..Default::default() + } + } + pub fn timestamp(seconds: i64) -> Value { + Value { + ty: Type::Timestamp, + bits: seconds as u64, + ts_nsec: 0, + ..Default::default() + } + } + pub fn timestamp_ns(seconds: i64, nanoseconds: u32) -> Value { + Value { + ty: Type::Timestamp, + bits: seconds as u64, + ts_nsec: nanoseconds, + ..Default::default() + } + } + + fn fixed(width: IntWidth, bits: u64) -> Value { + Value { + ty: Type::Integer, + bits, + int_width: width, + ..Default::default() + } + } + pub fn int8(x: i8) -> Value { + Value::fixed(IntWidth::Int8, x as i64 as u64) + } + pub fn int16(x: i16) -> Value { + Value::fixed(IntWidth::Int16, x as i64 as u64) + } + pub fn int32(x: i32) -> Value { + Value::fixed(IntWidth::Int32, x as i64 as u64) + } + pub fn int64(x: i64) -> Value { + Value::fixed(IntWidth::Int64, x as u64) + } + pub fn uint8(x: u8) -> Value { + Value::fixed(IntWidth::Uint8, x as u64) + } + pub fn uint16(x: u16) -> Value { + Value::fixed(IntWidth::Uint16, x as u64) + } + pub fn uint32(x: u32) -> Value { + Value::fixed(IntWidth::Uint32, x as u64) + } + pub fn uint64(x: u64) -> Value { + Value::fixed(IntWidth::Uint64, x) + } +} diff --git a/rust/tests/api.rs b/rust/tests/api.rs new file mode 100644 index 0000000..8ca727b --- /dev/null +++ b/rust/tests/api.rs @@ -0,0 +1,214 @@ +//! API behaviour and round-trip tests for the Rust port. + +use msgpack_blob::{type_str, Blob, Builder, Iterator, Type, Value}; + +#[test] +fn builder_matches_from_json() { + let mut b = Builder::new(); + b.map_header(3) + .string("name") + .string("Alice") + .string("age") + .integer(30) + .string("scores") + .array_header(3) + .real(95.5) + .real(87.5) + .real(91.0); + let built = b.build(); + let reference = Blob::from_json(r#"{"name":"Alice","age":30,"scores":[95.5,87.5,91.0]}"#); + assert_eq!(built.hex(), reference.hex()); +} + +#[test] +fn quote_roundtrips_type() { + let values = [ + Value::nil(), + Value::boolean(true), + Value::integer(-12345), + Value::real(3.25), + Value::real32(1.5), + Value::string("hello"), + Value::binary(&[0xde, 0xad]), + Value::ext(7, &[1, 2]), + Value::timestamp_ns(1_700_000_000, 123_456_789), + ]; + for v in &values { + let blob = Builder::quote(v); + assert!(blob.valid()); + assert_eq!(blob.extract("$").get_type(), v.get_type()); + } +} + +const ROUND_TRIP: &[&str] = &[ + "null", + "true", + "false", + "0", + "-1", + "127", + "128", + "65536", + "1.5", + "0.1", + "1e10", + "\"hi\"", + "[]", + "{}", + "[1,2,3]", + r#"{"a":1,"b":[2,3],"c":{"d":true}}"#, + r#"{"u":"caf\u00e9","emoji":"\ud83d\ude00"}"#, +]; + +#[test] +fn json_bytes_stable() { + for case in ROUND_TRIP { + let once = Blob::from_json(case); + let twice = Blob::from_json(&once.to_json()); + assert_eq!(once.hex(), twice.hex(), "{}", case); + } +} + +#[test] +fn integers_64bit_roundtrip() { + let blob = Builder::quote(&Value::uint64(u64::MAX)); + assert_eq!(blob.hex(), "cfffffffffffffffff"); + assert_eq!(blob.extract("$").as_u64(), u64::MAX); + assert_eq!(blob.to_json(), "18446744073709551615"); + + let neg = Builder::quote(&Value::int64(i64::MIN)); + assert_eq!(neg.extract("$").as_i64(), i64::MIN); +} + +#[test] +fn extraction() { + let blob = Blob::from_json( + r#"{"name":"Alice","age":30,"tall":true,"pets":["cat","dog"],"addr":{"city":"NYC"}}"#, + ); + assert_eq!(blob.extract("$.name").as_string(), "Alice"); + assert_eq!(blob.extract("$.age").as_i64(), 30); + assert!(blob.extract("$.tall").as_bool()); + assert_eq!(blob.extract("$.pets[1]").as_string(), "dog"); + assert_eq!(blob.extract("$.addr.city").as_string(), "NYC"); + assert!(blob.extract("$.nope").is_nil()); + assert_eq!(blob.type_str_at("$.pets"), "array"); + assert_eq!(blob.array_length_at("$.pets"), 2); + assert_eq!(blob.array_length_at("$.name"), -1); +} + +#[test] +fn binary_ext_timestamp() { + let bin = Builder::new().binary(&[1, 2, 3, 4]).build(); + assert_eq!(bin.extract("$").get_type(), Type::Binary); + assert_eq!(bin.extract("$").blob_data(), &[1, 2, 3, 4]); + assert_eq!(bin.to_json(), "\"01020304\""); + + let ext = Builder::new().ext(42, &[0xaa, 0xbb]).build(); + assert_eq!(ext.extract("$").ext_type(), 42); + + let ts = Builder::new() + .timestamp_ns(1_700_000_000, 500_000_000) + .build(); + assert_eq!(ts.extract("$").timestamp_seconds(), 1_700_000_000); + assert_eq!(ts.extract("$").timestamp_nanoseconds(), 500_000_000); +} + +#[test] +fn copy_on_write_mutation() { + let orig = Blob::from_json(r#"{"a":1}"#); + let updated = orig.set("$.b", &Value::integer(2)); + assert_eq!(orig.to_json(), r#"{"a":1}"#); + assert_eq!(updated.to_json(), r#"{"a":1,"b":2}"#); + + let b = Blob::from_json(r#"{"a":1,"b":2,"c":3}"#); + assert_eq!(b.remove("$.b").to_json(), r#"{"a":1,"c":3}"#); + assert_eq!( + b.patch(&Blob::from_json(r#"{"b":null,"d":4}"#)).to_json(), + r#"{"a":1,"c":3,"d":4}"# + ); + + let arr = Blob::from_json("[1,2,3]"); + assert_eq!( + arr.array_insert("$[1]", &Value::integer(9)).to_json(), + "[1,9,2,3]" + ); + assert_eq!(arr.set("$[3]", &Value::integer(4)).to_json(), "[1,2,3,4]"); +} + +#[test] +fn iterator_each_and_tree() { + let map = Blob::from_json(r#"{"a":1,"b":2,"c":3}"#); + let keys: Vec = Iterator::new(&map, "$", false) + .rows() + .iter() + .map(|r| r.key.clone()) + .collect(); + assert_eq!(keys, vec!["a", "b", "c"]); + + let nested = Blob::from_json(r#"{"x":{"y":[1,2]}}"#); + let fullkeys: Vec = Iterator::new(&nested, "$", true) + .into_iter() + .map(|r| r.fullkey) + .collect(); + assert_eq!(fullkeys, vec!["$", "$.x", "$.x.y", "$.x.y[0]", "$.x.y[1]"]); + + let arr = Blob::from_json("[1,2]"); + let mut it = Iterator::new(&arr, "$", false); + let mut seen = Vec::new(); + while it.next() { + seen.push(it.current().value.as_i64()); + } + assert_eq!(seen, vec![1, 2]); +} + +#[test] +fn validity() { + assert!(Blob::from_json("[1,2,3]").valid()); + assert!(!Blob::new(&[]).valid()); + assert!(!Blob::new(&[0x91]).valid()); +} + +#[test] +fn type_str_labels() { + assert_eq!(type_str(Type::Nil), "null"); + assert_eq!(type_str(Type::String), "text"); + assert_eq!(type_str(Type::Float32), "float32"); + assert_eq!(type_str(Type::Timestamp), "timestamp"); +} + +#[test] +fn truncated_map_key_does_not_panic() { + // One-entry map whose str8 key declares length 10 but supplies only 2 bytes. + // The C++ reference bails via skip_one; the port must not panic. + let blob = Blob::new(&[0x81, 0xd9, 0x0a, 0x61, 0x62]); + // Mutation returns the original blob unchanged. + assert_eq!(blob.set("$.x", &Value::integer(1)).data(), blob.data()); + assert_eq!(blob.remove("$.x").data(), blob.data()); + assert_eq!( + blob.patch(&Blob::from_json(r#"{"a":1}"#)).data(), + blob.data() + ); + // Flat and recursive iteration yield no usable rows (no panic). + assert_eq!(Iterator::new(&blob, "$", false).rows().len(), 0); + let _ = Iterator::new(&blob, "$", true).rows(); + // Truncated fixstr key as well. + let fix = Blob::new(&[0x81, 0xa5, 0x68, 0x69]); + assert_eq!(fix.set("$.x", &Value::integer(1)).data(), fix.data()); + assert_eq!(Iterator::new(&fix, "$", false).rows().len(), 0); +} + +#[test] +fn non_utf8_string_bytes_preserved() { + // {"k": <0xff 0x80 0xfe 0xc0>} — a str with non-UTF-8 payload, as a foreign + // encoder (C++/SQLite) may produce. C++ passes raw bytes through verbatim. + let blob = Blob::new(&[0x81, 0xa1, 0x6b, 0xa4, 0xff, 0x80, 0xfe, 0xc0]); + assert_eq!( + blob.to_json_bytes(), + vec![0x7b, 0x22, 0x6b, 0x22, 0x3a, 0x22, 0xff, 0x80, 0xfe, 0xc0, 0x22, 0x7d] + ); + let v = blob.extract("$.k"); + assert_eq!(v.as_bytes(), &[0xff, 0x80, 0xfe, 0xc0]); + // Value::string_bytes round-trips arbitrary bytes back to identical output. + let rebuilt = Builder::new().string_bytes(v.as_bytes()).build(); + assert_eq!(rebuilt.hex(), "a4ff80fec0"); +} diff --git a/rust/tests/common/mod.rs b/rust/tests/common/mod.rs new file mode 100644 index 0000000..856effc --- /dev/null +++ b/rust/tests/common/mod.rs @@ -0,0 +1,312 @@ +//! Shared test support: a tiny std-only JSON reader (so the crate stays +//! dependency-free) plus helpers to replay the cross-language vectors. + +#![allow(dead_code)] + +use std::collections::BTreeMap; + +use msgpack_blob::Value; + +/// A minimal JSON value. +#[derive(Clone, Debug)] +pub enum Json { + Null, + Bool(bool), + Num(f64), + Str(String), + Arr(Vec), + Obj(BTreeMap), +} + +impl Json { + pub fn as_str(&self) -> &str { + match self { + Json::Str(s) => s, + _ => panic!("expected string, got {:?}", self), + } + } + pub fn as_f64(&self) -> f64 { + match self { + Json::Num(n) => *n, + _ => panic!("expected number, got {:?}", self), + } + } + pub fn as_i64(&self) -> i64 { + self.as_f64() as i64 + } + pub fn as_bool(&self) -> bool { + match self { + Json::Bool(b) => *b, + _ => panic!("expected bool, got {:?}", self), + } + } + pub fn as_arr(&self) -> &[Json] { + match self { + Json::Arr(a) => a, + _ => panic!("expected array, got {:?}", self), + } + } + pub fn get(&self, key: &str) -> &Json { + match self { + Json::Obj(m) => m.get(key).unwrap_or_else(|| panic!("missing key {}", key)), + _ => panic!("expected object, got {:?}", self), + } + } + pub fn has(&self, key: &str) -> bool { + matches!(self, Json::Obj(m) if m.contains_key(key)) + } +} + +struct Parser<'a> { + b: &'a [u8], + i: usize, +} + +impl<'a> Parser<'a> { + fn ws(&mut self) { + while self.i < self.b.len() && matches!(self.b[self.i], b' ' | b'\t' | b'\n' | b'\r') { + self.i += 1; + } + } + fn value(&mut self) -> Json { + self.ws(); + match self.b[self.i] { + b'{' => self.object(), + b'[' => self.array(), + b'"' => Json::Str(self.string()), + b't' => { + self.i += 4; + Json::Bool(true) + } + b'f' => { + self.i += 5; + Json::Bool(false) + } + b'n' => { + self.i += 4; + Json::Null + } + _ => self.number(), + } + } + fn object(&mut self) -> Json { + let mut m = BTreeMap::new(); + self.i += 1; // { + self.ws(); + if self.b[self.i] == b'}' { + self.i += 1; + return Json::Obj(m); + } + loop { + self.ws(); + let key = self.string(); + self.ws(); + assert_eq!(self.b[self.i], b':'); + self.i += 1; + let val = self.value(); + m.insert(key, val); + self.ws(); + match self.b[self.i] { + b',' => { + self.i += 1; + } + b'}' => { + self.i += 1; + break; + } + c => panic!("unexpected {} in object", c as char), + } + } + Json::Obj(m) + } + fn array(&mut self) -> Json { + let mut v = Vec::new(); + self.i += 1; // [ + self.ws(); + if self.b[self.i] == b']' { + self.i += 1; + return Json::Arr(v); + } + loop { + v.push(self.value()); + self.ws(); + match self.b[self.i] { + b',' => { + self.i += 1; + } + b']' => { + self.i += 1; + break; + } + c => panic!("unexpected {} in array", c as char), + } + } + Json::Arr(v) + } + fn string(&mut self) -> String { + assert_eq!(self.b[self.i], b'"'); + self.i += 1; + let mut out: Vec = Vec::new(); + while self.b[self.i] != b'"' { + let c = self.b[self.i]; + if c == b'\\' { + self.i += 1; + let esc = self.b[self.i]; + self.i += 1; + match esc { + b'"' => out.push(b'"'), + b'\\' => out.push(b'\\'), + b'/' => out.push(b'/'), + b'n' => out.push(b'\n'), + b'r' => out.push(b'\r'), + b't' => out.push(b'\t'), + b'b' => out.push(0x08), + b'f' => out.push(0x0c), + b'u' => { + let mut cp = hex4(self.b, self.i); + self.i += 4; + if (0xd800..=0xdbff).contains(&cp) + && self.b[self.i] == b'\\' + && self.b[self.i + 1] == b'u' + { + let lo = hex4(self.b, self.i + 2); + if (0xdc00..=0xdfff).contains(&lo) { + self.i += 6; + cp = 0x10000 + ((cp - 0xd800) << 10) + (lo - 0xdc00); + } + } + push_utf8(&mut out, cp as u32); + } + other => out.push(other), + } + } else { + out.push(c); + self.i += 1; + } + } + self.i += 1; // closing " + String::from_utf8(out).expect("vectors strings are valid UTF-8") + } + fn number(&mut self) -> Json { + let start = self.i; + while self.i < self.b.len() + && matches!( + self.b[self.i], + b'0'..=b'9' | b'-' | b'+' | b'.' | b'e' | b'E' + ) + { + self.i += 1; + } + let s = std::str::from_utf8(&self.b[start..self.i]).unwrap(); + Json::Num(s.parse().unwrap()) + } +} + +fn hex4(b: &[u8], off: usize) -> i32 { + let mut v = 0; + for j in 0..4 { + let c = b[off + j]; + let h = match c { + b'0'..=b'9' => (c - b'0') as i32, + b'a'..=b'f' => (c - b'a' + 10) as i32, + b'A'..=b'F' => (c - b'A' + 10) as i32, + _ => 0, + }; + v = (v << 4) | h; + } + v +} + +fn push_utf8(out: &mut Vec, cp: u32) { + if cp < 0x80 { + out.push(cp as u8); + } else if cp < 0x800 { + out.push(0xc0 | (cp >> 6) as u8); + out.push(0x80 | (cp & 0x3f) as u8); + } else if cp < 0x10000 { + out.push(0xe0 | (cp >> 12) as u8); + out.push(0x80 | ((cp >> 6) & 0x3f) as u8); + out.push(0x80 | (cp & 0x3f) as u8); + } else { + out.push(0xf0 | (cp >> 18) as u8); + out.push(0x80 | ((cp >> 12) & 0x3f) as u8); + out.push(0x80 | ((cp >> 6) & 0x3f) as u8); + out.push(0x80 | (cp & 0x3f) as u8); + } +} + +pub fn parse_json(text: &str) -> Json { + let mut p = Parser { + b: text.as_bytes(), + i: 0, + }; + p.value() +} + +/// Load and parse the shared cross-language vector file. +pub fn load_vectors() -> Json { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../tests/vectors/blob_vectors.json" + ); + let text = std::fs::read_to_string(path).expect("read blob_vectors.json"); + parse_json(&text) +} + +pub fn hex_to_bytes(h: &str) -> Vec { + let bytes = h.as_bytes(); + let mut out = Vec::with_capacity(bytes.len() / 2); + let nib = |c: u8| -> u8 { + match c { + b'0'..=b'9' => c - b'0', + b'a'..=b'f' => c - b'a' + 10, + b'A'..=b'F' => c - b'A' + 10, + _ => 0, + } + }; + let mut i = 0; + while i + 1 < bytes.len() { + out.push((nib(bytes[i]) << 4) | nib(bytes[i + 1])); + i += 2; + } + out +} + +pub fn bytes_to_hex(b: &[u8]) -> String { + let mut s = String::with_capacity(b.len() * 2); + for &x in b { + s.push_str(&format!("{:02x}", x)); + } + s +} + +/// Build a `Value` from a ValueSpec object (see `cpp/tests/gen_blob_vectors.cpp`). +pub fn build_value(spec: &Json) -> Value { + match spec.get("k").as_str() { + "nil" => Value::nil(), + "bool" => Value::boolean(spec.get("v").as_bool()), + "int" => Value::integer(spec.get("v").as_str().parse().unwrap()), + "uint" => Value::unsigned_integer(spec.get("v").as_str().parse().unwrap()), + "int8" => Value::int8(spec.get("v").as_str().parse().unwrap()), + "int16" => Value::int16(spec.get("v").as_str().parse().unwrap()), + "int32" => Value::int32(spec.get("v").as_str().parse().unwrap()), + "int64" => Value::int64(spec.get("v").as_str().parse().unwrap()), + "uint8" => Value::uint8(spec.get("v").as_str().parse().unwrap()), + "uint16" => Value::uint16(spec.get("v").as_str().parse().unwrap()), + "uint32" => Value::uint32(spec.get("v").as_str().parse().unwrap()), + "uint64" => Value::uint64(spec.get("v").as_str().parse().unwrap()), + "real" => Value::real(spec.get("v").as_f64()), + "real32" => Value::real32(spec.get("v").as_f64() as f32), + "str" => Value::string(spec.get("v").as_str()), + "binary" => Value::binary(&hex_to_bytes(spec.get("hex").as_str())), + "ext" => Value::ext( + spec.get("type").as_i64() as i8, + &hex_to_bytes(spec.get("hex").as_str()), + ), + "timestamp" => Value::timestamp_ns( + spec.get("sec").as_str().parse().unwrap(), + spec.get("nsec").as_i64() as u32, + ), + other => panic!("unknown spec kind {}", other), + } +} diff --git a/rust/tests/vectors.rs b/rust/tests/vectors.rs new file mode 100644 index 0000000..72ed702 --- /dev/null +++ b/rust/tests/vectors.rs @@ -0,0 +1,168 @@ +//! Replay the shared cross-language vectors (tests/vectors/blob_vectors.json). +//! Generated from the C++ reference implementation, so passing them proves the +//! Rust port is byte-identical. + +mod common; + +use common::{build_value, bytes_to_hex, hex_to_bytes, load_vectors}; +use msgpack_blob::{Blob, Builder, Iterator}; + +#[test] +fn from_json_byte_identical() { + let v = load_vectors(); + for case in v.get("from_json").as_arr() { + let blob = Blob::from_json(case.get("json").as_str()); + assert_eq!( + blob.hex(), + case.get("hex").as_str(), + "json={}", + case.get("json").as_str() + ); + } +} + +#[test] +fn to_json_matches() { + let v = load_vectors(); + for case in v.get("to_json").as_arr() { + let blob = Blob::new(&hex_to_bytes(case.get("hex").as_str())); + assert_eq!( + blob.to_json(), + case.get("json").as_str(), + "hex={}", + case.get("hex").as_str() + ); + } +} + +#[test] +fn to_json_pretty_matches() { + let v = load_vectors(); + for case in v.get("to_json_pretty").as_arr() { + let blob = Blob::new(&hex_to_bytes(case.get("hex").as_str())); + let indent = case.get("indent").as_i64() as i32; + assert_eq!(blob.to_json_pretty(indent), case.get("json").as_str()); + } +} + +#[test] +fn typed_quote_matches() { + let v = load_vectors(); + for case in v.get("typed").as_arr() { + let value = build_value(case.get("spec")); + let blob = Builder::quote(&value); + assert_eq!( + blob.hex(), + case.get("hex").as_str(), + "spec={:?}", + case.get("spec") + ); + } +} + +#[test] +fn mutate_matches() { + let v = load_vectors(); + for case in v.get("mutate").as_arr() { + let base = Blob::from_json(case.get("base").as_str()); + let op = case.get("op").as_str(); + let result = match op { + "set" => base.set(case.get("path").as_str(), &build_value(case.get("spec"))), + "insert" => base.insert(case.get("path").as_str(), &build_value(case.get("spec"))), + "replace" => base.replace(case.get("path").as_str(), &build_value(case.get("spec"))), + "array_insert" => { + base.array_insert(case.get("path").as_str(), &build_value(case.get("spec"))) + } + "remove" => base.remove(case.get("path").as_str()), + "set_blob" => base.set_blob( + case.get("path").as_str(), + &Blob::from_json(case.get("spec").get("json").as_str()), + ), + "patch" => base.patch(&Blob::from_json(case.get("patch").as_str())), + other => panic!("unknown op {}", other), + }; + assert_eq!( + result.hex(), + case.get("hex").as_str(), + "op={} base={}", + op, + case.get("base").as_str() + ); + } +} + +#[test] +fn extract_matches() { + let v = load_vectors(); + for case in v.get("extract").as_arr() { + let blob = Blob::from_json(case.get("base").as_str()); + let path = case.get("path").as_str(); + assert_eq!( + blob.type_str_at(path), + case.get("type").as_str(), + "type {} {}", + case.get("base").as_str(), + path + ); + let value = blob.extract(path); + assert_eq!(Builder::quote(&value).to_json(), case.get("vjson").as_str()); + } +} + +#[test] +fn array_length_matches() { + let v = load_vectors(); + for case in v.get("array_length").as_arr() { + let blob = Blob::from_json(case.get("base").as_str()); + let path = case.get("path").as_str(); + let got = if path == "$" { + blob.array_length() + } else { + blob.array_length_at(path) + }; + assert_eq!( + got, + case.get("len").as_i64(), + "{} {}", + case.get("base").as_str(), + path + ); + } +} + +#[test] +fn iterate_matches() { + let v = load_vectors(); + for case in v.get("iterate").as_arr() { + let blob = Blob::from_json(case.get("base").as_str()); + let rows = Iterator::new( + &blob, + case.get("path").as_str(), + case.get("recursive").as_bool(), + ) + .rows(); + let expected = case.get("rows").as_arr(); + assert_eq!( + rows.len(), + expected.len(), + "{} {}", + case.get("base").as_str(), + case.get("path").as_str() + ); + for (got, exp) in rows.iter().zip(expected.iter()) { + assert_eq!(got.fullkey, exp.get("fullkey").as_str()); + assert_eq!(got.path, exp.get("path").as_str()); + assert_eq!(got.id as i64, exp.get("id").as_i64()); + assert_eq!(msgpack_blob::type_str(got.ty), exp.get("type").as_str()); + if exp.has("key") { + assert_eq!(got.key, exp.get("key").as_str()); + assert_eq!(got.index, exp.get("index").as_i64()); + } + } + } +} + +#[test] +fn vector_helpers_roundtrip() { + assert_eq!(bytes_to_hex(&hex_to_bytes("deadbeef")), "deadbeef"); +} diff --git a/src/msgpack_blob.cpp b/src/msgpack_blob.cpp deleted file mode 100644 index b58dad0..0000000 --- a/src/msgpack_blob.cpp +++ /dev/null @@ -1,2250 +0,0 @@ -/* -** msgpack_blob.cpp — Standalone C++ MsgPack Blob API -** -** Ports the core internal functions from the sqlite-msgpack extension -** (msgpack.c) into a self-contained C++ implementation with zero -** SQLite dependency. Produces byte-identical msgpack encoding. -*/ - -#include "msgpack_blob.hpp" - -#include -#include -#include -#include -#include -#include -#include - -namespace msgpack { - -/* ── MessagePack format constants (same as msgpack.c) ─────────────── */ - -static constexpr uint8_t MP_NIL = 0xc0; -static constexpr uint8_t MP_FALSE = 0xc2; -static constexpr uint8_t MP_TRUE = 0xc3; -static constexpr uint8_t MP_BIN8 = 0xc4; -static constexpr uint8_t MP_BIN16 = 0xc5; -static constexpr uint8_t MP_BIN32 = 0xc6; -static constexpr uint8_t MP_EXT8 = 0xc7; -static constexpr uint8_t MP_EXT16 = 0xc8; -static constexpr uint8_t MP_EXT32 = 0xc9; -static constexpr uint8_t MP_FLOAT32 = 0xca; -static constexpr uint8_t MP_FLOAT64 = 0xcb; -static constexpr uint8_t MP_UINT8 = 0xcc; -static constexpr uint8_t MP_UINT16 = 0xcd; -static constexpr uint8_t MP_UINT32 = 0xce; -static constexpr uint8_t MP_UINT64 = 0xcf; -static constexpr uint8_t MP_INT8 = 0xd0; -static constexpr uint8_t MP_INT16 = 0xd1; -static constexpr uint8_t MP_INT32 = 0xd2; -static constexpr uint8_t MP_INT64 = 0xd3; -static constexpr uint8_t MP_FIXEXT1 = 0xd4; -static constexpr uint8_t MP_FIXEXT2 = 0xd5; -static constexpr uint8_t MP_FIXEXT4 = 0xd6; -static constexpr uint8_t MP_FIXEXT8 = 0xd7; -static constexpr uint8_t MP_FIXEXT16 = 0xd8; -static constexpr uint8_t MP_STR8 = 0xd9; -static constexpr uint8_t MP_STR16 = 0xda; -static constexpr uint8_t MP_STR32 = 0xdb; -static constexpr uint8_t MP_ARRAY16 = 0xdc; -static constexpr uint8_t MP_ARRAY32 = 0xdd; -static constexpr uint8_t MP_MAP16 = 0xde; -static constexpr uint8_t MP_MAP32 = 0xdf; - -static constexpr uint8_t MP_FIXMAP_MASK = 0x80; -static constexpr uint8_t MP_FIXARRAY_MASK = 0x90; -static constexpr uint8_t MP_FIXSTR_MASK = 0xa0; - -/* Edit modes */ -static constexpr int EDIT_SET = 0; -static constexpr int EDIT_INSERT = 1; -static constexpr int EDIT_REPLACE = 2; -static constexpr int EDIT_REMOVE = 3; -static constexpr int EDIT_ARRAY_INS = 4; - -/* Result codes (internal, not exposed) */ -static constexpr int RC_OK = 0; -static constexpr int RC_ERROR = 1; -static constexpr int RC_NOTFOUND = 2; - -/* ── Big-endian byte-order helpers ────────────────────────────────── */ - -static inline uint16_t read16(const uint8_t* p) { - return static_cast((static_cast(p[0]) << 8) | p[1]); -} -static inline uint32_t read32(const uint8_t* p) { - return (static_cast(p[0]) << 24) | - (static_cast(p[1]) << 16) | - (static_cast(p[2]) << 8) | p[3]; -} -static inline uint64_t read64(const uint8_t* p) { - return (static_cast(read32(p)) << 32) | read32(p + 4); -} -static inline void write16(uint8_t* p, uint16_t v) { - p[0] = static_cast(v >> 8); - p[1] = static_cast(v); -} -static inline void write32(uint8_t* p, uint32_t v) { - p[0] = static_cast(v >> 24); - p[1] = static_cast(v >> 16); - p[2] = static_cast(v >> 8); - p[3] = static_cast(v); -} -static inline void write64(uint8_t* p, uint64_t v) { - write32(p, static_cast(v >> 32)); - write32(p + 4, static_cast(v)); -} - -/* ── Buf — growable output buffer ─────────────────────────────────── */ - -class Buf { -public: - std::vector data; - - void append(const uint8_t* p, size_t n) { - data.insert(data.end(), p, p + n); - } - void append1(uint8_t b) { - data.push_back(b); - } - uint8_t* reserve(size_t n) { - size_t old = data.size(); - data.resize(old + n); - return data.data() + old; - } - void clear() { data.clear(); } - size_t size() const { return data.size(); } - const uint8_t* ptr() const { return data.data(); } -}; - -/* ── skip_one — skip one complete msgpack element ─────────────────── */ - -static uint32_t skip_one_d(const uint8_t* a, uint32_t n, uint32_t i, int depth); -static uint32_t skip_one(const uint8_t* a, uint32_t n, uint32_t i) { - return skip_one_d(a, n, i, 0); -} -static uint32_t skip_one_d(const uint8_t* a, uint32_t n, uint32_t i, int depth) { - if (depth > kMaxDepth) return 0; - if (i >= n) return 0; - uint8_t b = a[i++]; - - if (b <= 0x7f) return i; /* positive fixint */ - if (b >= 0xe0) return i; /* negative fixint */ - - switch (b) { - case MP_NIL: case MP_FALSE: case MP_TRUE: - return i; - case MP_FLOAT32: - return (i + 4 <= n) ? i + 4 : 0; - case MP_FLOAT64: case MP_INT64: case MP_UINT64: - return (i + 8 <= n) ? i + 8 : 0; - case MP_UINT8: case MP_INT8: - return (i + 1 <= n) ? i + 1 : 0; - case MP_UINT16: case MP_INT16: - return (i + 2 <= n) ? i + 2 : 0; - case MP_UINT32: case MP_INT32: - return (i + 4 <= n) ? i + 4 : 0; - case MP_BIN8: { - if (i + 1 > n) return 0; - uint32_t sz = a[i]; i++; - return (sz <= n - i) ? i + sz : 0; - } - case MP_BIN16: { - if (i + 2 > n) return 0; - uint32_t sz = read16(a + i); i += 2; - return (sz <= n - i) ? i + sz : 0; - } - case MP_BIN32: { - if (i + 4 > n) return 0; - uint32_t sz = read32(a + i); i += 4; - return (sz <= n - i) ? i + sz : 0; - } - case MP_STR8: { - if (i + 1 > n) return 0; - uint32_t sz = a[i]; i++; - return (sz <= n - i) ? i + sz : 0; - } - case MP_STR16: { - if (i + 2 > n) return 0; - uint32_t sz = read16(a + i); i += 2; - return (sz <= n - i) ? i + sz : 0; - } - case MP_STR32: { - if (i + 4 > n) return 0; - uint32_t sz = read32(a + i); i += 4; - return (sz <= n - i) ? i + sz : 0; - } - case MP_FIXEXT1: return (i + 2 <= n) ? i + 2 : 0; - case MP_FIXEXT2: return (i + 3 <= n) ? i + 3 : 0; - case MP_FIXEXT4: return (i + 5 <= n) ? i + 5 : 0; - case MP_FIXEXT8: return (i + 9 <= n) ? i + 9 : 0; - case MP_FIXEXT16: return (i + 17 <= n) ? i + 17 : 0; - case MP_EXT8: { - if (i + 2 > n) return 0; - uint32_t sz = a[i]; i += 2; - return (sz <= n - i) ? i + sz : 0; - } - case MP_EXT16: { - if (i + 3 > n) return 0; - uint32_t sz = read16(a + i); i += 3; - return (sz <= n - i) ? i + sz : 0; - } - case MP_EXT32: { - if (i + 5 > n) return 0; - uint32_t sz = read32(a + i); i += 5; - return (sz <= n - i) ? i + sz : 0; - } - default: break; - } - - /* fixstr */ - if (b >= 0xa0 && b <= 0xbf) { - uint32_t sz = b & 0x1f; - return (sz <= n - i) ? i + sz : 0; - } - - /* fixarray */ - if (b >= 0x90 && b <= 0x9f) { - uint32_t count = b & 0x0f; - for (uint32_t j = 0; j < count; j++) { - i = skip_one_d(a, n, i, depth + 1); - if (!i) return 0; - } - return i; - } - - /* fixmap */ - if (b >= 0x80 && b <= 0x8f) { - uint32_t count = b & 0x0f; - for (uint32_t j = 0; j < count; j++) { - i = skip_one_d(a, n, i, depth + 1); if (!i) return 0; - i = skip_one_d(a, n, i, depth + 1); if (!i) return 0; - } - return i; - } - - /* array16/32 */ - if (b == MP_ARRAY16 || b == MP_ARRAY32) { - uint32_t count; - if (b == MP_ARRAY16) { - if (i + 2 > n) return 0; - count = read16(a + i); i += 2; - } else { - if (i + 4 > n) return 0; - count = read32(a + i); i += 4; - } - for (uint32_t j = 0; j < count; j++) { - i = skip_one_d(a, n, i, depth + 1); - if (!i) return 0; - } - return i; - } - - /* map16/32 */ - if (b == MP_MAP16 || b == MP_MAP32) { - uint32_t count; - if (b == MP_MAP16) { - if (i + 2 > n) return 0; - count = read16(a + i); i += 2; - } else { - if (i + 4 > n) return 0; - count = read32(a + i); i += 4; - } - for (uint32_t j = 0; j < count; j++) { - i = skip_one_d(a, n, i, depth + 1); if (!i) return 0; - i = skip_one_d(a, n, i, depth + 1); if (!i) return 0; - } - return i; - } - - return 0; -} - -/* ── is_valid ─────────────────────────────────────────────────────── */ - -static bool is_valid(const uint8_t* a, uint32_t n) { - if (n == 0) return false; - uint32_t end = skip_one(a, n, 0); - return end == n; -} - -/* ── error_position_of — byte offset of first error ───────────────── */ - -static size_t error_position_of(const uint8_t* a, uint32_t n) { - if (n == 0) return 0; - uint32_t end = skip_one(a, n, 0); - if (end == n) return 0; - /* Walk byte by byte to find where it goes wrong */ - for (uint32_t i = 0; i < n;) { - uint32_t next = skip_one(a, n, i); - if (!next) return i; - i = next; - } - return 0; -} - -/* ── encode helpers ───────────────────────────────────────────────── */ - -static void encode_array_header(Buf& buf, uint32_t count) { - if (count <= 15) { - buf.append1(static_cast(MP_FIXARRAY_MASK | count)); - } else if (count <= 0xffff) { - uint8_t h[3]; h[0] = MP_ARRAY16; write16(h + 1, static_cast(count)); - buf.append(h, 3); - } else { - uint8_t h[5]; h[0] = MP_ARRAY32; write32(h + 1, count); - buf.append(h, 5); - } -} - -static void encode_map_header(Buf& buf, uint32_t count) { - if (count <= 15) { - buf.append1(static_cast(MP_FIXMAP_MASK | count)); - } else if (count <= 0xffff) { - uint8_t h[3]; h[0] = MP_MAP16; write16(h + 1, static_cast(count)); - buf.append(h, 3); - } else { - uint8_t h[5]; h[0] = MP_MAP32; write32(h + 1, count); - buf.append(h, 5); - } -} - -static void encode_string(Buf& buf, const char* s, uint32_t len) { - if (len <= 31) { - buf.append1(static_cast(MP_FIXSTR_MASK | len)); - } else if (len <= 0xff) { - uint8_t h[2] = {MP_STR8, static_cast(len)}; - buf.append(h, 2); - } else if (len <= 0xffff) { - uint8_t h[3]; h[0] = MP_STR16; write16(h + 1, static_cast(len)); - buf.append(h, 3); - } else { - uint8_t h[5]; h[0] = MP_STR32; write32(h + 1, len); - buf.append(h, 5); - } - buf.append(reinterpret_cast(s), len); -} - -/* ── is_timestamp_ext — check if element at offset is a timestamp ext ── */ - -static constexpr uint8_t MP_TIMESTAMP_TYPE = 0xFF; - -static bool is_timestamp_ext(const uint8_t* a, uint32_t n, uint32_t i) { - if (i >= n) return false; - uint8_t b = a[i]; - if (b == MP_FIXEXT4 && i + 6 <= n && a[i+1] == MP_TIMESTAMP_TYPE) return true; - if (b == MP_FIXEXT8 && i + 10 <= n && a[i+1] == MP_TIMESTAMP_TYPE) return true; - if (b == MP_EXT8 && i + 3 <= n && a[i+1] == 12 && a[i+2] == MP_TIMESTAMP_TYPE) return true; - return false; -} - -static bool decode_timestamp(const uint8_t* a, uint32_t n, uint32_t i, - int64_t* pSec, uint32_t* pNsec) { - if (i >= n) return false; - uint8_t b = a[i]; - if (b == MP_FIXEXT4 && i + 6 <= n && a[i+1] == MP_TIMESTAMP_TYPE) { - *pSec = static_cast(read32(a + i + 2)); - *pNsec = 0; - return true; - } - if (b == MP_FIXEXT8 && i + 10 <= n && a[i+1] == MP_TIMESTAMP_TYPE) { - uint64_t v = read64(a + i + 2); - *pNsec = static_cast(v >> 34); - *pSec = static_cast(v & 0x3FFFFFFFFULL); - return true; - } - if (b == MP_EXT8 && i + 15 <= n && a[i+1] == 12 && a[i+2] == MP_TIMESTAMP_TYPE) { - *pNsec = read32(a + i + 3); - *pSec = static_cast(read64(a + i + 7)); - return true; - } - return false; -} - -/* ── get_type — return Type for element at offset ─────────────────── */ - -static Type get_type(const uint8_t* a, uint32_t n, uint32_t i) { - if (i >= n) return Type::Nil; - uint8_t b = a[i]; - if (b == MP_NIL) return Type::Nil; - if (b == MP_TRUE) return Type::True; - if (b == MP_FALSE) return Type::False; - if (b <= 0x7f || b >= 0xe0) return Type::Integer; - if (b >= 0xa0 && b <= 0xbf) return Type::String; - if (b >= 0x90 && b <= 0x9f) return Type::Array; - if (b >= 0x80 && b <= 0x8f) return Type::Map; - switch (b) { - case MP_UINT8: case MP_UINT16: case MP_UINT32: case MP_UINT64: - case MP_INT8: case MP_INT16: case MP_INT32: case MP_INT64: - return Type::Integer; - case MP_FLOAT32: - return Type::Float32; - case MP_FLOAT64: - return Type::Real; - case MP_STR8: case MP_STR16: case MP_STR32: - return Type::String; - case MP_BIN8: case MP_BIN16: case MP_BIN32: - return Type::Binary; - case MP_ARRAY16: case MP_ARRAY32: - return Type::Array; - case MP_MAP16: case MP_MAP32: - return Type::Map; - case MP_EXT8: case MP_EXT16: case MP_EXT32: - case MP_FIXEXT1: case MP_FIXEXT2: case MP_FIXEXT4: - case MP_FIXEXT8: case MP_FIXEXT16: - if (is_timestamp_ext(a, n, i)) return Type::Timestamp; - return Type::Ext; - default: - return Type::Nil; - } -} - -static const char* get_type_str_at(const uint8_t* a, uint32_t n, uint32_t i) { - switch (get_type(a, n, i)) { - case Type::Nil: return "null"; - case Type::True: return "true"; - case Type::False: return "false"; - case Type::Integer: return "integer"; - case Type::Real: return "real"; - case Type::Float32: return "float32"; - case Type::String: return "text"; - case Type::Binary: return "binary"; - case Type::Array: return "array"; - case Type::Map: return "map"; - case Type::Ext: return "ext"; - case Type::Timestamp: return "timestamp"; - } - return "null"; -} - -/* ── get_container_count ──────────────────────────────────────────── */ - -static int64_t get_container_count(const uint8_t* a, uint32_t n, uint32_t i) { - if (i >= n) return -1; - uint8_t b = a[i]; - if (b >= 0x90 && b <= 0x9f) return b & 0x0f; - if (b >= 0x80 && b <= 0x8f) return b & 0x0f; - if (b == MP_ARRAY16 && i + 3 <= n) return read16(a + i + 1); - if (b == MP_ARRAY32 && i + 5 <= n) return read32(a + i + 1); - if (b == MP_MAP16 && i + 3 <= n) return read16(a + i + 1); - if (b == MP_MAP32 && i + 5 <= n) return read32(a + i + 1); - return -1; -} - -/* ── path_step — parse one step of $.path[0].key syntax ───────────── */ - -static int path_step( - const char* zPath, int* pi, - const char** pKey, int* nKey, - int64_t* pIdx -) { - int i = *pi; - if (zPath[i] == '\0') return 0; - if (zPath[i] == '.') { - int start; - i++; - start = i; - while (zPath[i] && zPath[i] != '.' && zPath[i] != '[') i++; - *pKey = zPath + start; - *nKey = i - start; - *pi = i; - return 'k'; - } - if (zPath[i] == '[') { - int64_t idx = 0; - int hasDigit = 0; - i++; - while (zPath[i] >= '0' && zPath[i] <= '9') { - idx = idx * 10 + (zPath[i] - '0'); - i++; - hasDigit = 1; - } - if (!hasDigit || zPath[i] != ']') return -1; - i++; - *pIdx = idx; - *pi = i; - return 'i'; - } - return -1; -} - -/* ── lookup — resolve path to byte range ──────────────────────────── */ - -static int lookup( - const uint8_t* a, uint32_t n, uint32_t iRoot, - const char* zPath, - uint32_t* piStart, uint32_t* piEnd -) { - int pi; - uint32_t iCur = iRoot; - if (!zPath || zPath[0] != '$') return RC_ERROR; - pi = 1; - - for (;;) { - const char* zKey = nullptr; - int nKey = 0; - int64_t idx = 0; - int step = path_step(zPath, &pi, &zKey, &nKey, &idx); - - if (step == 0) { - uint32_t iNext = skip_one(a, n, iCur); - *piStart = iCur; - *piEnd = iNext ? iNext : n; - return (iNext || iCur == n) ? RC_OK : RC_ERROR; - } - if (step < 0) return RC_ERROR; - if (iCur >= n) return RC_NOTFOUND; - - if (step == 'i') { - uint8_t b = a[iCur]; - uint32_t count, elemOff; - if (b >= 0x90 && b <= 0x9f) { - count = b & 0x0f; elemOff = iCur + 1; - } else if (b == MP_ARRAY16) { - if (iCur + 3 > n) return RC_ERROR; - count = read16(a + iCur + 1); elemOff = iCur + 3; - } else if (b == MP_ARRAY32) { - if (iCur + 5 > n) return RC_ERROR; - count = read32(a + iCur + 1); elemOff = iCur + 5; - } else { - return RC_NOTFOUND; - } - if (idx < 0 || static_cast(idx) >= count) return RC_NOTFOUND; - iCur = elemOff; - for (int64_t j = 0; j < idx; j++) { - iCur = skip_one(a, n, iCur); - if (!iCur) return RC_ERROR; - } - } else { - uint8_t b = a[iCur]; - uint32_t count, elemOff; - bool found = false; - if (b >= 0x80 && b <= 0x8f) { - count = b & 0x0f; elemOff = iCur + 1; - } else if (b == MP_MAP16) { - if (iCur + 3 > n) return RC_ERROR; - count = read16(a + iCur + 1); elemOff = iCur + 3; - } else if (b == MP_MAP32) { - if (iCur + 5 > n) return RC_ERROR; - count = read32(a + iCur + 1); elemOff = iCur + 5; - } else { - return RC_NOTFOUND; - } - iCur = elemOff; - for (uint32_t j = 0; j < count && !found; j++) { - if (iCur >= n) return RC_ERROR; - uint8_t kb = a[iCur]; - const char* kStr = nullptr; - uint32_t kLen = 0; - if (kb >= 0xa0 && kb <= 0xbf) { - kLen = kb & 0x1f; kStr = reinterpret_cast(a + iCur + 1); - } else if (kb == MP_STR8 && iCur + 2 <= n) { - kLen = a[iCur + 1]; kStr = reinterpret_cast(a + iCur + 2); - } else if (kb == MP_STR16 && iCur + 3 <= n) { - kLen = read16(a + iCur + 1); kStr = reinterpret_cast(a + iCur + 3); - } else if (kb == MP_STR32 && iCur + 5 <= n) { - kLen = read32(a + iCur + 1); kStr = reinterpret_cast(a + iCur + 5); - } - uint32_t valOff = skip_one(a, n, iCur); - if (!valOff) return RC_ERROR; - if (kStr && static_cast(kLen) == nKey && - std::memcmp(kStr, zKey, static_cast(nKey)) == 0) { - iCur = valOff; - found = true; - } else { - iCur = skip_one(a, n, valOff); - if (!iCur) return RC_ERROR; - } - } - if (!found) return RC_NOTFOUND; - } - } -} - -/* ── decode_element — decode element at offset into Value ─────────── */ - -static Value decode_element(const uint8_t* a, uint32_t n, uint32_t iStart, uint32_t iEnd) { - if (iStart >= n || iStart >= iEnd) return Value::nil(); - uint8_t b = a[iStart]; - - if (b == MP_NIL) return Value::nil(); - if (b == MP_FALSE) return Value::boolean(false); - if (b == MP_TRUE) return Value::boolean(true); - if (b <= 0x7f) return Value::integer(static_cast(b)); - if (b >= 0xe0) return Value::integer(static_cast(static_cast(b))); - - switch (b) { - case MP_UINT8: - if (iStart + 2 <= n) return Value::integer(static_cast(a[iStart + 1])); - break; - case MP_UINT16: - if (iStart + 3 <= n) return Value::integer(static_cast(read16(a + iStart + 1))); - break; - case MP_UINT32: - if (iStart + 5 <= n) return Value::integer(static_cast(read32(a + iStart + 1))); - break; - case MP_UINT64: - if (iStart + 9 <= n) { - uint64_t v = read64(a + iStart + 1); - return Value::unsigned_integer(v); - } - break; - case MP_INT8: - if (iStart + 2 <= n) return Value::integer(static_cast(static_cast(a[iStart + 1]))); - break; - case MP_INT16: - if (iStart + 3 <= n) return Value::integer(static_cast(static_cast(read16(a + iStart + 1)))); - break; - case MP_INT32: - if (iStart + 5 <= n) return Value::integer(static_cast(static_cast(read32(a + iStart + 1)))); - break; - case MP_INT64: - if (iStart + 9 <= n) return Value::integer(static_cast(read64(a + iStart + 1))); - break; - case MP_FLOAT32: - if (iStart + 5 <= n) { - uint32_t bits = read32(a + iStart + 1); - float f; - std::memcpy(&f, &bits, 4); - return Value::real32(f); - } - break; - case MP_FLOAT64: - if (iStart + 9 <= n) { - uint64_t bits = read64(a + iStart + 1); - double d; - std::memcpy(&d, &bits, 8); - return Value::real(d); - } - break; - default: break; - } - - /* str → String */ - uint32_t sLen = 0, sOff = 0; - if (b >= 0xa0 && b <= 0xbf) { - sLen = b & 0x1f; sOff = iStart + 1; - } else if (b == MP_STR8 && iStart + 2 <= n) { - sLen = a[iStart + 1]; sOff = iStart + 2; - } else if (b == MP_STR16 && iStart + 3 <= n) { - sLen = read16(a + iStart + 1); sOff = iStart + 3; - } else if (b == MP_STR32 && iStart + 5 <= n) { - sLen = read32(a + iStart + 1); sOff = iStart + 5; - } - if (sOff) { - if (sLen > n - sOff) sLen = n - sOff; - return Value::string(std::string_view(reinterpret_cast(a + sOff), sLen)); - } - - /* bin → Binary (payload only, no header) */ - { - uint32_t bLen = 0, bOff = 0; - if (b == MP_BIN8 && iStart + 2 <= n) { - bLen = a[iStart + 1]; bOff = iStart + 2; - } else if (b == MP_BIN16 && iStart + 3 <= n) { - bLen = read16(a + iStart + 1); bOff = iStart + 3; - } else if (b == MP_BIN32 && iStart + 5 <= n) { - bLen = read32(a + iStart + 1); bOff = iStart + 5; - } - if (bOff) { - if (bLen > n - bOff) bLen = n - bOff; - return Value::binary(a + bOff, bLen); - } - } - - /* timestamp ext → Timestamp value */ - { - int64_t tsec; uint32_t tnsec; - if (decode_timestamp(a, n, iStart, &tsec, &tnsec)) { - return Value::timestamp(tsec, tnsec); - } - } - - /* ext → Ext (type code + payload, no header) */ - { - int8_t tc = 0; - uint32_t elen = 0, eOff = 0; - switch (b) { - case MP_FIXEXT1: if (iStart+3<=n) { tc=static_cast(a[iStart+1]); elen=1; eOff=iStart+2; } break; - case MP_FIXEXT2: if (iStart+4<=n) { tc=static_cast(a[iStart+1]); elen=2; eOff=iStart+2; } break; - case MP_FIXEXT4: if (iStart+6<=n) { tc=static_cast(a[iStart+1]); elen=4; eOff=iStart+2; } break; - case MP_FIXEXT8: if (iStart+10<=n){ tc=static_cast(a[iStart+1]); elen=8; eOff=iStart+2; } break; - case MP_FIXEXT16: if (iStart+18<=n){ tc=static_cast(a[iStart+1]); elen=16; eOff=iStart+2; } break; - case MP_EXT8: - if (iStart+3<=n) { elen=a[iStart+1]; tc=static_cast(a[iStart+2]); eOff=iStart+3; } break; - case MP_EXT16: - if (iStart+4<=n) { elen=read16(a+iStart+1); tc=static_cast(a[iStart+3]); eOff=iStart+4; } break; - case MP_EXT32: - if (iStart+6<=n) { elen=read32(a+iStart+1); tc=static_cast(a[iStart+5]); eOff=iStart+6; } break; - default: break; - } - if (eOff) { - if (elen > n - eOff) elen = n - eOff; - return Value::ext(tc, a + eOff, elen); - } - } - - /* containers → raw binary blob (includes header) */ - return Value::binary(a + iStart, iEnd - iStart); -} - -/* ── Mutation internals ───────────────────────────────────────────── */ - -static int edit_step(Buf& out, const uint8_t* a, uint32_t n, uint32_t iCur, - const char* zPath, int pi, - const uint8_t* newBin, uint32_t nNew, - int mode, int* pSkip); - -static int edit_map( - Buf& out, const uint8_t* a, uint32_t n, uint32_t iCur, - const char* zKey, int nKey, - const char* zPath, int pi, - const uint8_t* newBin, uint32_t nNew, int mode -) { - if (iCur >= n) return RC_ERROR; - uint8_t b = a[iCur]; - uint32_t count, dataOff; - - if (b >= 0x80 && b <= 0x8f) { count = b & 0x0f; dataOff = iCur + 1; } - else if (b == MP_MAP16) { - if (iCur + 3 > n) return RC_ERROR; - count = read16(a + iCur + 1); dataOff = iCur + 3; - } else if (b == MP_MAP32) { - if (iCur + 5 > n) return RC_ERROR; - count = read32(a + iCur + 1); dataOff = iCur + 5; - } else { - if (mode == EDIT_REPLACE || mode == EDIT_REMOVE) { - uint32_t iEnd = skip_one(a, n, iCur); - if (iEnd) out.append(a + iCur, iEnd - iCur); - return RC_OK; - } - return RC_ERROR; - } - - uint32_t newCount = count; - Buf tmp; - uint32_t cur2 = dataOff; - bool foundKey = false; - int rc = RC_OK; - - for (uint32_t j = 0; j < count; j++) { - if (cur2 >= n) return RC_ERROR; - uint8_t kb = a[cur2]; - const char* kStr = nullptr; uint32_t kLen = 0; - if (kb >= 0xa0 && kb <= 0xbf) { - kLen = kb & 0x1f; kStr = reinterpret_cast(a + cur2 + 1); - } else if (kb == MP_STR8 && cur2 + 2 <= n) { - kLen = a[cur2 + 1]; kStr = reinterpret_cast(a + cur2 + 2); - } else if (kb == MP_STR16 && cur2 + 3 <= n) { - kLen = read16(a + cur2 + 1); kStr = reinterpret_cast(a + cur2 + 3); - } else if (kb == MP_STR32 && cur2 + 5 <= n) { - kLen = read32(a + cur2 + 1); kStr = reinterpret_cast(a + cur2 + 5); - } - - uint32_t valOff = skip_one(a, n, cur2); - if (!valOff) return RC_ERROR; - uint32_t pairEnd = skip_one(a, n, valOff); - if (!pairEnd) return RC_ERROR; - - bool isMatch = (kStr && static_cast(kLen) == nKey && - std::memcmp(kStr, zKey, static_cast(nKey)) == 0); - - if (isMatch) { - foundKey = true; - if (mode == EDIT_INSERT) { - tmp.append(a + cur2, pairEnd - cur2); - } else { - Buf vbuf; int skip = 0; - rc = edit_step(vbuf, a, n, valOff, zPath, pi, newBin, nNew, mode, &skip); - if (rc != RC_OK) return rc; - if (skip) { - newCount--; - } else { - tmp.append(a + cur2, valOff - cur2); - tmp.append(vbuf.ptr(), vbuf.size()); - } - } - } else { - tmp.append(a + cur2, pairEnd - cur2); - } - cur2 = pairEnd; - } - - if (!foundKey) { - if (mode == EDIT_SET || mode == EDIT_INSERT) { - int pi2 = pi; const char* zk2; int nk2; int64_t idx2; - if (path_step(zPath, &pi2, &zk2, &nk2, &idx2) != 0) { - uint32_t iEnd = skip_one(a, n, iCur); - if (iEnd) out.append(a + iCur, iEnd - iCur); - return RC_OK; - } - encode_string(tmp, zKey, static_cast(nKey)); - tmp.append(newBin, nNew); - newCount++; - } else { - uint32_t iEnd = skip_one(a, n, iCur); - if (iEnd) out.append(a + iCur, iEnd - iCur); - return RC_OK; - } - } - - encode_map_header(out, newCount); - out.append(tmp.ptr(), tmp.size()); - return RC_OK; -} - -static int edit_array( - Buf& out, const uint8_t* a, uint32_t n, uint32_t iCur, - int64_t stepIdx, - const char* zPath, int pi, - const uint8_t* newBin, uint32_t nNew, int mode -) { - if (iCur >= n) return RC_ERROR; - uint8_t b = a[iCur]; - uint32_t count, dataOff; - - if (b >= 0x90 && b <= 0x9f) { count = b & 0x0f; dataOff = iCur + 1; } - else if (b == MP_ARRAY16) { - if (iCur + 3 > n) return RC_ERROR; - count = read16(a + iCur + 1); dataOff = iCur + 3; - } else if (b == MP_ARRAY32) { - if (iCur + 5 > n) return RC_ERROR; - count = read32(a + iCur + 1); dataOff = iCur + 5; - } else { - if (mode == EDIT_REPLACE || mode == EDIT_REMOVE) { - uint32_t iEnd = skip_one(a, n, iCur); - if (iEnd) out.append(a + iCur, iEnd - iCur); - return RC_OK; - } - return RC_ERROR; - } - - uint32_t newCount = count; - Buf tmp; - uint32_t cur2 = dataOff; - bool foundIt = false; - int rc = RC_OK; - - for (uint32_t j = 0; j < count; j++) { - uint32_t eEnd = skip_one(a, n, cur2); - if (!eEnd) return RC_ERROR; - - if (static_cast(j) == stepIdx) { - foundIt = true; - if (mode == EDIT_ARRAY_INS) { - tmp.append(newBin, nNew); - tmp.append(a + cur2, eEnd - cur2); - newCount++; - } else if (mode == EDIT_INSERT) { - tmp.append(a + cur2, eEnd - cur2); - } else { - Buf ebuf; int skip = 0; - rc = edit_step(ebuf, a, n, cur2, zPath, pi, newBin, nNew, mode, &skip); - if (rc != RC_OK) return rc; - if (skip) { - newCount--; - } else { - tmp.append(ebuf.ptr(), ebuf.size()); - } - } - } else { - tmp.append(a + cur2, eEnd - cur2); - } - cur2 = eEnd; - } - - if (!foundIt) { - if (mode == EDIT_ARRAY_INS) { - tmp.append(newBin, nNew); - newCount++; - } else if ((mode == EDIT_SET || mode == EDIT_INSERT) && - static_cast(stepIdx) == count) { - tmp.append(newBin, nNew); - newCount++; - } else if (mode == EDIT_REPLACE || mode == EDIT_REMOVE) { - uint32_t iEnd = skip_one(a, n, iCur); - if (iEnd) out.append(a + iCur, iEnd - iCur); - return RC_OK; - } else { - return RC_NOTFOUND; - } - } - - encode_array_header(out, newCount); - out.append(tmp.ptr(), tmp.size()); - return RC_OK; -} - -static int edit_step( - Buf& out, const uint8_t* a, uint32_t n, uint32_t iCur, - const char* zPath, int pi, - const uint8_t* newBin, uint32_t nNew, - int mode, int* pSkip -) { - const char* zKey = nullptr; int nKey = 0; int64_t stepIdx = 0; - int step = path_step(zPath, &pi, &zKey, &nKey, &stepIdx); - if (pSkip) *pSkip = 0; - - if (step == 0) { - if (mode == EDIT_REMOVE) { - if (pSkip) *pSkip = 1; - return RC_OK; - } - if (mode == EDIT_ARRAY_INS) return RC_ERROR; - if (mode == EDIT_INSERT) { - uint32_t iEnd = skip_one(a, n, iCur); - if (iEnd) out.append(a + iCur, iEnd - iCur); - return RC_OK; - } - out.append(newBin, nNew); - return RC_OK; - } - if (step < 0) return RC_ERROR; - - if (step == 'k') { - return edit_map(out, a, n, iCur, zKey, nKey, zPath, pi, newBin, nNew, mode); - } else { - return edit_array(out, a, n, iCur, stepIdx, zPath, pi, newBin, nNew, mode); - } -} - -static int apply_edit( - Buf& out, - const uint8_t* a, uint32_t n, - const char* zPath, - const uint8_t* newBin, uint32_t nNew, - int mode -) { - if (!zPath || zPath[0] != '$') return RC_ERROR; - return edit_step(out, a, n, 0, zPath, 1, newBin, nNew, mode, nullptr); -} - -/* ── merge_patch (RFC 7386) ───────────────────────────────────────── */ - -static int merge_patch( - Buf& out, - const uint8_t* a, uint32_t n, uint32_t ia, - const uint8_t* p, uint32_t np, uint32_t ip, - int depth -) { - if (ip >= np) return RC_ERROR; - if (depth > kMaxDepth) return RC_ERROR; - uint8_t pb = p[ip]; - - if (pb == MP_NIL) { out.append1(MP_NIL); return RC_OK; } - - bool pIsMap = (pb >= 0x80 && pb <= 0x8f) || pb == MP_MAP16 || pb == MP_MAP32; - if (!pIsMap) { - uint32_t pEnd = skip_one(p, np, ip); - if (pEnd) out.append(p + ip, pEnd - ip); - return RC_OK; - } - - uint8_t ab = (ia < n) ? a[ia] : 0; - bool aIsMap = (ab >= 0x80 && ab <= 0x8f) || ab == MP_MAP16 || ab == MP_MAP32; - - uint32_t pCount, pDataOff; - if (pb >= 0x80 && pb <= 0x8f) { pCount = pb & 0x0f; pDataOff = ip + 1; } - else if (pb == MP_MAP16) { - if (ip + 3 > np) return RC_ERROR; - pCount = read16(p + ip + 1); pDataOff = ip + 3; - } else { - if (ip + 5 > np) return RC_ERROR; - pCount = read32(p + ip + 1); pDataOff = ip + 5; - } - - uint32_t aCount = 0, aDataOff = 0; - if (aIsMap) { - if (ab >= 0x80 && ab <= 0x8f) { aCount = ab & 0x0f; aDataOff = ia + 1; } - else if (ab == MP_MAP16) { - if (ia + 3 > n) { aIsMap = false; } - else { aCount = read16(a + ia + 1); aDataOff = ia + 3; } - } else { - if (ia + 5 > n) { aIsMap = false; } - else { aCount = read32(a + ia + 1); aDataOff = ia + 5; } - } - } - - /* Pre-scan patch keys */ - struct PatchEntry { - const char* zKey; uint32_t nKey; - uint32_t keyOff, valOff, pairEnd; - bool matched; - }; - /* Sanity: each map pair needs at least 2 bytes; reject implausible counts */ - if (pCount > (np - pDataOff) / 2 + 1) return RC_ERROR; - std::vector pIdx(pCount); - { - uint32_t pc2 = pDataOff; - for (uint32_t k = 0; k < pCount; k++) { - if (pc2 >= np) return RC_ERROR; - uint8_t pkb = p[pc2]; - pIdx[k] = {nullptr, 0, pc2, 0, 0, false}; - if (pkb >= 0xa0 && pkb <= 0xbf) { - pIdx[k].nKey = pkb & 0x1f; - pIdx[k].zKey = reinterpret_cast(p + pc2 + 1); - } else if (pkb == MP_STR8 && pc2 + 2 <= np) { - pIdx[k].nKey = p[pc2 + 1]; - pIdx[k].zKey = reinterpret_cast(p + pc2 + 2); - } else if (pkb == MP_STR16 && pc2 + 3 <= np) { - pIdx[k].nKey = read16(p + pc2 + 1); - pIdx[k].zKey = reinterpret_cast(p + pc2 + 3); - } else if (pkb == MP_STR32 && pc2 + 5 <= np) { - pIdx[k].nKey = read32(p + pc2 + 1); - pIdx[k].zKey = reinterpret_cast(p + pc2 + 5); - } - pIdx[k].valOff = skip_one(p, np, pc2); - if (!pIdx[k].valOff) return RC_ERROR; - pIdx[k].pairEnd = skip_one(p, np, pIdx[k].valOff); - if (!pIdx[k].pairEnd) return RC_ERROR; - pc2 = pIdx[k].pairEnd; - } - } - - Buf tmp; - uint32_t newCount = 0; - - /* Phase 1: iterate target pairs */ - if (aIsMap) { - uint32_t ac = aDataOff; - for (uint32_t j = 0; j < aCount; j++) { - if (ac >= n) return RC_ERROR; - uint8_t kb = a[ac]; - const char* kStr = nullptr; uint32_t kLen = 0; - if (kb >= 0xa0 && kb <= 0xbf) { - kLen = kb & 0x1f; kStr = reinterpret_cast(a + ac + 1); - } else if (kb == MP_STR8 && ac + 2 <= n) { - kLen = a[ac + 1]; kStr = reinterpret_cast(a + ac + 2); - } else if (kb == MP_STR16 && ac + 3 <= n) { - kLen = read16(a + ac + 1); kStr = reinterpret_cast(a + ac + 3); - } else if (kb == MP_STR32 && ac + 5 <= n) { - kLen = read32(a + ac + 1); kStr = reinterpret_cast(a + ac + 5); - } - - uint32_t aValOff = skip_one(a, n, ac); - if (!aValOff) return RC_ERROR; - uint32_t aPairEnd = skip_one(a, n, aValOff); - if (!aPairEnd) return RC_ERROR; - - bool foundInPatch = false, patchIsNil = false; - uint32_t pMatchVal = 0; - for (uint32_t k = 0; k < pCount; k++) { - if (pIdx[k].zKey && kStr && pIdx[k].nKey == kLen && - std::memcmp(pIdx[k].zKey, kStr, kLen) == 0) { - foundInPatch = true; - pMatchVal = pIdx[k].valOff; - patchIsNil = (pIdx[k].valOff < np && p[pIdx[k].valOff] == MP_NIL); - pIdx[k].matched = true; - break; - } - } - - if (foundInPatch && patchIsNil) { - /* Drop this pair */ - } else if (foundInPatch) { - Buf mb; - int mrc = merge_patch(mb, a, n, aValOff, p, np, pMatchVal, depth + 1); - if (mrc == RC_OK) { - tmp.append(a + ac, aValOff - ac); - tmp.append(mb.ptr(), mb.size()); - newCount++; - } - } else { - tmp.append(a + ac, aPairEnd - ac); - newCount++; - } - ac = aPairEnd; - } - } - - /* Phase 2: add unmatched patch pairs */ - for (uint32_t k = 0; k < pCount; k++) { - if (!pIdx[k].matched && pIdx[k].valOff < np && p[pIdx[k].valOff] != MP_NIL) { - tmp.append(p + pIdx[k].keyOff, pIdx[k].pairEnd - pIdx[k].keyOff); - newCount++; - } - } - - encode_map_header(out, newCount); - out.append(tmp.ptr(), tmp.size()); - return RC_OK; -} - -/* ── JSON output ──────────────────────────────────────────────────── */ - -static void json_escape_str(Buf& out, const uint8_t* s, uint32_t len) { - out.append1('"'); - uint32_t start = 0; - for (uint32_t j = 0; j < len; j++) { - uint8_t c = s[j]; - if (c >= 0x20 && c != '"' && c != '\\') continue; - if (j > start) out.append(s + start, j - start); - if (c == '"') { uint8_t b[2] = {'\\', '"'}; out.append(b, 2); } - else if (c == '\\') { uint8_t b[2] = {'\\', '\\'}; out.append(b, 2); } - else if (c == '\n') { uint8_t b[2] = {'\\', 'n'}; out.append(b, 2); } - else if (c == '\r') { uint8_t b[2] = {'\\', 'r'}; out.append(b, 2); } - else if (c == '\t') { uint8_t b[2] = {'\\', 't'}; out.append(b, 2); } - else { - char esc[8]; std::snprintf(esc, 8, "\\u%04x", static_cast(c)); - out.append(reinterpret_cast(esc), 6); - } - start = j + 1; - } - if (len > start) out.append(s + start, len - start); - out.append1('"'); -} - -static void json_newline(Buf& out, int depth, int indentW) { - static const char spaces[] = - " "; - int nSpaces = depth * indentW; - out.append1('\n'); - while (nSpaces > 0) { - int chunk = nSpaces > static_cast(sizeof(spaces) - 1) - ? static_cast(sizeof(spaces) - 1) : nSpaces; - out.append(reinterpret_cast(spaces), static_cast(chunk)); - nSpaces -= chunk; - } -} - -static void to_json_at( - Buf& out, const uint8_t* a, uint32_t n, uint32_t i, - bool pretty, int depth, int indentW -) { - char s[64]; - if (i >= n || depth > kMaxDepth) { - out.append(reinterpret_cast("null"), 4); return; - } - uint8_t b = a[i]; - - if (b == MP_NIL) { out.append(reinterpret_cast("null"), 4); return; } - if (b == MP_FALSE) { out.append(reinterpret_cast("false"), 5); return; } - if (b == MP_TRUE) { out.append(reinterpret_cast("true"), 4); return; } - if (b <= 0x7f) { - int len = std::snprintf(s, sizeof(s), "%d", static_cast(b)); - out.append(reinterpret_cast(s), static_cast(len)); return; - } - if (b >= 0xe0) { - int len = std::snprintf(s, sizeof(s), "%d", static_cast(static_cast(b))); - out.append(reinterpret_cast(s), static_cast(len)); return; - } - - switch (b) { - case MP_UINT8: if (i+2>n) break; { int l=std::snprintf(s,sizeof(s),"%u",static_cast(a[i+1])); out.append(reinterpret_cast(s),static_cast(l)); return; } - case MP_UINT16: if (i+3>n) break; { int l=std::snprintf(s,sizeof(s),"%u",static_cast(read16(a+i+1))); out.append(reinterpret_cast(s),static_cast(l)); return; } - case MP_UINT32: if (i+5>n) break; { int l=std::snprintf(s,sizeof(s),"%u",static_cast(read32(a+i+1))); out.append(reinterpret_cast(s),static_cast(l)); return; } - case MP_UINT64: if (i+9>n) break; { int l=std::snprintf(s,sizeof(s),"%llu",static_cast(read64(a+i+1))); out.append(reinterpret_cast(s),static_cast(l)); return; } - case MP_INT8: if (i+2>n) break; { int l=std::snprintf(s,sizeof(s),"%d",static_cast(static_cast(a[i+1]))); out.append(reinterpret_cast(s),static_cast(l)); return; } - case MP_INT16: if (i+3>n) break; { int l=std::snprintf(s,sizeof(s),"%d",static_cast(static_cast(read16(a+i+1)))); out.append(reinterpret_cast(s),static_cast(l)); return; } - case MP_INT32: if (i+5>n) break; { int l=std::snprintf(s,sizeof(s),"%d",static_cast(static_cast(read32(a+i+1)))); out.append(reinterpret_cast(s),static_cast(l)); return; } - case MP_INT64: if (i+9>n) break; { int l=std::snprintf(s,sizeof(s),"%lld",static_cast(read64(a+i+1))); out.append(reinterpret_cast(s),static_cast(l)); return; } - case MP_FLOAT32: { - if (i+5>n) break; - uint32_t bits = read32(a+i+1); float f; std::memcpy(&f, &bits, 4); - if (!std::isfinite(static_cast(f))) { out.append(reinterpret_cast("null"),4); return; } - int l=std::snprintf(s,sizeof(s),"%.7g",static_cast(f)); - out.append(reinterpret_cast(s),static_cast(l)); return; - } - case MP_FLOAT64: { - if (i+9>n) break; - uint64_t bits = read64(a+i+1); double d; std::memcpy(&d, &bits, 8); - if (!std::isfinite(d)) { out.append(reinterpret_cast("null"),4); return; } - int l = std::snprintf(s,sizeof(s),"%.17g",d); - if (!std::strchr(s,'.') && !std::strchr(s,'e') && !std::strchr(s,'E')) - l = std::snprintf(s,sizeof(s),"%.1f",d); - out.append(reinterpret_cast(s),static_cast(l)); return; - } - default: break; - } - - /* str */ - { - uint32_t sLen = 0, sOff = 0; - if (b >= 0xa0 && b <= 0xbf) { sLen = b & 0x1f; sOff = i + 1; } - else if (b == MP_STR8 && i + 2 <= n) { sLen = a[i+1]; sOff = i + 2; } - else if (b == MP_STR16 && i + 3 <= n) { sLen = read16(a+i+1); sOff = i + 3; } - else if (b == MP_STR32 && i + 5 <= n) { sLen = read32(a+i+1); sOff = i + 5; } - if (sOff) { - if (sLen > n - sOff) sLen = n - sOff; - json_escape_str(out, a + sOff, sLen); - return; - } - } - - /* bin → hex string */ - { - uint32_t bLen = 0, bOff = 0; - if (b == MP_BIN8 && i + 2 <= n) { bLen = a[i+1]; bOff = i + 2; } - else if (b == MP_BIN16 && i + 3 <= n) { bLen = read16(a+i+1); bOff = i + 3; } - else if (b == MP_BIN32 && i + 5 <= n) { bLen = read32(a+i+1); bOff = i + 5; } - if (bOff) { - static const char hex[] = "0123456789abcdef"; - if (bLen > n - bOff) bLen = n - bOff; - out.append1('"'); - for (uint32_t j = 0; j < bLen; j++) { - uint8_t by = a[bOff + j]; - out.append1(static_cast(hex[by >> 4])); - out.append1(static_cast(hex[by & 0xf])); - } - out.append1('"'); - return; - } - } - - /* array */ - { - bool isArr = false; uint32_t count = 0, dataOff = 0; - if (b >= 0x90 && b <= 0x9f) { isArr = true; count = b & 0x0f; dataOff = i + 1; } - else if (b == MP_ARRAY16 && i + 3 <= n) { isArr = true; count = read16(a+i+1); dataOff = i + 3; } - else if (b == MP_ARRAY32 && i + 5 <= n) { isArr = true; count = read32(a+i+1); dataOff = i + 5; } - if (isArr) { - uint32_t cur = dataOff; - out.append1('['); - for (uint32_t j = 0; j < count; j++) { - if (cur >= n) break; - uint32_t next = skip_one(a, n, cur); - if (j > 0) out.append1(','); - if (pretty) json_newline(out, depth + 1, indentW); - to_json_at(out, a, n, cur, pretty, depth + 1, indentW); - cur = next ? next : n; - } - if (pretty && count > 0) json_newline(out, depth, indentW); - out.append1(']'); - return; - } - } - - /* map */ - { - bool isMap = false; uint32_t count = 0, dataOff = 0; - if (b >= 0x80 && b <= 0x8f) { isMap = true; count = b & 0x0f; dataOff = i + 1; } - else if (b == MP_MAP16 && i + 3 <= n) { isMap = true; count = read16(a+i+1); dataOff = i + 3; } - else if (b == MP_MAP32 && i + 5 <= n) { isMap = true; count = read32(a+i+1); dataOff = i + 5; } - if (isMap) { - uint32_t cur = dataOff; - out.append1('{'); - for (uint32_t j = 0; j < count; j++) { - if (cur >= n) break; - uint32_t valOff = skip_one(a, n, cur); - uint32_t pairEnd = valOff ? skip_one(a, n, valOff) : 0; - if (j > 0) out.append1(','); - if (pretty) json_newline(out, depth + 1, indentW); - to_json_at(out, a, n, cur, pretty, depth + 1, indentW); - out.append1(':'); - if (pretty) out.append1(' '); - to_json_at(out, a, n, valOff ? valOff : n, pretty, depth + 1, indentW); - cur = pairEnd ? pairEnd : n; - } - if (pretty && count > 0) json_newline(out, depth, indentW); - out.append1('}'); - return; - } - } - - /* ext / unknown → null */ - out.append(reinterpret_cast("null"), 4); -} - -/* ── JSON parser → msgpack ────────────────────────────────────────── */ - -struct JsonParser { - const char* z; - int n, i; -}; - -static void jp_skip_ws(JsonParser& p) { - while (p.i < p.n && (p.z[p.i] == ' ' || p.z[p.i] == '\t' || - p.z[p.i] == '\n' || p.z[p.i] == '\r')) p.i++; -} - -static int jp_hex4(const char* z) { - int v = 0; - for (int j = 0; j < 4; j++) { - char c = z[j]; int h; - if (c >= '0' && c <= '9') h = c - '0'; - else if (c >= 'a' && c <= 'f') h = c - 'a' + 10; - else if (c >= 'A' && c <= 'F') h = c - 'A' + 10; - else return -1; - v = (v << 4) | h; - } - return v; -} - -static int jp_codepoint_to_utf8(uint32_t cp, uint8_t* buf) { - if (cp < 0x80) { buf[0] = static_cast(cp); return 1; } - if (cp < 0x800) { buf[0] = static_cast(0xc0 | (cp >> 6)); buf[1] = static_cast(0x80 | (cp & 0x3f)); return 2; } - if (cp < 0x10000) { buf[0] = static_cast(0xe0 | (cp >> 12)); buf[1] = static_cast(0x80 | ((cp >> 6) & 0x3f)); buf[2] = static_cast(0x80 | (cp & 0x3f)); return 3; } - buf[0] = static_cast(0xf0 | (cp >> 18)); buf[1] = static_cast(0x80 | ((cp >> 12) & 0x3f)); - buf[2] = static_cast(0x80 | ((cp >> 6) & 0x3f)); buf[3] = static_cast(0x80 | (cp & 0x3f)); return 4; -} - -static int jp_parse_value(JsonParser& p, Buf& out); - -static int jp_parse_string(JsonParser& p, Buf& out) { - Buf sb; - p.i++; /* skip '"' */ - while (p.i < p.n) { - auto c = static_cast(p.z[p.i]); - if (c == '"') { p.i++; break; } - if (c == '\\') { - p.i++; - if (p.i >= p.n) return RC_ERROR; - char esc = p.z[p.i++]; - switch (esc) { - case '"': sb.append1('"'); break; - case '\\': sb.append1('\\'); break; - case '/': sb.append1('/'); break; - case 'n': sb.append1('\n'); break; - case 'r': sb.append1('\r'); break; - case 't': sb.append1('\t'); break; - case 'b': sb.append1('\b'); break; - case 'f': sb.append1('\f'); break; - case 'u': { - if (p.i + 4 > p.n) return RC_ERROR; - int cp = jp_hex4(p.z + p.i); p.i += 4; - if (cp < 0) return RC_ERROR; - if (cp >= 0xD800 && cp <= 0xDBFF && p.i + 6 <= p.n && - p.z[p.i] == '\\' && p.z[p.i + 1] == 'u') { - int lo = jp_hex4(p.z + p.i + 2); - if (lo >= 0xDC00 && lo <= 0xDFFF) { - p.i += 6; - cp = 0x10000 + ((cp - 0xD800) << 10) + (lo - 0xDC00); - } - } - uint8_t utf[4]; - int ulen = jp_codepoint_to_utf8(static_cast(cp), utf); - sb.append(utf, static_cast(ulen)); - break; - } - default: sb.append1(static_cast(esc)); break; - } - } else { - sb.append1(c); p.i++; - } - } - encode_string(out, reinterpret_cast(sb.ptr()), - static_cast(sb.size())); - return RC_OK; -} - -static int jp_parse_number(JsonParser& p, Buf& out) { - int start = p.i; - bool isFloat = false; - if (p.i < p.n && p.z[p.i] == '-') p.i++; - while (p.i < p.n && p.z[p.i] >= '0' && p.z[p.i] <= '9') p.i++; - if (p.i < p.n && p.z[p.i] == '.') { - isFloat = true; p.i++; - while (p.i < p.n && p.z[p.i] >= '0' && p.z[p.i] <= '9') p.i++; - } - if (p.i < p.n && (p.z[p.i] == 'e' || p.z[p.i] == 'E')) { - isFloat = true; p.i++; - if (p.i < p.n && (p.z[p.i] == '+' || p.z[p.i] == '-')) p.i++; - while (p.i < p.n && p.z[p.i] >= '0' && p.z[p.i] <= '9') p.i++; - } - int len = p.i - start; - if (len <= 0 || len >= 64) return RC_ERROR; - char buf[64]; - std::memcpy(buf, p.z + start, static_cast(len)); buf[len] = '\0'; - - if (isFloat) { - double d = std::strtod(buf, nullptr); - uint8_t b[9]; uint64_t bits; b[0] = MP_FLOAT64; - std::memcpy(&bits, &d, 8); write64(b + 1, bits); - out.append(b, 9); - } else { - int64_t v = static_cast(std::strtoll(buf, nullptr, 10)); - if (v >= 0) { - if (v <= 0x7f) out.append1(static_cast(v)); - else if (v <= 0xff) { uint8_t b[2] = {MP_UINT8, static_cast(v)}; out.append(b, 2); } - else if (v <= 0xffff) { uint8_t b[3]; b[0] = MP_UINT16; write16(b+1, static_cast(v)); out.append(b, 3); } - else if (v <= static_cast(0xffffffff)) { uint8_t b[5]; b[0] = MP_UINT32; write32(b+1, static_cast(v)); out.append(b, 5); } - else { uint8_t b[9]; b[0] = MP_UINT64; write64(b+1, static_cast(v)); out.append(b, 9); } - } else { - if (v >= -32) out.append1(static_cast(v)); - else if (v >= -128) { uint8_t b[2] = {MP_INT8, static_cast(v)}; out.append(b, 2); } - else if (v >= -32768) { uint8_t b[3]; b[0] = MP_INT16; write16(b+1, static_cast(v)); out.append(b, 3); } - else if (v >= static_cast(-2147483648LL)) { uint8_t b[5]; b[0] = MP_INT32; write32(b+1, static_cast(v)); out.append(b, 5); } - else { uint8_t b[9]; b[0] = MP_INT64; write64(b+1, static_cast(v)); out.append(b, 9); } - } - } - return RC_OK; -} - -static int jp_parse_array(JsonParser& p, Buf& out) { - Buf tmp; uint32_t count = 0; - p.i++; /* skip '[' */ - jp_skip_ws(p); - while (p.i < p.n && p.z[p.i] != ']') { - if (count > 0) { - jp_skip_ws(p); - if (p.i >= p.n || p.z[p.i] != ',') return RC_ERROR; - p.i++; - } - jp_skip_ws(p); - if (jp_parse_value(p, tmp) != RC_OK) return RC_ERROR; - count++; - jp_skip_ws(p); - } - if (p.i >= p.n) return RC_ERROR; - p.i++; /* skip ']' */ - encode_array_header(out, count); - out.append(tmp.ptr(), tmp.size()); - return RC_OK; -} - -static int jp_parse_object(JsonParser& p, Buf& out) { - Buf tmp; uint32_t count = 0; - p.i++; /* skip '{' */ - jp_skip_ws(p); - while (p.i < p.n && p.z[p.i] != '}') { - if (count > 0) { - jp_skip_ws(p); - if (p.i >= p.n || p.z[p.i] != ',') return RC_ERROR; - p.i++; - } - jp_skip_ws(p); - if (p.i >= p.n || p.z[p.i] != '"') return RC_ERROR; - if (jp_parse_string(p, tmp) != RC_OK) return RC_ERROR; - jp_skip_ws(p); - if (p.i >= p.n || p.z[p.i] != ':') return RC_ERROR; - p.i++; - jp_skip_ws(p); - if (jp_parse_value(p, tmp) != RC_OK) return RC_ERROR; - count++; - jp_skip_ws(p); - } - if (p.i >= p.n) return RC_ERROR; - p.i++; /* skip '}' */ - encode_map_header(out, count); - out.append(tmp.ptr(), tmp.size()); - return RC_OK; -} - -static int jp_parse_value(JsonParser& p, Buf& out) { - jp_skip_ws(p); - if (p.i >= p.n) return RC_ERROR; - char c = p.z[p.i]; - if (c == 'n' && p.i + 4 <= p.n && std::memcmp(p.z + p.i, "null", 4) == 0) { - p.i += 4; out.append1(MP_NIL); return RC_OK; - } - if (c == 't' && p.i + 4 <= p.n && std::memcmp(p.z + p.i, "true", 4) == 0) { - p.i += 4; out.append1(MP_TRUE); return RC_OK; - } - if (c == 'f' && p.i + 5 <= p.n && std::memcmp(p.z + p.i, "false", 5) == 0) { - p.i += 5; out.append1(MP_FALSE); return RC_OK; - } - if (c == '"') return jp_parse_string(p, out); - if (c == '[') return jp_parse_array(p, out); - if (c == '{') return jp_parse_object(p, out); - if (c == '-' || (c >= '0' && c <= '9')) return jp_parse_number(p, out); - return RC_ERROR; -} - -/* ── Iteration internals ──────────────────────────────────────────── */ - -static void each_iter( - const uint8_t* a, uint32_t n, uint32_t iCont, - const std::string& zBase, std::vector& rows -) { - if (iCont >= n) return; - uint8_t b = a[iCont]; - bool isArr = false, isMap = false; - uint32_t count = 0, dataOff = 0; - - if (b >= 0x90 && b <= 0x9f) { isArr = true; count = b & 0x0f; dataOff = iCont + 1; } - else if (b == MP_ARRAY16 && iCont + 3 <= n) { isArr = true; count = read16(a + iCont + 1); dataOff = iCont + 3; } - else if (b == MP_ARRAY32 && iCont + 5 <= n) { isArr = true; count = read32(a + iCont + 1); dataOff = iCont + 5; } - else if (b >= 0x80 && b <= 0x8f) { isMap = true; count = b & 0x0f; dataOff = iCont + 1; } - else if (b == MP_MAP16 && iCont + 3 <= n) { isMap = true; count = read16(a + iCont + 1); dataOff = iCont + 3; } - else if (b == MP_MAP32 && iCont + 5 <= n) { isMap = true; count = read32(a + iCont + 1); dataOff = iCont + 5; } - - if (!isArr && !isMap) return; - - /* Sanity: reject counts that exceed remaining data capacity */ - uint32_t remaining = (dataOff <= n) ? (n - dataOff) : 0; - uint32_t minBytesPerElem = isMap ? 2u : 1u; - if (count > remaining / minBytesPerElem + 1) return; - - uint32_t cur = dataOff; - - for (uint32_t j = 0; j < count; j++) { - if (cur >= n) break; - if (isArr) { - uint32_t cEnd = skip_one(a, n, cur); - if (!cEnd) break; - EachRow row; - row.index = static_cast(j); - row.fullkey = zBase + "[" + std::to_string(j) + "]"; - row.path = zBase; - row.id = cur; - row.type = get_type(a, n, cur); - row.value = decode_element(a, n, cur, cEnd); - rows.push_back(std::move(row)); - cur = cEnd; - } else { - uint8_t kb = a[cur]; - const char* zKey = nullptr; uint32_t nKey = 0; - if (kb >= 0xa0 && kb <= 0xbf) { nKey = kb & 0x1f; zKey = reinterpret_cast(a + cur + 1); } - else if (kb == MP_STR8 && cur + 2 <= n) { nKey = a[cur + 1]; zKey = reinterpret_cast(a + cur + 2); } - else if (kb == MP_STR16 && cur + 3 <= n) { nKey = read16(a + cur + 1); zKey = reinterpret_cast(a + cur + 3); } - else if (kb == MP_STR32 && cur + 5 <= n) { nKey = read32(a + cur + 1); zKey = reinterpret_cast(a + cur + 5); } - uint32_t vOff = skip_one(a, n, cur); - if (!vOff) break; - uint32_t pEnd = skip_one(a, n, vOff); - if (!pEnd) break; - - EachRow row; - row.key = zKey ? std::string(zKey, nKey) : "?"; - row.index = static_cast(j); - row.fullkey = zBase + "." + row.key; - row.path = zBase; - row.id = vOff; - row.type = get_type(a, n, vOff); - row.value = decode_element(a, n, vOff, pEnd); - rows.push_back(std::move(row)); - cur = pEnd; - } - } -} - -static void tree_walk( - const uint8_t* a, uint32_t n, uint32_t iOff, - const std::string& zFull, const std::string& zParPath, - int depth, std::vector& rows -) { - if (depth > kMaxDepth || iOff >= n) return; - uint32_t iEnd = skip_one(a, n, iOff); - if (!iEnd) return; - - /* Yield this element */ - { - EachRow row; - row.fullkey = zFull; - row.path = zParPath; - row.id = iOff; - row.type = get_type(a, n, iOff); - row.value = decode_element(a, n, iOff, iEnd); - rows.push_back(std::move(row)); - } - - uint8_t b = a[iOff]; - bool isArr = false, isMap = false; - uint32_t count = 0, dataOff = 0; - - if (b >= 0x90 && b <= 0x9f) { isArr = true; count = b & 0x0f; dataOff = iOff + 1; } - else if (b == MP_ARRAY16 && iOff + 3 <= n) { isArr = true; count = read16(a + iOff + 1); dataOff = iOff + 3; } - else if (b == MP_ARRAY32 && iOff + 5 <= n) { isArr = true; count = read32(a + iOff + 1); dataOff = iOff + 5; } - else if (b >= 0x80 && b <= 0x8f) { isMap = true; count = b & 0x0f; dataOff = iOff + 1; } - else if (b == MP_MAP16 && iOff + 3 <= n) { isMap = true; count = read16(a + iOff + 1); dataOff = iOff + 3; } - else if (b == MP_MAP32 && iOff + 5 <= n) { isMap = true; count = read32(a + iOff + 1); dataOff = iOff + 5; } - - if (!isArr && !isMap) return; - - /* Sanity: reject counts that exceed remaining data capacity */ - uint32_t tRemaining = (dataOff <= n) ? (n - dataOff) : 0; - uint32_t tMinBytes = isMap ? 2u : 1u; - if (count > tRemaining / tMinBytes + 1) return; - - uint32_t cur = dataOff; - - for (uint32_t j = 0; j < count; j++) { - if (cur >= n) break; - if (isArr) { - uint32_t cEnd = skip_one(a, n, cur); if (!cEnd) break; - std::string childFull = zFull + "[" + std::to_string(j) + "]"; - tree_walk(a, n, cur, childFull, zFull, depth + 1, rows); - cur = cEnd; - } else { - uint8_t kb = a[cur]; - const char* zKey = nullptr; uint32_t nKey = 0; - if (kb >= 0xa0 && kb <= 0xbf) { nKey = kb & 0x1f; zKey = reinterpret_cast(a + cur + 1); } - else if (kb == MP_STR8 && cur + 2 <= n) { nKey = a[cur + 1]; zKey = reinterpret_cast(a + cur + 2); } - else if (kb == MP_STR16 && cur + 3 <= n) { nKey = read16(a + cur + 1); zKey = reinterpret_cast(a + cur + 3); } - else if (kb == MP_STR32 && cur + 5 <= n) { nKey = read32(a + cur + 1); zKey = reinterpret_cast(a + cur + 5); } - uint32_t vOff = skip_one(a, n, cur); if (!vOff) break; - uint32_t pEnd = skip_one(a, n, vOff); if (!pEnd) break; - std::string keyStr = zKey ? std::string(zKey, nKey) : "?"; - std::string childFull = zFull + "." + keyStr; - tree_walk(a, n, vOff, childFull, zFull, depth + 1, rows); - cur = pEnd; - } - } -} - -/* ══════════════════════════════════════════════════════════════════════ -** Public API implementations -** ══════════════════════════════════════════════════════════════════════ */ - -/* ── type_str ─────────────────────────────────────────────────────── */ - -const char* type_str(Type t) noexcept { - switch (t) { - case Type::Nil: return "null"; - case Type::True: return "true"; - case Type::False: return "false"; - case Type::Integer: return "integer"; - case Type::Real: return "real"; - case Type::Float32: return "float32"; - case Type::String: return "text"; - case Type::Binary: return "binary"; - case Type::Array: return "array"; - case Type::Map: return "map"; - case Type::Ext: return "ext"; - case Type::Timestamp: return "timestamp"; - } - return "null"; -} - -/* ── Value ────────────────────────────────────────────────────────── */ - -Value::Value() noexcept : type_(Type::Nil), i64_(0) {} - -Type Value::type() const noexcept { return type_; } -bool Value::is_nil() const noexcept { return type_ == Type::Nil; } - -bool Value::as_bool() const noexcept { - return type_ == Type::True; -} - -int64_t Value::as_int64() const noexcept { - if (type_ == Type::Integer) return i64_; - if (type_ == Type::Real) return static_cast(f64_); - if (type_ == Type::Float32) return static_cast(f32_); - if (type_ == Type::Timestamp) return i64_; - if (type_ == Type::True) return 1; - return 0; -} - -uint64_t Value::as_uint64() const noexcept { - if (type_ == Type::Integer) return u64_; - return 0; -} - -double Value::as_double() const noexcept { - if (type_ == Type::Real) return f64_; - if (type_ == Type::Float32) return static_cast(f32_); - if (type_ == Type::Integer) return static_cast(i64_); - return 0.0; -} - -float Value::as_float() const noexcept { - if (type_ == Type::Float32) return f32_; - if (type_ == Type::Real) return static_cast(f64_); - return 0.0f; -} - -int8_t Value::ext_type() const noexcept { return ext_type_; } - -int64_t Value::timestamp_seconds() const noexcept { - if (type_ == Type::Timestamp) return i64_; - return 0; -} - -uint32_t Value::timestamp_nanoseconds() const noexcept { - if (type_ == Type::Timestamp) return ts_nsec_; - return 0; -} - -IntWidth Value::int_width() const noexcept { return int_width_; } - -std::string_view Value::as_string() const noexcept { - if (type_ == Type::String) return str_; - return {}; -} - -const uint8_t* Value::blob_data() const noexcept { - return !owned_blob_.empty() ? owned_blob_.data() : blob_ptr_; -} -size_t Value::blob_size() const noexcept { return blob_len_; } - -Value Value::nil() { - Value v; v.type_ = Type::Nil; return v; -} - -Value Value::boolean(bool b) { - Value v; v.type_ = b ? Type::True : Type::False; return v; -} - -Value Value::integer(int64_t x) { - Value v; v.type_ = Type::Integer; v.i64_ = x; return v; -} - -Value Value::unsigned_integer(uint64_t x) { - Value v; v.type_ = Type::Integer; v.u64_ = x; - /* Values that don't fit in int64 must be encoded as unsigned to round-trip. */ - if (x > static_cast(INT64_MAX)) { - v.int_width_ = IntWidth::Uint64; - } - return v; -} - -Value Value::real(double d) { - Value v; v.type_ = Type::Real; v.f64_ = d; return v; -} - -Value Value::real32(float f) { - Value v; v.type_ = Type::Float32; v.f32_ = f; return v; -} - -Value Value::string(std::string_view s) { - Value v; v.type_ = Type::String; v.str_ = std::string(s); return v; -} - -Value Value::binary(const uint8_t* data, size_t len) { - Value v; - v.type_ = Type::Binary; - if (len > 0 && data) { - v.owned_blob_.assign(data, data + len); - v.blob_ptr_ = v.owned_blob_.data(); - } else { - v.blob_ptr_ = nullptr; - } - v.blob_len_ = len; - return v; -} - -Value Value::ext(int8_t type_code, const uint8_t* data, size_t len) { - Value v; - v.type_ = Type::Ext; - v.ext_type_ = type_code; - v.owned_blob_.assign(data, data + len); - v.blob_ptr_ = v.owned_blob_.data(); - v.blob_len_ = len; - return v; -} - -Value Value::timestamp(int64_t seconds) { - Value v; v.type_ = Type::Timestamp; v.i64_ = seconds; v.ts_nsec_ = 0; return v; -} - -Value Value::timestamp(int64_t seconds, uint32_t nanoseconds) { - Value v; v.type_ = Type::Timestamp; v.i64_ = seconds; v.ts_nsec_ = nanoseconds; return v; -} - -Value Value::int8(int8_t x) { - Value v; v.type_ = Type::Integer; v.i64_ = x; v.int_width_ = IntWidth::Int8; return v; -} - -Value Value::int16(int16_t x) { - Value v; v.type_ = Type::Integer; v.i64_ = x; v.int_width_ = IntWidth::Int16; return v; -} - -Value Value::int32(int32_t x) { - Value v; v.type_ = Type::Integer; v.i64_ = x; v.int_width_ = IntWidth::Int32; return v; -} - -Value Value::int64(int64_t x) { - Value v; v.type_ = Type::Integer; v.i64_ = x; v.int_width_ = IntWidth::Int64; return v; -} - -Value Value::uint8(uint8_t x) { - Value v; v.type_ = Type::Integer; v.u64_ = x; v.int_width_ = IntWidth::Uint8; return v; -} - -Value Value::uint16(uint16_t x) { - Value v; v.type_ = Type::Integer; v.u64_ = x; v.int_width_ = IntWidth::Uint16; return v; -} - -Value Value::uint32(uint32_t x) { - Value v; v.type_ = Type::Integer; v.u64_ = x; v.int_width_ = IntWidth::Uint32; return v; -} - -Value Value::uint64(uint64_t x) { - Value v; v.type_ = Type::Integer; v.u64_ = x; v.int_width_ = IntWidth::Uint64; return v; -} - -/* ── Blob ─────────────────────────────────────────────────────────── */ - -Blob::Blob() = default; - -Blob::Blob(const uint8_t* data, size_t size) - : data_(data, data + size) {} - -Blob::Blob(std::vector data) - : data_(std::move(data)) {} - -const uint8_t* Blob::data() const noexcept { return data_.data(); } -size_t Blob::size() const noexcept { return data_.size(); } -bool Blob::empty() const noexcept { return data_.empty(); } - -bool Blob::valid() const { - if (data_.empty()) return false; - return is_valid(data_.data(), static_cast(data_.size())); -} - -size_t Blob::error_position() const { - if (data_.empty()) return 0; - return error_position_of(data_.data(), static_cast(data_.size())); -} - -Type Blob::type() const { - if (data_.empty()) return Type::Nil; - return get_type(data_.data(), static_cast(data_.size()), 0); -} - -Type Blob::type(const char* path) const { - uint32_t iStart, iEnd; - int rc = lookup(data_.data(), static_cast(data_.size()), 0, path, &iStart, &iEnd); - if (rc != RC_OK) return Type::Nil; - return get_type(data_.data(), static_cast(data_.size()), iStart); -} - -const char* Blob::type_str() const { - if (data_.empty()) return "null"; - return get_type_str_at(data_.data(), static_cast(data_.size()), 0); -} - -const char* Blob::type_str(const char* path) const { - uint32_t iStart, iEnd; - int rc = lookup(data_.data(), static_cast(data_.size()), 0, path, &iStart, &iEnd); - if (rc != RC_OK) return "null"; - return get_type_str_at(data_.data(), static_cast(data_.size()), iStart); -} - -Value Blob::extract(const char* path) const { - uint32_t iStart, iEnd; - int rc = lookup(data_.data(), static_cast(data_.size()), 0, path, &iStart, &iEnd); - if (rc != RC_OK) return Value::nil(); - return decode_element(data_.data(), static_cast(data_.size()), iStart, iEnd); -} - -int64_t Blob::array_length() const { - if (data_.empty()) return -1; - return get_container_count(data_.data(), static_cast(data_.size()), 0); -} - -int64_t Blob::array_length(const char* path) const { - uint32_t iStart, iEnd; - int rc = lookup(data_.data(), static_cast(data_.size()), 0, path, &iStart, &iEnd); - if (rc != RC_OK) return -1; - return get_container_count(data_.data(), static_cast(data_.size()), iStart); -} - -/* Mutation helpers */ - -static Blob apply_mutation(const Blob& blob, const char* path, const Value& val, int mode) { - Builder enc; - enc.value(val); - Buf out; - int rc = apply_edit(out, blob.data(), static_cast(blob.size()), - path, enc.buf_data(), static_cast(enc.buf_size()), mode); - if (rc != RC_OK) return blob; - return Blob(std::move(out.data)); -} - -Blob Blob::set(const char* path, const Value& val) const { - return apply_mutation(*this, path, val, EDIT_SET); -} - -Blob Blob::set(const char* path, const Blob& sub) const { - Buf out; - int rc = apply_edit(out, data_.data(), static_cast(data_.size()), - path, sub.data(), static_cast(sub.size()), EDIT_SET); - if (rc != RC_OK) return *this; - return Blob(std::move(out.data)); -} - -Blob Blob::insert(const char* path, const Value& val) const { - return apply_mutation(*this, path, val, EDIT_INSERT); -} - -Blob Blob::replace(const char* path, const Value& val) const { - return apply_mutation(*this, path, val, EDIT_REPLACE); -} - -Blob Blob::remove(const char* path) const { - Buf out; - int rc = apply_edit(out, data_.data(), static_cast(data_.size()), - path, nullptr, 0, EDIT_REMOVE); - if (rc != RC_OK) return *this; - return Blob(std::move(out.data)); -} - -Blob Blob::array_insert(const char* path, const Value& val) const { - return apply_mutation(*this, path, val, EDIT_ARRAY_INS); -} - -Blob Blob::patch(const Blob& mp) const { - Buf out; - int rc = merge_patch(out, - data_.data(), static_cast(data_.size()), 0, - mp.data(), static_cast(mp.size()), 0, 0); - if (rc != RC_OK) return *this; - return Blob(std::move(out.data)); -} - -std::string Blob::to_json() const { - if (data_.empty()) return "null"; - Buf out; - to_json_at(out, data_.data(), static_cast(data_.size()), 0, false, 0, 0); - return std::string(reinterpret_cast(out.ptr()), out.size()); -} - -std::string Blob::to_json_pretty(int indent) const { - if (data_.empty()) return "null"; - if (indent < 0) indent = 0; - if (indent > 8) indent = 8; - Buf out; - to_json_at(out, data_.data(), static_cast(data_.size()), 0, true, 0, indent); - return std::string(reinterpret_cast(out.ptr()), out.size()); -} - -Blob Blob::from_json(const char* json) { - if (!json) return Blob(); - JsonParser p{json, static_cast(std::strlen(json)), 0}; - Buf out; - if (jp_parse_value(p, out) != RC_OK) return Blob(); - return Blob(std::move(out.data)); -} - -Blob Blob::from_json(const std::string& json) { - return from_json(json.c_str()); -} - -/* ── Builder ──────────────────────────────────────────────────────── */ - -Builder::Builder() = default; - -void Builder::append(const uint8_t* data, size_t n) { - buf_.insert(buf_.end(), data, data + n); -} -void Builder::append1(uint8_t b) { buf_.push_back(b); } -uint8_t* Builder::reserve(size_t n) { - size_t old = buf_.size(); - buf_.resize(old + n); - return buf_.data() + old; -} - -Builder& Builder::nil() { append1(MP_NIL); return *this; } - -Builder& Builder::boolean(bool v) { - append1(v ? MP_TRUE : MP_FALSE); - return *this; -} - -Builder& Builder::integer(int64_t x) { - if (x >= 0) { - if (x <= 0x7f) { - append1(static_cast(x)); - } else if (x <= 0xff) { - uint8_t b[2] = {MP_UINT8, static_cast(x)}; - append(b, 2); - } else if (x <= 0xffff) { - uint8_t b[3]; b[0] = MP_UINT16; write16(b + 1, static_cast(x)); - append(b, 3); - } else if (x <= static_cast(0xffffffff)) { - uint8_t b[5]; b[0] = MP_UINT32; write32(b + 1, static_cast(x)); - append(b, 5); - } else { - uint8_t b[9]; b[0] = MP_UINT64; write64(b + 1, static_cast(x)); - append(b, 9); - } - } else { - if (x >= -32) { - append1(static_cast(x)); - } else if (x >= -128) { - uint8_t b[2] = {MP_INT8, static_cast(x)}; - append(b, 2); - } else if (x >= -32768) { - uint8_t b[3]; b[0] = MP_INT16; write16(b + 1, static_cast(x)); - append(b, 3); - } else if (x >= static_cast(-2147483648LL)) { - uint8_t b[5]; b[0] = MP_INT32; write32(b + 1, static_cast(x)); - append(b, 5); - } else { - uint8_t b[9]; b[0] = MP_INT64; write64(b + 1, static_cast(x)); - append(b, 9); - } - } - return *this; -} - -Builder& Builder::unsigned_integer(uint64_t x) { - if (x <= 0x7f) { - append1(static_cast(x)); - } else if (x <= 0xff) { - uint8_t b[2] = {MP_UINT8, static_cast(x)}; - append(b, 2); - } else if (x <= 0xffff) { - uint8_t b[3]; b[0] = MP_UINT16; write16(b + 1, static_cast(x)); - append(b, 3); - } else if (x <= 0xffffffff) { - uint8_t b[5]; b[0] = MP_UINT32; write32(b + 1, static_cast(x)); - append(b, 5); - } else { - uint8_t b[9]; b[0] = MP_UINT64; write64(b + 1, x); - append(b, 9); - } - return *this; -} - -Builder& Builder::real(double d) { - uint8_t b[9]; uint64_t bits; - b[0] = MP_FLOAT64; - std::memcpy(&bits, &d, 8); - write64(b + 1, bits); - append(b, 9); - return *this; -} - -Builder& Builder::real32(float f) { - uint8_t b[5]; uint32_t bits; - b[0] = MP_FLOAT32; - std::memcpy(&bits, &f, 4); - write32(b + 1, bits); - append(b, 5); - return *this; -} - -Builder& Builder::string(std::string_view s) { - auto len = static_cast(s.size()); - if (len <= 31) { - append1(static_cast(MP_FIXSTR_MASK | len)); - } else if (len <= 0xff) { - uint8_t h[2] = {MP_STR8, static_cast(len)}; - append(h, 2); - } else if (len <= 0xffff) { - uint8_t h[3]; h[0] = MP_STR16; write16(h + 1, static_cast(len)); - append(h, 3); - } else { - uint8_t h[5]; h[0] = MP_STR32; write32(h + 1, len); - append(h, 5); - } - append(reinterpret_cast(s.data()), len); - return *this; -} - -Builder& Builder::binary(const uint8_t* data, size_t len) { - auto n = static_cast(len); - if (n <= 0xff) { - uint8_t h[2] = {MP_BIN8, static_cast(n)}; - append(h, 2); - } else if (n <= 0xffff) { - uint8_t h[3]; h[0] = MP_BIN16; write16(h + 1, static_cast(n)); - append(h, 3); - } else { - uint8_t h[5]; h[0] = MP_BIN32; write32(h + 1, n); - append(h, 5); - } - if (data) append(data, n); - return *this; -} - -Builder& Builder::ext(int8_t type_code, const uint8_t* data, size_t len) { - auto n = static_cast(len); - switch (n) { - case 1: append1(MP_FIXEXT1); break; - case 2: append1(MP_FIXEXT2); break; - case 4: append1(MP_FIXEXT4); break; - case 8: append1(MP_FIXEXT8); break; - case 16: append1(MP_FIXEXT16); break; - default: - if (n <= 0xff) { - uint8_t h[2] = {MP_EXT8, static_cast(n)}; - append(h, 2); - } else if (n <= 0xffff) { - uint8_t h[3]; h[0] = MP_EXT16; write16(h + 1, static_cast(n)); - append(h, 3); - } else { - uint8_t h[5]; h[0] = MP_EXT32; write32(h + 1, n); - append(h, 5); - } - break; - } - append1(static_cast(type_code)); - if (data) append(data, n); - return *this; -} - -Builder& Builder::int8(int8_t x) { - uint8_t b[2] = {MP_INT8, static_cast(x)}; - append(b, 2); return *this; -} - -Builder& Builder::int16(int16_t x) { - uint8_t b[3]; b[0] = MP_INT16; write16(b + 1, static_cast(x)); - append(b, 3); return *this; -} - -Builder& Builder::int32(int32_t x) { - uint8_t b[5]; b[0] = MP_INT32; write32(b + 1, static_cast(x)); - append(b, 5); return *this; -} - -Builder& Builder::int64(int64_t x) { - uint8_t b[9]; b[0] = MP_INT64; write64(b + 1, static_cast(x)); - append(b, 9); return *this; -} - -Builder& Builder::uint8(uint8_t x) { - uint8_t b[2] = {MP_UINT8, x}; - append(b, 2); return *this; -} - -Builder& Builder::uint16(uint16_t x) { - uint8_t b[3]; b[0] = MP_UINT16; write16(b + 1, x); - append(b, 3); return *this; -} - -Builder& Builder::uint32(uint32_t x) { - uint8_t b[5]; b[0] = MP_UINT32; write32(b + 1, x); - append(b, 5); return *this; -} - -Builder& Builder::uint64(uint64_t x) { - uint8_t b[9]; b[0] = MP_UINT64; write64(b + 1, x); - append(b, 9); return *this; -} - -Builder& Builder::array_header(uint32_t count) { - if (count <= 15) { - append1(static_cast(MP_FIXARRAY_MASK | count)); - } else if (count <= 0xffff) { - uint8_t h[3]; h[0] = MP_ARRAY16; write16(h + 1, static_cast(count)); - append(h, 3); - } else { - uint8_t h[5]; h[0] = MP_ARRAY32; write32(h + 1, count); - append(h, 5); - } - return *this; -} - -Builder& Builder::map_header(uint32_t count) { - if (count <= 15) { - append1(static_cast(MP_FIXMAP_MASK | count)); - } else if (count <= 0xffff) { - uint8_t h[3]; h[0] = MP_MAP16; write16(h + 1, static_cast(count)); - append(h, 3); - } else { - uint8_t h[5]; h[0] = MP_MAP32; write32(h + 1, count); - append(h, 5); - } - return *this; -} - -Builder& Builder::raw(const uint8_t* data, size_t len) { - append(data, len); - return *this; -} - -Builder& Builder::raw(const Blob& blob) { - append(blob.data(), blob.size()); - return *this; -} - -Builder& Builder::timestamp(int64_t sec) { - return timestamp(sec, 0); -} - -Builder& Builder::timestamp(int64_t sec, uint32_t nsec) { - if (nsec == 0 && sec >= 0 && sec <= static_cast(0xFFFFFFFFLL)) { - uint8_t b[6]; b[0] = MP_FIXEXT4; b[1] = 0xFF; - write32(b + 2, static_cast(sec)); - append(b, 6); - } else if (sec >= 0 && sec <= static_cast(0x3FFFFFFFFLL)) { - uint8_t b[10]; b[0] = MP_FIXEXT8; b[1] = 0xFF; - uint64_t v64 = (static_cast(nsec) << 34) | static_cast(sec); - write64(b + 2, v64); - append(b, 10); - } else { - uint8_t b[15]; b[0] = MP_EXT8; b[1] = 12; b[2] = 0xFF; - write32(b + 3, nsec); - write64(b + 7, static_cast(sec)); - append(b, 15); - } - return *this; -} - -const uint8_t* Builder::buf_data() const noexcept { return buf_.data(); } -size_t Builder::buf_size() const noexcept { return buf_.size(); } - -Builder& Builder::value(const Value& v) { - switch (v.type()) { - case Type::Nil: return nil(); - case Type::True: return boolean(true); - case Type::False: return boolean(false); - case Type::Integer: { - IntWidth w = v.int_width(); - switch (w) { - case IntWidth::Int8: return int8(static_cast(v.as_int64())); - case IntWidth::Int16: return int16(static_cast(v.as_int64())); - case IntWidth::Int32: return int32(static_cast(v.as_int64())); - case IntWidth::Int64: return int64(v.as_int64()); - case IntWidth::Uint8: return uint8(static_cast(v.as_uint64())); - case IntWidth::Uint16: return uint16(static_cast(v.as_uint64())); - case IntWidth::Uint32: return uint32(static_cast(v.as_uint64())); - case IntWidth::Uint64: return uint64(v.as_uint64()); - case IntWidth::Auto: break; - } - return integer(v.as_int64()); - } - case Type::Real: return real(v.as_double()); - case Type::Float32: return real32(v.as_float()); - case Type::String: return string(v.as_string()); - case Type::Binary: return binary(v.blob_data(), v.blob_size()); - case Type::Ext: return ext(v.ext_type(), v.blob_data(), v.blob_size()); - case Type::Timestamp: return timestamp(v.timestamp_seconds(), v.timestamp_nanoseconds()); - default: return nil(); - } -} - -Blob Builder::build() { - return Blob(std::move(buf_)); -} - -Blob Builder::quote(const Value& v) { - Builder b; - b.value(v); - return b.build(); -} - -/* ── Iterator ─────────────────────────────────────────────────────── */ - -Iterator::Iterator(const Blob& blob, const char* path, bool recursive) - : blob_(blob), base_path_(path ? path : "$"), - recursive_(recursive), cursor_(-1), populated_(false) {} - -void Iterator::populate() { - if (populated_) return; - populated_ = true; - rows_.clear(); - - const uint8_t* a = blob_.data(); - auto n = static_cast(blob_.size()); - if (!a || n == 0) return; - - uint32_t iRoot = 0; - std::string zBase = base_path_; - - if (base_path_ != "$") { - uint32_t iStart, iEnd; - if (lookup(a, n, 0, base_path_.c_str(), &iStart, &iEnd) == RC_OK) { - iRoot = iStart; - } else { - return; - } - } - - if (recursive_) { - tree_walk(a, n, iRoot, zBase, zBase, 0, rows_); - } else { - each_iter(a, n, iRoot, zBase, rows_); - } -} - -bool Iterator::next() { - populate(); - cursor_++; - return cursor_ < static_cast(rows_.size()); -} - -const EachRow& Iterator::current() const { - return rows_[static_cast(cursor_)]; -} - -void Iterator::reset() { - cursor_ = -1; -} - -} /* namespace msgpack */ diff --git a/tests/vectors/blob_vectors.json b/tests/vectors/blob_vectors.json new file mode 100644 index 0000000..6b25ecb --- /dev/null +++ b/tests/vectors/blob_vectors.json @@ -0,0 +1,211 @@ +{ + "from_json":[ + {"json":"null","hex":"c0"}, + {"json":"true","hex":"c3"}, + {"json":"false","hex":"c2"}, + {"json":"0","hex":"00"}, + {"json":"1","hex":"01"}, + {"json":"127","hex":"7f"}, + {"json":"128","hex":"cc80"}, + {"json":"255","hex":"ccff"}, + {"json":"256","hex":"cd0100"}, + {"json":"65535","hex":"cdffff"}, + {"json":"65536","hex":"ce00010000"}, + {"json":"4294967295","hex":"ceffffffff"}, + {"json":"4294967296","hex":"cf0000000100000000"}, + {"json":"9007199254740991","hex":"cf001fffffffffffff"}, + {"json":"-1","hex":"ff"}, + {"json":"-32","hex":"e0"}, + {"json":"-33","hex":"d0df"}, + {"json":"-128","hex":"d080"}, + {"json":"-129","hex":"d1ff7f"}, + {"json":"-32768","hex":"d18000"}, + {"json":"-32769","hex":"d2ffff7fff"}, + {"json":"-2147483648","hex":"d280000000"}, + {"json":"-2147483649","hex":"d3ffffffff7fffffff"}, + {"json":"1.5","hex":"cb3ff8000000000000"}, + {"json":"0.1","hex":"cb3fb999999999999a"}, + {"json":"-0.0","hex":"cb8000000000000000"}, + {"json":"3.0","hex":"cb4008000000000000"}, + {"json":"1e10","hex":"cb4202a05f20000000"}, + {"json":"1.25e-3","hex":"cb3f547ae147ae147b"}, + {"json":"95.5","hex":"cb4057e00000000000"}, + {"json":"1e-7","hex":"cb3e7ad7f29abcaf48"}, + {"json":"1e-4","hex":"cb3f1a36e2eb1c432d"}, + {"json":"1e-5","hex":"cb3ee4f8b588e368f1"}, + {"json":"1e20","hex":"cb4415af1d78b58c40"}, + {"json":"1e21","hex":"cb444b1ae4d6e2ef50"}, + {"json":"1e22","hex":"cb4480f0cf064dd592"}, + {"json":"9.999999e-8","hex":"cb3e7ad7f26db3783b"}, + {"json":"0.30000000000000004","hex":"cb3fd3333333333334"}, + {"json":"123456789012345680000","hex":"cf7fffffffffffffff"}, + {"json":"\"\"","hex":"a0"}, + {"json":"\"hello\"","hex":"a568656c6c6f"}, + {"json":"\"a string longer than thirty-one chars!!\"","hex":"d9276120737472696e67206c6f6e676572207468616e207468697274792d6f6e652063686172732121"}, + {"json":"\"unicode: \\u00e9\\u4e2d\\ud83d\\ude00\"","hex":"b2756e69636f64653a20c3a9e4b8adf09f9880"}, + {"json":"[]","hex":"90"}, + {"json":"{}","hex":"80"}, + {"json":"[1,2,3]","hex":"93010203"}, + {"json":"{\"a\":1,\"b\":2}","hex":"82a16101a16202"}, + {"json":"{\"name\":\"Alice\",\"age\":30,\"scores\":[95,87,91]}","hex":"83a46e616d65a5416c696365a36167651ea673636f726573935f575b"}, + {"json":"[[1,2],[3,4],{\"x\":[true,null,false]}]","hex":"9392010292030481a17893c3c0c2"}, + {"json":"{\"nested\":{\"deep\":{\"value\":42}}}","hex":"81a66e657374656481a46465657081a576616c75652a"}], + "to_json":[ + {"hex":"c0","json":"null"}, + {"hex":"c3","json":"true"}, + {"hex":"c2","json":"false"}, + {"hex":"00","json":"0"}, + {"hex":"01","json":"1"}, + {"hex":"7f","json":"127"}, + {"hex":"cc80","json":"128"}, + {"hex":"ccff","json":"255"}, + {"hex":"cd0100","json":"256"}, + {"hex":"cdffff","json":"65535"}, + {"hex":"ce00010000","json":"65536"}, + {"hex":"ceffffffff","json":"4294967295"}, + {"hex":"cf0000000100000000","json":"4294967296"}, + {"hex":"cf001fffffffffffff","json":"9007199254740991"}, + {"hex":"ff","json":"-1"}, + {"hex":"e0","json":"-32"}, + {"hex":"d0df","json":"-33"}, + {"hex":"d080","json":"-128"}, + {"hex":"d1ff7f","json":"-129"}, + {"hex":"d18000","json":"-32768"}, + {"hex":"d2ffff7fff","json":"-32769"}, + {"hex":"d280000000","json":"-2147483648"}, + {"hex":"d3ffffffff7fffffff","json":"-2147483649"}, + {"hex":"cb3ff8000000000000","json":"1.5"}, + {"hex":"cb3fb999999999999a","json":"0.10000000000000001"}, + {"hex":"cb8000000000000000","json":"-0.0"}, + {"hex":"cb4008000000000000","json":"3.0"}, + {"hex":"cb4202a05f20000000","json":"10000000000.0"}, + {"hex":"cb3f547ae147ae147b","json":"0.00125"}, + {"hex":"cb4057e00000000000","json":"95.5"}, + {"hex":"cb3e7ad7f29abcaf48","json":"9.9999999999999995e-08"}, + {"hex":"cb3f1a36e2eb1c432d","json":"0.0001"}, + {"hex":"cb3ee4f8b588e368f1","json":"1.0000000000000001e-05"}, + {"hex":"cb4415af1d78b58c40","json":"1e+20"}, + {"hex":"cb444b1ae4d6e2ef50","json":"1e+21"}, + {"hex":"cb4480f0cf064dd592","json":"1e+22"}, + {"hex":"cb3e7ad7f26db3783b","json":"9.9999989999999999e-08"}, + {"hex":"cb3fd3333333333334","json":"0.30000000000000004"}, + {"hex":"cf7fffffffffffffff","json":"9223372036854775807"}, + {"hex":"a0","json":"\"\""}, + {"hex":"a568656c6c6f","json":"\"hello\""}, + {"hex":"d9276120737472696e67206c6f6e676572207468616e207468697274792d6f6e652063686172732121","json":"\"a string longer than thirty-one chars!!\""}, + {"hex":"b2756e69636f64653a20c3a9e4b8adf09f9880","json":"\"unicode: é中😀\""}, + {"hex":"90","json":"[]"}, + {"hex":"80","json":"{}"}, + {"hex":"93010203","json":"[1,2,3]"}, + {"hex":"82a16101a16202","json":"{\"a\":1,\"b\":2}"}, + {"hex":"83a46e616d65a5416c696365a36167651ea673636f726573935f575b","json":"{\"name\":\"Alice\",\"age\":30,\"scores\":[95,87,91]}"}, + {"hex":"9392010292030481a17893c3c0c2","json":"[[1,2],[3,4],{\"x\":[true,null,false]}]"}, + {"hex":"81a66e657374656481a46465657081a576616c75652a","json":"{\"nested\":{\"deep\":{\"value\":42}}}"}, + {"hex":"ca3f800000","json":"1"}, + {"hex":"ca40490fdb","json":"3.141593"}, + {"hex":"cb3ff0000000000000","json":"1.0"}, + {"hex":"cb3fb999999999999a","json":"0.10000000000000001"}, + {"hex":"c403abcdef","json":"\"abcdef\""}, + {"hex":"d6ff0102","json":"null"}, + {"hex":"d40102","json":"null"}, + {"hex":"92c2c3","json":"[false,true]"}, + {"hex":"81a16382","json":"{\"c\":{}}"}], + "to_json_pretty":[ + {"hex":"82a16101a162920203","indent":2,"json":"{\n \"a\": 1,\n \"b\": [\n 2,\n 3\n ]\n}"}, + {"hex":"920192029103","indent":4,"json":"[\n 1,\n [\n 2,\n [\n 3\n ]\n ]\n]"}, + {"hex":"80","indent":2,"json":"{}"}, + {"hex":"90","indent":2,"json":"[]"}, + {"hex":"81a17881a17901","indent":0,"json":"{\n\"x\": {\n\"y\": 1\n}\n}"}], + "typed":[ + {"spec":{"k":"nil"},"hex":"c0"}, + {"spec":{"k":"bool","v":true},"hex":"c3"}, + {"spec":{"k":"bool","v":false},"hex":"c2"}, + {"spec":{"k":"int","v":"0"},"hex":"00"}, + {"spec":{"k":"int","v":"127"},"hex":"7f"}, + {"spec":{"k":"int","v":"128"},"hex":"cc80"}, + {"spec":{"k":"int","v":"255"},"hex":"ccff"}, + {"spec":{"k":"int","v":"256"},"hex":"cd0100"}, + {"spec":{"k":"int","v":"65536"},"hex":"ce00010000"}, + {"spec":{"k":"int","v":"4294967296"},"hex":"cf0000000100000000"}, + {"spec":{"k":"int","v":"-1"},"hex":"ff"}, + {"spec":{"k":"int","v":"-32"},"hex":"e0"}, + {"spec":{"k":"int","v":"-33"},"hex":"d0df"}, + {"spec":{"k":"int","v":"-128"},"hex":"d080"}, + {"spec":{"k":"int","v":"-129"},"hex":"d1ff7f"}, + {"spec":{"k":"int","v":"-32769"},"hex":"d2ffff7fff"}, + {"spec":{"k":"uint","v":"0"},"hex":"00"}, + {"spec":{"k":"uint","v":"255"},"hex":"ccff"}, + {"spec":{"k":"uint","v":"18446744073709551615"},"hex":"cfffffffffffffffff"}, + {"spec":{"k":"int8","v":"-5"},"hex":"d0fb"}, + {"spec":{"k":"int16","v":"500"},"hex":"d101f4"}, + {"spec":{"k":"int32","v":"-70000"},"hex":"d2fffeee90"}, + {"spec":{"k":"int64","v":"-5"},"hex":"d3fffffffffffffffb"}, + {"spec":{"k":"uint8","v":"200"},"hex":"ccc8"}, + {"spec":{"k":"uint16","v":"60000"},"hex":"cdea60"}, + {"spec":{"k":"uint32","v":"4000000000"},"hex":"ceee6b2800"}, + {"spec":{"k":"uint64","v":"42"},"hex":"cf000000000000002a"}, + {"spec":{"k":"real","v":1.5},"hex":"cb3ff8000000000000"}, + {"spec":{"k":"real","v":0.1},"hex":"cb3fb999999999999a"}, + {"spec":{"k":"real","v":3.0},"hex":"cb4008000000000000"}, + {"spec":{"k":"real32","v":1.5},"hex":"ca3fc00000"}, + {"spec":{"k":"real32","v":0.5},"hex":"ca3f000000"}, + {"spec":{"k":"str","v":""},"hex":"a0"}, + {"spec":{"k":"str","v":"hello"},"hex":"a568656c6c6f"}, + {"spec":{"k":"str","v":"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"},"hex":"d92878787878787878787878787878787878787878787878787878787878787878787878787878787878"}, + {"spec":{"k":"binary","hex":"deadbeef"},"hex":"c404deadbeef"}, + {"spec":{"k":"binary","hex":""},"hex":"c400"}, + {"spec":{"k":"ext","type":42,"hex":"0102"},"hex":"d52a0102"}, + {"spec":{"k":"ext","type":1,"hex":"00"},"hex":"d40100"}, + {"spec":{"k":"ext","type":7,"hex":"0102030405"},"hex":"c705070102030405"}, + {"spec":{"k":"timestamp","sec":"0","nsec":0},"hex":"d6ff00000000"}, + {"spec":{"k":"timestamp","sec":"1700000000","nsec":0},"hex":"d6ff6553f100"}, + {"spec":{"k":"timestamp","sec":"1700000000","nsec":500000000},"hex":"d7ff773594006553f100"}, + {"spec":{"k":"timestamp","sec":"17000000000","nsec":0},"hex":"d7ff00000003f5476a00"}, + {"spec":{"k":"timestamp","sec":"-1","nsec":0},"hex":"c70cff00000000ffffffffffffffff"}], + "mutate":[ + {"base":"{\"a\":1}","op":"set","path":"$.b","spec":{"k":"int","v":"2"},"hex":"82a16101a16202"}, + {"base":"{\"a\":1}","op":"set","path":"$.a","spec":{"k":"int","v":"99"},"hex":"81a16163"}, + {"base":"{\"a\":1}","op":"set","path":"$.a","spec":{"k":"int16","v":"1000"},"hex":"81a161d103e8"}, + {"base":"{\"x\":0}","op":"set","path":"$.created","spec":{"k":"timestamp","sec":"1700000000","nsec":0},"hex":"82a17800a763726561746564d6ff6553f100"}, + {"base":"{\"a\":1}","op":"insert","path":"$.b","spec":{"k":"str","v":"new"},"hex":"82a16101a162a36e6577"}, + {"base":"{\"a\":1}","op":"insert","path":"$.a","spec":{"k":"int","v":"5"},"hex":"81a16101"}, + {"base":"{\"a\":1,\"b\":2}","op":"replace","path":"$.a","spec":{"k":"real","v":2.5},"hex":"82a161cb4004000000000000a16202"}, + {"base":"{\"a\":1}","op":"replace","path":"$.zzz","spec":{"k":"int","v":"9"},"hex":"81a16101"}, + {"base":"[1,2,3]","op":"set","path":"$[1]","spec":{"k":"int","v":"20"},"hex":"93011403"}, + {"base":"[1,2,3]","op":"set","path":"$[3]","spec":{"k":"int","v":"4"},"hex":"9401020304"}, + {"base":"[1,2,3]","op":"array_insert","path":"$[1]","spec":{"k":"int","v":"99"},"hex":"9401630203"}, + {"base":"[1,2,3]","op":"array_insert","path":"$[0]","spec":{"k":"str","v":"head"},"hex":"94a468656164010203"}, + {"base":"{\"a\":1,\"b\":2}","op":"remove","path":"$.a","hex":"81a16202"}, + {"base":"{\"a\":1,\"b\":2}","op":"remove","path":"$.b","hex":"81a16101"}, + {"base":"[1,2,3]","op":"remove","path":"$[1]","hex":"920103"}, + {"base":"{\"a\":{\"b\":1,\"c\":2}}","op":"remove","path":"$.a.b","hex":"81a16181a16302"}, + {"base":"{\"a\":1}","op":"set_blob","path":"$.b","spec":{"k":"blob","json":"[1,2,3]"},"hex":"82a16101a16293010203"}, + {"base":"{\"a\":1,\"b\":2}","op":"patch","patch":"{\"a\":9}","hex":"82a16109a16202"}, + {"base":"{\"a\":1,\"b\":2}","op":"patch","patch":"{\"b\":null}","hex":"81a16101"}, + {"base":"{\"a\":1}","op":"patch","patch":"{\"c\":3}","hex":"82a16101a16303"}, + {"base":"{\"a\":{\"x\":1,\"y\":2}}","op":"patch","patch":"{\"a\":{\"y\":null,\"z\":3}}","hex":"81a16182a17801a17a03"}, + {"base":"{\"a\":1}","op":"patch","patch":"{\"a\":{\"nested\":true}}","hex":"81a16181a66e6573746564c3"}], + "extract":[ + {"base":"{\"name\":\"Alice\",\"age\":30}","path":"$.name","type":"text","vjson":"\"Alice\""}, + {"base":"{\"name\":\"Alice\",\"age\":30}","path":"$.age","type":"integer","vjson":"30"}, + {"base":"{\"a\":[10,20,30]}","path":"$.a[1]","type":"integer","vjson":"20"}, + {"base":"{\"a\":[10,20,30]}","path":"$.a","type":"array","vjson":"\"930a141e\""}, + {"base":"{\"a\":1}","path":"$.missing","type":"null","vjson":"null"}, + {"base":"{\"f\":1.5}","path":"$.f","type":"real","vjson":"1.5"}, + {"base":"{\"b\":true,\"n\":null}","path":"$.b","type":"true","vjson":"true"}, + {"base":"{\"b\":true,\"n\":null}","path":"$.n","type":"null","vjson":"null"}, + {"base":"[{\"x\":1}]","path":"$[0].x","type":"integer","vjson":"1"}], + "array_length":[ + {"base":"[1,2,3]","path":"$","len":3}, + {"base":"{\"a\":[1,2,3,4]}","path":"$.a","len":4}, + {"base":"{\"a\":1}","path":"$","len":1}, + {"base":"{\"m\":{\"x\":1,\"y\":2}}","path":"$.m","len":2}, + {"base":"[]","path":"$","len":0}], + "iterate":[ + {"base":"{\"a\":1,\"b\":2,\"c\":3}","path":"$","recursive":false,"rows":[{"key":"a","index":0,"fullkey":"$.a","path":"$","id":3,"type":"integer"},{"key":"b","index":1,"fullkey":"$.b","path":"$","id":6,"type":"integer"},{"key":"c","index":2,"fullkey":"$.c","path":"$","id":9,"type":"integer"}]}, + {"base":"[10,20,30]","path":"$","recursive":false,"rows":[{"key":"","index":0,"fullkey":"$[0]","path":"$","id":1,"type":"integer"},{"key":"","index":1,"fullkey":"$[1]","path":"$","id":2,"type":"integer"},{"key":"","index":2,"fullkey":"$[2]","path":"$","id":3,"type":"integer"}]}, + {"base":"{\"x\":{\"y\":[1,2,3]}}","path":"$","recursive":true,"rows":[{"fullkey":"$","path":"$","id":0,"type":"map"},{"fullkey":"$.x","path":"$","id":3,"type":"map"},{"fullkey":"$.x.y","path":"$.x","id":6,"type":"array"},{"fullkey":"$.x.y[0]","path":"$.x.y","id":7,"type":"integer"},{"fullkey":"$.x.y[1]","path":"$.x.y","id":8,"type":"integer"},{"fullkey":"$.x.y[2]","path":"$.x.y","id":9,"type":"integer"}]}, + {"base":"{\"users\":[{\"name\":\"A\"},{\"name\":\"B\"}]}","path":"$","recursive":true,"rows":[{"fullkey":"$","path":"$","id":0,"type":"map"},{"fullkey":"$.users","path":"$","id":7,"type":"array"},{"fullkey":"$.users[0]","path":"$.users","id":8,"type":"map"},{"fullkey":"$.users[0].name","path":"$.users[0]","id":14,"type":"text"},{"fullkey":"$.users[1]","path":"$.users","id":16,"type":"map"},{"fullkey":"$.users[1].name","path":"$.users[1]","id":22,"type":"text"}]}, + {"base":"{\"users\":[{\"name\":\"A\"},{\"name\":\"B\"}]}","path":"$.users","recursive":false,"rows":[{"key":"","index":0,"fullkey":"$.users[0]","path":"$.users","id":8,"type":"map"},{"key":"","index":1,"fullkey":"$.users[1]","path":"$.users","id":16,"type":"map"}]}, + {"base":"[1,[2,3],4]","path":"$","recursive":true,"rows":[{"fullkey":"$","path":"$","id":0,"type":"array"},{"fullkey":"$[0]","path":"$","id":1,"type":"integer"},{"fullkey":"$[1]","path":"$","id":2,"type":"array"},{"fullkey":"$[1][0]","path":"$[1]","id":3,"type":"integer"},{"fullkey":"$[1][1]","path":"$[1]","id":4,"type":"integer"},{"fullkey":"$[2]","path":"$","id":5,"type":"integer"}]}] +}