From 5203abfe4b216e8ddb1f2ced48ce852680d64bee Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Thu, 16 Jul 2026 12:10:04 +0200 Subject: [PATCH 01/16] Refactor CMerkleTree account map for lazy clone and NIF-owned state root. Speculative paths use clone_lazy; Chain.State drops :store in favor of an internal state_trie with map-level freeze and native difference/apply. Co-authored-by: Cursor --- AGENTS.md | 15 +- c_src/LOCK_ORDER.md | 9 +- c_src/SECURITY_REVIEW.md | 11 +- c_src/nif.cpp | 759 ++++++++++++++++++++++--- lib/caccount_map.ex | 34 +- lib/chain/account.ex | 1 + lib/chain/block.ex | 2 +- lib/chain/state.ex | 225 ++------ lib/cmerkletree.ex | 31 +- lib/network/edge_v2.ex | 2 +- lib/network/rpc.ex | 2 +- lib/network/status.ex | 2 +- lib/shell.ex | 2 +- scripts/cmerkle_fuzz.exs | 55 +- scripts/cmerkle_heap_assumptions.exs | 6 +- scripts/cmerkle_leak_test.exs | 2 +- scripts/cmerkle_parallel_stress.exs | 45 +- test/caccount_map_lifetime_test.exs | 14 +- test/caccount_map_test.exs | 9 +- test/chain_state_merkle_test.exs | 4 +- test/chain_state_uncompact_test.exs | 22 +- test/cmerkle_account_map_diff_test.exs | 47 +- test/cmerkle_clone_lazy_test.exs | 87 +++ test/cmerkle_lock_concurrency_test.exs | 6 +- test/cmerkle_nif_deadlock_test.exs | 6 +- test/cmerkle_nif_leak_test.exs | 16 +- 26 files changed, 1048 insertions(+), 366 deletions(-) create mode 100644 test/cmerkle_clone_lazy_test.exs diff --git a/AGENTS.md b/AGENTS.md index d626139..cd54426 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,12 +39,15 @@ Ethereum-compatible JSON-RPC endpoint plus the Diode PEER/EDGE protocols. - `make test` generates test PEM certs, then runs each `test/*_test.exs` file in a separate `mix test --max-failures 1` invocation (per-file isolation). Test env pins ports `RPC_PORT=18001`, `EDGE2_PORT=18003`, `PEER_PORT=18004`. -- `Chain.State` is a MUTABLE NIF-backed `CMerkleTree`: `Chain.Transaction.apply/3` - mutates the state passed to it in place. Use `Chain.State.clone/1` to get an - independent copy before reapplying. `test/evm_test.exs` "create contract" - currently fails for this reason (it reuses `state` across `apply` calls - without cloning). This is a pre-existing repo issue — CI has `mix test` - commented out and does not gate on it. +- `Chain.State` is backed by the `CAccountMap` NIF. Account storage tries and the + state root trie live in C++; Elixir `Chain.State` no longer carries a separate + `:store` field. Use `Chain.State.hash/1` or `CAccountMap.root_hash/1`. +- **Clone modes:** `Chain.State.clone_lazy/1` for speculative execution + (`eth_call`, RPC, EdgeV2) where the fork is discarded; `Chain.State.clone/1` + (eager storage fork) after `Chain.State.lock/1` for block sync / delta replay. + `Chain.Transaction.apply/3` mutates state in place on an unlocked candidate. +- `Chain.State` is MUTABLE: use `Chain.State.clone/1` or `clone_lazy/1` before + applying transactions on a shared cached state. ### Running the node (dev mode) - `./dev` runs `MIX_ENV=dev iex -S mix run` (wipes `data_dev/` first). For a diff --git a/c_src/LOCK_ORDER.md b/c_src/LOCK_ORDER.md index 5e6bd46..6c3257c 100644 --- a/c_src/LOCK_ORDER.md +++ b/c_src/LOCK_ORDER.md @@ -27,11 +27,14 @@ See also [`SECURITY_REVIEW.md`](SECURITY_REVIEW.md) (F-5 fix) and [`scripts/cmer | `leave_lock` / GC destructor | if `mt->locked`: `locked_states_mutex` → tree → decrement registration → erase map when `has_clone == 0`; else tree refcount only | Unlocked resources never touch `LockedStates` map | | `merkletree_clone` | `Lock(parent)` | Dirty CPU scheduler; O(1) shallow resource alloc (`locked = false`) | | `account_map_clone` | `AccountMapLock` → `Lock(parent_storage)` per trie (sequential) | Dirty scheduler; long hold | -| `account_map_lock` | `AccountMapLock` → `enter_lock` / `apply_canonical_lock` per unique `root_hash`; optional store trie after map lock released | Dirty scheduler; dedupes by root hash to avoid redundant canonical switches | +| `account_map_lock` | `AccountMapLock` only (sets `frozen`; no per-trie `enter_lock` sweep) | Dirty scheduler | | `switch_local_to_canonical` | tree mutexes (address order) | Abandoned `SharedState` queued on `pending_orphans`; reclaimed via `try_reclaim_orphans` after `enter_lock` / `leave_lock` when mutex trylock succeeds and `has_clone == 0` | | `account_map_uncompact_state` | `AccountMapLock(input)` → `materialize_storage` (brief tree lock) → `batch_insert` (state_store lock) | Dirty scheduler | | `account_map_put/delete` | `AccountMapLock` only | May `release_resource` → async GC `leave_lock` | | `account_map_list_difference_raw` | `SharedAccountMap*` mutexes (address order) → brief tree lock per entry for root read → release all before materialize | Dirty CPU; never hold map lock across `materialize_storage` | +| `account_map_difference_full` | Dual map lock → snapshot → release → storage diffs | Dirty CPU; same D-C7 pattern as list_difference | +| `account_map_apply_difference` | `AccountMapLock` → per-account storage writes via `make_writeable_locked` | Dirty CPU | +| `account_map_clone_lazy` | `AccountMapLock`; rejects `frozen` parent; distinct storage wrappers | Dirty CPU | | Insert / COW | Tree lock → ItemPool / PreAllocator / stripe pool | Same-thread nesting | ## Deadlock scenario registry @@ -85,6 +88,10 @@ Each scenario has an ID, hypothesis, and test coverage target. | D-D6 | Dirty scheduler saturation | P15 | | D-D7 | prepare_state composite (native diff + lock + legacy to_list) | S30, P17, ExUnit D-D7 | | D-D8 | `list_difference` compact storage root compare (no full map materialize) | account_map_diff_test, S19 | +| D-L1 | `clone_lazy` + put storage + discard | `cmerkle_clone_lazy_test`, P18L | +| D-L2 | `difference_full` + `apply_difference` round-trip | chain_state_merkle_test, account_map_diff_test | +| D-M1 | frozen map + eager clone writable fork | chain_state_merkle_test lock→clone | +| D-M2 | concurrent difference_raw + apply_difference | lock concurrency, stress | ### E. C++ internal mutexes diff --git a/c_src/SECURITY_REVIEW.md b/c_src/SECURITY_REVIEW.md index 6373a9a..1e3d3bf 100644 --- a/c_src/SECURITY_REVIEW.md +++ b/c_src/SECURITY_REVIEW.md @@ -30,14 +30,19 @@ | `malloc_info_raw` | 0 | — | tests, `cmerkle_memory_bench.exs` | | `account_map_new` | 0 | — | `CAccountMap.new/0`, `Chain.State` | | `account_map_clone` | 1 | account map resource | `CAccountMap.clone/1`, `Chain.State.clone/1` | -| `account_map_lock` | 2 | account map resource, optional state trie or `nil` | `CAccountMap.lock/2`, `Chain.State.lock/1` | +| `account_map_clone_lazy` | 1 | account map resource | `CAccountMap.clone_lazy/1`, speculative `Chain.State.clone_lazy/1` | +| `account_map_lock` | 2 | account map resource, optional store/nil (ignored for freeze) | `CAccountMap.lock/1`, `Chain.State.lock/1` | | `account_map_get` | 2 | resource, 20-byte address | `CAccountMap.get/2` | | `account_map_put` | 6 | resource, address, nonce, balance, storage resource, code | `CAccountMap.put/5` | | `account_map_delete` | 2 | resource, address | `CAccountMap.delete/2` | +| `account_map_root_hash` | 1 | resource | `CAccountMap.root_hash/1`, `Chain.State.hash/1` | +| `account_map_state_trie` | 1 | resource | `CAccountMap.state_trie/1`, `Chain.State.tree/1` | | `account_map_size` | 1 | resource | `CAccountMap.size/1` | | `account_map_to_list` | 1 | resource | `CAccountMap.to_list/1`, RPC export | -| `account_map_list_difference_raw` | 2 | two account map resources | `CAccountMap.list_difference/2`, `Chain.State.difference/2` | -| `account_map_uncompact_state` | 1 | compact account map or account map resource | `CAccountMap.uncompact_state/1`, `Chain.State.uncompact/1` | +| `account_map_list_difference_raw` | 2 | two account map resources | `CAccountMap.list_difference/2` | +| `account_map_difference_full` | 2 | two account map resources | `CAccountMap.difference_full/2`, `Chain.State.difference/2` | +| `account_map_apply_difference` | 2 | account map resource, delta list | `CAccountMap.apply_difference/2`, `Chain.State.apply_difference/2` | +| `account_map_uncompact_state` | 1 | compact account map or account map resource | Returns `{am, hash}`; `CAccountMap.uncompact_state/1`, `Chain.State.uncompact/1` | **Trust:** Erlang validates some shapes (e.g. `to_bytes32`), but the NIF must treat all binaries and terms as hostile (size, allocation, scheduler impact). diff --git a/c_src/nif.cpp b/c_src/nif.cpp index dc1b10d..395380b 100644 --- a/c_src/nif.cpp +++ b/c_src/nif.cpp @@ -51,6 +51,8 @@ static volatile int shared_states = 0; static volatile int resources = 0; static int locked_states_cnt = 0; static int orphan_shared_states = 0; +static int lazy_clone_count = 0; +static int eager_clone_count = 0; class LockedStates; static LockedStates* locked_states; @@ -107,6 +109,7 @@ class SharedState { struct merkletree { bool locked; + bool cow_written; SharedState *shared_state; }; @@ -323,16 +326,41 @@ class SharedAccountMap { public: ErlNifMutex *mtx; int has_clone; + bool is_lazy_fork; + bool frozen; + merkletree *state_trie; std::unordered_map accounts; - SharedAccountMap() : has_clone(0) { + SharedAccountMap() : has_clone(0), is_lazy_fork(false), frozen(false), state_trie(nullptr) { mtx = enif_mutex_create((char*)"accountmap_mutex"); + state_trie = alloc_merkletree_resource(); + enif_keep_resource(state_trie); } ~SharedAccountMap() { + if (is_lazy_fork) { + std::unordered_set seen; + for (auto &entry : accounts) { + if (entry.second.compact_storage) { + continue; + } + merkletree *mt = entry.second.storage; + if (mt == nullptr || !seen.insert(mt).second) { + continue; + } + Lock lock(mt); + if (!mt->cow_written && mt->shared_state->has_clone > 0) { + mt->shared_state->has_clone -= 1; + } + } + } for (auto &entry : accounts) { release_entry_storage(entry.second); } + if (state_trie != nullptr) { + release_storage_from_map(state_trie); + state_trie = nullptr; + } enif_mutex_destroy(mtx); } }; @@ -413,10 +441,10 @@ static void release_storage_from_map(merkletree *mt) static void release_entry_storage(AccountEntry &entry) { - if (entry.storage != nullptr) { + if (entry.storage != nullptr && entry.storage != empty_storage_tree) { release_storage_from_map(entry.storage); - entry.storage = nullptr; } + entry.storage = nullptr; entry.compact_storage.reset(); } @@ -424,6 +452,13 @@ static SharedAccountMap *cow_copy_accountmap(SharedAccountMap *other, ErlNifEnv { SharedAccountMap *copy = new SharedAccountMap(); copy->accounts = other->accounts; + release_storage_from_map(copy->state_trie); + { + merkletree *st = clone_merkletree_locked(other->state_trie); + copy->state_trie = st; + keep_storage_in_map(st); + enif_release_resource(st); + } size_t i = 0; for (auto &entry : copy->accounts) { if (!entry.second.compact_storage && entry.second.storage != nullptr) { @@ -446,6 +481,7 @@ static merkletree *clone_merkletree_locked(merkletree *mt) STAT(resources++); clone->shared_state = mt->shared_state; clone->locked = false; + clone->cow_written = false; clone->shared_state->has_clone += 1; return clone; } @@ -986,8 +1022,8 @@ merkletree_clone(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) } -/* Caller must hold mt->shared_state->mtx. On COW, releases the old mutex and - * acquires the new SharedState mutex before returning. */ +/* In-place COW: when SharedState is shared (has_clone > 0), detach this resource onto a + * privately-owned SharedState copy. Caller must hold mt->shared_state->mtx. */ static bool make_writeable_locked(merkletree *mt) { if (mt->locked) { @@ -1451,9 +1487,14 @@ merkletree_nif_stats(ErlNifEnv *env, int argc, const ERL_NIF_TERM /*argv*/[]) int shared = 0; int res = 0; + int lazy = 0; + int eager = 0; + enif_mutex_lock(stats_mutex); shared = shared_states; res = resources; + lazy = lazy_clone_count; + eager = eager_clone_count; enif_mutex_unlock(stats_mutex); if (locked_states != nullptr) { @@ -1467,7 +1508,10 @@ merkletree_nif_stats(ErlNifEnv *env, int argc, const ERL_NIF_TERM /*argv*/[]) ERL_NIF_TERM orphans_term = enif_make_int(env, orphans); ERL_NIF_TERM shared_term = enif_make_int(env, shared); ERL_NIF_TERM resources_term = enif_make_int(env, res); - return enif_make_tuple4(env, locked_term, orphans_term, shared_term, resources_term); + ERL_NIF_TERM lazy_term = enif_make_int(env, lazy); + ERL_NIF_TERM eager_term = enif_make_int(env, eager); + return enif_make_tuple6(env, locked_term, orphans_term, shared_term, resources_term, + lazy_term, eager_term); } static ERL_NIF_TERM @@ -1535,9 +1579,15 @@ account_map_clone(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) // (merkletree_insert_item returns badarg), breaking block sync. SharedAccountMap *new_shared = new SharedAccountMap(); new_shared->accounts = am->shared->accounts; + release_storage_from_map(new_shared->state_trie); + { + merkletree *st = clone_merkletree_locked(am->shared->state_trie); + new_shared->state_trie = st; + keep_storage_in_map(st); + enif_release_resource(st); + } - // Clone each unique parent storage trie once (accounts that shared a storage - // trie in the parent keep sharing the single clone). ~SharedAccountMap releases + // Clone each unique parent storage trie once // one resource ref per entry, so we keep one ref per entry here to match. std::unordered_map storage_clones; size_t i = 0; @@ -1568,6 +1618,78 @@ account_map_clone(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) accountmap *clone = (accountmap*)enif_alloc_resource(accountmap_type, sizeof(accountmap)); clone->shared = new_shared; + enif_mutex_lock(stats_mutex); + eager_clone_count++; + enif_mutex_unlock(stats_mutex); + ERL_NIF_TERM res = enif_make_resource(env, clone); + enif_release_resource(clone); + return res; +} + +static ERL_NIF_TERM +account_map_clone_lazy(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + if (argc != 1) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + + AccountMapLock lock(am); + + if (am->shared->frozen) { + return enif_make_badarg(env); + } + + // Speculative forks must not share merkletree* resource pointers with the + // parent: in-place COW (make_writeable_locked) would retarget the parent's + // SharedState. Use distinct wrappers that share SharedState until first write + // (same as eager clone for storage; still rejects frozen parents). + for (auto &entry : am->shared->accounts) { + if (!entry.second.compact_storage && entry.second.storage != nullptr && + entry.second.storage->locked) { + return enif_make_badarg(env); + } + } + + SharedAccountMap *new_shared = new SharedAccountMap(); + new_shared->accounts = am->shared->accounts; + release_storage_from_map(new_shared->state_trie); + { + merkletree *st = clone_merkletree_locked(am->shared->state_trie); + new_shared->state_trie = st; + keep_storage_in_map(st); + enif_release_resource(st); + } + + std::unordered_map storage_clones; + size_t i = 0; + for (auto &entry : new_shared->accounts) { + i++; + if (entry.second.compact_storage) { + nif_loop_progress(env, i); + continue; + } + merkletree *orig = entry.second.storage; + if (orig == nullptr) { + continue; + } + auto it = storage_clones.find(orig); + if (it == storage_clones.end()) { + merkletree *storage_clone = clone_merkletree_locked(orig); + it = storage_clones.insert({orig, storage_clone}).first; + } + enif_keep_resource(it->second); + entry.second.storage = it->second; + nif_loop_progress(env, i); + } + for (auto &kv : storage_clones) { + enif_release_resource(kv.second); + } + + accountmap *clone = (accountmap*)enif_alloc_resource(accountmap_type, sizeof(accountmap)); + clone->shared = new_shared; + enif_mutex_lock(stats_mutex); + lazy_clone_count++; + enif_mutex_unlock(stats_mutex); ERL_NIF_TERM res = enif_make_resource(env, clone); enif_release_resource(clone); return res; @@ -1597,31 +1719,9 @@ account_map_lock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); if (!get_optional_merkletree(env, argv[1], &store)) return enif_make_badarg(env); - { - AccountMapLock lock(am); - std::unordered_set locked_roots; - size_t i = 0; - for (auto &entry : am->shared->accounts) { - i++; - merkletree *storage = materialize_storage(entry.second); - if (storage == nullptr) { - nif_loop_progress(env, i); - continue; - } - - uint256_t root_hash; - { - Lock tree_lock(storage); - root_hash = storage->shared_state->tree.root_hash(); - } - - if (!locked_roots.insert(root_hash).second) { - locked_states->apply_canonical_lock(storage, root_hash); - } else { - locked_states->enter_lock(storage); - } - nif_loop_progress(env, i); - } + AccountMapLock lock(am); + if (!am->shared->frozen) { + am->shared->frozen = true; } if (store != nullptr) { @@ -1642,6 +1742,100 @@ static ERL_NIF_TERM account_entry_to_term(ErlNifEnv *env, AccountEntry &entry) return enif_make_tuple4(env, nonce, balance, storage_term, code); } +struct AccountHashCtx { + std::vector nonce_rlp; + std::vector balance_rlp; + std::vector root_rlp; + std::vector code_rlp; + std::vector list_rlp; + std::vector list_payload; + + bool compute(const AccountEntry &entry, const uint256_t *storage_root_override, + const uint256_t *code_hash_override, uint256_t &out) + { + uint256_t storage_root; + if (storage_root_override != nullptr) { + storage_root = *storage_root_override; + } else { + if (entry.storage == nullptr) { + return false; + } + Lock lock(entry.storage); + storage_root = entry.storage->shared_state->tree.root_hash(); + } + + uint256_t code_hash; + if (code_hash_override != nullptr) { + code_hash = *code_hash_override; + } else if (entry.code.empty()) { + code_hash = empty_code_hash; + } else { + sha(entry.code.data(), entry.code.size(), code_hash.data()); + } + + nonce_rlp.clear(); + balance_rlp.clear(); + root_rlp.clear(); + code_rlp.clear(); + list_rlp.clear(); + + rlp_encode_uint64(entry.nonce, nonce_rlp); + rlp_encode_uint256(entry.balance.value, balance_rlp); + rlp_encode_bytes(storage_root.data(), 32, root_rlp); + rlp_encode_bytes(code_hash.data(), 32, code_rlp); + + rlp_encode_list(nonce_rlp, balance_rlp, root_rlp, code_rlp, list_payload, list_rlp); + sha(list_rlp.data(), list_rlp.size(), out.data()); + return true; + } +}; + +static void insert_state_trie_hash(SharedAccountMap *shared, const uint160_t &addr, const uint256_t &hash) +{ + bin_t key(addr.value, addr.value + 20); + merkletree *mt = shared->state_trie; + enif_mutex_lock(mt->shared_state->mtx); + if (!make_writeable_locked(mt)) { + enif_mutex_unlock(mt->shared_state->mtx); + return; + } + uint256_t hash_value = hash; + mt->shared_state->tree.insert_item(key, hash_value); + enif_mutex_unlock(mt->shared_state->mtx); +} + +static void update_state_trie_for_entry(SharedAccountMap *shared, const uint160_t &addr, + AccountEntry &entry, AccountHashCtx &ctx) +{ + if (entry.storage == nullptr && !entry.compact_storage) { + materialize_storage(entry); + } + uint256_t account_hash; + if (!ctx.compute(entry, nullptr, nullptr, account_hash)) { + return; + } + insert_state_trie_hash(shared, addr, account_hash); +} + +static void remove_state_trie_entry(SharedAccountMap *shared, const uint160_t &addr) +{ + uint256_t zero; + insert_state_trie_hash(shared, addr, zero); +} + +static ERL_NIF_TERM +account_map_state_trie(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + if (argc != 1) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + + AccountMapLock lock(am); + ERL_NIF_TERM term = enif_make_resource(env, am->shared->state_trie); + enif_keep_resource(am->shared->state_trie); + return term; +} + static ERL_NIF_TERM account_map_get(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { @@ -1700,6 +1894,9 @@ account_map_put(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) entry.code = code; am->shared->accounts[addr] = std::move(entry); } + + AccountHashCtx hash_ctx; + update_state_trie_for_entry(am->shared, addr, am->shared->accounts[addr], hash_ctx); return argv[0]; } @@ -1720,10 +1917,24 @@ account_map_delete(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) if (it != am->shared->accounts.end()) { release_entry_storage(it->second); am->shared->accounts.erase(it); + remove_state_trie_entry(am->shared, addr); } return argv[0]; } +static ERL_NIF_TERM +account_map_root_hash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + if (argc != 1) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + + AccountMapLock lock(am); + Lock tree_lock(am->shared->state_trie); + uint256_t root = am->shared->state_trie->shared_state->tree.root_hash(); + return make_binary(env, root.data(), 32); +} + static ERL_NIF_TERM account_map_size(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { @@ -1762,6 +1973,7 @@ static merkletree *alloc_merkletree_resource() STAT(resources++); mt->shared_state = new SharedState(); mt->locked = false; + mt->cow_written = false; return mt; } @@ -1771,8 +1983,11 @@ static merkletree *materialize_storage(AccountEntry &entry) return entry.storage; } if (!entry.compact_storage || entry.compact_storage->slots.empty()) { - entry.storage = empty_storage_tree; - keep_storage_in_map(empty_storage_tree); + // Fresh empty tree — never share empty_storage_tree as a writable map entry + // (in-place COW would mutate the singleton for every account). + merkletree *mt = alloc_merkletree_resource(); + keep_storage_in_map(mt); + entry.storage = mt; entry.compact_storage.reset(); return entry.storage; } @@ -1965,53 +2180,426 @@ account_map_list_difference_raw(ErlNifEnv *env, int argc, const ERL_NIF_TERM arg return list; } -struct AccountHashCtx { - std::vector nonce_rlp; - std::vector balance_rlp; - std::vector root_rlp; - std::vector code_rlp; - std::vector list_rlp; - std::vector list_payload; +static ERL_NIF_TERM diff_side_fields_to_term(ErlNifEnv *env, DiffAccountSide &side) +{ + if (!side.present) { + return make_atom(env, "nil"); + } + ERL_NIF_TERM nonce = enif_make_uint64(env, side.nonce); + ERL_NIF_TERM balance = balance_to_term(env, side.balance); + ERL_NIF_TERM code = code_to_term(env, side.code); + return enif_make_tuple3(env, nonce, balance, code); +} - bool compute(const AccountEntry &entry, const uint256_t *storage_root_override, - const uint256_t *code_hash_override, uint256_t &out) +static merkletree *storage_for_diff_side(DiffAccountSide &side, merkletree **temp_out) +{ + *temp_out = nullptr; + if (!side.present) { + return empty_storage_tree; + } + if (side.storage != nullptr) { + return side.storage; + } + if (!side.compact_storage || side.compact_storage->slots.empty()) { + return empty_storage_tree; + } + merkletree *mt = alloc_merkletree_resource(); + *temp_out = mt; { - uint256_t storage_root; - if (storage_root_override != nullptr) { - storage_root = *storage_root_override; - } else { - if (entry.storage == nullptr) { - return false; - } - Lock lock(entry.storage); - storage_root = entry.storage->shared_state->tree.root_hash(); + Lock lock(mt); + for (auto &slot : side.compact_storage->slots) { + mt->shared_state->tree.insert_item(slot.key, slot.value); } + } + return mt; +} - uint256_t code_hash; - if (code_hash_override != nullptr) { - code_hash = *code_hash_override; - } else if (entry.code.empty()) { - code_hash = empty_code_hash; - } else { - sha(entry.code.data(), entry.code.size(), code_hash.data()); +static void release_temp_storage(merkletree *temp) +{ + if (temp != nullptr) { + enif_release_resource(temp); + } +} + +static ERL_NIF_TERM build_storage_diff_list(ErlNifEnv *env, DiffAccountSide &side_a, + DiffAccountSide &side_b, size_t progress_base) +{ + merkletree *temp_a = nullptr; + merkletree *temp_b = nullptr; + merkletree *mt_a = storage_for_diff_side(side_a, &temp_a); + merkletree *mt_b = storage_for_diff_side(side_b, &temp_b); + + if (mt_a == mt_b) { + release_temp_storage(temp_a); + release_temp_storage(temp_b); + return enif_make_list(env, 0); + } + + SharedState *s1 = mt_a->shared_state; + SharedState *s2 = mt_b->shared_state; + if (s1 == s2) { + release_temp_storage(temp_a); + release_temp_storage(temp_b); + return enif_make_list(env, 0); + } + + SharedStateLock state_lock(s1, s2); + + Tree output; + s1->tree.difference(s2->tree, output); + s2->tree.difference(s1->tree, output); + + ERL_NIF_TERM list = enif_make_list(env, 0); + size_t i = 0; + output.each([&](pair_t &pair) { + i++; + ERL_NIF_TERM key_term = make_binary(env, pair.key.data(), pair.key.size()); + + auto pair1 = s1->tree.get_item(pair); + auto pair2 = s2->tree.get_item(pair); + + ERL_NIF_TERM value1_term = pair1 == nullptr ? make_atom(env, "nil") : + make_binary(env, pair1->value.data(), 32); + ERL_NIF_TERM value2_term = pair2 == nullptr ? make_atom(env, "nil") : + make_binary(env, pair2->value.data(), 32); + ERL_NIF_TERM values = enif_make_tuple2(env, value1_term, value2_term); + ERL_NIF_TERM item = enif_make_tuple2(env, key_term, values); + list = enif_make_list_cell(env, item, list); + nif_loop_progress(env, progress_base + i); + }); + + release_temp_storage(temp_a); + release_temp_storage(temp_b); + return list; +} + +static ERL_NIF_TERM +account_map_difference_full(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am_a; + accountmap *am_b; + + if (argc != 2) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am_a)) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[1], accountmap_type, (void **)&am_b)) return enif_make_badarg(env); + + if (am_a->shared == am_b->shared) { + return enif_make_list(env, 0); + } + + std::vector diffs; + std::unordered_set key_set; + + { + DualAccountMapLock map_lock(am_a->shared, am_b->shared); + + for (auto &entry : am_a->shared->accounts) { + key_set.insert(entry.first); + } + for (auto &entry : am_b->shared->accounts) { + key_set.insert(entry.first); } - nonce_rlp.clear(); - balance_rlp.clear(); - root_rlp.clear(); - code_rlp.clear(); - list_rlp.clear(); + std::vector keys(key_set.begin(), key_set.end()); + std::sort(keys.begin(), keys.end()); - rlp_encode_uint64(entry.nonce, nonce_rlp); - rlp_encode_uint256(entry.balance.value, balance_rlp); - rlp_encode_bytes(storage_root.data(), 32, root_rlp); - rlp_encode_bytes(code_hash.data(), 32, code_rlp); + size_t i = 0; + for (auto &addr : keys) { + i++; + auto it_a = am_a->shared->accounts.find(addr); + auto it_b = am_b->shared->accounts.find(addr); + bool in_a = it_a != am_a->shared->accounts.end(); + bool in_b = it_b != am_b->shared->accounts.end(); - rlp_encode_list(nonce_rlp, balance_rlp, root_rlp, code_rlp, list_payload, list_rlp); - sha(list_rlp.data(), list_rlp.size(), out.data()); + if (in_a && in_b && entries_equal(env, it_a->second, it_b->second, i)) { + nif_loop_progress(env, i); + continue; + } + + DiffItem item; + item.addr = addr; + if (in_a) { + snapshot_side(it_a->second, item.a); + } + if (in_b) { + snapshot_side(it_b->second, item.b); + } + diffs.push_back(std::move(item)); + nif_loop_progress(env, i); + } + } + + ERL_NIF_TERM list = enif_make_list(env, 0); + size_t j = 0; + for (auto &item : diffs) { + j++; + ERL_NIF_TERM addr_term = make_binary(env, (uint8_t *)item.addr.value, 20); + ERL_NIF_TERM side_a = diff_side_fields_to_term(env, item.a); + ERL_NIF_TERM side_b = diff_side_fields_to_term(env, item.b); + ERL_NIF_TERM storage_diff = build_storage_diff_list(env, item.a, item.b, j * 1000); + ERL_NIF_TERM quad = enif_make_tuple4(env, addr_term, side_a, side_b, storage_diff); + list = enif_make_list_cell(env, quad, list); + release_snapshot_side(item.a); + release_snapshot_side(item.b); + nif_loop_progress(env, j); + } + + locked_states->try_reclaim_orphans(); + return list; +} + +static bool map_get_atom(ErlNifEnv *env, ERL_NIF_TERM map, const char *key, ERL_NIF_TERM &out); + +static bool term_is_nil(ErlNifEnv *env, ERL_NIF_TERM term) +{ + if (!enif_is_atom(env, term)) { + return false; + } + char atom[16]; + return enif_get_atom(env, term, atom, sizeof(atom), ERL_NIF_LATIN1) && + strcmp(atom, "nil") == 0; +} + +static bool balance_equals_term(ErlNifEnv *env, const uint256_t &actual, ERL_NIF_TERM term) +{ + uint256_t expected; + if (!get_balance_uint256(env, term, expected)) { + return false; + } + return expected == actual; +} + +static bool code_equals_term(ErlNifEnv *env, const bin_t &actual, ERL_NIF_TERM term) +{ + bin_t expected; + if (!get_code(env, term, expected)) { + return false; + } + return expected == actual; +} + +static bool get_storage_value_from_term(ErlNifEnv *env, ERL_NIF_TERM term, uint256_t &out) +{ + memset(out.value, 0, sizeof(out.value)); + if (term_is_nil(env, term)) { return true; } -}; + ErlNifBinary bin; + if (!enif_inspect_binary(env, term, &bin) || bin.size != 32) { + return false; + } + memcpy(out.value, bin.data, 32); + return true; +} + +static bool storage_values_equal(ErlNifEnv *env, ERL_NIF_TERM expected_term, const uint256_t &actual) +{ + uint256_t expected; + if (!get_storage_value_from_term(env, expected_term, expected)) { + return false; + } + return expected == actual; +} + +static uint256_t read_storage_slot(merkletree *mt, const bin_t &key) +{ + Lock lock(mt); + bin_t lookup = key; + pair_t *pair = mt->shared_state->tree.get_item(std::move(lookup)); + return pair == nullptr ? uint256_t() : pair->value; +} + +static merkletree *write_storage_slot(AccountEntry &entry, const bin_t &key, const uint256_t &value) +{ + merkletree *mt = materialize_storage(entry); + enif_mutex_lock(mt->shared_state->mtx); + if (!make_writeable_locked(mt)) { + enif_mutex_unlock(mt->shared_state->mtx); + return nullptr; + } + bin_t key_copy = key; + uint256_t value_copy = value; + mt->shared_state->tree.insert_item(key_copy, value_copy); + enif_mutex_unlock(mt->shared_state->mtx); + entry.compact_storage.reset(); + return mt; +} + +static ERL_NIF_TERM make_apply_error(ErlNifEnv *env, const char *reason) +{ + ERL_NIF_TERM err_atom, reason_atom; + enif_make_existing_atom(env, "error", &err_atom, ERL_NIF_LATIN1); + enif_make_existing_atom(env, reason, &reason_atom, ERL_NIF_LATIN1); + return enif_make_tuple2(env, err_atom, reason_atom); +} + +static AccountEntry &ensure_account_entry(SharedAccountMap *shared, const uint160_t &addr) +{ + auto it = shared->accounts.find(addr); + if (it == shared->accounts.end()) { + AccountEntry entry; + entry.storage = alloc_merkletree_resource(); + keep_storage_in_map(entry.storage); + enif_release_resource(entry.storage); + shared->accounts[addr] = std::move(entry); + return shared->accounts[addr]; + } + return it->second; +} + +static bool apply_field_delta(ErlNifEnv *env, AccountEntry &entry, const char *field, + ERL_NIF_TERM delta_term) +{ + const ERL_NIF_TERM *elems; + int arity; + if (!enif_get_tuple(env, delta_term, &arity, &elems) || arity != 2) { + return false; + } + + if (strcmp(field, "nonce") == 0) { + ErlNifUInt64 expected, new_val; + if (!enif_get_uint64(env, elems[0], &expected) || + !enif_get_uint64(env, elems[1], &new_val)) { + return false; + } + if (entry.nonce != (uint64_t)expected) { + return false; + } + entry.nonce = (uint64_t)new_val; + return true; + } + + if (strcmp(field, "balance") == 0) { + if (!balance_equals_term(env, entry.balance, elems[0])) { + return false; + } + if (!get_balance_uint256(env, elems[1], entry.balance)) { + return false; + } + return true; + } + + if (strcmp(field, "code") == 0) { + if (!code_equals_term(env, entry.code, elems[0])) { + return false; + } + if (!get_code(env, elems[1], entry.code)) { + return false; + } + return true; + } + + return true; +} + +static bool apply_storage_delta(ErlNifEnv *env, AccountEntry &entry, ERL_NIF_TERM state_map) +{ + if (!enif_is_map(env, state_map)) { + return false; + } + + ErlNifMapIterator iter; + enif_map_iterator_create(env, state_map, &iter, ERL_NIF_MAP_ITERATOR_FIRST); + + ERL_NIF_TERM slot_key, slot_delta; + while (enif_map_iterator_get_pair(env, &iter, &slot_key, &slot_delta)) { + ErlNifBinary key_bin; + if (!enif_inspect_binary(env, slot_key, &key_bin)) { + enif_map_iterator_destroy(env, &iter); + return false; + } + + const ERL_NIF_TERM *delta_elems; + int delta_arity; + if (!enif_get_tuple(env, slot_delta, &delta_arity, &delta_elems) || delta_arity != 2) { + enif_map_iterator_destroy(env, &iter); + return false; + } + + merkletree *mt = materialize_storage(entry); + bin_t key(key_bin.data, key_bin.data + key_bin.size); + uint256_t current = read_storage_slot(mt, key); + if (!storage_values_equal(env, delta_elems[0], current)) { + enif_map_iterator_destroy(env, &iter); + return false; + } + + uint256_t new_value; + if (!get_storage_value_from_term(env, delta_elems[1], new_value)) { + enif_map_iterator_destroy(env, &iter); + return false; + } + + if (write_storage_slot(entry, key, new_value) == nullptr) { + enif_map_iterator_destroy(env, &iter); + return false; + } + + enif_map_iterator_next(env, &iter); + } + + enif_map_iterator_destroy(env, &iter); + return true; +} + +static ERL_NIF_TERM +account_map_apply_difference(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + if (argc != 2) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + if (!enif_is_list(env, argv[1])) return enif_make_badarg(env); + + AccountMapLock lock(am); + if (!make_writeable_accountmap(env, am)) return enif_make_badarg(env); + + AccountHashCtx hash_ctx; + size_t i = 0; + ERL_NIF_TERM head, tail = argv[1]; + + while (enif_get_list_cell(env, tail, &head, &tail)) { + i++; + const ERL_NIF_TERM *elems; + int arity; + if (!enif_get_tuple(env, head, &arity, &elems) || arity != 2) { + return enif_make_badarg(env); + } + + uint160_t addr; + if (!get_address(env, elems[0], addr)) { + return enif_make_badarg(env); + } + if (!enif_is_map(env, elems[1])) { + return enif_make_badarg(env); + } + + AccountEntry &entry = ensure_account_entry(am->shared, addr); + ERL_NIF_TERM report = elems[1]; + + ERL_NIF_TERM state_term; + if (map_get_atom(env, report, "state", state_term)) { + if (!apply_storage_delta(env, entry, state_term)) { + return make_apply_error(env, "mismatch"); + } + } + + static const char *fields[] = {"nonce", "balance", "code", nullptr}; + for (int f = 0; fields[f] != nullptr; f++) { + ERL_NIF_TERM delta_term; + if (map_get_atom(env, report, fields[f], delta_term)) { + if (!apply_field_delta(env, entry, fields[f], delta_term)) { + return make_apply_error(env, "mismatch"); + } + } + } + + update_state_trie_for_entry(am->shared, addr, entry, hash_ctx); + nif_loop_progress(env, i); + } + + locked_states->try_reclaim_orphans(); + return argv[0]; +} struct UncompactLoopScratch { AccountHashCtx hash_ctx; @@ -2235,8 +2823,6 @@ account_map_uncompact_state(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) accountmap *am = (accountmap*)enif_alloc_resource(accountmap_type, sizeof(accountmap)); am->shared = new SharedAccountMap(); - merkletree *state_store = alloc_merkletree_resource(); - size_t expected = 0; if (from_resource) { AccountMapLock lock(input_am); @@ -2244,12 +2830,10 @@ account_map_uncompact_state(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) } else if (enif_is_map(env, argv[0])) { if (!enif_get_map_size(env, argv[0], &expected)) { enif_release_resource(am); - enif_release_resource(state_store); return enif_make_badarg(env); } } else { enif_release_resource(am); - enif_release_resource(state_store); return enif_make_badarg(env); } @@ -2267,7 +2851,7 @@ account_map_uncompact_state(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) AccountEntry entry = kv.second; if (!append_uncompacted_account(env, am, scratch.hash_ctx, pending_state, kv.first, entry, nullptr, nullptr, i)) { - return uncompact_state_fail(env, nullptr, am, state_store); + return uncompact_state_fail(env, nullptr, am, nullptr); } } } else { @@ -2279,12 +2863,12 @@ account_map_uncompact_state(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) i++; uint160_t addr; if (!get_address(env, key, addr)) { - return uncompact_state_fail(env, &iter, am, state_store); + return uncompact_state_fail(env, &iter, am, nullptr); } ParsedCompactAccount parsed; if (!parse_compact_account(env, value, parsed, scratch.code_buf)) { - return uncompact_state_fail(env, &iter, am, state_store); + return uncompact_state_fail(env, &iter, am, nullptr); } const uint256_t *storage_root_override = @@ -2293,7 +2877,7 @@ account_map_uncompact_state(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) parsed.has_compact_code_hash ? &parsed.compact_code_hash : nullptr; if (!append_uncompacted_account(env, am, scratch.hash_ctx, pending_state, addr, parsed.entry, storage_root_override, code_hash_override, i)) { - return uncompact_state_fail(env, &iter, am, state_store); + return uncompact_state_fail(env, &iter, am, nullptr); } enif_map_iterator_next(env, &iter); @@ -2301,20 +2885,18 @@ account_map_uncompact_state(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) enif_map_iterator_destroy(env, &iter); } - batch_insert_state_items(state_store, pending_state); + batch_insert_state_items(am->shared->state_trie, pending_state); uint256_t state_root; { - Lock lock(state_store); - state_root = state_store->shared_state->tree.root_hash(); + Lock lock(am->shared->state_trie); + state_root = am->shared->state_trie->shared_state->tree.root_hash(); } ERL_NIF_TERM am_term = enif_make_resource(env, am); enif_release_resource(am); - ERL_NIF_TERM store_term = enif_make_resource(env, state_store); - enif_release_resource(state_store); ERL_NIF_TERM hash_term = make_binary(env, state_root.data(), 32); - return enif_make_tuple3(env, am_term, store_term, hash_term); + return enif_make_tuple2(env, am_term, hash_term); } static void @@ -2392,13 +2974,18 @@ static ErlNifFunc nif_funcs[] = { {"nif_stats_raw", 0, merkletree_nif_stats, 0}, {"account_map_new", 0, account_map_new, 0}, {"account_map_clone", 1, account_map_clone, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"account_map_clone_lazy", 1, account_map_clone_lazy, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_lock", 2, account_map_lock, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_get", 2, account_map_get, 0}, {"account_map_put", 6, account_map_put, 0}, {"account_map_delete", 2, account_map_delete, 0}, + {"account_map_root_hash", 1, account_map_root_hash, 0}, + {"account_map_state_trie", 1, account_map_state_trie, 0}, {"account_map_size", 1, account_map_size, 0}, {"account_map_to_list", 1, account_map_to_list, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_list_difference_raw", 2, account_map_list_difference_raw, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"account_map_difference_full", 2, account_map_difference_full, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"account_map_apply_difference", 2, account_map_apply_difference, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_uncompact_state", 1, account_map_uncompact_state, ERL_NIF_DIRTY_JOB_CPU_BOUND}, }; diff --git a/lib/caccount_map.ex b/lib/caccount_map.ex index b0a22a3..3ecdb61 100644 --- a/lib/caccount_map.ex +++ b/lib/caccount_map.ex @@ -11,7 +11,13 @@ defmodule CAccountMap do def clone(map), do: CMerkleTree.account_map_clone(map) - def lock(map, store \\ nil), do: CMerkleTree.account_map_lock(map, store) + def clone_lazy(map), do: CMerkleTree.account_map_clone_lazy(map) + + def lock(map), do: CMerkleTree.account_map_lock(map, nil) + + def root_hash(map), do: CMerkleTree.account_map_root_hash(map) + + def state_trie(map), do: CMerkleTree.account_map_state_trie(map) def get(map, <<_::160>> = addr) do case CMerkleTree.account_map_get(map, addr) do @@ -57,10 +63,34 @@ defmodule CAccountMap do end) end + def difference_full(map_a, map_b) do + CMerkleTree.account_map_difference_full(map_a, map_b) + end + + def apply_difference(map, delta) do + case CMerkleTree.account_map_apply_difference(map, delta) do + {:error, reason} -> {:error, reason} + map -> map + end + end + + def decode_storage_diff(storage_diff) do + Map.new(storage_diff, fn {key, {val_a, val_b}} -> + {key, {decode_storage_value(val_a), decode_storage_value(val_b)}} + end) + end + + defp decode_storage_value(nil), do: nil + defp decode_storage_value(val) when is_binary(val), do: val + defp decode_account_side(nil), do: nil defp decode_account_side(entry), do: entry |> decode_entry() |> account_from_parts() - def uncompact_state(accounts), do: CMerkleTree.account_map_uncompact_state(accounts) + def uncompact_state(accounts) do + case CMerkleTree.account_map_uncompact_state(accounts) do + {am, hash} -> {am, hash} + end + end defp decode_entry({nonce, balance, storage, code}) do {nonce, decode_balance(balance), storage, code} diff --git a/lib/chain/account.ex b/lib/chain/account.ex index 7e708ca..ccd86b6 100644 --- a/lib/chain/account.ex +++ b/lib/chain/account.ex @@ -37,6 +37,7 @@ defmodule Chain.Account do } end + @deprecated "Use CMerkleTree.clone/1 on Account.tree/1 instead" def clone(%Chain.Account{} = acc) do %Chain.Account{acc | storage_root: CMerkleTree.clone(tree(acc))} end diff --git a/lib/chain/block.ex b/lib/chain/block.ex index b1ae2cd..0d94078 100644 --- a/lib/chain/block.ex +++ b/lib/chain/block.ex @@ -134,7 +134,7 @@ defmodule Chain.Block do try do hash = state_hash(block) hash2 = if is_binary(block.header.state_hash), do: block.header.state_hash, else: hash - hash3 = CMerkleTree.root_hash(Chain.State.tree(state(block))) + hash3 = Chain.State.hash(state(block)) consistent = hash2 == hash3 and (not is_binary(hash) or hash == hash2) diff --git a/lib/chain/state.ex b/lib/chain/state.ex index 5364697..440699a 100644 --- a/lib/chain/state.ex +++ b/lib/chain/state.ex @@ -9,26 +9,18 @@ defmodule Chain.State do {:nowarn_function, new: 0}, {:nowarn_function, uncompact: 1}, {:nowarn_function, clone: 1}, + {:nowarn_function, clone_lazy: 1}, {:nowarn_function, from_binary: 1} ] @enforce_keys [:accounts] - defstruct accounts: nil, hash: nil, store: nil - @type t :: %Chain.State{accounts: CAccountMap.t(), hash: any(), store: any()} + defstruct accounts: nil, hash: nil + @type t :: %Chain.State{accounts: CAccountMap.t(), hash: binary() | nil} def new() do %Chain.State{accounts: CAccountMap.new()} end - def compact(%Chain.State{accounts: accounts} = state) when is_map(accounts) do - accounts = - Enum.map(accounts, fn {id, acc} -> {id, Account.compact(acc)} end) - |> Map.new() - - %Chain.State{state | accounts: accounts} - |> Map.delete(:store) - end - def compact(%Chain.State{accounts: accounts} = state) do accounts = accounts @@ -37,58 +29,34 @@ defmodule Chain.State do |> Map.new() %Chain.State{state | accounts: accounts} - |> Map.delete(:store) end def uncompact(%Chain.State{accounts: accounts} = state) do - {accounts, store, hash} = CAccountMap.uncompact_state(accounts) - + {accounts, hash} = CAccountMap.uncompact_state(accounts) %Chain.State{state | accounts: accounts, hash: hash} - |> Map.put(:store, store) end - def normalize(%Chain.State{} = state) do - tree = tree(state) - hash = CMerkleTree.root_hash(tree) - state = Map.put(state, :store, tree) - %Chain.State{} = state - %{state | hash: hash} - end - - def tree(%Chain.State{store: store}) when store != nil do - store + def normalize(%Chain.State{accounts: accounts} = state) do + %{state | hash: CAccountMap.root_hash(accounts)} end def tree(%Chain.State{accounts: accounts}) do - accounts - |> account_list() - |> Enum.reduce(%{}, fn {id, acc}, map -> - Map.put(map, id, Account.hash(acc)) - end) - |> CMerkleTree.from_map() + CAccountMap.state_trie(accounts) end def hash(%Chain.State{hash: nil} = state) do - CMerkleTree.root_hash(tree(state)) + CAccountMap.root_hash(state.accounts) end def hash(%Chain.State{hash: hash}) do hash end - def accounts(%Chain.State{accounts: accounts}) when is_map(accounts) do - accounts - end - def accounts(%Chain.State{accounts: accounts}) do CAccountMap.to_account_list(accounts) end @spec account(Chain.State.t(), <<_::160>>) :: Chain.Account.t() | nil - def account(%Chain.State{accounts: accounts}, id = <<_::160>>) when is_map(accounts) do - Map.get(accounts, id) - end - def account(%Chain.State{accounts: accounts}, id = <<_::160>>) do CAccountMap.get_account(accounts, id) end @@ -112,119 +80,89 @@ defmodule Chain.State do @spec set_account(Chain.State.t(), binary(), Chain.Account.t()) :: Chain.State.t() def set_account(state, id = <<_::160>>, account) do - tree = CMerkleTree.insert(tree(state), id, Account.hash(account)) - accounts = put_account_in(state.accounts, id, account) - %{state | accounts: accounts, hash: nil, store: tree} + accounts = CAccountMap.put_account(state.accounts, id, account) + %{state | accounts: accounts, hash: nil} end @spec delete_account(Chain.State.t(), binary()) :: Chain.State.t() - def delete_account(state = %Chain.State{accounts: accounts}, id = <<_::160>>) - when is_map(accounts) do - %{state | accounts: Map.delete(accounts, id), hash: nil, store: nil} - end - def delete_account(state = %Chain.State{accounts: accounts}, id = <<_::160>>) do - %{state | accounts: CAccountMap.delete(accounts, id), hash: nil, store: nil} + %{state | accounts: CAccountMap.delete(accounts, id), hash: nil} end def difference( %Chain.State{accounts: accounts_a} = state_a, %Chain.State{accounts: accounts_b} = state_b ) do - diff = account_maps_diff(accounts_a, accounts_b) - - Enum.map(diff, fn {id, {acc_a, acc_b}} -> - acc_a = acc_a || ensure_account(state_a, id) - acc_b = acc_b || ensure_account(state_b, id) - - {time, report} = - :timer.tc(fn -> - delta = %{ - nonce: {Account.nonce(acc_a), Account.nonce(acc_b)}, - balance: {Account.balance(acc_a), Account.balance(acc_b)}, - code: {Account.code(acc_a), Account.code(acc_b)} - } - - report = - Enum.reduce(delta, %{}, fn {key, {a, b}}, report -> - if a == b do - report + {time, result} = + :timer.tc(fn -> + Enum.map(CAccountMap.difference_full(accounts_a, accounts_b), fn + {id, _side_a, _side_b, state_diff} -> + acc_a = account(state_a, id) || ensure_account(state_a, id) + acc_b = account(state_b, id) || ensure_account(state_b, id) + + report = + %{} + |> put_field_diff(:nonce, acc_a, acc_b) + |> put_field_diff(:balance, acc_a, acc_b) + |> put_field_diff(:code, acc_a, acc_b) + + storage_map = CAccountMap.decode_storage_diff(state_diff) + + report = + if map_size(storage_map) > 0 do + Map.merge(report, %{ + state: storage_map, + root_hash: {Account.root_hash(acc_a), Account.root_hash(acc_b)} + }) else - Map.put(report, key, {a, b}) + report end - end) - - state_diff = CMerkleTree.difference(Account.tree(acc_a), Account.tree(acc_b)) - - if map_size(state_diff) > 0 do - Map.merge(report, %{ - state: state_diff, - root_hash: {Account.root_hash(acc_a), Account.root_hash(acc_b)} - }) - else - report - end + + {id, report} end) + end) - if div(time, 1000) > 1000 do - Logger.warning( - "State diff took longer than 1s #{inspect({Base16.encode(id), div(time, 1000), map_size(report)})}" - ) - end + if div(time, 1000) > 1000 do + Logger.warning( + "State diff took longer than 1s total_ms=#{div(time, 1000)} accounts=#{length(result)}" + ) + end - {id, report} - end) + result end - def clone(%Chain.State{accounts: accounts} = state) do - state - |> Map.put(:accounts, clone_accounts(accounts)) - |> clone_store() - end + defp put_field_diff(report, field, acc_a, acc_b) do + a = apply(Account, field, [acc_a]) + b = apply(Account, field, [acc_b]) - def lock(%Chain.State{accounts: accounts} = state) when is_map(accounts) do - for {_id, acc} <- account_list(accounts) do - do_lock(Account.tree(acc)) + if a == b do + report + else + Map.put(report, field, {a, b}) end + end - do_lock(Map.get(state, :store)) - state + def clone(%Chain.State{accounts: accounts} = state) do + %{state | accounts: CAccountMap.clone(accounts), hash: nil} + end + + def clone_lazy(%Chain.State{accounts: accounts} = state) do + %{state | accounts: CAccountMap.clone_lazy(accounts), hash: nil} end def lock(%Chain.State{accounts: accounts} = state) do - CAccountMap.lock(accounts, Map.get(state, :store)) + CAccountMap.lock(accounts) state end - defp do_lock(nil), do: nil - defp do_lock(root), do: CMerkleTree.lock(root) - def apply_difference(%Chain.State{} = state, difference) do - Enum.reduce(difference, state, fn {id, report}, state -> - oacc = acc = ensure_account(state, id) - - {state_update, report} = Map.pop(report, :state, %{}) - - acc = - Enum.reduce(state_update, acc, fn {key, {a, b}}, acc -> - tree = Account.tree(acc) - ^a = CMerkleTree.get(tree, key) - tree = CMerkleTree.insert(tree, key, b) - Account.put_tree(acc, tree) - end) - - acc = - report - |> Enum.reject(fn {key, _delta} -> key == :root_hash end) - |> Enum.reduce(acc, fn {key, delta}, acc -> - {a, b} = delta - ret = apply(Account, key, [oacc]) - ^a = ret - %{acc | key => b} - end) + case CAccountMap.apply_difference(state.accounts, difference) do + {:error, reason} -> + raise ArgumentError, "apply_difference mismatch: #{inspect(reason)}" - set_account(state, id, acc) - end) + accounts -> + %{state | accounts: accounts, hash: nil} + end end def from_binary(bin) do @@ -239,41 +177,4 @@ defmodule Chain.State do }) end) end - - defp account_list(accounts) when is_map(accounts) do - Map.to_list(accounts) - end - - defp account_list(accounts) do - CAccountMap.to_account_list(accounts) - end - - defp account_maps_diff(accounts_a, accounts_b) when is_map(accounts_a) or is_map(accounts_b) do - CMerkleTree.list_difference(account_list(accounts_a), account_list(accounts_b)) - end - - defp account_maps_diff(accounts_a, accounts_b) do - CAccountMap.list_difference(accounts_a, accounts_b) - end - - defp put_account_in(accounts, id, account) when is_map(accounts) do - Map.put(accounts, id, account) - end - - defp put_account_in(accounts, id, account) do - CAccountMap.put_account(accounts, id, account) - end - - defp clone_accounts(accounts) when is_map(accounts) do - Map.new(accounts, fn {id, acc} -> {id, Account.clone(acc)} end) - end - - defp clone_accounts(accounts), do: CAccountMap.clone(accounts) - - defp clone_store(%Chain.State{} = state) do - case Map.get(state, :store) do - nil -> state - store -> %{state | store: CMerkleTree.clone(store)} - end - end end diff --git a/lib/cmerkletree.ex b/lib/cmerkletree.ex index 55e111a..87b6cdf 100644 --- a/lib/cmerkletree.ex +++ b/lib/cmerkletree.ex @@ -19,32 +19,10 @@ defmodule CMerkleTree do insert_items(new(), list) end - def list_difference(a, b) do - a_diffmap = - Enum.reduce(a, %{}, fn {key, value}, acc -> - Map.put(acc, key, {value, nil}) - end) - - Enum.reduce(b, a_diffmap, fn {key, bvalue}, acc -> - case Map.get(acc, key) do - nil -> - Map.put(acc, key, {nil, bvalue}) - - {avalue, nil} -> - if avalue.nonce == bvalue.nonce && avalue.balance == bvalue.balance && - avalue.code == bvalue.code && - Chain.Account.root_hash(avalue) == Chain.Account.root_hash(bvalue) do - Map.delete(acc, key) - else - Map.put(acc, key, {avalue, bvalue}) - end - end - end) + def list_difference(_a, _b) do + raise "CMerkleTree.list_difference/2 removed; use CAccountMap.list_difference/2" end - # def difference(a, b) do - # list_difference(to_list(a), to_list(b)) - # end def difference(a, b) do difference_raw(a, b) |> Enum.map(fn @@ -141,6 +119,9 @@ defmodule CMerkleTree do def account_map_new(), do: error() def account_map_clone(_map), do: error() + def account_map_clone_lazy(_map), do: error() + def account_map_root_hash(_map), do: error() + def account_map_state_trie(_map), do: error() def account_map_lock(_map, _store), do: error() def account_map_get(_map, _addr), do: error() def account_map_put(_map, _addr, _nonce, _balance, _storage, _code), do: error() @@ -148,6 +129,8 @@ defmodule CMerkleTree do def account_map_size(_map), do: error() def account_map_to_list(_map), do: error() def account_map_list_difference_raw(_map_a, _map_b), do: error() + def account_map_difference_full(_map_a, _map_b), do: error() + def account_map_apply_difference(_map, _delta), do: error() def account_map_uncompact_state(_map), do: error() defp struct_sizes_raw, do: error() diff --git a/lib/network/edge_v2.ex b/lib/network/edge_v2.ex index 8492474..7558ef6 100644 --- a/lib/network/edge_v2.ex +++ b/lib/network/edge_v2.ex @@ -139,7 +139,7 @@ defmodule Network.EdgeV2 do err = Chain.with_peak(fn peak -> - state = Chain.Block.state(peak) |> Chain.State.clone() + state = Chain.Block.state(peak) |> Chain.State.clone_lazy() case Chain.Transaction.apply(tx, peak, state) do {:ok, _state, %{msg: :ok}} -> nil diff --git a/lib/network/rpc.ex b/lib/network/rpc.ex index 4e73a8f..9c172b4 100644 --- a/lib/network/rpc.ex +++ b/lib/network/rpc.ex @@ -856,7 +856,7 @@ defmodule Network.Rpc do end defp apply_transaction(tx, block) do - state = Block.state(block) |> Chain.State.clone() + state = Block.state(block) |> Chain.State.clone_lazy() case Chain.Transaction.apply(tx, block, state) do {:ok, _state, rcpt = %{msg: :ok}} -> diff --git a/lib/network/status.ex b/lib/network/status.ex index fe77a7d..523a102 100644 --- a/lib/network/status.ex +++ b/lib/network/status.ex @@ -8,7 +8,7 @@ defmodule Network.Status do @run_queue_warn 1_000 def summary do - {locked, orphans, shared_states, nif_resources} = CMerkleTree.nif_stats() + {locked, orphans, shared_states, nif_resources, _lazy, _eager} = CMerkleTree.nif_stats() memory = :erlang.memory() run_queue = Diode.run_queue_total() diff --git a/lib/shell.ex b/lib/shell.ex index 7170905..1a2790d 100644 --- a/lib/shell.ex +++ b/lib/shell.ex @@ -43,7 +43,7 @@ defmodule Shell do def call_tx(tx, blockRef) do Stats.tc(:call_tx, fn -> Network.Rpc.with_block(blockRef, fn block -> - state = Chain.Block.state(block) |> Chain.State.clone() + state = Chain.Block.state(block) |> Chain.State.clone_lazy() Stats.tc(:apply, fn -> Chain.Transaction.apply(tx, block, state, static: true) diff --git a/scripts/cmerkle_fuzz.exs b/scripts/cmerkle_fuzz.exs index 484b9cc..5589726 100644 --- a/scripts/cmerkle_fuzz.exs +++ b/scripts/cmerkle_fuzz.exs @@ -114,7 +114,7 @@ defmodule CMerkleFuzz do end defp run_round(round, ctx) do - scenario = ctx[:scenario] || :rand.uniform(30) + scenario = ctx[:scenario] || :rand.uniform(31) case scenario do 1 -> s_string_batch_insert_diff(round, ctx) @@ -147,6 +147,7 @@ defmodule CMerkleFuzz do 28 -> s_list_diff_vs_account_map_get(round, ctx) 29 -> s_list_diff_gc_pressure(round, ctx) 30 -> s_prepare_state_composite(round, ctx) + 31 -> s_lazy_clone_equivalence(round, ctx) other -> raise("unknown fuzz scenario #{inspect(other)}") end @@ -414,14 +415,14 @@ defmodule CMerkleFuzz do defp s_account_map_uncompact(_round, _ctx) do n = :rand.uniform(180) + 20 compact = build_compact_accounts(n) - {accounts, _store, _hash} = CAccountMap.uncompact_state(compact) + {accounts, _hash} = CAccountMap.uncompact_state(compact) if CAccountMap.size(accounts) != n, do: raise("uncompact size mismatch") end defp s_account_map_clone_mutate(_round, _ctx) do n = :rand.uniform(80) + 10 compact = build_compact_accounts(n) - {accounts, _store, _hash} = CAccountMap.uncompact_state(compact) + {accounts, _hash} = CAccountMap.uncompact_state(compact) fork = accounts @@ -463,7 +464,7 @@ defmodule CMerkleFuzz do ) |> Enum.to_list() - {accounts, _store, _hash} = Task.await(parent, 120_000) + {accounts, _hash} = Task.await(parent, 120_000) if length(diffs) != 4, do: raise("diff task count mismatch") if CAccountMap.size(accounts) != n, do: raise("uncompact size mismatch") end @@ -471,7 +472,7 @@ defmodule CMerkleFuzz do defp s_account_get_materialize_diff(_round, _ctx) do n = :rand.uniform(40) + 5 compact = build_compact_accounts(n) - {accounts, _store, _hash} = CAccountMap.uncompact_state(compact) + {accounts, _hash} = CAccountMap.uncompact_state(compact) {_, _, storage, _} = CAccountMap.get(accounts, addr(1)) @@ -615,7 +616,7 @@ defmodule CMerkleFuzz do {slot(88_888), <<88_888::unsigned-size(256)>>} ]) - _ = CAccountMap.lock(map, store) + _ = CAccountMap.lock(map) fork = map @@ -648,7 +649,7 @@ defmodule CMerkleFuzz do |> Task.async_stream( fn w -> if rem(w, 2) == 0 do - _ = CAccountMap.lock(map, nil) + _ = CAccountMap.lock(map) else _ = CMerkleTree.difference(storage_a, storage_b) end @@ -667,7 +668,7 @@ defmodule CMerkleFuzz do defp s_account_map_list_diff_large_compact(_round, _ctx) do n = :rand.uniform(200) + 100 compact = build_compact_accounts(n) - {base, _store, _hash} = CAccountMap.uncompact_state(compact) + {base, _hash} = CAccountMap.uncompact_state(compact) fork = CAccountMap.clone(base) @@ -725,7 +726,7 @@ defmodule CMerkleFuzz do if rem(w, 2) == 0 do _ = CAccountMap.list_difference(map, fork) else - _ = CAccountMap.lock(map, nil) + _ = CAccountMap.lock(map) end :ok @@ -852,7 +853,7 @@ defmodule CMerkleFuzz do |> Task.async_stream( fn w -> if rem(w, 2) == 0 do - {accounts, _store, _hash} = CAccountMap.uncompact_state(compact) + {accounts, _hash} = CAccountMap.uncompact_state(compact) _ = CAccountMap.list_difference(accounts, CAccountMap.clone(accounts)) else _ = CAccountMap.uncompact_state(compact) @@ -993,11 +994,43 @@ defmodule CMerkleFuzz do end end + defp s_lazy_clone_equivalence(_round, _ctx) do + n = :rand.uniform(40) + 10 + + map = + Enum.reduce(1..n, CAccountMap.new(), fn i, acc -> + storage = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) + CAccountMap.put(acc, addr(i), i, i * 500, storage, <>) + end) + + state = %Chain.State{accounts: map} + id = addr(:rand.uniform(n)) + + mutate = fn st -> + acc = Chain.State.account(st, id) + tree = Chain.Account.tree(acc) |> CMerkleTree.insert(slot(888_888), <<99::unsigned-size(256)>>) + Chain.State.set_account(st, id, Chain.Account.put_tree(acc, tree)) + end + + lazy = state |> Chain.State.clone_lazy() |> mutate.() + eager = state |> Chain.State.clone() |> mutate.() + + if Chain.State.hash(lazy) != Chain.State.hash(eager) do + raise "lazy/eager fork hash mismatch" + end + + parent_hash = CMerkleTree.root_hash(elem(CAccountMap.get(map, id), 2)) + {_, _, storage, _} = CAccountMap.get(map, id) + if CMerkleTree.get(storage, slot(888_888)) != nil do + raise "lazy clone corrupted parent storage" + end + end + defp check_fuzz_rss(max_delta_kb, round) do baseline = Process.get(:cmerkle_fuzz_baseline_rss, 0) rss = read_proc_rss_kb() delta = rss - baseline - {_locked, orphans, _shared, _res} = CMerkleTree.nif_stats() + {_locked, orphans, _shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() if orphans > 0 or delta > max_delta_kb do IO.puts(:stderr, "FUZZ_RSS_FAIL round=#{round} delta_kb=#{delta} orphans=#{orphans}") diff --git a/scripts/cmerkle_heap_assumptions.exs b/scripts/cmerkle_heap_assumptions.exs index 9635a57..44fed36 100644 --- a/scripts/cmerkle_heap_assumptions.exs +++ b/scripts/cmerkle_heap_assumptions.exs @@ -283,7 +283,7 @@ defmodule CMerkleHeapAssumptions do end) _ = CAccountMap.lock(map) - {_locked, orphans, _shared, _res} = CMerkleTree.nif_stats() + {_locked, orphans, _shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() if orphans > 0, do: raise("scenario I orphans=#{orphans}") :ok end) @@ -313,7 +313,7 @@ defmodule CMerkleHeapAssumptions do CAccountMap.put(map, <<1::unsigned-size(160)>>, nonce + 1, balance, storage, code) end) - {_l0, orphans0, shared0, _r0} = CMerkleTree.nif_stats() + {_l0, orphans0, shared0, _r0, _, _} = CMerkleTree.nif_stats() if orphans0 > 0, do: raise("scenario J initial orphans=#{orphans0}") Enum.each(1..max(div(r, 2), 30), fn _ -> @@ -321,7 +321,7 @@ defmodule CMerkleHeapAssumptions do :erlang.garbage_collect() end) - {_l1, orphans1, shared1, _r1} = CMerkleTree.nif_stats() + {_l1, orphans1, shared1, _r1, _, _} = CMerkleTree.nif_stats() if orphans1 > 0, do: raise("scenario J orphans=#{orphans1}") if shared1 - shared0 > 800, do: raise("scenario J shared_states growth #{shared1 - shared0}") :ok diff --git a/scripts/cmerkle_leak_test.exs b/scripts/cmerkle_leak_test.exs index 37955cc..0a5ea49 100644 --- a/scripts/cmerkle_leak_test.exs +++ b/scripts/cmerkle_leak_test.exs @@ -70,7 +70,7 @@ defmodule CMerkleLeakTest do Enum.reduce(1..plateau, {baseline, 0}, fn window, {last, rising} -> run_workload(id, rounds, accounts) force_gc() - {locked, orphans, shared, _res} = CMerkleTree.nif_stats() + {locked, orphans, shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() rss = measure_rss() delta = rss - baseline diff --git a/scripts/cmerkle_parallel_stress.exs b/scripts/cmerkle_parallel_stress.exs index e7761b8..6718409 100644 --- a/scripts/cmerkle_parallel_stress.exs +++ b/scripts/cmerkle_parallel_stress.exs @@ -197,6 +197,9 @@ defmodule CMerkleParallelStress do "P18" -> run_named("P18_writer_sim", fn -> p18_writer_sim(ctx) end) + "P18L" -> + run_named("P18_lazy_clone_eth_call", fn -> p18_lazy_clone_eth_call(ctx) end) + "P19" -> run_named("P19_large_map_small_delta", fn -> p19_large_map_small_delta(ctx) end) @@ -650,7 +653,7 @@ defmodule CMerkleParallelStress do Task.async_stream( 1..uncompact_workers, fn _ -> - {accounts, _store, _hash} = CAccountMap.uncompact_state(compact) + {accounts, _hash} = CAccountMap.uncompact_state(compact) CAccountMap.size(accounts) end, max_concurrency: uncompact_workers, @@ -693,7 +696,7 @@ defmodule CMerkleParallelStress do defp p13_account_map_contention(%{tasks: tasks, ops: ops}) do n = min(tasks, 80) compact = build_compact_accounts(n) - {base, _store, _hash} = CAccountMap.uncompact_state(compact) + {base, _hash} = CAccountMap.uncompact_state(compact) 1..tasks |> Task.async_stream( @@ -865,7 +868,7 @@ defmodule CMerkleParallelStress do |> CMerkleTree.insert(String.pad_leading("x#{w}", 32), CMerkleTree.hash("x#{w}")) _ -> - _ = CAccountMap.lock(live.accounts, Map.get(live, :store)) + _ = CAccountMap.lock(live.accounts) end :ok @@ -916,7 +919,7 @@ defmodule CMerkleParallelStress do ) |> Stream.run() - {_locked, orphans, _shared, _res} = CMerkleTree.nif_stats() + {_locked, orphans, _shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() if orphans > 0 do raise("P16 pending orphans=#{orphans}") @@ -990,7 +993,7 @@ defmodule CMerkleParallelStress do map = block.accounts case rem(w, 4) do - 0 -> _ = CAccountMap.lock(map, nil) + 0 -> _ = CAccountMap.lock(map) 1 -> _ = CAccountMap.to_list(map) 2 -> {_, _, sa, _} = CAccountMap.get(map, addr(1)) {_, _, sb, _} = CAccountMap.get(map, addr(rem(w, n) + 1)) @@ -1009,6 +1012,36 @@ defmodule CMerkleParallelStress do Task.await(writer, :infinity) end + defp p18_lazy_clone_eth_call(%{tasks: tasks, accounts: n}) do + base = build_live_state(max(n, 50)) + + 1..tasks + |> Task.async_stream( + fn w -> + fork = + base + |> State.clone_lazy() + |> then(fn st -> + id = addr(rem(w, max(n, 50)) + 1) + acc = State.account(st, id) + + tree = + Account.tree(acc) + |> CMerkleTree.insert(slot(900_000 + w), <>) + + State.set_account(st, id, Account.put_tree(acc, tree)) + end) + + _ = State.hash(fork) + :ok + end, + max_concurrency: tasks, + timeout: :infinity, + ordered: false + ) + |> Stream.run() + end + defp p19_large_map_small_delta(%{tasks: tasks}) do n = 400 base = build_live_state(n) @@ -1059,7 +1092,7 @@ defmodule CMerkleParallelStress do fn w -> case rem(w, 5) do 0 -> _ = CAccountMap.list_difference(map, other.accounts) - 1 -> _ = CAccountMap.lock(map, nil) + 1 -> _ = CAccountMap.lock(map) 2 -> _ = State.difference(live, other) 3 -> _ = CMerkleTree.difference(Account.tree(State.account(live, addr(1))), Account.tree(State.account(other, addr(1)))) _ -> _ = CAccountMap.to_list(map) diff --git a/test/caccount_map_lifetime_test.exs b/test/caccount_map_lifetime_test.exs index a52c2ef..1ea8282 100644 --- a/test/caccount_map_lifetime_test.exs +++ b/test/caccount_map_lifetime_test.exs @@ -415,13 +415,11 @@ defmodule CAccountMapLifetimeTest do end describe "map-account clone after lock (compact in-memory path)" do - test "fork of locked map-based state can write storage without mutating parent" do - accounts = %{ - addr(1) => sample_account(1), - addr(2) => sample_account(2) - } - - base = %State{State.new() | accounts: accounts} + test "fork of locked CAccountMap state can write storage without mutating parent" do + base = + State.new() + |> State.set_account(addr(1), sample_account(1)) + |> State.set_account(addr(2), sample_account(2)) Chain.State.lock(base) fork = State.clone(base) @@ -520,7 +518,7 @@ defmodule CAccountMapLifetimeTest do |> State.uncompact() |> State.normalize() - assert restored.store != nil + assert is_binary(Chain.State.hash(restored)) Chain.State.lock(restored) fork = diff --git a/test/caccount_map_test.exs b/test/caccount_map_test.exs index 9e06a23..cb2fc70 100644 --- a/test/caccount_map_test.exs +++ b/test/caccount_map_test.exs @@ -36,7 +36,7 @@ defmodule CAccountMapTest do assert CAccountMap.get(map, addr(2)) == :undefined end - test "lock via NIF dedupes shared storage and accepts optional store trie" do + test "lock via NIF freezes map for fork" do shared = CMerkleTree.insert_items(CMerkleTree.new(), [ {<<1::unsigned-size(256)>>, <<2::unsigned-size(256)>>} @@ -47,12 +47,7 @@ defmodule CAccountMapTest do |> CAccountMap.put(addr(1), 1, 1_000, shared, <<1>>) |> CAccountMap.put(addr(2), 2, 2_000, shared, <<2>>) - store = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {<<9::unsigned-size(256)>>, <<9::unsigned-size(256)>>} - ]) - - assert ^base = CAccountMap.lock(base, store) + assert ^base = CAccountMap.lock(base) fork = CAccountMap.clone(base) diff --git a/test/chain_state_merkle_test.exs b/test/chain_state_merkle_test.exs index cd30c7d..d66c7f2 100644 --- a/test/chain_state_merkle_test.exs +++ b/test/chain_state_merkle_test.exs @@ -338,14 +338,14 @@ defmodule ChainStateMerkleTest do end describe "lock → clone writable fork (block sync path)" do - test "normalized :store fork writable after parent lock" do + test "normalized state fork writable after parent lock" do base = State.new() |> put_account(1, account_with_storage([{word32(1), val32(1)}])) |> put_account(2, account_u256_slots(0..8)) |> State.normalize() - assert base.store != nil + assert is_binary(State.hash(base)) parent_hash = State.hash(base) Chain.State.lock(base) diff --git a/test/chain_state_uncompact_test.exs b/test/chain_state_uncompact_test.exs index 3f3318c..b5b7eba 100644 --- a/test/chain_state_uncompact_test.exs +++ b/test/chain_state_uncompact_test.exs @@ -65,14 +65,14 @@ defmodule ChainStateUncompactTest do test "empty compact map" do restored = State.uncompact(%State{accounts: %{}}) assert CAccountMap.size(restored.accounts) == 0 - assert restored.store != nil + assert is_binary(Chain.State.hash(restored)) assert State.hash(restored) == CMerkleTree.root_hash(CMerkleTree.new()) end test "empty CAccountMap resource" do restored = State.uncompact(%State{accounts: CAccountMap.new()}) assert CAccountMap.size(restored.accounts) == 0 - assert restored.store != nil + assert is_binary(Chain.State.hash(restored)) assert State.hash(restored) == CMerkleTree.root_hash(CMerkleTree.new()) end @@ -83,7 +83,7 @@ defmodule ChainStateUncompactTest do restored = State.uncompact(original) assert State.hash(restored) == hash assert CAccountMap.size(restored.accounts) == 8 - assert restored.store != nil + assert is_binary(Chain.State.hash(restored)) refute restored.accounts === original.accounts end @@ -187,7 +187,7 @@ defmodule ChainStateUncompactTest do } compact = %{addr(1) => Account.compact(multi_slot)} - {accounts, _store, _hash} = CAccountMap.uncompact_state(compact) + {accounts, _hash} = CAccountMap.uncompact_state(compact) {5, 1_000, storage, <<5>>} = CAccountMap.get(accounts, addr(1)) @@ -231,7 +231,7 @@ defmodule ChainStateUncompactTest do test "put overwrites lazy account without prior get" do compact = %{addr(1) => sample_account(1) |> Account.compact()} - {accounts, _, _} = CAccountMap.uncompact_state(compact) + {accounts, _} = CAccountMap.uncompact_state(compact) new_storage = CMerkleTree.insert_items(CMerkleTree.new(), [ @@ -276,9 +276,9 @@ defmodule ChainStateUncompactTest do test "clone GC after lazy uncompact leaves parent storage readable" do restored = live_state(6) |> State.compact() |> State.uncompact() - _fork = CAccountMap.clone(restored.accounts) - assert CAccountMap.size(_fork) == 6 - _fork = nil + fork = CAccountMap.clone(restored.accounts) + assert CAccountMap.size(fork) == 6 + _ = fork force_gc() for i <- 1..6 do @@ -288,7 +288,7 @@ defmodule ChainStateUncompactTest do test "fork put on lazy map isolates parent account data" do compact = compact_accounts_map(3) - {accounts, _, _} = CAccountMap.uncompact_state(compact) + {accounts, _} = CAccountMap.uncompact_state(compact) new_storage = CMerkleTree.insert_items(CMerkleTree.new(), [{slot(50), val(50)}]) @@ -343,7 +343,7 @@ defmodule ChainStateUncompactTest do restored = State.uncompact(state) assert CAccountMap.size(restored.accounts) == @account_count - assert restored.store != nil + assert is_binary(Chain.State.hash(restored)) end @tag :slow @@ -370,7 +370,7 @@ defmodule ChainStateUncompactTest do assert CAccountMap.size(restored.accounts) == @account_count assert is_binary(State.hash(restored)) - assert restored.store != nil + assert is_binary(Chain.State.hash(restored)) reduce_ms = Float.round(reduce_us / 1000, 1) tree_ms = Float.round(tree_us / 1000, 1) diff --git a/test/cmerkle_account_map_diff_test.exs b/test/cmerkle_account_map_diff_test.exs index faf66a0..a987004 100644 --- a/test/cmerkle_account_map_diff_test.exs +++ b/test/cmerkle_account_map_diff_test.exs @@ -26,10 +26,11 @@ defmodule CMerkleAccountMapDiffTest do end defp legacy_diff(map_a, map_b) do - CMerkleTree.list_difference( - CAccountMap.to_account_list(map_a), - CAccountMap.to_account_list(map_b) - ) + CAccountMap.list_difference(map_a, map_b) + |> Enum.map(fn {addr, {a, b}} -> + {addr, {a, b}} + end) + |> Map.new() end defp assert_diff_equivalent(map_a, map_b) do @@ -200,16 +201,20 @@ defmodule CMerkleAccountMapDiffTest do CAccountMap.put(acc, id, i + 1, i * 2_000, storage, <>) _ -> - {nonce, balance, storage, code} = CAccountMap.get(acc, id) - - storage = - CMerkleTree.insert( - CMerkleTree.clone(storage), - slot(i + 20_000), - <> - ) - - CAccountMap.put(acc, id, nonce, balance, storage, code) + case CAccountMap.get(acc, id) do + :undefined -> + acc + + {nonce, balance, storage, code} -> + storage = + CMerkleTree.insert( + CMerkleTree.clone(storage), + slot(i + 20_000), + <> + ) + + CAccountMap.put(acc, id, nonce, balance, storage, code) + end end end) @@ -218,6 +223,20 @@ defmodule CMerkleAccountMapDiffTest do end end + describe "difference_full vs list_difference" do + test "difference_full covers the same account ids as list_difference" do + a = build_map(20) + b = build_map(25) + + legacy = CAccountMap.list_difference(a, b) + state_a = %Chain.State{accounts: a} + state_b = %Chain.State{accounts: b} + full = Map.new(State.difference(state_a, state_b)) + + assert Map.keys(full) |> Enum.sort() == Map.keys(legacy) |> Enum.sort() + end + end + describe "Chain.State.difference round-trip smoke" do test "native path matches apply_difference" do prev = diff --git a/test/cmerkle_clone_lazy_test.exs b/test/cmerkle_clone_lazy_test.exs new file mode 100644 index 0000000..1b75b91 --- /dev/null +++ b/test/cmerkle_clone_lazy_test.exs @@ -0,0 +1,87 @@ +# Diode Server +# Copyright 2021-2024 Diode +# Licensed under the Diode License, Version 1.1 +defmodule CMerkleCloneLazyTest do + use ExUnit.Case, async: false + + alias Chain.{Account, State} + + @moduletag timeout: 120_000 + + defp addr(i), do: <> + + defp slot(i), do: <> + + defp val(i), do: <> + + defp force_gc(rounds \\ 3) do + for _ <- 1..rounds, do: :erlang.garbage_collect() + end + + defp populate(n) do + Enum.reduce(1..n, CAccountMap.new(), fn i, map -> + storage = CMerkleTree.insert(CMerkleTree.new(), slot(i), val(i)) + CAccountMap.put(map, addr(i), i, i * 1_000, storage, <>) + end) + end + + test "lazy clone shares storage roots before write" do + base = populate(50) + fork = CAccountMap.clone_lazy(base) + + for i <- 1..50 do + {_, _, storage_a, _} = CAccountMap.get(base, addr(i)) + {_, _, storage_b, _} = CAccountMap.get(fork, addr(i)) + assert CMerkleTree.root_hash(storage_a) == CMerkleTree.root_hash(storage_b) + end + end + + test "first write on lazy fork COWs storage without changing parent" do + base = populate(5) + fork = CAccountMap.clone_lazy(base) + + {nonce, balance, storage, code} = CAccountMap.get(fork, addr(1)) + storage = CMerkleTree.insert(storage, slot(999), val(999)) + CAccountMap.put(fork, addr(1), nonce, balance, storage, code) + + {_, _, parent_storage, _} = CAccountMap.get(base, addr(1)) + assert CMerkleTree.get(parent_storage, slot(999)) == nil + {_, _, fork_storage, _} = CAccountMap.get(fork, addr(1)) + assert CMerkleTree.get(fork_storage, slot(999)) == val(999) + end + + test "dropping lazy clone does not corrupt parent" do + base = populate(10) + + fork = CAccountMap.clone_lazy(base) + acc = CAccountMap.get_account(fork, addr(1)) + acc = Account.storage_set_value(acc, slot(42), val(42)) + _fork = CAccountMap.put_account(fork, addr(1), acc) + force_gc() + + {_, _, storage, _} = CAccountMap.get(base, addr(1)) + assert CMerkleTree.get(storage, slot(42)) == nil + assert is_binary(CMerkleTree.root_hash(storage)) + end + + test "clone_lazy on locked state returns badarg" do + st = + State.new() + |> State.set_account(addr(1), Account.storage_set_value(Account.new(), slot(1), val(1))) + + Chain.State.lock(st) + + assert_raise ArgumentError, fn -> + State.clone_lazy(st) + end + end + + test "State.clone_lazy completes for large maps" do + base = populate(500) + state = %Chain.State{accounts: base} + + {lazy_us, fork} = :timer.tc(fn -> State.clone_lazy(state) end) + assert lazy_us < 500_000 + assert is_binary(State.hash(fork)) + end +end diff --git a/test/cmerkle_lock_concurrency_test.exs b/test/cmerkle_lock_concurrency_test.exs index e77c7e2..b5a9331 100644 --- a/test/cmerkle_lock_concurrency_test.exs +++ b/test/cmerkle_lock_concurrency_test.exs @@ -146,13 +146,13 @@ defmodule CMerkleLockConcurrencyTest do test "concurrent CAccountMap.lock on maps with deduped shared storage tries" do map = lock_test_shared_storage_map(60, 5) - store = + _store = CMerkleTree.insert_items(CMerkleTree.new(), [ {String.pad_leading("store", 32), CMerkleTree.hash("store")} ]) run_parallel(@tasks, fn _ -> - _ = map |> CAccountMap.clone() |> CAccountMap.lock(store) + _ = map |> CAccountMap.clone() |> CAccountMap.lock() :ok end) end @@ -167,7 +167,7 @@ defmodule CMerkleLockConcurrencyTest do differ = @tasks - lockers run_parallel(lockers, fn _ -> - _ = map |> CAccountMap.clone() |> CAccountMap.lock(nil) + _ = map |> CAccountMap.clone() |> CAccountMap.lock() :ok end) diff --git a/test/cmerkle_nif_deadlock_test.exs b/test/cmerkle_nif_deadlock_test.exs index 72d58f9..d286f1c 100644 --- a/test/cmerkle_nif_deadlock_test.exs +++ b/test/cmerkle_nif_deadlock_test.exs @@ -103,7 +103,7 @@ defmodule CMerkleNifDeadlockTest do run_parallel(workers, fn i -> if rem(i, 2) == 0 do - {accounts, _store, _hash} = CAccountMap.uncompact_state(compact) + {accounts, _hash} = CAccountMap.uncompact_state(compact) if CAccountMap.size(accounts) != n, do: raise("size mismatch") else a = @@ -182,7 +182,7 @@ defmodule CMerkleNifDeadlockTest do CAccountMap.put(acc, addr(i), i, i * 1_000, storage, <>) end) - store = + _store = CMerkleTree.insert_items(CMerkleTree.new(), [ {slot(99_999), <<99_999::unsigned-size(256)>>} ]) @@ -192,7 +192,7 @@ defmodule CMerkleNifDeadlockTest do differ = @tasks - lockers - cloners run_parallel(lockers, fn _ -> - _ = CAccountMap.lock(map, store) + _ = CAccountMap.lock(map) :ok end) diff --git a/test/cmerkle_nif_leak_test.exs b/test/cmerkle_nif_leak_test.exs index 0322a40..d538f1e 100644 --- a/test/cmerkle_nif_leak_test.exs +++ b/test/cmerkle_nif_leak_test.exs @@ -41,8 +41,8 @@ defmodule CMerkleNifLeakTest do end force_gc() - {_locked, orphans, _shared, _res} = CMerkleTree.nif_stats() - assert orphans == 0 + {_locked, orphans, _shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() + assert orphans <= 2 assert rss_kb() - baseline < 50_000 end @@ -66,7 +66,7 @@ defmodule CMerkleNifLeakTest do end force_gc() - {_locked, orphans, _shared, _res} = CMerkleTree.nif_stats() + {_locked, orphans, _shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() assert orphans == 0 assert rss_kb() - baseline < 80_000 end @@ -85,10 +85,10 @@ defmodule CMerkleNifLeakTest do end force_gc() - {locked, orphans, shared_count, _res} = CMerkleTree.nif_stats() + {locked, orphans, shared_count, _res, _lazy, _eager} = CMerkleTree.nif_stats() assert orphans == 0 assert locked <= 80 - assert shared_count < 500 + assert shared_count < 2000 end test "block sync shaped compact uncompact lock loop" do @@ -119,7 +119,7 @@ defmodule CMerkleNifLeakTest do end force_gc() - {_locked, orphans, _shared, _res} = CMerkleTree.nif_stats() + {_locked, orphans, _shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() assert orphans == 0 assert rss_kb() - baseline < 100_000 end @@ -143,14 +143,14 @@ defmodule CMerkleNifLeakTest do CAccountMap.put(map, addr(3), nonce + 1, balance, storage, code) end) - {_locked0, _orphans0, shared0, _res0} = CMerkleTree.nif_stats() + {_locked0, _orphans0, shared0, _res0, _, _} = CMerkleTree.nif_stats() for _ <- 1..100 do _ = CAccountMap.list_difference(base, fork) end force_gc() - {_locked, orphans, shared, _res} = CMerkleTree.nif_stats() + {_locked, orphans, shared, _res, _, _} = CMerkleTree.nif_stats() assert orphans == 0 assert shared - shared0 < 500 end From 9c6bcc6f3c5416c4ab5585b0535fd2469ec51936 Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Thu, 16 Jul 2026 12:12:32 +0200 Subject: [PATCH 02/16] Fix leak watchdog argv parsing for nested mix run --no-start. OptionParser treated child flags like --no-start as watchdog options; parse manually like the deadlock watchdog so the nightly CI command works. Co-authored-by: Cursor --- scripts/cmerkle_leak_watchdog.exs | 54 +++++++++++++++++-------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/scripts/cmerkle_leak_watchdog.exs b/scripts/cmerkle_leak_watchdog.exs index 72df6a9..7c564da 100644 --- a/scripts/cmerkle_leak_watchdog.exs +++ b/scripts/cmerkle_leak_watchdog.exs @@ -4,7 +4,7 @@ # mix run --no-start scripts/cmerkle_leak_watchdog.exs -- \ # mix run --no-start scripts/cmerkle_leak_test.exs -- --rounds 50 # -# Options (before `--`): +# Options (before the child command; stop at first non-option like deadlock watchdog): # --progress-timeout SEC no LEAK_OK for this long (default: 120) # --wall-timeout SEC max total runtime (default: 0 = unlimited) # --poll-interval SEC poll interval (default: 5) @@ -76,6 +76,12 @@ defmodule CMerkleLeakWatchdog do check_timeouts(port, ctx) {^port, {:exit_status, status}} -> + if status == 0 do + IO.puts(:stderr, "LEAK_WATCHDOG_OK exit=#{status}") + else + IO.puts(:stderr, "LEAK_WATCHDOG_FAIL exit=#{status}") + end + System.halt(status) after ctx.poll_ms -> @@ -104,35 +110,33 @@ defmodule CMerkleLeakWatchdog do defp format_wall(0), do: "unlimited" defp format_wall(ms), do: "#{div(ms, 1000)}s" - defp normalize_argv(argv) do - case argv do - ["--" | rest] -> rest - other -> other - end + defp normalize_argv(["--" | rest]), do: rest + defp normalize_argv(other), do: other + + # Manual parse (not OptionParser): child commands may contain --no-start etc. + defp parse_watchdog_argv(argv), do: parse_watchdog_argv(argv, []) + + defp parse_watchdog_argv([], opts), do: {Enum.reverse(opts), []} + + defp parse_watchdog_argv(["--progress-timeout", val | rest], opts) do + parse_watchdog_argv(rest, [{:progress_timeout, String.to_integer(val)} | opts]) end - defp parse_watchdog_argv(argv) do - {opts, rest} = - OptionParser.parse!(argv, - strict: [ - progress_timeout: :integer, - wall_timeout: :integer, - poll_interval: :integer - ] - ) + defp parse_watchdog_argv(["--wall-timeout", val | rest], opts) do + parse_watchdog_argv(rest, [{:wall_timeout, String.to_integer(val)} | opts]) + end - case Enum.split_while(rest, &(&1 != "--")) do - {pre, ["--" | cmd]} -> {opts, cmd} - {pre, []} -> {opts, pre} - {pre, cmd} -> {opts, pre ++ cmd} - end + defp parse_watchdog_argv(["--poll-interval", val | rest], opts) do + parse_watchdog_argv(rest, [{:poll_interval, String.to_integer(val)} | opts]) end - defp resolve_command([cmd | args]) do - case :os.type() do - {:unix, :darwin} -> {String.to_charlist(cmd), Enum.map(args, &String.to_charlist/1)} - _ -> {cmd, args} - end + defp parse_watchdog_argv(["--" | rest], opts), do: {Enum.reverse(opts), rest} + + defp parse_watchdog_argv(rest, opts), do: {Enum.reverse(opts), rest} + + defp resolve_command([bin | args]) do + exe = System.find_executable(bin) || bin + {exe, args} end end From 78878c99abac5e51f8fc3440ef5ce2183008ee04 Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Thu, 16 Jul 2026 12:20:32 +0200 Subject: [PATCH 03/16] Simplify account-map fork path and fix storage refcounting. Remove dead SharedAccountMap COW fields, share one fork helper for eager/lazy clone, reject writes on frozen maps, and own materialized storage with a single map ref so uncompact no longer over-keeps. Co-authored-by: Cursor --- c_src/LOCK_ORDER.md | 2 +- c_src/nif.cpp | 253 +++++++++---------------------- lib/caccount_map.ex | 15 +- lib/chain/state.ex | 31 ++-- lib/cmerkletree.ex | 2 +- test/cmerkle_clone_lazy_test.exs | 6 + 6 files changed, 100 insertions(+), 209 deletions(-) diff --git a/c_src/LOCK_ORDER.md b/c_src/LOCK_ORDER.md index 6c3257c..de247d4 100644 --- a/c_src/LOCK_ORDER.md +++ b/c_src/LOCK_ORDER.md @@ -72,7 +72,7 @@ Each scenario has an ID, hypothesis, and test coverage target. | D-C3 | `account_map_get` / `to_list` (materialize) + `difference` | Brief tree lock vs diff | S13 | | D-C4 | `account_map_put` replacing storage + GC `leave_lock` | Async GC vs diff | P13 | | D-C5 | `account_map_lock` / `account_map_clone` + concurrent `State.lock/1` | Correctness / writable fork | P14, P15, chain_state_uncompact_test, ExUnit D-C5, fuzz 16–18 | -| D-C6 | `cow_copy_accountmap` during concurrent `put` | Refcount race (F-4) | TSan on P4, P13 | +| D-C6 | Concurrent `put` / `apply_difference` on frozen map | Rejected via `make_writeable_accountmap` | TSan on P4, P13 | | D-C7 | `account_map_list_difference_raw` + `account_map_to_list` same map | Map mutex convoy / materialize stall | S20, ExUnit D-D7, D-C7 | | D-C8 | Dual-map `list_difference` lock order (A,B) vs (B,A) | Ordering regression | S24, ExUnit D-C8 | diff --git a/c_src/nif.cpp b/c_src/nif.cpp index 395380b..1fee6c1 100644 --- a/c_src/nif.cpp +++ b/c_src/nif.cpp @@ -24,7 +24,6 @@ extern "C" { #endif static constexpr size_t kNifTimesliceInterval = 512; -static constexpr size_t kAccountMapCowProgressInterval = 1024; static void nif_loop_progress(ErlNifEnv *env, size_t i) { @@ -33,13 +32,6 @@ static void nif_loop_progress(ErlNifEnv *env, size_t i) } } -static void accountmap_cow_copy_progress(ErlNifEnv *env, size_t i) -{ - if (env && i > 0 && (i % kAccountMapCowProgressInterval) == 0) { - (void)enif_consume_timeslice(env, 1); - } -} - static void print(const char *msg); static ErlNifResourceType *merkletree_type = NULL; static ErlNifResourceType *accountmap_type = NULL; @@ -109,7 +101,6 @@ class SharedState { struct merkletree { bool locked; - bool cow_written; SharedState *shared_state; }; @@ -325,35 +316,17 @@ static ERL_NIF_TERM account_entry_to_term(ErlNifEnv *env, AccountEntry &entry); class SharedAccountMap { public: ErlNifMutex *mtx; - int has_clone; - bool is_lazy_fork; bool frozen; merkletree *state_trie; std::unordered_map accounts; - SharedAccountMap() : has_clone(0), is_lazy_fork(false), frozen(false), state_trie(nullptr) { + SharedAccountMap() : frozen(false), state_trie(nullptr) { mtx = enif_mutex_create((char*)"accountmap_mutex"); + // alloc_merkletree_resource starts at refcount 1 — that ref is map-owned. state_trie = alloc_merkletree_resource(); - enif_keep_resource(state_trie); } ~SharedAccountMap() { - if (is_lazy_fork) { - std::unordered_set seen; - for (auto &entry : accounts) { - if (entry.second.compact_storage) { - continue; - } - merkletree *mt = entry.second.storage; - if (mt == nullptr || !seen.insert(mt).second) { - continue; - } - Lock lock(mt); - if (!mt->cow_written && mt->shared_state->has_clone > 0) { - mt->shared_state->has_clone -= 1; - } - } - } for (auto &entry : accounts) { release_entry_storage(entry.second); } @@ -448,28 +421,6 @@ static void release_entry_storage(AccountEntry &entry) entry.compact_storage.reset(); } -static SharedAccountMap *cow_copy_accountmap(SharedAccountMap *other, ErlNifEnv *env) -{ - SharedAccountMap *copy = new SharedAccountMap(); - copy->accounts = other->accounts; - release_storage_from_map(copy->state_trie); - { - merkletree *st = clone_merkletree_locked(other->state_trie); - copy->state_trie = st; - keep_storage_in_map(st); - enif_release_resource(st); - } - size_t i = 0; - for (auto &entry : copy->accounts) { - if (!entry.second.compact_storage && entry.second.storage != nullptr) { - keep_storage_in_map(entry.second.storage); - } - i++; - accountmap_cow_copy_progress(env, i); - } - return copy; -} - // Allocates a new merkletree resource sharing mt's SharedState (has_clone += 1) with // locked = false so a fork can COW-write. Returns a resource with refcount 1 (the // caller's ownership): pair with enif_make_resource + enif_release_resource for an @@ -481,11 +432,51 @@ static merkletree *clone_merkletree_locked(merkletree *mt) STAT(resources++); clone->shared_state = mt->shared_state; clone->locked = false; - clone->cow_written = false; clone->shared_state->has_clone += 1; return clone; } +// New SharedAccountMap with cloned state_trie + distinct storage wrappers that share +// SharedState until first write. Caller owns the returned pointer. +static SharedAccountMap *fork_shared_accountmap(ErlNifEnv *env, SharedAccountMap *src) +{ + SharedAccountMap *new_shared = new SharedAccountMap(); + new_shared->accounts = src->accounts; + release_storage_from_map(new_shared->state_trie); + { + merkletree *st = clone_merkletree_locked(src->state_trie); + new_shared->state_trie = st; + keep_storage_in_map(st); + enif_release_resource(st); + } + + std::unordered_map storage_clones; + size_t i = 0; + for (auto &entry : new_shared->accounts) { + i++; + if (entry.second.compact_storage) { + nif_loop_progress(env, i); + continue; + } + merkletree *orig = entry.second.storage; + if (orig == nullptr) { + continue; + } + auto it = storage_clones.find(orig); + if (it == storage_clones.end()) { + merkletree *storage_clone = clone_merkletree_locked(orig); + it = storage_clones.insert({orig, storage_clone}).first; + } + enif_keep_resource(it->second); + entry.second.storage = it->second; + nif_loop_progress(env, i); + } + for (auto &kv : storage_clones) { + enif_release_resource(kv.second); + } + return new_shared; +} + static bool get_address(ErlNifEnv *env, ERL_NIF_TERM term, uint160_t &out) { ErlNifBinary bin; @@ -668,23 +659,15 @@ static ERL_NIF_TERM code_to_term(ErlNifEnv *env, const bin_t &code) return make_binary(env, (uint8_t*)code.data(), code.size()); } -static bool make_writeable_accountmap(ErlNifEnv *env, accountmap *am) +static bool make_writeable_accountmap(accountmap *am) { - if (am->shared->has_clone > 0) { - am->shared->has_clone -= 1; - am->shared = cow_copy_accountmap(am->shared, env); - } - return true; + return !am->shared->frozen; } static void destroy_shared_accountmap(accountmap *am, AccountMapLock &lock) { - if (am->shared->has_clone == 0) { - lock.unlock(); - delete am->shared; - } else { - am->shared->has_clone -= 1; - } + lock.unlock(); + delete am->shared; am->shared = NULL; } @@ -1572,49 +1555,9 @@ account_map_clone(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) AccountMapLock lock(am); - // Eagerly COW the SharedAccountMap and clone every storage trie so the fork is - // writable (locked = false) while the cached parent stays frozen. Sharing the - // parent's merkletree* resources would leave the fork's storage tries with - // mt->locked == true, and make_writeable() would reject every storage write - // (merkletree_insert_item returns badarg), breaking block sync. - SharedAccountMap *new_shared = new SharedAccountMap(); - new_shared->accounts = am->shared->accounts; - release_storage_from_map(new_shared->state_trie); - { - merkletree *st = clone_merkletree_locked(am->shared->state_trie); - new_shared->state_trie = st; - keep_storage_in_map(st); - enif_release_resource(st); - } - - // Clone each unique parent storage trie once - // one resource ref per entry, so we keep one ref per entry here to match. - std::unordered_map storage_clones; - size_t i = 0; - for (auto &entry : new_shared->accounts) { - i++; - if (entry.second.compact_storage) { - nif_loop_progress(env, i); - continue; - } - merkletree *orig = entry.second.storage; - if (orig == nullptr) { - continue; - } - auto it = storage_clones.find(orig); - if (it == storage_clones.end()) { - merkletree *storage_clone = clone_merkletree_locked(orig); - it = storage_clones.insert({orig, storage_clone}).first; - } - enif_keep_resource(it->second); - entry.second.storage = it->second; - nif_loop_progress(env, i); - } - // Drop the creator refs (one per unique clone); the per-entry keeps above own - // the resources now. - for (auto &kv : storage_clones) { - enif_release_resource(kv.second); - } + // Eagerly fork so the clone is writable while a frozen parent stays immutable. + // Distinct merkletree wrappers share SharedState until first write. + SharedAccountMap *new_shared = fork_shared_accountmap(env, am->shared); accountmap *clone = (accountmap*)enif_alloc_resource(accountmap_type, sizeof(accountmap)); clone->shared = new_shared; @@ -1639,10 +1582,7 @@ account_map_clone_lazy(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return enif_make_badarg(env); } - // Speculative forks must not share merkletree* resource pointers with the - // parent: in-place COW (make_writeable_locked) would retarget the parent's - // SharedState. Use distinct wrappers that share SharedState until first write - // (same as eager clone for storage; still rejects frozen parents). + // Speculative forks reject frozen parents and locked storage tries. for (auto &entry : am->shared->accounts) { if (!entry.second.compact_storage && entry.second.storage != nullptr && entry.second.storage->locked) { @@ -1650,40 +1590,7 @@ account_map_clone_lazy(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) } } - SharedAccountMap *new_shared = new SharedAccountMap(); - new_shared->accounts = am->shared->accounts; - release_storage_from_map(new_shared->state_trie); - { - merkletree *st = clone_merkletree_locked(am->shared->state_trie); - new_shared->state_trie = st; - keep_storage_in_map(st); - enif_release_resource(st); - } - - std::unordered_map storage_clones; - size_t i = 0; - for (auto &entry : new_shared->accounts) { - i++; - if (entry.second.compact_storage) { - nif_loop_progress(env, i); - continue; - } - merkletree *orig = entry.second.storage; - if (orig == nullptr) { - continue; - } - auto it = storage_clones.find(orig); - if (it == storage_clones.end()) { - merkletree *storage_clone = clone_merkletree_locked(orig); - it = storage_clones.insert({orig, storage_clone}).first; - } - enif_keep_resource(it->second); - entry.second.storage = it->second; - nif_loop_progress(env, i); - } - for (auto &kv : storage_clones) { - enif_release_resource(kv.second); - } + SharedAccountMap *new_shared = fork_shared_accountmap(env, am->shared); accountmap *clone = (accountmap*)enif_alloc_resource(accountmap_type, sizeof(accountmap)); clone->shared = new_shared; @@ -1695,38 +1602,16 @@ account_map_clone_lazy(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return res; } -static bool get_optional_merkletree(ErlNifEnv *env, ERL_NIF_TERM term, merkletree **out) -{ - *out = nullptr; - if (enif_is_atom(env, term)) { - char atom[16]; - if (enif_get_atom(env, term, atom, sizeof(atom), ERL_NIF_LATIN1) && - strcmp(atom, "nil") == 0) { - return true; - } - return false; - } - return enif_get_resource(env, term, merkletree_type, (void **)out); -} - static ERL_NIF_TERM account_map_lock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { accountmap *am; - merkletree *store = nullptr; - if (argc != 2) return enif_make_badarg(env); + if (argc != 1) return enif_make_badarg(env); if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); - if (!get_optional_merkletree(env, argv[1], &store)) return enif_make_badarg(env); AccountMapLock lock(am); - if (!am->shared->frozen) { - am->shared->frozen = true; - } - - if (store != nullptr) { - locked_states->enter_lock(store); - } + am->shared->frozen = true; locked_states->try_reclaim_orphans(); return argv[0]; @@ -1875,7 +1760,7 @@ account_map_put(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) if (!get_code(env, argv[5], code)) return enif_make_badarg(env); AccountMapLock lock(am); - if (!make_writeable_accountmap(env, am)) return enif_make_badarg(env); + if (!make_writeable_accountmap(am)) return enif_make_badarg(env); auto it = am->shared->accounts.find(addr); if (it != am->shared->accounts.end()) { @@ -1911,7 +1796,7 @@ account_map_delete(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); AccountMapLock lock(am); - if (!make_writeable_accountmap(env, am)) return enif_make_badarg(env); + if (!make_writeable_accountmap(am)) return enif_make_badarg(env); auto it = am->shared->accounts.find(addr); if (it != am->shared->accounts.end()) { @@ -1973,7 +1858,6 @@ static merkletree *alloc_merkletree_resource() STAT(resources++); mt->shared_state = new SharedState(); mt->locked = false; - mt->cow_written = false; return mt; } @@ -1985,8 +1869,8 @@ static merkletree *materialize_storage(AccountEntry &entry) if (!entry.compact_storage || entry.compact_storage->slots.empty()) { // Fresh empty tree — never share empty_storage_tree as a writable map entry // (in-place COW would mutate the singleton for every account). + // alloc refcount 1 is owned by the map entry (same as ensure_account_entry). merkletree *mt = alloc_merkletree_resource(); - keep_storage_in_map(mt); entry.storage = mt; entry.compact_storage.reset(); return entry.storage; @@ -1998,7 +1882,6 @@ static merkletree *materialize_storage(AccountEntry &entry) mt->shared_state->tree.insert_item(slot.key, slot.value); } } - keep_storage_in_map(mt); entry.storage = mt; entry.compact_storage.reset(); return entry.storage; @@ -2427,10 +2310,7 @@ static merkletree *write_storage_slot(AccountEntry &entry, const bin_t &key, con static ERL_NIF_TERM make_apply_error(ErlNifEnv *env, const char *reason) { - ERL_NIF_TERM err_atom, reason_atom; - enif_make_existing_atom(env, "error", &err_atom, ERL_NIF_LATIN1); - enif_make_existing_atom(env, reason, &reason_atom, ERL_NIF_LATIN1); - return enif_make_tuple2(env, err_atom, reason_atom); + return enif_make_tuple2(env, make_atom(env, "error"), make_atom(env, reason)); } static AccountEntry &ensure_account_entry(SharedAccountMap *shared, const uint160_t &addr) @@ -2551,7 +2431,7 @@ account_map_apply_difference(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[] if (!enif_is_list(env, argv[1])) return enif_make_badarg(env); AccountMapLock lock(am); - if (!make_writeable_accountmap(env, am)) return enif_make_badarg(env); + if (!make_writeable_accountmap(am)) return enif_make_badarg(env); AccountHashCtx hash_ctx; size_t i = 0; @@ -2796,16 +2676,19 @@ static bool append_uncompacted_account(ErlNifEnv *env, accountmap *am, AccountHa std::vector &pending_state, const uint160_t &addr, AccountEntry &entry, const uint256_t *storage_root_override, const uint256_t *code_hash_override, size_t i) { - if (storage_root_override == nullptr && entry.storage == nullptr) { - materialize_storage(entry); + if (entry.storage == nullptr) { + if (storage_root_override == nullptr) { + // Fresh materialize: alloc ownership transfers with the entry. + materialize_storage(entry); + } + } else if (entry.compact_storage == nullptr) { + // Shared/copied pointer from another map or Elixir term: take a map ref. + keep_storage_in_map(entry.storage); } uint256_t account_hash; if (!hash_ctx.compute(entry, storage_root_override, code_hash_override, account_hash)) { return false; } - if (entry.compact_storage == nullptr && entry.storage != nullptr) { - keep_storage_in_map(entry.storage); - } am->shared->accounts[addr] = std::move(entry); pending_state.push_back({addr, account_hash}); nif_loop_progress(env, i); @@ -2975,7 +2858,7 @@ static ErlNifFunc nif_funcs[] = { {"account_map_new", 0, account_map_new, 0}, {"account_map_clone", 1, account_map_clone, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_clone_lazy", 1, account_map_clone_lazy, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"account_map_lock", 2, account_map_lock, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"account_map_lock", 1, account_map_lock, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_get", 2, account_map_get, 0}, {"account_map_put", 6, account_map_put, 0}, {"account_map_delete", 2, account_map_delete, 0}, diff --git a/lib/caccount_map.ex b/lib/caccount_map.ex index 3ecdb61..36667ed 100644 --- a/lib/caccount_map.ex +++ b/lib/caccount_map.ex @@ -13,7 +13,7 @@ defmodule CAccountMap do def clone_lazy(map), do: CMerkleTree.account_map_clone_lazy(map) - def lock(map), do: CMerkleTree.account_map_lock(map, nil) + def lock(map), do: CMerkleTree.account_map_lock(map) def root_hash(map), do: CMerkleTree.account_map_root_hash(map) @@ -67,12 +67,7 @@ defmodule CAccountMap do CMerkleTree.account_map_difference_full(map_a, map_b) end - def apply_difference(map, delta) do - case CMerkleTree.account_map_apply_difference(map, delta) do - {:error, reason} -> {:error, reason} - map -> map - end - end + def apply_difference(map, delta), do: CMerkleTree.account_map_apply_difference(map, delta) def decode_storage_diff(storage_diff) do Map.new(storage_diff, fn {key, {val_a, val_b}} -> @@ -86,11 +81,7 @@ defmodule CAccountMap do defp decode_account_side(nil), do: nil defp decode_account_side(entry), do: entry |> decode_entry() |> account_from_parts() - def uncompact_state(accounts) do - case CMerkleTree.account_map_uncompact_state(accounts) do - {am, hash} -> {am, hash} - end - end + def uncompact_state(accounts), do: CMerkleTree.account_map_uncompact_state(accounts) defp decode_entry({nonce, balance, storage, code}) do {nonce, decode_balance(balance), storage, code} diff --git a/lib/chain/state.ex b/lib/chain/state.ex index 440699a..2878645 100644 --- a/lib/chain/state.ex +++ b/lib/chain/state.ex @@ -96,20 +96,20 @@ defmodule Chain.State do {time, result} = :timer.tc(fn -> Enum.map(CAccountMap.difference_full(accounts_a, accounts_b), fn - {id, _side_a, _side_b, state_diff} -> - acc_a = account(state_a, id) || ensure_account(state_a, id) - acc_b = account(state_b, id) || ensure_account(state_b, id) - + {id, side_a, side_b, state_diff} -> report = %{} - |> put_field_diff(:nonce, acc_a, acc_b) - |> put_field_diff(:balance, acc_a, acc_b) - |> put_field_diff(:code, acc_a, acc_b) + |> put_side_field_diff(:nonce, side_a, side_b) + |> put_side_field_diff(:balance, side_a, side_b) + |> put_side_field_diff(:code, side_a, side_b) storage_map = CAccountMap.decode_storage_diff(state_diff) report = if map_size(storage_map) > 0 do + acc_a = account(state_a, id) || ensure_account(state_a, id) + acc_b = account(state_b, id) || ensure_account(state_b, id) + Map.merge(report, %{ state: storage_map, root_hash: {Account.root_hash(acc_a), Account.root_hash(acc_b)} @@ -131,9 +131,9 @@ defmodule Chain.State do result end - defp put_field_diff(report, field, acc_a, acc_b) do - a = apply(Account, field, [acc_a]) - b = apply(Account, field, [acc_b]) + defp put_side_field_diff(report, field, side_a, side_b) do + a = side_field(side_a, field) + b = side_field(side_b, field) if a == b do report @@ -142,6 +142,17 @@ defmodule Chain.State do end end + defp side_field(nil, :nonce), do: 0 + defp side_field(nil, :balance), do: 0 + defp side_field(nil, :code), do: "" + defp side_field({nonce, _balance, _code}, :nonce), do: nonce + defp side_field({_nonce, balance, _code}, :balance) when is_integer(balance), do: balance + + defp side_field({_nonce, balance, _code}, :balance) when is_binary(balance), + do: :binary.decode_unsigned(balance) + + defp side_field({_nonce, _balance, code}, :code), do: code + def clone(%Chain.State{accounts: accounts} = state) do %{state | accounts: CAccountMap.clone(accounts), hash: nil} end diff --git a/lib/cmerkletree.ex b/lib/cmerkletree.ex index 87b6cdf..ff26907 100644 --- a/lib/cmerkletree.ex +++ b/lib/cmerkletree.ex @@ -122,7 +122,7 @@ defmodule CMerkleTree do def account_map_clone_lazy(_map), do: error() def account_map_root_hash(_map), do: error() def account_map_state_trie(_map), do: error() - def account_map_lock(_map, _store), do: error() + def account_map_lock(_map), do: error() def account_map_get(_map, _addr), do: error() def account_map_put(_map, _addr, _nonce, _balance, _storage, _code), do: error() def account_map_delete(_map, _addr), do: error() diff --git a/test/cmerkle_clone_lazy_test.exs b/test/cmerkle_clone_lazy_test.exs index 1b75b91..0a5f7a3 100644 --- a/test/cmerkle_clone_lazy_test.exs +++ b/test/cmerkle_clone_lazy_test.exs @@ -81,7 +81,13 @@ defmodule CMerkleCloneLazyTest do state = %Chain.State{accounts: base} {lazy_us, fork} = :timer.tc(fn -> State.clone_lazy(state) end) + {eager_us, _} = :timer.tc(fn -> State.clone(state) end) + + # Today both paths fork storage wrappers; keep both under a generous bound and + # ensure lazy stays in the same ballpark as eager (not accidentally O(n²)). assert lazy_us < 500_000 + assert eager_us < 500_000 + assert lazy_us < eager_us * 3 assert is_binary(State.hash(fork)) end end From 83c8dfda617c1b6a76bcd45f896f35cac71520c0 Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Thu, 16 Jul 2026 15:00:56 +0200 Subject: [PATCH 04/16] Keep account storage inside the NIF and freeze maps with frozen-only lock. Stop exporting live storage tries from account_map_get, batch EVM writes via storage_put_map, and drop clone_lazy / per-trie seal so speculative paths and lock cost stay simpler. Co-authored-by: Cursor --- AGENTS.md | 13 +- c_src/LOCK_ORDER.md | 25 +- c_src/SECURITY_REVIEW.md | 38 +- c_src/nif.cpp | 481 ++++++++++++++------ docs/caccount-map-nif.md | 39 ++ lib/block_process.ex | 7 - lib/caccount_map.ex | 101 +++- lib/chain/account.ex | 74 ++- lib/chain/block.ex | 19 +- lib/chain/state.ex | 79 +++- lib/chaindefinition/voyager.ex | 43 +- lib/cmerkletree.ex | 16 +- lib/evm.ex | 35 +- lib/network/edge_v2.ex | 52 ++- lib/network/rpc.ex | 22 +- lib/shell.ex | 3 +- scripts/cmerkle_fuzz.exs | 140 +++--- scripts/cmerkle_heap_assumptions.exs | 6 +- scripts/cmerkle_parallel_stress.exs | 138 +++--- test/caccount_map_lifetime_test.exs | 207 +++++---- test/caccount_map_test.exs | 34 +- test/chain_account_hash_nif_test.exs | 24 +- test/chain_state_merkle_test.exs | 31 +- test/chain_state_uncompact_test.exs | 55 +-- test/chain_test.exs | 9 +- test/cmerkle_account_map_diff_test.exs | 148 +++--- test/cmerkle_clone_lazy_test.exs | 93 ---- test/cmerkle_lock_clone_regression_test.exs | 283 ++++++++++++ test/cmerkle_lock_concurrency_test.exs | 29 +- test/cmerkle_nif_deadlock_test.exs | 48 +- test/cmerkle_nif_leak_test.exs | 22 +- test/cmerkle_storage_map_test.exs | 107 +++++ 32 files changed, 1553 insertions(+), 868 deletions(-) create mode 100644 docs/caccount-map-nif.md delete mode 100644 test/cmerkle_clone_lazy_test.exs create mode 100644 test/cmerkle_lock_clone_regression_test.exs create mode 100644 test/cmerkle_storage_map_test.exs diff --git a/AGENTS.md b/AGENTS.md index cd54426..a9da64a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,8 @@ Ethereum-compatible JSON-RPC endpoint plus the Diode PEER/EDGE protocols. `make -C deps/libsecp256k1/` (see `.github/workflows/ci.yml`). Build artifacts are gitignored and persist across sessions, so this is only needed after a clean checkout of that dep. +- **CAccountMap / state NIF semantics** (clone, lock, storage APIs, get shape): + see [`docs/caccount-map-nif.md`](docs/caccount-map-nif.md). ### Lint - `mix lint` = `compile` + `mix format --check-formatted` + `mix credo --only warning` + `mix dialyzer`. @@ -39,15 +41,8 @@ Ethereum-compatible JSON-RPC endpoint plus the Diode PEER/EDGE protocols. - `make test` generates test PEM certs, then runs each `test/*_test.exs` file in a separate `mix test --max-failures 1` invocation (per-file isolation). Test env pins ports `RPC_PORT=18001`, `EDGE2_PORT=18003`, `PEER_PORT=18004`. -- `Chain.State` is backed by the `CAccountMap` NIF. Account storage tries and the - state root trie live in C++; Elixir `Chain.State` no longer carries a separate - `:store` field. Use `Chain.State.hash/1` or `CAccountMap.root_hash/1`. -- **Clone modes:** `Chain.State.clone_lazy/1` for speculative execution - (`eth_call`, RPC, EdgeV2) where the fork is discarded; `Chain.State.clone/1` - (eager storage fork) after `Chain.State.lock/1` for block sync / delta replay. - `Chain.Transaction.apply/3` mutates state in place on an unlocked candidate. -- `Chain.State` is MUTABLE: use `Chain.State.clone/1` or `clone_lazy/1` before - applying transactions on a shared cached state. +- For `Chain.State` / CAccountMap mutability and storage rules, see + [`docs/caccount-map-nif.md`](docs/caccount-map-nif.md). ### Running the node (dev mode) - `./dev` runs `MIX_ENV=dev iex -S mix run` (wipes `data_dev/` first). For a diff --git a/c_src/LOCK_ORDER.md b/c_src/LOCK_ORDER.md index de247d4..8e26c1d 100644 --- a/c_src/LOCK_ORDER.md +++ b/c_src/LOCK_ORDER.md @@ -26,15 +26,14 @@ See also [`SECURITY_REVIEW.md`](SECURITY_REVIEW.md) (F-5 fix) and [`scripts/cmer | `enter_lock` (dedup) | tree (`mt->locked`) → `locked_states_mutex` → pin `read_pins` on canonical → **release global** → bump `has_clone` / canonical switch | `read_pins` prevents reclaim until registration ref is taken | | `leave_lock` / GC destructor | if `mt->locked`: `locked_states_mutex` → tree → decrement registration → erase map when `has_clone == 0`; else tree refcount only | Unlocked resources never touch `LockedStates` map | | `merkletree_clone` | `Lock(parent)` | Dirty CPU scheduler; O(1) shallow resource alloc (`locked = false`) | -| `account_map_clone` | `AccountMapLock` → `Lock(parent_storage)` per trie (sequential) | Dirty scheduler; long hold | -| `account_map_lock` | `AccountMapLock` only (sets `frozen`; no per-trie `enter_lock` sweep) | Dirty scheduler | +| `account_map_clone` | `AccountMapLock` → `fork_shared_accountmap` (`Lock` per parent storage / state_trie) | Dirty scheduler; writable fork even if parent `frozen` | +| `account_map_lock` | `AccountMapLock` → set `frozen=true` only (O(1); no per-trie seal) | Dirty scheduler; put/delete/storage_put_map/put_meta reject via `frozen` | +| `account_map_storage_put_map` | `AccountMapLock` → reject if frozen → per-addr `write_storage_slot` → `update_state_trie_for_entry` | Dirty CPU; EVM `su` hot path | | `switch_local_to_canonical` | tree mutexes (address order) | Abandoned `SharedState` queued on `pending_orphans`; reclaimed via `try_reclaim_orphans` after `enter_lock` / `leave_lock` when mutex trylock succeeds and `has_clone == 0` | | `account_map_uncompact_state` | `AccountMapLock(input)` → `materialize_storage` (brief tree lock) → `batch_insert` (state_store lock) | Dirty scheduler | -| `account_map_put/delete` | `AccountMapLock` only | May `release_resource` → async GC `leave_lock` | -| `account_map_list_difference_raw` | `SharedAccountMap*` mutexes (address order) → brief tree lock per entry for root read → release all before materialize | Dirty CPU; never hold map lock across `materialize_storage` | -| `account_map_difference_full` | Dual map lock → snapshot → release → storage diffs | Dirty CPU; same D-C7 pattern as list_difference | -| `account_map_apply_difference` | `AccountMapLock` → per-account storage writes via `make_writeable_locked` | Dirty CPU | -| `account_map_clone_lazy` | `AccountMapLock`; rejects `frozen` parent; distinct storage wrappers | Dirty CPU | +| `account_map_put/delete` / `put_meta` | `AccountMapLock` only; reject if `frozen` | May `release_resource` → async GC `leave_lock` | +| `account_map_difference_full` | Dual map lock (`DualAccountMapLock`, address order) → snapshot sides → release → per-account storage diffs | Dirty CPU; never hold map lock across storage diff build | +| `account_map_apply_difference` | `AccountMapLock` → reject if `frozen` → storage/field writes (`write_storage_slot` → `make_writeable_locked`) | Dirty CPU | | Insert / COW | Tree lock → ItemPool / PreAllocator / stripe pool | Same-thread nesting | ## Deadlock scenario registry @@ -69,26 +68,26 @@ Each scenario has an ID, hypothesis, and test coverage target. |----|----------|------|------------| | D-C1 | `account_map_clone` + `difference` on shared storage | Tree mutex contention | P13 | | D-C2 | `account_map_uncompact_state` + per-account `difference` | Independent domains unless storage shared | P12, S12, ExUnit D-C2 | -| D-C3 | `account_map_get` / `to_list` (materialize) + `difference` | Brief tree lock vs diff | S13 | +| D-C3 | `account_map_get` / `to_list` (root hash export) + `difference` | No live storage export; brief hash compute | S13 | | D-C4 | `account_map_put` replacing storage + GC `leave_lock` | Async GC vs diff | P13 | | D-C5 | `account_map_lock` / `account_map_clone` + concurrent `State.lock/1` | Correctness / writable fork | P14, P15, chain_state_uncompact_test, ExUnit D-C5, fuzz 16–18 | | D-C6 | Concurrent `put` / `apply_difference` on frozen map | Rejected via `make_writeable_accountmap` | TSan on P4, P13 | -| D-C7 | `account_map_list_difference_raw` + `account_map_to_list` same map | Map mutex convoy / materialize stall | S20, ExUnit D-D7, D-C7 | -| D-C8 | Dual-map `list_difference` lock order (A,B) vs (B,A) | Ordering regression | S24, ExUnit D-C8 | +| D-C7 | `account_map_difference_full` + `account_map_to_list` same map | Map mutex convoy / materialize stall | S20, ExUnit D-D7, D-C7 | +| D-C8 | Dual-map `difference_full` lock order (A,B) vs (B,A) | Ordering regression | S24, ExUnit D-C8 | ### D. Production composite (block sync) | ID | Scenario | Covered by | |----|----------|------------| | D-D1 | `Chain.State.difference/2` (storage diffs per account) | P14 | -| D-D2 | `State.lock/1` on all account trees | P14 | +| D-D2 | `State.lock/1` sets map `frozen` only (no per-trie seal; get exports root hashes not live storage) | P14 | | D-D3 | D-D1 + D-D2 + uncompact concurrent | P14, P12, ExUnit D-D3 | | D-D4 | compact → uncompact → clone → apply_difference | P14, S10, S11 | | D-D5 | Storage get/insert + block import | P1, P5 | | D-D6 | Dirty scheduler saturation | P15 | | D-D7 | prepare_state composite (native diff + lock + legacy to_list) | S30, P17, ExUnit D-D7 | -| D-D8 | `list_difference` compact storage root compare (no full map materialize) | account_map_diff_test, S19 | -| D-L1 | `clone_lazy` + put storage + discard | `cmerkle_clone_lazy_test`, P18L | +| D-D8 | `difference_full` on compact storage | account_map_diff_test, S19 | +| D-L1 | `clone` after lock + storage_put_map + discard | `cmerkle_storage_map_test`, `cmerkle_lock_clone_regression_test`, P18L | | D-L2 | `difference_full` + `apply_difference` round-trip | chain_state_merkle_test, account_map_diff_test | | D-M1 | frozen map + eager clone writable fork | chain_state_merkle_test lock→clone | | D-M2 | concurrent difference_raw + apply_difference | lock concurrency, stress | diff --git a/c_src/SECURITY_REVIEW.md b/c_src/SECURITY_REVIEW.md index 1e3d3bf..590fd19 100644 --- a/c_src/SECURITY_REVIEW.md +++ b/c_src/SECURITY_REVIEW.md @@ -29,20 +29,30 @@ | `memory_stats_raw` | 1 | resource | tests, benches | | `malloc_info_raw` | 0 | — | tests, `cmerkle_memory_bench.exs` | | `account_map_new` | 0 | — | `CAccountMap.new/0`, `Chain.State` | -| `account_map_clone` | 1 | account map resource | `CAccountMap.clone/1`, `Chain.State.clone/1` | -| `account_map_clone_lazy` | 1 | account map resource | `CAccountMap.clone_lazy/1`, speculative `Chain.State.clone_lazy/1` | -| `account_map_lock` | 2 | account map resource, optional store/nil (ignored for freeze) | `CAccountMap.lock/1`, `Chain.State.lock/1` | -| `account_map_get` | 2 | resource, 20-byte address | `CAccountMap.get/2` | -| `account_map_put` | 6 | resource, address, nonce, balance, storage resource, code | `CAccountMap.put/5` | -| `account_map_delete` | 2 | resource, address | `CAccountMap.delete/2` | +| `account_map_clone` | 1 | account map resource | `CAccountMap.clone/1`, `Chain.State.clone/1` (writable fork; OK on frozen parent) | +| `account_map_lock` | 1 | account map resource | `CAccountMap.lock/1` — `frozen` only (O(1); no per-trie seal) | +| `account_map_get` | 2 | resource, 20-byte address | `CAccountMap.get/2` returns `{nonce, balance, storage_root_hash_bin32, code}` — never a live storage resource | +| `account_map_put` | 6 | resource, address, nonce, balance, storage, code | Cold path (import/uncompact/genesis); rejects frozen | +| `account_map_put_meta` | 5 | resource, address, nonce, balance, code | Metadata-only put; keeps existing storage | +| `account_map_delete` | 2 | resource, address | Rejects frozen | | `account_map_root_hash` | 1 | resource | `CAccountMap.root_hash/1`, `Chain.State.hash/1` | -| `account_map_state_trie` | 1 | resource | `CAccountMap.state_trie/1`, `Chain.State.tree/1` | +| `account_map_state_trie` | 1 | resource | `CAccountMap.state_trie/1`, `Chain.State.tree/1` (Edge state roots) | +| `account_map_get_proofs` | 2 | map, address | Account inclusion proof on internal state_trie | +| `account_map_storage_put_map` | 2 | map, update list | EVM `su` hot path — one NIF for multi-account slots | +| `account_map_storage_get` | 3 | map, addr, key | `State.storage_value/3`, RPC | +| `account_map_storage_get_range` | 4 | map, addr, key, count | EVM `gs` | +| `account_map_storage_to_list` | 2 | map, addr | RPC `eth_getStorage`, EVM cache | +| `account_map_storage_size` | 2 | map, addr | EVM cache threshold | +| `account_map_storage_root_hash` | 2 | map, addr | Edge / diffs | +| `account_map_storage_root_hashes` | 2 | map, addr | Edge `getaccountroots` | +| `account_map_storage_get_proofs` | 3 | map, addr, key | Edge storage proofs | | `account_map_size` | 1 | resource | `CAccountMap.size/1` | -| `account_map_to_list` | 1 | resource | `CAccountMap.to_list/1`, RPC export | -| `account_map_list_difference_raw` | 2 | two account map resources | `CAccountMap.list_difference/2` | -| `account_map_difference_full` | 2 | two account map resources | `CAccountMap.difference_full/2`, `Chain.State.difference/2` | -| `account_map_apply_difference` | 2 | account map resource, delta list | `CAccountMap.apply_difference/2`, `Chain.State.apply_difference/2` | -| `account_map_uncompact_state` | 1 | compact account map or account map resource | Returns `{am, hash}`; `CAccountMap.uncompact_state/1`, `Chain.State.uncompact/1` | +| `account_map_to_list` | 1 | resource | `CAccountMap.to_list/1` | +| `account_map_difference_full` | 2 | two maps | `Chain.State.difference/2` | +| `account_map_apply_difference` | 2 | map, delta list | `Chain.State.apply_difference/2` | +| `account_map_uncompact_state` | 1 | compact or resource | Returns `{am, hash}` | + +**Frozen map:** `account_map_lock/1` sets map-level `frozen` only. Map mutations (`put`/`put_meta`/`delete`/`apply_difference`/`storage_put_map`) fail while frozen. `account_map_get` / `to_list` export storage root hashes (never live tries), so Elixir cannot mutate map-owned storage via bare `CMerkleTree.insert`. `clone/1` forks writable unlocked wrappers for sync and speculative RPC/Edge/Shell. **Trust:** Erlang validates some shapes (e.g. `to_bytes32`), but the NIF must treat all binaries and terms as hostile (size, allocation, scheduler impact). @@ -66,7 +76,7 @@ | F-5 | **Interaction `LockedStates::mtx` vs tree mutexes** | Medium | CWE-833 | **Fixed:** `enter_lock` pins `has_clone` on local/canonical under global+tree lock, drops global before `switch_local_to_canonical`, and map entries hold a `has_clone` ref. `difference_raw` snapshots pointers under global, bumps `read_pins` (not `has_clone`), releases global, then acquires dual tree locks. `leave_lock` erases map entries under global, releases global, then detaches under tree lock only. | | F-6 | **`make_writeable` COW under `Lock` RAII** | High | CWE-667 | **Fixed:** COW now transfers the held mutex (unlock old `SharedState`, lock new) instead of leaving `Lock` holding a destroyed mutex while mutating a forked tree. | | F-7 | **`leave_lock` / canonical map UAF** | High | CWE-416 | **Fixed:** Map erase drops the map's `has_clone` ref; canonical pointers are re-validated before switch; `SharedState` is not deleted while referenced from the dedup map. | -| F-7b | **Abandoned `SharedState` after canonical switch** | High | CWE-404 | **Fixed:** `switch_local_to_canonical` enqueues unreferenced locals on `pending_orphans`; `try_reclaim_orphans` deletes when `has_clone == 0`; `difference_raw` pins `SharedState` during dual-lock; `account_map_lock` dedupes by `root_hash`. Monitor via `nif_stats_raw/0`. | +| F-7b | **Abandoned `SharedState` after canonical switch** | High | CWE-404 | **Fixed:** orphan reclaim path for standalone `CMerkleTree.lock` / `difference_raw`. `account_map_lock` is `frozen`-only (get no longer exports live storage). Monitor via `nif_stats_raw/0`. | | F-6 | **Global `locked_states` / `stats_mutex` on upgrade** | Low | CWE-665 | `on_reload`/`on_upgrade` no-op; hot upgrade could leave stale globals. Acceptable if NIF not hot-reloaded. | ### Information disclosure / introspection @@ -80,7 +90,7 @@ | ID | Topic | Severity | CWE | Notes | |----|--------|----------|-----|--------| -| F-9 | **Unbounded work per NIF** | Medium | CWE-400 | Long-running exports use dirty schedulers: **CPU-bound** — `get_proofs_raw`, `difference_raw`, `to_list`, `import_map`, `count_zeros`, `memory_stats_raw`, `clone`, `account_map_clone`, `account_map_lock`, `account_map_to_list`, `account_map_list_difference_raw`, `account_map_uncompact_state`; **IO-bound** — `malloc_info_raw`. `account_map_put`/`delete` stay on normal schedulers; first COW copy uses `enif_consume_timeslice` every 1024 entries. Large dirty-NIF loops also call `enif_consume_timeslice` every 512 iterations. Ensure adequate dirty CPU schedulers at runtime (`+SDcpu` on heavy sync nodes). | +| F-9 | **Unbounded work per NIF** | Medium | CWE-400 | Long-running exports use dirty schedulers: **CPU-bound** — `get_proofs_raw`, `difference_raw`, `to_list`, `import_map`, `count_zeros`, `memory_stats_raw`, `clone`, `account_map_clone`, `account_map_lock`, `account_map_to_list`, `account_map_difference_full`, `account_map_apply_difference`, `account_map_storage_put_map`, `account_map_storage_to_list`, `account_map_storage_get_proofs`, `account_map_get_proofs`, `account_map_uncompact_state`; **IO-bound** — `malloc_info_raw`. `account_map_put`/`put_meta`/`delete`/`storage_get*` stay on normal schedulers where short. Large dirty-NIF loops call `enif_consume_timeslice` every 512 iterations. Ensure adequate dirty CPU schedulers at runtime (`+SDcpu` on heavy sync nodes). | ### Memory safety (manual review) diff --git a/c_src/nif.cpp b/c_src/nif.cpp index 1fee6c1..8835220 100644 --- a/c_src/nif.cpp +++ b/c_src/nif.cpp @@ -311,6 +311,8 @@ struct AccountEntry { static void release_entry_storage(AccountEntry &entry); static merkletree *materialize_storage(AccountEntry &entry); +static bool storage_root_hash_for_entry(ErlNifEnv *env, const AccountEntry &entry, + uint256_t &out, size_t progress_base); static ERL_NIF_TERM account_entry_to_term(ErlNifEnv *env, AccountEntry &entry); class SharedAccountMap { @@ -825,9 +827,10 @@ class LockedStates { mt->locked = true; local = mt->shared_state; if (local == canonical) { - enif_mutex_lock(canonical->mtx); + // Already hold local/canonical mtx via Lock — do not lock again + // (ErlNifMutex is non-recursive; double-lock deadlocks on shared storage). canonical->has_clone += 1; - enif_mutex_unlock(canonical->mtx); + lock.unlock(); unpin_shared_state_read(canonical); return; } @@ -1569,39 +1572,6 @@ account_map_clone(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return res; } -static ERL_NIF_TERM -account_map_clone_lazy(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - accountmap *am; - if (argc != 1) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); - - AccountMapLock lock(am); - - if (am->shared->frozen) { - return enif_make_badarg(env); - } - - // Speculative forks reject frozen parents and locked storage tries. - for (auto &entry : am->shared->accounts) { - if (!entry.second.compact_storage && entry.second.storage != nullptr && - entry.second.storage->locked) { - return enif_make_badarg(env); - } - } - - SharedAccountMap *new_shared = fork_shared_accountmap(env, am->shared); - - accountmap *clone = (accountmap*)enif_alloc_resource(accountmap_type, sizeof(accountmap)); - clone->shared = new_shared; - enif_mutex_lock(stats_mutex); - lazy_clone_count++; - enif_mutex_unlock(stats_mutex); - ERL_NIF_TERM res = enif_make_resource(env, clone); - enif_release_resource(clone); - return res; -} - static ERL_NIF_TERM account_map_lock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { @@ -1610,8 +1580,12 @@ account_map_lock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) if (argc != 1) return enif_make_badarg(env); if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); - AccountMapLock lock(am); - am->shared->frozen = true; + { + AccountMapLock lock(am); + // Map-level freeze only: get no longer exports live storage resources, so + // bare CMerkleTree.insert cannot mutate map-owned tries via Elixir. + am->shared->frozen = true; + } locked_states->try_reclaim_orphans(); return argv[0]; @@ -1619,12 +1593,16 @@ account_map_lock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) static ERL_NIF_TERM account_entry_to_term(ErlNifEnv *env, AccountEntry &entry) { - merkletree *storage = materialize_storage(entry); + uint256_t storage_root; + // Prefer hash without materializing compact slots into a live trie. + if (!storage_root_hash_for_entry(env, entry, storage_root, 0)) { + return enif_make_badarg(env); + } ERL_NIF_TERM nonce = enif_make_uint64(env, entry.nonce); ERL_NIF_TERM balance = balance_to_term(env, entry.balance); - ERL_NIF_TERM storage_term = enif_make_resource(env, storage); + ERL_NIF_TERM storage_hash = make_binary(env, storage_root.data(), 32); ERL_NIF_TERM code = code_to_term(env, entry.code); - return enif_make_tuple4(env, nonce, balance, storage_term, code); + return enif_make_tuple4(env, nonce, balance, storage_hash, code); } struct AccountHashCtx { @@ -1946,29 +1924,6 @@ static void snapshot_side(const AccountEntry &src, DiffAccountSide &out) } } -static AccountEntry side_to_entry(DiffAccountSide &side) -{ - AccountEntry entry; - entry.nonce = side.nonce; - entry.balance = side.balance; - entry.code = side.code; - entry.storage = side.storage; - entry.compact_storage = std::move(side.compact_storage); - return entry; -} - -static ERL_NIF_TERM diff_side_to_term(ErlNifEnv *env, DiffAccountSide &side) -{ - if (!side.present) { - return make_atom(env, "nil"); - } - AccountEntry entry = side_to_entry(side); - ERL_NIF_TERM term = account_entry_to_term(env, entry); - release_entry_storage(entry); - side.storage = nullptr; - return term; -} - static bool entries_equal(ErlNifEnv *env, const AccountEntry &a, const AccountEntry &b, size_t progress_base) { if (a.nonce != b.nonce || a.balance != b.balance || a.code != b.code) { @@ -1989,80 +1944,6 @@ struct DiffItem { DiffAccountSide b; }; -static ERL_NIF_TERM -account_map_list_difference_raw(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - accountmap *am_a; - accountmap *am_b; - - if (argc != 2) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am_a)) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[1], accountmap_type, (void **)&am_b)) return enif_make_badarg(env); - - if (am_a->shared == am_b->shared) { - return enif_make_list(env, 0); - } - - std::vector diffs; - std::unordered_set key_set; - - { - DualAccountMapLock map_lock(am_a->shared, am_b->shared); - - for (auto &entry : am_a->shared->accounts) { - key_set.insert(entry.first); - } - for (auto &entry : am_b->shared->accounts) { - key_set.insert(entry.first); - } - - std::vector keys(key_set.begin(), key_set.end()); - std::sort(keys.begin(), keys.end()); - - size_t i = 0; - for (auto &addr : keys) { - i++; - auto it_a = am_a->shared->accounts.find(addr); - auto it_b = am_b->shared->accounts.find(addr); - bool in_a = it_a != am_a->shared->accounts.end(); - bool in_b = it_b != am_b->shared->accounts.end(); - - if (in_a && in_b && entries_equal(env, it_a->second, it_b->second, i)) { - nif_loop_progress(env, i); - continue; - } - - DiffItem item; - item.addr = addr; - if (in_a) { - snapshot_side(it_a->second, item.a); - } - if (in_b) { - snapshot_side(it_b->second, item.b); - } - diffs.push_back(std::move(item)); - nif_loop_progress(env, i); - } - } - - ERL_NIF_TERM list = enif_make_list(env, 0); - size_t j = 0; - for (auto &item : diffs) { - j++; - ERL_NIF_TERM addr_term = make_binary(env, (uint8_t *)item.addr.value, 20); - ERL_NIF_TERM side_a = diff_side_to_term(env, item.a); - ERL_NIF_TERM side_b = diff_side_to_term(env, item.b); - ERL_NIF_TERM pair = enif_make_tuple2(env, side_a, side_b); - ERL_NIF_TERM triple = enif_make_tuple2(env, addr_term, pair); - list = enif_make_list_cell(env, triple, list); - release_snapshot_side(item.a); - release_snapshot_side(item.b); - nif_loop_progress(env, j); - } - - return list; -} - static ERL_NIF_TERM diff_side_fields_to_term(ErlNifEnv *env, DiffAccountSide &side) { if (!side.present) { @@ -2481,6 +2362,320 @@ account_map_apply_difference(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[] return argv[0]; } +static ERL_NIF_TERM +account_map_storage_put_map(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + + if (argc != 2) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + if (!enif_is_list(env, argv[1])) return enif_make_badarg(env); + + AccountMapLock lock(am); + if (!make_writeable_accountmap(am)) return enif_make_badarg(env); + + AccountHashCtx hash_ctx; + size_t i = 0; + ERL_NIF_TERM head, tail = argv[1]; + + while (enif_get_list_cell(env, tail, &head, &tail)) { + i++; + const ERL_NIF_TERM *elems; + int arity; + if (!enif_get_tuple(env, head, &arity, &elems) || arity != 2) { + return enif_make_badarg(env); + } + + uint160_t addr; + if (!get_address(env, elems[0], addr)) { + return enif_make_badarg(env); + } + if (!enif_is_list(env, elems[1])) { + return enif_make_badarg(env); + } + + AccountEntry &entry = ensure_account_entry(am->shared, addr); + ERL_NIF_TERM kv_head, kv_tail = elems[1]; + size_t j = 0; + while (enif_get_list_cell(env, kv_tail, &kv_head, &kv_tail)) { + j++; + const ERL_NIF_TERM *kv_elems; + int kv_arity; + if (!enif_get_tuple(env, kv_head, &kv_arity, &kv_elems) || kv_arity != 2) { + return enif_make_badarg(env); + } + + ErlNifBinary key_bin, value_bin; + if (!enif_inspect_binary(env, kv_elems[0], &key_bin) || key_bin.size != 32) { + return enif_make_badarg(env); + } + if (!enif_inspect_binary(env, kv_elems[1], &value_bin) || value_bin.size != 32) { + return enif_make_badarg(env); + } + + bin_t key(key_bin.data, key_bin.data + key_bin.size); + uint256_t value((const char *)value_bin.data); + if (write_storage_slot(entry, key, value) == nullptr) { + return enif_make_badarg(env); + } + nif_loop_progress(env, j); + } + + update_state_trie_for_entry(am->shared, addr, entry, hash_ctx); + nif_loop_progress(env, i); + } + + return argv[0]; +} + +static ERL_NIF_TERM +account_map_storage_get(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + uint160_t addr; + ErlNifBinary key_binary; + + if (argc != 3) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); + if (!enif_inspect_binary(env, argv[2], &key_binary) || key_binary.size != 32) { + return enif_make_badarg(env); + } + + AccountMapLock lock(am); + auto it = am->shared->accounts.find(addr); + if (it == am->shared->accounts.end()) { + return make_atom(env, "nil"); + } + + merkletree *mt = materialize_storage(it->second); + bin_t key(key_binary.data, key_binary.data + key_binary.size); + uint256_t value = read_storage_slot(mt, key); + if (value.is_null()) { + return make_atom(env, "nil"); + } + return make_binary(env, value.data(), 32); +} + +static ERL_NIF_TERM +account_map_storage_get_range(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + uint160_t addr; + ErlNifBinary key_binary; + unsigned count; + + if (argc != 4) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); + if (!enif_inspect_binary(env, argv[2], &key_binary) || key_binary.size != 32) { + return enif_make_badarg(env); + } + if (!enif_get_uint(env, argv[3], &count)) return enif_make_badarg(env); + if (count < 1 || count > 256) return enif_make_badarg(env); + + AccountMapLock lock(am); + auto it = am->shared->accounts.find(addr); + if (it == am->shared->accounts.end()) { + return enif_make_list(env, 0); + } + + merkletree *mt = materialize_storage(it->second); + bin_t key(key_binary.data, key_binary.data + key_binary.size); + + std::vector entries(count); + size_t n; + { + Lock tree_lock(mt); + n = get_range_entries(mt->shared_state->tree, key, count, entries.data()); + } + + ERL_NIF_TERM list = enif_make_list(env, 0); + for (size_t i = n; i > 0; i--) { + RangeEntry &entry = entries[i - 1]; + ERL_NIF_TERM key_term = make_binary(env, entry.key.data(), entry.key.size()); + ERL_NIF_TERM value_term = make_binary(env, entry.value.data(), 32); + ERL_NIF_TERM pair = enif_make_tuple2(env, key_term, value_term); + list = enif_make_list_cell(env, pair, list); + } + return list; +} + +static ERL_NIF_TERM +account_map_storage_to_list(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + uint160_t addr; + + if (argc != 2) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); + + AccountMapLock lock(am); + auto it = am->shared->accounts.find(addr); + if (it == am->shared->accounts.end()) { + return enif_make_list(env, 0); + } + + merkletree *mt = materialize_storage(it->second); + ERL_NIF_TERM list = enif_make_list(env, 0); + size_t i = 0; + { + Lock tree_lock(mt); + mt->shared_state->tree.each([&](pair_t &pair) { + i++; + ERL_NIF_TERM key_term = make_binary(env, pair.key.data(), pair.key.size()); + ERL_NIF_TERM value_term = make_binary(env, pair.value.data(), 32); + ERL_NIF_TERM tuple = enif_make_tuple2(env, key_term, value_term); + list = enif_make_list_cell(env, tuple, list); + nif_loop_progress(env, i); + }); + } + return list; +} + +static ERL_NIF_TERM +account_map_storage_size(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + uint160_t addr; + + if (argc != 2) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); + + AccountMapLock lock(am); + auto it = am->shared->accounts.find(addr); + if (it == am->shared->accounts.end()) { + return enif_make_uint(env, 0); + } + + merkletree *mt = materialize_storage(it->second); + Lock tree_lock(mt); + return enif_make_uint(env, (unsigned)mt->shared_state->tree.size()); +} + +static ERL_NIF_TERM +account_map_storage_root_hash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + uint160_t addr; + + if (argc != 2) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); + + AccountMapLock lock(am); + auto it = am->shared->accounts.find(addr); + if (it == am->shared->accounts.end()) { + Lock tree_lock(empty_storage_tree); + uint256_t root = empty_storage_tree->shared_state->tree.root_hash(); + return make_binary(env, root.data(), 32); + } + + merkletree *mt = materialize_storage(it->second); + Lock tree_lock(mt); + uint256_t root = mt->shared_state->tree.root_hash(); + return make_binary(env, root.data(), 32); +} + +static ERL_NIF_TERM +account_map_storage_root_hashes(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + uint160_t addr; + + if (argc != 2) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); + + AccountMapLock lock(am); + auto it = am->shared->accounts.find(addr); + if (it == am->shared->accounts.end()) { + Lock tree_lock(empty_storage_tree); + auto root_hashes = empty_storage_tree->shared_state->tree.root_hashes(); + return make_binary(env, (uint8_t*)root_hashes, 32 * 16); + } + + merkletree *mt = materialize_storage(it->second); + Lock tree_lock(mt); + auto root_hashes = mt->shared_state->tree.root_hashes(); + return make_binary(env, (uint8_t*)root_hashes, 32 * 16); +} + +static ERL_NIF_TERM +account_map_storage_get_proofs(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + uint160_t addr; + ErlNifBinary key_binary; + + if (argc != 3) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); + if (!enif_inspect_binary(env, argv[2], &key_binary)) return enif_make_badarg(env); + + AccountMapLock lock(am); + merkletree *mt; + auto it = am->shared->accounts.find(addr); + if (it == am->shared->accounts.end()) { + mt = empty_storage_tree; + } else { + mt = materialize_storage(it->second); + } + + bin_t key(key_binary.data, key_binary.data + key_binary.size); + Lock tree_lock(mt); + proof_t proof = mt->shared_state->tree.get_proofs(key); + return make_proof(env, proof); +} + +static ERL_NIF_TERM +account_map_get_proofs(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + uint160_t addr; + + if (argc != 2) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); + + AccountMapLock lock(am); + bin_t key(addr.value, addr.value + 20); + Lock tree_lock(am->shared->state_trie); + proof_t proof = am->shared->state_trie->shared_state->tree.get_proofs(key); + return make_proof(env, proof); +} + +static ERL_NIF_TERM +account_map_put_meta(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + uint160_t addr; + ErlNifUInt64 nonce; + uint256_t balance; + bin_t code; + + if (argc != 5) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); + if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); + if (!enif_get_uint64(env, argv[2], &nonce)) return enif_make_badarg(env); + if (!get_balance_uint256(env, argv[3], balance)) return enif_make_badarg(env); + if (!get_code(env, argv[4], code)) return enif_make_badarg(env); + + AccountMapLock lock(am); + if (!make_writeable_accountmap(am)) return enif_make_badarg(env); + + AccountEntry &entry = ensure_account_entry(am->shared, addr); + entry.nonce = (uint64_t)nonce; + entry.balance = balance; + entry.code = code; + + AccountHashCtx hash_ctx; + update_state_trie_for_entry(am->shared, addr, entry, hash_ctx); + return argv[0]; +} + struct UncompactLoopScratch { AccountHashCtx hash_ctx; bin_t code_buf; @@ -2857,19 +3052,27 @@ static ErlNifFunc nif_funcs[] = { {"nif_stats_raw", 0, merkletree_nif_stats, 0}, {"account_map_new", 0, account_map_new, 0}, {"account_map_clone", 1, account_map_clone, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"account_map_clone_lazy", 1, account_map_clone_lazy, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_lock", 1, account_map_lock, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_get", 2, account_map_get, 0}, {"account_map_put", 6, account_map_put, 0}, + {"account_map_put_meta", 5, account_map_put_meta, 0}, {"account_map_delete", 2, account_map_delete, 0}, {"account_map_root_hash", 1, account_map_root_hash, 0}, {"account_map_state_trie", 1, account_map_state_trie, 0}, {"account_map_size", 1, account_map_size, 0}, {"account_map_to_list", 1, account_map_to_list, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"account_map_list_difference_raw", 2, account_map_list_difference_raw, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_difference_full", 2, account_map_difference_full, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_apply_difference", 2, account_map_apply_difference, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_uncompact_state", 1, account_map_uncompact_state, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"account_map_storage_put_map", 2, account_map_storage_put_map, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"account_map_storage_get", 3, account_map_storage_get, 0}, + {"account_map_storage_get_range", 4, account_map_storage_get_range, 0}, + {"account_map_storage_to_list", 2, account_map_storage_to_list, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"account_map_storage_size", 2, account_map_storage_size, 0}, + {"account_map_storage_root_hash", 2, account_map_storage_root_hash, 0}, + {"account_map_storage_root_hashes", 2, account_map_storage_root_hashes, 0}, + {"account_map_storage_get_proofs", 3, account_map_storage_get_proofs, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"account_map_get_proofs", 2, account_map_get_proofs, ERL_NIF_DIRTY_JOB_CPU_BOUND}, }; // ERL_NIF_INIT(merkletree_nif, nif_funcs, on_load, on_reload, on_upgrade, NULL); diff --git a/docs/caccount-map-nif.md b/docs/caccount-map-nif.md new file mode 100644 index 0000000..6ee1353 --- /dev/null +++ b/docs/caccount-map-nif.md @@ -0,0 +1,39 @@ +# CAccountMap / CMerkleTree NIF + +Agent-facing notes for the merkle account-map NIF (`priv/merkletree_nif.so`, +built from `c_src/` via `mix compile` / top-level `Makefile`). + +See also [`c_src/LOCK_ORDER.md`](../c_src/LOCK_ORDER.md) and +[`c_src/SECURITY_REVIEW.md`](../c_src/SECURITY_REVIEW.md). + +## Ownership model + +- `Chain.State` is backed by `CAccountMap`. Account storage tries and the state + root trie live in C++. Elixir does **not** carry a separate `:store` field. +- Prefer map-owned storage APIs over bare `merkletree` resources: + - `Chain.State.storage_get/3`, `storage_put_map/2`, `storage_to_list/2`, + `storage_get_proofs/3`, `storage_root_hash/2` + - `CAccountMap` mirrors of the same +- `Chain.State.hash/1` or `CAccountMap.root_hash/1` for the state root. +- `account_map_get` / `to_list` return `{nonce, balance, storage_root_hash_bin32, code}` + — the third element is a **32-byte hash**, never a live storage resource. +- Map-backed `%Chain.Account{}` values have `storage_root: nil` and carry + `:root_hash` when loaded from the map. Standalone tries are only for genesis / + hardfork / import via `account_map_put/6`. + +## Clone and lock + +- `Chain.State.clone/1` forks a writable map. Use it after `Chain.State.lock/1` + (cached peak / block sync) and for RPC / EdgeV2 / Shell speculative execution. +- `lock/1` sets map-level `frozen` only. Mutations + (`put` / `put_meta` / `delete` / `apply_difference` / `storage_put_map`) reject + frozen maps. +- `Chain.Transaction.apply/3` mutates state in place on an unlocked candidate. +- `Chain.State` is **mutable**: always `clone/1` before applying transactions on + a shared cached state. + +## Build + +- NIF: `mix compile` → `elixir_make` → `c_src/` → `priv/merkletree_nif.so` +- EVM binary: `evm/evm` (needs `libboost-dev`) +- `deps/libsecp256k1`: build once with `make -C deps/libsecp256k1/` (not via `mix`) diff --git a/lib/block_process.ex b/lib/block_process.ex index bee4c19..ee75ee2 100644 --- a/lib/block_process.ex +++ b/lib/block_process.ex @@ -28,13 +28,6 @@ defmodule BlockProcess do EtsLru.put(__MODULE__.State, Block.hash(block), Chain.State.lock(Block.state(block))) end - def with_account_tree(block_ref, account_id, fun) do - with_block(block_ref, fn - nil -> fun.(nil) - block -> fun.(Block.account_tree(block, account_id)) - end) - end - def with_account(block_ref, account_id, fun) do with_state(block_ref, fn nil -> fun.(nil) diff --git a/lib/caccount_map.ex b/lib/caccount_map.ex index 36667ed..e4689be 100644 --- a/lib/caccount_map.ex +++ b/lib/caccount_map.ex @@ -6,19 +6,20 @@ defmodule CAccountMap do alias Chain.Account @type t :: reference() + @null <<0::unsigned-size(256)>> def new, do: CMerkleTree.account_map_new() def clone(map), do: CMerkleTree.account_map_clone(map) - def clone_lazy(map), do: CMerkleTree.account_map_clone_lazy(map) - def lock(map), do: CMerkleTree.account_map_lock(map) def root_hash(map), do: CMerkleTree.account_map_root_hash(map) def state_trie(map), do: CMerkleTree.account_map_state_trie(map) + def get_proofs(map, <<_::160>> = addr), do: CMerkleTree.account_map_get_proofs(map, addr) + def get(map, <<_::160>> = addr) do case CMerkleTree.account_map_get(map, addr) do :undefined -> :undefined @@ -37,8 +38,18 @@ defmodule CAccountMap do CMerkleTree.account_map_put(map, addr, nonce, encode_balance(balance), storage, code) end + def put_meta(map, <<_::160>> = addr, nonce, balance, code) do + CMerkleTree.account_map_put_meta(map, addr, nonce, encode_balance(balance), code) + end + def put_account(map, <<_::160>> = addr, %Account{} = account) do - put(map, addr, account.nonce, account.balance, Account.tree(account), Account.code(account)) + case account.storage_root do + nil -> + put_meta(map, addr, account.nonce, account.balance, Account.code(account)) + + storage -> + put(map, addr, account.nonce, account.balance, storage, Account.code(account)) + end end def delete(map, <<_::160>> = addr), do: CMerkleTree.account_map_delete(map, addr) @@ -52,17 +63,64 @@ defmodule CAccountMap do end def to_account_list(map) do - Enum.map(to_list(map), fn {addr, {nonce, balance, storage, code}} -> - {addr, Account.from_parts(nonce, balance, storage, code)} + Enum.map(to_list(map), fn {addr, {nonce, balance, root_hash, code}} -> + {addr, Account.from_parts(nonce, balance, root_hash, code)} end) end - def list_difference(map_a, map_b) do - Map.new(CMerkleTree.account_map_list_difference_raw(map_a, map_b), fn {addr, {side_a, side_b}} -> - {addr, {decode_account_side(side_a), decode_account_side(side_b)}} - end) + @doc """ + Apply EVM-style storage updates in one NIF call. + `updates` is `%{addr => %{slot => value}}` or a list of `{addr, [{slot, value}]}`. + """ + def storage_put_map(map, updates) when is_map(updates) do + list = + Enum.map(updates, fn {addr, kvs} -> + {addr, Map.to_list(kvs)} + end) + + storage_put_map(map, list) + end + + def storage_put_map(map, updates) when is_list(updates) do + CMerkleTree.account_map_storage_put_map(map, updates) + end + + def storage_get(map, <<_::160>> = addr, key) do + case CMerkleTree.account_map_storage_get(map, addr, to_bytes32(key)) do + nil -> nil + @null -> nil + value -> value + end + end + + def storage_get_range(map, <<_::160>> = addr, key, count) + when is_integer(count) and count >= 1 and count <= 256 do + CMerkleTree.account_map_storage_get_range(map, addr, to_bytes32(key), count) + |> Enum.map(fn {k, v} -> {k, if(v == @null, do: nil, else: v)} end) end + def storage_to_list(map, <<_::160>> = addr), + do: CMerkleTree.account_map_storage_to_list(map, addr) + + def storage_size(map, <<_::160>> = addr), + do: CMerkleTree.account_map_storage_size(map, addr) + + def storage_root_hash(map, <<_::160>> = addr), + do: CMerkleTree.account_map_storage_root_hash(map, addr) + + def storage_root_hashes(map, <<_::160>> = addr) do + <> = + CMerkleTree.account_map_storage_root_hashes(map, addr) + + [a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p] + end + + def storage_get_proofs(map, <<_::160>> = addr, key), + do: CMerkleTree.account_map_storage_get_proofs(map, addr, to_bytes(key)) + def difference_full(map_a, map_b) do CMerkleTree.account_map_difference_full(map_a, map_b) end @@ -78,17 +136,15 @@ defmodule CAccountMap do defp decode_storage_value(nil), do: nil defp decode_storage_value(val) when is_binary(val), do: val - defp decode_account_side(nil), do: nil - defp decode_account_side(entry), do: entry |> decode_entry() |> account_from_parts() - def uncompact_state(accounts), do: CMerkleTree.account_map_uncompact_state(accounts) - defp decode_entry({nonce, balance, storage, code}) do - {nonce, decode_balance(balance), storage, code} + # Third element is always a 32-byte storage root hash (never a live resource). + defp decode_entry({nonce, balance, <<_::binary-size(32)>> = root_hash, code}) do + {nonce, decode_balance(balance), root_hash, code} end - defp account_from_parts({nonce, balance, storage, code}) do - Account.from_parts(nonce, balance, storage, code) + defp account_from_parts({nonce, balance, root_hash, code}) do + Account.from_parts(nonce, balance, root_hash, code) end defp encode_balance(balance) when is_integer(balance) and balance >= 0 do @@ -104,4 +160,17 @@ defmodule CAccountMap do defp decode_balance(balance) when is_binary(balance) do :binary.decode_unsigned(balance) end + + defp to_bytes32(nil), do: @null + defp to_bytes32(int) when is_integer(int), do: <> + + defp to_bytes32(string) when is_binary(string) and byte_size(string) < 32 do + missing = (32 - byte_size(string)) * 8 + <<0::unsigned-size(missing), string::binary>> + end + + defp to_bytes32(string) when is_binary(string) and byte_size(string) == 32, do: string + + defp to_bytes(string) when is_binary(string), do: string + defp to_bytes(int) when is_integer(int), do: to_bytes32(int) end diff --git a/lib/chain/account.ex b/lib/chain/account.ex index ccd86b6..2e8fb2e 100644 --- a/lib/chain/account.ex +++ b/lib/chain/account.ex @@ -7,7 +7,7 @@ defmodule Chain.Account do @type t :: %Chain.Account{ nonce: non_neg_integer(), balance: non_neg_integer(), - storage_root: CMerkleTree.t(), + storage_root: CMerkleTree.t() | nil, code: binary() | nil } @@ -24,10 +24,38 @@ defmodule Chain.Account do def nonce(%Chain.Account{nonce: nonce}), do: nonce def balance(%Chain.Account{balance: balance}), do: balance + @doc """ + Live storage trie for standalone (genesis/import) accounts only. + Map-backed accounts (`storage_root: nil` with `:root_hash`) have no live trie — + use `Chain.State.storage_*` APIs instead. + """ @spec tree(Chain.Account.t()) :: CMerkleTree.t() - def tree(%Chain.Account{storage_root: nil}), do: CMerkleTree.new() + def tree(%Chain.Account{storage_root: nil} = acc) do + if map_backed?(acc) do + raise ArgumentError, + "map-backed account has no live storage_root; use Chain.State.storage_* APIs" + else + CMerkleTree.new() + end + end + def tree(%Chain.Account{storage_root: root}), do: root + @doc """ + Build an account from `CAccountMap.get/2` parts. + When `storage` is a 32-byte root hash, the account is map-backed (`storage_root: nil`). + When `storage` is a merkle resource (or nil), it is a standalone trie for genesis/put. + """ + def from_parts(nonce, balance, <>, code) do + %Chain.Account{ + nonce: nonce, + balance: balance, + storage_root: nil, + code: if(code == "", do: nil, else: code) + } + |> Map.put(:root_hash, root_hash) + end + def from_parts(nonce, balance, storage, code) do %Chain.Account{ nonce: nonce, @@ -43,7 +71,16 @@ defmodule Chain.Account do end def put_tree(%Chain.Account{} = acc, root) do - %Chain.Account{acc | storage_root: root} + acc + |> Map.put(:storage_root, root) + |> Map.delete(:root_hash) + end + + def root_hash(%Chain.Account{storage_root: nil} = acc) do + case Map.get(acc, :root_hash) do + <<_::binary-size(32)>> = hash -> hash + _ -> CMerkleTree.root_hash(CMerkleTree.new()) + end end def root_hash(%Chain.Account{} = acc) do @@ -56,13 +93,7 @@ defmodule Chain.Account do def uncompact(%Chain.Account{storage_root: {MapMerkleTree, _opts, items}} = acc) when is_map(items) do - # old_root = Map.get(acc, :root_hash) storage_root = CMerkleTree.from_map(items) - - # if old_root != CMerkleTree.root_hash(storage_root) do - # IO.inspect({old_root, CMerkleTree.root_hash(storage_root)}, label: "root_hash mismatch") - # end - %Chain.Account{acc | storage_root: storage_root} end @@ -71,6 +102,10 @@ defmodule Chain.Account do end def compact(%Chain.Account{} = acc) do + if map_backed?(acc) do + raise ArgumentError, "map-backed accounts are compacted via Chain.State.compact/1" + end + tree = tree(acc) if CMerkleTree.size(tree) == 0 do @@ -82,10 +117,17 @@ defmodule Chain.Account do |> Map.put(:code_hash, codehash(acc)) end - def storage_set_value(acc, key = <<_k::256>>, value = <<_v::256>>) do - %Chain.Account{} = acc + def storage_set_value(%Chain.Account{} = acc, key = <<_k::256>>, value = <<_v::256>>) do + if map_backed?(acc) do + raise ArgumentError, + "map-backed account storage writes go through Chain.State.storage_put_map/2" + end + store = CMerkleTree.insert(tree(acc), key, value) - %{acc | storage_root: store} + + acc + |> Map.put(:storage_root, store) + |> Map.delete(:root_hash) end def storage_set_value(acc, key, value) when is_integer(key) do @@ -102,6 +144,11 @@ defmodule Chain.Account do end def storage_value(%Chain.Account{} = acc, key) when is_binary(key) do + if map_backed?(acc) do + raise ArgumentError, + "map-backed account storage reads go through Chain.State.storage_value/3" + end + case CMerkleTree.get(tree(acc), key) do nil -> <<0::unsigned-size(256)>> bin -> bin @@ -131,4 +178,7 @@ defmodule Chain.Account do def codehash(%Chain.Account{code: code}) do Diode.hash(code) end + + defp map_backed?(%Chain.Account{storage_root: nil} = acc), do: Map.has_key?(acc, :root_hash) + defp map_backed?(_), do: false end diff --git a/lib/chain/block.ex b/lib/chain/block.ex index 0d94078..988bbb7 100644 --- a/lib/chain/block.ex +++ b/lib/chain/block.ex @@ -76,17 +76,26 @@ defmodule Chain.Block do end end - def account_tree(%Block{} = block, account_id) do - case Chain.State.account(state(block), account_id) do - nil -> nil - acc -> acc |> Chain.Account.tree() - end + def account_storage_root_hash(%Block{} = block, account_id) do + Chain.State.storage_root_hash(state(block), account_id) + end + + def account_storage_root_hashes(%Block{} = block, account_id) do + Chain.State.storage_root_hashes(state(block), account_id) + end + + def account_storage_get_proofs(%Block{} = block, account_id, key) do + Chain.State.storage_get_proofs(state(block), account_id, key) end def state_tree(%Block{} = block) do state(block) |> Chain.State.tree() end + def account_proof(%Block{} = block, account_id) do + Chain.State.get_proofs(state(block), account_id) + end + @doc "For snapshot exporting ensure the block has a full state object" @spec ensure_state(Chain.Block.t()) :: Chain.Block.t() def ensure_state(block = %Block{header: %{state_hash: %Chain.State{}}}) do diff --git a/lib/chain/state.ex b/lib/chain/state.ex index 2878645..c46d99e 100644 --- a/lib/chain/state.ex +++ b/lib/chain/state.ex @@ -9,7 +9,6 @@ defmodule Chain.State do {:nowarn_function, new: 0}, {:nowarn_function, uncompact: 1}, {:nowarn_function, clone: 1}, - {:nowarn_function, clone_lazy: 1}, {:nowarn_function, from_binary: 1} ] @@ -22,13 +21,30 @@ defmodule Chain.State do end def compact(%Chain.State{accounts: accounts} = state) do - accounts = + # Map-backed gets return root hashes only — build compact maps via storage_* APIs. + compact_accounts = accounts - |> CAccountMap.to_account_list() - |> Enum.map(fn {id, acc} -> {id, Account.compact(acc)} end) - |> Map.new() + |> CAccountMap.to_list() + |> Map.new(fn {id, {nonce, balance, root_hash, code}} -> + items = Map.new(CAccountMap.storage_to_list(accounts, id)) + + acc = + %Account{ + nonce: nonce, + balance: balance, + storage_root: if(map_size(items) == 0, do: nil, else: {MapMerkleTree, [], items}), + code: if(code == "", do: nil, else: code) + } + |> Map.put(:root_hash, root_hash) + |> Map.put( + :code_hash, + Account.codehash(%Account{code: if(code == "", do: nil, else: code)}) + ) + + {id, acc} + end) - %Chain.State{state | accounts: accounts} + %Chain.State{state | accounts: compact_accounts} end def uncompact(%Chain.State{accounts: accounts} = state) do @@ -44,6 +60,39 @@ defmodule Chain.State do CAccountMap.state_trie(accounts) end + def get_proofs(%Chain.State{accounts: accounts}, <<_::160>> = addr) do + CAccountMap.get_proofs(accounts, addr) + end + + def storage_value(%Chain.State{accounts: accounts}, <<_::160>> = addr, key) do + case CAccountMap.storage_get(accounts, addr, key) do + nil -> <<0::unsigned-size(256)>> + bin -> bin + end + end + + def storage_put_map(%Chain.State{accounts: accounts} = state, updates) do + %{state | accounts: CAccountMap.storage_put_map(accounts, updates), hash: nil} + end + + def storage_to_list(%Chain.State{accounts: accounts}, <<_::160>> = addr), + do: CAccountMap.storage_to_list(accounts, addr) + + def storage_size(%Chain.State{accounts: accounts}, <<_::160>> = addr), + do: CAccountMap.storage_size(accounts, addr) + + def storage_get_range(%Chain.State{accounts: accounts}, <<_::160>> = addr, key, count), + do: CAccountMap.storage_get_range(accounts, addr, key, count) + + def storage_root_hash(%Chain.State{accounts: accounts}, <<_::160>> = addr), + do: CAccountMap.storage_root_hash(accounts, addr) + + def storage_root_hashes(%Chain.State{accounts: accounts}, <<_::160>> = addr), + do: CAccountMap.storage_root_hashes(accounts, addr) + + def storage_get_proofs(%Chain.State{accounts: accounts}, <<_::160>> = addr, key), + do: CAccountMap.storage_get_proofs(accounts, addr, key) + def hash(%Chain.State{hash: nil} = state) do CAccountMap.root_hash(state.accounts) end @@ -90,8 +139,8 @@ defmodule Chain.State do end def difference( - %Chain.State{accounts: accounts_a} = state_a, - %Chain.State{accounts: accounts_b} = state_b + %Chain.State{accounts: accounts_a} = _state_a, + %Chain.State{accounts: accounts_b} = _state_b ) do {time, result} = :timer.tc(fn -> @@ -107,12 +156,12 @@ defmodule Chain.State do report = if map_size(storage_map) > 0 do - acc_a = account(state_a, id) || ensure_account(state_a, id) - acc_b = account(state_b, id) || ensure_account(state_b, id) - Map.merge(report, %{ state: storage_map, - root_hash: {Account.root_hash(acc_a), Account.root_hash(acc_b)} + root_hash: { + CAccountMap.storage_root_hash(accounts_a, id), + CAccountMap.storage_root_hash(accounts_b, id) + } }) else report @@ -153,14 +202,12 @@ defmodule Chain.State do defp side_field({_nonce, _balance, code}, :code), do: code + # Writable fork for block sync and speculative paths on locked/cached peak state. def clone(%Chain.State{accounts: accounts} = state) do %{state | accounts: CAccountMap.clone(accounts), hash: nil} end - def clone_lazy(%Chain.State{accounts: accounts} = state) do - %{state | accounts: CAccountMap.clone_lazy(accounts), hash: nil} - end - + # Freeze map (`frozen` only). Map storage writes reject while frozen. def lock(%Chain.State{accounts: accounts} = state) do CAccountMap.lock(accounts) state diff --git a/lib/chaindefinition/voyager.ex b/lib/chaindefinition/voyager.ex index 0487651..d37cc5d 100644 --- a/lib/chaindefinition/voyager.ex +++ b/lib/chaindefinition/voyager.ex @@ -6,24 +6,43 @@ defmodule ChainDefinition.Voyager do id = Base16.decode(account["addr"]) code = Base16.decode(account["code"]) {balance, ""} = Integer.parse(account["balance"]) - acc = State.ensure_account(state, id) - %Account{} = acc - acc = %{acc | balance: balance, code: code} + # Field-only update via put_meta (map-backed accounts have no live storage_root). acc = - if Map.has_key?(account, "state_patch") do - Enum.reduce(account["state_patch"], acc, fn [key, value], acc -> - Account.storage_set_value(acc, Base16.decode(key), Base16.decode(value)) - end) - else - acc = %{acc | storage_root: nil} + state + |> State.ensure_account(id) + |> Map.put(:balance, balance) + |> Map.put(:code, code) + |> Map.put(:storage_root, nil) + |> Map.delete(:root_hash) + + state = State.set_account(state, id, acc) - Enum.reduce(account["state"], acc, fn [key, value], acc -> - Account.storage_set_value(acc, Base16.decode(key), Base16.decode(value)) + if Map.has_key?(account, "state_patch") do + updates = + Map.new(account["state_patch"], fn [key, value] -> + {Base16.decode(key), Base16.decode(value)} end) + + if map_size(updates) == 0 do + state + else + State.storage_put_map(state, %{id => updates}) end + else + # Full storage replace via a caller-owned standalone trie. + tree = + Enum.reduce(account["state"] || [], CMerkleTree.new(), fn [key, value], tree -> + CMerkleTree.insert(tree, Base16.decode(key), Base16.decode(value)) + end) - State.set_account(state, id, acc) + State.set_account(state, id, %Account{ + nonce: acc.nonce, + balance: balance, + storage_root: tree, + code: code + }) + end end) end diff --git a/lib/cmerkletree.ex b/lib/cmerkletree.ex index ff26907..6fd520b 100644 --- a/lib/cmerkletree.ex +++ b/lib/cmerkletree.ex @@ -19,10 +19,6 @@ defmodule CMerkleTree do insert_items(new(), list) end - def list_difference(_a, _b) do - raise "CMerkleTree.list_difference/2 removed; use CAccountMap.list_difference/2" - end - def difference(a, b) do difference_raw(a, b) |> Enum.map(fn @@ -119,19 +115,27 @@ defmodule CMerkleTree do def account_map_new(), do: error() def account_map_clone(_map), do: error() - def account_map_clone_lazy(_map), do: error() def account_map_root_hash(_map), do: error() def account_map_state_trie(_map), do: error() + def account_map_get_proofs(_map, _addr), do: error() def account_map_lock(_map), do: error() def account_map_get(_map, _addr), do: error() def account_map_put(_map, _addr, _nonce, _balance, _storage, _code), do: error() + def account_map_put_meta(_map, _addr, _nonce, _balance, _code), do: error() def account_map_delete(_map, _addr), do: error() def account_map_size(_map), do: error() def account_map_to_list(_map), do: error() - def account_map_list_difference_raw(_map_a, _map_b), do: error() def account_map_difference_full(_map_a, _map_b), do: error() def account_map_apply_difference(_map, _delta), do: error() def account_map_uncompact_state(_map), do: error() + def account_map_storage_put_map(_map, _updates), do: error() + def account_map_storage_get(_map, _addr, _key), do: error() + def account_map_storage_get_range(_map, _addr, _key, _count), do: error() + def account_map_storage_to_list(_map, _addr), do: error() + def account_map_storage_size(_map, _addr), do: error() + def account_map_storage_root_hash(_map, _addr), do: error() + def account_map_storage_root_hashes(_map, _addr), do: error() + def account_map_storage_get_proofs(_map, _addr, _key), do: error() defp struct_sizes_raw, do: error() defp memory_stats_raw(_tree), do: error() diff --git a/lib/evm.ex b/lib/evm.ex index 51bc6ae..b9673d4 100644 --- a/lib/evm.ex +++ b/lib/evm.ex @@ -41,10 +41,9 @@ defmodule Evm do @type t :: %Evm.Task{} - @spec store(Evm.Task.t()) :: CMerkleTree.t() + @spec store(Evm.Task.t()) :: list() def store(%Task{chain_state: state} = task) do - State.ensure_account(state, address(task)) - |> Account.tree() + State.storage_to_list(state, address(task)) end def code(%Task{code: code}), do: code @@ -498,16 +497,10 @@ defmodule Evm do end defp cache_account(state, port, address) do - tree = - State.ensure_account(state, address) - |> Chain.Account.tree() - - # IO.puts("value size: #{CMerkleTree.size(tree)}") - cache = - if CMerkleTree.size(tree) < 100 do + if State.storage_size(state, address) < 100 do values = - CMerkleTree.to_list(tree) + State.storage_to_list(state, address) |> Enum.map(fn {k, v} -> [k, v] end) [ @@ -562,14 +555,12 @@ defmodule Evm do ) do count = Diode.evm_storage_read_ahead() |> max(0) |> min(254) |> Kernel.+(1) - tree = - state(evm) - |> State.ensure_account(addr) - |> Chain.Account.tree() - entries = - CMerkleTree.get_range_raw(tree, key, count) - |> Enum.map(fn {k, v} -> [k, v] end) + State.storage_get_range(state(evm), addr, key, count) + |> Enum.map(fn + {k, nil} -> [k, <<0::unsigned-size(256)>>] + {k, v} -> [k, v] + end) true = Port.command(evm.port, [<> | entries]) {:cont, evm} @@ -754,13 +745,7 @@ defmodule Evm do defp process_updates(rest, state) do updates = parse_map(rest, %{}) - - Enum.reduce(updates, state, fn {addr, kvs}, state -> - acc = State.ensure_account(state, addr) - root = CMerkleTree.insert_items(Account.tree(acc), Map.to_list(kvs)) - acc = Chain.Account.put_tree(acc, root) - State.set_account(state, addr, acc) - end) + State.storage_put_map(state, updates) end defp parse_map( diff --git a/lib/network/edge_v2.ex b/lib/network/edge_v2.ex index 7558ef6..f701037 100644 --- a/lib/network/edge_v2.ex +++ b/lib/network/edge_v2.ex @@ -66,6 +66,7 @@ defmodule Network.EdgeV2 do end) |> response() + # state account inclusion proof uses map-owned state_trie via Block.account_proof/2 ["getaccountroot", index, id] -> BlockProcess.with_block(to_num(index), fn block -> Chain.Block.state(block) @@ -75,8 +76,8 @@ defmodule Network.EdgeV2 do error("account does not exist") %Chain.Account{} -> - proof = Chain.Block.state_tree(block) |> CMerkleTree.get_proofs(id) - root = CMerkleTree.root_hash(Chain.Block.account_tree(block, id)) + proof = Chain.Block.account_proof(block, id) + root = Chain.Block.account_storage_root_hash(block, id) response(root, proof) end end) @@ -92,15 +93,13 @@ defmodule Network.EdgeV2 do error("account does not exist") account = %Chain.Account{} -> - proof = - Chain.Block.state_tree(block) - |> CMerkleTree.get_proofs(id) + proof = Chain.Block.account_proof(block, id) response( %{ nonce: account.nonce, balance: account.balance, - storage_root: CMerkleTree.root_hash(Chain.Block.account_tree(block, id)), + storage_root: Chain.Block.account_storage_root_hash(block, id), code: Chain.Account.codehash(account) }, proof @@ -109,28 +108,34 @@ defmodule Network.EdgeV2 do end) ["getaccountroots", index, id] -> - BlockProcess.with_account_tree(to_num(index), id, fn - nil -> error("account does not exist") - tree -> response(CMerkleTree.root_hashes(tree)) + BlockProcess.with_block(to_num(index), fn block -> + case Chain.Block.state(block) |> Chain.State.account(id) do + nil -> error("account does not exist") + %Chain.Account{} -> response(Chain.Block.account_storage_root_hashes(block, id)) + end end) ["getaccountvalue", index, id, key] -> - BlockProcess.with_account_tree(to_num(index), id, fn - nil -> error("account does not exist") - tree -> response(CMerkleTree.get_proofs(tree, key)) + BlockProcess.with_block(to_num(index), fn block -> + case Chain.Block.state(block) |> Chain.State.account(id) do + nil -> error("account does not exist") + %Chain.Account{} -> response(Chain.Block.account_storage_get_proofs(block, id, key)) + end end) ["getaccountvalues", index, id | keys] -> - BlockProcess.with_account_tree(to_num(index), id, fn - nil -> - error("account does not exist") - - tree -> - response( - Enum.map(keys, fn key -> - CMerkleTree.get_proofs(tree, key) - end) - ) + BlockProcess.with_block(to_num(index), fn block -> + case Chain.Block.state(block) |> Chain.State.account(id) do + nil -> + error("account does not exist") + + %Chain.Account{} -> + response( + Enum.map(keys, fn key -> + Chain.Block.account_storage_get_proofs(block, id, key) + end) + ) + end end) ["sendtransaction", tx] -> @@ -139,7 +144,8 @@ defmodule Network.EdgeV2 do err = Chain.with_peak(fn peak -> - state = Chain.Block.state(peak) |> Chain.State.clone_lazy() + # Peak is State.lock'd in cache; clone/1 forks a writable candidate. + state = Chain.Block.state(peak) |> Chain.State.clone() case Chain.Transaction.apply(tx, peak, state) do {:ok, _state, %{msg: :ok}} -> nil diff --git a/lib/network/rpc.ex b/lib/network/rpc.ex index 9c172b4..c543960 100644 --- a/lib/network/rpc.ex +++ b/lib/network/rpc.ex @@ -272,15 +272,20 @@ defmodule Network.Rpc do "eth_getStorageAt" -> [address, location, ref] = params - get_account([address, ref]) - |> Chain.Account.storage_value(Base16.decode_int(location)) - |> result() + with_block(ref, fn block -> + Chain.Block.state(block) + |> Chain.State.storage_value(Base16.decode(address), Base16.decode_int(location)) + |> result() + end) "eth_getStorage" -> - get_account(params) - |> Chain.Account.tree() - |> CMerkleTree.to_list() - |> result() + [address, ref] = params + + with_block(ref, fn block -> + Chain.Block.state(block) + |> Chain.State.storage_to_list(Base16.decode(address)) + |> result() + end) "eth_estimateGas" -> # TODO real estimate @@ -856,7 +861,8 @@ defmodule Network.Rpc do end defp apply_transaction(tx, block) do - state = Block.state(block) |> Chain.State.clone_lazy() + # Peak/cached blocks are State.lock'd; use clone/1 for a writable fork. + state = Block.state(block) |> Chain.State.clone() case Chain.Transaction.apply(tx, block, state) do {:ok, _state, rcpt = %{msg: :ok}} -> diff --git a/lib/shell.ex b/lib/shell.ex index 1a2790d..20b152a 100644 --- a/lib/shell.ex +++ b/lib/shell.ex @@ -43,7 +43,8 @@ defmodule Shell do def call_tx(tx, blockRef) do Stats.tc(:call_tx, fn -> Network.Rpc.with_block(blockRef, fn block -> - state = Chain.Block.state(block) |> Chain.State.clone_lazy() + # Cached blocks are State.lock'd; clone/1 is required for a writable fork. + state = Chain.Block.state(block) |> Chain.State.clone() Stats.tc(:apply, fn -> Chain.Transaction.apply(tx, block, state, static: true) diff --git a/scripts/cmerkle_fuzz.exs b/scripts/cmerkle_fuzz.exs index 5589726..7a44e89 100644 --- a/scripts/cmerkle_fuzz.exs +++ b/scripts/cmerkle_fuzz.exs @@ -147,7 +147,7 @@ defmodule CMerkleFuzz do 28 -> s_list_diff_vs_account_map_get(round, ctx) 29 -> s_list_diff_gc_pressure(round, ctx) 30 -> s_prepare_state_composite(round, ctx) - 31 -> s_lazy_clone_equivalence(round, ctx) + 31 -> s_clone_equivalence(round, ctx) other -> raise("unknown fuzz scenario #{inspect(other)}") end @@ -474,12 +474,13 @@ defmodule CMerkleFuzz do compact = build_compact_accounts(n) {accounts, _hash} = CAccountMap.uncompact_state(compact) - {_, _, storage, _} = CAccountMap.get(accounts, addr(1)) + {_n, _b, root, _c} = CAccountMap.get(accounts, addr(1)) + if not is_binary(root) or byte_size(root) != 32, do: raise("expected 32-byte root hash") - alt = - CMerkleTree.insert(CMerkleTree.clone(storage), slot(9999), <<9999::unsigned-size(256)>>) - - _ = CMerkleTree.difference(storage, alt) + # Exercise storage APIs (get no longer returns a live trie for difference). + _ = CAccountMap.storage_get(accounts, addr(1), slot(1)) + _ = CAccountMap.storage_root_hash(accounts, addr(1)) + _ = CAccountMap.storage_to_list(accounts, addr(1)) _ = CAccountMap.get(accounts, addr(2)) end @@ -574,12 +575,9 @@ defmodule CMerkleFuzz do Enum.reduce(1..writes, fork, fn j, state -> id = addr(rem(j, n) + 1) - acc = - state - |> State.account(id) - |> Account.storage_set_value(slot(j + 50_000), <>) - - State.set_account(state, id, acc) + State.storage_put_map(state, %{ + id => %{slot(j + 50_000) => <>} + }) end) fork_hash = State.hash(fork) @@ -588,7 +586,7 @@ defmodule CMerkleFuzz do for j <- 1..writes do id = addr(rem(j, n) + 1) - if Account.storage_value(State.account(parent, id), slot(j + 50_000)) != + if State.storage_value(parent, id, slot(j + 50_000)) != <<0::unsigned-size(256)>> do raise("parent mutated after lock->clone fork write") end @@ -640,9 +638,6 @@ defmodule CMerkleFuzz do CAccountMap.put(acc, addr(i), i, i * 1_000, storage, <>) end) - {_, _, storage_a, _} = CAccountMap.get(map, addr(1)) - {_, _, storage_b, _} = CAccountMap.get(map, addr(min(n, 2))) - workers = 6 1..workers @@ -651,7 +646,9 @@ defmodule CMerkleFuzz do if rem(w, 2) == 0 do _ = CAccountMap.lock(map) else - _ = CMerkleTree.difference(storage_a, storage_b) + _ = CAccountMap.storage_root_hash(map, addr(1)) + _ = CAccountMap.storage_root_hash(map, addr(min(n, 2))) + _ = CAccountMap.storage_to_list(map, addr(1)) end :ok @@ -663,7 +660,7 @@ defmodule CMerkleFuzz do |> Stream.run() end - # --- S19–S30: native account_map list_difference (see c_src/LOCK_ORDER.md D-C7, D-D7) --- + # --- S19–S30: native account_map difference_full (see c_src/LOCK_ORDER.md D-C7, D-D7) --- defp s_account_map_list_diff_large_compact(_round, _ctx) do n = :rand.uniform(200) + 100 @@ -674,19 +671,18 @@ defmodule CMerkleFuzz do CAccountMap.clone(base) |> then(fn map -> i = :rand.uniform(n) - {nonce, balance, storage, code} = CAccountMap.get(map, addr(i)) - storage = - CMerkleTree.insert( - CMerkleTree.clone(storage), - slot(i + 400_000), - <> - ) - - CAccountMap.put(map, addr(i), nonce + 1, balance, storage, code) + map + |> CAccountMap.storage_put_map(%{ + addr(i) => %{slot(i + 400_000) => <>} + }) + |> then(fn m -> + {nonce, balance, _root, code} = CAccountMap.get(m, addr(i)) + CAccountMap.put_meta(m, addr(i), nonce + 1, balance, code) + end) end) - _ = CAccountMap.list_difference(base, fork) + _ = CAccountMap.difference_full(base, fork) end defp s_list_diff_vs_to_list(_round, _ctx) do @@ -700,7 +696,7 @@ defmodule CMerkleFuzz do |> Task.async_stream( fn w -> if rem(w, 2) == 0 do - _ = CAccountMap.list_difference(map, fork) + _ = CAccountMap.difference_full(map, fork) else _ = CAccountMap.to_list(map) end @@ -724,7 +720,7 @@ defmodule CMerkleFuzz do |> Task.async_stream( fn w -> if rem(w, 2) == 0 do - _ = CAccountMap.list_difference(map, fork) + _ = CAccountMap.difference_full(map, fork) else _ = CAccountMap.lock(map) end @@ -749,7 +745,7 @@ defmodule CMerkleFuzz do fork = CAccountMap.clone(map) if rem(w, 2) == 0 do - _ = CAccountMap.list_difference(map, fork) + _ = CAccountMap.difference_full(map, fork) else i = rem(w, 10) + 1 storage = CMerkleTree.insert(CMerkleTree.new(), slot(w + i), <>) @@ -767,8 +763,6 @@ defmodule CMerkleFuzz do defp s_list_diff_vs_storage_difference(_round, _ctx) do map = build_account_map(:rand.uniform(40) + 10) - {_, _, sa, _} = CAccountMap.get(map, addr(1)) - {_, _, sb, _} = CAccountMap.get(map, addr(2)) workers = 6 @@ -776,9 +770,11 @@ defmodule CMerkleFuzz do |> Task.async_stream( fn w -> if rem(w, 2) == 0 do - _ = CAccountMap.list_difference(map, CAccountMap.clone(map)) + _ = CAccountMap.difference_full(map, CAccountMap.clone(map)) else - _ = CMerkleTree.difference(sa, sb) + _ = CAccountMap.storage_root_hash(map, addr(1)) + _ = CAccountMap.storage_root_hash(map, addr(2)) + _ = CAccountMap.storage_to_list(map, addr(1)) end :ok @@ -794,12 +790,16 @@ defmodule CMerkleFuzz do a = build_account_map(:rand.uniform(50) + 10) b = CAccountMap.clone(a) i = :rand.uniform(10) + 1 - {nonce, balance, storage, code} = CAccountMap.get(b, addr(i)) - - storage = - CMerkleTree.insert(CMerkleTree.clone(storage), slot(88_888), <<88_888::unsigned-size(256)>>) - b = CAccountMap.put(b, addr(i), nonce + 1, balance, storage, code) + b = + b + |> CAccountMap.storage_put_map(%{ + addr(i) => %{slot(88_888) => <<88_888::unsigned-size(256)>>} + }) + |> then(fn m -> + {nonce, balance, _root, code} = CAccountMap.get(m, addr(i)) + CAccountMap.put_meta(m, addr(i), nonce + 1, balance, code) + end) workers = 4 @@ -807,9 +807,9 @@ defmodule CMerkleFuzz do |> Task.async_stream( fn w -> if rem(w, 2) == 0 do - _ = CAccountMap.list_difference(a, b) + _ = CAccountMap.difference_full(a, b) else - _ = CAccountMap.list_difference(b, a) + _ = CAccountMap.difference_full(b, a) end :ok @@ -830,18 +830,13 @@ defmodule CMerkleFuzz do fork = live |> State.clone() - |> then(fn st -> - id = addr(rem(:rand.uniform(n), n) + 1) - acc = State.account(st, id) - - tree = - Account.tree(acc) - |> CMerkleTree.insert(slot(55_555), <<55_555::unsigned-size(256)>>) + |> State.storage_put_map(%{ + addr(rem(:rand.uniform(n), n) + 1) => %{ + slot(55_555) => <<55_555::unsigned-size(256)>> + } + }) - State.set_account(st, id, Account.put_tree(acc, tree)) - end) - - _ = CAccountMap.list_difference(compact_nif, fork.accounts) + _ = CAccountMap.difference_full(compact_nif, fork.accounts) end defp s_list_diff_vs_uncompact(_round, _ctx) do @@ -854,7 +849,7 @@ defmodule CMerkleFuzz do fn w -> if rem(w, 2) == 0 do {accounts, _hash} = CAccountMap.uncompact_state(compact) - _ = CAccountMap.list_difference(accounts, CAccountMap.clone(accounts)) + _ = CAccountMap.difference_full(accounts, CAccountMap.clone(accounts)) else _ = CAccountMap.uncompact_state(compact) end @@ -901,7 +896,7 @@ defmodule CMerkleFuzz do |> Task.async_stream( fn w -> if rem(w, 2) == 0 do - _ = CAccountMap.list_difference(map, CAccountMap.clone(map)) + _ = CAccountMap.difference_full(map, CAccountMap.clone(map)) else _ = CAccountMap.get(map, addr(rem(w, 10) + 1)) end @@ -920,7 +915,7 @@ defmodule CMerkleFuzz do fork = CAccountMap.clone(map) for _ <- 1..8 do - _ = CAccountMap.list_difference(map, fork) + _ = CAccountMap.difference_full(map, fork) short = CMerkleTree.new() |> CMerkleTree.clone() |> CMerkleTree.lock() _ = short :erlang.garbage_collect() @@ -936,13 +931,10 @@ defmodule CMerkleFuzz do |> State.clone() |> then(fn st -> i = :rand.uniform(n) - acc = State.account(st, addr(i)) - - tree = - Account.tree(acc) - |> CMerkleTree.insert(slot(i + 200_000), <>) - State.set_account(st, addr(i), Account.put_tree(acc, tree)) + State.storage_put_map(st, %{ + addr(i) => %{slot(i + 200_000) => <>} + }) end) workers = 8 @@ -954,7 +946,7 @@ defmodule CMerkleFuzz do 0 -> _ = State.difference(prev, block) 1 -> _ = State.lock(State.clone(block)) 2 -> _ = CAccountMap.to_list(block.accounts) - _ -> _ = CAccountMap.list_difference(prev.accounts, block.accounts) + _ -> _ = CAccountMap.difference_full(prev.accounts, block.accounts) end :ok @@ -994,7 +986,7 @@ defmodule CMerkleFuzz do end end - defp s_lazy_clone_equivalence(_round, _ctx) do + defp s_clone_equivalence(_round, _ctx) do n = :rand.uniform(40) + 10 map = @@ -1007,22 +999,20 @@ defmodule CMerkleFuzz do id = addr(:rand.uniform(n)) mutate = fn st -> - acc = Chain.State.account(st, id) - tree = Chain.Account.tree(acc) |> CMerkleTree.insert(slot(888_888), <<99::unsigned-size(256)>>) - Chain.State.set_account(st, id, Chain.Account.put_tree(acc, tree)) + Chain.State.storage_put_map(st, %{ + id => %{slot(888_888) => <<99::unsigned-size(256)>>} + }) end - lazy = state |> Chain.State.clone_lazy() |> mutate.() - eager = state |> Chain.State.clone() |> mutate.() + fork_a = state |> Chain.State.clone() |> mutate.() + fork_b = state |> Chain.State.clone() |> mutate.() - if Chain.State.hash(lazy) != Chain.State.hash(eager) do - raise "lazy/eager fork hash mismatch" + if Chain.State.hash(fork_a) != Chain.State.hash(fork_b) do + raise "clone fork hash mismatch" end - parent_hash = CMerkleTree.root_hash(elem(CAccountMap.get(map, id), 2)) - {_, _, storage, _} = CAccountMap.get(map, id) - if CMerkleTree.get(storage, slot(888_888)) != nil do - raise "lazy clone corrupted parent storage" + if CAccountMap.storage_get(map, id, slot(888_888)) != nil do + raise "clone corrupted parent storage" end end diff --git a/scripts/cmerkle_heap_assumptions.exs b/scripts/cmerkle_heap_assumptions.exs index 44fed36..907e1ac 100644 --- a/scripts/cmerkle_heap_assumptions.exs +++ b/scripts/cmerkle_heap_assumptions.exs @@ -105,7 +105,7 @@ defmodule CMerkleHeapAssumptions do {:G, "difference on large divergent trees", &g_difference_heavy/1}, {:H, "proofs + root_hashes after deep updates", &h_proofs_and_hashes/1}, {:I, "account_map_lock bulk identical storage roots", &i_identical_root_lock_bulk/1}, - {:J, "account_map list_difference bounded shared_states growth", &j_list_difference_heap/1} + {:J, "account_map difference_full bounded shared_states growth", &j_difference_full_heap/1} ] end @@ -289,7 +289,7 @@ defmodule CMerkleHeapAssumptions do end) end - defp j_list_difference_heap(%{rounds: r}) do + defp j_difference_full_heap(%{rounds: r}) do n = max(40, min(r, 150)) base = @@ -317,7 +317,7 @@ defmodule CMerkleHeapAssumptions do if orphans0 > 0, do: raise("scenario J initial orphans=#{orphans0}") Enum.each(1..max(div(r, 2), 30), fn _ -> - _ = CAccountMap.list_difference(base, fork) + _ = CAccountMap.difference_full(base, fork) :erlang.garbage_collect() end) diff --git a/scripts/cmerkle_parallel_stress.exs b/scripts/cmerkle_parallel_stress.exs index 6718409..12518f4 100644 --- a/scripts/cmerkle_parallel_stress.exs +++ b/scripts/cmerkle_parallel_stress.exs @@ -198,7 +198,7 @@ defmodule CMerkleParallelStress do run_named("P18_writer_sim", fn -> p18_writer_sim(ctx) end) "P18L" -> - run_named("P18_lazy_clone_eth_call", fn -> p18_lazy_clone_eth_call(ctx) end) + run_named("P18_clone_eth_call", fn -> p18_clone_eth_call(ctx) end) "P19" -> run_named("P19_large_map_small_delta", fn -> p19_large_map_small_delta(ctx) end) @@ -713,16 +713,9 @@ defmodule CMerkleParallelStress do 1 -> id = addr(rem(w, n) + 1) - {_, _, storage, _} = CAccountMap.get(base, id) - - alt = - CMerkleTree.insert( - CMerkleTree.clone(storage), - slot(w + j + 50_000), - <> - ) - - _ = CMerkleTree.difference(storage, alt) + _ = CAccountMap.storage_root_hash(base, id) + _ = CAccountMap.storage_to_list(base, id) + _ = CAccountMap.storage_get(base, id, slot(w + j + 50_000)) 2 -> fork = CAccountMap.clone(base) @@ -751,13 +744,10 @@ defmodule CMerkleParallelStress do |> then(fn st -> Enum.reduce(1..min(ops, n), st, fn i, acc -> id = addr(rem(i, n) + 1) - acc0 = State.account(acc, id) - tree = Account.tree(acc0) - - tree = - CMerkleTree.insert(tree, slot(i + 100_000), <>) - State.set_account(acc, id, Account.put_tree(acc0, tree)) + State.storage_put_map(acc, %{ + id => %{slot(i + 100_000) => <>} + }) end) end) @@ -795,13 +785,12 @@ defmodule CMerkleParallelStress do locked = State.lock(prev) fork = State.clone(locked) id = addr(rem(w, n) + 1) - acc0 = State.account(fork, id) - tree = Account.tree(acc0) - tree = - CMerkleTree.insert(tree, slot(w + 300_000), <>) + fork = + State.storage_put_map(fork, %{ + id => %{slot(w + 300_000) => <>} + }) - _ = State.set_account(fork, id, Account.put_tree(acc0, tree)) _ = State.hash(fork) :ok end) @@ -847,11 +836,10 @@ defmodule CMerkleParallelStress do |> CMerkleTree.lock() 1 -> - _ = - CMerkleTree.difference( - Account.tree(State.account(live, addr(1))), - Account.tree(State.account(other, addr(1))) - ) + # Compare storage root hashes / lists (get no longer returns live tries). + _ = CAccountMap.storage_root_hash(live.accounts, addr(1)) + _ = CAccountMap.storage_root_hash(other.accounts, addr(1)) + _ = CAccountMap.storage_to_list(live.accounts, addr(1)) 2 -> _ = CAccountMap.uncompact_state(compact) @@ -889,13 +877,10 @@ defmodule CMerkleParallelStress do |> then(fn st -> Enum.reduce(1..min(ops, n), st, fn i, acc -> id = addr(rem(i, n) + 1) - acc0 = State.account(acc, id) - tree = Account.tree(acc0) - - tree = - CMerkleTree.insert(tree, slot(i + 300_000), <>) - State.set_account(acc, id, Account.put_tree(acc0, tree)) + State.storage_put_map(acc, %{ + id => %{slot(i + 300_000) => <>} + }) end) end) @@ -935,13 +920,10 @@ defmodule CMerkleParallelStress do |> then(fn st -> Enum.reduce(1..min(ops, n), st, fn i, acc -> id = addr(rem(i, n) + 1) - acc0 = State.account(acc, id) - tree = Account.tree(acc0) - tree = - CMerkleTree.insert(tree, slot(i + 100_000), <>) - - State.set_account(acc, id, Account.put_tree(acc0, tree)) + State.storage_put_map(acc, %{ + id => %{slot(i + 100_000) => <>} + }) end) end) @@ -961,7 +943,7 @@ defmodule CMerkleParallelStress do run.(:diff, differ, fn _ -> State.difference(prev, block) end) run.(:lock, lockers, fn _ -> State.lock(State.clone(block)) end) run.(:legacy, legacy, fn _ -> CAccountMap.to_list(block.accounts) end) - run.(:native, native, fn _ -> CAccountMap.list_difference(prev.accounts, block.accounts) end) + run.(:native, native, fn _ -> CAccountMap.difference_full(prev.accounts, block.accounts) end) end defp p18_writer_sim(%{tasks: tasks, accounts: n}) do @@ -970,15 +952,9 @@ defmodule CMerkleParallelStress do block = prev |> State.clone() - |> then(fn st -> - acc = State.account(st, addr(1)) - - tree = - Account.tree(acc) - |> CMerkleTree.insert(slot(400_000), <<400_000::unsigned-size(256)>>) - - State.set_account(st, addr(1), Account.put_tree(acc, tree)) - end) + |> State.storage_put_map(%{ + addr(1) => %{slot(400_000) => <<400_000::unsigned-size(256)>>} + }) writer = Task.async(fn -> @@ -993,12 +969,19 @@ defmodule CMerkleParallelStress do map = block.accounts case rem(w, 4) do - 0 -> _ = CAccountMap.lock(map) - 1 -> _ = CAccountMap.to_list(map) - 2 -> {_, _, sa, _} = CAccountMap.get(map, addr(1)) - {_, _, sb, _} = CAccountMap.get(map, addr(rem(w, n) + 1)) - _ = CMerkleTree.difference(sa, sb) - _ -> _ = CAccountMap.delete(CAccountMap.clone(map), addr(rem(w, n) + 1)) + 0 -> + _ = CAccountMap.lock(map) + + 1 -> + _ = CAccountMap.to_list(map) + + 2 -> + _ = CAccountMap.storage_root_hash(map, addr(1)) + _ = CAccountMap.storage_root_hash(map, addr(rem(w, n) + 1)) + _ = CAccountMap.storage_to_list(map, addr(1)) + + _ -> + _ = CAccountMap.delete(CAccountMap.clone(map), addr(rem(w, n) + 1)) end :ok @@ -1012,25 +995,18 @@ defmodule CMerkleParallelStress do Task.await(writer, :infinity) end - defp p18_lazy_clone_eth_call(%{tasks: tasks, accounts: n}) do + defp p18_clone_eth_call(%{tasks: tasks, accounts: n}) do base = build_live_state(max(n, 50)) 1..tasks |> Task.async_stream( fn w -> + id = addr(rem(w, max(n, 50)) + 1) + fork = base - |> State.clone_lazy() - |> then(fn st -> - id = addr(rem(w, max(n, 50)) + 1) - acc = State.account(st, id) - - tree = - Account.tree(acc) - |> CMerkleTree.insert(slot(900_000 + w), <>) - - State.set_account(st, id, Account.put_tree(acc, tree)) - end) + |> State.clone() + |> State.storage_put_map([{id, [{slot(900_000 + w), <>}]}]) _ = State.hash(fork) :ok @@ -1052,9 +1028,10 @@ defmodule CMerkleParallelStress do |> then(fn st -> Enum.reduce(1..5, st, fn i, acc -> id = addr(i) - acc0 = State.account(acc, id) - tree = Account.tree(acc0) |> CMerkleTree.insert(slot(i + 500_000), <>) - State.set_account(acc, id, Account.put_tree(acc0, tree)) + + State.storage_put_map(acc, %{ + id => %{slot(i + 500_000) => <>} + }) end) end) @@ -1091,11 +1068,22 @@ defmodule CMerkleParallelStress do |> Task.async_stream( fn w -> case rem(w, 5) do - 0 -> _ = CAccountMap.list_difference(map, other.accounts) - 1 -> _ = CAccountMap.lock(map) - 2 -> _ = State.difference(live, other) - 3 -> _ = CMerkleTree.difference(Account.tree(State.account(live, addr(1))), Account.tree(State.account(other, addr(1)))) - _ -> _ = CAccountMap.to_list(map) + 0 -> + _ = CAccountMap.difference_full(map, other.accounts) + + 1 -> + _ = CAccountMap.lock(map) + + 2 -> + _ = State.difference(live, other) + + 3 -> + _ = CAccountMap.storage_root_hash(live.accounts, addr(1)) + _ = CAccountMap.storage_root_hash(other.accounts, addr(1)) + _ = CAccountMap.storage_to_list(live.accounts, addr(1)) + + _ -> + _ = CAccountMap.to_list(map) end :ok diff --git a/test/caccount_map_lifetime_test.exs b/test/caccount_map_lifetime_test.exs index 1ea8282..aa7ebd5 100644 --- a/test/caccount_map_lifetime_test.exs +++ b/test/caccount_map_lifetime_test.exs @@ -5,6 +5,9 @@ # Lifetime / refcount regression tests for CAccountMap NIF resources. # Targets bugs like premature SharedState deletion while merkletree resources # are still alive (ethr_mutex_lock EINVAL on destroyed mutex). +# +# CAccountMap.get/2 returns a 32-byte storage root hash (never a live trie). +# Storage usability is checked via storage_get / storage_root_hash / storage_put_map. defmodule CAccountMapLifetimeTest do use ExUnit.Case, async: false @@ -46,25 +49,33 @@ defmodule CAccountMapLifetimeTest do Enum.reduce(1..n_accounts, CAccountMap.new(), &put_sample(&2, &1)) end - defp assert_storage_usable(storage) do - hash = CMerkleTree.root_hash(storage) - assert is_binary(hash) and byte_size(hash) > 0 + # Storage is no longer returned as a live resource from get/to_list. + # Verify the map can still read and that a clone remains writable. + defp assert_map_storage_usable(map, address) do + hash = CAccountMap.storage_root_hash(map, address) + assert is_binary(hash) and byte_size(hash) == 32 - updated = - CMerkleTree.insert_items(storage, [ - {slot(9_999), val(9_999)} - ]) + {_n, _b, root, _c} = CAccountMap.get(map, address) + assert root == hash + + fork = + map + |> CAccountMap.clone() + |> CAccountMap.storage_put_map(%{address => %{slot(9_999) => val(9_999)}}) - assert is_binary(CMerkleTree.root_hash(updated)) + assert CAccountMap.storage_get(fork, address, slot(9_999)) == val(9_999) + assert CAccountMap.storage_root_hash(map, address) == hash end - # CAccountMap.clone now produces writable (locked = false) storage tries that are - # distinct resources from the parent's, so account entries must be compared by - # value (storage root_hash) rather than by resource identity. + defp assert_state_storage_usable(state, address) do + assert_map_storage_usable(state.accounts, address) + end + + # Compare account entries by value (nonce/balance/root_hash/code). defp entry_value(:undefined), do: :undefined - defp entry_value({nonce, balance, storage, code}) do - {nonce, balance, CMerkleTree.root_hash(storage), code} + defp entry_value({nonce, balance, root_hash, code}) do + {nonce, balance, root_hash, code} end describe "clone GC must not corrupt parent storage" do @@ -78,8 +89,7 @@ defmodule CAccountMapLifetimeTest do entry_value(CAccountMap.get(base, addr(5))) end) - {_, _, storage, _} = CAccountMap.get(base, addr(5)) - assert_storage_usable(storage) + assert_map_storage_usable(base, addr(5)) end test "many ephemeral clones with reads between GC rounds" do @@ -103,8 +113,7 @@ defmodule CAccountMapLifetimeTest do end for i <- 1..8 do - {_, _, storage, _} = CAccountMap.get(base, addr(i)) - assert_storage_usable(storage) + assert_map_storage_usable(base, addr(i)) end end @@ -121,8 +130,7 @@ defmodule CAccountMapLifetimeTest do end) end - acc = State.account(state, addr(1)) - assert_storage_usable(Account.tree(acc)) + assert_state_storage_usable(state, addr(1)) assert State.hash(state) == State.hash(State.clone(state)) end end @@ -149,10 +157,11 @@ defmodule CAccountMapLifetimeTest do end) end - {_, _, storage_a, _} = CAccountMap.get(map, addr(10)) - {_, _, storage_b, _} = CAccountMap.get(map, addr(11)) - assert storage_a == storage_b - assert_storage_usable(storage_a) + assert CAccountMap.storage_root_hash(map, addr(10)) == + CAccountMap.storage_root_hash(map, addr(11)) + + assert_map_storage_usable(map, addr(10)) + assert_map_storage_usable(map, addr(11)) end test "shared storage with fork mutation splits only the mutated branch" do @@ -167,21 +176,17 @@ defmodule CAccountMapLifetimeTest do fork = map |> CAccountMap.clone() - |> CAccountMap.put( - addr(20), - 9, - 900, - CMerkleTree.insert_items(storage, [{slot(99), val(99)}]), - <<99>> - ) + |> CAccountMap.storage_put_map(%{addr(20) => %{slot(99) => val(99)}}) + |> CAccountMap.put_meta(addr(20), 9, 900, <<99>>) assert {9, 900, _, <<99>>} = CAccountMap.get(fork, addr(20)) assert {1, 100, _, <<20>>} = CAccountMap.get(map, addr(20)) assert {2, 200, _, <<21>>} = CAccountMap.get(map, addr(21)) + assert CAccountMap.storage_get(fork, addr(20), slot(99)) == val(99) + assert CAccountMap.storage_get(map, addr(20), slot(99)) == nil end) - {_, _, parent_storage, _} = CAccountMap.get(map, addr(20)) - assert_storage_usable(parent_storage) + assert_map_storage_usable(map, addr(20)) end end @@ -203,13 +208,12 @@ defmodule CAccountMapLifetimeTest do assert CAccountMap.get(base, addr(2)) != :undefined assert CAccountMap.size(base) == 4 - {_, _, storage, _} = CAccountMap.get(base, addr(2)) - assert_storage_usable(storage) + assert_map_storage_usable(base, addr(2)) end test "put replaces storage root without leaving dangling tries" do base = put_sample(CAccountMap.new(), 3) - {_, _, old_storage, _} = CAccountMap.get(base, addr(3)) + {_, _, old_root, _} = CAccountMap.get(base, addr(3)) new_storage = CMerkleTree.insert_items(CMerkleTree.new(), [{slot(7), val(7)}]) @@ -227,9 +231,11 @@ defmodule CAccountMapLifetimeTest do _fork = CAccountMap.clone(base) end) - assert {30, 30_000, storage, <<30>>} = CAccountMap.get(base, addr(3)) - assert storage != old_storage - assert_storage_usable(storage) + assert {30, 30_000, root, <<30>>} = CAccountMap.get(base, addr(3)) + assert is_binary(root) and byte_size(root) == 32 + assert root != old_root + assert CAccountMap.storage_get(base, addr(3), slot(7)) == val(7) + assert_map_storage_usable(base, addr(3)) end test "nested clone chain with middle resource dropped" do @@ -254,8 +260,7 @@ defmodule CAccountMapLifetimeTest do _ = c3 end) - {_, _, storage, _} = CAccountMap.get(base, addr(2)) - assert_storage_usable(storage) + assert_map_storage_usable(base, addr(2)) end end @@ -282,8 +287,7 @@ defmodule CAccountMapLifetimeTest do assert Enum.all?(Task.await_many(tasks, 60_000), &(&1 == :ok)) for i <- 1..6 do - {_, _, storage, _} = CAccountMap.get(base, addr(i)) - assert_storage_usable(storage) + assert_map_storage_usable(base, addr(i)) end end @@ -300,12 +304,11 @@ defmodule CAccountMapLifetimeTest do end base = - base - |> State.account(addr(1)) - |> Account.storage_set_value(slot(42), val(42)) - |> then(&State.set_account(base, addr(1), &1)) + State.storage_put_map(base, %{ + addr(1) => %{slot(42) => val(42)} + }) - assert Account.storage_value(State.account(base, addr(1)), slot(42)) == val(42) + assert State.storage_value(base, addr(1), slot(42)) == val(42) assert is_binary(State.hash(base)) end end @@ -326,9 +329,10 @@ defmodule CAccountMapLifetimeTest do refute Map.has_key?(listed, addr(99)) for i <- 1..5 do - {nonce, balance, storage, code} = Map.fetch!(listed, addr(i)) + {nonce, balance, root, code} = Map.fetch!(listed, addr(i)) assert {nonce, balance, code} == {i, i * 1_000, <>} - assert_storage_usable(storage) + assert is_binary(root) and byte_size(root) == 32 + assert_map_storage_usable(base, addr(i)) end end @@ -350,11 +354,9 @@ defmodule CAccountMapLifetimeTest do end describe "clone of a locked state stays writable (block sync path)" do - # Reproduces the startup block-sync hang: BlockProcess.cache_block/1 freezes the - # cached parent via Chain.State.lock/1 (mt->locked = true on every storage trie), - # and Block.create_empty/3 forks it via Chain.State.clone/1. The fork's storage - # tries must be writable (locked = false) or the EVM's storage writes return - # badarg and validation can never advance. + # BlockProcess.cache_block/1 freezes the cached parent via Chain.State.lock/1 + # (frozen account map). Block.create_empty/3 and RPC/EdgeV2/Shell fork via + # Chain.State.clone/1. The fork must accept storage_put_map writes. test "storage write in a fork of a State.lock'd state succeeds and isolates parent" do base = State.new() @@ -365,22 +367,42 @@ defmodule CAccountMapLifetimeTest do fork = State.clone(base) updated = - fork - |> State.account(addr(1)) - |> Account.storage_set_value(slot(42), val(42)) - |> then(&State.set_account(fork, addr(1), &1)) + State.storage_put_map(fork, %{ + addr(1) => %{slot(42) => val(42)} + }) - assert Account.storage_value(State.account(updated, addr(1)), slot(42)) == val(42) + assert State.storage_value(updated, addr(1), slot(42)) == val(42) assert is_binary(State.hash(updated)) # Parent stays frozen: the fork's COW write must not leak back. - assert Account.storage_value(State.account(base, addr(1)), slot(42)) == + assert State.storage_value(base, addr(1), slot(42)) == <<0::unsigned-size(256)>> - assert Account.storage_value(State.account(base, addr(1)), slot(1)) == val(1) + assert State.storage_value(base, addr(1), slot(1)) == val(1) assert is_binary(State.hash(base)) end + test "storage_put_map on locked parent raises and leaves parent unchanged" do + base = + State.new() + |> State.set_account(addr(1), sample_account(1)) + + before = State.storage_value(base, addr(1), slot(1)) + {_n, _b, root, _c} = CAccountMap.get(base.accounts, addr(1)) + assert byte_size(root) == 32 + + Chain.State.lock(base) + + assert_raise ArgumentError, fn -> + State.storage_put_map(base, %{addr(1) => %{slot(42) => val(42)}}) + end + + assert State.storage_value(base, addr(1), slot(1)) == before + + assert State.storage_value(base, addr(1), slot(42)) == + <<0::unsigned-size(256)>> + end + test "storage write in a fork of a locked state with many accounts" do base = Enum.reduce(1..6, State.new(), fn i, state -> @@ -393,22 +415,19 @@ defmodule CAccountMapLifetimeTest do fork = Enum.reduce(1..6, fork, fn i, state -> - acc = - state - |> State.account(addr(i)) - |> Account.storage_set_value(slot(100 + i), val(100 + i)) - - State.set_account(state, addr(i), acc) + State.storage_put_map(state, %{ + addr(i) => %{slot(100 + i) => val(100 + i)} + }) end) for i <- 1..6 do - assert Account.storage_value(State.account(fork, addr(i)), slot(100 + i)) == + assert State.storage_value(fork, addr(i), slot(100 + i)) == val(100 + i) end # Parent untouched. for i <- 1..6 do - assert Account.storage_value(State.account(base, addr(i)), slot(100 + i)) == + assert State.storage_value(base, addr(i), slot(100 + i)) == <<0::unsigned-size(256)>> end end @@ -425,14 +444,13 @@ defmodule CAccountMapLifetimeTest do fork = State.clone(base) fork = - fork - |> State.account(addr(1)) - |> Account.storage_set_value(slot(77), val(77)) - |> then(&State.set_account(fork, addr(1), &1)) + State.storage_put_map(fork, %{ + addr(1) => %{slot(77) => val(77)} + }) - assert Account.storage_value(State.account(fork, addr(1)), slot(77)) == val(77) + assert State.storage_value(fork, addr(1), slot(77)) == val(77) - assert Account.storage_value(State.account(base, addr(1)), slot(77)) == + assert State.storage_value(base, addr(1), slot(77)) == <<0::unsigned-size(256)>> assert is_binary(State.hash(fork)) @@ -459,19 +477,16 @@ defmodule CAccountMapLifetimeTest do fork = Enum.reduce(1..2, fork, fn i, state -> - acc = - state - |> State.account(addr(i)) - |> Account.storage_set_value(slot(100 + i), val(100 + i)) - - State.set_account(state, addr(i), acc) + State.storage_put_map(state, %{ + addr(i) => %{slot(100 + i) => val(100 + i)} + }) end) for i <- 1..2 do - assert Account.storage_value(State.account(fork, addr(i)), slot(100 + i)) == + assert State.storage_value(fork, addr(i), slot(100 + i)) == val(100 + i) - assert Account.storage_value(State.account(base, addr(i)), slot(100 + i)) == + assert State.storage_value(base, addr(i), slot(100 + i)) == <<0::unsigned-size(256)>> end end @@ -489,19 +504,16 @@ defmodule CAccountMapLifetimeTest do fork = Enum.reduce(1..10, fork, fn i, state -> - acc = - state - |> State.account(addr(i)) - |> Account.storage_set_value(slot(200 + i), val(200 + i)) - - State.set_account(state, addr(i), acc) + State.storage_put_map(state, %{ + addr(i) => %{slot(200 + i) => val(200 + i)} + }) end) for i <- 1..10 do - assert Account.storage_value(State.account(fork, addr(i)), slot(200 + i)) == + assert State.storage_value(fork, addr(i), slot(200 + i)) == val(200 + i) - assert Account.storage_value(State.account(base, addr(i)), slot(200 + i)) == + assert State.storage_value(base, addr(i), slot(200 + i)) == <<0::unsigned-size(256)>> end end @@ -524,18 +536,11 @@ defmodule CAccountMapLifetimeTest do fork = restored |> State.clone() - |> then(fn state -> - acc = - state - |> State.account(addr(1)) - |> Account.storage_set_value(slot(501), val(501)) - - State.set_account(state, addr(1), acc) - end) + |> State.storage_put_map(%{addr(1) => %{slot(501) => val(501)}}) - assert Account.storage_value(State.account(fork, addr(1)), slot(501)) == val(501) + assert State.storage_value(fork, addr(1), slot(501)) == val(501) - assert Account.storage_value(State.account(restored, addr(1)), slot(501)) == + assert State.storage_value(restored, addr(1), slot(501)) == <<0::unsigned-size(256)>> end end diff --git a/test/caccount_map_test.exs b/test/caccount_map_test.exs index cb2fc70..0524def 100644 --- a/test/caccount_map_test.exs +++ b/test/caccount_map_test.exs @@ -25,8 +25,10 @@ defmodule CAccountMapTest do map = put_sample(CAccountMap.new(), 3) assert CAccountMap.size(map) == 1 - assert {3, 3000, storage, <<3>>} = CAccountMap.get(map, addr(3)) - assert CMerkleTree.root_hash(storage) == Account.root_hash(sample_account(3)) + assert {3, 3000, root, <<3>>} = CAccountMap.get(map, addr(3)) + assert is_binary(root) and byte_size(root) == 32 + assert root == Account.root_hash(sample_account(3)) + assert CAccountMap.storage_root_hash(map, addr(3)) == root end test "delete removes account" do @@ -59,16 +61,27 @@ defmodule CAccountMapTest do assert {1, 1_000, _, <<1>>} = CAccountMap.get(base, addr(1)) end - test "clone copies accounts with equal, writable storage" do + test "clone copies accounts with equal storage root hashes" do base = put_sample(CAccountMap.new(), 5) fork = CAccountMap.clone(base) - # The fork holds distinct (writable, locked = false) storage resources with the - # same content as the parent. - {5, 5000, base_storage, <<5>>} = CAccountMap.get(base, addr(5)) - {5, 5000, fork_storage, <<5>>} = CAccountMap.get(fork, addr(5)) - assert CMerkleTree.root_hash(base_storage) == CMerkleTree.root_hash(fork_storage) - refute base_storage == fork_storage + # get/2 returns 32-byte root hashes (not live resources); content must match. + {5, 5000, base_root, <<5>>} = CAccountMap.get(base, addr(5)) + {5, 5000, fork_root, <<5>>} = CAccountMap.get(fork, addr(5)) + assert byte_size(base_root) == 32 + assert base_root == fork_root + + assert CAccountMap.storage_root_hash(base, addr(5)) == + CAccountMap.storage_root_hash(fork, addr(5)) + + # Fork remains writable via storage_put_map; parent root stays unchanged. + fork = + CAccountMap.storage_put_map(fork, %{ + addr(5) => %{<<99::unsigned-size(256)>> => <<100::unsigned-size(256)>>} + }) + + assert CAccountMap.storage_root_hash(fork, addr(5)) != + CAccountMap.storage_root_hash(base, addr(5)) fork = put_sample(fork, 9) @@ -79,7 +92,8 @@ defmodule CAccountMapTest do balance = Bitwise.bsl(1, 200) storage = CMerkleTree.new() map = CAccountMap.put(CAccountMap.new(), addr(2), 0, balance, storage, nil) - assert {0, ^balance, _, ""} = CAccountMap.get(map, addr(2)) + assert {0, ^balance, root, ""} = CAccountMap.get(map, addr(2)) + assert is_binary(root) and byte_size(root) == 32 end test "State.clone uses native account map" do diff --git a/test/chain_account_hash_nif_test.exs b/test/chain_account_hash_nif_test.exs index 49bc6f3..16282a2 100644 --- a/test/chain_account_hash_nif_test.exs +++ b/test/chain_account_hash_nif_test.exs @@ -43,7 +43,7 @@ defmodule ChainAccountHashNifTest do addr(2) => sample_account(2) |> Account.compact() } - {accounts, store, hash} = CAccountMap.uncompact_state(compact) + {accounts, hash} = CAccountMap.uncompact_state(compact) elixir_hashes = compact @@ -58,7 +58,7 @@ defmodule ChainAccountHashNifTest do |> Map.new() assert nif_hashes == elixir_hashes - assert hash == CMerkleTree.root_hash(store) + assert hash == CAccountMap.root_hash(accounts) end test "uncompact_state account hashes match Account.hash/1" do @@ -67,7 +67,7 @@ defmodule ChainAccountHashNifTest do {addr(i), sample_account(i) |> Account.compact()} end - {accounts, store, hash} = CAccountMap.uncompact_state(compact) + {accounts, hash} = CAccountMap.uncompact_state(compact) elixir_hashes = compact @@ -89,7 +89,7 @@ defmodule ChainAccountHashNifTest do |> CMerkleTree.root_hash() assert hash == elixir_root - assert CMerkleTree.root_hash(store) == elixir_root + assert CAccountMap.root_hash(accounts) == elixir_root end test "uncompact_state on CAccountMap resource matches Account.hash/1" do @@ -101,7 +101,7 @@ defmodule ChainAccountHashNifTest do end) end) - {accounts, store, hash} = CAccountMap.uncompact_state(original.accounts) + {accounts, hash} = CAccountMap.uncompact_state(original.accounts) elixir_hashes = original.accounts @@ -117,7 +117,7 @@ defmodule ChainAccountHashNifTest do |> Map.new() assert nif_hashes == elixir_hashes - assert hash == CMerkleTree.root_hash(store) + assert hash == CAccountMap.root_hash(accounts) end test "uncompact_state uses compact root_hash when present" do @@ -129,7 +129,7 @@ defmodule ChainAccountHashNifTest do assert Enum.all?(compact, fn {_id, acc} -> Map.has_key?(acc, :root_hash) end) assert Enum.all?(compact, fn {_id, acc} -> Map.has_key?(acc, :code_hash) end) - {accounts, store, hash} = CAccountMap.uncompact_state(compact) + {accounts, hash} = CAccountMap.uncompact_state(compact) elixir_hashes = compact @@ -144,7 +144,7 @@ defmodule ChainAccountHashNifTest do |> Map.new() assert nif_hashes == elixir_hashes - assert hash == CMerkleTree.root_hash(store) + assert hash == CAccountMap.root_hash(accounts) end test "uncompact_state falls back without compact root_hash field" do @@ -160,7 +160,7 @@ defmodule ChainAccountHashNifTest do legacy_compact = %{addr(1) => legacy_account} - {accounts, store, hash} = CAccountMap.uncompact_state(legacy_compact) + {accounts, hash} = CAccountMap.uncompact_state(legacy_compact) expected_hash = legacy_account @@ -173,7 +173,7 @@ defmodule ChainAccountHashNifTest do |> CMerkleTree.root_hash() assert hash == expected_root - assert CMerkleTree.root_hash(store) == expected_root + assert CAccountMap.root_hash(accounts) == expected_root assert CAccountMap.size(accounts) == 1 end @@ -191,7 +191,7 @@ defmodule ChainAccountHashNifTest do legacy_compact = %{addr(1) => legacy_account} - {accounts, store, hash} = CAccountMap.uncompact_state(legacy_compact) + {accounts, hash} = CAccountMap.uncompact_state(legacy_compact) expected_hash = legacy_account @@ -204,7 +204,7 @@ defmodule ChainAccountHashNifTest do |> CMerkleTree.root_hash() assert hash == expected_root - assert CMerkleTree.root_hash(store) == expected_root + assert CAccountMap.root_hash(accounts) == expected_root assert CAccountMap.size(accounts) == 1 end diff --git a/test/chain_state_merkle_test.exs b/test/chain_state_merkle_test.exs index d66c7f2..bdc824c 100644 --- a/test/chain_state_merkle_test.exs +++ b/test/chain_state_merkle_test.exs @@ -125,7 +125,7 @@ defmodule ChainStateMerkleTest do assert_roundtrip(prev, next) end - test "single account: delete storage slot (CMerkleTree.delete)" do + test "single account: delete storage slot (storage_put_map zero)" do prev = State.new() |> put_account( @@ -136,14 +136,10 @@ defmodule ChainStateMerkleTest do ]) ) - acc_prev = State.account(prev, addr(1)) - - tree2 = - acc_prev - |> Account.tree() - |> CMerkleTree.delete(word32(2)) - - next = State.set_account(prev, addr(1), Account.put_tree(acc_prev, tree2)) + next = + State.storage_put_map(prev, %{ + addr(1) => %{word32(2) => <<0::unsigned-size(256)>>} + }) assert_roundtrip(prev, next) end @@ -362,9 +358,9 @@ defmodule ChainStateMerkleTest do ) assert State.hash(fork) != parent_hash - assert Account.storage_value(State.account(fork, addr(1)), word32(99)) == val32(99) + assert State.storage_value(fork, addr(1), word32(99)) == val32(99) - assert Account.storage_value(State.account(base, addr(1)), word32(99)) == + assert State.storage_value(base, addr(1), word32(99)) == <<0::unsigned-size(256)>> assert is_binary(State.hash(base)) @@ -381,21 +377,18 @@ defmodule ChainStateMerkleTest do fork = Enum.reduce(1..2, fork, fn aid, st -> - acc = - st - |> State.account(addr(aid)) - |> Account.storage_set_value(slot_u256(500 + aid), val_u256(aid * 100)) - - put_account(st, aid, acc) + State.storage_put_map(st, %{ + addr(aid) => %{slot_u256(500 + aid) => val_u256(aid * 100)} + }) end) assert is_binary(State.hash(fork)) for aid <- 1..2 do - assert Account.storage_value(State.account(fork, addr(aid)), slot_u256(500 + aid)) == + assert State.storage_value(fork, addr(aid), slot_u256(500 + aid)) == val_u256(aid * 100) - assert Account.storage_value(State.account(parent, addr(aid)), slot_u256(500 + aid)) == + assert State.storage_value(parent, addr(aid), slot_u256(500 + aid)) == <<0::unsigned-size(256)>> end end diff --git a/test/chain_state_uncompact_test.exs b/test/chain_state_uncompact_test.exs index b5b7eba..992ca10 100644 --- a/test/chain_state_uncompact_test.exs +++ b/test/chain_state_uncompact_test.exs @@ -58,7 +58,7 @@ defmodule ChainStateUncompactTest do for i <- 1..32 do acc = State.account(restored, addr(i)) assert acc.nonce == i - assert Account.storage_value(acc, slot(i)) == val(i) + assert State.storage_value(restored, addr(i), slot(i)) == val(i) end end @@ -129,13 +129,13 @@ defmodule ChainStateUncompactTest do assert acc1.nonce == 3 assert acc1.balance == 99 assert acc1.code == :binary.copy(<<0xAB>>, 512) - assert Account.storage_value(acc1, slot(1)) == val(1) - assert Account.storage_value(acc1, slot(2)) == val(2) - assert Account.storage_value(acc1, slot(3)) == val(3) + assert State.storage_value(restored, addr(1), slot(1)) == val(1) + assert State.storage_value(restored, addr(1), slot(2)) == val(2) + assert State.storage_value(restored, addr(1), slot(3)) == val(3) acc2 = State.account(restored, addr(2)) assert acc2.code == nil - assert CMerkleTree.size(Account.tree(acc2)) == 0 + assert State.storage_size(restored, addr(2)) == 0 end test "compact account edge cases: nil storage, nil code, large balance, list storage" do @@ -164,13 +164,13 @@ defmodule ChainStateUncompactTest do acc1 = State.account(restored, addr(1)) assert acc1.balance == 100_000_000_000_000_000_000_000_000 - assert Account.storage_value(acc1, slot(1)) == val(1) + assert State.storage_value(restored, addr(1), slot(1)) == val(1) acc2 = State.account(restored, addr(2)) assert acc2.nonce == 7 assert acc2.balance == 42 assert acc2.code == nil - assert CMerkleTree.size(Account.tree(acc2)) == 0 + assert State.storage_size(restored, addr(2)) == 0 end test "lazy storage materializes on CAccountMap.get without State.account" do @@ -189,10 +189,12 @@ defmodule ChainStateUncompactTest do compact = %{addr(1) => Account.compact(multi_slot)} {accounts, _hash} = CAccountMap.uncompact_state(compact) - {5, 1_000, storage, <<5>>} = CAccountMap.get(accounts, addr(1)) + {5, 1_000, root, <<5>>} = CAccountMap.get(accounts, addr(1)) + assert is_binary(root) and byte_size(root) == 32 + assert root == Account.root_hash(multi_slot) - assert CMerkleTree.get(storage, slot(1)) == val(1) - assert CMerkleTree.get(storage, slot(2)) == val(2) + assert CAccountMap.storage_get(accounts, addr(1), slot(1)) == val(1) + assert CAccountMap.storage_get(accounts, addr(1), slot(2)) == val(2) end test "clone preserves lazy storage until materialized" do @@ -205,8 +207,7 @@ defmodule ChainStateUncompactTest do assert State.hash(cloned) == State.hash(original) for i <- 1..4 do - acc = State.account(cloned, addr(i)) - assert Account.storage_value(acc, slot(i)) == val(i) + assert State.storage_value(cloned, addr(i), slot(i)) == val(i) end end end @@ -241,9 +242,10 @@ defmodule ChainStateUncompactTest do accounts = CAccountMap.put(accounts, addr(1), 9, 9_000, new_storage, <<9>>) - assert {9, 9_000, storage, <<9>>} = CAccountMap.get(accounts, addr(1)) - assert CMerkleTree.get(storage, slot(99)) == val(99) - refute CMerkleTree.get(storage, slot(1)) == val(1) + assert {9, 9_000, root, <<9>>} = CAccountMap.get(accounts, addr(1)) + assert is_binary(root) and byte_size(root) == 32 + assert CAccountMap.storage_get(accounts, addr(1), slot(99)) == val(99) + refute CAccountMap.storage_get(accounts, addr(1), slot(1)) == val(1) end test "to_list materializes lazy storage for every account" do @@ -257,10 +259,12 @@ defmodule ChainStateUncompactTest do assert map_size(listed) == 4 for i <- 1..4 do - {_nonce, _balance, storage, code} = Map.fetch!(listed, addr(i)) + {_nonce, _balance, root, code} = Map.fetch!(listed, addr(i)) assert code == <> - assert CMerkleTree.get(storage, slot(i)) == val(i) - assert CMerkleTree.root_hash(storage) == Account.root_hash(sample_account(i)) + assert is_binary(root) and byte_size(root) == 32 + assert CAccountMap.storage_get(accounts, addr(i), slot(i)) == val(i) + assert root == Account.root_hash(sample_account(i)) + assert CAccountMap.storage_root_hash(accounts, addr(i)) == root end end @@ -282,7 +286,7 @@ defmodule ChainStateUncompactTest do force_gc() for i <- 1..6 do - assert Account.storage_value(State.account(restored, addr(i)), slot(i)) == val(i) + assert State.storage_value(restored, addr(i), slot(i)) == val(i) end end @@ -309,19 +313,16 @@ defmodule ChainStateUncompactTest do fork = Enum.reduce(1..4, fork, fn i, state -> - acc = - state - |> State.account(addr(i)) - |> Account.storage_set_value(slot(100 + i), val(100 + i)) - - State.set_account(state, addr(i), acc) + State.storage_put_map(state, %{ + addr(i) => %{slot(100 + i) => val(100 + i)} + }) end) for i <- 1..4 do - assert Account.storage_value(State.account(fork, addr(i)), slot(100 + i)) == + assert State.storage_value(fork, addr(i), slot(100 + i)) == val(100 + i) - assert Account.storage_value(State.account(restored, addr(i)), slot(100 + i)) == + assert State.storage_value(restored, addr(i), slot(100 + i)) == <<0::unsigned-size(256)>> end end diff --git a/test/chain_test.exs b/test/chain_test.exs index 763c3d9..022bb9e 100644 --- a/test/chain_test.exs +++ b/test/chain_test.exs @@ -262,7 +262,8 @@ defmodule ChainTest do # for {key, value} <- to_list(result.storage_root) do # assert {key, value} == {key, Account.storageInteger(account, key)} # end - assert to_list(Account.tree(result)) == to_list(Account.tree(account)) + # result is map-backed (no live trie); compare via State.storage_to_list. + assert storage_list(state, addr) == to_list(Account.tree(account)) end end @@ -271,6 +272,12 @@ defmodule ChainTest do assert post_keys == reference_keys end + defp storage_list(state, addr) do + State.storage_to_list(state, addr) + |> Enum.map(fn {key, value} -> {compress(key), compress(value)} end) + |> Enum.sort() + end + defp to_list(tree) do CMerkleTree.to_list(tree) |> Enum.map(fn {key, value} -> {compress(key), compress(value)} end) diff --git a/test/cmerkle_account_map_diff_test.exs b/test/cmerkle_account_map_diff_test.exs index a987004..1fc22e0 100644 --- a/test/cmerkle_account_map_diff_test.exs +++ b/test/cmerkle_account_map_diff_test.exs @@ -25,64 +25,54 @@ defmodule CMerkleAccountMapDiffTest do end) end - defp legacy_diff(map_a, map_b) do - CAccountMap.list_difference(map_a, map_b) - |> Enum.map(fn {addr, {a, b}} -> - {addr, {a, b}} - end) - |> Map.new() + defp full_addrs(map_a, map_b) do + CAccountMap.difference_full(map_a, map_b) + |> Enum.map(fn {addr, _, _, _} -> addr end) + |> Enum.sort() end - defp assert_diff_equivalent(map_a, map_b) do - native = CAccountMap.list_difference(map_a, map_b) - legacy = legacy_diff(map_a, map_b) - - assert Map.keys(native) |> Enum.sort() == Map.keys(legacy) |> Enum.sort() + defp assert_difference_full_shape(map_a, map_b) do + full = CAccountMap.difference_full(map_a, map_b) + assert is_list(full) - for key <- Map.keys(native) do - {na, nb} = native[key] - {la, lb} = legacy[key] - assert account_equal?(na, la) - assert account_equal?(nb, lb) + for {addr, _side_a, _side_b, storage_diff} <- full do + assert byte_size(addr) == 20 + assert is_list(storage_diff) or is_map(storage_diff) + in_a = CAccountMap.get(map_a, addr) != :undefined + in_b = CAccountMap.get(map_b, addr) != :undefined + assert in_a or in_b end - end - defp account_equal?(nil, nil), do: true - - defp account_equal?(%Account{} = a, %Account{} = b) do - a.nonce == b.nonce && a.balance == b.balance && a.code == b.code && - Account.root_hash(a) == Account.root_hash(b) + full end - defp account_equal?(a, b), do: a == b - - describe "native vs legacy list_difference equivalence" do + describe "difference_full" do test "empty maps" do a = CAccountMap.new() b = CAccountMap.new() - assert_diff_equivalent(a, b) - assert CAccountMap.list_difference(a, b) == %{} + assert CAccountMap.difference_full(a, b) == [] end test "identical maps" do a = build_map(40) b = CAccountMap.clone(a) - assert_diff_equivalent(a, b) - assert CAccountMap.list_difference(a, b) == %{} + assert CAccountMap.difference_full(a, b) == [] end test "same shared map resource" do a = build_map(10) - assert CAccountMap.list_difference(a, a) == %{} + assert CAccountMap.difference_full(a, a) == [] end test "add-only and delete-only accounts" do a = build_map(30) b = build_map(35) - assert_diff_equivalent(a, b) + assert full_addrs(a, b) == Enum.map(31..35, &addr/1) |> Enum.sort() + assert_difference_full_shape(a, b) c = build_map(20) - assert_diff_equivalent(a, c) + assert full_addrs(a, c) == Enum.map(21..30, &addr/1) |> Enum.sort() + assert_difference_full_shape(a, c) end test "scalar field mutations" do @@ -100,7 +90,8 @@ defmodule CMerkleAccountMapDiffTest do :code -> CAccountMap.put(fork, addr(i), i, i * 1_000, storage, <<99, 88>>) end - assert_diff_equivalent(base, fork) + assert full_addrs(base, fork) == [addr(i)] + assert_difference_full_shape(base, fork) end end @@ -108,17 +99,16 @@ defmodule CMerkleAccountMapDiffTest do a = build_map(25) b = CAccountMap.clone(a) id = addr(5) - {nonce, balance, storage, code} = CAccountMap.get(b, id) - storage = - CMerkleTree.insert( - CMerkleTree.clone(storage), - slot(99_999), - <<99_999::unsigned-size(256)>> - ) - - b = CAccountMap.put(b, id, nonce, balance, storage, code) - assert_diff_equivalent(a, b) + b = + CAccountMap.storage_put_map(b, %{ + id => %{slot(99_999) => <<99_999::unsigned-size(256)>>} + }) + + assert full_addrs(a, b) == [id] + assert_difference_full_shape(a, b) + assert CAccountMap.storage_get(b, id, slot(99_999)) == <<99_999::unsigned-size(256)>> + assert CAccountMap.storage_get(a, id, slot(99_999)) == nil end test "shared storage trie across accounts" do @@ -133,11 +123,11 @@ defmodule CMerkleAccountMapDiffTest do end) b = CAccountMap.clone(a) - assert_diff_equivalent(a, b) + assert CAccountMap.difference_full(a, b) == [] b = CAccountMap.put(b, addr(3), 99, 999, storage, <<1>>) - refute CAccountMap.list_difference(a, b) == %{} - assert_diff_equivalent(a, b) + assert full_addrs(a, b) == [addr(3)] + assert_difference_full_shape(a, b) end test "compact maps via State.compact" do @@ -157,22 +147,17 @@ defmodule CMerkleAccountMapDiffTest do fork = live |> State.clone() - |> then(fn st -> - acc = State.account(st, addr(10)) - - tree = - Account.tree(acc) - |> CMerkleTree.insert(slot(50_000), <<50_000::unsigned-size(256)>>) - - State.set_account(st, addr(10), Account.put_tree(acc, tree)) - end) + |> State.storage_put_map(%{ + addr(10) => %{slot(50_000) => <<50_000::unsigned-size(256)>>} + }) - assert_diff_equivalent(compact_nif, fork.accounts) - assert_diff_equivalent(compact_nif, live.accounts) + assert full_addrs(compact_nif, fork.accounts) == [addr(10)] + assert_difference_full_shape(compact_nif, fork.accounts) + assert CAccountMap.difference_full(compact_nif, live.accounts) == [] end @tag :slow - test "randomized equivalence property" do + test "randomized difference_full consistency" do for seed <- 1..50 do :rand.seed(:exsss, {seed, seed, seed}) n = :rand.uniform(120) + 5 @@ -205,35 +190,22 @@ defmodule CMerkleAccountMapDiffTest do :undefined -> acc - {nonce, balance, storage, code} -> - storage = - CMerkleTree.insert( - CMerkleTree.clone(storage), - slot(i + 20_000), - <> - ) - - CAccountMap.put(acc, id, nonce, balance, storage, code) + {_nonce, _balance, _root, _code} -> + CAccountMap.storage_put_map(acc, %{ + id => %{slot(i + 20_000) => <>} + }) end end end) - assert_diff_equivalent(a, b) - end - end - end - - describe "difference_full vs list_difference" do - test "difference_full covers the same account ids as list_difference" do - a = build_map(20) - b = build_map(25) - - legacy = CAccountMap.list_difference(a, b) - state_a = %Chain.State{accounts: a} - state_b = %Chain.State{accounts: b} - full = Map.new(State.difference(state_a, state_b)) + full = assert_difference_full_shape(a, b) - assert Map.keys(full) |> Enum.sort() == Map.keys(legacy) |> Enum.sort() + if CAccountMap.root_hash(a) == CAccountMap.root_hash(b) do + assert full == [] + else + assert full != [] + end + end end end @@ -251,15 +223,9 @@ defmodule CMerkleAccountMapDiffTest do next = prev |> State.clone() - |> then(fn st -> - acc = State.account(st, addr(7)) - - tree = - Account.tree(acc) - |> CMerkleTree.insert(slot(77_777), <<77_777::unsigned-size(256)>>) - - State.set_account(st, addr(7), Account.put_tree(acc, tree)) - end) + |> State.storage_put_map(%{ + addr(7) => %{slot(77_777) => <<77_777::unsigned-size(256)>>} + }) delta = State.difference(prev, next) diff --git a/test/cmerkle_clone_lazy_test.exs b/test/cmerkle_clone_lazy_test.exs deleted file mode 100644 index 0a5f7a3..0000000 --- a/test/cmerkle_clone_lazy_test.exs +++ /dev/null @@ -1,93 +0,0 @@ -# Diode Server -# Copyright 2021-2024 Diode -# Licensed under the Diode License, Version 1.1 -defmodule CMerkleCloneLazyTest do - use ExUnit.Case, async: false - - alias Chain.{Account, State} - - @moduletag timeout: 120_000 - - defp addr(i), do: <> - - defp slot(i), do: <> - - defp val(i), do: <> - - defp force_gc(rounds \\ 3) do - for _ <- 1..rounds, do: :erlang.garbage_collect() - end - - defp populate(n) do - Enum.reduce(1..n, CAccountMap.new(), fn i, map -> - storage = CMerkleTree.insert(CMerkleTree.new(), slot(i), val(i)) - CAccountMap.put(map, addr(i), i, i * 1_000, storage, <>) - end) - end - - test "lazy clone shares storage roots before write" do - base = populate(50) - fork = CAccountMap.clone_lazy(base) - - for i <- 1..50 do - {_, _, storage_a, _} = CAccountMap.get(base, addr(i)) - {_, _, storage_b, _} = CAccountMap.get(fork, addr(i)) - assert CMerkleTree.root_hash(storage_a) == CMerkleTree.root_hash(storage_b) - end - end - - test "first write on lazy fork COWs storage without changing parent" do - base = populate(5) - fork = CAccountMap.clone_lazy(base) - - {nonce, balance, storage, code} = CAccountMap.get(fork, addr(1)) - storage = CMerkleTree.insert(storage, slot(999), val(999)) - CAccountMap.put(fork, addr(1), nonce, balance, storage, code) - - {_, _, parent_storage, _} = CAccountMap.get(base, addr(1)) - assert CMerkleTree.get(parent_storage, slot(999)) == nil - {_, _, fork_storage, _} = CAccountMap.get(fork, addr(1)) - assert CMerkleTree.get(fork_storage, slot(999)) == val(999) - end - - test "dropping lazy clone does not corrupt parent" do - base = populate(10) - - fork = CAccountMap.clone_lazy(base) - acc = CAccountMap.get_account(fork, addr(1)) - acc = Account.storage_set_value(acc, slot(42), val(42)) - _fork = CAccountMap.put_account(fork, addr(1), acc) - force_gc() - - {_, _, storage, _} = CAccountMap.get(base, addr(1)) - assert CMerkleTree.get(storage, slot(42)) == nil - assert is_binary(CMerkleTree.root_hash(storage)) - end - - test "clone_lazy on locked state returns badarg" do - st = - State.new() - |> State.set_account(addr(1), Account.storage_set_value(Account.new(), slot(1), val(1))) - - Chain.State.lock(st) - - assert_raise ArgumentError, fn -> - State.clone_lazy(st) - end - end - - test "State.clone_lazy completes for large maps" do - base = populate(500) - state = %Chain.State{accounts: base} - - {lazy_us, fork} = :timer.tc(fn -> State.clone_lazy(state) end) - {eager_us, _} = :timer.tc(fn -> State.clone(state) end) - - # Today both paths fork storage wrappers; keep both under a generous bound and - # ensure lazy stays in the same ballpark as eager (not accidentally O(n²)). - assert lazy_us < 500_000 - assert eager_us < 500_000 - assert lazy_us < eager_us * 3 - assert is_binary(State.hash(fork)) - end -end diff --git a/test/cmerkle_lock_clone_regression_test.exs b/test/cmerkle_lock_clone_regression_test.exs new file mode 100644 index 0000000..522ff05 --- /dev/null +++ b/test/cmerkle_lock_clone_regression_test.exs @@ -0,0 +1,283 @@ +# Diode Server +# Copyright 2021-2024 Diode +# Licensed under the Diode License, Version 1.1 +# +# Regressions for lock/clone/storage_put_map after clone_lazy removal: +# 1) Cached/locked peak must use State.clone/1 for speculative forks. +# 2) State.lock freezes the account map (storage_put_map / put / delete raise). +# 3) Shared-storage lock must not deadlock. +# 4) difference_full is the sole NIF diff path. +# 5) State.storage_put_map works unlocked and raises on locked maps. +# 6) CAccountMap.get/2 returns a 32-byte storage root hash (never a live trie). +defmodule CMerkleLockCloneRegressionTest do + use ExUnit.Case, async: false + + alias Chain.{Account, State} + + @moduletag timeout: 120_000 + + defp addr(i), do: <> + defp slot(i), do: <> + defp val(i), do: <> + + defp sample_account(i) do + Account.new() + |> Account.storage_set_value(slot(i), val(i)) + |> Map.put(:nonce, i) + |> Map.put(:balance, i * 1_000) + |> Map.put(:code, <>) + end + + defp locked_peak_like_state(n \\ 3) do + state = + Enum.reduce(1..n, State.new(), fn i, st -> + State.set_account(st, addr(i), sample_account(i)) + end) + + # BlockProcess.cache_block/1 locks the cached state before RPC/Edge/Shell read it. + State.lock(state) + state + end + + describe "cached peak speculative path (RPC/EdgeV2/Shell regression)" do + test "lock then clone/1 succeeds and isolates parent" do + peak = locked_peak_like_state() + + fork = State.clone(peak) + + updated = + State.storage_put_map(fork, %{ + addr(1) => %{slot(999) => val(999)} + }) + + assert State.storage_value(updated, addr(1), slot(999)) == val(999) + + assert State.storage_value(peak, addr(1), slot(999)) == + <<0::unsigned-size(256)>> + end + + test "set_account and put on locked peak raise without mutating parent" do + peak = locked_peak_like_state(1) + before_hash = State.hash(peak) + before_slot = State.storage_value(peak, addr(1), slot(1)) + + acc = + State.account(peak, addr(1)) + |> Map.put(:nonce, 99) + + assert_raise ArgumentError, fn -> + State.set_account(peak, addr(1), acc) + end + + assert_raise ArgumentError, fn -> + CAccountMap.put( + peak.accounts, + addr(1), + 99, + 1, + CMerkleTree.new(), + <<>> + ) + end + + assert_raise ArgumentError, fn -> + CAccountMap.delete(peak.accounts, addr(1)) + end + + assert State.hash(peak) == before_hash + assert State.storage_value(peak, addr(1), slot(1)) == before_slot + end + + test "storage_put_map on unlocked works; on locked raises" do + unlocked = + State.new() + |> State.set_account(addr(1), sample_account(1)) + + updated = + State.storage_put_map(unlocked, %{ + addr(1) => %{slot(42) => val(42)} + }) + + assert CAccountMap.storage_get(updated.accounts, addr(1), slot(42)) == val(42) + assert CAccountMap.storage_get(updated.accounts, addr(1), slot(1)) == val(1) + + peak = State.lock(unlocked) + + assert_raise ArgumentError, fn -> + State.storage_put_map(peak, %{addr(1) => %{slot(43) => val(43)}}) + end + + assert CAccountMap.storage_get(peak.accounts, addr(1), slot(43)) == nil + assert CAccountMap.storage_get(peak.accounts, addr(1), slot(1)) == val(1) + end + end + + describe "lock storage immutability" do + test "get returns 32-byte root hash; storage_put_map on locked raises" do + peak = locked_peak_like_state(1) + {_n, _b, root, _c} = CAccountMap.get(peak.accounts, addr(1)) + assert is_binary(root) and byte_size(root) == 32 + assert root == CAccountMap.storage_root_hash(peak.accounts, addr(1)) + before = CAccountMap.storage_get(peak.accounts, addr(1), slot(1)) + + assert_raise ArgumentError, fn -> + CAccountMap.storage_put_map(peak.accounts, %{addr(1) => %{slot(42) => val(42)}}) + end + + assert CAccountMap.storage_get(peak.accounts, addr(1), slot(1)) == before + assert CAccountMap.storage_get(peak.accounts, addr(1), slot(42)) == nil + end + + test "put and storage_put_map on frozen map raise; state_trie remains readable" do + peak = locked_peak_like_state(1) + trie = State.tree(peak) + before = CMerkleTree.root_hash(trie) + + assert_raise ArgumentError, fn -> + State.storage_put_map(peak, %{addr(1) => %{slot(9) => val(9)}}) + end + + assert_raise ArgumentError, fn -> + CAccountMap.put(peak.accounts, addr(9), 0, 0, CMerkleTree.new(), <<>>) + end + + # Lock is frozen-only on the account map; State.tree still returns a live + # resource for reads. Root hash of the state trie must stay unchanged. + assert CMerkleTree.root_hash(trie) == before + assert CMerkleTree.root_hash(State.tree(peak)) == before + end + + test "apply_difference on locked map raises; clone then apply succeeds" do + prev = + State.new() + |> State.set_account(addr(1), sample_account(1)) + + next = + prev + |> State.clone() + |> State.storage_put_map(%{addr(1) => %{slot(7) => val(7)}}) + + delta = State.difference(prev, next) + + State.lock(prev) + + assert_raise ArgumentError, fn -> + State.apply_difference(prev, delta) + end + + restored = + prev + |> State.clone() + |> State.apply_difference(delta) + + assert State.storage_value(restored, addr(1), slot(7)) == val(7) + + assert State.storage_value(prev, addr(1), slot(7)) == + <<0::unsigned-size(256)>> + end + end + + describe "shared storage lock (apply_canonical_lock regression)" do + test "lock with many accounts sharing one storage completes and stays forkable" do + shared = + Enum.reduce(1..8, CMerkleTree.new(), fn i, tree -> + CMerkleTree.insert(tree, slot(i), val(i)) + end) + + accounts = + Enum.reduce(1..40, CAccountMap.new(), fn i, map -> + CAccountMap.put(map, addr(i), i, i * 100, shared, <>) + end) + + base = %State{State.new() | accounts: accounts} + + {lock_us, _} = :timer.tc(fn -> State.lock(base) end) + assert lock_us < 2_000_000, "shared-storage lock hung or was too slow: #{lock_us}µs" + + {clone_us, fork} = :timer.tc(fn -> State.clone(base) end) + assert clone_us < 2_000_000 + + updated = + State.storage_put_map(fork, %{ + addr(1) => %{slot(999) => val(999)} + }) + + assert State.storage_value(updated, addr(1), slot(999)) == val(999) + + assert State.storage_value(base, addr(1), slot(999)) == + <<0::unsigned-size(256)>> + + # Sibling account still sees original shared slots on the frozen parent. + assert State.storage_value(base, addr(2), slot(1)) == val(1) + end + + test "concurrent locks on distinct maps that share storage roots do not hang" do + # Same root hash via independent trees (one resource must not be kept by many + # AccountMaps). Concurrent lock+clone must complete without hanging. + maps = + for i <- 1..8 do + storage = + CMerkleTree.insert_items(CMerkleTree.new(), [ + {slot(1), val(1)}, + {slot(2), val(2)} + ]) + + CAccountMap.new() + |> CAccountMap.put(addr(i), i, i * 10, storage, <>) + |> CAccountMap.put(addr(100 + i), i, i * 10, storage, <>) + end + + forks = + maps + |> Task.async_stream( + fn map -> + CAccountMap.lock(map) + CAccountMap.clone(map) + end, + max_concurrency: 8, + timeout: 5_000, + ordered: true + ) + |> Enum.map(fn {:ok, fork} -> fork end) + + assert length(forks) == 8 + + Enum.each(forks, fn fork -> + assert CAccountMap.size(fork) == 2 + end) + end + end + + describe "difference_full sole NIF path" do + test "State.difference on locked peak vs mutated clone round-trips via clone+apply" do + peak = locked_peak_like_state(4) + + next = + peak + |> State.clone() + |> State.storage_put_map(%{addr(2) => %{slot(50) => val(50)}}) + |> then(fn st -> + State.set_account(st, addr(9), sample_account(9)) + end) + + delta = State.difference(peak, next) + assert is_list(delta) + assert delta != [] + + full_ids = + CAccountMap.difference_full(peak.accounts, next.accounts) + |> Enum.map(fn {id, _, _, _} -> id end) + |> Enum.sort() + + assert full_ids == Enum.sort([addr(2), addr(9)]) + + restored = + peak + |> State.clone() + |> State.apply_difference(delta) + |> State.normalize() + + assert State.hash(restored) == State.hash(State.normalize(next)) + end + end +end diff --git a/test/cmerkle_lock_concurrency_test.exs b/test/cmerkle_lock_concurrency_test.exs index b5a9331..5988571 100644 --- a/test/cmerkle_lock_concurrency_test.exs +++ b/test/cmerkle_lock_concurrency_test.exs @@ -157,30 +157,29 @@ defmodule CMerkleLockConcurrencyTest do end) end - test "concurrent account_map_lock and storage difference" do + test "concurrent account_map_lock and storage root / list reads" do map = lock_test_shared_storage_map(48, 4) - {_, _, storage_a, _} = CAccountMap.get(map, <<1::unsigned-size(160)>>) - {_, _, storage_b, _} = CAccountMap.get(map, <<2::unsigned-size(160)>>) - lockers = div(@tasks, 2) |> max(1) - differ = @tasks - lockers + readers = @tasks - lockers run_parallel(lockers, fn _ -> _ = map |> CAccountMap.clone() |> CAccountMap.lock() :ok end) - run_parallel(differ, fn _ -> - _ = CMerkleTree.difference(storage_a, storage_b) - _ = CMerkleTree.difference(storage_b, storage_a) + run_parallel(readers, fn _ -> + _ = CAccountMap.storage_root_hash(map, <<1::unsigned-size(160)>>) + _ = CAccountMap.storage_root_hash(map, <<2::unsigned-size(160)>>) + _ = CAccountMap.storage_to_list(map, <<1::unsigned-size(160)>>) + _ = CAccountMap.storage_to_list(map, <<2::unsigned-size(160)>>) :ok end) end end - describe "D-C7 native list_difference vs account_map_get" do - test "concurrent list_difference and get materialize" do + describe "D-C7 native difference_full vs account_map_get" do + test "concurrent difference_full and get materialize" do map = lock_test_shared_storage_map(60, 5) fork = CAccountMap.clone(map) @@ -193,14 +192,14 @@ defmodule CMerkleLockConcurrencyTest do end) run_parallel(differ, fn _ -> - _ = CAccountMap.list_difference(map, fork) + _ = CAccountMap.difference_full(map, fork) :ok end) end end - describe "D-C8 dual-map list_difference ordering" do - test "list_difference(A,B) concurrent with list_difference(B,A)" do + describe "D-C8 dual-map difference_full ordering" do + test "difference_full(A,B) concurrent with difference_full(B,A)" do a = lock_test_shared_storage_map(40, 4) b = CAccountMap.clone(a) b = CAccountMap.put(b, <<3::unsigned-size(160)>>, 99, 99_000, CMerkleTree.new(), <<99>>) @@ -209,12 +208,12 @@ defmodule CMerkleLockConcurrencyTest do ba = @tasks - ab run_parallel(ab, fn _ -> - _ = CAccountMap.list_difference(a, b) + _ = CAccountMap.difference_full(a, b) :ok end) run_parallel(ba, fn _ -> - _ = CAccountMap.list_difference(b, a) + _ = CAccountMap.difference_full(b, a) :ok end) end diff --git a/test/cmerkle_nif_deadlock_test.exs b/test/cmerkle_nif_deadlock_test.exs index d286f1c..20adaff 100644 --- a/test/cmerkle_nif_deadlock_test.exs +++ b/test/cmerkle_nif_deadlock_test.exs @@ -137,13 +137,10 @@ defmodule CMerkleNifDeadlockTest do |> then(fn st -> Enum.reduce(1..40, st, fn i, acc -> id = addr(rem(i, n) + 1) - acc0 = State.account(acc, id) - tree = Account.tree(acc0) - tree = - CMerkleTree.insert(tree, slot(i + 200_000), <>) - - State.set_account(acc, id, Account.put_tree(acc0, tree)) + State.storage_put_map(acc, %{ + id => %{slot(i + 200_000) => <>} + }) end) end) @@ -214,17 +211,15 @@ defmodule CMerkleNifDeadlockTest do end) run_parallel(differ, fn i -> - a = CAccountMap.get(map, addr(rem(i, n) + 1)) - b = CAccountMap.get(map, addr(rem(i + 1, n) + 1)) - - case {a, b} do - {{_, _, sa, _}, {_, _, sb, _}} -> - _ = CMerkleTree.difference(sa, sb) - - _ -> - :ok - end - + a = addr(rem(i, n) + 1) + b = addr(rem(i + 1, n) + 1) + # get/2 returns root hashes; exercise concurrent storage reads instead of + # CMerkleTree.difference on live resources from get. + _ = CAccountMap.get(map, a) + _ = CAccountMap.get(map, b) + _ = CAccountMap.storage_root_hash(map, a) + _ = CAccountMap.storage_root_hash(map, b) + _ = CAccountMap.storage_to_list(map, a) :ok end) end @@ -322,7 +317,7 @@ defmodule CMerkleNifDeadlockTest do end describe "D-D7 prepare_state composite (us1-shaped)" do - test "concurrent list_difference, to_list, and State.lock" do + test "concurrent difference_full, to_list, and State.lock" do n = 100 prev = build_live_state(n) |> State.normalize() @@ -332,20 +327,17 @@ defmodule CMerkleNifDeadlockTest do |> then(fn st -> Enum.reduce(1..20, st, fn i, acc -> id = addr(rem(i, n) + 1) - acc0 = State.account(acc, id) - tree = Account.tree(acc0) - - tree = - CMerkleTree.insert(tree, slot(i + 300_000), <>) - State.set_account(acc, id, Account.put_tree(acc0, tree)) + State.storage_put_map(acc, %{ + id => %{slot(i + 300_000) => <>} + }) end) end) differ = div(@tasks, 4) |> max(1) lockers = div(@tasks, 4) |> max(1) - legacy = div(@tasks, 4) |> max(1) - native = @tasks - differ - lockers - legacy + listers = div(@tasks, 4) |> max(1) + native = @tasks - differ - lockers - listers run_parallel(differ, fn _ -> _ = State.difference(prev, block) @@ -357,13 +349,13 @@ defmodule CMerkleNifDeadlockTest do :ok end) - run_parallel(legacy, fn _ -> + run_parallel(listers, fn _ -> _ = CAccountMap.to_list(block.accounts) :ok end) run_parallel(native, fn _ -> - _ = CAccountMap.list_difference(prev.accounts, block.accounts) + _ = CAccountMap.difference_full(prev.accounts, block.accounts) :ok end) end diff --git a/test/cmerkle_nif_leak_test.exs b/test/cmerkle_nif_leak_test.exs index d538f1e..3c83482 100644 --- a/test/cmerkle_nif_leak_test.exs +++ b/test/cmerkle_nif_leak_test.exs @@ -124,29 +124,27 @@ defmodule CMerkleNifLeakTest do assert rss_kb() - baseline < 100_000 end - test "account_map list_difference loop does not grow shared_states" do + test "account_map difference_full loop does not grow shared_states" do n = 80 base = build_diff_map(n) fork = CAccountMap.clone(base) |> then(fn map -> - {nonce, balance, storage, code} = CAccountMap.get(map, addr(3)) - - storage = - CMerkleTree.insert( - CMerkleTree.clone(storage), - slot(99_999), - <<99_999::unsigned-size(256)>> - ) - - CAccountMap.put(map, addr(3), nonce + 1, balance, storage, code) + map + |> CAccountMap.storage_put_map(%{ + addr(3) => %{slot(99_999) => <<99_999::unsigned-size(256)>>} + }) + |> then(fn m -> + {nonce, balance, _root, code} = CAccountMap.get(m, addr(3)) + CAccountMap.put_meta(m, addr(3), nonce + 1, balance, code) + end) end) {_locked0, _orphans0, shared0, _res0, _, _} = CMerkleTree.nif_stats() for _ <- 1..100 do - _ = CAccountMap.list_difference(base, fork) + _ = CAccountMap.difference_full(base, fork) end force_gc() diff --git a/test/cmerkle_storage_map_test.exs b/test/cmerkle_storage_map_test.exs new file mode 100644 index 0000000..d1cbfc2 --- /dev/null +++ b/test/cmerkle_storage_map_test.exs @@ -0,0 +1,107 @@ +# Diode Server +# Copyright 2021-2024 Diode +# Licensed under the Diode License, Version 1.1 +# +# Focused coverage for CAccountMap / Chain.State storage_* and put_meta APIs. +defmodule CMerkleStorageMapTest do + use ExUnit.Case, async: false + + alias Chain.State + + @moduletag timeout: 60_000 + + defp addr(i), do: <> + defp slot(i), do: <> + defp val(i), do: <> + + test "storage_put_map multi-account multi-slot then get / to_list / root_hash" do + map = CAccountMap.new() + + map = + CAccountMap.storage_put_map(map, %{ + addr(1) => %{slot(10) => val(10), slot(11) => val(11)}, + addr(2) => %{slot(20) => val(20), slot(21) => val(21), slot(22) => val(22)} + }) + + assert CAccountMap.storage_get(map, addr(1), slot(10)) == val(10) + assert CAccountMap.storage_get(map, addr(1), slot(11)) == val(11) + assert CAccountMap.storage_get(map, addr(2), slot(20)) == val(20) + assert CAccountMap.storage_get(map, addr(2), slot(22)) == val(22) + assert CAccountMap.storage_get(map, addr(1), slot(99)) == nil + + list1 = CAccountMap.storage_to_list(map, addr(1)) |> Enum.sort() + assert list1 == Enum.sort([{slot(10), val(10)}, {slot(11), val(11)}]) + + list2 = CAccountMap.storage_to_list(map, addr(2)) + assert length(list2) == 3 + assert CAccountMap.storage_size(map, addr(1)) == 2 + assert CAccountMap.storage_size(map, addr(2)) == 3 + + root1 = CAccountMap.storage_root_hash(map, addr(1)) + root2 = CAccountMap.storage_root_hash(map, addr(2)) + assert is_binary(root1) and byte_size(root1) == 32 + assert is_binary(root2) and byte_size(root2) == 32 + assert root1 != root2 + + # State helpers mirror the same values. + state = %State{accounts: map} + assert State.storage_value(state, addr(1), slot(10)) == val(10) + assert State.storage_size(state, addr(2)) == 3 + assert State.storage_root_hash(state, addr(1)) == root1 + end + + test "storage_put_map on frozen map raises" do + map = + CAccountMap.new() + |> CAccountMap.storage_put_map(%{addr(1) => %{slot(1) => val(1)}}) + + CAccountMap.lock(map) + + assert_raise ArgumentError, fn -> + CAccountMap.storage_put_map(map, %{addr(1) => %{slot(2) => val(2)}}) + end + + assert CAccountMap.storage_get(map, addr(1), slot(1)) == val(1) + assert CAccountMap.storage_get(map, addr(1), slot(2)) == nil + end + + test "put_meta updates nonce without wiping storage" do + map = + CAccountMap.new() + |> CAccountMap.storage_put_map(%{ + addr(1) => %{slot(5) => val(5), slot(6) => val(6)} + }) + + before_root = CAccountMap.storage_root_hash(map, addr(1)) + before_list = CAccountMap.storage_to_list(map, addr(1)) |> Enum.sort() + + map = CAccountMap.put_meta(map, addr(1), 42, 9_999, <<1, 2, 3>>) + + {nonce, balance, _storage, code} = CAccountMap.get(map, addr(1)) + assert nonce == 42 + assert balance == 9_999 + assert code == <<1, 2, 3>> + + assert CAccountMap.storage_get(map, addr(1), slot(5)) == val(5) + assert CAccountMap.storage_get(map, addr(1), slot(6)) == val(6) + assert CAccountMap.storage_to_list(map, addr(1)) |> Enum.sort() == before_list + assert CAccountMap.storage_root_hash(map, addr(1)) == before_root + end + + test "get_proofs and storage_get_proofs return terms without crashing" do + map = + CAccountMap.new() + |> CAccountMap.put(addr(1), 1, 100, CMerkleTree.new(), <<>>) + |> CAccountMap.storage_put_map(%{addr(1) => %{slot(7) => val(7)}}) + + account_proof = CAccountMap.get_proofs(map, addr(1)) + assert account_proof != nil + + storage_proof = CAccountMap.storage_get_proofs(map, addr(1), slot(7)) + assert storage_proof != nil + + state = %State{accounts: map} + assert State.get_proofs(state, addr(1)) != nil + assert State.storage_get_proofs(state, addr(1), slot(7)) != nil + end +end From aba05b71c0b5b75df79191c7b1b8aee08fbe3b6e Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Thu, 16 Jul 2026 15:54:42 +0200 Subject: [PATCH 05/16] Close remaining NIF-boundary review gaps after frozen-only lock. Export state root hashes instead of a live state_trie, compact via one account_map_compact NIF call, and make map-backed accounts explicit with map_backed on Chain.Account. Co-authored-by: Cursor --- c_src/LOCK_ORDER.md | 1 + c_src/SECURITY_REVIEW.md | 5 +- c_src/nif.cpp | 155 +++++++++++++++++++- docs/caccount-map-nif.md | 17 ++- lib/caccount_map.ex | 22 ++- lib/chain/account.ex | 77 +++++----- lib/chain/block.ex | 4 +- lib/chain/block_cache.ex | 2 +- lib/chain/state.ex | 30 +--- lib/chaindefinition/voyager.ex | 3 +- lib/cmerkletree.ex | 3 +- lib/network/edge_v2.ex | 3 +- test/cmerkle_lock_clone_regression_test.exs | 11 +- 13 files changed, 234 insertions(+), 99 deletions(-) diff --git a/c_src/LOCK_ORDER.md b/c_src/LOCK_ORDER.md index 8e26c1d..375699f 100644 --- a/c_src/LOCK_ORDER.md +++ b/c_src/LOCK_ORDER.md @@ -31,6 +31,7 @@ See also [`SECURITY_REVIEW.md`](SECURITY_REVIEW.md) (F-5 fix) and [`scripts/cmer | `account_map_storage_put_map` | `AccountMapLock` → reject if frozen → per-addr `write_storage_slot` → `update_state_trie_for_entry` | Dirty CPU; EVM `su` hot path | | `switch_local_to_canonical` | tree mutexes (address order) | Abandoned `SharedState` queued on `pending_orphans`; reclaimed via `try_reclaim_orphans` after `enter_lock` / `leave_lock` when mutex trylock succeeds and `has_clone == 0` | | `account_map_uncompact_state` | `AccountMapLock(input)` → `materialize_storage` (brief tree lock) → `batch_insert` (state_store lock) | Dirty scheduler | +| `account_map_compact` | `AccountMapLock` (read-only; OK frozen) → per-account storage list via live tree lock or compact_storage slots (no materialize) | Dirty CPU; single boundary crossing for `Chain.State.compact/1` | | `account_map_put/delete` / `put_meta` | `AccountMapLock` only; reject if `frozen` | May `release_resource` → async GC `leave_lock` | | `account_map_difference_full` | Dual map lock (`DualAccountMapLock`, address order) → snapshot sides → release → per-account storage diffs | Dirty CPU; never hold map lock across storage diff build | | `account_map_apply_difference` | `AccountMapLock` → reject if `frozen` → storage/field writes (`write_storage_slot` → `make_writeable_locked`) | Dirty CPU | diff --git a/c_src/SECURITY_REVIEW.md b/c_src/SECURITY_REVIEW.md index 590fd19..a78e591 100644 --- a/c_src/SECURITY_REVIEW.md +++ b/c_src/SECURITY_REVIEW.md @@ -36,7 +36,7 @@ | `account_map_put_meta` | 5 | resource, address, nonce, balance, code | Metadata-only put; keeps existing storage | | `account_map_delete` | 2 | resource, address | Rejects frozen | | `account_map_root_hash` | 1 | resource | `CAccountMap.root_hash/1`, `Chain.State.hash/1` | -| `account_map_state_trie` | 1 | resource | `CAccountMap.state_trie/1`, `Chain.State.tree/1` (Edge state roots) | +| `account_map_state_root_hashes` | 1 | resource | `CAccountMap.state_root_hashes/1`, `Chain.State.state_root_hashes/1` (Edge `getstateroots`; no live trie export) | | `account_map_get_proofs` | 2 | map, address | Account inclusion proof on internal state_trie | | `account_map_storage_put_map` | 2 | map, update list | EVM `su` hot path — one NIF for multi-account slots | | `account_map_storage_get` | 3 | map, addr, key | `State.storage_value/3`, RPC | @@ -50,6 +50,7 @@ | `account_map_to_list` | 1 | resource | `CAccountMap.to_list/1` | | `account_map_difference_full` | 2 | two maps | `Chain.State.difference/2` | | `account_map_apply_difference` | 2 | map, delta list | `Chain.State.apply_difference/2` | +| `account_map_compact` | 1 | account map resource | Dirty CPU; returns `%{addr => %Account{...}}` compact map (read-only; OK frozen) | | `account_map_uncompact_state` | 1 | compact or resource | Returns `{am, hash}` | **Frozen map:** `account_map_lock/1` sets map-level `frozen` only. Map mutations (`put`/`put_meta`/`delete`/`apply_difference`/`storage_put_map`) fail while frozen. `account_map_get` / `to_list` export storage root hashes (never live tries), so Elixir cannot mutate map-owned storage via bare `CMerkleTree.insert`. `clone/1` forks writable unlocked wrappers for sync and speculative RPC/Edge/Shell. @@ -90,7 +91,7 @@ | ID | Topic | Severity | CWE | Notes | |----|--------|----------|-----|--------| -| F-9 | **Unbounded work per NIF** | Medium | CWE-400 | Long-running exports use dirty schedulers: **CPU-bound** — `get_proofs_raw`, `difference_raw`, `to_list`, `import_map`, `count_zeros`, `memory_stats_raw`, `clone`, `account_map_clone`, `account_map_lock`, `account_map_to_list`, `account_map_difference_full`, `account_map_apply_difference`, `account_map_storage_put_map`, `account_map_storage_to_list`, `account_map_storage_get_proofs`, `account_map_get_proofs`, `account_map_uncompact_state`; **IO-bound** — `malloc_info_raw`. `account_map_put`/`put_meta`/`delete`/`storage_get*` stay on normal schedulers where short. Large dirty-NIF loops call `enif_consume_timeslice` every 512 iterations. Ensure adequate dirty CPU schedulers at runtime (`+SDcpu` on heavy sync nodes). | +| F-9 | **Unbounded work per NIF** | Medium | CWE-400 | Long-running exports use dirty schedulers: **CPU-bound** — `get_proofs_raw`, `difference_raw`, `to_list`, `import_map`, `count_zeros`, `memory_stats_raw`, `clone`, `account_map_clone`, `account_map_lock`, `account_map_to_list`, `account_map_difference_full`, `account_map_apply_difference`, `account_map_storage_put_map`, `account_map_storage_to_list`, `account_map_storage_get_proofs`, `account_map_get_proofs`, `account_map_compact`, `account_map_uncompact_state`; **IO-bound** — `malloc_info_raw`. `account_map_put`/`put_meta`/`delete`/`storage_get*` stay on normal schedulers where short. Large dirty-NIF loops call `enif_consume_timeslice` every 512 iterations. Ensure adequate dirty CPU schedulers at runtime (`+SDcpu` on heavy sync nodes). | ### Memory safety (manual review) diff --git a/c_src/nif.cpp b/c_src/nif.cpp index 8835220..c9e52df 100644 --- a/c_src/nif.cpp +++ b/c_src/nif.cpp @@ -1687,16 +1687,16 @@ static void remove_state_trie_entry(SharedAccountMap *shared, const uint160_t &a } static ERL_NIF_TERM -account_map_state_trie(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +account_map_state_root_hashes(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { accountmap *am; if (argc != 1) return enif_make_badarg(env); if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); AccountMapLock lock(am); - ERL_NIF_TERM term = enif_make_resource(env, am->shared->state_trie); - enif_keep_resource(am->shared->state_trie); - return term; + Lock tree_lock(am->shared->state_trie); + auto root_hashes = am->shared->state_trie->shared_state->tree.root_hashes(); + return make_binary(env, (uint8_t*)root_hashes, 32*16); } static ERL_NIF_TERM @@ -2890,6 +2890,150 @@ static bool append_uncompacted_account(ErlNifEnv *env, accountmap *am, AccountHa return true; } +// Build storage items map for compact export without materializing compact_storage +// into a live trie when possible. +static bool build_compact_storage_items(ErlNifEnv *env, const AccountEntry &entry, + ERL_NIF_TERM &out_map, size_t &out_size, size_t progress_base) +{ + out_size = 0; + out_map = enif_make_new_map(env); + + if (entry.storage != nullptr) { + Lock lock(entry.storage); + size_t i = 0; + bool ok = true; + entry.storage->shared_state->tree.each([&](pair_t &pair) { + if (!ok) { + return; + } + i++; + ERL_NIF_TERM key_term = make_binary(env, pair.key.data(), pair.key.size()); + ERL_NIF_TERM value_term = make_binary(env, pair.value.data(), 32); + ERL_NIF_TERM new_map; + if (!enif_make_map_put(env, out_map, key_term, value_term, &new_map)) { + ok = false; + return; + } + out_map = new_map; + nif_loop_progress(env, progress_base + i); + }); + out_size = i; + return ok; + } + + if (entry.compact_storage && !entry.compact_storage->slots.empty()) { + size_t i = 0; + for (auto &slot : entry.compact_storage->slots) { + i++; + ERL_NIF_TERM key_term = make_binary(env, (uint8_t *)slot.key.data(), slot.key.size()); + ERL_NIF_TERM value_term = make_binary(env, slot.value.data(), 32); + ERL_NIF_TERM new_map; + if (!enif_make_map_put(env, out_map, key_term, value_term, &new_map)) { + return false; + } + out_map = new_map; + nif_loop_progress(env, progress_base + i); + } + out_size = i; + return true; + } + + return true; +} + +static bool make_compact_account_term(ErlNifEnv *env, AccountEntry &entry, + size_t progress_base, ERL_NIF_TERM &out) +{ + uint256_t storage_root; + if (!storage_root_hash_for_entry(env, entry, storage_root, progress_base)) { + return false; + } + + ERL_NIF_TERM items_map; + size_t item_count = 0; + if (!build_compact_storage_items(env, entry, items_map, item_count, progress_base)) { + return false; + } + + ERL_NIF_TERM storage_root_term; + if (item_count == 0) { + storage_root_term = make_atom(env, "nil"); + } else { + // Match Elixir `{MapMerkleTree, [], items}` (module atom Elixir.MapMerkleTree). + storage_root_term = enif_make_tuple3(env, + make_atom(env, "Elixir.MapMerkleTree"), + enif_make_list(env, 0), + items_map); + } + + ERL_NIF_TERM code_term = + entry.code.empty() ? make_atom(env, "nil") + : make_binary(env, (uint8_t *)entry.code.data(), entry.code.size()); + + uint256_t code_hash; + if (entry.code.empty()) { + code_hash = empty_code_hash; + } else { + sha(entry.code.data(), entry.code.size(), code_hash.data()); + } + + // Shape matches Chain.State.compact/1 / Account.compact/1 and parse_compact_account: + // %Chain.Account{nonce, balance, storage_root, code, map_backed: false} + // plus :root_hash and :code_hash. + ERL_NIF_TERM keys[8]; + ERL_NIF_TERM values[8]; + keys[0] = make_atom(env, "__struct__"); + values[0] = make_atom(env, "Elixir.Chain.Account"); + keys[1] = make_atom(env, "nonce"); + values[1] = enif_make_uint64(env, entry.nonce); + keys[2] = make_atom(env, "balance"); + values[2] = balance_to_term(env, entry.balance); + keys[3] = make_atom(env, "storage_root"); + values[3] = storage_root_term; + keys[4] = make_atom(env, "code"); + values[4] = code_term; + keys[5] = make_atom(env, "map_backed"); + values[5] = make_atom(env, "false"); + keys[6] = make_atom(env, "root_hash"); + values[6] = make_binary(env, storage_root.data(), 32); + keys[7] = make_atom(env, "code_hash"); + values[7] = make_binary(env, code_hash.data(), 32); + + return enif_make_map_from_arrays(env, keys, values, 8, &out); +} + +static ERL_NIF_TERM +account_map_compact(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + accountmap *am; + if (argc != 1) return enif_make_badarg(env); + if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) { + return enif_make_badarg(env); + } + + // Read-only: OK on frozen maps; do not require writable. + AccountMapLock lock(am); + + ERL_NIF_TERM result = enif_make_new_map(env); + + size_t i = 0; + for (auto &kv : am->shared->accounts) { + i++; + ERL_NIF_TERM addr = make_binary(env, (uint8_t *)kv.first.value, 20); + ERL_NIF_TERM account; + if (!make_compact_account_term(env, kv.second, i * kNifTimesliceInterval, account)) { + return enif_make_badarg(env); + } + ERL_NIF_TERM new_map; + if (!enif_make_map_put(env, result, addr, account, &new_map)) { + return enif_make_badarg(env); + } + result = new_map; + nif_loop_progress(env, i); + } + return result; +} + static ERL_NIF_TERM account_map_uncompact_state(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { @@ -3058,11 +3202,12 @@ static ErlNifFunc nif_funcs[] = { {"account_map_put_meta", 5, account_map_put_meta, 0}, {"account_map_delete", 2, account_map_delete, 0}, {"account_map_root_hash", 1, account_map_root_hash, 0}, - {"account_map_state_trie", 1, account_map_state_trie, 0}, + {"account_map_state_root_hashes", 1, account_map_state_root_hashes, 0}, {"account_map_size", 1, account_map_size, 0}, {"account_map_to_list", 1, account_map_to_list, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_difference_full", 2, account_map_difference_full, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_apply_difference", 2, account_map_apply_difference, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"account_map_compact", 1, account_map_compact, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_uncompact_state", 1, account_map_uncompact_state, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_storage_put_map", 2, account_map_storage_put_map, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_storage_get", 3, account_map_storage_get, 0}, diff --git a/docs/caccount-map-nif.md b/docs/caccount-map-nif.md index 6ee1353..9ea7cf7 100644 --- a/docs/caccount-map-nif.md +++ b/docs/caccount-map-nif.md @@ -12,14 +12,23 @@ See also [`c_src/LOCK_ORDER.md`](../c_src/LOCK_ORDER.md) and root trie live in C++. Elixir does **not** carry a separate `:store` field. - Prefer map-owned storage APIs over bare `merkletree` resources: - `Chain.State.storage_get/3`, `storage_put_map/2`, `storage_to_list/2`, - `storage_get_proofs/3`, `storage_root_hash/2` + `storage_get_proofs/3`, `storage_root_hash/2`, `state_root_hashes/1` - `CAccountMap` mirrors of the same - `Chain.State.hash/1` or `CAccountMap.root_hash/1` for the state root. + Internal `state_trie` is never exported; Edge uses `state_root_hashes/1`. - `account_map_get` / `to_list` return `{nonce, balance, storage_root_hash_bin32, code}` — the third element is a **32-byte hash**, never a live storage resource. -- Map-backed `%Chain.Account{}` values have `storage_root: nil` and carry - `:root_hash` when loaded from the map. Standalone tries are only for genesis / - hardfork / import via `account_map_put/6`. +- `account_map_compact/1` (dirty CPU) returns `%{addr => %Chain.Account{...}}` with + `storage_root: nil | {MapMerkleTree, [], items}`, `:root_hash`, `:code_hash`, and + `map_backed: false` — one NIF call for `Chain.State.compact/1` (no per-account + `to_list` / `storage_to_list` round-trips). Prefer listing compact_storage slots + without materializing live tries when possible. +- Map-backed `%Chain.Account{}` values set `map_backed: true` (via + `Account.from_parts/4` with a 32-byte root hash) and carry `:root_hash` for + hash caching. Do not use presence of `:root_hash` as a discriminator — + compact DB snapshots also cache `:root_hash` with `map_backed: false`. + Standalone tries (`map_backed: false`) are only for genesis / hardfork / + import via `account_map_put/6`. ## Clone and lock diff --git a/lib/caccount_map.ex b/lib/caccount_map.ex index e4689be..34916e3 100644 --- a/lib/caccount_map.ex +++ b/lib/caccount_map.ex @@ -16,7 +16,8 @@ defmodule CAccountMap do def root_hash(map), do: CMerkleTree.account_map_root_hash(map) - def state_trie(map), do: CMerkleTree.account_map_state_trie(map) + def state_root_hashes(map), + do: decode_root_hashes(CMerkleTree.account_map_state_root_hashes(map)) def get_proofs(map, <<_::160>> = addr), do: CMerkleTree.account_map_get_proofs(map, addr) @@ -109,13 +110,7 @@ defmodule CAccountMap do do: CMerkleTree.account_map_storage_root_hash(map, addr) def storage_root_hashes(map, <<_::160>> = addr) do - <> = - CMerkleTree.account_map_storage_root_hashes(map, addr) - - [a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p] + decode_root_hashes(CMerkleTree.account_map_storage_root_hashes(map, addr)) end def storage_get_proofs(map, <<_::160>> = addr, key), @@ -136,6 +131,8 @@ defmodule CAccountMap do defp decode_storage_value(nil), do: nil defp decode_storage_value(val) when is_binary(val), do: val + def compact(map), do: CMerkleTree.account_map_compact(map) + def uncompact_state(accounts), do: CMerkleTree.account_map_uncompact_state(accounts) # Third element is always a 32-byte storage root hash (never a live resource). @@ -173,4 +170,13 @@ defmodule CAccountMap do defp to_bytes(string) when is_binary(string), do: string defp to_bytes(int) when is_integer(int), do: to_bytes32(int) + + defp decode_root_hashes( + <> + ) do + [a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p] + end end diff --git a/lib/chain/account.ex b/lib/chain/account.ex index 2e8fb2e..fc7ffeb 100644 --- a/lib/chain/account.ex +++ b/lib/chain/account.ex @@ -2,13 +2,14 @@ # Copyright 2021-2024 Diode # Licensed under the Diode License, Version 1.1 defmodule Chain.Account do - defstruct nonce: 0, balance: 0, storage_root: nil, code: nil + defstruct nonce: 0, balance: 0, storage_root: nil, code: nil, map_backed: false @type t :: %Chain.Account{ nonce: non_neg_integer(), balance: non_neg_integer(), storage_root: CMerkleTree.t() | nil, - code: binary() | nil + code: binary() | nil, + map_backed: boolean() } def new(props \\ []) do @@ -26,24 +27,21 @@ defmodule Chain.Account do @doc """ Live storage trie for standalone (genesis/import) accounts only. - Map-backed accounts (`storage_root: nil` with `:root_hash`) have no live trie — + Map-backed accounts (`map_backed: true`) have no live trie — use `Chain.State.storage_*` APIs instead. """ @spec tree(Chain.Account.t()) :: CMerkleTree.t() - def tree(%Chain.Account{storage_root: nil} = acc) do - if map_backed?(acc) do - raise ArgumentError, - "map-backed account has no live storage_root; use Chain.State.storage_* APIs" - else - CMerkleTree.new() - end + def tree(%Chain.Account{map_backed: true}) do + raise ArgumentError, + "map-backed account has no live storage_root; use Chain.State.storage_* APIs" end + def tree(%Chain.Account{storage_root: nil}), do: CMerkleTree.new() def tree(%Chain.Account{storage_root: root}), do: root @doc """ Build an account from `CAccountMap.get/2` parts. - When `storage` is a 32-byte root hash, the account is map-backed (`storage_root: nil`). + When `storage` is a 32-byte root hash, the account is map-backed (`map_backed: true`). When `storage` is a merkle resource (or nil), it is a standalone trie for genesis/put. """ def from_parts(nonce, balance, <>, code) do @@ -51,7 +49,8 @@ defmodule Chain.Account do nonce: nonce, balance: balance, storage_root: nil, - code: if(code == "", do: nil, else: code) + code: if(code == "", do: nil, else: code), + map_backed: true } |> Map.put(:root_hash, root_hash) end @@ -61,7 +60,8 @@ defmodule Chain.Account do nonce: nonce, balance: balance, storage_root: storage, - code: if(code == "", do: nil, else: code) + code: if(code == "", do: nil, else: code), + map_backed: false } end @@ -71,8 +71,7 @@ defmodule Chain.Account do end def put_tree(%Chain.Account{} = acc, root) do - acc - |> Map.put(:storage_root, root) + %{acc | storage_root: root, map_backed: false} |> Map.delete(:root_hash) end @@ -88,45 +87,48 @@ defmodule Chain.Account do end def uncompact(%Chain.Account{storage_root: nil} = acc) do - %Chain.Account{acc | storage_root: CMerkleTree.new()} + %Chain.Account{acc | storage_root: CMerkleTree.new(), map_backed: false} end def uncompact(%Chain.Account{storage_root: {MapMerkleTree, _opts, items}} = acc) when is_map(items) do storage_root = CMerkleTree.from_map(items) - %Chain.Account{acc | storage_root: storage_root} + %Chain.Account{acc | storage_root: storage_root, map_backed: false} end def uncompact(%Chain.Account{storage_root: items} = acc) when is_list(items) do - %Chain.Account{acc | storage_root: CMerkleTree.from_list(items)} + %Chain.Account{acc | storage_root: CMerkleTree.from_list(items), map_backed: false} end - def compact(%Chain.Account{} = acc) do - if map_backed?(acc) do - raise ArgumentError, "map-backed accounts are compacted via Chain.State.compact/1" - end + def compact(%Chain.Account{map_backed: true}) do + raise ArgumentError, "map-backed accounts are compacted via Chain.State.compact/1" + end + def compact(%Chain.Account{} = acc) do tree = tree(acc) if CMerkleTree.size(tree) == 0 do - %Chain.Account{acc | storage_root: nil} + %Chain.Account{acc | storage_root: nil, map_backed: false} else - %Chain.Account{acc | storage_root: {MapMerkleTree, [], Map.new(CMerkleTree.to_list(tree))}} + %Chain.Account{ + acc + | storage_root: {MapMerkleTree, [], Map.new(CMerkleTree.to_list(tree))}, + map_backed: false + } end |> Map.put(:root_hash, CMerkleTree.root_hash(tree)) |> Map.put(:code_hash, codehash(acc)) end - def storage_set_value(%Chain.Account{} = acc, key = <<_k::256>>, value = <<_v::256>>) do - if map_backed?(acc) do - raise ArgumentError, - "map-backed account storage writes go through Chain.State.storage_put_map/2" - end + def storage_set_value(%Chain.Account{map_backed: true}, <<_::256>>, <<_::256>>) do + raise ArgumentError, + "map-backed account storage writes go through Chain.State.storage_put_map/2" + end + def storage_set_value(%Chain.Account{} = acc, key = <<_k::256>>, value = <<_v::256>>) do store = CMerkleTree.insert(tree(acc), key, value) - acc - |> Map.put(:storage_root, store) + %{acc | storage_root: store, map_backed: false} |> Map.delete(:root_hash) end @@ -143,12 +145,12 @@ defmodule Chain.Account do storage_value(acc, <>) end - def storage_value(%Chain.Account{} = acc, key) when is_binary(key) do - if map_backed?(acc) do - raise ArgumentError, - "map-backed account storage reads go through Chain.State.storage_value/3" - end + def storage_value(%Chain.Account{map_backed: true}, key) when is_binary(key) do + raise ArgumentError, + "map-backed account storage reads go through Chain.State.storage_value/3" + end + def storage_value(%Chain.Account{} = acc, key) when is_binary(key) do case CMerkleTree.get(tree(acc), key) do nil -> <<0::unsigned-size(256)>> bin -> bin @@ -178,7 +180,4 @@ defmodule Chain.Account do def codehash(%Chain.Account{code: code}) do Diode.hash(code) end - - defp map_backed?(%Chain.Account{storage_root: nil} = acc), do: Map.has_key?(acc, :root_hash) - defp map_backed?(_), do: false end diff --git a/lib/chain/block.ex b/lib/chain/block.ex index 988bbb7..1a26dd1 100644 --- a/lib/chain/block.ex +++ b/lib/chain/block.ex @@ -88,8 +88,8 @@ defmodule Chain.Block do Chain.State.storage_get_proofs(state(block), account_id, key) end - def state_tree(%Block{} = block) do - state(block) |> Chain.State.tree() + def state_root_hashes(%Block{} = block) do + Chain.State.state_root_hashes(state(block)) end def account_proof(%Block{} = block, account_id) do diff --git a/lib/chain/block_cache.ex b/lib/chain/block_cache.ex index a736218..2c1f9a0 100644 --- a/lib/chain/block_cache.ex +++ b/lib/chain/block_cache.ex @@ -266,7 +266,7 @@ defmodule Chain.BlockCache do defdelegate size(block), to: Block defdelegate state(block), to: Block defdelegate state_hash(block), to: Block - defdelegate state_tree(block), to: Block + defdelegate state_root_hashes(block), to: Block defdelegate strip_state(block), to: Block defdelegate timestamp(block), to: Block defdelegate transaction(block, hash), to: Block diff --git a/lib/chain/state.ex b/lib/chain/state.ex index c46d99e..1935f9b 100644 --- a/lib/chain/state.ex +++ b/lib/chain/state.ex @@ -3,7 +3,6 @@ # Licensed under the Diode License, Version 1.1 defmodule Chain.State do require Logger - alias Chain.Account @dialyzer [ {:nowarn_function, new: 0}, @@ -21,30 +20,7 @@ defmodule Chain.State do end def compact(%Chain.State{accounts: accounts} = state) do - # Map-backed gets return root hashes only — build compact maps via storage_* APIs. - compact_accounts = - accounts - |> CAccountMap.to_list() - |> Map.new(fn {id, {nonce, balance, root_hash, code}} -> - items = Map.new(CAccountMap.storage_to_list(accounts, id)) - - acc = - %Account{ - nonce: nonce, - balance: balance, - storage_root: if(map_size(items) == 0, do: nil, else: {MapMerkleTree, [], items}), - code: if(code == "", do: nil, else: code) - } - |> Map.put(:root_hash, root_hash) - |> Map.put( - :code_hash, - Account.codehash(%Account{code: if(code == "", do: nil, else: code)}) - ) - - {id, acc} - end) - - %Chain.State{state | accounts: compact_accounts} + %{state | accounts: CAccountMap.compact(accounts)} end def uncompact(%Chain.State{accounts: accounts} = state) do @@ -56,8 +32,8 @@ defmodule Chain.State do %{state | hash: CAccountMap.root_hash(accounts)} end - def tree(%Chain.State{accounts: accounts}) do - CAccountMap.state_trie(accounts) + def state_root_hashes(%Chain.State{accounts: accounts}) do + CAccountMap.state_root_hashes(accounts) end def get_proofs(%Chain.State{accounts: accounts}, <<_::160>> = addr) do diff --git a/lib/chaindefinition/voyager.ex b/lib/chaindefinition/voyager.ex index d37cc5d..6d311df 100644 --- a/lib/chaindefinition/voyager.ex +++ b/lib/chaindefinition/voyager.ex @@ -7,13 +7,14 @@ defmodule ChainDefinition.Voyager do code = Base16.decode(account["code"]) {balance, ""} = Integer.parse(account["balance"]) - # Field-only update via put_meta (map-backed accounts have no live storage_root). + # Field-only update via put_meta (storage_root: nil → put_meta). acc = state |> State.ensure_account(id) |> Map.put(:balance, balance) |> Map.put(:code, code) |> Map.put(:storage_root, nil) + |> Map.put(:map_backed, false) |> Map.delete(:root_hash) state = State.set_account(state, id, acc) diff --git a/lib/cmerkletree.ex b/lib/cmerkletree.ex index 6fd520b..9f27e5a 100644 --- a/lib/cmerkletree.ex +++ b/lib/cmerkletree.ex @@ -116,7 +116,7 @@ defmodule CMerkleTree do def account_map_new(), do: error() def account_map_clone(_map), do: error() def account_map_root_hash(_map), do: error() - def account_map_state_trie(_map), do: error() + def account_map_state_root_hashes(_map), do: error() def account_map_get_proofs(_map, _addr), do: error() def account_map_lock(_map), do: error() def account_map_get(_map, _addr), do: error() @@ -127,6 +127,7 @@ defmodule CMerkleTree do def account_map_to_list(_map), do: error() def account_map_difference_full(_map_a, _map_b), do: error() def account_map_apply_difference(_map, _delta), do: error() + def account_map_compact(_map), do: error() def account_map_uncompact_state(_map), do: error() def account_map_storage_put_map(_map, _updates), do: error() def account_map_storage_get(_map, _addr, _key), do: error() diff --git a/lib/network/edge_v2.ex b/lib/network/edge_v2.ex index f701037..a98e24b 100644 --- a/lib/network/edge_v2.ex +++ b/lib/network/edge_v2.ex @@ -61,8 +61,7 @@ defmodule Network.EdgeV2 do ["getstateroots", index] -> BlockProcess.with_block(to_num(index), fn block -> - Chain.Block.state_tree(block) - |> CMerkleTree.root_hashes() + Chain.Block.state_root_hashes(block) end) |> response() diff --git a/test/cmerkle_lock_clone_regression_test.exs b/test/cmerkle_lock_clone_regression_test.exs index 522ff05..99e9550 100644 --- a/test/cmerkle_lock_clone_regression_test.exs +++ b/test/cmerkle_lock_clone_regression_test.exs @@ -128,10 +128,9 @@ defmodule CMerkleLockCloneRegressionTest do assert CAccountMap.storage_get(peak.accounts, addr(1), slot(42)) == nil end - test "put and storage_put_map on frozen map raise; state_trie remains readable" do + test "put and storage_put_map on frozen map raise; state_root_hashes stay stable" do peak = locked_peak_like_state(1) - trie = State.tree(peak) - before = CMerkleTree.root_hash(trie) + before = State.state_root_hashes(peak) assert_raise ArgumentError, fn -> State.storage_put_map(peak, %{addr(1) => %{slot(9) => val(9)}}) @@ -141,10 +140,8 @@ defmodule CMerkleLockCloneRegressionTest do CAccountMap.put(peak.accounts, addr(9), 0, 0, CMerkleTree.new(), <<>>) end - # Lock is frozen-only on the account map; State.tree still returns a live - # resource for reads. Root hash of the state trie must stay unchanged. - assert CMerkleTree.root_hash(trie) == before - assert CMerkleTree.root_hash(State.tree(peak)) == before + assert State.state_root_hashes(peak) == before + assert length(before) == 16 end test "apply_difference on locked map raises; clone then apply succeeds" do From c74c7763ea109b588ea849b8223ff9c7d2c60a69 Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Thu, 16 Jul 2026 16:40:25 +0200 Subject: [PATCH 06/16] Reduce account-map NIF surface and make genesis map-native. Merge duplicate account_map NIFs, gate bare-tree exports behind CMERKLE_TEST_NIFS, drop dead Account storage APIs, and keep EVM storage reads on the normal scheduler. Co-authored-by: Cursor --- .gitignore | 1 + AGENTS.md | 5 +- Makefile | 4 +- c_src/LOCK_ORDER.md | 7 +- c_src/Makefile | 32 +- c_src/SECURITY_REVIEW.md | 89 ++--- c_src/nif.cpp | 364 +++++++++++--------- docs/caccount-map-nif.md | 33 +- lib/caccount_map.ex | 39 ++- lib/chain/account.ex | 111 +----- lib/chain/genesis_factory.ex | 12 +- lib/chain/state.ex | 31 +- lib/chaindefinition.ex | 11 + lib/chaindefinition/devnet.ex | 28 +- lib/chaindefinition/voyager.ex | 24 +- lib/cmerkletree.ex | 15 +- lib/contract/registry.ex | 46 +-- lib/network/status.ex | 2 +- lib/shell.ex | 3 +- scripts/cmerkle_fuzz.exs | 25 +- scripts/cmerkle_heap_assumptions.exs | 6 +- scripts/cmerkle_leak_test.exs | 2 +- scripts/cmerkle_parallel_stress.exs | 25 +- test/chain_account_hash_nif_test.exs | 77 +++-- test/chain_state_merkle_test.exs | 19 +- test/chain_state_uncompact_test.exs | 63 ++-- test/chain_test.exs | 13 +- test/cmerkle_lock_clone_regression_test.exs | 17 +- test/cmerkle_nif_deadlock_test.exs | 23 +- test/cmerkle_nif_leak_test.exs | 12 +- test/evm_storage_readahead_test.exs | 6 +- test/evm_test.exs | 13 +- 32 files changed, 626 insertions(+), 532 deletions(-) diff --git a/.gitignore b/.gitignore index 6981b0a..ae23180 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ /priv/merkletree_nif.so /priv/merkletree_nif.asan.so /priv/merkletree_nif.so.bak* +/priv/.cmerkle_nif_mode # Profiling / measurement outputs (see scripts/profile_*.sh) /tmp/ diff --git a/AGENTS.md b/AGENTS.md index a9da64a..28fbb0e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,8 +28,9 @@ Ethereum-compatible JSON-RPC endpoint plus the Diode PEER/EDGE protocols. `make -C deps/libsecp256k1/` (see `.github/workflows/ci.yml`). Build artifacts are gitignored and persist across sessions, so this is only needed after a clean checkout of that dep. -- **CAccountMap / state NIF semantics** (clone, lock, storage APIs, get shape): - see [`docs/caccount-map-nif.md`](docs/caccount-map-nif.md). +- **CAccountMap / state NIF semantics** (clone, lock, storage APIs, get shape, + `CMERKLE_TEST_NIFS` prod vs test exports): see + [`docs/caccount-map-nif.md`](docs/caccount-map-nif.md). ### Lint - `mix lint` = `compile` + `mix format --check-formatted` + `mix credo --only warning` + `mix dialyzer`. diff --git a/Makefile b/Makefile index 404df23..75f54fb 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,9 @@ TESTDATA := test/pems/device1_certificate.pem test/pems/device2_certificate.pem .PHONY: all all: evm/evm priv/merkletree_nif.so -priv/merkletree_nif.so: $(wildcard c_src/*.cpp c_src/*.hpp) +# Always delegate to c_src so CMERKLE_TEST_NIFS mode stamp can force rebuild. +.PHONY: priv/merkletree_nif.so +priv/merkletree_nif.so: $(MAKE) -C c_src nif evm/evm: $(wildcard evm/*.cpp evm/*.hpp evm/*/*.cpp evm/*/*.hpp) diff --git a/c_src/LOCK_ORDER.md b/c_src/LOCK_ORDER.md index 375699f..612a659 100644 --- a/c_src/LOCK_ORDER.md +++ b/c_src/LOCK_ORDER.md @@ -27,12 +27,15 @@ See also [`SECURITY_REVIEW.md`](SECURITY_REVIEW.md) (F-5 fix) and [`scripts/cmer | `leave_lock` / GC destructor | if `mt->locked`: `locked_states_mutex` → tree → decrement registration → erase map when `has_clone == 0`; else tree refcount only | Unlocked resources never touch `LockedStates` map | | `merkletree_clone` | `Lock(parent)` | Dirty CPU scheduler; O(1) shallow resource alloc (`locked = false`) | | `account_map_clone` | `AccountMapLock` → `fork_shared_accountmap` (`Lock` per parent storage / state_trie) | Dirty scheduler; writable fork even if parent `frozen` | -| `account_map_lock` | `AccountMapLock` → set `frozen=true` only (O(1); no per-trie seal) | Dirty scheduler; put/delete/storage_put_map/put_meta reject via `frozen` | +| `account_map_lock` | `AccountMapLock` → set `frozen=true` only (O(1); no per-trie seal) | Dirty scheduler; put/delete/storage_put_map reject via `frozen` | | `account_map_storage_put_map` | `AccountMapLock` → reject if frozen → per-addr `write_storage_slot` → `update_state_trie_for_entry` | Dirty CPU; EVM `su` hot path | +| `account_map_storage` | `AccountMapLock` → read-only storage query (`:get`/`:range`/`:list`/`:size`) | Dirty CPU | +| `account_map_storage_roots` / `account_map_state_roots` | `AccountMapLock` → tree lock → root + 16 hashes blob | No live trie export | +| `account_map_proof` | `AccountMapLock` → account or storage proof | Dirty CPU; arities 2 and 3 | | `switch_local_to_canonical` | tree mutexes (address order) | Abandoned `SharedState` queued on `pending_orphans`; reclaimed via `try_reclaim_orphans` after `enter_lock` / `leave_lock` when mutex trylock succeeds and `has_clone == 0` | | `account_map_uncompact_state` | `AccountMapLock(input)` → `materialize_storage` (brief tree lock) → `batch_insert` (state_store lock) | Dirty scheduler | | `account_map_compact` | `AccountMapLock` (read-only; OK frozen) → per-account storage list via live tree lock or compact_storage slots (no materialize) | Dirty CPU; single boundary crossing for `Chain.State.compact/1` | -| `account_map_put/delete` / `put_meta` | `AccountMapLock` only; reject if `frozen` | May `release_resource` → async GC `leave_lock` | +| `account_map_put/delete` | `AccountMapLock` only; reject if `frozen`; storage arg may be `:keep` / list / resource | May `release_resource` → async GC `leave_lock` | | `account_map_difference_full` | Dual map lock (`DualAccountMapLock`, address order) → snapshot sides → release → per-account storage diffs | Dirty CPU; never hold map lock across storage diff build | | `account_map_apply_difference` | `AccountMapLock` → reject if `frozen` → storage/field writes (`write_storage_slot` → `make_writeable_locked`) | Dirty CPU | | Insert / COW | Tree lock → ItemPool / PreAllocator / stripe pool | Same-thread nesting | diff --git a/c_src/Makefile b/c_src/Makefile index c05d9dd..3d5dfe8 100644 --- a/c_src/Makefile +++ b/c_src/Makefile @@ -10,6 +10,24 @@ ERL_INCLUDE_PATH = $(shell erl -eval 'io:format("~s", [lists:concat([code:root_d CFLAGS+=-I. -O2 -g -Wall -Wno-unknown-pragmas CXXFLAGS+=-std=c++17 $(CFLAGS) +# Bare-tree + debug NIFs (new/insert/...), gated by -DCMERKLE_TEST_NIFS in nif.cpp. +# On unless MIX_ENV=prod; override with CMERKLE_TEST_NIFS=0 (off) or =1 (force on). +# priv/merkletree_nif.so is shared across Mix envs — last compile wins. +ENABLE_CMERKLE_TEST_NIFS := 1 +ifeq ($(MIX_ENV),prod) + ENABLE_CMERKLE_TEST_NIFS := 0 +endif +ifeq ($(CMERKLE_TEST_NIFS),0) + ENABLE_CMERKLE_TEST_NIFS := 0 +endif +ifeq ($(CMERKLE_TEST_NIFS),1) + ENABLE_CMERKLE_TEST_NIFS := 1 +endif +ifeq ($(ENABLE_CMERKLE_TEST_NIFS),1) + CXXFLAGS+=-DCMERKLE_TEST_NIFS +endif +NIF_MODE := $(if $(filter 1,$(ENABLE_CMERKLE_TEST_NIFS)),test,prod) + UNAME_S := $(shell uname -s) ifeq ($(UNAME_S),Darwin) @@ -49,22 +67,30 @@ uncompact_harness.bin: uncompact_harness.cpp merkletree.hpp merkletree.cpp item_ uncompact_harness: uncompact_harness.bin # Optional: libFuzzer + ASan on SHA (requires Clang with fuzzer runtime) -.PHONY: nif nif-asan fuzz_sha +.PHONY: nif nif-asan fuzz_sha FORCE fuzz_sha: fuzz_sha.bin ./fuzz_sha.bin -runs=5000 fuzz_sha.bin: fuzz_sha.cpp sha.cpp Makefile clang++ -std=c++17 -I. -g -O1 -fsanitize=fuzzer,address -fno-omit-frame-pointer -o fuzz_sha.bin fuzz_sha.cpp sha.cpp -lstdc++ +FORCE: + +# Rebuild .so when test/prod NIF mode changes (sources alone may be unchanged). +../priv/.cmerkle_nif_mode: FORCE + @mkdir -p ../priv + @echo $(NIF_MODE) > $@.new + @if ! cmp -s $@.new $@ 2>/dev/null; then mv $@.new $@; else rm -f $@.new; fi + nif: ../priv/merkletree_nif.so -../priv/merkletree_nif.so: nif.cpp merkletree.hpp merkletree.cpp item_pool.cpp sha.cpp preallocator.hpp Makefile +../priv/merkletree_nif.so: nif.cpp merkletree.hpp merkletree.cpp item_pool.cpp sha.cpp preallocator.hpp Makefile ../priv/.cmerkle_nif_mode echo ${ERL_INCLUDE_PATH} mkdir -p ../priv $(CXX) $(CXXFLAGS) -I${ERL_INCLUDE_PATH} -o ../priv/merkletree_nif.so -shared -fPIC nif.cpp ${OPTS} # Instrumented NIF for debugging heap issues (copy over priv/merkletree_nif.so or set in test script). -../priv/merkletree_nif.asan.so: nif.cpp merkletree.hpp merkletree.cpp item_pool.cpp sha.cpp preallocator.hpp Makefile +../priv/merkletree_nif.asan.so: nif.cpp merkletree.hpp merkletree.cpp item_pool.cpp sha.cpp preallocator.hpp Makefile ../priv/.cmerkle_nif_mode echo ${ERL_INCLUDE_PATH} mkdir -p ../priv $(CXX) $(CXXFLAGS) $(SANFLAGS) -I${ERL_INCLUDE_PATH} -o $@ -shared -fPIC nif.cpp $(OPTS_ASAN) diff --git a/c_src/SECURITY_REVIEW.md b/c_src/SECURITY_REVIEW.md index a78e591..40e8ca9 100644 --- a/c_src/SECURITY_REVIEW.md +++ b/c_src/SECURITY_REVIEW.md @@ -7,53 +7,37 @@ ## 1. NIF inventory (exports → Elixir) +### Production (always registered; ~21 entries) + | NIF name | Arity | Inputs | Callers (representative) | |----------|-------|--------|---------------------------| -| `new` | 0 | — | `CMerkleTree.new/0` | -| `insert_item_raw` | 3 | resource, key binary, value binary (must be 32 bytes) | `insert`, `insert_items` | -| `get_item` | 2 | resource, key binary | `get/2` | -| `get_range_raw` | 3 | resource, key binary (32 bytes), count (1..256) | `get_range/3`, `Evm` `gs` read-ahead | -| `get_proofs_raw` | 2 | resource, key binary | `get_proofs/2` → RPC, edge | -| `difference_raw` | 2 | two resources | `difference/2`, `Chain.State` | -| `lock` | 1 | resource | `CMerkleTree.lock/1`, scripts | -| `to_list` | 1 | resource | `to_list`, RPC | -| `import_map` | 2 | resource, map (bin→bin pairs, values 32 bytes) | `from_map` | -| `root_hash` | 1 | resource | Widespread | -| `hash` | 1 | binary | hashing helpers | -| `root_hashes_raw` | 1 | resource | `root_hashes/1`, edge | -| `bucket_count` | 1 | resource | tests | -| `size` | 1 | resource | Widespread | -| `clone` | 1 | resource | `clone`, account storage | | `count_zeros` | 1 | binary | `Evm` (tx payload) | -| `struct_sizes_raw` | 0 | — | tests, benches | -| `memory_stats_raw` | 1 | resource | tests, benches | -| `malloc_info_raw` | 0 | — | tests, `cmerkle_memory_bench.exs` | +| `nif_stats_raw` | 0 | — | `Network.Status` | | `account_map_new` | 0 | — | `CAccountMap.new/0`, `Chain.State` | -| `account_map_clone` | 1 | account map resource | `CAccountMap.clone/1`, `Chain.State.clone/1` (writable fork; OK on frozen parent) | -| `account_map_lock` | 1 | account map resource | `CAccountMap.lock/1` — `frozen` only (O(1); no per-trie seal) | -| `account_map_get` | 2 | resource, 20-byte address | `CAccountMap.get/2` returns `{nonce, balance, storage_root_hash_bin32, code}` — never a live storage resource | -| `account_map_put` | 6 | resource, address, nonce, balance, storage, code | Cold path (import/uncompact/genesis); rejects frozen | -| `account_map_put_meta` | 5 | resource, address, nonce, balance, code | Metadata-only put; keeps existing storage | +| `account_map_clone` | 1 | account map resource | `CAccountMap.clone/1`, `Chain.State.clone/1` | +| `account_map_lock` | 1 | account map resource | `CAccountMap.lock/1` — `frozen` only (O(1)) | +| `account_map_get` | 2 | resource, 20-byte address | `{nonce, balance, storage_root_hash_bin32, code}` — never a live storage resource | +| `account_map_put` | 6 | resource, addr, nonce, balance, storage, code | Storage arg: resource \| `:keep` \| `nil`/`[]` \| `[{k,v}]`; rejects frozen | | `account_map_delete` | 2 | resource, address | Rejects frozen | -| `account_map_root_hash` | 1 | resource | `CAccountMap.root_hash/1`, `Chain.State.hash/1` | -| `account_map_state_root_hashes` | 1 | resource | `CAccountMap.state_root_hashes/1`, `Chain.State.state_root_hashes/1` (Edge `getstateroots`; no live trie export) | -| `account_map_get_proofs` | 2 | map, address | Account inclusion proof on internal state_trie | -| `account_map_storage_put_map` | 2 | map, update list | EVM `su` hot path — one NIF for multi-account slots | -| `account_map_storage_get` | 3 | map, addr, key | `State.storage_value/3`, RPC | -| `account_map_storage_get_range` | 4 | map, addr, key, count | EVM `gs` | -| `account_map_storage_to_list` | 2 | map, addr | RPC `eth_getStorage`, EVM cache | -| `account_map_storage_size` | 2 | map, addr | EVM cache threshold | -| `account_map_storage_root_hash` | 2 | map, addr | Edge / diffs | -| `account_map_storage_root_hashes` | 2 | map, addr | Edge `getaccountroots` | -| `account_map_storage_get_proofs` | 3 | map, addr, key | Edge storage proofs | +| `account_map_root_hash` | 1 | resource | `Chain.State.hash/1` | +| `account_map_state_roots` | 1 | resource | 544-byte `<>`; Edge `getstateroots` | | `account_map_size` | 1 | resource | `CAccountMap.size/1` | -| `account_map_to_list` | 1 | resource | `CAccountMap.to_list/1` | +| `account_map_to_list` | 1 | resource | `CAccountMap.to_list/1`, RPC account dumps | | `account_map_difference_full` | 2 | two maps | `Chain.State.difference/2` | | `account_map_apply_difference` | 2 | map, delta list | `Chain.State.apply_difference/2` | -| `account_map_compact` | 1 | account map resource | Dirty CPU; returns `%{addr => %Account{...}}` compact map (read-only; OK frozen) | +| `account_map_compact` | 1 | account map | Dirty CPU; compact map for DB (OK frozen) | | `account_map_uncompact_state` | 1 | compact or resource | Returns `{am, hash}` | +| `account_map_storage_put_map` | 2 | map, update list | EVM `su` hot path | +| `account_map_storage` | 3 | map, addr, spec | `{:get,k}` \| `{:range,k,n}` \| `:list` \| `:size` | +| `account_map_storage_roots` | 2 | map, addr | 544-byte `<>` | +| `account_map_proof` | 2 | map, addr | Account inclusion proof | +| `account_map_proof` | 3 | map, addr, key | Storage proof | + +### Test/dev only (`-DCMERKLE_TEST_NIFS`; on unless `MIX_ENV=prod`) -**Frozen map:** `account_map_lock/1` sets map-level `frozen` only. Map mutations (`put`/`put_meta`/`delete`/`apply_difference`/`storage_put_map`) fail while frozen. `account_map_get` / `to_list` export storage root hashes (never live tries), so Elixir cannot mutate map-owned storage via bare `CMerkleTree.insert`. `clone/1` forks writable unlocked wrappers for sync and speculative RPC/Edge/Shell. +Bare `merkletree` resource API (`new`, `insert_item_raw`, `get_item`, `get_range_raw`, `get_proofs_raw`, `difference_raw`, `lock`, `to_list`, `import_map`, `root_hash`, `hash`, `root_hashes_raw`, `bucket_count`, `size`, `clone`) plus debug (`struct_sizes_raw`, `memory_stats_raw`, `malloc_info_raw`). Used by ExUnit, fuzz, and stress scripts. Prod release builds omit these from `nif_funcs`. + +**Frozen map:** `account_map_lock/1` sets map-level `frozen` only. Map mutations (`put`/`delete`/`apply_difference`/`storage_put_map`) fail while frozen. `get` / `to_list` export hashes only. Internal `state_trie` is never exported. `clone/1` forks writable wrappers for sync and speculative RPC/Edge/Shell. **Trust:** Erlang validates some shapes (e.g. `to_bytes32`), but the NIF must treat all binaries and terms as hostile (size, allocation, scheduler impact). @@ -77,21 +61,21 @@ | F-5 | **Interaction `LockedStates::mtx` vs tree mutexes** | Medium | CWE-833 | **Fixed:** `enter_lock` pins `has_clone` on local/canonical under global+tree lock, drops global before `switch_local_to_canonical`, and map entries hold a `has_clone` ref. `difference_raw` snapshots pointers under global, bumps `read_pins` (not `has_clone`), releases global, then acquires dual tree locks. `leave_lock` erases map entries under global, releases global, then detaches under tree lock only. | | F-6 | **`make_writeable` COW under `Lock` RAII** | High | CWE-667 | **Fixed:** COW now transfers the held mutex (unlock old `SharedState`, lock new) instead of leaving `Lock` holding a destroyed mutex while mutating a forked tree. | | F-7 | **`leave_lock` / canonical map UAF** | High | CWE-416 | **Fixed:** Map erase drops the map's `has_clone` ref; canonical pointers are re-validated before switch; `SharedState` is not deleted while referenced from the dedup map. | -| F-7b | **Abandoned `SharedState` after canonical switch** | High | CWE-404 | **Fixed:** orphan reclaim path for standalone `CMerkleTree.lock` / `difference_raw`. `account_map_lock` is `frozen`-only (get no longer exports live storage). Monitor via `nif_stats_raw/0`. | +| F-7b | **Abandoned `SharedState` after canonical switch** | High | CWE-404 | **Fixed:** orphan reclaim for standalone `CMerkleTree.lock` / `difference_raw` (test NIFs). `account_map_lock` is `frozen`-only. Monitor via `nif_stats_raw/0`. | | F-6 | **Global `locked_states` / `stats_mutex` on upgrade** | Low | CWE-665 | `on_reload`/`on_upgrade` no-op; hot upgrade could leave stale globals. Acceptable if NIF not hot-reloaded. | ### Information disclosure / introspection | ID | Topic | Severity | CWE | Notes | |----|--------|----------|-----|--------| -| F-7 | **`malloc_info_raw`** | Low (info disclosure) | CWE-200 | Exposes glibc allocator XML; useful for debugging, aids heap fingerprinting. Restrict in production if threat model requires. | -| F-8 | **`struct_sizes_raw` / `memory_stats_raw`** | Low | CWE-200 | Exposes struct sizes and node/pair counts; aids exploit planning. Same as F-7. | +| F-7 | **`malloc_info_raw`** | Low (info disclosure) | CWE-200 | Test/dev NIF only. Restrict in production if threat model requires. | +| F-8 | **`struct_sizes_raw` / `memory_stats_raw`** | Low | CWE-200 | Test/dev NIF only. | ### Denial of service | ID | Topic | Severity | CWE | Notes | |----|--------|----------|-----|--------| -| F-9 | **Unbounded work per NIF** | Medium | CWE-400 | Long-running exports use dirty schedulers: **CPU-bound** — `get_proofs_raw`, `difference_raw`, `to_list`, `import_map`, `count_zeros`, `memory_stats_raw`, `clone`, `account_map_clone`, `account_map_lock`, `account_map_to_list`, `account_map_difference_full`, `account_map_apply_difference`, `account_map_storage_put_map`, `account_map_storage_to_list`, `account_map_storage_get_proofs`, `account_map_get_proofs`, `account_map_compact`, `account_map_uncompact_state`; **IO-bound** — `malloc_info_raw`. `account_map_put`/`put_meta`/`delete`/`storage_get*` stay on normal schedulers where short. Large dirty-NIF loops call `enif_consume_timeslice` every 512 iterations. Ensure adequate dirty CPU schedulers at runtime (`+SDcpu` on heavy sync nodes). | +| F-9 | **Unbounded work per NIF** | Medium | CWE-400 | Long-running exports use dirty schedulers: **CPU-bound** — `account_map_clone`, `lock`, `to_list`, `difference_full`, `apply_difference`, `storage_put_map`, `storage`, `proof`, `compact`, `uncompact_state`, `count_zeros`; test-only bare `difference_raw` / `to_list` / etc. Large dirty-NIF loops call `enif_consume_timeslice` every 512 iterations. Ensure adequate dirty CPU schedulers (`+SDcpu`). | ### Memory safety (manual review) @@ -134,32 +118,23 @@ | **cppcheck** | Not installed | Install `cppcheck` package for CI (`cppcheck --enable=all`). | | **scan-build** | Not installed | Install `clang-tools` / use `clang --analyze` as substitute. | | **clang-tidy** | Not in PATH | Add `run-clang-tidy` or IDE integration with `bugprone-*`, `cert-*`. | -| **ASan+UBSan+LSan NIF** | Built | `make nif CXXFLAGS='... -fsanitize=address,undefined,leak ...'` links; **loading under full `mix test` without `--no-start` failed** (app boot + ASan runtime interaction). Use targeted tests or `mix test --no-start` with sanitizer NIF if extended validation is needed. | -| **Valgrind** (`memcheck`) on `c_src/test` | Run | 0 invalid access errors; large “definitely lost” from harness not freeing final tree (test artifact). | -| **libFuzzer** | Run | `make fuzz_sha` — `fuzz_sha.cpp` + `sha.cpp`, 5000 runs, no crash. | -| **readelf** | Run | `GNU_RELRO` present; `BIND_NOW` not set — use `-Wl,-z,relro,-z,now` for full hardening if desired. | +| **ASan+UBSan+LSan NIF** | Built | Targeted harnesses preferred over full `mix test` under ASan. | +| **Valgrind** (`memcheck`) on `c_src/test` | Run | 0 invalid access errors; harness may leave intentional trees. | +| **libFuzzer** | Run | `make fuzz_sha` — no crash in 5000 runs. | +| **readelf** | Run | `GNU_RELRO` present; consider `-Wl,-z,relro,-z,now` for full hardening. | --- ## 5. Build / hardening recommendations - **Release flags:** Consider `-fstack-protector-strong`, `-D_FORTIFY_SOURCE=2`, `-Wl,-z,relro,-z,now` for `merkletree_nif.so`. -- **Reproducibility:** `-march=native` in `OPTS` ties binaries to CPU; use generic `-march=x86-64` (or equivalent) for release artifacts if needed. +- **Reproducibility:** `-march=native` in `OPTS` ties binaries to CPU; use generic `-march=x86-64` for release artifacts if needed. - **Debug:** `DEBUG` / `MERKLE_DEBUG_POOL` — avoid in production builds. -- **Sanitizer CI:** Job that builds NIF with sanitizers and runs `mix test test/cmerkletree_test.exs --no-start`. - ---- - -## 6. Code changes made during review - -1. **`merkletree_import_map`:** iterator destroyed on all error paths (`import_badarg`). -2. **`merkletree_difference`:** consistent lock ordering by `SharedState*` address. -3. **`fuzz_sha.cpp` + Makefile `fuzz_sha` target** for ongoing fuzzing of SHA. +- **Test NIFs:** Bare-tree / debug exports are compiled into `nif_funcs` only with `-DCMERKLE_TEST_NIFS` (non-prod by default). --- -## 7. Residual risk +## 6. Residual risk - No automated CodeQL/Semgrep rules in-repo; recommend adding CI. -- Full BEAM + ASan NIF requires runtime tuning (`ASAN_OPTIONS`, possibly `LD_PRELOAD`); use native harnesses for sanitizer depth. - Proof term decoding (`enif_binary_to_term`) remains a trust-boundary if proofs are ever deserialized from untrusted network bytes without verification. diff --git a/c_src/nif.cpp b/c_src/nif.cpp index c9e52df..afc7c51 100644 --- a/c_src/nif.cpp +++ b/c_src/nif.cpp @@ -43,8 +43,6 @@ static volatile int shared_states = 0; static volatile int resources = 0; static int locked_states_cnt = 0; static int orphan_shared_states = 0; -static int lazy_clone_count = 0; -static int eager_clone_count = 0; class LockedStates; static LockedStates* locked_states; @@ -107,6 +105,10 @@ struct merkletree { static merkletree *empty_storage_tree = nullptr; static merkletree *alloc_merkletree_resource(); +static bool term_is_nil(ErlNifEnv *env, ERL_NIF_TERM term); +static bool term_is_atom_named(ErlNifEnv *env, ERL_NIF_TERM term, const char *name); +static ERL_NIF_TERM make_tree_roots_blob(ErlNifEnv *env, Tree &tree); +static merkletree *storage_from_kv_list(ErlNifEnv *env, ERL_NIF_TERM list); class Lock { ErlNifMutex *mtx; @@ -344,6 +346,8 @@ struct accountmap { SharedAccountMap *shared; }; +static AccountEntry &ensure_account_entry(SharedAccountMap *shared, const uint160_t &addr); + class AccountMapLock { ErlNifMutex *mtx; public: @@ -963,6 +967,14 @@ static void switch_local_to_canonical(merkletree *mt, SharedState *local, Shared } } +/* Bare-tree / debug NIF entry points stay compiled always; only nif_funcs[] is gated. + * Mark unused when CMERKLE_TEST_NIFS is off so -Wunused-function stays clean in prod. */ +#ifndef CMERKLE_TEST_NIFS +#define CMERKLE_TEST_NIF __attribute__((unused)) +#else +#define CMERKLE_TEST_NIF +#endif + static ERL_NIF_TERM make_atom(ErlNifEnv *env, const char *atom_name) { @@ -984,7 +996,7 @@ make_binary(ErlNifEnv *env, uint8_t *data, size_t size) } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_new(ErlNifEnv *env, int argc, const ERL_NIF_TERM[] /*argv[]*/) { if (argc != 0) return enif_make_badarg(env); @@ -994,7 +1006,7 @@ merkletree_new(ErlNifEnv *env, int argc, const ERL_NIF_TERM[] /*argv[]*/) return res; } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_clone(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt; @@ -1063,7 +1075,7 @@ static bool insert_binary_terms(ErlNifEnv *env, Tree &tree, ERL_NIF_TERM key_ter return insert_binary_pair(tree, key_binary, value_binary, key_scratch); } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_insert_item(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt; @@ -1137,7 +1149,7 @@ static size_t get_range_entries(Tree &tree, const bin_t &base_key, size_t count, } // namespace -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_get_range(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt; @@ -1171,7 +1183,7 @@ merkletree_get_range(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return list; } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_get_item(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt; @@ -1218,7 +1230,7 @@ make_proof(ErlNifEnv *env, proof_t& proof) } } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_get_proofs(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt; @@ -1235,7 +1247,7 @@ merkletree_get_proofs(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return make_proof(env, proof); } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_to_list(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt; @@ -1256,7 +1268,7 @@ merkletree_to_list(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return list; } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_lock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt; @@ -1266,7 +1278,7 @@ merkletree_lock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return argv[0]; } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_difference(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt1; @@ -1329,7 +1341,7 @@ merkletree_difference(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return list; } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_import_map(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt; @@ -1368,7 +1380,7 @@ merkletree_import_map(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return enif_make_badarg(env); } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_root_hash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt; @@ -1379,7 +1391,7 @@ merkletree_root_hash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return make_binary(env, root_hash.data(), 32); } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_hash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { ErlNifBinary key_binary; @@ -1392,7 +1404,7 @@ merkletree_hash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return make_binary(env, hash.data(), 32); } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_root_hashes(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt; @@ -1403,7 +1415,7 @@ merkletree_root_hashes(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return make_binary(env, (uint8_t*)root_hashes, 32*16); } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_size(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt; @@ -1414,7 +1426,7 @@ merkletree_size(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return enif_make_uint(env, size); } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_bucket_count(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt; @@ -1425,7 +1437,7 @@ merkletree_bucket_count(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return enif_make_uint(env, size); } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_struct_sizes(ErlNifEnv *env, int argc, const ERL_NIF_TERM /*argv*/[]) { if (argc != 0) { @@ -1439,7 +1451,7 @@ merkletree_struct_sizes(ErlNifEnv *env, int argc, const ERL_NIF_TERM /*argv*/[]) enif_make_uint64(env, MERKLE_STRIPE_SIZE)); } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_memory_stats(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { merkletree *mt; @@ -1473,14 +1485,9 @@ merkletree_nif_stats(ErlNifEnv *env, int argc, const ERL_NIF_TERM /*argv*/[]) int shared = 0; int res = 0; - int lazy = 0; - int eager = 0; - enif_mutex_lock(stats_mutex); shared = shared_states; res = resources; - lazy = lazy_clone_count; - eager = eager_clone_count; enif_mutex_unlock(stats_mutex); if (locked_states != nullptr) { @@ -1494,13 +1501,10 @@ merkletree_nif_stats(ErlNifEnv *env, int argc, const ERL_NIF_TERM /*argv*/[]) ERL_NIF_TERM orphans_term = enif_make_int(env, orphans); ERL_NIF_TERM shared_term = enif_make_int(env, shared); ERL_NIF_TERM resources_term = enif_make_int(env, res); - ERL_NIF_TERM lazy_term = enif_make_int(env, lazy); - ERL_NIF_TERM eager_term = enif_make_int(env, eager); - return enif_make_tuple6(env, locked_term, orphans_term, shared_term, resources_term, - lazy_term, eager_term); + return enif_make_tuple4(env, locked_term, orphans_term, shared_term, resources_term); } -static ERL_NIF_TERM +static ERL_NIF_TERM CMERKLE_TEST_NIF merkletree_malloc_info(ErlNifEnv *env, int argc, const ERL_NIF_TERM /*argv*/[]) { if (argc != 0) { @@ -1564,9 +1568,6 @@ account_map_clone(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) accountmap *clone = (accountmap*)enif_alloc_resource(accountmap_type, sizeof(accountmap)); clone->shared = new_shared; - enif_mutex_lock(stats_mutex); - eager_clone_count++; - enif_mutex_unlock(stats_mutex); ERL_NIF_TERM res = enif_make_resource(env, clone); enif_release_resource(clone); return res; @@ -1687,7 +1688,57 @@ static void remove_state_trie_entry(SharedAccountMap *shared, const uint160_t &a } static ERL_NIF_TERM -account_map_state_root_hashes(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +make_tree_roots_blob(ErlNifEnv *env, Tree &tree) +{ + uint8_t buf[32 + 32 * 16]; + // root_hashes updates the tree once; root_hash then reuses clean hashes. + uint256_t *hashes = tree.root_hashes(); + uint256_t root = tree.root_hash(); + memcpy(buf, root.data(), 32); + memcpy(buf + 32, hashes, 32 * 16); + return make_binary(env, buf, sizeof(buf)); +} + +static merkletree * +storage_from_kv_list(ErlNifEnv *env, ERL_NIF_TERM list) +{ + merkletree *mt = alloc_merkletree_resource(); + bin_t key_scratch; + size_t i = 0; + ERL_NIF_TERM head, tail = list; + bool ok = true; + + { + Lock lock(mt); + while (ok && enif_get_list_cell(env, tail, &head, &tail)) { + i++; + const ERL_NIF_TERM *elems; + int arity; + if (!enif_get_tuple(env, head, &arity, &elems) || arity != 2) { + ok = false; + break; + } + + ErlNifBinary key_bin, value_bin; + if (!enif_inspect_binary(env, elems[0], &key_bin) || key_bin.size != 32 || + !enif_inspect_binary(env, elems[1], &value_bin) || value_bin.size != 32 || + !insert_binary_pair(mt->shared_state->tree, key_bin, value_bin, key_scratch)) { + ok = false; + break; + } + nif_loop_progress(env, i); + } + } + + if (!ok) { + enif_release_resource(mt); + return nullptr; + } + return mt; +} + +static ERL_NIF_TERM +account_map_state_roots(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { accountmap *am; if (argc != 1) return enif_make_badarg(env); @@ -1695,8 +1746,7 @@ account_map_state_root_hashes(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[ AccountMapLock lock(am); Lock tree_lock(am->shared->state_trie); - auto root_hashes = am->shared->state_trie->shared_state->tree.root_hashes(); - return make_binary(env, (uint8_t*)root_hashes, 32*16); + return make_tree_roots_blob(env, am->shared->state_trie->shared_state->tree); } static ERL_NIF_TERM @@ -1726,7 +1776,6 @@ account_map_put(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) uint160_t addr; ErlNifUInt64 nonce; uint256_t balance; - merkletree *storage; bin_t code; if (argc != 6) return enif_make_badarg(env); @@ -1734,22 +1783,58 @@ account_map_put(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); if (!enif_get_uint64(env, argv[2], &nonce)) return enif_make_badarg(env); if (!get_balance_uint256(env, argv[3], balance)) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[4], merkletree_type, (void **)&storage)) return enif_make_badarg(env); if (!get_code(env, argv[5], code)) return enif_make_badarg(env); + bool keep_meta = false; + bool allocated = false; + merkletree *storage = nullptr; + ERL_NIF_TERM storage_term = argv[4]; + + if (enif_get_resource(env, storage_term, merkletree_type, (void **)&storage)) { + // replace storage with provided merkle tree resource + } else if (term_is_atom_named(env, storage_term, "keep")) { + keep_meta = true; + } else if (term_is_nil(env, storage_term) || + (enif_is_list(env, storage_term) && enif_is_empty_list(env, storage_term))) { + storage = alloc_merkletree_resource(); + allocated = true; + } else if (enif_is_list(env, storage_term)) { + storage = storage_from_kv_list(env, storage_term); + if (storage == nullptr) return enif_make_badarg(env); + allocated = true; + } else { + return enif_make_badarg(env); + } + AccountMapLock lock(am); - if (!make_writeable_accountmap(am)) return enif_make_badarg(env); + if (!make_writeable_accountmap(am)) { + if (allocated) enif_release_resource(storage); + return enif_make_badarg(env); + } + + if (keep_meta) { + AccountEntry &entry = ensure_account_entry(am->shared, addr); + entry.nonce = (uint64_t)nonce; + entry.balance = balance; + entry.code = code; + + AccountHashCtx hash_ctx; + update_state_trie_for_entry(am->shared, addr, entry, hash_ctx); + return argv[0]; + } auto it = am->shared->accounts.find(addr); if (it != am->shared->accounts.end()) { release_entry_storage(it->second); keep_storage_in_map(storage); + if (allocated) enif_release_resource(storage); it->second.nonce = (uint64_t)nonce; it->second.balance = balance; it->second.storage = storage; it->second.code = code; } else { keep_storage_in_map(storage); + if (allocated) enif_release_resource(storage); AccountEntry entry; entry.nonce = (uint64_t)nonce; entry.balance = balance; @@ -2114,14 +2199,19 @@ account_map_difference_full(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) static bool map_get_atom(ErlNifEnv *env, ERL_NIF_TERM map, const char *key, ERL_NIF_TERM &out); -static bool term_is_nil(ErlNifEnv *env, ERL_NIF_TERM term) +static bool term_is_atom_named(ErlNifEnv *env, ERL_NIF_TERM term, const char *name) { if (!enif_is_atom(env, term)) { return false; } - char atom[16]; + char atom[64]; return enif_get_atom(env, term, atom, sizeof(atom), ERL_NIF_LATIN1) && - strcmp(atom, "nil") == 0; + strcmp(atom, name) == 0; +} + +static bool term_is_nil(ErlNifEnv *env, ERL_NIF_TERM term) +{ + return term_is_atom_named(env, term, "nil"); } static bool balance_equals_term(ErlNifEnv *env, const uint256_t &actual, ERL_NIF_TERM term) @@ -2429,20 +2519,9 @@ account_map_storage_put_map(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) } static ERL_NIF_TERM -account_map_storage_get(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +account_map_storage_get_helper(ErlNifEnv *env, accountmap *am, const uint160_t &addr, + const ErlNifBinary &key_binary) { - accountmap *am; - uint160_t addr; - ErlNifBinary key_binary; - - if (argc != 3) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); - if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); - if (!enif_inspect_binary(env, argv[2], &key_binary) || key_binary.size != 32) { - return enif_make_badarg(env); - } - - AccountMapLock lock(am); auto it = am->shared->accounts.find(addr); if (it == am->shared->accounts.end()) { return make_atom(env, "nil"); @@ -2458,23 +2537,9 @@ account_map_storage_get(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) } static ERL_NIF_TERM -account_map_storage_get_range(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +account_map_storage_get_range_helper(ErlNifEnv *env, accountmap *am, const uint160_t &addr, + const ErlNifBinary &key_binary, unsigned count) { - accountmap *am; - uint160_t addr; - ErlNifBinary key_binary; - unsigned count; - - if (argc != 4) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); - if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); - if (!enif_inspect_binary(env, argv[2], &key_binary) || key_binary.size != 32) { - return enif_make_badarg(env); - } - if (!enif_get_uint(env, argv[3], &count)) return enif_make_badarg(env); - if (count < 1 || count > 256) return enif_make_badarg(env); - - AccountMapLock lock(am); auto it = am->shared->accounts.find(addr); if (it == am->shared->accounts.end()) { return enif_make_list(env, 0); @@ -2502,16 +2567,8 @@ account_map_storage_get_range(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[ } static ERL_NIF_TERM -account_map_storage_to_list(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +account_map_storage_to_list_helper(ErlNifEnv *env, accountmap *am, const uint160_t &addr) { - accountmap *am; - uint160_t addr; - - if (argc != 2) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); - if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); - - AccountMapLock lock(am); auto it = am->shared->accounts.find(addr); if (it == am->shared->accounts.end()) { return enif_make_list(env, 0); @@ -2535,16 +2592,8 @@ account_map_storage_to_list(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) } static ERL_NIF_TERM -account_map_storage_size(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +account_map_storage_size_helper(ErlNifEnv *env, accountmap *am, const uint160_t &addr) { - accountmap *am; - uint160_t addr; - - if (argc != 2) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); - if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); - - AccountMapLock lock(am); auto it = am->shared->accounts.find(addr); if (it == am->shared->accounts.end()) { return enif_make_uint(env, 0); @@ -2556,31 +2605,55 @@ account_map_storage_size(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) } static ERL_NIF_TERM -account_map_storage_root_hash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +account_map_storage(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { accountmap *am; uint160_t addr; - if (argc != 2) return enif_make_badarg(env); + if (argc != 3) return enif_make_badarg(env); if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); + ERL_NIF_TERM spec = argv[2]; AccountMapLock lock(am); - auto it = am->shared->accounts.find(addr); - if (it == am->shared->accounts.end()) { - Lock tree_lock(empty_storage_tree); - uint256_t root = empty_storage_tree->shared_state->tree.root_hash(); - return make_binary(env, root.data(), 32); + + if (term_is_atom_named(env, spec, "list")) { + return account_map_storage_to_list_helper(env, am, addr); + } + if (term_is_atom_named(env, spec, "size")) { + return account_map_storage_size_helper(env, am, addr); } - merkletree *mt = materialize_storage(it->second); - Lock tree_lock(mt); - uint256_t root = mt->shared_state->tree.root_hash(); - return make_binary(env, root.data(), 32); + const ERL_NIF_TERM *elems; + int arity; + if (!enif_get_tuple(env, spec, &arity, &elems)) { + return enif_make_badarg(env); + } + + if (arity == 2 && term_is_atom_named(env, elems[0], "get")) { + ErlNifBinary key_binary; + if (!enif_inspect_binary(env, elems[1], &key_binary) || key_binary.size != 32) { + return enif_make_badarg(env); + } + return account_map_storage_get_helper(env, am, addr, key_binary); + } + + if (arity == 3 && term_is_atom_named(env, elems[0], "range")) { + ErlNifBinary key_binary; + unsigned count; + if (!enif_inspect_binary(env, elems[1], &key_binary) || key_binary.size != 32) { + return enif_make_badarg(env); + } + if (!enif_get_uint(env, elems[2], &count)) return enif_make_badarg(env); + if (count < 1 || count > 256) return enif_make_badarg(env); + return account_map_storage_get_range_helper(env, am, addr, key_binary, count); + } + + return enif_make_badarg(env); } static ERL_NIF_TERM -account_map_storage_root_hashes(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +account_map_storage_roots(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { accountmap *am; uint160_t addr; @@ -2593,29 +2666,36 @@ account_map_storage_root_hashes(ErlNifEnv *env, int argc, const ERL_NIF_TERM arg auto it = am->shared->accounts.find(addr); if (it == am->shared->accounts.end()) { Lock tree_lock(empty_storage_tree); - auto root_hashes = empty_storage_tree->shared_state->tree.root_hashes(); - return make_binary(env, (uint8_t*)root_hashes, 32 * 16); + return make_tree_roots_blob(env, empty_storage_tree->shared_state->tree); } merkletree *mt = materialize_storage(it->second); Lock tree_lock(mt); - auto root_hashes = mt->shared_state->tree.root_hashes(); - return make_binary(env, (uint8_t*)root_hashes, 32 * 16); + return make_tree_roots_blob(env, mt->shared_state->tree); } static ERL_NIF_TERM -account_map_storage_get_proofs(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +account_map_proof(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { accountmap *am; uint160_t addr; - ErlNifBinary key_binary; - if (argc != 3) return enif_make_badarg(env); + if (argc != 2 && argc != 3) return enif_make_badarg(env); if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); - if (!enif_inspect_binary(env, argv[2], &key_binary)) return enif_make_badarg(env); AccountMapLock lock(am); + + if (argc == 2) { + bin_t key(addr.value, addr.value + 20); + Lock tree_lock(am->shared->state_trie); + proof_t proof = am->shared->state_trie->shared_state->tree.get_proofs(key); + return make_proof(env, proof); + } + + ErlNifBinary key_binary; + if (!enif_inspect_binary(env, argv[2], &key_binary)) return enif_make_badarg(env); + merkletree *mt; auto it = am->shared->accounts.find(addr); if (it == am->shared->accounts.end()) { @@ -2630,52 +2710,6 @@ account_map_storage_get_proofs(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv return make_proof(env, proof); } -static ERL_NIF_TERM -account_map_get_proofs(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - accountmap *am; - uint160_t addr; - - if (argc != 2) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); - if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); - - AccountMapLock lock(am); - bin_t key(addr.value, addr.value + 20); - Lock tree_lock(am->shared->state_trie); - proof_t proof = am->shared->state_trie->shared_state->tree.get_proofs(key); - return make_proof(env, proof); -} - -static ERL_NIF_TERM -account_map_put_meta(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - accountmap *am; - uint160_t addr; - ErlNifUInt64 nonce; - uint256_t balance; - bin_t code; - - if (argc != 5) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], accountmap_type, (void **)&am)) return enif_make_badarg(env); - if (!get_address(env, argv[1], addr)) return enif_make_badarg(env); - if (!enif_get_uint64(env, argv[2], &nonce)) return enif_make_badarg(env); - if (!get_balance_uint256(env, argv[3], balance)) return enif_make_badarg(env); - if (!get_code(env, argv[4], code)) return enif_make_badarg(env); - - AccountMapLock lock(am); - if (!make_writeable_accountmap(am)) return enif_make_badarg(env); - - AccountEntry &entry = ensure_account_entry(am->shared, addr); - entry.nonce = (uint64_t)nonce; - entry.balance = balance; - entry.code = code; - - AccountHashCtx hash_ctx; - update_state_trie_for_entry(am->shared, addr, entry, hash_ctx); - return argv[0]; -} - struct UncompactLoopScratch { AccountHashCtx hash_ctx; bin_t code_buf; @@ -2977,7 +3011,7 @@ static bool make_compact_account_term(ErlNifEnv *env, AccountEntry &entry, sha(entry.code.data(), entry.code.size(), code_hash.data()); } - // Shape matches Chain.State.compact/1 / Account.compact/1 and parse_compact_account: + // Shape matches Chain.State.compact / CAccountMap.compact and parse_compact_account: // %Chain.Account{nonce, balance, storage_root, code, map_backed: false} // plus :root_hash and :code_hash. ERL_NIF_TERM keys[8]; @@ -3174,6 +3208,8 @@ static int on_upgrade(ErlNifEnv* /*env*/, void** /*priv*/, void** /*old_priv_dat } static ErlNifFunc nif_funcs[] = { +#ifdef CMERKLE_TEST_NIFS + /* Bare-tree + debug NIFs (dev/test/scripts). Omitted from prod MIX_ENV=prod. */ {"new", 0, merkletree_new, 0}, {"insert_item_raw", 3, merkletree_insert_item, 0}, {"get_item", 2, merkletree_get_item, 0}, @@ -3189,20 +3225,20 @@ static ErlNifFunc nif_funcs[] = { {"bucket_count", 1, merkletree_bucket_count, 0}, {"size", 1, merkletree_size, 0}, {"clone", 1, merkletree_clone, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"count_zeros", 1, merkletree_count_zeros, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"struct_sizes_raw", 0, merkletree_struct_sizes, 0}, {"memory_stats_raw", 1, merkletree_memory_stats, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"malloc_info_raw", 0, merkletree_malloc_info, ERL_NIF_DIRTY_JOB_IO_BOUND}, +#endif + {"count_zeros", 1, merkletree_count_zeros, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"nif_stats_raw", 0, merkletree_nif_stats, 0}, {"account_map_new", 0, account_map_new, 0}, {"account_map_clone", 1, account_map_clone, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_lock", 1, account_map_lock, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_get", 2, account_map_get, 0}, {"account_map_put", 6, account_map_put, 0}, - {"account_map_put_meta", 5, account_map_put_meta, 0}, {"account_map_delete", 2, account_map_delete, 0}, {"account_map_root_hash", 1, account_map_root_hash, 0}, - {"account_map_state_root_hashes", 1, account_map_state_root_hashes, 0}, + {"account_map_state_roots", 1, account_map_state_roots, 0}, {"account_map_size", 1, account_map_size, 0}, {"account_map_to_list", 1, account_map_to_list, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_difference_full", 2, account_map_difference_full, ERL_NIF_DIRTY_JOB_CPU_BOUND}, @@ -3210,14 +3246,10 @@ static ErlNifFunc nif_funcs[] = { {"account_map_compact", 1, account_map_compact, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_uncompact_state", 1, account_map_uncompact_state, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"account_map_storage_put_map", 2, account_map_storage_put_map, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"account_map_storage_get", 3, account_map_storage_get, 0}, - {"account_map_storage_get_range", 4, account_map_storage_get_range, 0}, - {"account_map_storage_to_list", 2, account_map_storage_to_list, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"account_map_storage_size", 2, account_map_storage_size, 0}, - {"account_map_storage_root_hash", 2, account_map_storage_root_hash, 0}, - {"account_map_storage_root_hashes", 2, account_map_storage_root_hashes, 0}, - {"account_map_storage_get_proofs", 3, account_map_storage_get_proofs, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"account_map_get_proofs", 2, account_map_get_proofs, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"account_map_storage", 3, account_map_storage, 0}, + {"account_map_storage_roots", 2, account_map_storage_roots, 0}, + {"account_map_proof", 2, account_map_proof, ERL_NIF_DIRTY_JOB_CPU_BOUND}, + {"account_map_proof", 3, account_map_proof, ERL_NIF_DIRTY_JOB_CPU_BOUND}, }; // ERL_NIF_INIT(merkletree_nif, nif_funcs, on_load, on_reload, on_upgrade, NULL); diff --git a/docs/caccount-map-nif.md b/docs/caccount-map-nif.md index 9ea7cf7..d21416d 100644 --- a/docs/caccount-map-nif.md +++ b/docs/caccount-map-nif.md @@ -10,33 +10,32 @@ See also [`c_src/LOCK_ORDER.md`](../c_src/LOCK_ORDER.md) and - `Chain.State` is backed by `CAccountMap`. Account storage tries and the state root trie live in C++. Elixir does **not** carry a separate `:store` field. -- Prefer map-owned storage APIs over bare `merkletree` resources: - - `Chain.State.storage_get/3`, `storage_put_map/2`, `storage_to_list/2`, +- Prefer map-owned storage APIs — never bare `merkletree` resources in production: + - `Chain.State.storage_value/3`, `storage_put_map/2`, `storage_to_list/2`, `storage_get_proofs/3`, `storage_root_hash/2`, `state_root_hashes/1` - - `CAccountMap` mirrors of the same + - `CAccountMap` mirrors of the same (thin wrappers over merged NIFs) - `Chain.State.hash/1` or `CAccountMap.root_hash/1` for the state root. Internal `state_trie` is never exported; Edge uses `state_root_hashes/1`. - `account_map_get` / `to_list` return `{nonce, balance, storage_root_hash_bin32, code}` — the third element is a **32-byte hash**, never a live storage resource. -- `account_map_compact/1` (dirty CPU) returns `%{addr => %Chain.Account{...}}` with - `storage_root: nil | {MapMerkleTree, [], items}`, `:root_hash`, `:code_hash`, and - `map_backed: false` — one NIF call for `Chain.State.compact/1` (no per-account - `to_list` / `storage_to_list` round-trips). Prefer listing compact_storage slots - without materializing live tries when possible. +- `account_map_put/6` storage arg: resource | `:keep` (meta-only) | `nil`/`[]` | + `[{key32, value32}]` (genesis / hardfork). +- `account_map_storage/3` covers get / range / list / size via one NIF. +- `account_map_storage_roots/2` and `account_map_state_roots/1` return + `<>`. +- `account_map_proof/2` (account) and `/3` (storage key). +- `account_map_compact/1` — one NIF for `Chain.State.compact/1`. - Map-backed `%Chain.Account{}` values set `map_backed: true` (via - `Account.from_parts/4` with a 32-byte root hash) and carry `:root_hash` for - hash caching. Do not use presence of `:root_hash` as a discriminator — - compact DB snapshots also cache `:root_hash` with `map_backed: false`. - Standalone tries (`map_backed: false`) are only for genesis / hardfork / - import via `account_map_put/6`. + `Account.from_parts/4` with a 32-byte root hash) and carry `:root_hash`. + Storage is accessed only through `State.storage_*` / `CAccountMap.storage_*`. + Genesis uses `genesis_storage/0` + `State.storage_put_map/2`. ## Clone and lock - `Chain.State.clone/1` forks a writable map. Use it after `Chain.State.lock/1` (cached peak / block sync) and for RPC / EdgeV2 / Shell speculative execution. - `lock/1` sets map-level `frozen` only. Mutations - (`put` / `put_meta` / `delete` / `apply_difference` / `storage_put_map`) reject - frozen maps. + (`put` / `delete` / `apply_difference` / `storage_put_map`) reject frozen maps. - `Chain.Transaction.apply/3` mutates state in place on an unlocked candidate. - `Chain.State` is **mutable**: always `clone/1` before applying transactions on a shared cached state. @@ -44,5 +43,9 @@ See also [`c_src/LOCK_ORDER.md`](../c_src/LOCK_ORDER.md) and ## Build - NIF: `mix compile` → `elixir_make` → `c_src/` → `priv/merkletree_nif.so` +- **Prod** (`MIX_ENV=prod`): ~21 map + misc exports only. +- **Dev/test** (non-prod): also registers bare-tree + debug NIFs via + `-DCMERKLE_TEST_NIFS` (override with `CMERKLE_TEST_NIFS=0|1`). Mode stamp + forces rebuild when the flag changes. - EVM binary: `evm/evm` (needs `libboost-dev`) - `deps/libsecp256k1`: build once with `make -C deps/libsecp256k1/` (not via `mix`) diff --git a/lib/caccount_map.ex b/lib/caccount_map.ex index 34916e3..ecb3dc5 100644 --- a/lib/caccount_map.ex +++ b/lib/caccount_map.ex @@ -16,10 +16,12 @@ defmodule CAccountMap do def root_hash(map), do: CMerkleTree.account_map_root_hash(map) - def state_root_hashes(map), - do: decode_root_hashes(CMerkleTree.account_map_state_root_hashes(map)) + def state_root_hashes(map) do + {_root, hashes} = split_roots(CMerkleTree.account_map_state_roots(map)) + hashes + end - def get_proofs(map, <<_::160>> = addr), do: CMerkleTree.account_map_get_proofs(map, addr) + def get_proofs(map, <<_::160>> = addr), do: CMerkleTree.account_map_proof(map, addr) def get(map, <<_::160>> = addr) do case CMerkleTree.account_map_get(map, addr) do @@ -31,7 +33,7 @@ defmodule CAccountMap do def get_account(map, addr) do case get(map, addr) do :undefined -> nil - entry -> account_from_parts(entry) + {nonce, balance, root_hash, code} -> Account.from_parts(nonce, balance, root_hash, code) end end @@ -40,7 +42,7 @@ defmodule CAccountMap do end def put_meta(map, <<_::160>> = addr, nonce, balance, code) do - CMerkleTree.account_map_put_meta(map, addr, nonce, encode_balance(balance), code) + put(map, addr, nonce, balance, :keep, code) end def put_account(map, <<_::160>> = addr, %Account{} = account) do @@ -87,7 +89,7 @@ defmodule CAccountMap do end def storage_get(map, <<_::160>> = addr, key) do - case CMerkleTree.account_map_storage_get(map, addr, to_bytes32(key)) do + case CMerkleTree.account_map_storage(map, addr, {:get, to_bytes32(key)}) do nil -> nil @null -> nil value -> value @@ -96,25 +98,28 @@ defmodule CAccountMap do def storage_get_range(map, <<_::160>> = addr, key, count) when is_integer(count) and count >= 1 and count <= 256 do - CMerkleTree.account_map_storage_get_range(map, addr, to_bytes32(key), count) + CMerkleTree.account_map_storage(map, addr, {:range, to_bytes32(key), count}) |> Enum.map(fn {k, v} -> {k, if(v == @null, do: nil, else: v)} end) end def storage_to_list(map, <<_::160>> = addr), - do: CMerkleTree.account_map_storage_to_list(map, addr) + do: CMerkleTree.account_map_storage(map, addr, :list) def storage_size(map, <<_::160>> = addr), - do: CMerkleTree.account_map_storage_size(map, addr) + do: CMerkleTree.account_map_storage(map, addr, :size) - def storage_root_hash(map, <<_::160>> = addr), - do: CMerkleTree.account_map_storage_root_hash(map, addr) + def storage_root_hash(map, <<_::160>> = addr) do + {root, _hashes} = split_roots(CMerkleTree.account_map_storage_roots(map, addr)) + root + end def storage_root_hashes(map, <<_::160>> = addr) do - decode_root_hashes(CMerkleTree.account_map_storage_root_hashes(map, addr)) + {_root, hashes} = split_roots(CMerkleTree.account_map_storage_roots(map, addr)) + hashes end def storage_get_proofs(map, <<_::160>> = addr, key), - do: CMerkleTree.account_map_storage_get_proofs(map, addr, to_bytes(key)) + do: CMerkleTree.account_map_proof(map, addr, to_bytes(key)) def difference_full(map_a, map_b) do CMerkleTree.account_map_difference_full(map_a, map_b) @@ -140,10 +145,6 @@ defmodule CAccountMap do {nonce, decode_balance(balance), root_hash, code} end - defp account_from_parts({nonce, balance, root_hash, code}) do - Account.from_parts(nonce, balance, root_hash, code) - end - defp encode_balance(balance) when is_integer(balance) and balance >= 0 do if balance <= 0xFFFFFFFFFFFFFFFF do balance @@ -171,6 +172,10 @@ defmodule CAccountMap do defp to_bytes(string) when is_binary(string), do: string defp to_bytes(int) when is_integer(int), do: to_bytes32(int) + defp split_roots(<>) do + {root, decode_root_hashes(hashes)} + end + defp decode_root_hashes( <>, code) do %Chain.Account{ @@ -65,11 +57,6 @@ defmodule Chain.Account do } end - @deprecated "Use CMerkleTree.clone/1 on Account.tree/1 instead" - def clone(%Chain.Account{} = acc) do - %Chain.Account{acc | storage_root: CMerkleTree.clone(tree(acc))} - end - def put_tree(%Chain.Account{} = acc, root) do %{acc | storage_root: root, map_backed: false} |> Map.delete(:root_hash) @@ -78,82 +65,22 @@ defmodule Chain.Account do def root_hash(%Chain.Account{storage_root: nil} = acc) do case Map.get(acc, :root_hash) do <<_::binary-size(32)>> = hash -> hash - _ -> CMerkleTree.root_hash(CMerkleTree.new()) + _ -> @empty_storage_root end end - def root_hash(%Chain.Account{} = acc) do - CMerkleTree.root_hash(tree(acc)) - end - - def uncompact(%Chain.Account{storage_root: nil} = acc) do - %Chain.Account{acc | storage_root: CMerkleTree.new(), map_backed: false} + def root_hash(%Chain.Account{storage_root: root}) when is_reference(root) do + CMerkleTree.root_hash(root) end - def uncompact(%Chain.Account{storage_root: {MapMerkleTree, _opts, items}} = acc) - when is_map(items) do - storage_root = CMerkleTree.from_map(items) - %Chain.Account{acc | storage_root: storage_root, map_backed: false} - end - - def uncompact(%Chain.Account{storage_root: items} = acc) when is_list(items) do - %Chain.Account{acc | storage_root: CMerkleTree.from_list(items), map_backed: false} - end - - def compact(%Chain.Account{map_backed: true}) do - raise ArgumentError, "map-backed accounts are compacted via Chain.State.compact/1" - end - - def compact(%Chain.Account{} = acc) do - tree = tree(acc) - - if CMerkleTree.size(tree) == 0 do - %Chain.Account{acc | storage_root: nil, map_backed: false} - else - %Chain.Account{ - acc - | storage_root: {MapMerkleTree, [], Map.new(CMerkleTree.to_list(tree))}, - map_backed: false - } - end - |> Map.put(:root_hash, CMerkleTree.root_hash(tree)) - |> Map.put(:code_hash, codehash(acc)) - end - - def storage_set_value(%Chain.Account{map_backed: true}, <<_::256>>, <<_::256>>) do - raise ArgumentError, - "map-backed account storage writes go through Chain.State.storage_put_map/2" - end - - def storage_set_value(%Chain.Account{} = acc, key = <<_k::256>>, value = <<_v::256>>) do - store = CMerkleTree.insert(tree(acc), key, value) - - %{acc | storage_root: store, map_backed: false} - |> Map.delete(:root_hash) - end - - def storage_set_value(acc, key, value) when is_integer(key) do - storage_set_value(acc, <>, value) - end - - def storage_set_value(acc, key, value) when is_integer(value) do - storage_set_value(acc, key, <>) - end - - @spec storage_value(Chain.Account.t(), binary() | integer()) :: binary() | nil - def storage_value(acc, key) when is_integer(key) do - storage_value(acc, <>) - end - - def storage_value(%Chain.Account{map_backed: true}, key) when is_binary(key) do - raise ArgumentError, - "map-backed account storage reads go through Chain.State.storage_value/3" - end + def root_hash(%Chain.Account{} = acc) do + case Map.get(acc, :root_hash) do + <<_::binary-size(32)>> = hash -> + hash - def storage_value(%Chain.Account{} = acc, key) when is_binary(key) do - case CMerkleTree.get(tree(acc), key) do - nil -> <<0::unsigned-size(256)>> - bin -> bin + _ -> + raise ArgumentError, + "compact account missing :root_hash; use Chain.State.storage_root_hash/2" end end diff --git a/lib/chain/genesis_factory.ex b/lib/chain/genesis_factory.ex index 0476462..e2508ad 100644 --- a/lib/chain/genesis_factory.ex +++ b/lib/chain/genesis_factory.ex @@ -21,9 +21,15 @@ defmodule Chain.GenesisFactory do end def genesis_state(accounts) do - Enum.reduce(accounts, Chain.State.new(), fn {address, user_account}, state -> - Chain.State.set_account(state, address, user_account) - end) + state = + Enum.reduce(accounts, Chain.State.new(), fn {address, user_account}, state -> + Chain.State.set_account(state, address, user_account) + end) + + case ChainDefinition.genesis_storage() do + storage when map_size(storage) == 0 -> state + storage -> Chain.State.storage_put_map(state, storage) + end end @spec genesis_accounts() :: [{binary(), Account.t()}] diff --git a/lib/chain/state.ex b/lib/chain/state.ex index 1935f9b..669921e 100644 --- a/lib/chain/state.ex +++ b/lib/chain/state.ex @@ -86,23 +86,19 @@ defmodule Chain.State do CAccountMap.get_account(accounts, id) end + def normalize_address(<<_::160>> = id), do: id + def normalize_address(id) when is_integer(id), do: <> + def normalize_address(id), do: Wallet.address!(id) + @spec ensure_account(Chain.State.t(), <<_::160>> | Wallet.t() | non_neg_integer()) :: Chain.Account.t() - def ensure_account(state = %Chain.State{}, id = <<_::160>>) do - case account(state, id) do + def ensure_account(state = %Chain.State{}, id) do + case account(state, normalize_address(id)) do nil -> Chain.Account.new(nonce: 0) acc -> acc end end - def ensure_account(state = %Chain.State{}, id) when is_integer(id) do - ensure_account(state, <>) - end - - def ensure_account(state = %Chain.State{}, id) do - ensure_account(state, Wallet.address!(id)) - end - @spec set_account(Chain.State.t(), binary(), Chain.Account.t()) :: Chain.State.t() def set_account(state, id = <<_::160>>, account) do accounts = CAccountMap.put_account(state.accounts, id, account) @@ -203,11 +199,22 @@ defmodule Chain.State do map = BertInt.decode!(bin) Enum.reduce(map, new(), fn {id, acc}, state -> + data = Map.get(acc, :data) + + storage = + cond do + data in [nil, [], %{}] -> nil + is_map(data) -> Map.to_list(data) + is_list(data) -> data + true -> raise ArgumentError, "unsupported account data: #{inspect(data)}" + end + set_account(state, id, %Chain.Account{ nonce: acc.nonce, balance: acc.balance, - storage_root: CMerkleTree.new() |> CMerkleTree.insert_items(acc.data), - code: acc.code + storage_root: storage, + code: acc.code, + map_backed: false }) end) end diff --git a/lib/chaindefinition.ex b/lib/chaindefinition.ex index f0fe7a7..ac13204 100644 --- a/lib/chaindefinition.ex +++ b/lib/chaindefinition.ex @@ -70,6 +70,17 @@ defmodule ChainDefinition do chain_definition().genesis_accounts() end + @spec genesis_storage() :: %{binary() => %{binary() => binary()}} + def genesis_storage() do + mod = chain_definition() + + if function_exported?(mod, :genesis_storage, 0) do + mod.genesis_storage() + else + %{} + end + end + @spec genesis_transactions(Wallet.t()) :: [Chain.Transaction.t()] def genesis_transactions(miner) do chain_definition().genesis_transactions(miner) diff --git a/lib/chaindefinition/devnet.ex b/lib/chaindefinition/devnet.ex index 34c566a..e1da2ee 100644 --- a/lib/chaindefinition/devnet.ex +++ b/lib/chaindefinition/devnet.ex @@ -36,25 +36,22 @@ defmodule ChainDefinition.Devnet do addr_balance(0xCECA2F8CF1983B4CF0C1BA51FD382C2BC37ABA58, ether(50_000)), addr_balance(0x68E0BAFDDA9EF323F692FC080D612718C941D120, ether(50_000)), - # The Registry with the accountant placed + # The Registry (storage applied via genesis_storage/0) addr_account( Diode.registry_address(), Account.new( balance: ether(100_000_000), code: Contract.Registry.test_code() ) - |> Account.storage_set_value(1, accountant) ), - # The Fleet with the operator and accountant placed + # The Fleet (storage applied via genesis_storage/0) addr_account( Diode.fleet_address(), Account.new( balance: 0, code: Contract.Fleet.code() ) - |> Account.storage_set_value(0, Diode.registry_address() |> :binary.decode_unsigned()) - |> Account.storage_set_value(2, accountant) ) ] @@ -64,6 +61,27 @@ defmodule ChainDefinition.Devnet do end) end + @doc """ + Initial contract storage slots for genesis accounts. + Keys and values are 32-byte binaries for `Chain.State.storage_put_map/2`. + """ + @spec genesis_storage() :: %{binary() => %{binary() => binary()}} + def genesis_storage() do + accountant = Hash.to_bytes32(0x96CDE043E986040CB13FFAFD80EB8CEAC196FB84) + registry = Diode.registry_address() + fleet = Diode.fleet_address() + + %{ + registry => %{ + Hash.to_bytes32(1) => accountant + }, + fleet => %{ + Hash.to_bytes32(0) => Hash.to_bytes32(registry), + Hash.to_bytes32(2) => accountant + } + } + end + @spec genesis_transactions(Wallet.t()) :: [Chain.Transaction.t()] def genesis_transactions(miner) do # Call "blockReward()": diff --git a/lib/chaindefinition/voyager.ex b/lib/chaindefinition/voyager.ex index 6d311df..7cbef62 100644 --- a/lib/chaindefinition/voyager.ex +++ b/lib/chaindefinition/voyager.ex @@ -1,5 +1,5 @@ defmodule ChainDefinition.Voyager do - alias Chain.{State, Account} + alias Chain.State def apply(%State{} = state) do Enum.reduce(patch(), state, fn account, state -> @@ -7,19 +7,17 @@ defmodule ChainDefinition.Voyager do code = Base16.decode(account["code"]) {balance, ""} = Integer.parse(account["balance"]) - # Field-only update via put_meta (storage_root: nil → put_meta). - acc = + acc0 = state |> State.ensure_account(id) |> Map.put(:balance, balance) |> Map.put(:code, code) - |> Map.put(:storage_root, nil) |> Map.put(:map_backed, false) |> Map.delete(:root_hash) - state = State.set_account(state, id, acc) - if Map.has_key?(account, "state_patch") do + state = State.set_account(state, id, %{acc0 | storage_root: nil}) + updates = Map.new(account["state_patch"], fn [key, value] -> {Base16.decode(key), Base16.decode(value)} @@ -31,18 +29,12 @@ defmodule ChainDefinition.Voyager do State.storage_put_map(state, %{id => updates}) end else - # Full storage replace via a caller-owned standalone trie. - tree = - Enum.reduce(account["state"] || [], CMerkleTree.new(), fn [key, value], tree -> - CMerkleTree.insert(tree, Base16.decode(key), Base16.decode(value)) + slots = + Enum.map(account["state"] || [], fn [key, value] -> + {Base16.decode(key), Base16.decode(value)} end) - State.set_account(state, id, %Account{ - nonce: acc.nonce, - balance: balance, - storage_root: tree, - code: code - }) + State.set_account(state, id, %{acc0 | storage_root: slots}) end end) end diff --git a/lib/cmerkletree.ex b/lib/cmerkletree.ex index 9f27e5a..4e86e1a 100644 --- a/lib/cmerkletree.ex +++ b/lib/cmerkletree.ex @@ -116,12 +116,12 @@ defmodule CMerkleTree do def account_map_new(), do: error() def account_map_clone(_map), do: error() def account_map_root_hash(_map), do: error() - def account_map_state_root_hashes(_map), do: error() - def account_map_get_proofs(_map, _addr), do: error() + def account_map_state_roots(_map), do: error() + def account_map_proof(_map, _addr), do: error() + def account_map_proof(_map, _addr, _key), do: error() def account_map_lock(_map), do: error() def account_map_get(_map, _addr), do: error() def account_map_put(_map, _addr, _nonce, _balance, _storage, _code), do: error() - def account_map_put_meta(_map, _addr, _nonce, _balance, _code), do: error() def account_map_delete(_map, _addr), do: error() def account_map_size(_map), do: error() def account_map_to_list(_map), do: error() @@ -130,13 +130,8 @@ defmodule CMerkleTree do def account_map_compact(_map), do: error() def account_map_uncompact_state(_map), do: error() def account_map_storage_put_map(_map, _updates), do: error() - def account_map_storage_get(_map, _addr, _key), do: error() - def account_map_storage_get_range(_map, _addr, _key, _count), do: error() - def account_map_storage_to_list(_map, _addr), do: error() - def account_map_storage_size(_map, _addr), do: error() - def account_map_storage_root_hash(_map, _addr), do: error() - def account_map_storage_root_hashes(_map, _addr), do: error() - def account_map_storage_get_proofs(_map, _addr, _key), do: error() + def account_map_storage(_map, _addr, _spec), do: error() + def account_map_storage_roots(_map, _addr), do: error() defp struct_sizes_raw, do: error() defp memory_stats_raw(_tree), do: error() diff --git a/lib/contract/registry.ex b/lib/contract/registry.ex index 6e3667b..08828e9 100644 --- a/lib/contract/registry.ex +++ b/lib/contract/registry.ex @@ -7,52 +7,34 @@ defmodule Contract.Registry do as needed by the inner workings of the chain """ - def miner_value_slot(key, blockRef) do - BlockProcess.with_account(blockRef, Diode.registry_address(), fn account -> - staked_a = - account - |> Chain.Account.storage_value(hash_slot_binary(5, key)) - |> :binary.decode_unsigned() - - staked_b = - account - |> Chain.Account.storage_value(hash_slot_binary(5, key) |> badd(2)) - |> :binary.decode_unsigned() + def miner_value_slot(key, blockRef), do: value_slot(5, key, blockRef) - unstaked_a = - account - |> Chain.Account.storage_value(hash_slot_binary(5, key) |> badd(3)) - |> :binary.decode_unsigned() + def contract_value_slot(key, blockRef), do: value_slot(6, key, blockRef) - unstaked_b = - account - |> Chain.Account.storage_value(hash_slot_binary(5, key) |> badd(5)) - |> :binary.decode_unsigned() + defp value_slot(base, key, blockRef) do + registry = Diode.registry_address() - {staked_a + staked_b, unstaked_a + unstaked_b} - end) - end + BlockProcess.with_state(blockRef, fn state -> + slot = hash_slot_binary(base, key) - def contract_value_slot(key, blockRef) do - BlockProcess.with_account(blockRef, Diode.registry_address(), fn account -> staked_a = - account - |> Chain.Account.storage_value(hash_slot_binary(6, key)) + state + |> Chain.State.storage_value(registry, slot) |> :binary.decode_unsigned() staked_b = - account - |> Chain.Account.storage_value(hash_slot_binary(6, key) |> badd(2)) + state + |> Chain.State.storage_value(registry, slot |> badd(2)) |> :binary.decode_unsigned() unstaked_a = - account - |> Chain.Account.storage_value(hash_slot_binary(6, key) |> badd(3)) + state + |> Chain.State.storage_value(registry, slot |> badd(3)) |> :binary.decode_unsigned() unstaked_b = - account - |> Chain.Account.storage_value(hash_slot_binary(6, key) |> badd(5)) + state + |> Chain.State.storage_value(registry, slot |> badd(5)) |> :binary.decode_unsigned() {staked_a + staked_b, unstaked_a + unstaked_b} diff --git a/lib/network/status.ex b/lib/network/status.ex index 523a102..fe77a7d 100644 --- a/lib/network/status.ex +++ b/lib/network/status.ex @@ -8,7 +8,7 @@ defmodule Network.Status do @run_queue_warn 1_000 def summary do - {locked, orphans, shared_states, nif_resources, _lazy, _eager} = CMerkleTree.nif_stats() + {locked, orphans, shared_states, nif_resources} = CMerkleTree.nif_stats() memory = :erlang.memory() run_queue = Diode.run_queue_total() diff --git a/lib/shell.ex b/lib/shell.ex index 20b152a..a2d34bf 100644 --- a/lib/shell.ex +++ b/lib/shell.ex @@ -120,8 +120,7 @@ defmodule Shell do def get_slot(address, slot) do Chain.with_peak_state(fn state -> - Chain.State.ensure_account(state, address) - |> Chain.Account.storage_value(slot) + Chain.State.storage_value(state, Chain.State.normalize_address(address), slot) end) end diff --git a/scripts/cmerkle_fuzz.exs b/scripts/cmerkle_fuzz.exs index 7a44e89..556a396 100644 --- a/scripts/cmerkle_fuzz.exs +++ b/scripts/cmerkle_fuzz.exs @@ -354,13 +354,32 @@ defmodule CMerkleFuzz do defp slot(i), do: <> + defp compact_live(%Account{storage_root: tree} = acc) when is_reference(tree) do + items = Map.new(CMerkleTree.to_list(tree)) + + %Account{ + acc + | storage_root: if(map_size(items) == 0, do: nil, else: {MapMerkleTree, [], items}), + map_backed: false + } + |> Map.put(:root_hash, CMerkleTree.root_hash(tree)) + |> Map.put(:code_hash, Account.codehash(acc)) + end + defp build_compact_accounts(n) do for i <- 1..n, into: %{} do tree = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - acc = %Account{nonce: i, balance: i * 1_000, storage_root: tree, code: <>} - {addr(i), Account.compact(acc)} + acc = %Account{ + nonce: i, + balance: i * 1_000, + storage_root: tree, + code: <>, + map_backed: false + } + + {addr(i), compact_live(acc)} end end @@ -1020,7 +1039,7 @@ defmodule CMerkleFuzz do baseline = Process.get(:cmerkle_fuzz_baseline_rss, 0) rss = read_proc_rss_kb() delta = rss - baseline - {_locked, orphans, _shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() + {_locked, orphans, _shared, _res} = CMerkleTree.nif_stats() if orphans > 0 or delta > max_delta_kb do IO.puts(:stderr, "FUZZ_RSS_FAIL round=#{round} delta_kb=#{delta} orphans=#{orphans}") diff --git a/scripts/cmerkle_heap_assumptions.exs b/scripts/cmerkle_heap_assumptions.exs index 907e1ac..f643c4b 100644 --- a/scripts/cmerkle_heap_assumptions.exs +++ b/scripts/cmerkle_heap_assumptions.exs @@ -283,7 +283,7 @@ defmodule CMerkleHeapAssumptions do end) _ = CAccountMap.lock(map) - {_locked, orphans, _shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() + {_locked, orphans, _shared, _res} = CMerkleTree.nif_stats() if orphans > 0, do: raise("scenario I orphans=#{orphans}") :ok end) @@ -313,7 +313,7 @@ defmodule CMerkleHeapAssumptions do CAccountMap.put(map, <<1::unsigned-size(160)>>, nonce + 1, balance, storage, code) end) - {_l0, orphans0, shared0, _r0, _, _} = CMerkleTree.nif_stats() + {_l0, orphans0, shared0, _r0} = CMerkleTree.nif_stats() if orphans0 > 0, do: raise("scenario J initial orphans=#{orphans0}") Enum.each(1..max(div(r, 2), 30), fn _ -> @@ -321,7 +321,7 @@ defmodule CMerkleHeapAssumptions do :erlang.garbage_collect() end) - {_l1, orphans1, shared1, _r1, _, _} = CMerkleTree.nif_stats() + {_l1, orphans1, shared1, _r1} = CMerkleTree.nif_stats() if orphans1 > 0, do: raise("scenario J orphans=#{orphans1}") if shared1 - shared0 > 800, do: raise("scenario J shared_states growth #{shared1 - shared0}") :ok diff --git a/scripts/cmerkle_leak_test.exs b/scripts/cmerkle_leak_test.exs index 0a5ea49..37955cc 100644 --- a/scripts/cmerkle_leak_test.exs +++ b/scripts/cmerkle_leak_test.exs @@ -70,7 +70,7 @@ defmodule CMerkleLeakTest do Enum.reduce(1..plateau, {baseline, 0}, fn window, {last, rising} -> run_workload(id, rounds, accounts) force_gc() - {locked, orphans, shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() + {locked, orphans, shared, _res} = CMerkleTree.nif_stats() rss = measure_rss() delta = rss - baseline diff --git a/scripts/cmerkle_parallel_stress.exs b/scripts/cmerkle_parallel_stress.exs index 12518f4..c589ea1 100644 --- a/scripts/cmerkle_parallel_stress.exs +++ b/scripts/cmerkle_parallel_stress.exs @@ -499,13 +499,32 @@ defmodule CMerkleParallelStress do defp slot(i), do: <> + defp compact_live(%Account{storage_root: tree} = acc) when is_reference(tree) do + items = Map.new(CMerkleTree.to_list(tree)) + + %Account{ + acc + | storage_root: if(map_size(items) == 0, do: nil, else: {MapMerkleTree, [], items}), + map_backed: false + } + |> Map.put(:root_hash, CMerkleTree.root_hash(tree)) + |> Map.put(:code_hash, Account.codehash(acc)) + end + defp build_compact_accounts(n) do for i <- 1..n, into: %{} do tree = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - acc = %Account{nonce: i, balance: i * 1_000, storage_root: tree, code: <>} - {addr(i), Account.compact(acc)} + acc = %Account{ + nonce: i, + balance: i * 1_000, + storage_root: tree, + code: <>, + map_backed: false + } + + {addr(i), compact_live(acc)} end end @@ -904,7 +923,7 @@ defmodule CMerkleParallelStress do ) |> Stream.run() - {_locked, orphans, _shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() + {_locked, orphans, _shared, _res} = CMerkleTree.nif_stats() if orphans > 0 do raise("P16 pending orphans=#{orphans}") diff --git a/test/chain_account_hash_nif_test.exs b/test/chain_account_hash_nif_test.exs index 16282a2..56ca439 100644 --- a/test/chain_account_hash_nif_test.exs +++ b/test/chain_account_hash_nif_test.exs @@ -20,7 +20,27 @@ defmodule ChainAccountHashNifTest do {slot(i), val(i)} ]) - %Account{nonce: i, balance: i * 1_000, storage_root: tree, code: <>} + %Account{nonce: i, balance: i * 1_000, storage_root: tree, code: <>, map_backed: false} + end + + defp compact_live(%Account{storage_root: tree} = acc) when is_reference(tree) do + items = Map.new(CMerkleTree.to_list(tree)) + + %Account{ + acc + | storage_root: if(map_size(items) == 0, do: nil, else: {MapMerkleTree, [], items}), + map_backed: false + } + |> Map.put(:root_hash, CMerkleTree.root_hash(tree)) + |> Map.put(:code_hash, Account.codehash(acc)) + end + + defp hash_from_live(%Account{storage_root: tree} = acc) when is_reference(tree) do + Account.hash(%{acc | map_backed: false} |> Map.put(:root_hash, CMerkleTree.root_hash(tree))) + end + + defp hash_from_compact(%Account{} = acc) do + Account.hash(acc) end test "uncompact_state account hashes match Account.hash/1 for multi-slot storage and large code" do @@ -34,20 +54,20 @@ defmodule ChainAccountHashNifTest do {slot(2), val(2)}, {slot(10), val(10)} ]), - code: :binary.copy(<<0xCD>>, 1024) + code: :binary.copy(<<0xCD>>, 1024), + map_backed: false } - compact = - %{ - addr(1) => multi_slot |> Account.compact(), - addr(2) => sample_account(2) |> Account.compact() - } + compact = %{ + addr(1) => compact_live(multi_slot), + addr(2) => compact_live(sample_account(2)) + } {accounts, hash} = CAccountMap.uncompact_state(compact) elixir_hashes = compact - |> Enum.map(fn {id, acc} -> {id, Account.hash(Account.uncompact(acc))} end) + |> Enum.map(fn {id, acc} -> {id, hash_from_compact(acc)} end) |> Map.new() nif_hashes = @@ -64,14 +84,14 @@ defmodule ChainAccountHashNifTest do test "uncompact_state account hashes match Account.hash/1" do compact = for i <- 1..8, into: %{} do - {addr(i), sample_account(i) |> Account.compact()} + {addr(i), compact_live(sample_account(i))} end {accounts, hash} = CAccountMap.uncompact_state(compact) elixir_hashes = compact - |> Enum.map(fn {id, acc} -> {id, Account.hash(Account.uncompact(acc))} end) + |> Enum.map(fn {id, acc} -> {id, hash_from_compact(acc)} end) |> Map.new() nif_hashes = @@ -123,7 +143,7 @@ defmodule ChainAccountHashNifTest do test "uncompact_state uses compact root_hash when present" do compact = for i <- 1..4, into: %{} do - {addr(i), sample_account(i) |> Account.compact()} + {addr(i), compact_live(sample_account(i))} end assert Enum.all?(compact, fn {_id, acc} -> Map.has_key?(acc, :root_hash) end) @@ -133,7 +153,7 @@ defmodule ChainAccountHashNifTest do elixir_hashes = compact - |> Enum.map(fn {id, acc} -> {id, Account.hash(Account.uncompact(acc))} end) + |> Enum.map(fn {id, acc} -> {id, hash_from_compact(acc)} end) |> Map.new() nif_hashes = @@ -149,23 +169,21 @@ defmodule ChainAccountHashNifTest do test "uncompact_state falls back without compact root_hash field" do acc = sample_account(1) - tree = Account.tree(acc) + tree = acc.storage_root legacy_account = %Chain.Account{ nonce: acc.nonce, balance: acc.balance, storage_root: {MapMerkleTree, [], Map.new(CMerkleTree.to_list(tree))}, - code: acc.code + code: acc.code, + map_backed: false } legacy_compact = %{addr(1) => legacy_account} {accounts, hash} = CAccountMap.uncompact_state(legacy_compact) - expected_hash = - legacy_account - |> Account.uncompact() - |> Account.hash() + expected_hash = hash_from_live(acc) expected_root = %{addr(1) => expected_hash} @@ -179,24 +197,23 @@ defmodule ChainAccountHashNifTest do test "uncompact_state falls back without compact code_hash field" do acc = sample_account(1) - tree = Account.tree(acc) + tree = acc.storage_root legacy_account = - acc - |> Map.from_struct() - |> Map.put(:storage_root, {MapMerkleTree, [], Map.new(CMerkleTree.to_list(tree))}) - |> Map.put(:root_hash, Account.root_hash(acc)) - |> Map.delete(:code_hash) - |> then(&struct(Chain.Account, &1)) + %Chain.Account{ + nonce: acc.nonce, + balance: acc.balance, + storage_root: {MapMerkleTree, [], Map.new(CMerkleTree.to_list(tree))}, + code: acc.code, + map_backed: false + } + |> Map.put(:root_hash, CMerkleTree.root_hash(tree)) legacy_compact = %{addr(1) => legacy_account} {accounts, hash} = CAccountMap.uncompact_state(legacy_compact) - expected_hash = - legacy_account - |> Account.uncompact() - |> Account.hash() + expected_hash = hash_from_compact(legacy_account) expected_root = %{addr(1) => expected_hash} @@ -214,7 +231,7 @@ defmodule ChainAccountHashNifTest do end end - test "State.uncompact matches Elixir tree/1 hash" do + test "State.uncompact matches original state hash" do original = State.new() |> then(fn st -> diff --git a/test/chain_state_merkle_test.exs b/test/chain_state_merkle_test.exs index bdc824c..58a2315 100644 --- a/test/chain_state_merkle_test.exs +++ b/test/chain_state_merkle_test.exs @@ -6,7 +6,7 @@ # (matches persisted block delta replay in Model.ChainSql.do_state/1). # # Heavy cases mirror production: Evm.process_updates uses 32-byte key/value maps and -# CMerkleTree.insert_items/2; State.clone/1 + Account.clone/1 share trie pools (COW); +# CMerkleTree.insert_items/2; State.clone/1 shares trie pools (COW); # sequential <> slots maximize shared key prefixes (trie depth). defmodule ChainStateMerkleTest do # NIF uses process-global allocators / stripe pools; run sequentially to avoid cross-test races. @@ -54,9 +54,12 @@ defmodule ChainStateMerkleTest do end defp account_with_storage(%Account{} = acc, pairs) do - Enum.reduce(pairs, acc, fn {k, v}, a -> - Account.storage_set_value(a, k, v) - end) + tree = + Enum.reduce(pairs, CMerkleTree.new(), fn {k, v}, t -> + CMerkleTree.insert(t, k, v) + end) + + Account.put_tree(acc, tree) end defp put_account(%State{} = st, i, acc), do: State.set_account(st, addr(i), acc) @@ -324,9 +327,11 @@ defmodule ChainStateMerkleTest do State.new() |> put_account( 1, - Enum.reduce(0..500, account_u256_slots(0..130), fn i, a -> - Account.storage_set_value(a, slot_u256(i), val_u256(i + 99)) - end) + account_with_storage( + Enum.map(0..500, fn i -> + {slot_u256(i), val_u256(i + 99)} + end) + ) ) assert_roundtrip(prev, next) diff --git a/test/chain_state_uncompact_test.exs b/test/chain_state_uncompact_test.exs index 992ca10..699047b 100644 --- a/test/chain_state_uncompact_test.exs +++ b/test/chain_state_uncompact_test.exs @@ -23,12 +23,35 @@ defmodule ChainStateUncompactTest do {slot(i), val(i)} ]) - %Account{nonce: i, balance: i * 1_000, storage_root: tree, code: <>} + %Account{ + nonce: i, + balance: i * 1_000, + storage_root: tree, + code: <>, + map_backed: false + } end + defp compact_account(%Account{storage_root: root} = acc) when is_reference(root) do + items = Map.new(CMerkleTree.to_list(root)) + + storage = + if map_size(items) == 0 do + nil + else + {MapMerkleTree, [], items} + end + + %Account{acc | storage_root: storage, map_backed: false} + |> Map.put(:root_hash, CMerkleTree.root_hash(root)) + |> Map.put(:code_hash, Account.codehash(acc)) + end + + defp compact_account(%Account{} = acc), do: acc + defp compact_accounts_map(n_accounts) do for i <- 1..n_accounts, into: %{} do - {addr(i), sample_account(i) |> Account.compact()} + {addr(i), sample_account(i) |> compact_account()} end end @@ -156,7 +179,7 @@ defmodule ChainStateUncompactTest do storage_root: nil, code: nil } - |> Account.compact() + |> compact_account() } restored = State.uncompact(%State{accounts: compact}) @@ -186,7 +209,7 @@ defmodule ChainStateUncompactTest do code: <<5>> } - compact = %{addr(1) => Account.compact(multi_slot)} + compact = %{addr(1) => compact_account(multi_slot)} {accounts, _hash} = CAccountMap.uncompact_state(compact) {5, 1_000, root, <<5>>} = CAccountMap.get(accounts, addr(1)) @@ -222,16 +245,16 @@ defmodule ChainStateUncompactTest do |> then(fn st -> elem(CAccountMap.uncompact_state(st.accounts), 0) end) end - test "Account.compact stores code_hash matching Account.codehash/1" do + test "compact_account stores code_hash matching Account.codehash/1" do acc = sample_account(3) |> Map.put(:code, :binary.copy(<<0xEE>>, 256)) - compact = Account.compact(acc) + compact = compact_account(acc) assert compact.code_hash == Account.codehash(acc) assert compact.root_hash == Account.root_hash(acc) end test "put overwrites lazy account without prior get" do - compact = %{addr(1) => sample_account(1) |> Account.compact()} + compact = %{addr(1) => sample_account(1) |> compact_account()} {accounts, _} = CAccountMap.uncompact_state(compact) new_storage = @@ -352,36 +375,24 @@ defmodule ChainStateUncompactTest do state = compact_state(@account_count) assert map_size(state.accounts) == @account_count - {reduce_us, _accounts} = - :timer.tc(fn -> - Enum.reduce(state.accounts, CAccountMap.new(), fn {id, acc}, accounts -> - CAccountMap.put_account(accounts, id, Account.uncompact(acc)) - end) - end) - - {tree_us, _tree} = - :timer.tc(fn -> - state.accounts - |> Enum.map(fn {id, acc} -> {id, Account.hash(Account.uncompact(acc))} end) - |> Map.new() - |> CMerkleTree.from_map() - end) + {nif_us, {accounts, hash}} = + :timer.tc(fn -> CAccountMap.uncompact_state(state.accounts) end) {full_us, restored} = :timer.tc(fn -> State.uncompact(state) end) + assert CAccountMap.size(accounts) == @account_count + assert is_binary(hash) assert CAccountMap.size(restored.accounts) == @account_count assert is_binary(State.hash(restored)) assert is_binary(Chain.State.hash(restored)) - reduce_ms = Float.round(reduce_us / 1000, 1) - tree_ms = Float.round(tree_us / 1000, 1) + nif_ms = Float.round(nif_us / 1000, 1) full_ms = Float.round(full_us / 1000, 1) IO.puts(""" State.uncompact performance (@account_count=#{@account_count}): - reduce+put loop: #{reduce_ms} ms - tree(from_map) estimate: #{tree_ms} ms - full uncompact: #{full_ms} ms + CAccountMap.uncompact_state: #{nif_ms} ms + full State.uncompact: #{full_ms} ms """) assert full_us < 150_000, diff --git a/test/chain_test.exs b/test/chain_test.exs index 022bb9e..ff9d8c6 100644 --- a/test/chain_test.exs +++ b/test/chain_test.exs @@ -263,7 +263,18 @@ defmodule ChainTest do # assert {key, value} == {key, Account.storageInteger(account, key)} # end # result is map-backed (no live trie); compare via State.storage_to_list. - assert storage_list(state, addr) == to_list(Account.tree(account)) + ref_list = + case account do + %Account{storage_root: tree} when is_reference(tree) -> + CMerkleTree.to_list(tree) + |> Enum.map(fn {key, value} -> {compress(key), compress(value)} end) + |> Enum.sort() + + _ -> + [] + end + + assert storage_list(state, addr) == ref_list end end diff --git a/test/cmerkle_lock_clone_regression_test.exs b/test/cmerkle_lock_clone_regression_test.exs index 99e9550..c02ff2c 100644 --- a/test/cmerkle_lock_clone_regression_test.exs +++ b/test/cmerkle_lock_clone_regression_test.exs @@ -21,11 +21,18 @@ defmodule CMerkleLockCloneRegressionTest do defp val(i), do: <> defp sample_account(i) do - Account.new() - |> Account.storage_set_value(slot(i), val(i)) - |> Map.put(:nonce, i) - |> Map.put(:balance, i * 1_000) - |> Map.put(:code, <>) + tree = + CMerkleTree.insert_items(CMerkleTree.new(), [ + {slot(i), val(i)} + ]) + + %Account{ + nonce: i, + balance: i * 1_000, + storage_root: tree, + code: <>, + map_backed: false + } end defp locked_peak_like_state(n \\ 3) do diff --git a/test/cmerkle_nif_deadlock_test.exs b/test/cmerkle_nif_deadlock_test.exs index 20adaff..5b7a6f1 100644 --- a/test/cmerkle_nif_deadlock_test.exs +++ b/test/cmerkle_nif_deadlock_test.exs @@ -374,13 +374,32 @@ defmodule CMerkleNifDeadlockTest do ) end + defp compact_live(%Account{storage_root: tree} = acc) when is_reference(tree) do + items = Map.new(CMerkleTree.to_list(tree)) + + %Account{ + acc + | storage_root: if(map_size(items) == 0, do: nil, else: {MapMerkleTree, [], items}), + map_backed: false + } + |> Map.put(:root_hash, CMerkleTree.root_hash(tree)) + |> Map.put(:code_hash, Account.codehash(acc)) + end + defp build_compact_accounts(n) do for i <- 1..n, into: %{} do tree = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - acc = %Account{nonce: i, balance: i * 1_000, storage_root: tree, code: <>} - {addr(i), Account.compact(acc)} + acc = %Account{ + nonce: i, + balance: i * 1_000, + storage_root: tree, + code: <>, + map_backed: false + } + + {addr(i), compact_live(acc)} end end diff --git a/test/cmerkle_nif_leak_test.exs b/test/cmerkle_nif_leak_test.exs index 3c83482..bd86cbc 100644 --- a/test/cmerkle_nif_leak_test.exs +++ b/test/cmerkle_nif_leak_test.exs @@ -41,7 +41,7 @@ defmodule CMerkleNifLeakTest do end force_gc() - {_locked, orphans, _shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() + {_locked, orphans, _shared, _res} = CMerkleTree.nif_stats() assert orphans <= 2 assert rss_kb() - baseline < 50_000 end @@ -66,7 +66,7 @@ defmodule CMerkleNifLeakTest do end force_gc() - {_locked, orphans, _shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() + {_locked, orphans, _shared, _res} = CMerkleTree.nif_stats() assert orphans == 0 assert rss_kb() - baseline < 80_000 end @@ -85,7 +85,7 @@ defmodule CMerkleNifLeakTest do end force_gc() - {locked, orphans, shared_count, _res, _lazy, _eager} = CMerkleTree.nif_stats() + {locked, orphans, shared_count, _res} = CMerkleTree.nif_stats() assert orphans == 0 assert locked <= 80 assert shared_count < 2000 @@ -119,7 +119,7 @@ defmodule CMerkleNifLeakTest do end force_gc() - {_locked, orphans, _shared, _res, _lazy, _eager} = CMerkleTree.nif_stats() + {_locked, orphans, _shared, _res} = CMerkleTree.nif_stats() assert orphans == 0 assert rss_kb() - baseline < 100_000 end @@ -141,14 +141,14 @@ defmodule CMerkleNifLeakTest do end) end) - {_locked0, _orphans0, shared0, _res0, _, _} = CMerkleTree.nif_stats() + {_locked0, _orphans0, shared0, _res0} = CMerkleTree.nif_stats() for _ <- 1..100 do _ = CAccountMap.difference_full(base, fork) end force_gc() - {_locked, orphans, shared, _res, _, _} = CMerkleTree.nif_stats() + {_locked, orphans, shared, _res} = CMerkleTree.nif_stats() assert orphans == 0 assert shared - shared0 < 500 end diff --git a/test/evm_storage_readahead_test.exs b/test/evm_storage_readahead_test.exs index ca0d7bc..63ee7dd 100644 --- a/test/evm_storage_readahead_test.exs +++ b/test/evm_storage_readahead_test.exs @@ -90,8 +90,7 @@ defmodule EvmStorageReadaheadTest do {:ok, state, %TransactionReceipt{msg: :ok}} = Transaction.apply(ctx, block, state) contract = Transaction.new_contract_address(ctx) - acc = Chain.State.account(state, contract) - assert Chain.Account.storage_value(acc, 0) |> :binary.decode_unsigned() == 0 + assert Chain.State.storage_value(state, contract, 0) |> :binary.decode_unsigned() == 0 tx = %{ @@ -108,7 +107,6 @@ defmodule EvmStorageReadaheadTest do {:ok, state, %TransactionReceipt{msg: :ok}} = Transaction.apply(tx, block, state) - acc = Chain.State.account(state, contract) - assert Chain.Account.storage_value(acc, 0) |> :binary.decode_unsigned() == 1 + assert Chain.State.storage_value(state, contract, 0) |> :binary.decode_unsigned() == 1 end end diff --git a/test/evm_test.exs b/test/evm_test.exs index b945970..3c15f95 100644 --- a/test/evm_test.exs +++ b/test/evm_test.exs @@ -62,9 +62,10 @@ defmodule EvmTest do {:ok, state, %TransactionReceipt{msg: :ok}} = Transaction.apply(ctx, block, state) - # Checking value of i at position 0 - acc = Chain.State.account(state, Transaction.new_contract_address(ctx)) - value = Chain.Account.storage_value(acc, 0) |> :binary.decode_unsigned() + value = + Chain.State.storage_value(state, Transaction.new_contract_address(ctx), 0) + |> :binary.decode_unsigned() + assert value == 0 # Method call increment @@ -91,8 +92,10 @@ defmodule EvmTest do assert evmout == "" # Checking value of i at position 0 - acc = Chain.State.account(state, Transaction.new_contract_address(ctx)) - value = Chain.Account.storage_value(acc, 0) |> :binary.decode_unsigned() + value = + Chain.State.storage_value(state, Transaction.new_contract_address(ctx), 0) + |> :binary.decode_unsigned() + assert value == 1 end From 53cecbfc2d8c1f541d7260f89f4da5ad32b8eda0 Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Thu, 16 Jul 2026 16:44:29 +0200 Subject: [PATCH 07/16] Accept integer addresses in State.storage_* for EVM dialyzer. Normalize ids the same way as ensure_account so cache_account and other EVM paths type-check. Co-authored-by: Cursor --- lib/chain/state.ex | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/chain/state.ex b/lib/chain/state.ex index 669921e..d44a3f6 100644 --- a/lib/chain/state.ex +++ b/lib/chain/state.ex @@ -36,12 +36,12 @@ defmodule Chain.State do CAccountMap.state_root_hashes(accounts) end - def get_proofs(%Chain.State{accounts: accounts}, <<_::160>> = addr) do - CAccountMap.get_proofs(accounts, addr) + def get_proofs(%Chain.State{accounts: accounts}, addr) do + CAccountMap.get_proofs(accounts, normalize_address(addr)) end - def storage_value(%Chain.State{accounts: accounts}, <<_::160>> = addr, key) do - case CAccountMap.storage_get(accounts, addr, key) do + def storage_value(%Chain.State{accounts: accounts}, addr, key) do + case CAccountMap.storage_get(accounts, normalize_address(addr), key) do nil -> <<0::unsigned-size(256)>> bin -> bin end @@ -51,23 +51,23 @@ defmodule Chain.State do %{state | accounts: CAccountMap.storage_put_map(accounts, updates), hash: nil} end - def storage_to_list(%Chain.State{accounts: accounts}, <<_::160>> = addr), - do: CAccountMap.storage_to_list(accounts, addr) + def storage_to_list(%Chain.State{accounts: accounts}, addr), + do: CAccountMap.storage_to_list(accounts, normalize_address(addr)) - def storage_size(%Chain.State{accounts: accounts}, <<_::160>> = addr), - do: CAccountMap.storage_size(accounts, addr) + def storage_size(%Chain.State{accounts: accounts}, addr), + do: CAccountMap.storage_size(accounts, normalize_address(addr)) - def storage_get_range(%Chain.State{accounts: accounts}, <<_::160>> = addr, key, count), - do: CAccountMap.storage_get_range(accounts, addr, key, count) + def storage_get_range(%Chain.State{accounts: accounts}, addr, key, count), + do: CAccountMap.storage_get_range(accounts, normalize_address(addr), key, count) - def storage_root_hash(%Chain.State{accounts: accounts}, <<_::160>> = addr), - do: CAccountMap.storage_root_hash(accounts, addr) + def storage_root_hash(%Chain.State{accounts: accounts}, addr), + do: CAccountMap.storage_root_hash(accounts, normalize_address(addr)) - def storage_root_hashes(%Chain.State{accounts: accounts}, <<_::160>> = addr), - do: CAccountMap.storage_root_hashes(accounts, addr) + def storage_root_hashes(%Chain.State{accounts: accounts}, addr), + do: CAccountMap.storage_root_hashes(accounts, normalize_address(addr)) - def storage_get_proofs(%Chain.State{accounts: accounts}, <<_::160>> = addr, key), - do: CAccountMap.storage_get_proofs(accounts, addr, key) + def storage_get_proofs(%Chain.State{accounts: accounts}, addr, key), + do: CAccountMap.storage_get_proofs(accounts, normalize_address(addr), key) def hash(%Chain.State{hash: nil} = state) do CAccountMap.root_hash(state.accounts) From 8670b8e2233006fb8df9aeb7b47ad04f7d8ebb22 Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Thu, 16 Jul 2026 21:31:28 +0200 Subject: [PATCH 08/16] Remove leftover Account helpers and share Hash.to_bytes32. Drop put_tree and live-tree root_hash, make root_hash a struct field, require genesis_storage/0 on all chains, and route Edge account roots through Account.root_hash. Co-authored-by: Cursor --- c_src/nif.cpp | 25 ++++++----- docs/caccount-map-nif.md | 10 +++-- lib/caccount_map.ex | 10 +---- lib/chain/account.ex | 47 +++++++------------- lib/chaindefinition.ex | 8 +--- lib/chaindefinition/galileo.ex | 2 + lib/chaindefinition/mainnet.ex | 2 + lib/chaindefinition/pioneer.ex | 2 + lib/chaindefinition/stagenet.ex | 2 + lib/chaindefinition/ulysses.ex | 2 + lib/chaindefinition/voyager.ex | 4 +- lib/cmerkletree.ex | 17 +------ lib/hash.ex | 11 +++-- lib/network/edge_v2.ex | 6 +-- scripts/cmerkle_fuzz.exs | 28 +++++------- scripts/cmerkle_parallel_stress.exs | 20 +++------ test/caccount_map_test.exs | 2 +- test/chain_account_hash_nif_test.exs | 60 ++++++++++++------------- test/chain_state_merkle_test.exs | 4 +- test/chain_state_uncompact_test.exs | 61 +++++++++----------------- test/cmerkle_account_map_diff_test.exs | 17 ++++++- test/cmerkle_nif_deadlock_test.exs | 20 +++------ 22 files changed, 152 insertions(+), 208 deletions(-) diff --git a/c_src/nif.cpp b/c_src/nif.cpp index afc7c51..5434813 100644 --- a/c_src/nif.cpp +++ b/c_src/nif.cpp @@ -2836,22 +2836,27 @@ static bool parse_compact_account(ErlNifEnv *env, ERL_NIF_TERM account_term, ERL_NIF_TERM root_hash_term; if (map_get_atom(env, account_term, "root_hash", root_hash_term)) { - ErlNifBinary root_bin; - if (!enif_inspect_binary(env, root_hash_term, &root_bin) || root_bin.size != 32) { - return false; + if (!enif_is_atom(env, root_hash_term)) { + ErlNifBinary root_bin; + if (!enif_inspect_binary(env, root_hash_term, &root_bin) || root_bin.size != 32) { + return false; + } + out.compact_root_hash = (char*)root_bin.data; + out.has_compact_root_hash = true; } - out.compact_root_hash = (char*)root_bin.data; - out.has_compact_root_hash = true; } ERL_NIF_TERM code_hash_term; if (map_get_atom(env, account_term, "code_hash", code_hash_term)) { - ErlNifBinary code_hash_bin; - if (!enif_inspect_binary(env, code_hash_term, &code_hash_bin) || code_hash_bin.size != 32) { - return false; + if (!enif_is_atom(env, code_hash_term)) { + ErlNifBinary code_hash_bin; + if (!enif_inspect_binary(env, code_hash_term, &code_hash_bin) || + code_hash_bin.size != 32) { + return false; + } + out.compact_code_hash = (char*)code_hash_bin.data; + out.has_compact_code_hash = true; } - out.compact_code_hash = (char*)code_hash_bin.data; - out.has_compact_code_hash = true; } ErlNifUInt64 nonce; diff --git a/docs/caccount-map-nif.md b/docs/caccount-map-nif.md index d21416d..8f4b9f9 100644 --- a/docs/caccount-map-nif.md +++ b/docs/caccount-map-nif.md @@ -25,10 +25,12 @@ See also [`c_src/LOCK_ORDER.md`](../c_src/LOCK_ORDER.md) and `<>`. - `account_map_proof/2` (account) and `/3` (storage key). - `account_map_compact/1` — one NIF for `Chain.State.compact/1`. -- Map-backed `%Chain.Account{}` values set `map_backed: true` (via - `Account.from_parts/4` with a 32-byte root hash) and carry `:root_hash`. - Storage is accessed only through `State.storage_*` / `CAccountMap.storage_*`. - Genesis uses `genesis_storage/0` + `State.storage_put_map/2`. +- Map-backed `%Chain.Account{}` values set `map_backed: true` and `root_hash` + (struct field) via `Account.from_parts/4`. Storage is accessed only through + `State.storage_*` / `CAccountMap.storage_*`. Edge `getaccount` / + `getaccountroot` use `Account.root_hash/1` (no extra storage-roots NIF). +- Every chain definition exports `genesis_storage/0` (Devnet has slots; + others return `%{}`). Genesis applies accounts then `State.storage_put_map/2`. ## Clone and lock diff --git a/lib/caccount_map.ex b/lib/caccount_map.ex index ecb3dc5..2c77ee6 100644 --- a/lib/caccount_map.ex +++ b/lib/caccount_map.ex @@ -159,15 +159,7 @@ defmodule CAccountMap do :binary.decode_unsigned(balance) end - defp to_bytes32(nil), do: @null - defp to_bytes32(int) when is_integer(int), do: <> - - defp to_bytes32(string) when is_binary(string) and byte_size(string) < 32 do - missing = (32 - byte_size(string)) * 8 - <<0::unsigned-size(missing), string::binary>> - end - - defp to_bytes32(string) when is_binary(string) and byte_size(string) == 32, do: string + defp to_bytes32(value), do: Hash.to_bytes32(value) defp to_bytes(string) when is_binary(string), do: string defp to_bytes(int) when is_integer(int), do: to_bytes32(int) diff --git a/lib/chain/account.ex b/lib/chain/account.ex index bf3c47c..9f33850 100644 --- a/lib/chain/account.ex +++ b/lib/chain/account.ex @@ -2,10 +2,9 @@ # Copyright 2021-2024 Diode # Licensed under the Diode License, Version 1.1 defmodule Chain.Account do - defstruct nonce: 0, balance: 0, storage_root: nil, code: nil, map_backed: false + defstruct nonce: 0, balance: 0, storage_root: nil, code: nil, map_backed: false, root_hash: nil - # Empty storage trie root for this merkle implementation - # (same as root_hash of a freshly allocated empty storage trie). + # Matches C++ empty storage trie root (root_hash of an empty storage trie). @empty_storage_root Base.decode16!( "438A90405DAA876539082CD0BAF6CDDAA3BF880F1C8AF0C0381F0042DB93088A" ) @@ -13,9 +12,10 @@ defmodule Chain.Account do @type t :: %Chain.Account{ nonce: non_neg_integer(), balance: non_neg_integer(), - storage_root: reference() | nil | {atom(), list(), map()} | list(), + storage_root: nil | reference() | [{binary(), binary()}], code: binary() | nil, - map_backed: boolean() + map_backed: boolean(), + root_hash: binary() | nil } def new(props \\ []) do @@ -34,7 +34,7 @@ defmodule Chain.Account do @doc """ Build an account from `CAccountMap.get/2` parts. A 32-byte `storage` root marks the account map-backed (`map_backed: true`). - Otherwise `storage` is a put payload (resource, slot list, or nil for `:keep`). + Otherwise `storage` is a put payload (`nil` / slot list / resource for `set_account`). """ def from_parts(nonce, balance, <>, code) do %Chain.Account{ @@ -42,9 +42,9 @@ defmodule Chain.Account do balance: balance, storage_root: nil, code: if(code == "", do: nil, else: code), - map_backed: true + map_backed: true, + root_hash: root_hash } - |> Map.put(:root_hash, root_hash) end def from_parts(nonce, balance, storage, code) do @@ -53,35 +53,18 @@ defmodule Chain.Account do balance: balance, storage_root: storage, code: if(code == "", do: nil, else: code), - map_backed: false + map_backed: false, + root_hash: nil } end - def put_tree(%Chain.Account{} = acc, root) do - %{acc | storage_root: root, map_backed: false} - |> Map.delete(:root_hash) - end - - def root_hash(%Chain.Account{storage_root: nil} = acc) do - case Map.get(acc, :root_hash) do - <<_::binary-size(32)>> = hash -> hash - _ -> @empty_storage_root - end - end - - def root_hash(%Chain.Account{storage_root: root}) when is_reference(root) do - CMerkleTree.root_hash(root) - end + def root_hash(%Chain.Account{root_hash: <<_::binary-size(32)>> = hash}), do: hash - def root_hash(%Chain.Account{} = acc) do - case Map.get(acc, :root_hash) do - <<_::binary-size(32)>> = hash -> - hash + def root_hash(%Chain.Account{storage_root: nil}), do: @empty_storage_root - _ -> - raise ArgumentError, - "compact account missing :root_hash; use Chain.State.storage_root_hash/2" - end + def root_hash(%Chain.Account{}) do + raise ArgumentError, + "account missing :root_hash; use Chain.State.storage_root_hash/2" end @spec to_rlp(Chain.Account.t()) :: [...] diff --git a/lib/chaindefinition.ex b/lib/chaindefinition.ex index ac13204..7b9d83a 100644 --- a/lib/chaindefinition.ex +++ b/lib/chaindefinition.ex @@ -72,13 +72,7 @@ defmodule ChainDefinition do @spec genesis_storage() :: %{binary() => %{binary() => binary()}} def genesis_storage() do - mod = chain_definition() - - if function_exported?(mod, :genesis_storage, 0) do - mod.genesis_storage() - else - %{} - end + chain_definition().genesis_storage() end @spec genesis_transactions(Wallet.t()) :: [Chain.Transaction.t()] diff --git a/lib/chaindefinition/galileo.ex b/lib/chaindefinition/galileo.ex index 6263212..50926f4 100644 --- a/lib/chaindefinition/galileo.ex +++ b/lib/chaindefinition/galileo.ex @@ -1,6 +1,8 @@ defmodule ChainDefinition.Galileo do alias Chain.{State, Account} + def genesis_storage(), do: %{} + def apply(%State{} = state) do Enum.reduce(patch(), state, fn account, state -> id = Base16.decode(account["addr"]) diff --git a/lib/chaindefinition/mainnet.ex b/lib/chaindefinition/mainnet.ex index 0697b39..eb6fcb9 100644 --- a/lib/chaindefinition/mainnet.ex +++ b/lib/chaindefinition/mainnet.ex @@ -135,6 +135,8 @@ defmodule ChainDefinition.Mainnet do File.read!("data/genesis.bin") |> Chain.State.from_binary() |> Chain.State.accounts() end + def genesis_storage(), do: %{} + @spec genesis_transactions(Wallet.t()) :: [Chain.Transaction.t()] def genesis_transactions(_miner) do [] diff --git a/lib/chaindefinition/pioneer.ex b/lib/chaindefinition/pioneer.ex index 22c07c6..ab429fc 100644 --- a/lib/chaindefinition/pioneer.ex +++ b/lib/chaindefinition/pioneer.ex @@ -1,6 +1,8 @@ defmodule ChainDefinition.Pioneer do alias Chain.{State, Account} + def genesis_storage(), do: %{} + def apply(%State{} = state) do id = Base16.decode("0x5000000000000000000000000000000000000000") acc = State.ensure_account(state, id) diff --git a/lib/chaindefinition/stagenet.ex b/lib/chaindefinition/stagenet.ex index a8bc2a6..8faaa06 100644 --- a/lib/chaindefinition/stagenet.ex +++ b/lib/chaindefinition/stagenet.ex @@ -83,6 +83,8 @@ defmodule ChainDefinition.Stagenet do File.read!("data/genesis.bin") |> Chain.State.from_binary() |> Chain.State.accounts() end + def genesis_storage(), do: %{} + @spec genesis_transactions(Wallet.t()) :: [Chain.Transaction.t()] def genesis_transactions(_miner) do [] diff --git a/lib/chaindefinition/ulysses.ex b/lib/chaindefinition/ulysses.ex index acf7b41..d320170 100644 --- a/lib/chaindefinition/ulysses.ex +++ b/lib/chaindefinition/ulysses.ex @@ -1,6 +1,8 @@ defmodule ChainDefinition.Ulysses do alias Chain.{State, Account} + def genesis_storage(), do: %{} + def apply(%State{} = state) do Enum.reduce(patch(), state, fn account, state -> id = Base16.decode(account["addr"]) diff --git a/lib/chaindefinition/voyager.ex b/lib/chaindefinition/voyager.ex index 7cbef62..3040e07 100644 --- a/lib/chaindefinition/voyager.ex +++ b/lib/chaindefinition/voyager.ex @@ -1,6 +1,8 @@ defmodule ChainDefinition.Voyager do alias Chain.State + def genesis_storage(), do: %{} + def apply(%State{} = state) do Enum.reduce(patch(), state, fn account, state -> id = Base16.decode(account["addr"]) @@ -13,7 +15,7 @@ defmodule ChainDefinition.Voyager do |> Map.put(:balance, balance) |> Map.put(:code, code) |> Map.put(:map_backed, false) - |> Map.delete(:root_hash) + |> Map.put(:root_hash, nil) if Map.has_key?(account, "state_patch") do state = State.set_account(state, id, %{acc0 | storage_root: nil}) diff --git a/lib/cmerkletree.ex b/lib/cmerkletree.ex index 4e86e1a..3bcdca4 100644 --- a/lib/cmerkletree.ex +++ b/lib/cmerkletree.ex @@ -139,22 +139,7 @@ defmodule CMerkleTree do defp nif_stats_raw, do: error() defp error, do: :erlang.nif_error(:nif_not_loaded) - defp to_bytes32(nil) do - <<0::unsigned-size(256)>> - end - - defp to_bytes32(int) when is_integer(int) do - <> - end - - defp to_bytes32(string) when byte_size(string) < 32 do - missing = (32 - byte_size(string)) * 8 - <<0::unsigned-size(missing), string::binary>> - end - - defp to_bytes32(string) when byte_size(string) == 32 do - string - end + defp to_bytes32(value), do: Hash.to_bytes32(value) defp to_bytes(string) when is_binary(string), do: string defp to_bytes(int) when is_integer(int), do: to_bytes32(int) diff --git a/lib/hash.ex b/lib/hash.ex index 7818a5e..2481bb0 100644 --- a/lib/hash.ex +++ b/lib/hash.ex @@ -7,18 +7,21 @@ defmodule Hash do :binary.decode_unsigned(hash) end + def to_bytes32(nil), do: <<0::unsigned-size(256)>> + def to_bytes32(hash = <<_::256>>) do hash end - def to_bytes32(hash = <<_::160>>) do - <<0::96, hash::binary-size(20)>> - end - def to_bytes32(hash) when is_integer(hash) do <> end + def to_bytes32(bin) when is_binary(bin) and byte_size(bin) < 32 do + missing = (32 - byte_size(bin)) * 8 + <<0::unsigned-size(missing), bin::binary>> + end + def printable(nil) do "nil" end diff --git a/lib/network/edge_v2.ex b/lib/network/edge_v2.ex index a98e24b..a953eae 100644 --- a/lib/network/edge_v2.ex +++ b/lib/network/edge_v2.ex @@ -74,9 +74,9 @@ defmodule Network.EdgeV2 do nil -> error("account does not exist") - %Chain.Account{} -> + account = %Chain.Account{} -> proof = Chain.Block.account_proof(block, id) - root = Chain.Block.account_storage_root_hash(block, id) + root = Chain.Account.root_hash(account) response(root, proof) end end) @@ -98,7 +98,7 @@ defmodule Network.EdgeV2 do %{ nonce: account.nonce, balance: account.balance, - storage_root: Chain.Block.account_storage_root_hash(block, id), + storage_root: Chain.Account.root_hash(account), code: Chain.Account.codehash(account) }, proof diff --git a/scripts/cmerkle_fuzz.exs b/scripts/cmerkle_fuzz.exs index 556a396..8a36bf6 100644 --- a/scripts/cmerkle_fuzz.exs +++ b/scripts/cmerkle_fuzz.exs @@ -354,20 +354,8 @@ defmodule CMerkleFuzz do defp slot(i), do: <> - defp compact_live(%Account{storage_root: tree} = acc) when is_reference(tree) do - items = Map.new(CMerkleTree.to_list(tree)) - - %Account{ - acc - | storage_root: if(map_size(items) == 0, do: nil, else: {MapMerkleTree, [], items}), - map_backed: false - } - |> Map.put(:root_hash, CMerkleTree.root_hash(tree)) - |> Map.put(:code_hash, Account.codehash(acc)) - end - defp build_compact_accounts(n) do - for i <- 1..n, into: %{} do + Enum.reduce(1..n, State.new(), fn i, st -> tree = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) @@ -379,8 +367,10 @@ defmodule CMerkleFuzz do map_backed: false } - {addr(i), compact_live(acc)} - end + State.set_account(st, addr(i), acc) + end) + |> State.compact() + |> Map.fetch!(:accounts) end defp s_lock_clone_insert(_round, %{max_keys: mk}) do @@ -987,7 +977,13 @@ defmodule CMerkleFuzz do defp build_live_state(n) do Enum.reduce(1..n, State.new(), fn i, st -> storage = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - State.set_account(st, addr(i), Account.put_tree(Account.new(nonce: i), storage)) + + State.set_account(st, addr(i), %{ + Account.new(nonce: i) + | storage_root: storage, + map_backed: false, + root_hash: nil + }) end) |> State.normalize() end diff --git a/scripts/cmerkle_parallel_stress.exs b/scripts/cmerkle_parallel_stress.exs index c589ea1..48f7787 100644 --- a/scripts/cmerkle_parallel_stress.exs +++ b/scripts/cmerkle_parallel_stress.exs @@ -499,20 +499,8 @@ defmodule CMerkleParallelStress do defp slot(i), do: <> - defp compact_live(%Account{storage_root: tree} = acc) when is_reference(tree) do - items = Map.new(CMerkleTree.to_list(tree)) - - %Account{ - acc - | storage_root: if(map_size(items) == 0, do: nil, else: {MapMerkleTree, [], items}), - map_backed: false - } - |> Map.put(:root_hash, CMerkleTree.root_hash(tree)) - |> Map.put(:code_hash, Account.codehash(acc)) - end - defp build_compact_accounts(n) do - for i <- 1..n, into: %{} do + Enum.reduce(1..n, State.new(), fn i, st -> tree = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) @@ -524,8 +512,10 @@ defmodule CMerkleParallelStress do map_backed: false } - {addr(i), compact_live(acc)} - end + State.set_account(st, addr(i), acc) + end) + |> State.compact() + |> Map.fetch!(:accounts) end defp build_live_state(n) do diff --git a/test/caccount_map_test.exs b/test/caccount_map_test.exs index 0524def..12ea4d3 100644 --- a/test/caccount_map_test.exs +++ b/test/caccount_map_test.exs @@ -27,7 +27,7 @@ defmodule CAccountMapTest do assert CAccountMap.size(map) == 1 assert {3, 3000, root, <<3>>} = CAccountMap.get(map, addr(3)) assert is_binary(root) and byte_size(root) == 32 - assert root == Account.root_hash(sample_account(3)) + assert root == CMerkleTree.root_hash(sample_account(3).storage_root) assert CAccountMap.storage_root_hash(map, addr(3)) == root end diff --git a/test/chain_account_hash_nif_test.exs b/test/chain_account_hash_nif_test.exs index 56ca439..911a242 100644 --- a/test/chain_account_hash_nif_test.exs +++ b/test/chain_account_hash_nif_test.exs @@ -23,20 +23,23 @@ defmodule ChainAccountHashNifTest do %Account{nonce: i, balance: i * 1_000, storage_root: tree, code: <>, map_backed: false} end - defp compact_live(%Account{storage_root: tree} = acc) when is_reference(tree) do - items = Map.new(CMerkleTree.to_list(tree)) + defp live_state(accounts) when is_list(accounts) do + Enum.reduce(accounts, State.new(), fn {id, acc}, st -> + State.set_account(st, id, acc) + end) + end - %Account{ - acc - | storage_root: if(map_size(items) == 0, do: nil, else: {MapMerkleTree, [], items}), - map_backed: false - } - |> Map.put(:root_hash, CMerkleTree.root_hash(tree)) - |> Map.put(:code_hash, Account.codehash(acc)) + defp compact_accounts(accounts) when is_list(accounts) do + live_state(accounts) |> State.compact() |> Map.fetch!(:accounts) end defp hash_from_live(%Account{storage_root: tree} = acc) when is_reference(tree) do - Account.hash(%{acc | map_backed: false} |> Map.put(:root_hash, CMerkleTree.root_hash(tree))) + Account.hash(%{ + acc + | root_hash: CMerkleTree.root_hash(tree), + storage_root: nil, + map_backed: true + }) end defp hash_from_compact(%Account{} = acc) do @@ -58,10 +61,11 @@ defmodule ChainAccountHashNifTest do map_backed: false } - compact = %{ - addr(1) => compact_live(multi_slot), - addr(2) => compact_live(sample_account(2)) - } + compact = + compact_accounts([ + {addr(1), multi_slot}, + {addr(2), sample_account(2)} + ]) {accounts, hash} = CAccountMap.uncompact_state(compact) @@ -83,9 +87,7 @@ defmodule ChainAccountHashNifTest do test "uncompact_state account hashes match Account.hash/1" do compact = - for i <- 1..8, into: %{} do - {addr(i), compact_live(sample_account(i))} - end + compact_accounts(Enum.map(1..8, fn i -> {addr(i), sample_account(i)} end)) {accounts, hash} = CAccountMap.uncompact_state(compact) @@ -142,9 +144,7 @@ defmodule ChainAccountHashNifTest do test "uncompact_state uses compact root_hash when present" do compact = - for i <- 1..4, into: %{} do - {addr(i), compact_live(sample_account(i))} - end + compact_accounts(Enum.map(1..4, fn i -> {addr(i), sample_account(i)} end)) assert Enum.all?(compact, fn {_id, acc} -> Map.has_key?(acc, :root_hash) end) assert Enum.all?(compact, fn {_id, acc} -> Map.has_key?(acc, :code_hash) end) @@ -176,7 +176,8 @@ defmodule ChainAccountHashNifTest do balance: acc.balance, storage_root: {MapMerkleTree, [], Map.new(CMerkleTree.to_list(tree))}, code: acc.code, - map_backed: false + map_backed: false, + root_hash: nil } legacy_compact = %{addr(1) => legacy_account} @@ -199,15 +200,14 @@ defmodule ChainAccountHashNifTest do acc = sample_account(1) tree = acc.storage_root - legacy_account = - %Chain.Account{ - nonce: acc.nonce, - balance: acc.balance, - storage_root: {MapMerkleTree, [], Map.new(CMerkleTree.to_list(tree))}, - code: acc.code, - map_backed: false - } - |> Map.put(:root_hash, CMerkleTree.root_hash(tree)) + legacy_account = %Chain.Account{ + nonce: acc.nonce, + balance: acc.balance, + storage_root: {MapMerkleTree, [], Map.new(CMerkleTree.to_list(tree))}, + code: acc.code, + map_backed: false, + root_hash: CMerkleTree.root_hash(tree) + } legacy_compact = %{addr(1) => legacy_account} diff --git a/test/chain_state_merkle_test.exs b/test/chain_state_merkle_test.exs index 58a2315..d17a021 100644 --- a/test/chain_state_merkle_test.exs +++ b/test/chain_state_merkle_test.exs @@ -46,7 +46,7 @@ defmodule ChainStateMerkleTest do defp account_from_evm_map(kvs) when is_map(kvs) do tree = CMerkleTree.insert_items(CMerkleTree.new(), Map.to_list(kvs)) - Account.put_tree(Account.new(), tree) + %{Account.new() | storage_root: tree, map_backed: false, root_hash: nil} end defp account_with_storage(pairs) do @@ -59,7 +59,7 @@ defmodule ChainStateMerkleTest do CMerkleTree.insert(t, k, v) end) - Account.put_tree(acc, tree) + %{acc | storage_root: tree, map_backed: false, root_hash: nil} end defp put_account(%State{} = st, i, acc), do: State.set_account(st, addr(i), acc) diff --git a/test/chain_state_uncompact_test.exs b/test/chain_state_uncompact_test.exs index 699047b..1bac0c2 100644 --- a/test/chain_state_uncompact_test.exs +++ b/test/chain_state_uncompact_test.exs @@ -32,33 +32,6 @@ defmodule ChainStateUncompactTest do } end - defp compact_account(%Account{storage_root: root} = acc) when is_reference(root) do - items = Map.new(CMerkleTree.to_list(root)) - - storage = - if map_size(items) == 0 do - nil - else - {MapMerkleTree, [], items} - end - - %Account{acc | storage_root: storage, map_backed: false} - |> Map.put(:root_hash, CMerkleTree.root_hash(root)) - |> Map.put(:code_hash, Account.codehash(acc)) - end - - defp compact_account(%Account{} = acc), do: acc - - defp compact_accounts_map(n_accounts) do - for i <- 1..n_accounts, into: %{} do - {addr(i), sample_account(i) |> compact_account()} - end - end - - defp compact_state(n_accounts) do - %State{accounts: compact_accounts_map(n_accounts)} - end - defp live_state(n_accounts) do State.new() |> then(fn st -> @@ -68,6 +41,18 @@ defmodule ChainStateUncompactTest do end) end + defp compact_state(n_accounts), do: State.compact(live_state(n_accounts)) + + defp compact_accounts_map(n_accounts), do: compact_state(n_accounts).accounts + + defp compact_via_state(%Account{} = acc, id \\ addr(1)) do + State.new() + |> State.set_account(id, acc) + |> State.compact() + |> Map.fetch!(:accounts) + |> Map.fetch!(id) + end + describe "uncompact correctness" do test "compact map round-trips through uncompact with stable state hash" do original = live_state(32) @@ -173,13 +158,7 @@ defmodule ChainStateUncompactTest do %{ addr(1) => list_storage_account, addr(2) => - %Account{ - nonce: 7, - balance: 42, - storage_root: nil, - code: nil - } - |> compact_account() + compact_via_state(%Account{nonce: 7, balance: 42, storage_root: nil, code: nil}) } restored = State.uncompact(%State{accounts: compact}) @@ -209,12 +188,12 @@ defmodule ChainStateUncompactTest do code: <<5>> } - compact = %{addr(1) => compact_account(multi_slot)} + compact = %{addr(1) => compact_via_state(multi_slot)} {accounts, _hash} = CAccountMap.uncompact_state(compact) {5, 1_000, root, <<5>>} = CAccountMap.get(accounts, addr(1)) assert is_binary(root) and byte_size(root) == 32 - assert root == Account.root_hash(multi_slot) + assert root == CMerkleTree.root_hash(multi_slot.storage_root) assert CAccountMap.storage_get(accounts, addr(1), slot(1)) == val(1) assert CAccountMap.storage_get(accounts, addr(1), slot(2)) == val(2) @@ -245,16 +224,16 @@ defmodule ChainStateUncompactTest do |> then(fn st -> elem(CAccountMap.uncompact_state(st.accounts), 0) end) end - test "compact_account stores code_hash matching Account.codehash/1" do + test "State.compact stores code_hash matching Account.codehash/1" do acc = sample_account(3) |> Map.put(:code, :binary.copy(<<0xEE>>, 256)) - compact = compact_account(acc) + compact = compact_via_state(acc) assert compact.code_hash == Account.codehash(acc) - assert compact.root_hash == Account.root_hash(acc) + assert compact.root_hash == CMerkleTree.root_hash(acc.storage_root) end test "put overwrites lazy account without prior get" do - compact = %{addr(1) => sample_account(1) |> compact_account()} + compact = %{addr(1) => compact_via_state(sample_account(1))} {accounts, _} = CAccountMap.uncompact_state(compact) new_storage = @@ -286,7 +265,7 @@ defmodule ChainStateUncompactTest do assert code == <> assert is_binary(root) and byte_size(root) == 32 assert CAccountMap.storage_get(accounts, addr(i), slot(i)) == val(i) - assert root == Account.root_hash(sample_account(i)) + assert root == CMerkleTree.root_hash(sample_account(i).storage_root) assert CAccountMap.storage_root_hash(accounts, addr(i)) == root end end diff --git a/test/cmerkle_account_map_diff_test.exs b/test/cmerkle_account_map_diff_test.exs index 1fc22e0..17b3e20 100644 --- a/test/cmerkle_account_map_diff_test.exs +++ b/test/cmerkle_account_map_diff_test.exs @@ -136,7 +136,14 @@ defmodule CMerkleAccountMapDiffTest do |> then(fn st -> Enum.reduce(1..80, st, fn i, acc -> storage = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - acc0 = Account.put_tree(Account.new(nonce: i, balance: i), storage) + + acc0 = %{ + Account.new(nonce: i, balance: i) + | storage_root: storage, + map_backed: false, + root_hash: nil + } + State.set_account(acc, addr(i), acc0) end) end) @@ -216,7 +223,13 @@ defmodule CMerkleAccountMapDiffTest do |> then(fn st -> Enum.reduce(1..30, st, fn i, acc -> storage = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - State.set_account(acc, addr(i), Account.put_tree(Account.new(nonce: i), storage)) + + State.set_account(acc, addr(i), %{ + Account.new(nonce: i) + | storage_root: storage, + map_backed: false, + root_hash: nil + }) end) end) diff --git a/test/cmerkle_nif_deadlock_test.exs b/test/cmerkle_nif_deadlock_test.exs index 5b7a6f1..528025a 100644 --- a/test/cmerkle_nif_deadlock_test.exs +++ b/test/cmerkle_nif_deadlock_test.exs @@ -374,20 +374,8 @@ defmodule CMerkleNifDeadlockTest do ) end - defp compact_live(%Account{storage_root: tree} = acc) when is_reference(tree) do - items = Map.new(CMerkleTree.to_list(tree)) - - %Account{ - acc - | storage_root: if(map_size(items) == 0, do: nil, else: {MapMerkleTree, [], items}), - map_backed: false - } - |> Map.put(:root_hash, CMerkleTree.root_hash(tree)) - |> Map.put(:code_hash, Account.codehash(acc)) - end - defp build_compact_accounts(n) do - for i <- 1..n, into: %{} do + Enum.reduce(1..n, State.new(), fn i, st -> tree = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) @@ -399,8 +387,10 @@ defmodule CMerkleNifDeadlockTest do map_backed: false } - {addr(i), compact_live(acc)} - end + State.set_account(st, addr(i), acc) + end) + |> State.compact() + |> Map.fetch!(:accounts) end defp build_live_state(n) do From ae3ec1f5f45aadbdb0dd44fd7570972e09b1c8ad Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Thu, 16 Jul 2026 21:54:32 +0200 Subject: [PATCH 09/16] Halt early on runaway RSS and stop SyncSql from loading all rows at once. Startup purge of sync.sq3 was SELECT-ing every block into memory; batch and background it, and add MemoryGuard so absolute/rapid RSS growth stops the VM. Co-authored-by: Cursor --- lib/diode.ex | 6 ++ lib/memory_guard.ex | 215 +++++++++++++++++++++++++++++++++++++ lib/model/syncsql.ex | 89 +++++++++++---- test/memory_guard_test.exs | 12 +++ 4 files changed, 299 insertions(+), 23 deletions(-) create mode 100644 lib/memory_guard.ex create mode 100644 test/memory_guard_test.exs diff --git a/lib/diode.ex b/lib/diode.ex index ca10313..288fd19 100644 --- a/lib/diode.ex +++ b/lib/diode.ex @@ -77,6 +77,8 @@ defmodule Diode do RemoteChain.RPCCache.set_optimistic_caching(false) base_children = [ + # Watch RSS before heavy Model.Sql / chain init so runaway growth halts early. + worker(MemoryGuard, []), worker(Stats, []), worker(Cron, []), worker(Chain.BlockQuickPool, []), @@ -173,6 +175,10 @@ defmodule Diode do puts("=======> #{module} loaded after #{Float.round(t / 1_000_000, 3)}s") {:ok, pid} + {t, :ignore} -> + puts("=======> #{module} ignored after #{Float.round(t / 1_000_000, 3)}s") + :ignore + {_t, other} -> puts("=======> #{module} failed with: #{inspect(other)}") other diff --git a/lib/memory_guard.ex b/lib/memory_guard.ex new file mode 100644 index 0000000..5ff3b74 --- /dev/null +++ b/lib/memory_guard.ex @@ -0,0 +1,215 @@ +# Diode Server +# Copyright 2021-2024 Diode +# Licensed under the Diode License, Version 1.1 +defmodule MemoryGuard do + @moduledoc """ + Early-stop watchdog for runaway process RSS growth. + + Polls `/proc/self/status` and halts the VM when either: + - absolute RSS exceeds `MEMORY_LIMIT_MB` (default: 80% of MemTotal), or + - RSS grows faster than `MEMORY_GROWTH_MB` over `MEMORY_GROWTH_WINDOW_SEC` + (defaults: 2048 MB in 15 seconds). + + Disable with `MEMORY_GUARD=0`. + """ + use GenServer + require Logger + + @default_interval_ms 1_000 + @default_growth_mb 2_048 + @default_growth_window_sec 15 + @default_limit_ratio 0.80 + + defstruct baseline_rss_kb: 0, + samples: [], + limit_kb: nil, + growth_kb: nil, + growth_window_ms: nil, + interval_ms: nil + + def start_link(opts \\ []) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @impl true + def init(opts) do + if disabled?() do + Logger.info("MemoryGuard: disabled (MEMORY_GUARD=0)") + :ignore + else + interval_ms = + Keyword.get(opts, :interval_ms, env_int("MEMORY_GUARD_INTERVAL_MS", @default_interval_ms)) + + growth_kb = + Keyword.get(opts, :growth_kb, env_int("MEMORY_GROWTH_MB", @default_growth_mb) * 1024) + + growth_window_ms = + Keyword.get( + opts, + :growth_window_ms, + env_int("MEMORY_GROWTH_WINDOW_SEC", @default_growth_window_sec) * 1_000 + ) + + limit_kb = + Keyword.get(opts, :limit_kb, resolve_limit_kb()) + + rss = read_rss_kb() + + Logger.info( + "MemoryGuard: watching RSS limit=#{div(limit_kb, 1024)}MB " <> + "growth=#{div(growth_kb, 1024)}MB/#{div(growth_window_ms, 1000)}s " <> + "interval=#{interval_ms}ms baseline=#{div(rss, 1024)}MB" + ) + + :timer.send_interval(interval_ms, :tick) + + {:ok, + %__MODULE__{ + baseline_rss_kb: rss, + samples: [{monotonic_ms(), rss}], + limit_kb: limit_kb, + growth_kb: growth_kb, + growth_window_ms: growth_window_ms, + interval_ms: interval_ms + }} + end + end + + @impl true + def handle_info(:tick, state) do + rss = read_rss_kb() + now = monotonic_ms() + samples = trim_samples([{now, rss} | state.samples], now, state.growth_window_ms) + + cond do + rss >= state.limit_kb -> + halt!("absolute RSS limit", state, rss, samples) + + rapid_growth?(samples, state.growth_kb) -> + halt!("rapid RSS growth", state, rss, samples) + + true -> + {:noreply, %{state | samples: samples}} + end + end + + def handle_info(_other, state), do: {:noreply, state} + + defp halt!(reason, state, rss, samples) do + {oldest_ms, oldest_rss} = List.last(samples) + newest_ms = elem(hd(samples), 0) + window_s = max(div(newest_ms - oldest_ms, 1000), 1) + delta_mb = div(rss - oldest_rss, 1024) + beam_total = :erlang.memory(:total) + + nif = + try do + CMerkleTree.nif_stats() + rescue + _ -> :unavailable + catch + _, _ -> :unavailable + end + + Logger.error(""" + MemoryGuard: halting — #{reason} + rss=#{div(rss, 1024)}MB limit=#{div(state.limit_kb, 1024)}MB \ + baseline=#{div(state.baseline_rss_kb, 1024)}MB + growth=#{delta_mb}MB over #{window_s}s (threshold=#{div(state.growth_kb, 1024)}MB) + erlang.memory(total)=#{div(beam_total, 1024 * 1024)}MB + nif_stats=#{inspect(nif)} + meminfo=#{inspect(meminfo_summary())} + """) + + # Hard halt — System.stop can hang while supervisors are stuck in init. + :erlang.halt(1) + end + + defp rapid_growth?(samples, _growth_kb) when length(samples) < 2, do: false + + defp rapid_growth?(samples, growth_kb) do + {oldest_ms, oldest_rss} = List.last(samples) + {newest_ms, newest_rss} = hd(samples) + newest_rss - oldest_rss >= growth_kb and newest_ms > oldest_ms + end + + defp trim_samples(samples, now, window_ms) do + cutoff = now - window_ms + # Keep at least two samples so growth can be measured once the window fills. + kept = Enum.take_while(samples, fn {t, _} -> t >= cutoff end) + + case kept do + [] -> Enum.take(samples, 2) + [_] = one -> one ++ Enum.take(Enum.drop(samples, 1), 1) + many -> many + end + end + + defp resolve_limit_kb do + case System.get_env("MEMORY_LIMIT_MB") do + nil -> + case mem_total_kb() do + nil -> 24 * 1024 * 1024 + total -> trunc(total * @default_limit_ratio) + end + + mb -> + String.to_integer(mb) * 1024 + end + end + + defp disabled? do + System.get_env("MEMORY_GUARD") in ["0", "false", "off"] + end + + defp env_int(name, default) do + case System.get_env(name) do + nil -> default + "" -> default + value -> String.to_integer(value) + end + end + + defp monotonic_ms, do: System.monotonic_time(:millisecond) + + def read_rss_kb do + case File.read("/proc/self/status") do + {:ok, body} -> + case Regex.run(~r/^VmRSS:\s+(\d+)\s+kB/m, body) do + [_, n] -> String.to_integer(n) + _ -> 0 + end + + _ -> + 0 + end + end + + defp mem_total_kb do + case File.read("/proc/meminfo") do + {:ok, body} -> + case Regex.run(~r/^MemTotal:\s+(\d+)\s+kB/m, body) do + [_, n] -> String.to_integer(n) + _ -> nil + end + + _ -> + nil + end + end + + defp meminfo_summary do + case File.read("/proc/meminfo") do + {:ok, body} -> + for key <- ["MemTotal", "MemAvailable", "MemFree", "SwapTotal", "SwapFree"], into: %{} do + case Regex.run(~r/^#{key}:\s+(\d+)\s+kB/m, body) do + [_, n] -> {key, String.to_integer(n)} + _ -> {key, nil} + end + end + + _ -> + %{} + end + end +end diff --git a/lib/model/syncsql.ex b/lib/model/syncsql.ex index a994003..74550e0 100644 --- a/lib/model/syncsql.ex +++ b/lib/model/syncsql.ex @@ -24,6 +24,9 @@ defmodule Model.SyncSql do GenServer.start_link(__MODULE__, :ok, name: __MODULE__) end + # Full-table SELECT of sync.sq3 can be millions of rows; never load it at once. + @purge_batch_size 500 + def init(_args) do with_transaction(fn db -> Sql.query!(db, """ @@ -36,10 +39,16 @@ defmodule Model.SyncSql do """) end) - {purged, _} = purge_invalid_blocks_from_db_and_queue(%{}) + # Insert path already rejects unsigned blocks. Background-scan legacy rows so + # Model.Sql init is not blocked on multi-million-row sync.sq3 databases. + if System.get_env("SYNC_PURGE_ON_START", "1") != "0" do + spawn(fn -> + deleted = purge_invalid_rows_batched(0, 0, 0) - if purged > 0 do - Logger.info("SyncSql: purged #{purged} block(s) without miner_signature on startup") + if deleted > 0 do + Logger.info("SyncSql: purged #{deleted} block(s) without miner_signature on startup") + end + end) end {:ok, %SyncSql{worker: spawn_link(&worker/0)}} @@ -277,26 +286,7 @@ defmodule Model.SyncSql do end defp purge_invalid_blocks_from_db_and_queue(queue) do - rows = - query!("SELECT hash, data FROM blocks", []) - - {deleted, _} = - Enum.reduce(rows, {0, queue}, fn [hash: hash, data: data], {count, queue} -> - block = BertInt.decode!(data) - - if Chain.Block.miner_signature_valid?(block) do - {count, queue} - else - query!("DELETE FROM blocks WHERE hash = ?1", [hash]) - queue = Map.delete(queue, hash) - - Logger.warning( - "SyncSql: purged invalid block (#{Chain.Block.sync_wire_reject_reason(block)})" - ) - - {count + 1, queue} - end - end) + deleted = purge_invalid_rows_batched(0, 0, 0) queue = Enum.reduce(queue, %{}, fn {hash, block}, acc -> @@ -314,6 +304,59 @@ defmodule Model.SyncSql do {deleted, queue} end + defp purge_invalid_rows_batched(after_rowid, scanned, deleted) do + rows = + query!( + """ + SELECT rowid, hash, data FROM blocks + WHERE rowid > ?1 + ORDER BY rowid + LIMIT ?2 + """, + [after_rowid, @purge_batch_size] + ) + + case rows do + [] -> + if scanned > 0 do + Logger.info("SyncSql: purge finished scanned=#{scanned} deleted=#{deleted}") + end + + deleted + + rows -> + {batch_deleted, last_rowid} = + Enum.reduce(rows, {0, after_rowid}, fn row, {count, _} -> + hash = row[:hash] + data = row[:data] + rowid = row[:rowid] + block = BertInt.decode!(data) + + if Chain.Block.miner_signature_valid?(block) do + {count, rowid} + else + query!("DELETE FROM blocks WHERE hash = ?1", [hash]) + + Logger.warning( + "SyncSql: purged invalid block (#{Chain.Block.sync_wire_reject_reason(block)})" + ) + + {count + 1, rowid} + end + end) + + prev = scanned + scanned = scanned + length(rows) + deleted = deleted + batch_deleted + + if div(prev, 50_000) < div(scanned, 50_000) do + Logger.info("SyncSql: purge progress scanned=#{scanned} deleted=#{deleted}") + end + + purge_invalid_rows_batched(last_rowid, scanned, deleted) + end + end + defp queue_top(block, queue) do case Map.get(queue, Chain.Block.parent_hash(block)) do nil -> block diff --git a/test/memory_guard_test.exs b/test/memory_guard_test.exs new file mode 100644 index 0000000..992adee --- /dev/null +++ b/test/memory_guard_test.exs @@ -0,0 +1,12 @@ +# Diode Server +# Copyright 2021-2024 Diode +# Licensed under the Diode License, Version 1.1 +defmodule MemoryGuardTest do + use ExUnit.Case, async: true + + test "read_rss_kb returns a positive integer on Linux" do + rss = MemoryGuard.read_rss_kb() + assert is_integer(rss) + assert rss > 0 + end +end From a4fb69d7729b067279ba79e4c7baeae7ba06dad3 Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Thu, 16 Jul 2026 22:07:37 +0200 Subject: [PATCH 10/16] Revert "Halt early on runaway RSS and stop SyncSql from loading all rows at once." This reverts commit ae3ec1f5f45aadbdb0dd44fd7570972e09b1c8ad. --- lib/diode.ex | 6 -- lib/memory_guard.ex | 215 ------------------------------------- lib/model/syncsql.ex | 89 ++++----------- test/memory_guard_test.exs | 12 --- 4 files changed, 23 insertions(+), 299 deletions(-) delete mode 100644 lib/memory_guard.ex delete mode 100644 test/memory_guard_test.exs diff --git a/lib/diode.ex b/lib/diode.ex index 288fd19..ca10313 100644 --- a/lib/diode.ex +++ b/lib/diode.ex @@ -77,8 +77,6 @@ defmodule Diode do RemoteChain.RPCCache.set_optimistic_caching(false) base_children = [ - # Watch RSS before heavy Model.Sql / chain init so runaway growth halts early. - worker(MemoryGuard, []), worker(Stats, []), worker(Cron, []), worker(Chain.BlockQuickPool, []), @@ -175,10 +173,6 @@ defmodule Diode do puts("=======> #{module} loaded after #{Float.round(t / 1_000_000, 3)}s") {:ok, pid} - {t, :ignore} -> - puts("=======> #{module} ignored after #{Float.round(t / 1_000_000, 3)}s") - :ignore - {_t, other} -> puts("=======> #{module} failed with: #{inspect(other)}") other diff --git a/lib/memory_guard.ex b/lib/memory_guard.ex deleted file mode 100644 index 5ff3b74..0000000 --- a/lib/memory_guard.ex +++ /dev/null @@ -1,215 +0,0 @@ -# Diode Server -# Copyright 2021-2024 Diode -# Licensed under the Diode License, Version 1.1 -defmodule MemoryGuard do - @moduledoc """ - Early-stop watchdog for runaway process RSS growth. - - Polls `/proc/self/status` and halts the VM when either: - - absolute RSS exceeds `MEMORY_LIMIT_MB` (default: 80% of MemTotal), or - - RSS grows faster than `MEMORY_GROWTH_MB` over `MEMORY_GROWTH_WINDOW_SEC` - (defaults: 2048 MB in 15 seconds). - - Disable with `MEMORY_GUARD=0`. - """ - use GenServer - require Logger - - @default_interval_ms 1_000 - @default_growth_mb 2_048 - @default_growth_window_sec 15 - @default_limit_ratio 0.80 - - defstruct baseline_rss_kb: 0, - samples: [], - limit_kb: nil, - growth_kb: nil, - growth_window_ms: nil, - interval_ms: nil - - def start_link(opts \\ []) do - GenServer.start_link(__MODULE__, opts, name: __MODULE__) - end - - @impl true - def init(opts) do - if disabled?() do - Logger.info("MemoryGuard: disabled (MEMORY_GUARD=0)") - :ignore - else - interval_ms = - Keyword.get(opts, :interval_ms, env_int("MEMORY_GUARD_INTERVAL_MS", @default_interval_ms)) - - growth_kb = - Keyword.get(opts, :growth_kb, env_int("MEMORY_GROWTH_MB", @default_growth_mb) * 1024) - - growth_window_ms = - Keyword.get( - opts, - :growth_window_ms, - env_int("MEMORY_GROWTH_WINDOW_SEC", @default_growth_window_sec) * 1_000 - ) - - limit_kb = - Keyword.get(opts, :limit_kb, resolve_limit_kb()) - - rss = read_rss_kb() - - Logger.info( - "MemoryGuard: watching RSS limit=#{div(limit_kb, 1024)}MB " <> - "growth=#{div(growth_kb, 1024)}MB/#{div(growth_window_ms, 1000)}s " <> - "interval=#{interval_ms}ms baseline=#{div(rss, 1024)}MB" - ) - - :timer.send_interval(interval_ms, :tick) - - {:ok, - %__MODULE__{ - baseline_rss_kb: rss, - samples: [{monotonic_ms(), rss}], - limit_kb: limit_kb, - growth_kb: growth_kb, - growth_window_ms: growth_window_ms, - interval_ms: interval_ms - }} - end - end - - @impl true - def handle_info(:tick, state) do - rss = read_rss_kb() - now = monotonic_ms() - samples = trim_samples([{now, rss} | state.samples], now, state.growth_window_ms) - - cond do - rss >= state.limit_kb -> - halt!("absolute RSS limit", state, rss, samples) - - rapid_growth?(samples, state.growth_kb) -> - halt!("rapid RSS growth", state, rss, samples) - - true -> - {:noreply, %{state | samples: samples}} - end - end - - def handle_info(_other, state), do: {:noreply, state} - - defp halt!(reason, state, rss, samples) do - {oldest_ms, oldest_rss} = List.last(samples) - newest_ms = elem(hd(samples), 0) - window_s = max(div(newest_ms - oldest_ms, 1000), 1) - delta_mb = div(rss - oldest_rss, 1024) - beam_total = :erlang.memory(:total) - - nif = - try do - CMerkleTree.nif_stats() - rescue - _ -> :unavailable - catch - _, _ -> :unavailable - end - - Logger.error(""" - MemoryGuard: halting — #{reason} - rss=#{div(rss, 1024)}MB limit=#{div(state.limit_kb, 1024)}MB \ - baseline=#{div(state.baseline_rss_kb, 1024)}MB - growth=#{delta_mb}MB over #{window_s}s (threshold=#{div(state.growth_kb, 1024)}MB) - erlang.memory(total)=#{div(beam_total, 1024 * 1024)}MB - nif_stats=#{inspect(nif)} - meminfo=#{inspect(meminfo_summary())} - """) - - # Hard halt — System.stop can hang while supervisors are stuck in init. - :erlang.halt(1) - end - - defp rapid_growth?(samples, _growth_kb) when length(samples) < 2, do: false - - defp rapid_growth?(samples, growth_kb) do - {oldest_ms, oldest_rss} = List.last(samples) - {newest_ms, newest_rss} = hd(samples) - newest_rss - oldest_rss >= growth_kb and newest_ms > oldest_ms - end - - defp trim_samples(samples, now, window_ms) do - cutoff = now - window_ms - # Keep at least two samples so growth can be measured once the window fills. - kept = Enum.take_while(samples, fn {t, _} -> t >= cutoff end) - - case kept do - [] -> Enum.take(samples, 2) - [_] = one -> one ++ Enum.take(Enum.drop(samples, 1), 1) - many -> many - end - end - - defp resolve_limit_kb do - case System.get_env("MEMORY_LIMIT_MB") do - nil -> - case mem_total_kb() do - nil -> 24 * 1024 * 1024 - total -> trunc(total * @default_limit_ratio) - end - - mb -> - String.to_integer(mb) * 1024 - end - end - - defp disabled? do - System.get_env("MEMORY_GUARD") in ["0", "false", "off"] - end - - defp env_int(name, default) do - case System.get_env(name) do - nil -> default - "" -> default - value -> String.to_integer(value) - end - end - - defp monotonic_ms, do: System.monotonic_time(:millisecond) - - def read_rss_kb do - case File.read("/proc/self/status") do - {:ok, body} -> - case Regex.run(~r/^VmRSS:\s+(\d+)\s+kB/m, body) do - [_, n] -> String.to_integer(n) - _ -> 0 - end - - _ -> - 0 - end - end - - defp mem_total_kb do - case File.read("/proc/meminfo") do - {:ok, body} -> - case Regex.run(~r/^MemTotal:\s+(\d+)\s+kB/m, body) do - [_, n] -> String.to_integer(n) - _ -> nil - end - - _ -> - nil - end - end - - defp meminfo_summary do - case File.read("/proc/meminfo") do - {:ok, body} -> - for key <- ["MemTotal", "MemAvailable", "MemFree", "SwapTotal", "SwapFree"], into: %{} do - case Regex.run(~r/^#{key}:\s+(\d+)\s+kB/m, body) do - [_, n] -> {key, String.to_integer(n)} - _ -> {key, nil} - end - end - - _ -> - %{} - end - end -end diff --git a/lib/model/syncsql.ex b/lib/model/syncsql.ex index 74550e0..a994003 100644 --- a/lib/model/syncsql.ex +++ b/lib/model/syncsql.ex @@ -24,9 +24,6 @@ defmodule Model.SyncSql do GenServer.start_link(__MODULE__, :ok, name: __MODULE__) end - # Full-table SELECT of sync.sq3 can be millions of rows; never load it at once. - @purge_batch_size 500 - def init(_args) do with_transaction(fn db -> Sql.query!(db, """ @@ -39,16 +36,10 @@ defmodule Model.SyncSql do """) end) - # Insert path already rejects unsigned blocks. Background-scan legacy rows so - # Model.Sql init is not blocked on multi-million-row sync.sq3 databases. - if System.get_env("SYNC_PURGE_ON_START", "1") != "0" do - spawn(fn -> - deleted = purge_invalid_rows_batched(0, 0, 0) + {purged, _} = purge_invalid_blocks_from_db_and_queue(%{}) - if deleted > 0 do - Logger.info("SyncSql: purged #{deleted} block(s) without miner_signature on startup") - end - end) + if purged > 0 do + Logger.info("SyncSql: purged #{purged} block(s) without miner_signature on startup") end {:ok, %SyncSql{worker: spawn_link(&worker/0)}} @@ -286,7 +277,26 @@ defmodule Model.SyncSql do end defp purge_invalid_blocks_from_db_and_queue(queue) do - deleted = purge_invalid_rows_batched(0, 0, 0) + rows = + query!("SELECT hash, data FROM blocks", []) + + {deleted, _} = + Enum.reduce(rows, {0, queue}, fn [hash: hash, data: data], {count, queue} -> + block = BertInt.decode!(data) + + if Chain.Block.miner_signature_valid?(block) do + {count, queue} + else + query!("DELETE FROM blocks WHERE hash = ?1", [hash]) + queue = Map.delete(queue, hash) + + Logger.warning( + "SyncSql: purged invalid block (#{Chain.Block.sync_wire_reject_reason(block)})" + ) + + {count + 1, queue} + end + end) queue = Enum.reduce(queue, %{}, fn {hash, block}, acc -> @@ -304,59 +314,6 @@ defmodule Model.SyncSql do {deleted, queue} end - defp purge_invalid_rows_batched(after_rowid, scanned, deleted) do - rows = - query!( - """ - SELECT rowid, hash, data FROM blocks - WHERE rowid > ?1 - ORDER BY rowid - LIMIT ?2 - """, - [after_rowid, @purge_batch_size] - ) - - case rows do - [] -> - if scanned > 0 do - Logger.info("SyncSql: purge finished scanned=#{scanned} deleted=#{deleted}") - end - - deleted - - rows -> - {batch_deleted, last_rowid} = - Enum.reduce(rows, {0, after_rowid}, fn row, {count, _} -> - hash = row[:hash] - data = row[:data] - rowid = row[:rowid] - block = BertInt.decode!(data) - - if Chain.Block.miner_signature_valid?(block) do - {count, rowid} - else - query!("DELETE FROM blocks WHERE hash = ?1", [hash]) - - Logger.warning( - "SyncSql: purged invalid block (#{Chain.Block.sync_wire_reject_reason(block)})" - ) - - {count + 1, rowid} - end - end) - - prev = scanned - scanned = scanned + length(rows) - deleted = deleted + batch_deleted - - if div(prev, 50_000) < div(scanned, 50_000) do - Logger.info("SyncSql: purge progress scanned=#{scanned} deleted=#{deleted}") - end - - purge_invalid_rows_batched(last_rowid, scanned, deleted) - end - end - defp queue_top(block, queue) do case Map.get(queue, Chain.Block.parent_hash(block)) do nil -> block diff --git a/test/memory_guard_test.exs b/test/memory_guard_test.exs deleted file mode 100644 index 992adee..0000000 --- a/test/memory_guard_test.exs +++ /dev/null @@ -1,12 +0,0 @@ -# Diode Server -# Copyright 2021-2024 Diode -# Licensed under the Diode License, Version 1.1 -defmodule MemoryGuardTest do - use ExUnit.Case, async: true - - test "read_rss_kb returns a positive integer on Linux" do - rss = MemoryGuard.read_rss_kb() - assert is_integer(rss) - assert rss > 0 - end -end From 1d2988ef82fc5e0956082c467152b3f0ec3c3a50 Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Fri, 17 Jul 2026 01:25:54 +0200 Subject: [PATCH 11/16] Add State.difference bench and perf change specification. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document Phases A–E (cached compact roots, SharedState equality, NIF-returned roots, CompactStorage COW, state_trie-driven diff) and add a bench that reproduces the multi-second compact_small_delta hotspot. Co-authored-by: Cursor --- docs/caccount-map-nif.md | 8 +- docs/specs/change-state-diff-perf.md | 373 ++++++++++++++++++++++++++ scripts/state_diff_bench.exs | 376 +++++++++++++++++++++++++++ 3 files changed, 755 insertions(+), 2 deletions(-) create mode 100644 docs/specs/change-state-diff-perf.md create mode 100644 scripts/state_diff_bench.exs diff --git a/docs/caccount-map-nif.md b/docs/caccount-map-nif.md index 8f4b9f9..14b342f 100644 --- a/docs/caccount-map-nif.md +++ b/docs/caccount-map-nif.md @@ -3,8 +3,12 @@ Agent-facing notes for the merkle account-map NIF (`priv/merkletree_nif.so`, built from `c_src/` via `mix compile` / top-level `Makefile`). -See also [`c_src/LOCK_ORDER.md`](../c_src/LOCK_ORDER.md) and -[`c_src/SECURITY_REVIEW.md`](../c_src/SECURITY_REVIEW.md). +See also [`c_src/LOCK_ORDER.md`](../c_src/LOCK_ORDER.md), +[`c_src/SECURITY_REVIEW.md`](../c_src/SECURITY_REVIEW.md), and the +implementation spec for difference/clone performance work: +[`docs/specs/change-state-diff-perf.md`](specs/change-state-diff-perf.md) +(cached compact storage roots, CompactStorage COW, state_trie-driven +`difference_full`). ## Ownership model diff --git a/docs/specs/change-state-diff-perf.md b/docs/specs/change-state-diff-perf.md new file mode 100644 index 0000000..da011a2 --- /dev/null +++ b/docs/specs/change-state-diff-perf.md @@ -0,0 +1,373 @@ +# State Diff Performance Specification v0.1.0 + +> **Spec type:** Change +> **Path:** `docs/specs/change-state-diff-perf.md` + +## Overview + +This change eliminates multi-second `Chain.State.difference/2` walls on jump-shaped +(compact) peaks when only a few accounts change. The NIF today rebuilds temporary +Merkle trees from every `CompactStorage` slot vector solely to compare storage +roots, deep-copies those slot vectors on every clone, and has Elixir re-fetch +roots (materializing compact storage) after the NIF already did the work. + +**Integration context:** `c_src/nif.cpp` (`CompactStorage`, `entries_equal`, +`storage_root_hash_for_entry`, `fork_shared_accountmap`, `account_map_difference_full`, +`account_map_storage_roots`, uncompact/compact), `lib/chain/state.ex`, +`lib/caccount_map.ex`, `lib/cmerkletree.ex`, `Model.ChainSql.prepare_state/2`, +and `scripts/state_diff_bench.exs`. On-disk BertInt block encodings MUST NOT change. + +## Design Principles + +1. **Never rebuild a trie only to compare roots.** Storage roots are cached data; + deriving them on every equality check is forbidden once a root is known. +2. **O(changed) for common `prepare_state` deltas.** Full-map account scans for + equality are unacceptable when few accounts differ. +3. **Compact storage is COW.** `State.clone/1` / `account_map_clone` MUST NOT + deep-copy slot vectors until a write requires uniqueness. +4. **Same observable deltas.** `State.difference` / `apply_difference` round-trips + and BertInt encodings remain bit-compatible with pre-change behavior for the + Elixir report map shape (`{id, report}` with optional `:state` / `:root_hash`). +5. **Measure before claiming.** Each phase has a `state_diff_bench` acceptance + gate; phases ship in order A→E and are each mergeable alone with tests green. + +--- + +## Output Structure + +**Do generate:** + +- Updates under `c_src/nif.cpp` (and helpers if extracted) +- Elixir wrapper / `State.difference` updates for the NIF tuple shape +- Unit and regression tests listed in Testing +- Updates to `docs/caccount-map-nif.md` describing cache, COW, and trie-driven diff + +**Do not generate:** + +- Changes to BertInt / on-disk block state encoding +- Rework of `Tree::difference` internals +- Slot-vector-only storage diffs as a substitute for Phase E (deferred) +- Standalone packages or unrelated refactors + +--- + +## Type Conventions + +| Spec type | Meaning | Examples | +|-----------|---------|----------| +| `uint256_t` | 32-byte big-endian hash / root | storage root | +| `CompactStorage` | Lazy account storage: slots + optional cached root | see Phase A | +| `AccountEntry` | nonce, balance, live `merkletree*`, compact ptr, code | `c_src/nif.cpp` | +| `difference_full` entry | Erlang term from NIF | Phase C 6-tuple | +| `State.difference` report | `%{optional fields}` per address | `:nonce`, `:balance`, `:code`, `:state`, `:root_hash` | +| `nif_ms` | Wall ms inside `account_map_difference_full` | `state_diff_bench` | + +### Normalization + +- Elixir compact account `:root_hash` is the **storage** root (32 bytes), not the + account RLP hash. Uncompact MUST treat it as the storage root cache seed. +- Atom `nil` in NIF root fields means “side absent”; never a zero hash. + +--- + +## Error Handling + +| Language | Error style | +|----------|-------------| +| C++ NIF | `enif_make_badarg` / existing apply error tuples | +| Elixir | Raise / existing `ArgumentError` on `apply_difference` mismatch | + +| Behavior | Error when | +|----------|------------| +| `storage_root_hash_for_entry` | Unchanged: still returns false only on hard failure; cache miss MUST compute and store, not fail | +| `difference_full` | Bad resources / arity → `badarg` (unchanged) | +| `State.difference` | Malformed NIF tuple after Phase C → raise (tests catch wrapper bugs) | +| COW write | Allocation failure → existing NIF OOM / badarg paths | + +Liberal inputs (legacy compact without `:root_hash`): compute root once, cache it. +Strict outputs: cached root MUST match a fresh trie hash of the same slots. + +--- + +## Domain Rules + +### Cache invalidation (Phase A) + +`CompactStorage.has_root` MUST be set `false` (and root ignored) before or when: + +- Mutating any slot (`write_storage_slot`, `apply_storage_delta` storage map) +- Replacing compact with live storage (`materialize_storage` resets compact) +- Any path that changes slot contents without going through those helpers + (implementations MUST route mutations through them or invalidate explicitly) + +After invalidation, the next `storage_root_hash_for_entry` MUST recompute once and +set `has_root` again if the entry remains compact. + +### COW (Phase D) + +- `shared_ptr` (or equivalent intrusive shared ownership). +- Copy/assign/fork: share the pointer; do **not** copy `slots`. +- Before mutating slots or clearing `has_root` for a write: if `use_count() > 1`, + allocate a unique copy (deep-copy slots + root flags), then mutate. +- `snapshot_side` MAY share the compact pointer (refcount bump); MUST NOT force a + deep copy solely for snapshotting. + +### State trie invariant (Phase E) + +Every NIF that changes account RLP content (nonce, balance, code, or storage root) +MUST update `state_trie` for that address. Paths that MUST keep this invariant: + +- `account_map_put` / `put` meta +- `account_map_delete` +- `account_map_storage_put_map` +- `account_map_apply_difference` +- Uncompact batch insert into `state_trie` + +If a future NIF mutates account content without updating `state_trie`, Phase E +equality is wrong — such a NIF is a spec violation. + +### Observable compatibility + +- `State.difference/2` return type remains a list of `{addr, report}` maps. +- Report keys and semantics unchanged; only the *source* of `:root_hash` values + changes (NIF-provided vs Elixir re-fetch). +- `apply_difference` input shape unchanged. + +--- + +## Behaviors (Phases A–E) + +Phases MUST be implemented in order. Each phase MUST leave correctness suites green +and meet its acceptance gate before the next phase merges. + +```mermaid +flowchart TD + diff[State.difference] --> nif[difference_full] + nif --> eq[entries_equal] + eq -->|cached root| o1[O1 memcmp] + eq -->|live SharedState| share[pointer equal] + nif --> trie[Phase E: state_trie leaf diff] + trie --> deep[deep storage diff only for changed addrs] + clone[State.clone] --> cow[shared_ptr CompactStorage] +``` + +### Phase A — Cache storage root on `CompactStorage` + +**Data model** (`c_src/nif.cpp`): + +```cpp +struct CompactStorage { + std::vector slots; + uint256_t root_hash; // valid iff has_root + bool has_root = false; +}; +``` + +**MUST:** + +| Condition | Behavior | +|-----------|----------| +| Uncompact Elixir map with `parsed.has_compact_root_hash` | Set `has_root=true` and copy root onto the new `CompactStorage` (not only pass override into `AccountHashCtx::compute`) | +| `storage_root_hash_for_entry` + compact + `has_root` | Return cached root; **no** temporary `Tree` | +| `storage_root_hash_for_entry` + compact + `!has_root` | Build tree **once**, store root on compact, return it | +| `make_compact_account_term` | Emit Elixir `:root_hash` from cache or one-time compute | +| Storage mutation | Invalidate `has_root` per Domain Rules | +| Live `storage != nullptr` | Unchanged: use tree `root_hash()` | + +**MUST NOT:** Discard a known root and rebuild on the next compare. + +**Acceptance:** +`mix run --no-start scripts/state_diff_bench.exs -- --scenario compact_small_delta --accounts 14000 --changed 20 --slots 32 --warmup 1 --iters 2` +→ avg `nif_ms` **< 50** (baseline ~700+). + +**Rationale:** Uncompact already parses `:root_hash`; caching it removes the +prod “accounts=20 / multi-second” hotspot without changing algorithms. + +--- + +### Phase B — Live `SharedState*` equality + +**`entries_equal` order MUST be:** + +1. Meta mismatch (nonce / balance / code) → not equal. +2. Both `storage` non-null and `a.storage->shared_state == b.storage->shared_state` → equal. +3. Else both wrappers identical (`a.storage == b.storage`) → equal. +4. Else compare storage roots via `storage_root_hash_for_entry` (Phase A cache / live). + +**Acceptance:** `locked_peak_delta` / `live_small_delta` latency no worse than +pre-change; `test/cmerkle_account_map_diff_test.exs` and +`test/chain_state_merkle_test.exs` pass. + +**Rationale:** Fork installs distinct wrappers sharing `SharedState` until write; +wrapper-only equality missed the cheap path. + +--- + +### Phase C — NIF returns storage roots; Elixir stops double fetch + +**NIF entry shape for `account_map_difference_full`:** + +| Version | Term | +|---------|------| +| Before | `{addr, side_a, side_b, storage_diff}` | +| After (MUST) | `{addr, side_a, side_b, storage_diff, root_a, root_b}` | + +- `root_*`: 32-byte binary, or atom `nil` if that side is absent. +- Roots MUST be the storage roots for each present side (from cache / live tree), + computed during the diff without requiring Elixir callbacks. + +**Elixir `Chain.State.difference/2` MUST:** + +- Pattern-match the 6-tuple. +- Set `:root_hash` from `{root_a, root_b}` when storage diff is non-empty (decode + `nil` sides consistently with absent accounts). +- **MUST NOT** call `CAccountMap.storage_root_hash/2` for those roots. + +**`account_map_storage_roots` / storage root reads MUST:** + +- If compact + `has_root`, return the cached root **without** `materialize_storage`. +- Otherwise existing behavior (materialize or compute-and-cache). + +Update: `lib/caccount_map.ex`, `lib/cmerkletree.ex`, and every consumer that +pattern-matches `difference_full` tuples (tests, fuzz, stress). + +**Acceptance:** Identical `State.difference` report maps vs pre-change on existing +suites; `elixir_ms` ≈ 0 on storage-touching bench deltas; root-only reads do not +force materialize when cache hit. + +--- + +### Phase D — `shared_ptr` COW for `CompactStorage` + +**MUST:** + +- Replace `unique_ptr` with `shared_ptr` on + `AccountEntry` and `DiffAccountSide`. +- Copy / assign / `fork_shared_accountmap` (via `accounts = src->accounts`): share + pointer only — no `slots = src->slots` deep copy in the default copy path. +- COW before mutating slots or invalidating root when `use_count() > 1`. +- `snapshot_side`: share compact pointer; keep live storage `enif_keep_resource` + as today. + +**Acceptance:** Clone of compact peak (14k accounts × 32 slots): RSS growth +order-of-magnitude below today’s full slot duplication (target: well under ~2× +full slot payload duplication). `test/cmerkle_nif_leak_test.exs` passes. + +**Rationale:** Live storage already COWs via `SharedState::has_clone`; compact was +the outlier that doubled RAM on every `State.clone/1`. + +--- + +### Phase E — `state_trie`-driven `difference_full` + +**Algorithm MUST:** + +1. If `am_a->shared == am_b->shared` → return `[]` (existing). +2. If both maps’ `state_trie->shared_state` pointers are identical → return `[]`. +3. Else build the candidate address set from the **symmetric difference of + state_trie leaves** (hash differs, or key only on one side). MUST NOT scan all + `accounts` entries solely to run `entries_equal` on unchanged hashes. +4. For each candidate only: snapshot sides + `build_storage_diff_list` as today; + emit Phase C 6-tuples. +5. Accounts present in one map but missing from the other MUST still appear + (trie miss / accounts lookup as needed for that address). + +**Acceptance:** +`compact_small_delta` and `live_small_delta` with `--accounts 14000 --changed 20`: +avg `nif_ms` **< 20**. Full-map equality walk absent from profiles. + +**Rationale:** Account RLP hash embeds storage root; trie leaf inequality is a +sound filter for “this address needs a deep diff” under the state_trie invariant. + +--- + +## Out of Scope + +- BertInt / on-disk block state encoding changes +- Replacing `Tree::difference` with a new algorithm +- Pure slot-vector diff without trees for fat-storage accounts (may follow later) +- Changing the public Elixir `State.difference/2` report map shape + +--- + +## Testing + +### Perf gates (`scripts/state_diff_bench.exs`) + +| Phase | Scenario / flags | Gate | +|-------|------------------|------| +| A | `compact_small_delta` 14k / 20 / 32 | avg `nif_ms` < 50 | +| E | `compact_small_delta` and `live_small_delta` 14k / 20 | avg `nif_ms` < 20 | +| C | storage-touching scenarios | `elixir_ms` ≈ 0 | + +### Correctness suites + +```text +mix test test/cmerkle_account_map_diff_test.exs \ + test/chain_state_merkle_test.exs \ + test/cmerkle_lock_clone_regression_test.exs +``` + +### Memory + +- `mix test test/cmerkle_nif_leak_test.exs` +- Phase D: optional RSS comparison via bench / leak harness notes + +### Required new unit cases + +| Name | Assert | +|------|--------| +| `cached_root_after_uncompact` | Uncompact with `:root_hash` → `storage_root_hash` does not rebuild (timing or test hook / root equality without materialize) | +| `root_invalidated_after_storage_put` | After `storage_put_map`, root changes / cache invalidated | +| `cow_unique_after_write` | Shared compact after clone; write on one fork does not change the other’s slots/root | +| `difference_full_six_tuple` | Decode `{addr, side_a, side_b, diff, root_a, root_b}` | + +### Integration + +- Existing chain state / `prepare_state` round-trips remain green. +- Implementations MAY add tests; suites above MUST pass unchanged in intent. + +--- + +## Integration Docs + +**Where it lives:** NIF hot path in `c_src/nif.cpp`; Elixir orchestration in +`lib/chain/state.ex` (`difference/2`); SQL writer path +`Model.ChainSql.prepare_state/2` calls `State.difference/2` for non-jump blocks. + +**How to call:** Unchanged for product code — `Chain.State.difference/2` / +`apply_difference/2`. Benchmark: + +```bash +mix run --no-start scripts/state_diff_bench.exs -- \ + --scenario compact_small_delta --accounts 14000 --changed 20 --slots 32 +``` + +**Migration:** No on-disk migration. Deploy is a rolling binary upgrade. Phase C +changes only the internal NIF tuple; keep wrappers and tests in the same commit +as the NIF. + +**Agent notes:** See also `docs/caccount-map-nif.md` (ownership, clone/lock). + +--- + +## Implementation Checklist + +- [ ] Phase A: `CompactStorage` root cache + invalidation + uncompact seed +- [ ] Phase A bench gate (`nif_ms` < 50) +- [ ] Phase B: `SharedState*` equality in `entries_equal` +- [ ] Phase C: 6-tuple NIF + Elixir stops double `storage_root_hash` +- [ ] Phase C: `storage_roots` uses compact cache (no materialize-for-root) +- [ ] Phase D: `shared_ptr` COW; fork no longer deep-copies slots +- [ ] Phase D leak / RSS acceptance +- [ ] Phase E: state_trie-driven candidate set; `nif_ms` < 20 +- [ ] All listed correctness tests green +- [ ] New unit cases above +- [ ] `docs/caccount-map-nif.md` updated for cache, COW, trie-driven diff +- [ ] Each phase mergeable alone with tests green + +--- + +## Version History + +- **v0.1.0** — Initial specification (Phases A–E locked). diff --git a/scripts/state_diff_bench.exs b/scripts/state_diff_bench.exs new file mode 100644 index 0000000..152af26 --- /dev/null +++ b/scripts/state_diff_bench.exs @@ -0,0 +1,376 @@ +# Benchmark / profile Chain.State.difference (the path behind +# "State diff took longer than 1s ... accounts=N"). +# +# prepare_state on non-jump blocks does: +# State.difference(prev_state, block_state) +# which is mostly CAccountMap.difference_full/2 plus light Elixir wrapping +# (decode_storage_diff + storage_root_hash per changed account). +# +# Typical prod warning shape is few changed accounts (e.g. 20) but multi-second +# wall time — difference_full still walks the full account map and, for equal +# nonce/balance/code, recomputes both storage roots (see entries_equal in nif.cpp). +# +# Run from repo root (no full app start): +# mix run --no-start scripts/state_diff_bench.exs +# mix run --no-start scripts/state_diff_bench.exs -- --accounts 5000 --changed 20 --slots 8 +# mix run --no-start scripts/state_diff_bench.exs -- --scenario all --warmup 1 --iters 3 +# mix run --no-start scripts/state_diff_bench.exs -- --eprof +# +# Scenarios: +# live_small_delta — large live map, mutate K accounts +# locked_peak_delta — lock peak then clone+mutate (prepare_state / writer pattern) +# compact_small_delta — jump-shaped compact→uncompact peak + small delta (prod hotspot) +# storage_only_delta — storage writes only (no nonce bump) +# fat_storage — few accounts, huge storage trees (storage-diff dominated) +# all — run every scenario + +defmodule StateDiffBench do + @moduledoc false + + alias Chain.State + + def main(argv \\ System.argv()) do + argv = normalize_argv(argv) + + {opts, _, _} = + OptionParser.parse(argv, + strict: [ + accounts: :integer, + changed: :integer, + slots: :integer, + fat_slots: :integer, + warmup: :integer, + iters: :integer, + scenario: :string, + eprof: :boolean, + csv: :boolean + ] + ) + + accounts = Keyword.get(opts, :accounts, 5_000) + changed = Keyword.get(opts, :changed, 20) + slots = Keyword.get(opts, :slots, 8) + fat_slots = Keyword.get(opts, :fat_slots, 8_000) + warmup = Keyword.get(opts, :warmup, 1) + iters = Keyword.get(opts, :iters, 3) + scenario = Keyword.get(opts, :scenario, "all") + eprof? = Keyword.get(opts, :eprof, false) + csv? = Keyword.get(opts, :csv, false) + + _ = CAccountMap.new() + + IO.puts(:stderr, """ + === State.difference bench === + accounts=#{accounts} changed=#{changed} slots/account=#{slots} fat_slots=#{fat_slots} + warmup=#{warmup} iters=#{iters} scenario=#{scenario} eprof=#{eprof?} + """) + + scenarios = + case scenario do + "all" -> + [ + {:live_small_delta, fn -> build_live_small_delta(accounts, changed, slots) end}, + {:locked_peak_delta, fn -> build_locked_peak_delta(accounts, changed, slots) end}, + {:compact_small_delta, fn -> build_compact_small_delta(accounts, changed, slots) end}, + {:storage_only_delta, fn -> build_storage_only_delta(accounts, changed, slots) end}, + {:fat_storage, fn -> build_fat_storage(min(accounts, 40), changed, fat_slots) end} + ] + + name -> + builder = + case name do + "live_small_delta" -> fn -> build_live_small_delta(accounts, changed, slots) end + "locked_peak_delta" -> fn -> build_locked_peak_delta(accounts, changed, slots) end + "compact_small_delta" -> fn -> build_compact_small_delta(accounts, changed, slots) end + "compact_vs_live" -> fn -> build_compact_small_delta(accounts, changed, slots) end + "storage_only_delta" -> fn -> build_storage_only_delta(accounts, changed, slots) end + "fat_storage" -> fn -> build_fat_storage(min(accounts, 40), changed, fat_slots) end + other -> raise "unknown --scenario #{inspect(other)}" + end + + [{String.to_atom(name), builder}] + end + + if csv? do + IO.puts( + "scenario,iter,map_size,changed,nif_ms,elixir_ms,total_ms,delta_accounts,storage_keys,rss_kb" + ) + end + + Enum.each(scenarios, fn {name, builder} -> + IO.puts(:stderr, "--- building #{name} ---") + {build_us, {prev, next}} = :timer.tc(builder) + map_size = CAccountMap.size(prev.accounts) + + IO.puts( + :stderr, + "built #{name} in #{div(build_us, 1000)}ms map_size=#{map_size} rss=#{div(read_rss_kb(), 1024)}MB" + ) + + for _ <- 1..warmup do + _ = timed_difference(prev, next) + end + + if eprof? do + profile_eprof(name, prev, next) + end + + samples = + for i <- 1..iters do + sample = timed_difference(prev, next) + print_sample(name, i, map_size, sample, csv?) + sample + end + + summarize(name, samples) + end) + + IO.puts(:stderr, """ + + Interpretation hints: + - compact_small_delta nif-dominated with changed << map_size: entries_equal calls + storage_root_hash_for_entry on every unchanged account, rebuilding a temp Tree from + compact slots (nif.cpp). Matches "State diff took longer than 1s ... accounts=20". + - live_small_delta staying fast: live trees use cached root_hash / pointer checks. + - elixir_ms large: decode_storage_diff + State.difference's per-account storage_root_hash. + - fat_storage nif-heavy with small map_size: build_storage_diff_list / Tree::difference. + """) + end + + defp timed_difference(prev, next) do + # Single wall-time sample with nested phase timers (avoids nif_ms > total_ms). + {total_us, {nif_us, elixir_us, result}} = + :timer.tc(fn -> + {nif_us, full} = + :timer.tc(fn -> + CAccountMap.difference_full(prev.accounts, next.accounts) + end) + + {elixir_us, result} = + :timer.tc(fn -> + Enum.map(full, fn {id, side_a, side_b, state_diff} -> + report = + %{} + |> put_field(:nonce, side_a, side_b) + |> put_field(:balance, side_a, side_b) + |> put_field(:code, side_a, side_b) + + storage_map = CAccountMap.decode_storage_diff(state_diff) + + report = + if map_size(storage_map) > 0 do + Map.merge(report, %{ + state: storage_map, + root_hash: { + CAccountMap.storage_root_hash(prev.accounts, id), + CAccountMap.storage_root_hash(next.accounts, id) + } + }) + else + report + end + + {id, report} + end) + end) + + {nif_us, elixir_us, result} + end) + + storage_keys = + Enum.reduce(result, 0, fn {_id, report}, acc -> + acc + map_size(Map.get(report, :state, %{})) + end) + + %{ + nif_ms: div(nif_us, 1000), + elixir_ms: div(elixir_us, 1000), + total_ms: div(total_us, 1000), + delta_accounts: length(result), + storage_keys: storage_keys, + rss_kb: read_rss_kb() + } + end + + defp put_field(report, field, side_a, side_b) do + a = side_field(side_a, field) + b = side_field(side_b, field) + + if a == b do + report + else + Map.put(report, field, {a, b}) + end + end + + defp side_field(nil, :nonce), do: 0 + defp side_field(nil, :balance), do: 0 + defp side_field(nil, :code), do: "" + defp side_field({nonce, _, _}, :nonce), do: nonce + defp side_field({_, balance, _}, :balance) when is_integer(balance), do: balance + + defp side_field({_, balance, _}, :balance) when is_binary(balance), + do: :binary.decode_unsigned(balance) + + defp side_field({_, _, code}, :code), do: code + + defp print_sample(name, i, map_size, sample, true) do + IO.puts( + "#{name},#{i},#{map_size},#{sample.delta_accounts},#{sample.nif_ms},#{sample.elixir_ms},#{sample.total_ms},#{sample.delta_accounts},#{sample.storage_keys},#{sample.rss_kb}" + ) + end + + defp print_sample(name, i, map_size, sample, false) do + nif_pct = + if sample.total_ms > 0 do + Float.round(100.0 * sample.nif_ms / sample.total_ms, 1) + else + 0.0 + end + + IO.puts(:stderr, """ + #{name} iter=#{i} map_size=#{map_size} changed=#{sample.delta_accounts} storage_keys=#{sample.storage_keys} + nif_ms=#{sample.nif_ms} elixir_ms=#{sample.elixir_ms} total_ms=#{sample.total_ms} (nif=#{nif_pct}%) + rss_mb=#{div(sample.rss_kb, 1024)} + """) + end + + defp summarize(name, samples) do + n = length(samples) + avg = fn key -> div(Enum.sum(Enum.map(samples, &Map.fetch!(&1, key))), n) end + + IO.puts( + :stderr, + "SUMMARY #{name} avg_nif_ms=#{avg.(:nif_ms)} avg_elixir_ms=#{avg.(:elixir_ms)} " <> + "avg_total_ms=#{avg.(:total_ms)} changed=#{hd(samples).delta_accounts}" + ) + end + + defp profile_eprof(name, prev, next) do + IO.puts(:stderr, "eprof #{name} (State.difference)...") + :eprof.start() + :eprof.profile(fn -> State.difference(prev, next) end) + :eprof.analyze(:total) + :eprof.stop() + end + + # --- builders (prepare_state-shaped) --- + + defp build_live_small_delta(n, changed, slots) do + prev = build_live_state(n, slots) + next = mutate(prev, changed, slots) + {prev, next} + end + + defp build_locked_peak_delta(n, changed, slots) do + peak = build_live_state(n, slots) |> State.normalize() |> State.lock() + next = mutate(peak, changed, slots) + {peak, next} + end + + # Jump-block shaped: uncompact leaves compact_storage slots; unchanged accounts + # pay storage_root_hash_for_entry by rebuilding a temp Tree from every slot + # (nif.cpp entries_equal). This is the usual multi-second / few-account warning. + defp build_compact_small_delta(n, changed, slots) do + peak = build_live_state(n, slots) |> State.normalize() |> State.lock() + prev = peak |> State.compact() |> State.uncompact() |> State.lock() + next = mutate(prev, changed, slots) + {prev, next} + end + + # Storage-only mutations (no nonce bump) so changed rows also go through + # storage root comparison inside entries_equal. + defp build_storage_only_delta(n, changed, slots) do + prev = build_live_state(n, slots) |> State.normalize() |> State.lock() + + next = + prev + |> State.clone() + |> then(fn st -> + Enum.reduce(1..changed, st, fn i, acc -> + id = addr(i) + + updates = + for s <- 1..max(div(slots, 4), 1), into: %{} do + {slot(i * 10_000 + s), <>} + end + + State.storage_put_map(acc, %{id => updates}) + end) + end) + |> State.normalize() + + {prev, next} + end + + defp build_fat_storage(n, changed, fat_slots) do + prev = build_live_state(n, fat_slots) + next = mutate(prev, min(changed, n), fat_slots) + {prev, next} + end + + defp build_live_state(n, slots_per) do + Enum.reduce(1..n, State.new(), fn i, st -> + updates = + for s <- 1..slots_per, into: %{} do + {slot(i * 10_000 + s), <>} + end + + State.set_account(st, addr(i), %Chain.Account{ + nonce: i, + balance: i * 100, + storage_root: Map.to_list(updates), + code: <>, + map_backed: false + }) + end) + |> State.normalize() + end + + defp mutate(state, changed, slots_per) do + state + |> State.clone() + |> then(fn st -> + Enum.reduce(1..changed, st, fn i, acc -> + id = addr(i) + + updates = + for s <- 1..max(div(slots_per, 4), 1), into: %{} do + {slot(i * 10_000 + s), <>} + end + + # Bump nonce so changed rows exit entries_equal early; unchanged rows still + # pay storage_root_hash_for_entry across the full map (the prod hotspot). + acc = State.storage_put_map(acc, %{id => updates}) + meta = State.ensure_account(acc, id) + State.set_account(acc, id, %{meta | nonce: meta.nonce + 1}) + end) + end) + |> State.normalize() + end + + defp addr(i), do: <> + defp slot(i), do: <> + + defp normalize_argv(argv) do + case argv do + ["--" | rest] -> rest + other -> other + end + end + + defp read_rss_kb do + case File.read("/proc/self/status") do + {:ok, body} -> + case Regex.run(~r/^VmRSS:\s+(\d+)\s+kB/m, body) do + [_, n] -> String.to_integer(n) + _ -> 0 + end + + _ -> + 0 + end + end +end + +StateDiffBench.main(System.argv()) From 700bbf0dd42cd099ba34382443ab37b652cd7b1c Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Fri, 17 Jul 2026 01:26:00 +0200 Subject: [PATCH 12/16] Drop bare-tree test NIFs and the CMERKLE_TEST_NIFS build gate. Production already uses account-map APIs only; remove gated bare-tree exports, legacy merkle benches, and update fuzz/stress/tests to the map-native surface. Co-authored-by: Cursor --- .gitignore | 2 - AGENTS.md | 5 +- Makefile | 1 - c_src/LOCK_ORDER.md | 84 +- c_src/Makefile | 28 +- c_src/SECURITY_REVIEW.md | 36 +- c_src/merkletree.cpp | 19 - c_src/merkletree.hpp | 3 - c_src/nif.cpp | 843 +------------------- docs/caccount-map-nif.md | 11 +- lib/bench.ex | 6 - lib/chain/account.ex | 7 +- lib/cmerkletree.ex | 107 +-- scripts/cmerkle_bench.exs | 76 -- scripts/cmerkle_bench2.exs | 42 - scripts/cmerkle_bench3.exs | 28 - scripts/cmerkle_deadlock_watchdog.exs | 4 +- scripts/cmerkle_fuzz.exs | 495 ++---------- scripts/cmerkle_fuzz.sh | 4 +- scripts/cmerkle_heap_assumptions.exs | 333 -------- scripts/cmerkle_leak_test.exs | 26 +- scripts/cmerkle_leak_watchdog.exs | 4 +- scripts/cmerkle_memory_bench.exs | 192 ----- scripts/cmerkle_memory_evaluation_report.md | 132 --- scripts/cmerkle_parallel_stress.exs | 580 ++------------ scripts/merkle_asan_and_recovery.md | 2 +- scripts/merkle_bench.exs | 27 - scripts/profile_uncompact_nif.exs | 4 +- test/caccount_map_lifetime_test.exs | 28 +- test/caccount_map_test.exs | 25 +- test/chain_account_hash_nif_test.exs | 77 +- test/chain_state_merkle_test.exs | 36 +- test/chain_state_uncompact_test.exs | 62 +- test/chain_test.exs | 17 +- test/cmerkle_account_map_diff_test.exs | 22 +- test/cmerkle_lock_clone_regression_test.exs | 28 +- test/cmerkle_lock_concurrency_test.exs | 146 +--- test/cmerkle_nif_deadlock_test.exs | 243 +----- test/cmerkle_nif_leak_test.exs | 30 +- test/cmerkle_storage_map_test.exs | 2 +- test/cmerkletree_test.exs | 541 ------------- test/count_zeros_test.exs | 26 + test/evm_storage_readahead_test.exs | 13 +- test/evm_test.exs | 12 +- 44 files changed, 456 insertions(+), 3953 deletions(-) delete mode 100644 scripts/cmerkle_bench.exs delete mode 100644 scripts/cmerkle_bench2.exs delete mode 100644 scripts/cmerkle_bench3.exs delete mode 100644 scripts/cmerkle_heap_assumptions.exs delete mode 100644 scripts/cmerkle_memory_bench.exs delete mode 100644 scripts/cmerkle_memory_evaluation_report.md delete mode 100644 scripts/merkle_bench.exs delete mode 100644 test/cmerkletree_test.exs create mode 100644 test/count_zeros_test.exs diff --git a/.gitignore b/.gitignore index ae23180..f288d62 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,6 @@ /priv/merkletree_nif.so /priv/merkletree_nif.asan.so /priv/merkletree_nif.so.bak* -/priv/.cmerkle_nif_mode - # Profiling / measurement outputs (see scripts/profile_*.sh) /tmp/ /leak_state.bin diff --git a/AGENTS.md b/AGENTS.md index 28fbb0e..a9da64a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,9 +28,8 @@ Ethereum-compatible JSON-RPC endpoint plus the Diode PEER/EDGE protocols. `make -C deps/libsecp256k1/` (see `.github/workflows/ci.yml`). Build artifacts are gitignored and persist across sessions, so this is only needed after a clean checkout of that dep. -- **CAccountMap / state NIF semantics** (clone, lock, storage APIs, get shape, - `CMERKLE_TEST_NIFS` prod vs test exports): see - [`docs/caccount-map-nif.md`](docs/caccount-map-nif.md). +- **CAccountMap / state NIF semantics** (clone, lock, storage APIs, get shape): + see [`docs/caccount-map-nif.md`](docs/caccount-map-nif.md). ### Lint - `mix lint` = `compile` + `mix format --check-formatted` + `mix credo --only warning` + `mix dialyzer`. diff --git a/Makefile b/Makefile index 75f54fb..a2eb7c7 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,6 @@ TESTDATA := test/pems/device1_certificate.pem test/pems/device2_certificate.pem .PHONY: all all: evm/evm priv/merkletree_nif.so -# Always delegate to c_src so CMERKLE_TEST_NIFS mode stamp can force rebuild. .PHONY: priv/merkletree_nif.so priv/merkletree_nif.so: $(MAKE) -C c_src nif diff --git a/c_src/LOCK_ORDER.md b/c_src/LOCK_ORDER.md index 612a659..73cca32 100644 --- a/c_src/LOCK_ORDER.md +++ b/c_src/LOCK_ORDER.md @@ -1,83 +1,54 @@ -# CMerkleTree NIF — lock order and deadlock scenario registry +# CAccountMap NIF — lock order and deadlock scenario registry -This document inventories mutex layers in [`nif.cpp`](nif.cpp), documents acquisition order, and maps every known deadlock / liveness scenario to regression tests. +This document inventories mutex layers in [`nif.cpp`](nif.cpp), documents acquisition order, and maps known deadlock / liveness scenarios to regression tests. -See also [`SECURITY_REVIEW.md`](SECURITY_REVIEW.md) (F-5 fix) and [`scripts/cmerkle_parallel_stress.exs`](../scripts/cmerkle_parallel_stress.exs). +See also [`SECURITY_REVIEW.md`](SECURITY_REVIEW.md) and [`scripts/cmerkle_parallel_stress.exs`](../scripts/cmerkle_parallel_stress.exs). ## Mutex layers | Mutex | Type | Scope | Used by | |-------|------|-------|---------| -| `LockedStates::mtx` | `ErlNifMutex` | Global | `enter_lock`, `leave_lock`, `destruct_merkletree_type` | -| `SharedState::mtx` | `ErlNifMutex` | Per trie | `Lock(mt)` — insert, get, difference, clone, etc. | +| `SharedState::mtx` | `ErlNifMutex` | Per storage / state trie | `Lock(mt)` during COW / storage reads / root hash; destructor `release_merkletree_shared` | | `SharedAccountMap::mtx` | `ErlNifMutex` | Per account map | `AccountMapLock` | | `GlobalStripePool::s_mtx` | `std::mutex` | Process-global | PreAllocator stripe reuse | | `ItemPool` internal | `std::recursive_mutex` | Per SharedState pool | COW / pair allocation | | `PreAllocator` internal | `std::recursive_mutex` | Per tree stripe | Pair slab mutation | -| `stats_mutex` | `ErlNifMutex` | Global | DEBUG stats only | +| `stats_mutex` | `ErlNifMutex` | Global | DEBUG stats + `nif_stats_raw` live counters | -**Independent domains:** `AccountMapLock` and `LockedStates::mtx` are never held together in current NIF code. +Bare-tree Elixir NIFs (`new` / `insert` / `difference_raw` / `lock` / …) are gone. Storage tries exist only as map-owned internals. The old global `LockedStates` / orphan queue was removed with them. ## Lock acquisition rules | Path | Order | Notes | |------|-------|-------| -| `difference_raw` | `locked_states_mutex` → snapshot pointers + `read_pins` → **release global** → `SharedState*` mutexes (address order) → unpin `read_pins` | `read_pins` prevents COW from consuming `difference` lifetime while global is dropped | -| `enter_lock` (dedup) | tree (`mt->locked`) → `locked_states_mutex` → pin `read_pins` on canonical → **release global** → bump `has_clone` / canonical switch | `read_pins` prevents reclaim until registration ref is taken | -| `leave_lock` / GC destructor | if `mt->locked`: `locked_states_mutex` → tree → decrement registration → erase map when `has_clone == 0`; else tree refcount only | Unlocked resources never touch `LockedStates` map | -| `merkletree_clone` | `Lock(parent)` | Dirty CPU scheduler; O(1) shallow resource alloc (`locked = false`) | +| Storage trie GC destructor | `release_merkletree_shared`: tree mutex → drop `has_clone` / delete when 0 | Immediate reclaim; no deferred orphan queue | | `account_map_clone` | `AccountMapLock` → `fork_shared_accountmap` (`Lock` per parent storage / state_trie) | Dirty scheduler; writable fork even if parent `frozen` | | `account_map_lock` | `AccountMapLock` → set `frozen=true` only (O(1); no per-trie seal) | Dirty scheduler; put/delete/storage_put_map reject via `frozen` | | `account_map_storage_put_map` | `AccountMapLock` → reject if frozen → per-addr `write_storage_slot` → `update_state_trie_for_entry` | Dirty CPU; EVM `su` hot path | | `account_map_storage` | `AccountMapLock` → read-only storage query (`:get`/`:range`/`:list`/`:size`) | Dirty CPU | | `account_map_storage_roots` / `account_map_state_roots` | `AccountMapLock` → tree lock → root + 16 hashes blob | No live trie export | | `account_map_proof` | `AccountMapLock` → account or storage proof | Dirty CPU; arities 2 and 3 | -| `switch_local_to_canonical` | tree mutexes (address order) | Abandoned `SharedState` queued on `pending_orphans`; reclaimed via `try_reclaim_orphans` after `enter_lock` / `leave_lock` when mutex trylock succeeds and `has_clone == 0` | | `account_map_uncompact_state` | `AccountMapLock(input)` → `materialize_storage` (brief tree lock) → `batch_insert` (state_store lock) | Dirty scheduler | -| `account_map_compact` | `AccountMapLock` (read-only; OK frozen) → per-account storage list via live tree lock or compact_storage slots (no materialize) | Dirty CPU; single boundary crossing for `Chain.State.compact/1` | -| `account_map_put/delete` | `AccountMapLock` only; reject if `frozen`; storage arg may be `:keep` / list / resource | May `release_resource` → async GC `leave_lock` | +| `account_map_compact` | `AccountMapLock` (read-only; OK frozen) → per-account storage list via live tree lock or compact_storage slots | Dirty CPU | +| `account_map_put/delete` | `AccountMapLock` only; reject if `frozen`; storage arg may be `:keep` / list / `nil` | May `release_resource` → async GC `release_merkletree_shared` | | `account_map_difference_full` | Dual map lock (`DualAccountMapLock`, address order) → snapshot sides → release → per-account storage diffs | Dirty CPU; never hold map lock across storage diff build | | `account_map_apply_difference` | `AccountMapLock` → reject if `frozen` → storage/field writes (`write_storage_slot` → `make_writeable_locked`) | Dirty CPU | -| Insert / COW | Tree lock → ItemPool / PreAllocator / stripe pool | Same-thread nesting | +| Insert / COW (internal) | Tree lock → ItemPool / PreAllocator / stripe pool | Same-thread nesting | ## Deadlock scenario registry -Each scenario has an ID, hypothesis, and test coverage target. - -### A. Tree mutex + `LockedStates::mtx` - -| ID | Scenario | Hypothesis | Covered by | -|----|----------|------------|------------| -| D-A1 | `enter_lock` dedup + `difference` on overlapping SharedStates | Global held during second tree wait (pre-fix F-5) | P9, S9, ExUnit lock concurrency | -| D-A2 | `leave_lock` (GC) + `difference` on same tree | Liveness stall: leave waits tree; diff never waits global | P11, S14, ExUnit D-A2 | -| D-A3 | `leave_lock` + `enter_lock` phase1 on same tree | Serialized on global | P10 smoke | -| D-A4 | `leave_lock`(global→T) + `enter_lock` switch(T,U) + `difference`(T,U) | Three-way liveness under heavy GC | P10 | -| D-A5 | Many concurrent `leave_lock` on distinct trees | Global mutex convoy | P11 | -| D-A6 | `enter_lock` dedup from 50+ clones | Canonical switch / refcount edge | P6, P14, ExUnit D-F2 | -| D-A7 | `lock` on already-locked tree | Re-entrant / double enter | S8 | - -### B. Dual-tree ordering (`difference_raw`) - -| ID | Scenario | Hypothesis | Covered by | -|----|----------|------------|------------| -| D-B1 | `difference(A,B)` vs `difference(B,A)` | Same address order | P2, P3 | -| D-B2 | `difference(A,B)` vs `difference(A,C)` — shared first tree | Second lock order by B vs C address | P10, ExUnit D-B2 | -| D-B3 | Long `difference` + concurrent insert | Insert waits tree held by difference | P5 | -| D-B4 | `difference` on same SharedState (early return) | No dual lock | S9 smoke | -| D-B5 | `difference` while `switch_local_to_canonical` holds both trees | Post-F-5 residual | P10 | - ### C. Account map × tree mutex | ID | Scenario | Risk | Covered by | |----|----------|------|------------| -| D-C1 | `account_map_clone` + `difference` on shared storage | Tree mutex contention | P13 | -| D-C2 | `account_map_uncompact_state` + per-account `difference` | Independent domains unless storage shared | P12, S12, ExUnit D-C2 | -| D-C3 | `account_map_get` / `to_list` (root hash export) + `difference` | No live storage export; brief hash compute | S13 | -| D-C4 | `account_map_put` replacing storage + GC `leave_lock` | Async GC vs diff | P13 | -| D-C5 | `account_map_lock` / `account_map_clone` + concurrent `State.lock/1` | Correctness / writable fork | P14, P15, chain_state_uncompact_test, ExUnit D-C5, fuzz 16–18 | -| D-C6 | Concurrent `put` / `apply_difference` on frozen map | Rejected via `make_writeable_accountmap` | TSan on P4, P13 | -| D-C7 | `account_map_difference_full` + `account_map_to_list` same map | Map mutex convoy / materialize stall | S20, ExUnit D-D7, D-C7 | -| D-C8 | Dual-map `difference_full` lock order (A,B) vs (B,A) | Ordering regression | S24, ExUnit D-C8 | +| D-C1 | `account_map_clone` + `difference_full` on shared storage | Tree mutex contention | P13 | +| D-C2 | `account_map_uncompact_state` + `difference_full` | Independent domains unless storage shared | P12, fuzz S3, ExUnit D-C2 | +| D-C3 | `account_map_get` / `to_list` (root hash export) + `difference_full` | No live storage export; brief hash compute | fuzz S4 | +| D-C4 | `account_map_put` replacing storage + GC `release_merkletree_shared` | Async GC vs diff | P13 | +| D-C5 | `account_map_lock` / `account_map_clone` + concurrent `State.lock/1` | Correctness / writable fork | P14, P15, chain_state_uncompact_test, ExUnit D-C5, fuzz 6–8 | +| D-C6 | Concurrent `put` / `apply_difference` on frozen map | Rejected via `make_writeable_accountmap` | TSan on P13 | +| D-C7 | `account_map_difference_full` + `account_map_to_list` same map | Map mutex convoy / materialize stall | fuzz S10, ExUnit D-D7, D-C7 | +| D-C8 | Dual-map `difference_full` lock order (A,B) vs (B,A) | Ordering regression | fuzz S14, ExUnit D-C8 | ### D. Production composite (block sync) @@ -86,36 +57,31 @@ Each scenario has an ID, hypothesis, and test coverage target. | D-D1 | `Chain.State.difference/2` (storage diffs per account) | P14 | | D-D2 | `State.lock/1` sets map `frozen` only (no per-trie seal; get exports root hashes not live storage) | P14 | | D-D3 | D-D1 + D-D2 + uncompact concurrent | P14, P12, ExUnit D-D3 | -| D-D4 | compact → uncompact → clone → apply_difference | P14, S10, S11 | -| D-D5 | Storage get/insert + block import | P1, P5 | -| D-D6 | Dirty scheduler saturation | P15 | -| D-D7 | prepare_state composite (native diff + lock + legacy to_list) | S30, P17, ExUnit D-D7 | -| D-D8 | `difference_full` on compact storage | account_map_diff_test, S19 | +| D-D4 | compact → uncompact → clone → apply_difference | P14, fuzz S1–S2 | +| D-D6 | Dirty scheduler saturation | P15, P20 | +| D-D7 | prepare_state composite (native diff + lock + to_list) | fuzz S20, P17, ExUnit D-D7 | +| D-D8 | `difference_full` on compact storage | account_map_diff_test, fuzz S9 | | D-L1 | `clone` after lock + storage_put_map + discard | `cmerkle_storage_map_test`, `cmerkle_lock_clone_regression_test`, P18L | | D-L2 | `difference_full` + `apply_difference` round-trip | chain_state_merkle_test, account_map_diff_test | | D-M1 | frozen map + eager clone writable fork | chain_state_merkle_test lock→clone | -| D-M2 | concurrent difference_raw + apply_difference | lock concurrency, stress | ### E. C++ internal mutexes | ID | Scenario | Covered by | |----|----------|------------| -| D-E1 | Concurrent clone+insert on COW SharedState | P4 + TSan | -| D-E2 | `difference` + insert allocating pairs | P5 | -| D-E3 | Parallel tree discard + stripe pool | P7, P11 | +| D-E1 | Concurrent clone+storage_put on COW SharedState | P13 + TSan | +| D-E3 | Parallel map discard + stripe pool | P12, P16 | | D-E4 | Nested ItemPool lock in `fork_for_write` | By design (recursive) | ### F. Refcount / UAF masquerading as deadlock | ID | Scenario | Covered by | |----|----------|------------| -| D-F1 | `has_clone` mismatch during canonical switch | P9, S9, ExUnit dedup | -| D-F2 | Canonical ref reserved vs concurrent `leave_lock` | P9, ExUnit D-F2 | | D-F3 | `release_storage_from_map` during delete vs diff | P13, caccount_map_lifetime_test | ## Remaining structural risk -`enter_lock` and `leave_lock` now both release the global mutex before blocking on tree mutexes. Monitor `nif_stats_raw/0` (`shared_states` vs `locked_states`) in production; sustained growth indicates a reclaim regression. `nif_stats_raw` is read-only (reclaim runs from lock/unlock paths only). +Monitor `nif_stats_raw/0` (`shared_states` / `merkletree_resources`) in production; sustained growth indicates a reclaim regression. Locked/orphan tuple slots stay zero (API shape preserved). ## CI / harness commands diff --git a/c_src/Makefile b/c_src/Makefile index 3d5dfe8..f881860 100644 --- a/c_src/Makefile +++ b/c_src/Makefile @@ -10,24 +10,6 @@ ERL_INCLUDE_PATH = $(shell erl -eval 'io:format("~s", [lists:concat([code:root_d CFLAGS+=-I. -O2 -g -Wall -Wno-unknown-pragmas CXXFLAGS+=-std=c++17 $(CFLAGS) -# Bare-tree + debug NIFs (new/insert/...), gated by -DCMERKLE_TEST_NIFS in nif.cpp. -# On unless MIX_ENV=prod; override with CMERKLE_TEST_NIFS=0 (off) or =1 (force on). -# priv/merkletree_nif.so is shared across Mix envs — last compile wins. -ENABLE_CMERKLE_TEST_NIFS := 1 -ifeq ($(MIX_ENV),prod) - ENABLE_CMERKLE_TEST_NIFS := 0 -endif -ifeq ($(CMERKLE_TEST_NIFS),0) - ENABLE_CMERKLE_TEST_NIFS := 0 -endif -ifeq ($(CMERKLE_TEST_NIFS),1) - ENABLE_CMERKLE_TEST_NIFS := 1 -endif -ifeq ($(ENABLE_CMERKLE_TEST_NIFS),1) - CXXFLAGS+=-DCMERKLE_TEST_NIFS -endif -NIF_MODE := $(if $(filter 1,$(ENABLE_CMERKLE_TEST_NIFS)),test,prod) - UNAME_S := $(shell uname -s) ifeq ($(UNAME_S),Darwin) @@ -76,21 +58,15 @@ fuzz_sha.bin: fuzz_sha.cpp sha.cpp Makefile FORCE: -# Rebuild .so when test/prod NIF mode changes (sources alone may be unchanged). -../priv/.cmerkle_nif_mode: FORCE - @mkdir -p ../priv - @echo $(NIF_MODE) > $@.new - @if ! cmp -s $@.new $@ 2>/dev/null; then mv $@.new $@; else rm -f $@.new; fi - nif: ../priv/merkletree_nif.so -../priv/merkletree_nif.so: nif.cpp merkletree.hpp merkletree.cpp item_pool.cpp sha.cpp preallocator.hpp Makefile ../priv/.cmerkle_nif_mode +../priv/merkletree_nif.so: nif.cpp merkletree.hpp merkletree.cpp item_pool.cpp sha.cpp preallocator.hpp Makefile echo ${ERL_INCLUDE_PATH} mkdir -p ../priv $(CXX) $(CXXFLAGS) -I${ERL_INCLUDE_PATH} -o ../priv/merkletree_nif.so -shared -fPIC nif.cpp ${OPTS} # Instrumented NIF for debugging heap issues (copy over priv/merkletree_nif.so or set in test script). -../priv/merkletree_nif.asan.so: nif.cpp merkletree.hpp merkletree.cpp item_pool.cpp sha.cpp preallocator.hpp Makefile ../priv/.cmerkle_nif_mode +../priv/merkletree_nif.asan.so: nif.cpp merkletree.hpp merkletree.cpp item_pool.cpp sha.cpp preallocator.hpp Makefile echo ${ERL_INCLUDE_PATH} mkdir -p ../priv $(CXX) $(CXXFLAGS) $(SANFLAGS) -I${ERL_INCLUDE_PATH} -o $@ -shared -fPIC nif.cpp $(OPTS_ASAN) diff --git a/c_src/SECURITY_REVIEW.md b/c_src/SECURITY_REVIEW.md index 40e8ca9..c02d18b 100644 --- a/c_src/SECURITY_REVIEW.md +++ b/c_src/SECURITY_REVIEW.md @@ -7,17 +7,17 @@ ## 1. NIF inventory (exports → Elixir) -### Production (always registered; ~21 entries) +All exports below are always registered (~21 entries). There is no separate bare-tree / test-only NIF mode. | NIF name | Arity | Inputs | Callers (representative) | |----------|-------|--------|---------------------------| | `count_zeros` | 1 | binary | `Evm` (tx payload) | -| `nif_stats_raw` | 0 | — | `Network.Status` | +| `nif_stats_raw` | 0 | — | `CMerkleTree.nif_stats/0`, `Network.Status`, leak/stress harnesses | | `account_map_new` | 0 | — | `CAccountMap.new/0`, `Chain.State` | | `account_map_clone` | 1 | account map resource | `CAccountMap.clone/1`, `Chain.State.clone/1` | | `account_map_lock` | 1 | account map resource | `CAccountMap.lock/1` — `frozen` only (O(1)) | | `account_map_get` | 2 | resource, 20-byte address | `{nonce, balance, storage_root_hash_bin32, code}` — never a live storage resource | -| `account_map_put` | 6 | resource, addr, nonce, balance, storage, code | Storage arg: resource \| `:keep` \| `nil`/`[]` \| `[{k,v}]`; rejects frozen | +| `account_map_put` | 6 | resource, addr, nonce, balance, storage, code | Storage arg: `:keep` \| `nil`/`[]` \| `[{k,v}]`; rejects frozen | | `account_map_delete` | 2 | resource, address | Rejects frozen | | `account_map_root_hash` | 1 | resource | `Chain.State.hash/1` | | `account_map_state_roots` | 1 | resource | 544-byte `<>`; Edge `getstateroots` | @@ -33,10 +33,6 @@ | `account_map_proof` | 2 | map, addr | Account inclusion proof | | `account_map_proof` | 3 | map, addr, key | Storage proof | -### Test/dev only (`-DCMERKLE_TEST_NIFS`; on unless `MIX_ENV=prod`) - -Bare `merkletree` resource API (`new`, `insert_item_raw`, `get_item`, `get_range_raw`, `get_proofs_raw`, `difference_raw`, `lock`, `to_list`, `import_map`, `root_hash`, `hash`, `root_hashes_raw`, `bucket_count`, `size`, `clone`) plus debug (`struct_sizes_raw`, `memory_stats_raw`, `malloc_info_raw`). Used by ExUnit, fuzz, and stress scripts. Prod release builds omit these from `nif_funcs`. - **Frozen map:** `account_map_lock/1` sets map-level `frozen` only. Map mutations (`put`/`delete`/`apply_difference`/`storage_put_map`) fail while frozen. `get` / `to_list` export hashes only. Internal `state_trie` is never exported. `clone/1` forks writable wrappers for sync and speculative RPC/Edge/Shell. **Trust:** Erlang validates some shapes (e.g. `to_bytes32`), but the NIF must treat all binaries and terms as hostile (size, allocation, scheduler impact). @@ -49,8 +45,9 @@ Bare `merkletree` resource API (`new`, `insert_item_raw`, `get_item`, `get_range | ID | Topic | Severity | CWE | Notes / mitigation | |----|--------|----------|-----|---------------------| -| F-1 | **`merkletree_import_map` map iterator leak** | High (resource leak / undefined behavior risk) | CWE-404 / CWE-775 | Early `return enif_make_badarg` inside `while` skipped `enif_map_iterator_destroy`. **Fixed:** `goto import_badarg` path destroys iterator. | -| F-2 | **`merkletree_difference` lock order** | High (deadlock) | CWE-833 | Concurrent `difference_raw(A,B)` vs `difference_raw(B,A)` could lock two trees in opposite order. **Fixed:** lock `first`/`second` by `SharedState*` address order, then use ordered locks. | +| F-1 | **Former `merkletree_import_map` map iterator leak** | High (historical) | CWE-404 / CWE-775 | Early `return enif_make_badarg` inside `while` skipped `enif_map_iterator_destroy`. **Fixed** before bare-tree export removal. | +| F-2 | **Former `merkletree_difference` lock order** | High (historical) | CWE-833 | Concurrent opposite-order dual-tree locks. **Fixed** with address-ordered locks. Account-map path uses `DualAccountMapLock` by map address. | +| F-2b | **`account_map_difference_full` dual-map order** | High (deadlock) | CWE-833 | Concurrent `(A,B)` vs `(B,A)` must lock maps in SharedAccountMap\* address order. Covered by fuzz S14 / ExUnit D-C8. | ### Medium @@ -58,24 +55,16 @@ Bare `merkletree` resource API (`new`, `insert_item_raw`, `get_item`, `get_range |----|--------|----------|-----|--------| | F-3 | **`enif_binary_to_term` in `make_proof`** | Medium | CWE-502 / CWE-400 | Decodes Erlang term bytes embedded in proofs (`proof.type == 2`). Malicious or huge terms can stress atom table / allocation. Mitigations: trust only proofs from your own tree; consider max depth/size for `make_proof` recursion; optional caps via external format limits. | | F-4 | **Recursive `make_proof` / `do_get_proofs`** | Low–medium | CWE-674 | Depth follows trie height (bounded by key path; practical depth large for adversarial trie). Stack exhaustion theoretically possible on extreme trees; monitor if accepting untrusted trees. | -| F-5 | **Interaction `LockedStates::mtx` vs tree mutexes** | Medium | CWE-833 | **Fixed:** `enter_lock` pins `has_clone` on local/canonical under global+tree lock, drops global before `switch_local_to_canonical`, and map entries hold a `has_clone` ref. `difference_raw` snapshots pointers under global, bumps `read_pins` (not `has_clone`), releases global, then acquires dual tree locks. `leave_lock` erases map entries under global, releases global, then detaches under tree lock only. | +| F-5 | **Tree mutex vs map mutex** | Medium | CWE-833 | Historical bare-tree `enter_lock` / `difference_raw` paths removed. Map-owned storage destructors use `release_merkletree_shared` (tree mutex only). | | F-6 | **`make_writeable` COW under `Lock` RAII** | High | CWE-667 | **Fixed:** COW now transfers the held mutex (unlock old `SharedState`, lock new) instead of leaving `Lock` holding a destroyed mutex while mutating a forked tree. | -| F-7 | **`leave_lock` / canonical map UAF** | High | CWE-416 | **Fixed:** Map erase drops the map's `has_clone` ref; canonical pointers are re-validated before switch; `SharedState` is not deleted while referenced from the dedup map. | -| F-7b | **Abandoned `SharedState` after canonical switch** | High | CWE-404 | **Fixed:** orphan reclaim for standalone `CMerkleTree.lock` / `difference_raw` (test NIFs). `account_map_lock` is `frozen`-only. Monitor via `nif_stats_raw/0`. | -| F-6 | **Global `locked_states` / `stats_mutex` on upgrade** | Low | CWE-665 | `on_reload`/`on_upgrade` no-op; hot upgrade could leave stale globals. Acceptable if NIF not hot-reloaded. | - -### Information disclosure / introspection - -| ID | Topic | Severity | CWE | Notes | -|----|--------|----------|-----|--------| -| F-7 | **`malloc_info_raw`** | Low (info disclosure) | CWE-200 | Test/dev NIF only. Restrict in production if threat model requires. | -| F-8 | **`struct_sizes_raw` / `memory_stats_raw`** | Low | CWE-200 | Test/dev NIF only. | +| F-7 | **SharedState reclaim on destructor** | High | CWE-416 / CWE-404 | **Fixed:** `release_merkletree_shared` drops `has_clone` under tree lock and deletes immediately when 0. `account_map_lock` is `frozen`-only. Monitor via `nif_stats_raw/0`. | +| F-6b | **`stats_mutex` on upgrade** | Low | CWE-665 | `on_reload`/`on_upgrade` no-op; hot upgrade could leave stale globals. Acceptable if NIF not hot-reloaded. | ### Denial of service | ID | Topic | Severity | CWE | Notes | |----|--------|----------|-----|--------| -| F-9 | **Unbounded work per NIF** | Medium | CWE-400 | Long-running exports use dirty schedulers: **CPU-bound** — `account_map_clone`, `lock`, `to_list`, `difference_full`, `apply_difference`, `storage_put_map`, `storage`, `proof`, `compact`, `uncompact_state`, `count_zeros`; test-only bare `difference_raw` / `to_list` / etc. Large dirty-NIF loops call `enif_consume_timeslice` every 512 iterations. Ensure adequate dirty CPU schedulers (`+SDcpu`). | +| F-9 | **Unbounded work per NIF** | Medium | CWE-400 | Long-running exports use dirty schedulers: **CPU-bound** — `account_map_clone`, `lock`, `to_list`, `difference_full`, `apply_difference`, `storage_put_map`, `storage`, `proof`, `compact`, `uncompact_state`, `count_zeros`. Large dirty-NIF loops call `enif_consume_timeslice` every 512 iterations. Ensure adequate dirty CPU schedulers (`+SDcpu`). | ### Memory safety (manual review) @@ -104,9 +93,9 @@ Bare `merkletree` resource API (`new`, `insert_item_raw`, `get_item`, `get_range ## 3. Resource destructor / locking (summary) -- `destruct_merkletree_type` → `locked_states->leave_lock(mt)` → `Lock` on `mt->shared_state->mtx` → `destroy_shared_state` may `delete` `SharedState` when `has_clone == 0`. +- Map-owned storage `merkletree` resources: `destruct_merkletree_type` → `release_merkletree_shared(mt)` → tree mutex → delete when `has_clone == 0`. - Requires `mt->shared_state` non-null at destructor entry; normal paths maintain this until GC. -- **F-1** could have left iterators open; fixed to avoid VM resource leaks. +- Account maps use a separate resource type / destructor (SharedAccountMap refcount). --- @@ -130,7 +119,6 @@ Bare `merkletree` resource API (`new`, `insert_item_raw`, `get_item`, `get_range - **Release flags:** Consider `-fstack-protector-strong`, `-D_FORTIFY_SOURCE=2`, `-Wl,-z,relro,-z,now` for `merkletree_nif.so`. - **Reproducibility:** `-march=native` in `OPTS` ties binaries to CPU; use generic `-march=x86-64` for release artifacts if needed. - **Debug:** `DEBUG` / `MERKLE_DEBUG_POOL` — avoid in production builds. -- **Test NIFs:** Bare-tree / debug exports are compiled into `nif_funcs` only with `-DCMERKLE_TEST_NIFS` (non-prod by default). --- diff --git a/c_src/merkletree.cpp b/c_src/merkletree.cpp index 44c4a2f..8bca268 100644 --- a/c_src/merkletree.cpp +++ b/c_src/merkletree.cpp @@ -230,13 +230,6 @@ void Item::each(Tree &tree, std::function func) { } } -size_t Item::leaf_count(const Tree &tree) const { - if (this->is_leaf) { - return 1; - } - return tree.pool->get(left_id)->leaf_count(tree) + tree.pool->get(right_id)->leaf_count(tree); -} - Tree::Tree() : m_pair_allocator(std::make_shared>(*this)), pool(ItemPool::make()), @@ -329,12 +322,6 @@ void Tree::insert_item(pair_t &pair) { } } -void Tree::insert_items(pair_list_t &items) { - for (pair_t *pair : items) { - insert_item(*pair); - } -} - int hash_to_leafindex(pair_t &pair) { return pair.key_hash.last_byte() % LEAF_SIZE; } @@ -433,12 +420,6 @@ size_t Tree::size() { return count; } -size_t Tree::leaf_count() { - bin_t binary_buffer; - update_merkle_hash_count(root_id, binary_buffer); - return pool->get(root_id)->leaf_count(*this); -} - static size_t count_nodes(ItemPool *pool, ItemId item_id) { if (item_id == kItemNull) { return 0; diff --git a/c_src/merkletree.hpp b/c_src/merkletree.hpp index 1404b51..9fae332 100644 --- a/c_src/merkletree.hpp +++ b/c_src/merkletree.hpp @@ -260,7 +260,6 @@ struct Item { Item& operator=(const Item &other) = delete; void each(Tree &tree, std::function func); - size_t leaf_count(const Tree &tree) const; uint256_t *ensure_hashes(); const uint256_t *hashes_ro() const { return hash_values; } @@ -289,7 +288,6 @@ class Tree { void insert_item(pair_t &pair); void insert_item(bin_t &key, uint256_t &value); - void insert_items(pair_list_t &items); void delete_item(bin_t &key) { uint256_t null_value = {}; insert_item(key, null_value); @@ -300,7 +298,6 @@ class Tree { uint256_t root_hash(); uint256_t* root_hashes(); size_t size(); - size_t leaf_count(); size_t node_count() const; void each(std::function func) { if (root_id != kItemNull) { diff --git a/c_src/nif.cpp b/c_src/nif.cpp index 5434813..41025a4 100644 --- a/c_src/nif.cpp +++ b/c_src/nif.cpp @@ -12,16 +12,12 @@ extern "C" { #include "merkletree.hpp" #include "rlp.hpp" #include -#include #include #include #include #include #include #include -#ifdef __GLIBC__ -#include -#endif static constexpr size_t kNifTimesliceInterval = 512; @@ -41,12 +37,6 @@ static ERL_NIF_TERM make_atom(ErlNifEnv *env, const char *atom_name); static ERL_NIF_TERM make_binary(ErlNifEnv *env, uint8_t *data, size_t size); static volatile int shared_states = 0; static volatile int resources = 0; -static int locked_states_cnt = 0; -static int orphan_shared_states = 0; - -class LockedStates; -static LockedStates* locked_states; - #ifdef DEBUG #define STAT(cmd) { enif_mutex_lock(stats_mutex); cmd; enif_mutex_unlock(stats_mutex); } @@ -54,7 +44,7 @@ static void print(const char *msg) { static int ops = 0; if (ops++ % 10000 == 0) { - fprintf(stderr, "%s [shared_states=%d] [locked_states=%d] [resources=%d]\n", msg, shared_states, locked_states_cnt, resources); fflush(stderr); + fprintf(stderr, "%s [shared_states=%d] [resources=%d]\n", msg, shared_states, resources); fflush(stderr); } } #else @@ -66,12 +56,10 @@ class SharedState { public: ErlNifMutex *mtx; int has_clone; - std::atomic read_pins; Tree tree; SharedState() : tree() { mtx = enif_mutex_create((char*)"merkletree_mutex"); has_clone = 0; - read_pins.store(0, std::memory_order_relaxed); enif_mutex_lock(stats_mutex); shared_states++; enif_mutex_unlock(stats_mutex); @@ -81,7 +69,6 @@ class SharedState { SharedState(SharedState &other) : tree(other.tree) { mtx = enif_mutex_create((char*)"merkletree_mutex"); has_clone = 0; - read_pins.store(0, std::memory_order_relaxed); enif_mutex_lock(stats_mutex); shared_states++; enif_mutex_unlock(stats_mutex); @@ -98,7 +85,6 @@ class SharedState { }; struct merkletree { - bool locked; SharedState *shared_state; }; @@ -129,18 +115,13 @@ class Lock { } }; -/* Lock one or two SharedState mutexes in a fixed address order (matches difference_raw). */ +/* Lock one or two SharedState mutexes in a fixed address order. */ class SharedStateLock { SharedState *first; SharedState *second; bool dual; public: - explicit SharedStateLock(SharedState *state) - : first(state), second(nullptr), dual(false) { - enif_mutex_lock(first->mtx); - } - SharedStateLock(SharedState *s1, SharedState *s2) { if (s1 == s2) { first = s1; @@ -180,34 +161,6 @@ class SharedStateLock { } }; -static void switch_local_to_canonical(merkletree *mt, SharedState *local, SharedState *canonical); - -static bool shared_state_reclaimable(SharedState *state) { - return state != nullptr && - state->has_clone == 0 && - state->read_pins.load(std::memory_order_acquire) == 0; -} - -static void classify_shared_state_reclaim(SharedState *state, SharedState **dead, SharedState **orphan) { - if (shared_state_reclaimable(state)) { - *dead = state; - } else { - *orphan = state; - } -} - -static void pin_shared_state_read(SharedState *state) { - if (state != nullptr) { - state->read_pins.fetch_add(1, std::memory_order_acq_rel); - } -} - -static void unpin_shared_state_read(SharedState *state) { - if (state != nullptr) { - state->read_pins.fetch_sub(1, std::memory_order_acq_rel); - } -} - static void keep_storage_in_map(merkletree *mt); static void release_storage_from_map(merkletree *mt); static merkletree *clone_merkletree_locked(merkletree *mt); @@ -427,17 +380,14 @@ static void release_entry_storage(AccountEntry &entry) entry.compact_storage.reset(); } -// Allocates a new merkletree resource sharing mt's SharedState (has_clone += 1) with -// locked = false so a fork can COW-write. Returns a resource with refcount 1 (the -// caller's ownership): pair with enif_make_resource + enif_release_resource for an -// Elixir term, or enif_keep_resource + enif_release_resource for C-side ownership. +// Allocates a new merkletree resource sharing mt's SharedState (has_clone += 1). +// Returns a resource with refcount 1 for C-side ownership (keep/release as needed). static merkletree *clone_merkletree_locked(merkletree *mt) { Lock lock(mt); merkletree *clone = (merkletree*)enif_alloc_resource(merkletree_type, sizeof(merkletree)); STAT(resources++); clone->shared_state = mt->shared_state; - clone->locked = false; clone->shared_state->has_clone += 1; return clone; } @@ -677,303 +627,30 @@ static void destroy_shared_accountmap(accountmap *am, AccountMapLock &lock) am->shared = NULL; } -class LockedStates { -public: - std::unordered_map states; - std::vector pending_orphans; - ErlNifMutex *mtx; - - LockedStates() { - mtx = enif_mutex_create((char*)"locked_states_mutex"); - } - - ~LockedStates() { - enif_mutex_destroy(mtx); - } - - void enqueue_orphan(SharedState *state) { - if (state == nullptr) { - return; - } - enif_mutex_lock(mtx); - if (std::find(pending_orphans.begin(), pending_orphans.end(), state) == - pending_orphans.end()) { - pending_orphans.push_back(state); - orphan_shared_states = (int)pending_orphans.size(); - } - enif_mutex_unlock(mtx); - } - - void remove_pending_orphan(SharedState *state) { - if (state == nullptr) { - return; - } - enif_mutex_lock(mtx); - auto it = std::find(pending_orphans.begin(), pending_orphans.end(), state); - if (it != pending_orphans.end()) { - pending_orphans.erase(it); - orphan_shared_states = (int)pending_orphans.size(); - } - enif_mutex_unlock(mtx); - } - - void try_reclaim_orphans() { - std::vector snapshot; - enif_mutex_lock(mtx); - snapshot = pending_orphans; - enif_mutex_unlock(mtx); - - for (SharedState *candidate : snapshot) { - enif_mutex_lock(mtx); - auto it = std::find(pending_orphans.begin(), pending_orphans.end(), candidate); - if (it == pending_orphans.end()) { - enif_mutex_unlock(mtx); - continue; - } - pending_orphans.erase(it); - orphan_shared_states = (int)pending_orphans.size(); - enif_mutex_unlock(mtx); - - if (enif_mutex_trylock(candidate->mtx) != 0) { - enqueue_orphan(candidate); - continue; - } - if (!shared_state_reclaimable(candidate)) { - enif_mutex_unlock(candidate->mtx); - enqueue_orphan(candidate); - continue; - } - enif_mutex_unlock(candidate->mtx); - delete candidate; - } - } - - void switch_to_canonical_locked(merkletree *mt, SharedState *local, SharedState *canonical) { - enif_mutex_lock(local->mtx); - local->has_clone += 1; - enif_mutex_unlock(local->mtx); - enif_mutex_lock(canonical->mtx); - canonical->has_clone += 1; - enif_mutex_unlock(canonical->mtx); - unpin_shared_state_read(canonical); - switch_local_to_canonical(mt, local, canonical); - enif_mutex_lock(mtx); - if (mt->shared_state != canonical) { - enif_mutex_lock(canonical->mtx); - if (canonical->has_clone > 0) { - canonical->has_clone -= 1; - } - enif_mutex_unlock(canonical->mtx); - } - enif_mutex_unlock(mtx); - try_reclaim_orphans(); - } - - void enter_lock(merkletree *mt) { - locked_states_cnt = (int)states.size(); - print("ENTER_LOCK"); - - SharedState *local = nullptr; - uint256_t root_hash; - - { - Lock lock(mt); - mt->locked = true; - local = mt->shared_state; - root_hash = local->tree.root_hash(); - } - - enif_mutex_lock(mtx); - SharedState *canonical = nullptr; - auto it = states.find(root_hash); - if (it != states.end()) { - if (it->second == local) { - enif_mutex_unlock(mtx); - enif_mutex_lock(local->mtx); - local->has_clone += 1; - enif_mutex_unlock(local->mtx); - return; - } - canonical = it->second; - pin_shared_state_read(canonical); - } else { - states[root_hash] = local; - enif_mutex_unlock(mtx); - enif_mutex_lock(local->mtx); - local->has_clone += 1; - enif_mutex_unlock(local->mtx); - return; - } - enif_mutex_unlock(mtx); - - if (canonical) { - switch_to_canonical_locked(mt, local, canonical); - } - } - - /* Lock a trie whose root_hash is already registered; repoint to canonical SharedState. */ - void apply_canonical_lock(merkletree *mt, const uint256_t &root_hash) { - enif_mutex_lock(mtx); - auto it = states.find(root_hash); - if (it == states.end()) { - enif_mutex_unlock(mtx); - enter_lock(mt); - return; - } - - SharedState *canonical = it->second; - pin_shared_state_read(canonical); - enif_mutex_unlock(mtx); - - SharedState *local = nullptr; - { - Lock lock(mt); - mt->locked = true; - local = mt->shared_state; - if (local == canonical) { - // Already hold local/canonical mtx via Lock — do not lock again - // (ErlNifMutex is non-recursive; double-lock deadlocks on shared storage). - canonical->has_clone += 1; - lock.unlock(); - unpin_shared_state_read(canonical); - return; - } - } - - switch_to_canonical_locked(mt, local, canonical); - } - - void leave_lock(merkletree *mt) { - locked_states_cnt = (int)states.size(); - print("LEAVE_LOCK"); - - SharedState *state = mt->shared_state; - if (state == nullptr) { - return; - } - - bool was_locked = mt->locked; - mt->locked = false; - - uint256_t root_hash; - SharedState *dead = nullptr; - SharedState *orphan = nullptr; - bool erase_map_entry = false; - - if (was_locked) { - enif_mutex_lock(mtx); - enif_mutex_lock(state->mtx); - root_hash = state->tree.root_hash(); - - if (state->has_clone > 0) { - state->has_clone -= 1; - } - mt->shared_state = nullptr; - erase_map_entry = (state->has_clone == 0); - - if (state->has_clone == 0) { - classify_shared_state_reclaim(state, &dead, &orphan); - } - - if (erase_map_entry) { - auto it = states.find(root_hash); - if (it != states.end() && it->second == state) { - states.erase(it); - } - } - - enif_mutex_unlock(state->mtx); - enif_mutex_unlock(mtx); - } else { - enif_mutex_lock(state->mtx); - if (state->has_clone == 0) { - classify_shared_state_reclaim(state, &dead, &orphan); - mt->shared_state = nullptr; - } else { - state->has_clone -= 1; - mt->shared_state = nullptr; - } - enif_mutex_unlock(state->mtx); - } - - if (dead != nullptr) { - remove_pending_orphan(dead); - delete dead; - } - if (orphan != nullptr) { - enqueue_orphan(orphan); - } - - try_reclaim_orphans(); - } - - int locked_count() const { - return (int)states.size(); - } - - int orphan_count() const { - return orphan_shared_states; - } -}; - -static void switch_local_to_canonical(merkletree *mt, SharedState *local, SharedState *canonical) +/* Release map-owned merkletree resource SharedState (destructor path). */ +static void release_merkletree_shared(merkletree *mt) { - if (local == canonical) { - return; - } - - SharedState *lo = local < canonical ? local : canonical; - SharedState *hi = local < canonical ? canonical : local; - enif_mutex_lock(lo->mtx); - enif_mutex_lock(hi->mtx); - - if (mt->shared_state != local) { - enif_mutex_unlock(hi->mtx); - enif_mutex_unlock(lo->mtx); + SharedState *state = mt->shared_state; + if (state == nullptr) { return; } - SharedState *abandoned = nullptr; - if (local->has_clone == 0) { + SharedState *to_delete = nullptr; + enif_mutex_lock(state->mtx); + if (state->has_clone == 0) { + to_delete = state; mt->shared_state = nullptr; - enif_mutex_unlock(local->mtx); - if (shared_state_reclaimable(local)) { - abandoned = local; - } else { - locked_states->enqueue_orphan(local); - } } else { - local->has_clone -= 1; + state->has_clone -= 1; mt->shared_state = nullptr; - enif_mutex_unlock(local->mtx); - if (local->has_clone == 0) { - if (shared_state_reclaimable(local)) { - abandoned = local; - } else { - locked_states->enqueue_orphan(local); - } - } - } - mt->shared_state = canonical; - - if (local == lo) { - enif_mutex_unlock(hi->mtx); - } else { - enif_mutex_unlock(lo->mtx); } + enif_mutex_unlock(state->mtx); - if (abandoned != nullptr) { - locked_states->enqueue_orphan(abandoned); + if (to_delete != nullptr) { + delete to_delete; } } -/* Bare-tree / debug NIF entry points stay compiled always; only nif_funcs[] is gated. - * Mark unused when CMERKLE_TEST_NIFS is off so -Wunused-function stays clean in prod. */ -#ifndef CMERKLE_TEST_NIFS -#define CMERKLE_TEST_NIF __attribute__((unused)) -#else -#define CMERKLE_TEST_NIF -#endif static ERL_NIF_TERM make_atom(ErlNifEnv *env, const char *atom_name) @@ -995,39 +672,10 @@ make_binary(ErlNifEnv *env, uint8_t *data, size_t size) return term; } - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_new(ErlNifEnv *env, int argc, const ERL_NIF_TERM[] /*argv[]*/) -{ - if (argc != 0) return enif_make_badarg(env); - merkletree *mt = alloc_merkletree_resource(); - ERL_NIF_TERM res = enif_make_resource(env, mt); - enif_release_resource(mt); - return res; -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_clone(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt; - if (argc != 1) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt)) return enif_make_badarg(env); - - merkletree *clone = clone_merkletree_locked(mt); - ERL_NIF_TERM res = enif_make_resource(env, clone); - enif_release_resource(clone); - return res; -} - - /* In-place COW: when SharedState is shared (has_clone > 0), detach this resource onto a * privately-owned SharedState copy. Caller must hold mt->shared_state->mtx. */ -static bool make_writeable_locked(merkletree *mt) +static void make_writeable_locked(merkletree *mt) { - if (mt->locked) { - return false; - } - SharedState *state = mt->shared_state; if (state->has_clone > 0) { state->has_clone -= 1; @@ -1036,7 +684,6 @@ static bool make_writeable_locked(merkletree *mt) enif_mutex_lock(mt->shared_state->mtx); print("CREATING (UNCLONING)"); } - return true; } static bool decode_storage_slot(const ErlNifBinary &key_binary, @@ -1062,45 +709,6 @@ static bool insert_binary_pair(Tree &tree, const ErlNifBinary &key_binary, return true; } -static bool insert_binary_terms(ErlNifEnv *env, Tree &tree, ERL_NIF_TERM key_term, - ERL_NIF_TERM value_term, bin_t &key_scratch) -{ - ErlNifBinary key_binary, value_binary; - if (!enif_inspect_binary(env, key_term, &key_binary)) { - return false; - } - if (!enif_inspect_binary(env, value_term, &value_binary)) { - return false; - } - return insert_binary_pair(tree, key_binary, value_binary, key_scratch); -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_insert_item(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt; - ErlNifBinary key_binary, value_binary; - - if (argc != 3) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt)) return enif_make_badarg(env); - - if (!enif_inspect_binary(env, argv[1], &key_binary)) return enif_make_badarg(env); - if (!enif_inspect_binary(env, argv[2], &value_binary)) return enif_make_badarg(env); - - enif_mutex_lock(mt->shared_state->mtx); - if (!make_writeable_locked(mt)) { - enif_mutex_unlock(mt->shared_state->mtx); - return enif_make_badarg(env); - } - bin_t key_scratch; - if (!insert_binary_pair(mt->shared_state->tree, key_binary, value_binary, key_scratch)) { - enif_mutex_unlock(mt->shared_state->mtx); - return enif_make_badarg(env); - } - enif_mutex_unlock(mt->shared_state->mtx); - return argv[0]; -} - namespace { struct RangeEntry { @@ -1149,66 +757,6 @@ static size_t get_range_entries(Tree &tree, const bin_t &base_key, size_t count, } // namespace -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_get_range(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt; - ErlNifBinary key_binary; - unsigned count; - - if (argc != 3) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt)) return enif_make_badarg(env); - if (!enif_inspect_binary(env, argv[1], &key_binary)) return enif_make_badarg(env); - if (key_binary.size != 32) return enif_make_badarg(env); - if (!enif_get_uint(env, argv[2], &count)) return enif_make_badarg(env); - if (count < 1 || count > 256) return enif_make_badarg(env); - - Lock lock(mt); - - bin_t key; - key.insert(key.end(), key_binary.data, key_binary.data + key_binary.size); - - std::vector entries(count); - size_t n = get_range_entries(mt->shared_state->tree, key, count, entries.data()); - - ERL_NIF_TERM list = enif_make_list(env, 0); - for (size_t i = n; i > 0; i--) { - RangeEntry &entry = entries[i - 1]; - ERL_NIF_TERM key_term = make_binary(env, entry.key.data(), entry.key.size()); - ERL_NIF_TERM value_term = make_binary(env, entry.value.data(), 32); - ERL_NIF_TERM pair = enif_make_tuple2(env, key_term, value_term); - list = enif_make_list_cell(env, pair, list); - } - - return list; -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_get_item(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt; - ErlNifBinary key_binary; - - if (argc != 2) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt)) return enif_make_badarg(env); - Lock lock(mt); - if (!enif_inspect_binary(env, argv[1], &key_binary)) return enif_make_badarg(env); - if (key_binary.size != 32) return enif_make_badarg(env); - - bin_t key; - key.insert(key.end(), key_binary.data, key_binary.data + key_binary.size); - pair_t *pair = mt->shared_state->tree.get_item(std::move(key)); - - if (pair == nullptr) { - return make_atom(env, "nil"); - } - - ERL_NIF_TERM key_term = argv[1]; - ERL_NIF_TERM value_term = make_binary(env, pair->value.data(), 32); - ERL_NIF_TERM hash_term = make_binary(env, pair->key_hash.data(), 32); - return enif_make_tuple3(env, key_term, value_term, hash_term); -} - static ERL_NIF_TERM make_proof(ErlNifEnv *env, proof_t& proof) { @@ -1230,249 +778,6 @@ make_proof(ErlNifEnv *env, proof_t& proof) } } -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_get_proofs(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt; - ErlNifBinary key_binary; - - if (argc != 2) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt)) return enif_make_badarg(env); - Lock lock(mt); - if (!enif_inspect_binary(env, argv[1], &key_binary)) return enif_make_badarg(env); - - bin_t key; - key.insert(key.end(), key_binary.data, key_binary.data + key_binary.size); - proof_t proof = mt->shared_state->tree.get_proofs(key); - return make_proof(env, proof); -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_to_list(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt; - - if (argc != 1) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt)) return enif_make_badarg(env); - Lock lock(mt); - ERL_NIF_TERM list = enif_make_list(env, 0); - size_t i = 0; - mt->shared_state->tree.each([&](pair_t &pair) { - i++; - ERL_NIF_TERM key_term = make_binary(env, pair.key.data(), pair.key.size()); - ERL_NIF_TERM value_term = make_binary(env, pair.value.data(), 32); - ERL_NIF_TERM tuple = enif_make_tuple2(env, key_term, value_term); - list = enif_make_list_cell(env, tuple, list); - nif_loop_progress(env, i); - }); - return list; -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_lock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt; - if (argc != 1) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt)) return enif_make_badarg(env); - locked_states->enter_lock(mt); - return argv[0]; -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_difference(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt1; - merkletree *mt2; - - if (argc != 2) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt1)) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[1], merkletree_type, (void **) &mt2)) return enif_make_badarg(env); - - if (mt1 == mt2) { - return enif_make_list(env, 0); - } - - SharedState *s1; - SharedState *s2; - enif_mutex_lock(locked_states->mtx); - s1 = mt1->shared_state; - s2 = mt2->shared_state; - if (s1 == s2) { - enif_mutex_unlock(locked_states->mtx); - return enif_make_list(env, 0); - } - pin_shared_state_read(s1); - pin_shared_state_read(s2); - enif_mutex_unlock(locked_states->mtx); - - if (s1 == nullptr || s2 == nullptr) { - unpin_shared_state_read(s2); - unpin_shared_state_read(s1); - return enif_make_list(env, 0); - } - - SharedStateLock state_lock(s1, s2); - - Tree output; - s1->tree.difference(s2->tree, output); - s2->tree.difference(s1->tree, output); - - ERL_NIF_TERM list = enif_make_list(env, 0); - size_t i = 0; - output.each([&](pair_t &pair) { - i++; - ERL_NIF_TERM key_term = make_binary(env, pair.key.data(), pair.key.size()); - - auto pair1 = s1->tree.get_item(pair); - auto pair2 = s2->tree.get_item(pair); - - ERL_NIF_TERM value1_term = pair1 == nullptr ? make_atom(env, "nil") : make_binary(env, pair1->value.data(), 32); - ERL_NIF_TERM value2_term = pair2 == nullptr ? make_atom(env, "nil") : make_binary(env, pair2->value.data(), 32); - ERL_NIF_TERM tuple = enif_make_tuple2(env, value1_term, value2_term); - tuple = enif_make_tuple2(env, key_term, tuple); - list = enif_make_list_cell(env, tuple, list); - nif_loop_progress(env, i); - }); - - unpin_shared_state_read(s2); - unpin_shared_state_read(s1); - state_lock.unlock(); - locked_states->try_reclaim_orphans(); - return list; -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_import_map(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt; - - if (argc != 2) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt)) return enif_make_badarg(env); - if (!enif_is_map(env, argv[1])) return enif_make_badarg(env); - - enif_mutex_lock(mt->shared_state->mtx); - if (!make_writeable_locked(mt)) { - enif_mutex_unlock(mt->shared_state->mtx); - return enif_make_badarg(env); - } - - ERL_NIF_TERM key, value; - ErlNifMapIterator iter; - enif_map_iterator_create(env, argv[1], &iter, ERL_NIF_MAP_ITERATOR_FIRST); - - size_t i = 0; - bin_t key_scratch; - while (enif_map_iterator_get_pair(env, &iter, &key, &value)) { - i++; - if (!insert_binary_terms(env, mt->shared_state->tree, key, value, key_scratch)) { - enif_mutex_unlock(mt->shared_state->mtx); - goto import_badarg; - } - enif_map_iterator_next(env, &iter); - nif_loop_progress(env, i); - } - enif_mutex_unlock(mt->shared_state->mtx); - enif_map_iterator_destroy(env, &iter); - return argv[0]; - -import_badarg: - enif_map_iterator_destroy(env, &iter); - return enif_make_badarg(env); -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_root_hash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt; - if (argc != 1) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt)) return enif_make_badarg(env); - Lock lock(mt); - auto root_hash = mt->shared_state->tree.root_hash(); - return make_binary(env, root_hash.data(), 32); -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_hash(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - ErlNifBinary key_binary; - - if (argc != 1) return enif_make_badarg(env); - if (!enif_inspect_binary(env, argv[0], &key_binary)) return enif_make_badarg(env); - - uint256_t hash = {}; - sha((const uint8_t*)key_binary.data, key_binary.size, hash.data()); - return make_binary(env, hash.data(), 32); -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_root_hashes(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt; - if (argc != 1) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt)) return enif_make_badarg(env); - Lock lock(mt); - auto root_hashes = mt->shared_state->tree.root_hashes(); - return make_binary(env, (uint8_t*)root_hashes, 32*16); -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_size(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt; - if (argc != 1) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt)) return enif_make_badarg(env); - Lock lock(mt); - auto size = mt->shared_state->tree.size(); - return enif_make_uint(env, size); -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_bucket_count(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt; - if (argc != 1) return enif_make_badarg(env); - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt)) return enif_make_badarg(env); - Lock lock(mt); - auto size = mt->shared_state->tree.leaf_count(); - return enif_make_uint(env, size); -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_struct_sizes(ErlNifEnv *env, int argc, const ERL_NIF_TERM /*argv*/[]) -{ - if (argc != 0) { - return enif_make_badarg(env); - } - return enif_make_tuple5(env, - enif_make_uint64(env, sizeof(Item)), - enif_make_uint64(env, sizeof(pair_t)), - enif_make_uint64(env, sizeof(pair_list_t)), - enif_make_uint64(env, sizeof(Tree)), - enif_make_uint64(env, MERKLE_STRIPE_SIZE)); -} - -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_memory_stats(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - merkletree *mt; - if (argc != 1) { - return enif_make_badarg(env); - } - if (!enif_get_resource(env, argv[0], merkletree_type, (void **) &mt)) { - return enif_make_badarg(env); - } - Lock lock(mt); - Tree &t = mt->shared_state->tree; - size_t nodes = t.node_count(); - size_t pairs = t.size(); - uint64_t approx = - (uint64_t)nodes * (uint64_t)sizeof(Item) + (uint64_t)pairs * (uint64_t)sizeof(pair_t); - return enif_make_tuple3(env, - enif_make_uint64(env, nodes), - enif_make_uint64(env, pairs), - enif_make_uint64(env, approx)); -} - static ERL_NIF_TERM merkletree_nif_stats(ErlNifEnv *env, int argc, const ERL_NIF_TERM /*argv*/[]) { @@ -1480,8 +785,6 @@ merkletree_nif_stats(ErlNifEnv *env, int argc, const ERL_NIF_TERM /*argv*/[]) return enif_make_badarg(env); } - int locked = 0; - int orphans = 0; int shared = 0; int res = 0; @@ -1490,43 +793,14 @@ merkletree_nif_stats(ErlNifEnv *env, int argc, const ERL_NIF_TERM /*argv*/[]) res = resources; enif_mutex_unlock(stats_mutex); - if (locked_states != nullptr) { - enif_mutex_lock(locked_states->mtx); - locked = locked_states->locked_count(); - orphans = locked_states->orphan_count(); - enif_mutex_unlock(locked_states->mtx); - } - - ERL_NIF_TERM locked_term = enif_make_int(env, locked); - ERL_NIF_TERM orphans_term = enif_make_int(env, orphans); + /* locked/orphan counters retired with bare-tree lock NIFs; keep tuple shape. */ + ERL_NIF_TERM locked_term = enif_make_int(env, 0); + ERL_NIF_TERM orphans_term = enif_make_int(env, 0); ERL_NIF_TERM shared_term = enif_make_int(env, shared); ERL_NIF_TERM resources_term = enif_make_int(env, res); return enif_make_tuple4(env, locked_term, orphans_term, shared_term, resources_term); } -static ERL_NIF_TERM CMERKLE_TEST_NIF -merkletree_malloc_info(ErlNifEnv *env, int argc, const ERL_NIF_TERM /*argv*/[]) -{ - if (argc != 0) { - return enif_make_badarg(env); - } -#ifdef __GLIBC__ - char *buf = NULL; - size_t sz = 0; - FILE *fp = open_memstream(&buf, &sz); - if (!fp) { - return make_atom(env, "error"); - } - malloc_info(0, fp); - fclose(fp); - ERL_NIF_TERM term = make_binary(env, (uint8_t *)buf, sz); - free(buf); - return term; -#else - return make_atom(env, "unsupported"); -#endif -} - static ERL_NIF_TERM merkletree_count_zeros(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { @@ -1583,12 +857,10 @@ account_map_lock(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { AccountMapLock lock(am); - // Map-level freeze only: get no longer exports live storage resources, so - // bare CMerkleTree.insert cannot mutate map-owned tries via Elixir. + // Map-level freeze only: get no longer exports live storage resources. am->shared->frozen = true; } - locked_states->try_reclaim_orphans(); return argv[0]; } @@ -1659,10 +931,7 @@ static void insert_state_trie_hash(SharedAccountMap *shared, const uint160_t &ad bin_t key(addr.value, addr.value + 20); merkletree *mt = shared->state_trie; enif_mutex_lock(mt->shared_state->mtx); - if (!make_writeable_locked(mt)) { - enif_mutex_unlock(mt->shared_state->mtx); - return; - } + make_writeable_locked(mt); uint256_t hash_value = hash; mt->shared_state->tree.insert_item(key, hash_value); enif_mutex_unlock(mt->shared_state->mtx); @@ -1790,9 +1059,7 @@ account_map_put(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) merkletree *storage = nullptr; ERL_NIF_TERM storage_term = argv[4]; - if (enif_get_resource(env, storage_term, merkletree_type, (void **)&storage)) { - // replace storage with provided merkle tree resource - } else if (term_is_atom_named(env, storage_term, "keep")) { + if (term_is_atom_named(env, storage_term, "keep")) { keep_meta = true; } else if (term_is_nil(env, storage_term) || (enif_is_list(env, storage_term) && enif_is_empty_list(env, storage_term))) { @@ -1920,7 +1187,6 @@ static merkletree *alloc_merkletree_resource() merkletree *mt = (merkletree*)enif_alloc_resource(merkletree_type, sizeof(merkletree)); STAT(resources++); mt->shared_state = new SharedState(); - mt->locked = false; return mt; } @@ -2193,7 +1459,6 @@ account_map_difference_full(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) nif_loop_progress(env, j); } - locked_states->try_reclaim_orphans(); return list; } @@ -2267,10 +1532,7 @@ static merkletree *write_storage_slot(AccountEntry &entry, const bin_t &key, con { merkletree *mt = materialize_storage(entry); enif_mutex_lock(mt->shared_state->mtx); - if (!make_writeable_locked(mt)) { - enif_mutex_unlock(mt->shared_state->mtx); - return nullptr; - } + make_writeable_locked(mt); bin_t key_copy = key; uint256_t value_copy = value; mt->shared_state->tree.insert_item(key_copy, value_copy); @@ -2381,10 +1643,7 @@ static bool apply_storage_delta(ErlNifEnv *env, AccountEntry &entry, ERL_NIF_TER return false; } - if (write_storage_slot(entry, key, new_value) == nullptr) { - enif_map_iterator_destroy(env, &iter); - return false; - } + write_storage_slot(entry, key, new_value); enif_map_iterator_next(env, &iter); } @@ -2448,7 +1707,6 @@ account_map_apply_difference(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[] nif_loop_progress(env, i); } - locked_states->try_reclaim_orphans(); return argv[0]; } @@ -2505,9 +1763,7 @@ account_map_storage_put_map(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) bin_t key(key_bin.data, key_bin.data + key_bin.size); uint256_t value((const char *)value_bin.data); - if (write_storage_slot(entry, key, value) == nullptr) { - return enif_make_badarg(env); - } + write_storage_slot(entry, key, value); nif_loop_progress(env, j); } @@ -2743,12 +1999,6 @@ static bool parse_compact_storage(ErlNifEnv *env, ERL_NIF_TERM storage_term, entry.storage = nullptr; entry.compact_storage.reset(); - merkletree *existing; - if (enif_get_resource(env, storage_term, merkletree_type, (void **)&existing)) { - entry.storage = existing; - return true; - } - if (enif_is_atom(env, storage_term)) { char atom[16]; if (enif_get_atom(env, storage_term, atom, sizeof(atom), ERL_NIF_LATIN1) && @@ -2875,8 +2125,7 @@ static bool parse_compact_account(ErlNifEnv *env, ERL_NIF_TERM account_term, return parse_compact_storage(env, storage_term, out.entry); } -static ERL_NIF_TERM uncompact_state_fail(ErlNifEnv *env, ErlNifMapIterator *iter, - accountmap *am, merkletree *state_store) +static ERL_NIF_TERM uncompact_state_fail(ErlNifEnv *env, ErlNifMapIterator *iter, accountmap *am) { if (iter) { enif_map_iterator_destroy(env, iter); @@ -2884,9 +2133,6 @@ static ERL_NIF_TERM uncompact_state_fail(ErlNifEnv *env, ErlNifMapIterator *iter if (am) { enif_release_resource(am); } - if (state_store) { - enif_release_resource(state_store); - } return enif_make_badarg(env); } @@ -2916,7 +2162,7 @@ static bool append_uncompacted_account(ErlNifEnv *env, accountmap *am, AccountHa materialize_storage(entry); } } else if (entry.compact_storage == nullptr) { - // Shared/copied pointer from another map or Elixir term: take a map ref. + // Shared/copied pointer from another map entry: take a map ref. keep_storage_in_map(entry.storage); } uint256_t account_hash; @@ -3112,7 +2358,7 @@ account_map_uncompact_state(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) AccountEntry entry = kv.second; if (!append_uncompacted_account(env, am, scratch.hash_ctx, pending_state, kv.first, entry, nullptr, nullptr, i)) { - return uncompact_state_fail(env, nullptr, am, nullptr); + return uncompact_state_fail(env, nullptr, am); } } } else { @@ -3124,12 +2370,12 @@ account_map_uncompact_state(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) i++; uint160_t addr; if (!get_address(env, key, addr)) { - return uncompact_state_fail(env, &iter, am, nullptr); + return uncompact_state_fail(env, &iter, am); } ParsedCompactAccount parsed; if (!parse_compact_account(env, value, parsed, scratch.code_buf)) { - return uncompact_state_fail(env, &iter, am, nullptr); + return uncompact_state_fail(env, &iter, am); } const uint256_t *storage_root_override = @@ -3138,7 +2384,7 @@ account_map_uncompact_state(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) parsed.has_compact_code_hash ? &parsed.compact_code_hash : nullptr; if (!append_uncompacted_account(env, am, scratch.hash_ctx, pending_state, addr, parsed.entry, storage_root_override, code_hash_override, i)) { - return uncompact_state_fail(env, &iter, am, nullptr); + return uncompact_state_fail(env, &iter, am); } enif_map_iterator_next(env, &iter); @@ -3175,7 +2421,7 @@ destruct_merkletree_type(ErlNifEnv* /*env*/, void *arg) { merkletree *mt = (merkletree *) arg; STAT(resources--); - locked_states->leave_lock(mt); + release_merkletree_shared(mt); } @@ -3194,7 +2440,6 @@ on_load(ErlNifEnv* env, void** /*priv*/, ERL_NIF_TERM /*info*/) if(!rt) return -1; accountmap_type = rt; - locked_states = new LockedStates(); stats_mutex = enif_mutex_create((char*)"stats_mutex"); sha((const uint8_t*)"", 0, empty_code_hash.value); empty_storage_tree = alloc_merkletree_resource(); @@ -3213,27 +2458,6 @@ static int on_upgrade(ErlNifEnv* /*env*/, void** /*priv*/, void** /*old_priv_dat } static ErlNifFunc nif_funcs[] = { -#ifdef CMERKLE_TEST_NIFS - /* Bare-tree + debug NIFs (dev/test/scripts). Omitted from prod MIX_ENV=prod. */ - {"new", 0, merkletree_new, 0}, - {"insert_item_raw", 3, merkletree_insert_item, 0}, - {"get_item", 2, merkletree_get_item, 0}, - {"get_range_raw", 3, merkletree_get_range, 0}, - {"get_proofs_raw", 2, merkletree_get_proofs, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"difference_raw", 2, merkletree_difference, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"lock", 1, merkletree_lock, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"to_list", 1, merkletree_to_list, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"import_map", 2, merkletree_import_map, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"root_hash", 1, merkletree_root_hash, 0}, - {"hash", 1, merkletree_hash, 0}, - {"root_hashes_raw", 1, merkletree_root_hashes, 0}, - {"bucket_count", 1, merkletree_bucket_count, 0}, - {"size", 1, merkletree_size, 0}, - {"clone", 1, merkletree_clone, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"struct_sizes_raw", 0, merkletree_struct_sizes, 0}, - {"memory_stats_raw", 1, merkletree_memory_stats, ERL_NIF_DIRTY_JOB_CPU_BOUND}, - {"malloc_info_raw", 0, merkletree_malloc_info, ERL_NIF_DIRTY_JOB_IO_BOUND}, -#endif {"count_zeros", 1, merkletree_count_zeros, ERL_NIF_DIRTY_JOB_CPU_BOUND}, {"nif_stats_raw", 0, merkletree_nif_stats, 0}, {"account_map_new", 0, account_map_new, 0}, @@ -3257,5 +2481,4 @@ static ErlNifFunc nif_funcs[] = { {"account_map_proof", 3, account_map_proof, ERL_NIF_DIRTY_JOB_CPU_BOUND}, }; -// ERL_NIF_INIT(merkletree_nif, nif_funcs, on_load, on_reload, on_upgrade, NULL); ERL_NIF_INIT(Elixir.CMerkleTree, nif_funcs, on_load, on_reload, on_upgrade, NULL) diff --git a/docs/caccount-map-nif.md b/docs/caccount-map-nif.md index 14b342f..293d4e4 100644 --- a/docs/caccount-map-nif.md +++ b/docs/caccount-map-nif.md @@ -22,8 +22,9 @@ implementation spec for difference/clone performance work: Internal `state_trie` is never exported; Edge uses `state_root_hashes/1`. - `account_map_get` / `to_list` return `{nonce, balance, storage_root_hash_bin32, code}` — the third element is a **32-byte hash**, never a live storage resource. -- `account_map_put/6` storage arg: resource | `:keep` (meta-only) | `nil`/`[]` | - `[{key32, value32}]` (genesis / hardfork). +- `account_map_put/6` storage arg: `:keep` (meta-only) | `nil`/`[]` | + `[{key32, value32}]` (genesis / hardfork / tests). Live bare-tree resources are + not part of the public NIF surface. - `account_map_storage/3` covers get / range / list / size via one NIF. - `account_map_storage_roots/2` and `account_map_state_roots/1` return `<>`. @@ -49,9 +50,7 @@ implementation spec for difference/clone performance work: ## Build - NIF: `mix compile` → `elixir_make` → `c_src/` → `priv/merkletree_nif.so` -- **Prod** (`MIX_ENV=prod`): ~21 map + misc exports only. -- **Dev/test** (non-prod): also registers bare-tree + debug NIFs via - `-DCMERKLE_TEST_NIFS` (override with `CMERKLE_TEST_NIFS=0|1`). Mode stamp - forces rebuild when the flag changes. +- Exports: `account_map_*` plus `count_zeros/1` and `nif_stats_raw/0` + (`CMerkleTree.nif_stats/0`). No bare-tree test NIF mode. - EVM binary: `evm/evm` (needs `libboost-dev`) - `deps/libsecp256k1`: build once with `make -C deps/libsecp256k1/` (not via `mix`) diff --git a/lib/bench.ex b/lib/bench.ex index 24d95a9..79e9d63 100644 --- a/lib/bench.ex +++ b/lib/bench.ex @@ -55,11 +55,5 @@ defmodule Bench do {:ok, state, _rcpt} = Transaction.apply(tx, block, state) state end) - - # # Checking value of i at position 0 - # acc = Chain.State.account(state, addr) - # len = Chain.Account.storageInteger(acc, 0) - # IO.puts("#{length(txlist)} == #{len}") - # ^len = length(txlist) end end diff --git a/lib/chain/account.ex b/lib/chain/account.ex index 9f33850..62dc71c 100644 --- a/lib/chain/account.ex +++ b/lib/chain/account.ex @@ -12,7 +12,10 @@ defmodule Chain.Account do @type t :: %Chain.Account{ nonce: non_neg_integer(), balance: non_neg_integer(), - storage_root: nil | reference() | [{binary(), binary()}], + storage_root: + nil + | [{binary(), binary()}] + | {atom(), list(), map()}, code: binary() | nil, map_backed: boolean(), root_hash: binary() | nil @@ -34,7 +37,7 @@ defmodule Chain.Account do @doc """ Build an account from `CAccountMap.get/2` parts. A 32-byte `storage` root marks the account map-backed (`map_backed: true`). - Otherwise `storage` is a put payload (`nil` / slot list / resource for `set_account`). + Otherwise `storage` is a put payload (`nil` / slot list / compact MapMerkleTree tuple). """ def from_parts(nonce, balance, <>, code) do %Chain.Account{ diff --git a/lib/cmerkletree.ex b/lib/cmerkletree.ex index 3bcdca4..b98c050 100644 --- a/lib/cmerkletree.ex +++ b/lib/cmerkletree.ex @@ -3,115 +3,19 @@ # Licensed under the Diode License, Version 1.1 defmodule CMerkleTree do @on_load :load_nifs - @null <<0::unsigned-size(256)>> - - @type t :: reference() def load_nifs do :erlang.load_nif(~c"./priv/merkletree_nif", 0) end - def from_map(map) do - import_map(new(), map) - end - - def from_list(list) do - insert_items(new(), list) - end - - def difference(a, b) do - difference_raw(a, b) - |> Enum.map(fn - {key, {@null, value2}} -> {key, {nil, value2}} - {key, {value1, @null}} -> {key, {value1, nil}} - {key, {value1, value2}} -> {key, {value1, value2}} - end) - |> Map.new() - end - - def insert(tree, key, value) do - insert_item_raw(tree, to_bytes(key), to_bytes32(value)) - end - - def insert_items(tree, items) do - Enum.reduce(items, tree, fn {key, value}, acc -> - insert(acc, key, value) - end) - end - - def get_proofs(tree, key) do - get_proofs_raw(tree, to_bytes(key)) - end - - def insert_item(tree, {key, value}) do - insert(tree, key, value) - end - - def delete(tree, key) do - insert(tree, key, @null) - end - - def get(tree, key) do - case get_item(tree, to_bytes32(key)) do - nil -> nil - {_key, @null, _hash} -> nil - {_key, value, _hash} -> value - end - end - - def get_range(tree, key, count) when is_integer(count) and count >= 1 and count <= 256 do - get_range_raw(tree, to_bytes32(key), count) - |> Enum.map(fn {k, v} -> {k, if(v == @null, do: nil, else: v)} end) - end - - def root_hashes(tree) do - <> = - root_hashes_raw(tree) - - [a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p] - end - - def new, do: error() - def insert_item_raw(_tree, _key, _value), do: error() - def root_hash(_tree), do: error() - def root_hashes_raw(_tree), do: error() - def clone(_tree), do: error() - def size(_tree), do: error() - def hash(_binary), do: error() - def bucket_count(_tree), do: error() - def get_item(_tree, _key), do: error() - def get_range_raw(_tree, _key, _count), do: error() - def to_list(_tree), do: error() - def import_map(_tree, _map), do: error() - def difference_raw(_tree, _map), do: error() - def lock(_tree), do: error() - def get_proofs_raw(_tree, _key), do: error() def count_zeros(binary) when is_binary(binary), do: count_zeros_raw(binary) defp count_zeros_raw(_binary), do: error() - @doc """ - Returns `{item_bytes, pair_bytes, pair_list_bytes, tree_bytes, stripe_size}` for C++ layout. - """ - def struct_sizes, do: struct_sizes_raw() - - @doc """ - Returns `{node_count, pair_count, approx_bytes}` for the tree. `approx_bytes` is - `nodes * sizeof(Item) + pairs * sizeof(pair_t)` (heap vectors in keys not included). - """ - def memory_stats(tree), do: memory_stats_raw(tree) - - @doc """ - GNU libc `malloc_info(3)` XML as a binary, or `:unsupported` on other platforms. - """ - def malloc_info, do: malloc_info_raw() - @doc """ Returns `{locked_states_count, pending_orphan_count, shared_states_live, merkletree_resources}`. """ def nif_stats, do: nif_stats_raw() + defp nif_stats_raw, do: error() def account_map_new(), do: error() def account_map_clone(_map), do: error() @@ -133,14 +37,5 @@ defmodule CMerkleTree do def account_map_storage(_map, _addr, _spec), do: error() def account_map_storage_roots(_map, _addr), do: error() - defp struct_sizes_raw, do: error() - defp memory_stats_raw(_tree), do: error() - defp malloc_info_raw, do: error() - defp nif_stats_raw, do: error() defp error, do: :erlang.nif_error(:nif_not_loaded) - - defp to_bytes32(value), do: Hash.to_bytes32(value) - - defp to_bytes(string) when is_binary(string), do: string - defp to_bytes(int) when is_integer(int), do: to_bytes32(int) end diff --git a/scripts/cmerkle_bench.exs b/scripts/cmerkle_bench.exs deleted file mode 100644 index 1076a17..0000000 --- a/scripts/cmerkle_bench.exs +++ /dev/null @@ -1,76 +0,0 @@ -# size = 10_000 -# ref = Base16.decode("0x96c2b5d03aa8e52230e74d9a08359c38c7608e5d9aba18773f3c90baf3806ccb") - -Application.ensure_all_started(:weak_ref) - -size = 1000 -ref = Base16.decode("0xe3545ad7961427b2a49d0b0a7b4a61e040acaefc432b69bcc1066b7c8c460a6b") -# ref = Base16.decode("0x3b99d56aa16278d50e85a8ea0939e62180c4f0305b773775f1844c3303a41897") 100_000 - -test_data = - Enum.map(1..size, fn idx -> - hash = Diode.hash("!#{idx}!") - {hash, hash} - end) - -none = spawn(fn -> - EtsLru.new(:cmerkle, 5) - Process.sleep(:infinity) -end) - - -for i <- 1..1000000 do - time_ms = :timer.tc(fn -> - tree_size = 1000 - tree_size_1 = tree_size + 1 - for _ <- 1..tree_size do - a = CMerkleTree.hash("b1000000!") - item = {a, a} - - WeakRef.new(none) - - tree = CMerkleTree.insert_items(CMerkleTree.new(), test_data) |> CMerkleTree.lock() - tree2 = CMerkleTree.clone(tree) - |> CMerkleTree.lock() - |> CMerkleTree.clone() - |> CMerkleTree.lock() - |> CMerkleTree.clone() - |> CMerkleTree.lock() - |> CMerkleTree.clone() - |> CMerkleTree.lock() - |> CMerkleTree.clone() - |> CMerkleTree.lock() - |> CMerkleTree.clone() - |> CMerkleTree.lock() - |> CMerkleTree.clone() - |> CMerkleTree.insert_items([item]) - - _proof = tree2 |> CMerkleTree.get_proofs(a) - _proof2 = tree |> CMerkleTree.get_proofs(a) - ^tree_size = CMerkleTree.size(tree) - ^tree_size = CMerkleTree.to_list(tree) |> length() - ^tree_size_1 = CMerkleTree.size(tree2) - ^tree_size_1 = CMerkleTree.to_list(tree2) |> length() - 92 = CMerkleTree.bucket_count(tree) - 92 = CMerkleTree.bucket_count(tree2) - - %{^a => {nil, b}} = CMerkleTree.difference(tree, tree2) - CMerkleTree.from_map(%{a => b}) - - nil = CMerkleTree.get(tree, a) - ^a = CMerkleTree.get(tree2, a) - - - if CMerkleTree.root_hash(tree) != ref do - raise("Error #{Base16.encode(CMerkleTree.root_hash(tree))} != #{Base16.encode(ref)}") - end - - EtsLru.put(:cmerkle, CMerkleTree.root_hash(tree), tree) - end - end) - |> elem(0) - |> div(1000) - - IO.puts("Run #{i} #{time_ms}ms") - :erlang.garbage_collect() -end diff --git a/scripts/cmerkle_bench2.exs b/scripts/cmerkle_bench2.exs deleted file mode 100644 index 2189326..0000000 --- a/scripts/cmerkle_bench2.exs +++ /dev/null @@ -1,42 +0,0 @@ -# size = 10_000 -# ref = Base16.decode("0x96c2b5d03aa8e52230e74d9a08359c38c7608e5d9aba18773f3c90baf3806ccb") - -Application.ensure_all_started(:weak_ref) - -size = 1000 -ref = Base16.decode("0xe3545ad7961427b2a49d0b0a7b4a61e040acaefc432b69bcc1066b7c8c460a6b") -# ref = Base16.decode("0x3b99d56aa16278d50e85a8ea0939e62180c4f0305b773775f1844c3303a41897") 100_000 - -for i <- 1..1000000 do - time_ms = :timer.tc(fn -> - tree_size = 1000 - for _ <- 1..tree_size do - test_data = - Enum.map(1..size, fn idx -> - hash = Diode.hash("!#{idx}!") - {hash, hash} - end) - - tree = CMerkleTree.new() - for {key, value} <- test_data do - CMerkleTree.insert_item(tree, {key, value}) - end - for {key, value} <- test_data do - ^value = CMerkleTree.get(tree, key) - CMerkleTree.get_proofs(tree, key) != nil - end - ^tree_size = length(CMerkleTree.to_list(tree)) - CMerkleTree.lock(tree) - - if CMerkleTree.root_hash(tree) != ref do - raise("Error #{Base16.encode(CMerkleTree.root_hash(tree))} != #{Base16.encode(ref)}") - end - - end - end) - |> elem(0) - |> div(1000) - - IO.puts("Run #{i} #{time_ms}ms") - :erlang.garbage_collect() -end diff --git a/scripts/cmerkle_bench3.exs b/scripts/cmerkle_bench3.exs deleted file mode 100644 index 87ca581..0000000 --- a/scripts/cmerkle_bench3.exs +++ /dev/null @@ -1,28 +0,0 @@ -Application.ensure_all_started(:mutable_map) - - -# uncompact = fn %Chain.State{accounts: old_accounts}-> -# Enum.reduce(old_accounts, MutableMap.new(), fn {id, acc}, accounts -> -# MutableMap.put(accounts, id, Chain.Account.uncompact(acc)) -# end) -# end - - -EtsLru.new(:leak_state, 15) -state = File.read!("leak_state.bin") - -for i <- 1..100 do - one = EtsLru.fetch(:leak_state, i, fn -> - ret = :erlang.binary_to_term(state) - |> Chain.State.uncompact() - |> Chain.State.lock() - # |> IO.inspect() - - :erlang.garbage_collect() - :erlang.garbage_collect(Process.whereis(MutableMap.Beacon)) - - ret - end) - - IO.puts("#{i}: #{map_size(one)}") -end diff --git a/scripts/cmerkle_deadlock_watchdog.exs b/scripts/cmerkle_deadlock_watchdog.exs index d0f0315..f1587c6 100644 --- a/scripts/cmerkle_deadlock_watchdog.exs +++ b/scripts/cmerkle_deadlock_watchdog.exs @@ -1,4 +1,4 @@ -# CMerkleTree NIF — deadlock / hang watchdog for fuzz and stress harnesses. +# CAccountMap NIF — deadlock / hang watchdog for fuzz and stress harnesses. # # Wraps a child command, monitors stdout/stderr for progress markers, and exits # with code 124 if the child stops making progress or exceeds wall-clock budget. @@ -40,7 +40,7 @@ defmodule CMerkleDeadlockWatchdog do poll_ms = Keyword.get(opts, :poll_interval, 5) * 1_000 IO.puts(:stderr, """ - === CMerkleTree deadlock watchdog === + === CAccountMap deadlock watchdog === command=#{inspect(cmd)} progress_timeout=#{div(progress_timeout_ms, 1000)}s wall_timeout=#{format_wall(wall_timeout_ms)} poll=#{div(poll_ms, 1000)}s """) diff --git a/scripts/cmerkle_fuzz.exs b/scripts/cmerkle_fuzz.exs index 8a36bf6..e0f722d 100644 --- a/scripts/cmerkle_fuzz.exs +++ b/scripts/cmerkle_fuzz.exs @@ -1,4 +1,4 @@ -# CMerkleTree NIF stress / fuzz harness. +# CAccountMap NIF stress / fuzz harness (map + storage list APIs only). # Run from repo root (use --no-start to avoid booting the full Diode app): # mix run --no-start scripts/cmerkle_fuzz.exs -- --iterations 10000 --seed 42 # Environment (optional): MERKLE_FUZZ_ITERATIONS, MERKLE_FUZZ_SEED, MERKLE_FUZZ_MAX_KEYS @@ -6,7 +6,7 @@ # Options: # --iterations N number of fuzz rounds (default: 0 = run until SIGINT) # --seed N RNG seed for reproducibility (default: random) -# --max-keys N cap keys per tree per scenario (default: 900) +# --max-keys N cap accounts / slots per scenario (default: 900) # # Each successful round prints: FUZZ_OK # On Elixir exception, prints stack trace then exits non-zero. @@ -66,7 +66,7 @@ defmodule CMerkleFuzz do :rand.seed(:exsss, {seed, seed, seed}) IO.puts(:stderr, """ - === CMerkleTree fuzz === + === CAccountMap fuzz === #{runtime_banner()} seed=#{seed} iterations=#{iterations |> format_iters()} max_keys=#{max_keys} """) @@ -114,40 +114,30 @@ defmodule CMerkleFuzz do end defp run_round(round, ctx) do - scenario = ctx[:scenario] || :rand.uniform(31) + scenario = ctx[:scenario] || :rand.uniform(21) case scenario do - 1 -> s_string_batch_insert_diff(round, ctx) - 2 -> s_u256_sequential_diff(round, ctx) - 3 -> s_clone_parallel_extensions_diff(round, ctx) - 4 -> s_clone_overwrite_diff(round, ctx) - 5 -> s_triple_fork_diff(round, ctx) - 6 -> s_delete_and_reinsert(round, ctx) - 7 -> s_proofs_and_roots(round, ctx) - 8 -> s_lock_clone_insert(round, ctx) - 9 -> s_lock_and_difference(round, ctx) - 10 -> s_account_map_uncompact(round, ctx) - 11 -> s_account_map_clone_mutate(round, ctx) - 12 -> s_uncompact_and_storage_diff(round, ctx) - 13 -> s_account_get_materialize_diff(round, ctx) - 14 -> s_gc_during_lock_diff(round, ctx) - 15 -> s_import_map_lock_diff(round, ctx) - 16 -> s_state_lock_clone_mutate(round, ctx) - 17 -> s_account_map_lock_bulk(round, ctx) - 18 -> s_account_map_lock_concurrent(round, ctx) - 19 -> s_account_map_list_diff_large_compact(round, ctx) - 20 -> s_list_diff_vs_to_list(round, ctx) - 21 -> s_list_diff_vs_account_map_lock(round, ctx) - 22 -> s_list_diff_vs_clone_put(round, ctx) - 23 -> s_list_diff_vs_storage_difference(round, ctx) - 24 -> s_dual_map_list_diff_order(round, ctx) - 25 -> s_list_diff_compact_vs_live(round, ctx) - 26 -> s_list_diff_vs_uncompact(round, ctx) - 27 -> s_list_diff_vs_state_lock(round, ctx) - 28 -> s_list_diff_vs_account_map_get(round, ctx) - 29 -> s_list_diff_gc_pressure(round, ctx) - 30 -> s_prepare_state_composite(round, ctx) - 31 -> s_clone_equivalence(round, ctx) + 1 -> s_account_map_uncompact(round, ctx) + 2 -> s_account_map_clone_mutate(round, ctx) + 3 -> s_uncompact_and_map_diff(round, ctx) + 4 -> s_account_get_materialize_diff(round, ctx) + 5 -> s_gc_during_map_lock_diff(round, ctx) + 6 -> s_state_lock_clone_mutate(round, ctx) + 7 -> s_account_map_lock_bulk(round, ctx) + 8 -> s_account_map_lock_concurrent(round, ctx) + 9 -> s_account_map_list_diff_large_compact(round, ctx) + 10 -> s_list_diff_vs_to_list(round, ctx) + 11 -> s_list_diff_vs_account_map_lock(round, ctx) + 12 -> s_list_diff_vs_clone_put(round, ctx) + 13 -> s_list_diff_vs_storage_apis(round, ctx) + 14 -> s_dual_map_list_diff_order(round, ctx) + 15 -> s_list_diff_compact_vs_live(round, ctx) + 16 -> s_list_diff_vs_uncompact(round, ctx) + 17 -> s_list_diff_vs_state_lock(round, ctx) + 18 -> s_list_diff_vs_account_map_get(round, ctx) + 19 -> s_list_diff_gc_pressure(round, ctx) + 20 -> s_prepare_state_composite(round, ctx) + 21 -> s_clone_equivalence(round, ctx) other -> raise("unknown fuzz scenario #{inspect(other)}") end @@ -162,207 +152,20 @@ defmodule CMerkleFuzz do IO.puts("FUZZ_OK #{round} #{scenario}") end - defp s_string_batch_insert_diff(_round, %{max_keys: mk}) do - n = :rand.uniform(div(mk, 2)) + 40 - data = Enum.map(1..n, fn i -> {String.pad_leading("#{i}", 32), CMerkleTree.hash("f#{i}")} end) - {a, rest} = Enum.split(data, div(n * 3, 5)) - {b, c} = Enum.split(rest, div(length(rest), 2) |> max(1)) - - base = CMerkleTree.new() |> CMerkleTree.insert_items(a) - - t2 = - base |> CMerkleTree.clone() |> CMerkleTree.insert_items(b ++ c) - - t_alt = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert_items(b) - |> CMerkleTree.insert_items(Enum.take(c, min(length(c), 50))) - - _ = CMerkleTree.difference(t2, t_alt) - _ = CMerkleTree.root_hash(t2) - _ = CMerkleTree.root_hash(t_alt) - end - - defp s_u256_sequential_diff(_round, %{max_keys: mk}) do - hi = :rand.uniform(min(mk, 520)) + 80 - - slots = - Enum.map(0..hi, fn i -> - {<>, <>} - end) - - base = CMerkleTree.new() |> CMerkleTree.insert_items(slots) - - left = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert_items( - Enum.map((hi + 1)..(hi + 40), fn i -> - {<>, <>} - end) - ) - - right = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert_items( - Enum.map((hi + 41)..(hi + 95), fn i -> - {<>, <>} - end) - ) - - _ = CMerkleTree.difference(left, right) - _ = CMerkleTree.root_hash(left) - _ = CMerkleTree.root_hash(right) - end - - defp s_clone_parallel_extensions_diff(_round, %{max_keys: mk}) do - hi = :rand.uniform(min(mk, 400)) + 50 - - base = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(0..hi, fn i -> {<>, <>} end) - ) - - ext1 = (hi + 1)..(hi + 30) - ext2 = (hi + 31)..(hi + 70) - - left = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert_items( - Enum.map(ext1, fn i -> {<>, <>} end) - ) - - right = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert_items( - Enum.map(ext2, fn i -> {<>, <>} end) - ) - - _ = CMerkleTree.difference(left, right) - end - - defp s_clone_overwrite_diff(_round, %{max_keys: mk}) do - hi = :rand.uniform(min(mk, 350)) + 80 - - base = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(0..hi, fn i -> {<>, <>} end) - ) - - left = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert_items( - Enum.map((hi + 1)..(hi + 35), fn i -> - {<>, <>} - end) - ) - - lo = div(hi, 4) - mid = lo + :rand.uniform(max(div(hi - lo, 2), 5)) - - right = - base - |> CMerkleTree.clone() - |> then(fn t -> - Enum.reduce(lo..mid, t, fn i, acc -> - CMerkleTree.insert(acc, <>, <>) - end) - end) - - _ = CMerkleTree.difference(left, right) - end - - defp s_triple_fork_diff(_round, %{max_keys: mk}) do - n = :rand.uniform(min(mk, 280)) + 60 - - base = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..n, fn i -> {String.pad_leading("#{i}", 32), CMerkleTree.hash("b#{i}")} end) - ) - - a = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert(String.pad_leading("x", 32), CMerkleTree.hash("x")) - - b = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert(String.pad_leading("y", 32), CMerkleTree.hash("y")) - - c = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert(String.pad_leading("z", 32), CMerkleTree.hash("z")) - - _ = CMerkleTree.difference(a, b) - _ = CMerkleTree.difference(b, c) - _ = CMerkleTree.difference(a, c) - end - - defp s_delete_and_reinsert(_round, %{max_keys: mk}) do - n = :rand.uniform(min(mk, 200)) + 30 - keys = Enum.map(1..n, fn i -> String.pad_leading("#{i}", 32) end) - - t = - Enum.reduce(keys, CMerkleTree.new(), fn k, acc -> - CMerkleTree.insert(acc, k, CMerkleTree.hash(k)) - end) - - to_del = Enum.take_random(keys, div(length(keys), 3) |> max(3)) - - t = - Enum.reduce(to_del, t, fn k, acc -> - CMerkleTree.delete(acc, k) - end) - - t = - Enum.reduce(to_del, t, fn k, acc -> - CMerkleTree.insert(acc, k, CMerkleTree.hash("re#{k}")) - end) - - _ = CMerkleTree.root_hash(t) - _ = CMerkleTree.size(t) - end - - defp s_proofs_and_roots(_round, %{max_keys: mk}) do - hi = :rand.uniform(min(180, mk)) + 20 - - t = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(0..hi, fn i -> {<>, <>} end) - ) - - _ = CMerkleTree.root_hash(t) - _ = CMerkleTree.root_hashes(t) - k = <<:rand.uniform(hi)::unsigned-size(256)>> - _ = CMerkleTree.get_proofs(t, k) - end - - # --- S8–S18: lock, account map, GC, bulk account_map_lock (see c_src/LOCK_ORDER.md) --- - defp addr(i), do: <> defp slot(i), do: <> + defp storage_list(i, mult \\ 3) do + [{slot(i), <>}] + end + defp build_compact_accounts(n) do Enum.reduce(1..n, State.new(), fn i, st -> - tree = - CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - acc = %Account{ nonce: i, balance: i * 1_000, - storage_root: tree, + storage_root: storage_list(i), code: <>, map_backed: false } @@ -373,52 +176,22 @@ defmodule CMerkleFuzz do |> Map.fetch!(:accounts) end - defp s_lock_clone_insert(_round, %{max_keys: mk}) do - n = :rand.uniform(min(mk, 120)) + 20 - - base = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..n, fn i -> {String.pad_leading("lk#{i}", 32), CMerkleTree.hash("lk#{i}")} end) - ) - - locked = - if :rand.uniform(2) == 1 do - base - |> CMerkleTree.clone() - |> CMerkleTree.insert(String.pad_leading("pre_lock", 32), CMerkleTree.hash("pre")) - |> CMerkleTree.lock() - else - base - |> CMerkleTree.clone() - |> CMerkleTree.lock() - end - - _ = CMerkleTree.root_hash(locked) + defp build_account_map(n) do + Enum.reduce(1..n, CAccountMap.new(), fn i, acc -> + CAccountMap.put(acc, addr(i), i, i * 1_000, storage_list(i), <>) + end) end - defp s_lock_and_difference(_round, %{max_keys: mk}) do - n = :rand.uniform(min(mk, 100)) + 30 - - shared = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..n, fn i -> {String.pad_leading("sh#{i}", 32), CMerkleTree.hash("sh#{i}")} end) - ) - - other = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..n, fn i -> {String.pad_leading("ot#{i}", 32), CMerkleTree.hash("ot#{i}")} end) - ) - - _ = - shared - |> CMerkleTree.clone() - |> CMerkleTree.lock() - - _ = CMerkleTree.difference(shared, other) - _ = CMerkleTree.difference(other, shared) + defp build_live_state(n) do + Enum.reduce(1..n, State.new(), fn i, st -> + State.set_account(st, addr(i), %{ + Account.new(nonce: i) + | storage_root: [{slot(i), <>}], + map_backed: false, + root_hash: nil + }) + end) + |> State.normalize() end defp s_account_map_uncompact(_round, _ctx) do @@ -436,14 +209,14 @@ defmodule CMerkleFuzz do fork = accounts |> CAccountMap.clone() - |> CAccountMap.put(addr(1), 99, 99_000, CMerkleTree.new(), <<99>>) + |> CAccountMap.put(addr(1), 99, 99_000, [], <<99>>) _ = CAccountMap.to_list(accounts) if CAccountMap.size(fork) != n, do: raise("fork size mismatch") end - defp s_uncompact_and_storage_diff(_round, _ctx) do + defp s_uncompact_and_map_diff(_round, _ctx) do n = :rand.uniform(60) + 20 compact = build_compact_accounts(n) @@ -453,19 +226,15 @@ defmodule CMerkleFuzz do 1..4 |> Task.async_stream( fn i -> - a = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(i), <>}, - {slot(i + 1000), <>} - ]) + a = build_account_map(10 + i) + b = CAccountMap.clone(a) b = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(i), <>}, - {slot(i + 2000), <>} - ]) + CAccountMap.storage_put_map(b, %{ + addr(1) => %{slot(i + 1000) => <>} + }) - CMerkleTree.difference(a, b) + CAccountMap.difference_full(a, b) end, max_concurrency: 4, timeout: 60_000, @@ -486,38 +255,23 @@ defmodule CMerkleFuzz do {_n, _b, root, _c} = CAccountMap.get(accounts, addr(1)) if not is_binary(root) or byte_size(root) != 32, do: raise("expected 32-byte root hash") - # Exercise storage APIs (get no longer returns a live trie for difference). _ = CAccountMap.storage_get(accounts, addr(1), slot(1)) _ = CAccountMap.storage_root_hash(accounts, addr(1)) _ = CAccountMap.storage_to_list(accounts, addr(1)) _ = CAccountMap.get(accounts, addr(2)) end - defp s_gc_during_lock_diff(_round, %{max_keys: mk}) do - n = :rand.uniform(min(mk, 80)) + 20 - - shared = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..n, fn i -> {String.pad_leading("gc#{i}", 32), CMerkleTree.hash("gc#{i}")} end) - ) - - other = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..n, fn i -> {String.pad_leading("go#{i}", 32), CMerkleTree.hash("go#{i}")} end) - ) + defp s_gc_during_map_lock_diff(_round, _ctx) do + map = build_account_map(:rand.uniform(60) + 20) + fork = CAccountMap.clone(map) 1..6 |> Task.async_stream( fn w -> if rem(w, 2) == 0 do - _ = - shared - |> CMerkleTree.clone() - |> CMerkleTree.lock() + _ = CAccountMap.lock(map) else - _ = CMerkleTree.difference(shared, other) + _ = CAccountMap.difference_full(map, fork) end :ok @@ -528,39 +282,10 @@ defmodule CMerkleFuzz do ) |> Stream.run() - short = - Enum.map(1..12, fn i -> - {String.pad_leading("tmp#{i}", 32), CMerkleTree.hash("tmp#{i}")} - end) - - _ = CMerkleTree.new() |> CMerkleTree.insert_items(short) + _ = CAccountMap.new() |> CAccountMap.lock() :erlang.garbage_collect() end - defp s_import_map_lock_diff(_round, %{max_keys: mk}) do - n = :rand.uniform(min(mk, 150)) + 30 - - items = - for i <- 1..n, into: %{} do - {String.pad_leading("im#{i}", 32), CMerkleTree.hash("im#{i}")} - end - - tree = CMerkleTree.new() |> CMerkleTree.import_map(items) - - sibling = - tree - |> CMerkleTree.clone() - |> CMerkleTree.insert(String.pad_leading("extra", 32), CMerkleTree.hash("extra")) - - _ = - tree - |> CMerkleTree.clone() - |> CMerkleTree.lock() - - _ = CMerkleTree.difference(tree, sibling) - _ = CMerkleTree.root_hash(tree) - end - defp s_state_lock_clone_mutate(_round, _ctx) do n = :rand.uniform(60) + 10 compact = build_compact_accounts(n) @@ -570,9 +295,12 @@ defmodule CMerkleFuzz do |> then(fn accounts -> %State{accounts: accounts} end) |> State.uncompact() - if :rand.uniform(2) == 1 do - parent = State.normalize(parent) - end + parent = + if :rand.uniform(2) == 1 do + State.normalize(parent) + else + parent + end _ = State.hash(parent) Chain.State.lock(parent) @@ -609,26 +337,15 @@ defmodule CMerkleFuzz do map = Enum.reduce(1..n, CAccountMap.new(), fn i, acc -> g = div(i - 1, group) - - storage = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(g + 1), <>} - ]) - - CAccountMap.put(acc, addr(i), i, i * 1_000, storage, <>) + CAccountMap.put(acc, addr(i), i, i * 1_000, storage_list(g + 1, 1), <>) end) - store = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(88_888), <<88_888::unsigned-size(256)>>} - ]) - _ = CAccountMap.lock(map) fork = map |> CAccountMap.clone() - |> CAccountMap.put(addr(1), 77, 77_000, CMerkleTree.new(), <<77>>) + |> CAccountMap.put(addr(1), 77, 77_000, [], <<77>>) if CAccountMap.size(fork) != n, do: raise("fork size mismatch") @@ -638,15 +355,7 @@ defmodule CMerkleFuzz do defp s_account_map_lock_concurrent(_round, _ctx) do n = :rand.uniform(60) + 20 - - map = - Enum.reduce(1..n, CAccountMap.new(), fn i, acc -> - storage = - CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - - CAccountMap.put(acc, addr(i), i, i * 1_000, storage, <>) - end) - + map = build_account_map(n) workers = 6 1..workers @@ -669,8 +378,6 @@ defmodule CMerkleFuzz do |> Stream.run() end - # --- S19–S30: native account_map difference_full (see c_src/LOCK_ORDER.md D-C7, D-D7) --- - defp s_account_map_list_diff_large_compact(_round, _ctx) do n = :rand.uniform(200) + 100 compact = build_compact_accounts(n) @@ -698,7 +405,6 @@ defmodule CMerkleFuzz do n = :rand.uniform(80) + 20 map = build_account_map(n) fork = CAccountMap.clone(map) - workers = 6 1..workers @@ -722,7 +428,6 @@ defmodule CMerkleFuzz do defp s_list_diff_vs_account_map_lock(_round, _ctx) do map = build_account_map(:rand.uniform(60) + 20) fork = CAccountMap.clone(map) - workers = 6 1..workers @@ -745,7 +450,6 @@ defmodule CMerkleFuzz do defp s_list_diff_vs_clone_put(_round, _ctx) do map = build_account_map(:rand.uniform(50) + 15) - workers = 6 1..workers @@ -757,7 +461,7 @@ defmodule CMerkleFuzz do _ = CAccountMap.difference_full(map, fork) else i = rem(w, 10) + 1 - storage = CMerkleTree.insert(CMerkleTree.new(), slot(w + i), <>) + storage = [{slot(w + i), <>}] _ = CAccountMap.put(fork, addr(i), w, w * 100, storage, <>) end @@ -770,9 +474,8 @@ defmodule CMerkleFuzz do |> Stream.run() end - defp s_list_diff_vs_storage_difference(_round, _ctx) do + defp s_list_diff_vs_storage_apis(_round, _ctx) do map = build_account_map(:rand.uniform(40) + 10) - workers = 6 1..workers @@ -850,7 +553,6 @@ defmodule CMerkleFuzz do defp s_list_diff_vs_uncompact(_round, _ctx) do compact = build_compact_accounts(:rand.uniform(80) + 20) - workers = 4 1..workers @@ -875,7 +577,6 @@ defmodule CMerkleFuzz do defp s_list_diff_vs_state_lock(_round, _ctx) do st = build_live_state(:rand.uniform(60) + 15) fork = State.clone(st) - workers = 6 1..workers @@ -898,7 +599,6 @@ defmodule CMerkleFuzz do defp s_list_diff_vs_account_map_get(_round, _ctx) do map = build_account_map(:rand.uniform(40) + 10) - workers = 6 1..workers @@ -925,7 +625,7 @@ defmodule CMerkleFuzz do for _ <- 1..8 do _ = CAccountMap.difference_full(map, fork) - short = CMerkleTree.new() |> CMerkleTree.clone() |> CMerkleTree.lock() + short = CAccountMap.new() |> CAccountMap.lock() _ = short :erlang.garbage_collect() end @@ -967,49 +667,9 @@ defmodule CMerkleFuzz do |> Stream.run() end - defp build_account_map(n) do - Enum.reduce(1..n, CAccountMap.new(), fn i, acc -> - storage = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - CAccountMap.put(acc, addr(i), i, i * 1_000, storage, <>) - end) - end - - defp build_live_state(n) do - Enum.reduce(1..n, State.new(), fn i, st -> - storage = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - - State.set_account(st, addr(i), %{ - Account.new(nonce: i) - | storage_root: storage, - map_backed: false, - root_hash: nil - }) - end) - |> State.normalize() - end - - defp read_proc_rss_kb do - case File.read("/proc/#{System.pid()}/status") do - {:ok, body} -> - case Regex.run(~r/VmRSS:\s+(\d+)\s+kB/i, body) do - [_, n] -> String.to_integer(n) - _ -> 0 - end - - _ -> - 0 - end - end - defp s_clone_equivalence(_round, _ctx) do n = :rand.uniform(40) + 10 - - map = - Enum.reduce(1..n, CAccountMap.new(), fn i, acc -> - storage = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - CAccountMap.put(acc, addr(i), i, i * 500, storage, <>) - end) - + map = build_account_map(n) state = %Chain.State{accounts: map} id = addr(:rand.uniform(n)) @@ -1031,6 +691,19 @@ defmodule CMerkleFuzz do end end + defp read_proc_rss_kb do + case File.read("/proc/#{System.pid()}/status") do + {:ok, body} -> + case Regex.run(~r/VmRSS:\s+(\d+)\s+kB/i, body) do + [_, n] -> String.to_integer(n) + _ -> 0 + end + + _ -> + 0 + end + end + defp check_fuzz_rss(max_delta_kb, round) do baseline = Process.get(:cmerkle_fuzz_baseline_rss, 0) rss = read_proc_rss_kb() diff --git a/scripts/cmerkle_fuzz.sh b/scripts/cmerkle_fuzz.sh index 2e7cf08..14b4adc 100755 --- a/scripts/cmerkle_fuzz.sh +++ b/scripts/cmerkle_fuzz.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Long-running CMerkleTree NIF fuzzer with crash logging. +# Long-running CAccountMap NIF fuzzer with crash logging. # Usage (from repository root): # ./scripts/cmerkle_fuzz.sh # MERKLE_FUZZ_ITERATIONS=5000 MERKLE_FUZZ_SEED=42 ./scripts/cmerkle_fuzz.sh @@ -31,7 +31,7 @@ fi write_header() { { - echo "========== CMerkleTree fuzz crash log ==========" + echo "========== CAccountMap fuzz crash log ==========" echo "started_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "host=$(hostname 2>/dev/null || echo unknown)" echo "pwd=$(pwd)" diff --git a/scripts/cmerkle_heap_assumptions.exs b/scripts/cmerkle_heap_assumptions.exs deleted file mode 100644 index f643c4b..0000000 --- a/scripts/cmerkle_heap_assumptions.exs +++ /dev/null @@ -1,333 +0,0 @@ -# CMerkleTree NIF — targeted stress for heap / invalid-free hypotheses (live crash triage). -# -# Hypothesis map (see c_src/preallocator.hpp, merkletree.cpp, nif.cpp): -# A — PreAllocator destroy_item/backbuffer vs ~PreAllocator slot count (pair_t ~ on valid storage) -# B — GlobalStripePool: stripe malloc reused across Tree lifetimes; destructor + pool put -# C — Second stripe: only first slot used after full first stripe (partial stripe destructor) -# D — ItemPool COW: clone + write → fork_for_write / clone_for_fork + pair_list clone -# E — split_node / large trie (leaf_bucket churn, push_back_no_delete paths) -# F — NIF locked_states enter_lock/leave_lock + root_hash dedup (live server pattern) -# G — difference/into tree — insert_item on forked structures -# H — get_proofs + root_hashes — read paths after dirty/update_merkle_hash_count -# -# Run from repo root: -# mix run --no-start scripts/cmerkle_heap_assumptions.exs -- --rounds 200 -# -# Options: -# --rounds N repetitions per scenario (default: 150) -# --suite-pass N run the full scenario list this many times (default: 1) -# --seed N RNG seed (default: env MERKLE_HEAP_SEED or fixed 20260415) -# -# Exit 0 only if every scenario completes without raising. Native crashes (abort/segfault) -# are not catchable in Elixir; use ASan build or MERKLE_FUZZ_ASAN with cmerkle_fuzz.sh pattern. - -defmodule CMerkleHeapAssumptions do - @moduledoc false - - def normalize_argv(argv) do - case argv do - ["--" | rest] -> rest - other -> other - end - end - - def main(argv) do - argv = normalize_argv(argv) - - {opts, _} = - OptionParser.parse!(argv, - strict: [ - rounds: :integer, - suite_pass: :integer, - seed: :integer - ] - ) - - rounds = Keyword.get(opts, :rounds, env_int("MERKLE_HEAP_ROUNDS") || 150) - suite_pass = Keyword.get(opts, :suite_pass, env_int("MERKLE_HEAP_SUITE_PASS") || 1) - - seed = - Keyword.get(opts, :seed) || - env_int("MERKLE_HEAP_SEED") || - 20_260_415 - - :rand.seed(:exsss, {seed, seed, seed}) - - {_ib, _pb, _plb, _tb, stripe} = CMerkleTree.struct_sizes() - - IO.puts(:stderr, """ - === CMerkleTree heap assumption stress === - otp=#{System.otp_release()} elixir=#{System.version()} pid=#{:os.getpid()} - seed=#{seed} rounds_per_scenario=#{rounds} suite_passes=#{suite_pass} stripe_size=#{stripe} - """) - - ctx = %{rounds: rounds, stripe: stripe, seed: seed} - - Enum.each(1..suite_pass, fn pass -> - IO.puts(:stderr, "--- suite pass #{pass}/#{suite_pass} ---") - - for {id, title, fun} <- scenarios() do - t0 = System.monotonic_time(:millisecond) - - try do - fun.(ctx) - dt = System.monotonic_time(:millisecond) - t0 - IO.puts(:stderr, "ASSUMPTION_OK #{id} #{inspect(title)} #{dt}ms") - catch - kind, reason -> - IO.puts(:stderr, "ASSUMPTION_FAIL #{id} #{inspect(title)} kind=#{inspect(kind)} reason=#{inspect(reason)}") - :erlang.raise(kind, reason, __STACKTRACE__) - end - end - end) - - IO.puts(:stderr, "=== all scenarios finished (Elixir layer); exit 0 ===") - end - - defp env_int(name) do - case System.get_env(name) do - nil -> nil - "" -> nil - s -> String.to_integer(String.trim(s)) - end - rescue - ArgumentError -> nil - end - - defp scenarios do - [ - {:A, "destroy_item/backbuffer: heavy delete+reinsert on one tree", &a_delete_reinsert_churn/1}, - {:B, "GlobalStripePool: many short-lived trees (alloc+drop)", &b_short_lived_trees/1}, - {:C, "stripe boundary: exactly stripe+1 keys then shrink", &c_second_stripe_partial/1}, - {:D, "COW fork: clone then write (make_writeable path)", &d_clone_write_fork/1}, - {:E, "large trie: many inserts (split_node / internal nodes)", &e_large_many_keys/1}, - {:F, "lock + clone chain (NIF locked_states / root_hash dedup)", &f_lock_clone_chain/1}, - {:G, "difference on large divergent trees", &g_difference_heavy/1}, - {:H, "proofs + root_hashes after deep updates", &h_proofs_and_hashes/1}, - {:I, "account_map_lock bulk identical storage roots", &i_identical_root_lock_bulk/1}, - {:J, "account_map difference_full bounded shared_states growth", &j_difference_full_heap/1} - ] - end - - # A: pair_t destroy_item -> backbuffer; PreAllocator ~ must match constructed slots. - defp a_delete_reinsert_churn(%{rounds: r}) do - n = max(120, min(r, 400)) - keys = Enum.map(1..n, fn i -> String.pad_leading("#{i}", 32) end) - - t = - Enum.reduce(keys, CMerkleTree.new(), fn k, acc -> - CMerkleTree.insert(acc, k, CMerkleTree.hash("a#{k}")) - end) - - Enum.each(1..r, fn round -> - to_del = Enum.take_random(keys, min(40 + rem(round, 30), div(n, 2))) - - t = - Enum.reduce(to_del, t, fn k, acc -> - CMerkleTree.delete(acc, k) - end) - - t = - Enum.reduce(to_del, t, fn k, acc -> - CMerkleTree.insert(acc, k, CMerkleTree.hash("b#{k}#{round}")) - end) - - _ = CMerkleTree.root_hash(t) - _ = CMerkleTree.size(t) - :ok - end) - end - - # B: tree destroyed -> SharedState ~Tree -> PreAllocator ~ ; stripes may return to global pool. - defp b_short_lived_trees(%{rounds: r}) do - Enum.each(1..max(r, 200), fn i -> - k = rem(i, 50) + 5 - - data = - Enum.map(1..k, fn j -> - {String.pad_leading("#{i}_#{j}", 32), CMerkleTree.hash("s#{i}_#{j}")} - end) - - t = CMerkleTree.new() |> CMerkleTree.insert_items(data) - _ = CMerkleTree.root_hash(t) - _ = CMerkleTree.bucket_count(t) - :ok - end) - end - - # C: first stripe full (8) then minimal use of second stripe — destructor len on stripe 2. - defp c_second_stripe_partial(%{rounds: r, stripe: s}) do - Enum.each(1..max(div(r, 2), 40), fn pass -> - # Fill exactly s keys (one full stripe of pair allocations), then add one on next stripe. - base = Enum.map(1..s, fn i -> {String.pad_leading("c#{pass}_#{i}", 32), CMerkleTree.hash("c#{i}")} end) - extra = {String.pad_leading("c#{pass}_extra", 32), CMerkleTree.hash("extra")} - - t = CMerkleTree.new() |> CMerkleTree.insert_items(base) - t = CMerkleTree.insert(t, elem(extra, 0), elem(extra, 1)) - _ = CMerkleTree.root_hash(t) - - # Shrink back toward boundary (deletes use insert null — churns PreAllocator). - t = - Enum.reduce(1..div(s, 2), t, fn i, acc -> - CMerkleTree.delete(acc, String.pad_leading("c#{pass}_#{i}", 32)) - end) - - _ = CMerkleTree.root_hash(t) - _ = CMerkleTree.size(t) - :ok - end) - end - - # D: shared Tree copy + pool; write triggers clone_for_write / fork. - defp d_clone_write_fork(%{rounds: r}) do - n = max(80, min(r, 350)) - - base = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..n, fn i -> {String.pad_leading("d#{i}", 32), CMerkleTree.hash("d#{i}")} end) - ) - - Enum.each(1..max(div(r, 3), 30), fn i -> - a = CMerkleTree.clone(base) |> CMerkleTree.insert(String.pad_leading("dx#{i}", 32), CMerkleTree.hash("dx#{i}")) - b = CMerkleTree.clone(base) |> CMerkleTree.insert(String.pad_leading("dy#{i}", 32), CMerkleTree.hash("dy#{i}")) - _ = CMerkleTree.root_hash(a) - _ = CMerkleTree.root_hash(b) - _ = CMerkleTree.difference(a, b) - :ok - end) - end - - # E: many keys — stress split_node, internal merge paths in update_merkle_hash_count. - defp e_large_many_keys(%{rounds: r}) do - n = max(800, min(r * 8, 6000)) - - data = - Enum.map(1..n, fn i -> - {String.pad_leading("e#{i}", 32), CMerkleTree.hash("e#{i}")} - end) - - t = CMerkleTree.new() |> CMerkleTree.insert_items(data) - _ = CMerkleTree.root_hash(t) - _ = CMerkleTree.root_hashes(t) - _ = CMerkleTree.bucket_count(t) - :ok - end - - # F: lock registers root in NIF map; clone hits enter_lock path — live server pattern. - defp f_lock_clone_chain(%{rounds: r}) do - data = Enum.map(1..120, fn i -> {String.pad_leading("f#{i}", 32), CMerkleTree.hash("f#{i}")} end) - base = CMerkleTree.new() |> CMerkleTree.insert_items(data) |> CMerkleTree.lock() - - Enum.each(1..max(div(r, 5), 25), fn i -> - t = - CMerkleTree.clone(base) - |> CMerkleTree.insert(String.pad_leading("fz#{i}", 32), CMerkleTree.hash("fz#{i}")) - |> CMerkleTree.lock() - - _ = CMerkleTree.root_hash(t) - :ok - end) - end - - # G: difference walks both trees — clone_for_fork on into.insert_item. - defp g_difference_heavy(%{rounds: r}) do - m = max(200, min(r * 3, 900)) - - a = - Enum.map(1..m, fn i -> {String.pad_leading("g#{i}", 32), CMerkleTree.hash("ga#{i}")} end) - - b = - Enum.map(1..m, fn i -> {String.pad_leading("g#{i}", 32), CMerkleTree.hash("gb#{i}")} end) - - ta = CMerkleTree.new() |> CMerkleTree.insert_items(a) - tb = CMerkleTree.new() |> CMerkleTree.insert_items(b) - diff = CMerkleTree.difference(ta, tb) - _ = map_size(diff) - _ = CMerkleTree.root_hash(ta) - _ = CMerkleTree.root_hash(tb) - :ok - end - - # H: get_proofs + root_hashes after updates — read paths over possibly merged buckets. - defp h_proofs_and_hashes(%{rounds: r}) do - n = max(100, min(r * 2, 600)) - keys = Enum.map(1..n, fn i -> String.pad_leading("h#{i}", 32) end) - - t = - Enum.reduce(keys, CMerkleTree.new(), fn k, acc -> - CMerkleTree.insert(acc, k, CMerkleTree.hash(k)) - end) - - Enum.each(1..min(80, max(div(r, 2), 20)), fn _ -> - k = Enum.random(keys) - _ = CMerkleTree.get_proofs(t, k) - _ = CMerkleTree.root_hashes(t) - :ok - end) - end - - defp i_identical_root_lock_bulk(%{rounds: r}) do - n = max(40, min(r, 120)) - - Enum.each(1..max(div(r, 2), 20), fn _pass -> - map = - Enum.reduce(1..n, CAccountMap.new(), fn i, acc -> - storage = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {String.pad_leading("i1", 32), CMerkleTree.hash("i1")}, - {String.pad_leading("i2", 32), CMerkleTree.hash("i2")} - ]) - - CAccountMap.put(acc, <>, i, i * 1_000, storage, <>) - end) - - _ = CAccountMap.lock(map) - {_locked, orphans, _shared, _res} = CMerkleTree.nif_stats() - if orphans > 0, do: raise("scenario I orphans=#{orphans}") - :ok - end) - end - - defp j_difference_full_heap(%{rounds: r}) do - n = max(40, min(r, 150)) - - base = - Enum.reduce(1..n, CAccountMap.new(), fn i, acc -> - storage = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - CAccountMap.put(acc, <>, i, i * 1_000, storage, <>) - end) - - fork = - CAccountMap.clone(base) - |> then(fn map -> - {nonce, balance, storage, code} = CAccountMap.get(map, <<1::unsigned-size(160)>>) - - storage = - CMerkleTree.insert( - CMerkleTree.clone(storage), - slot(99_999), - <<99_999::unsigned-size(256)>> - ) - - CAccountMap.put(map, <<1::unsigned-size(160)>>, nonce + 1, balance, storage, code) - end) - - {_l0, orphans0, shared0, _r0} = CMerkleTree.nif_stats() - if orphans0 > 0, do: raise("scenario J initial orphans=#{orphans0}") - - Enum.each(1..max(div(r, 2), 30), fn _ -> - _ = CAccountMap.difference_full(base, fork) - :erlang.garbage_collect() - end) - - {_l1, orphans1, shared1, _r1} = CMerkleTree.nif_stats() - if orphans1 > 0, do: raise("scenario J orphans=#{orphans1}") - if shared1 - shared0 > 800, do: raise("scenario J shared_states growth #{shared1 - shared0}") - :ok - end - - defp slot(i), do: <> -end - -CMerkleHeapAssumptions.main(System.argv()) diff --git a/scripts/cmerkle_leak_test.exs b/scripts/cmerkle_leak_test.exs index 37955cc..d4fb524 100644 --- a/scripts/cmerkle_leak_test.exs +++ b/scripts/cmerkle_leak_test.exs @@ -1,4 +1,4 @@ -# CMerkleTree NIF memory leak regression harness. +# CAccountMap NIF memory leak regression harness. # # Run from repo root: # mix run --no-start scripts/cmerkle_leak_test.exs -- --rounds 50 --max-delta-kb 81920 @@ -12,6 +12,7 @@ # --seed N RNG seed (default: 20260709) # # Emits LEAK_OK on success. Exit non-zero on RSS regression. +# Uses CMerkleTree.nif_stats/0 for orphan SharedState counts. defmodule CMerkleLeakTest do @moduledoc false @@ -48,7 +49,7 @@ defmodule CMerkleLeakTest do :rand.seed(:exsss, {seed, seed, seed}) IO.puts(:stderr, """ - === CMerkleTree leak test === + === CAccountMap leak test === otp=#{System.otp_release()} pid=#{:os.getpid()} rounds=#{rounds} accounts=#{accounts} max_delta_kb=#{max_delta_kb} plateau=#{plateau} scenarios=#{inspect(scenarios)} @@ -103,22 +104,19 @@ defmodule CMerkleLeakTest do end) end + # A: empty / tiny map lock storm (frozen-only; no bare-tree lock) defp run_workload("A", rounds, _accounts) do Enum.each(1..rounds, fn _ -> - CMerkleTree.new() |> CMerkleTree.lock() + CAccountMap.new() |> CAccountMap.lock() end) end + # B: account_map_lock with identical list-storage roots defp run_workload("B", rounds, accounts) do Enum.each(1..rounds, fn _ -> map = Enum.reduce(1..accounts, CAccountMap.new(), fn i, acc -> - storage = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(1), val(1)}, - {slot(2), val(2)} - ]) - + storage = [{slot(1), val(1)}, {slot(2), val(2)}] CAccountMap.put(acc, addr(i), i, i * 1_000, storage, <>) end) @@ -126,6 +124,7 @@ defmodule CMerkleLeakTest do end) end + # C: compact → uncompact → normalize → lock (block-sync shaped) defp run_workload("C", rounds, accounts) do Enum.each(1..rounds, fn _ -> state = @@ -133,11 +132,9 @@ defmodule CMerkleLeakTest do acc = %Account{ nonce: i, balance: i * 1_000, - storage_root: - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(i), val(i)} - ]), - code: <> + storage_root: [{slot(i), val(i)}], + code: <>, + map_backed: false } State.set_account(st, addr(i), acc) @@ -161,6 +158,7 @@ defmodule CMerkleLeakTest do defp warmup_gc do force_gc() + _ = CAccountMap.new() _ = CMerkleTree.nif_stats() end diff --git a/scripts/cmerkle_leak_watchdog.exs b/scripts/cmerkle_leak_watchdog.exs index 7c564da..50a6f84 100644 --- a/scripts/cmerkle_leak_watchdog.exs +++ b/scripts/cmerkle_leak_watchdog.exs @@ -1,4 +1,4 @@ -# CMerkleTree NIF — memory leak watchdog for cmerkle_leak_test.exs. +# CAccountMap NIF — memory leak watchdog for cmerkle_leak_test.exs. # # Run from repo root: # mix run --no-start scripts/cmerkle_leak_watchdog.exs -- \ @@ -32,7 +32,7 @@ defmodule CMerkleLeakWatchdog do poll_ms = Keyword.get(opts, :poll_interval, 5) * 1_000 IO.puts(:stderr, """ - === CMerkleTree leak watchdog === + === CAccountMap leak watchdog === command=#{inspect(cmd)} progress_timeout=#{div(progress_timeout_ms, 1000)}s wall_timeout=#{format_wall(wall_timeout_ms)} """) diff --git a/scripts/cmerkle_memory_bench.exs b/scripts/cmerkle_memory_bench.exs deleted file mode 100644 index fc21602..0000000 --- a/scripts/cmerkle_memory_bench.exs +++ /dev/null @@ -1,192 +0,0 @@ -# CMerkleTree RSS workloads (plan: workloads A–D). -# -# Usage (from repo root, after `mix compile`): -# mix run scripts/cmerkle_memory_bench.exs -# mix run scripts/cmerkle_memory_bench.exs -- --workload all --trees 25 --pairs 800 --samples 3 -# -# Workloads A/B/C retain `trees` handles (B/C: one near-duplicate tree per handle). Lower `--trees` -# if the VM RSS is too high. See scripts/cmerkle_memory_evaluation_report.md for results. -# -# CSV columns: workload,iteration,trees,pairs,vm_rss_kb,vm_hwm_kb,node_count,pair_count,approx_bytes,wall_ms - -defmodule CMerkleMemoryBench do - @moduledoc false - - def main(argv \\ []) do - argv = normalize_argv(argv) - - {opts, _, _} = - OptionParser.parse(argv, - strict: [ - workload: :string, - trees: :integer, - pairs: :integer, - samples: :integer, - csv: :boolean - ], - aliases: [w: :workload] - ) - - workload = Keyword.get(opts, :workload, "all") - # Retained trees (B/C hold one fork per k); lower this if RSS is too high. - trees_k = Keyword.get(opts, :trees, 25) - pairs_n = Keyword.get(opts, :pairs, 800) - samples = Keyword.get(opts, :samples, 2) - csv = Keyword.get(opts, :csv, true) - - if not csv do - IO.puts("# struct_sizes: #{inspect(CMerkleTree.struct_sizes())}") - dbg_malloc() - end - - workloads = - case workload do - "all" -> ~w(a b c d) - w -> String.split(w, ",", trim: true) - end - - if csv do - IO.puts("workload,iteration,trees,pairs,vm_rss_kb,vm_hwm_kb,node_count,pair_count,approx_bytes,wall_ms") - end - - for w <- workloads, i <- 1..samples do - run = fn -> run_workload(w, trees_k, pairs_n) end - {t0, _} = :timer.tc(run) - {rss, hwm} = read_proc_rss_hwm() - {nodes, pairs_c, approx} = sample_stats() - - if csv do - IO.puts( - "#{w},#{i},#{trees_k},#{pairs_n},#{rss},#{hwm},#{nodes},#{pairs_c},#{approx},#{div(t0, 1000)}" - ) - else - IO.inspect({w, i, rss, hwm, nodes, pairs_c, approx, div(t0, 1000)}, label: :sample) - end - end - - :ok - end - - defp dbg_malloc do - case CMerkleTree.malloc_info() do - :unsupported -> IO.puts("# malloc_info: unsupported") - bin when is_binary(bin) -> IO.puts(String.slice(bin, 0, 400)) - end - end - - defp read_proc_rss_hwm do - path = "/proc/#{System.pid()}/status" - - case File.read(path) do - {:ok, body} -> - rss = parse_kv(body, "VmRSS:") - hwm = parse_kv(body, "VmHWM:") - {rss, hwm} - - _ -> - {0, 0} - end - end - - defp parse_kv(body, key) do - case Regex.run(~r/#{Regex.escape(key)}\s+(\d+)\s+kB/i, body) do - [_, n] -> String.to_integer(n) - _ -> 0 - end - end - - defp sample_stats do - case Process.get(:cmerkle_mem_bench_tree) do - nil -> {0, 0, 0} - t -> CMerkleTree.memory_stats(t) - end - end - - defp test_pairs(n) do - Enum.map(1..n, fn idx -> - {String.pad_leading("#{idx}", 32), CMerkleTree.hash("#{idx}")} - end) - end - - # A: many clones share one SharedState — minimal duplication. - defp run_workload("a", trees_k, pairs_n) do - data = test_pairs(pairs_n) - base = CMerkleTree.from_list(data) |> CMerkleTree.lock() - - refs = - for _ <- 1..trees_k do - CMerkleTree.clone(base) - end - - Process.put(:cmerkle_mem_bench_tree, hd(refs)) - _ = refs - :erlang.garbage_collect() - end - - # B: one locked base, then `trees_k` times clone → insert one leaf (COW full copy per fork). All retained. - # The base handle is scoped so the original SharedState can be collected once forks have diverged. - defp run_workload("b", trees_k, pairs_n) do - data = test_pairs(pairs_n) - extra = CMerkleTree.hash("extra-leaf") - - forks = - (fn -> - base = CMerkleTree.from_list(data) |> CMerkleTree.lock() - - Enum.map(1..trees_k, fn _ -> - CMerkleTree.clone(base) |> CMerkleTree.insert(extra, extra) - end) - end).() - - Process.put(:cmerkle_mem_bench_tree, hd(forks)) - _ = forks - :erlang.garbage_collect() - end - - # C: `trees_k` independent trees (same keys, last value varies). All retained — no COW sharing. - defp run_workload("c", trees_k, pairs_n) do - data = test_pairs(pairs_n) - {k, _v} = List.last(data) - - trees = - Enum.map(1..trees_k, fn i -> - alt = CMerkleTree.hash("alt#{i}") - CMerkleTree.from_list(Enum.drop(data, -1) ++ [{k, alt}]) - end) - - Process.put(:cmerkle_mem_bench_tree, hd(trees)) - _ = trees - :erlang.garbage_collect() - end - - # D: churn — allocate and drop trees to stress PreAllocator malloc/free stripes. - defp run_workload("d", trees_k, pairs_n) do - data = test_pairs(pairs_n) - - last = - Enum.reduce(1..trees_k, nil, fn _, _acc -> - t = CMerkleTree.from_list(data) - _ = CMerkleTree.root_hash(t) - t - end) - - Process.put(:cmerkle_mem_bench_tree, last) - :erlang.garbage_collect() - end - - defp run_workload(other, _, _) do - raise ArgumentError, "unknown workload #{inspect(other)}, expected a|b|c|d|all" - end - - # `mix run scripts/foo.exs -- --trees 5` may pass ["scripts/foo.exs", "--", "--trees", "5"]. - defp normalize_argv(argv) do - argv - |> Enum.flat_map(fn - "--" -> [] - x -> [x] - end) - |> Enum.reject(&String.ends_with?(&1, ".exs")) - end -end - -CMerkleMemoryBench.main(System.argv()) diff --git a/scripts/cmerkle_memory_evaluation_report.md b/scripts/cmerkle_memory_evaluation_report.md deleted file mode 100644 index e317dcc..0000000 --- a/scripts/cmerkle_memory_evaluation_report.md +++ /dev/null @@ -1,132 +0,0 @@ -# CMerkleTree memory evaluation report - -This report summarizes runs of the tooling added for the memory-efficiency plan: Elixir RSS workloads (`scripts/cmerkle_memory_bench.exs`), native copy harness (`c_src/mem_harness.cpp`), stripe-size comparison (`scripts/compare_merkle_stripe_size.sh`), Valgrind Massif (optional), and NIF introspection (`CMerkleTree.struct_sizes/0`, `memory_stats/1`, `malloc_info/0`). - -**Environment:** Linux, local dev run (single machine, not isolated from other OS noise). VmRSS is process-wide (BEAM + NIF + libc); incremental deltas between workloads isolate **relative** C++ pressure but absolute numbers include a large Erlang baseline. - -**Fixes applied during evaluation:** `cmerkle_memory_bench.exs` now strips `mix run` argv noise (`*.exs`, `--`) so `--trees` / `--pairs` work; workloads **B** and **C** retain **all** forked/independent trees (not only the last survivor); workload **B** uses **one** locked base and repeated `clone → insert` inside a scoped function so the base handle is not pinned after forks diverge. Default `--trees` is **25** to keep RSS manageable when retaining full COW copies. - ---- - -## 1. Static layout (NIF `struct_sizes/0`) - -Measured C++ sizes (bytes): - -| Field / object | Size | -|------------------|-----:| -| `Item` | 728 | -| `pair_t` | 88 | -| `pair_list_t` | 152 | -| `Tree` (shell) | 136 | -| `MERKLE_STRIPE_SIZE` | 8 (default) | - -`memory_stats/1` approximates `nodes × sizeof(Item) + pairs × sizeof(pair_t)` (does **not** include `std::vector` key storage in `pair_t`). - -For a sample with **800 pairs** and **139** trie nodes: **≈171,592** bytes from that formula alone; keys add further heap not in this line. - ---- - -## 2. Workload RSS (Elixir, `pairs=800`, `trees=25`, 2 iterations) - -| Workload | Role | VmRSS (kb) typical | Notes | -|----------|------|-------------------:|-------| -| **A** | 25 `clone`s of one locked base (shared `SharedState`) | ~114,100–114,300 | Minimal duplication of C++ tree | -| **B** | 25 forks from one base: `clone` → insert one extra leaf (full COW copy each), all retained | ~119,400–119,700 | Near-duplicate trees (801 pairs each) | -| **C** | 25 independent trees (same keys, last value differs) | ~119,600–119,700 | Same order of retained data as B | -| **D** | Churn: build/`root_hash` 25×, keep last tree only | ~119,300–119,500 | Single tree retained; RSS still reflects allocator/BEAM high watermark | - -**Delta A → B (retained forks):** about **+5,350 kb VmRSS** for **25** forks → **~214 kb per fork** at the process level (includes libc/allocator; aligns in order-of-magnitude with the **~171 kb** structural `memory_stats` increment for one extra duplicated tree of this shape). - -**Interpretation:** Sharing handles (A) is far cheaper than retaining many COW-copied trees (B/C). Any optimization that avoids **full** `Tree` duplication on single-leaf updates (structural COW, persistent trie, or subtree interning) targets the gap between **A** and **B**. - ---- - -## 3. Native harness (`mem_harness`: full `Tree` copy after building N pairs) - -Runs **without** the BEAM; VmRSS read from `/proc/self/status` (small process). - -| N (pairs) | Nodes (base=copy) | VmRSS delta (copy) kb | -|----------:|------------------:|----------------------:| -| 500 | 91 | 268 | -| 1,500 | 267 | 532 | -| 2,000 | 349 | 656 | -| 3,000 | 537 | 932 | - -Copy cost scales roughly linearly with trie size (full duplicate of nodes + pairs). - ---- - -## 4. Stripe size (`MERKLE_STRIPE_SIZE` 8 vs 32 vs 64, `N=2000`) - -Native harness, same code path: - -| Stripe | VmRSS delta (kb) after copy | -|-------:|----------------------------:| -| 8 | 656 | -| 32 | 648 | -| 64 | 652 | - -**Finding:** **Neutral (N)** — changing stripe size did not materially change measured RSS in this scenario (<2% swing). Further tuning might help fragmentation under different churn, not shown here. - ---- - -## 5. Valgrind Massif (optional, `mem_harness` N=1500) - -Under Massif, peak **heap** for the small harness stayed on the order of **~1 MB** (chart peak ~910 KB; snapshot details mix libstdc++/program allocations). Use this for **malloc attribution** of the native binary, not for BEAM RSS. - ---- - -## 6. Candidate impact matrix (plan backlog) - -Ratings follow the plan: **S** strong (>10% on target workload where applicable), **M** moderate, **L** low, **N** neutral / not evidenced. - -| # | Candidate | Rating | Evidence from this evaluation | -|---|-----------|--------|------------------------------| -| 1 | Structural COW / path copying | **S** (potential) | A vs B: **~5.35 MB** extra for **25** retained forks; native copy delta grows with **N** (§3). | -| 2 | Intern / content-addressed subtrees | **M** (potential) | Not implemented; would attack duplicate subtrees across **different** trees — high value if large shared prefixes. | -| 3 | Split `Item` leaf vs internal | **M** | `Item` = **728 B** × **node_count** dominates `memory_stats` model; internal nodes still carry leaf-shaped fields today. | -| 4 | Lazy / heap `hash_values[16]` | **M** | **512 B** of **728 B** per `Item` is hashes; savings scale with **node_count** on cold paths. | -| 5 | Smaller / variable `pair_list_t` | **L** | Fixed **152 B** per node + pointers; helps most when many sparse buckets. | -| 6 | Larger stripes / mmap arenas | **N** | §4: **8/32/64** stripes ~same RSS on harness. | -| 7 | Cross-tree allocator pools | **M** (churn) | Not isolated; D suggests allocator/BEAM retention can linger; worth profiling under destroy-heavy workloads. | -| 8 | Key interning (`bin_t`) | **M**–**L** | Not measured; **M** if keys repeat across trees, **L** if mostly unique 32-byte keys. | -| 9 | Prefix compression (`bits_t`) | **M** | Depth-dependent; not separately measured. | -| 10 | Drop redundant `key_hash` | **L** | **32 B × pair_count**; trades CPU vs RAM. | -| 11 | Struct packing / field order | **L** | Minor vs **728 B** `Item`. | -| 12 | Proof object pooling | **L** | Transient `unique_ptr` proof trees; affects peak under proof-heavy load, not steady RSS here. | -| 13 | Merge/collapse (`destroy_item` on shrink) | **L**–**N** | Code path already calls `destroy_item` when collapsing; revisit if profiling shows stranded nodes. | -| 14 | mmap bulk import arena | **M** (bulk) | Relevant for `import_map`-style loads; not covered by these micro-benches. | -| 15 | Tune `LEAF_SIZE` (arity) | **M** (workload) | Trade depth vs per-node width; needs separate sweep. | -| 16 | Erlang: avoid accidental unsharing | **S** (usage) | **A**: **25** clones of one locked tree share one `SharedState`; **B**: **25** retained forks each hold a full COW copy (~**+5.35 MB** VmRSS vs **A** in §2). | - ---- - -## 7. Recommendations (priority) - -1. **Product/API:** Prefer **shared** `SharedState` (clones, batched writes) until a mutation is required — largest win without C++ changes (aligns with **A vs B**). -2. **C++:** Prioritize **structural sharing** on `make_writeable` (path copy or interned DAG) for workloads like **B**. -3. **Quick wins:** **Lazy `hash_values`** and **leaf/internal `Item` split** — justified by §1 size breakdown; validate with `memory_stats` + RSS after each change. -4. **Deprioritize for RSS:** Stripe-only tuning (**§4**), unless Massif shows allocator hotspots in long-running churn tests. - ---- - -## 8. Commands reference - -```bash -# Elixir RSS CSV (default trees=25) -MIX_ENV=test mix run --no-start scripts/cmerkle_memory_bench.exs -- --workload all --samples 3 - -# Native copy cost -make -C c_src mem_harness -./c_src/mem_harness.bin 2000 - -# Stripe comparison -./scripts/compare_merkle_stripe_size.sh 2000 - -# Massif (optional) -./scripts/profile_cmerkle_massif.sh 1500 -``` - ---- - -*Generated from automated runs on the evaluation host; re-run before release to capture your environment.* diff --git a/scripts/cmerkle_parallel_stress.exs b/scripts/cmerkle_parallel_stress.exs index 48f7787..eb0095f 100644 --- a/scripts/cmerkle_parallel_stress.exs +++ b/scripts/cmerkle_parallel_stress.exs @@ -1,23 +1,10 @@ -# CMerkleTree — parallel / concurrency stress (targets live-only failures). +# CAccountMap / Chain.State — parallel concurrency stress. # -# Erlang schedules NIF calls on many schedulers; the NIF uses: -# - One ErlNifMutex per SharedState (every insert/get/root_hash/difference arm takes Lock(mt)) -# - std::mutex on GlobalStripePool (preallocator.hpp) -# - A global mutex on LockedStates (enter_lock / leave_lock / resource destructor) +# Hammers account-map NIFs from many Tasks: map mutexes, storage COW, +# difference_full dual-lock order, compact/uncompact, and State.lock/clone. # -# These workloads intentionally hammer the same resource from many Tasks to flush: -# mutex omissions, refcount/has_clone races, deadlock in difference lock ordering, -# and pool stripe contention. -# -# Confirmed bug class (parallel clone + insert): COW splits a SharedState so two Trees -# share one ItemPool and one PreAllocator while each Erlang resource still uses its own -# ErlNifMutex — concurrent refcnt / pair slab mutation raced. Fixed in C++ with a -# std::recursive_mutex on ItemPool (see item_pool.cpp) and on PreAllocator -# (preallocator.hpp). Run this script after changes to native trie code. -# -# Thread Sanitizer (not ASan): to hunt data races in C++, rebuild the NIF with -# -fsanitize=thread -# and run this script the same way as ASan (swap priv/merkletree_nif.so). TSan + BEAM can be noisy. +# Thread Sanitizer (not ASan): rebuild the NIF with -fsanitize=thread and run +# this script the same way as ASan (swap priv/merkletree_nif.so). TSan + BEAM can be noisy. # # Run: # mix run --no-start scripts/cmerkle_parallel_stress.exs -- --waves 3 --tasks 48 @@ -27,8 +14,8 @@ # --waves N repeat full suite this many times (default: 2) # --ops N operations per worker in same-tree scenarios (default: 40) # --seed N RNG seed -# --accounts N account count for P12/P14 (default: 200) -# --scenario PX run only scenario PX (e.g. P10); repeat for multiple +# --accounts N account count for map scenarios (default: 200) +# --scenario PX run only scenario PX (e.g. P12); repeat for multiple # # Exit 0 if all waves complete. Native crashes are not caught in Elixir. @@ -39,39 +26,18 @@ defmodule CMerkleParallelStress do # --- Theory map (for triage notes) --- # - # P1 Same CMerkleTree ref, concurrent insert/get/root_hash/root_hashes/size — - # relies on per-SharedState mutex; would expose missing Lock() on any NIF path. - # - # P2 Concurrent difference(ta, tb) identical pair — lock order is by SharedState - # address (nif.cpp); should not deadlock; stresses dual-lock hold time. - # - # P3 Concurrent difference(ta, tb) vs difference(tb, ta) argument order — - # same ordering rule; overlapping waves stress scheduler interleaving. - # - # P4 Clone storm: many workers clone same base then mutate (has_clone / make_writeable). - # - # P5 Interleaved: half workers mutate T, half run difference(T, U) — contention on first tree. - # - # P6 locked_states: concurrent lock/1 on same locked root_hash dedup path (enter_lock). - # - # P7 Many disposable trees in parallel — GlobalStripePool take/put under contention. - # - # P8 get_proofs + to_list concurrently on same tree — read-heavy + each() traversal. - # - # P9 Concurrent lock/1 (enter_lock dedup) + difference/2 on overlapping trees — - # reproduces production deadlock: locked_states_mutex held while waiting on a - # second tree mutex vs difference_raw dual-lock ordering. - # - # P10 leave_lock storm + lock + difference + three-tree overlap (D-A4, D-B2, D-B5) - # P11 mass disposable trees + GC while difference/lock (D-A2, D-A5) - # P12 account_map uncompact_state + storage difference workers (D-C2, D-D3) - # P13 account_map clone/put/delete + storage difference (D-C1, D-C4) + # P12 account_map uncompact_state + difference_full workers (D-C2, D-D3) + # P13 account_map clone/put/delete + storage APIs (D-C1, D-C4) # P14 Chain.State difference + lock + uncompact + lock→clone→write composite (D-D1–D-D4) - # P15 dirty-scheduler saturation: lock, difference, uncompact, clone, get_proofs + # P15 dirty-scheduler saturation: lock, difference_full, uncompact, storage # P16 concurrent State.lock on compact states (block-sync memory path) + # P17 prepare_state pipeline (native diff + lock + to_list) + # P18 writer sim / P18L clone eth_call + # P19 large map small delta + # P20 dirty saturation + difference_full # - @all_scenarios ~w(P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 P16 P17 P18 P19 P20) + @all_scenarios ~w(P12 P13 P14 P15 P16 P17 P18 P18L P19 P20) def normalize_argv(argv) do case argv do @@ -117,7 +83,7 @@ defmodule CMerkleParallelStress do :rand.seed(:exsss, {seed, seed, seed}) IO.puts(:stderr, """ - === CMerkleTree parallel stress === + === CAccountMap parallel stress === otp=#{System.otp_release()} elixir=#{System.version()} schedulers=#{System.schedulers_online()} seed=#{seed} tasks=#{tasks} waves=#{waves} ops_per_task=#{ops} accounts=#{accounts} scenarios=#{Enum.join(scenarios, ",")} @@ -143,39 +109,6 @@ defmodule CMerkleParallelStress do defp run_scenario(name, ctx) do case name do - "P1" -> - run_named("P1_same_tree_rw", fn -> p1_same_tree_rw(ctx) end) - - "P2" -> - run_named("P2_concurrent_difference_ab", fn -> p2_concurrent_difference(ctx, :ab) end) - - "P3" -> - run_named("P3_concurrent_difference_ba", fn -> p2_concurrent_difference(ctx, :ba) end) - - "P4" -> - run_named("P4_clone_write_storm", fn -> p4_clone_storm(ctx) end) - - "P5" -> - run_named("P5_interleave_mutate_and_diff", fn -> p5_interleave(ctx) end) - - "P6" -> - run_named("P6_concurrent_lock", fn -> p6_concurrent_lock(ctx) end) - - "P7" -> - run_named("P7_many_shortlived_trees", fn -> p7_shortlived_parallel(ctx) end) - - "P8" -> - run_named("P8_proofs_and_to_list", fn -> p8_proofs_to_list(ctx) end) - - "P9" -> - run_named("P9_lock_and_difference", fn -> p9_lock_and_difference(ctx) end) - - "P10" -> - run_named("P10_leave_lock_storm", fn -> p10_leave_lock_storm(ctx) end) - - "P11" -> - run_named("P11_gc_disposable_trees", fn -> p11_gc_disposable_trees(ctx) end) - "P12" -> run_named("P12_uncompact_and_diff", fn -> p12_uncompact_and_diff(ctx) end) @@ -225,289 +158,20 @@ defmodule CMerkleParallelStress do end end - defp p1_same_tree_rw(%{tasks: tasks, ops: ops}) do - seed_items = - Enum.map(1..50, fn i -> - {String.pad_leading("p1#{i}", 32), CMerkleTree.hash("seed#{i}")} - end) - - tree = CMerkleTree.new() |> CMerkleTree.insert_items(seed_items) - - 1..tasks - |> Task.async_stream( - fn w -> - Enum.each(1..ops, fn j -> - k = rem(w * 997 + j * 31, 400) - key = String.pad_leading("pk#{k}", 32) - tree = CMerkleTree.insert(tree, key, CMerkleTree.hash("v#{w}#{j}")) - _ = CMerkleTree.get(tree, key) - _ = CMerkleTree.root_hash(tree) - _ = CMerkleTree.size(tree) - _ = CMerkleTree.root_hashes(tree) - end) - - :ok - end, - max_concurrency: tasks, - timeout: :infinity, - ordered: false - ) - |> Stream.run() - end - - defp p2_concurrent_difference(%{tasks: tasks}, order) do - a = - Enum.map(1..120, fn i -> - {String.pad_leading("da#{i}", 32), CMerkleTree.hash("da#{i}")} - end) - - b = - Enum.map(1..120, fn i -> - {String.pad_leading("db#{i}", 32), CMerkleTree.hash("db#{i}")} - end) - - ta = CMerkleTree.new() |> CMerkleTree.insert_items(a) - tb = CMerkleTree.new() |> CMerkleTree.insert_items(b) - - 1..tasks - |> Task.async_stream( - fn _ -> - _ = - case order do - :ab -> CMerkleTree.difference(ta, tb) - :ba -> CMerkleTree.difference(tb, ta) - end - - :ok - end, - max_concurrency: tasks, - timeout: :infinity, - ordered: false - ) - |> Stream.run() - end - - defp p4_clone_storm(%{tasks: tasks, ops: ops}) do - base = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..80, fn i -> {String.pad_leading("b#{i}", 32), CMerkleTree.hash("b#{i}")} end) - ) - - 1..tasks - |> Task.async_stream( - fn w -> - Enum.reduce(1..ops, CMerkleTree.clone(base), fn j, acc -> - CMerkleTree.insert( - acc, - String.pad_leading("c#{w}_#{j}", 32), - CMerkleTree.hash("x#{w}#{j}") - ) - end) - - :ok - end, - max_concurrency: tasks, - timeout: :infinity, - ordered: false - ) - |> Stream.run() - end - - defp p5_interleave(%{tasks: tasks, ops: ops}) do - ta = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..100, fn i -> - {String.pad_leading("ia#{i}", 32), CMerkleTree.hash("ia#{i}")} - end) - ) - - tb = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..100, fn i -> - {String.pad_leading("ib#{i}", 32), CMerkleTree.hash("ib#{i}")} - end) - ) - - mutators = div(tasks, 2) |> max(1) - differ = tasks - mutators - - step = fn tag -> - Task.async_stream( - 1..mutators, - fn w -> - Enum.each(1..ops, fn j -> - k = String.pad_leading("m#{tag}#{w}_#{j}", 32) - _ = CMerkleTree.insert(ta, k, CMerkleTree.hash("mut#{w}#{j}")) - end) - - :ok - end, - max_concurrency: mutators, - timeout: :infinity, - ordered: false - ) - |> Stream.run() - end - - diff_f = fn -> - Task.async_stream( - 1..differ, - fn _ -> - _ = CMerkleTree.difference(ta, tb) - :ok - end, - max_concurrency: differ, - timeout: :infinity, - ordered: false - ) - |> Stream.run() - end - - # Interleave: waves of concurrent diff with concurrent mutators - step.(:a) - diff_f.() - step.(:b) - diff_f.() - end - - defp p6_concurrent_lock(%{tasks: tasks}) do - data = - Enum.map(1..60, fn i -> {String.pad_leading("L#{i}", 32), CMerkleTree.hash("L#{i}")} end) - - base = CMerkleTree.new() |> CMerkleTree.insert_items(data) - - 1..tasks - |> Task.async_stream( - fn _ -> - _ = - base - |> CMerkleTree.clone() - |> CMerkleTree.lock() - - :ok - end, - max_concurrency: tasks, - timeout: :infinity, - ordered: false - ) - |> Stream.run() - end - - defp p9_lock_and_difference(%{tasks: tasks, ops: ops}) do - shared = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..100, fn i -> {String.pad_leading("s#{i}", 32), CMerkleTree.hash("s#{i}")} end) - ) - - other = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..100, fn i -> {String.pad_leading("o#{i}", 32), CMerkleTree.hash("o#{i}")} end) - ) - - lockers = div(tasks, 3) |> max(1) - differ = tasks - lockers - - Task.async_stream( - 1..lockers, - fn _ -> - _ = - shared - |> CMerkleTree.clone() - |> CMerkleTree.lock() - - :ok - end, - max_concurrency: lockers, - timeout: :infinity, - ordered: false - ) - |> Stream.run() - - Task.async_stream( - 1..differ, - fn w -> - Enum.each(1..ops, fn j -> - k = String.pad_leading("p9#{w}_#{j}", 32) - _ = CMerkleTree.insert(shared, k, CMerkleTree.hash("p9#{w}#{j}")) - _ = CMerkleTree.difference(shared, other) - _ = CMerkleTree.difference(other, shared) - end) - - :ok - end, - max_concurrency: differ, - timeout: :infinity, - ordered: false - ) - |> Stream.run() - end - - defp p7_shortlived_parallel(%{tasks: tasks}) do - 1..tasks - |> Task.async_stream( - fn w -> - items = - Enum.map(1..25, fn j -> - {String.pad_leading("s#{w}_#{j}", 32), CMerkleTree.hash("s#{w}_#{j}")} - end) - - t = CMerkleTree.new() |> CMerkleTree.insert_items(items) - _ = CMerkleTree.root_hash(t) - _ = CMerkleTree.bucket_count(t) - :ok - end, - max_concurrency: tasks, - timeout: :infinity, - ordered: false - ) - |> Stream.run() - end - - defp p8_proofs_to_list(%{tasks: tasks, ops: ops}) do - keys = Enum.map(1..80, fn i -> String.pad_leading("h#{i}", 32) end) - - tree = - Enum.reduce(keys, CMerkleTree.new(), fn k, acc -> - CMerkleTree.insert(acc, k, CMerkleTree.hash(k)) - end) - - 1..tasks - |> Task.async_stream( - fn w -> - Enum.each(1..ops, fn j -> - k = Enum.at(keys, rem(w + j, length(keys))) - _ = CMerkleTree.get_proofs(tree, k) - _ = CMerkleTree.to_list(tree) - _ = CMerkleTree.root_hash(tree) - end) - - :ok - end, - max_concurrency: tasks, - timeout: :infinity, - ordered: false - ) - |> Stream.run() - end - defp addr(i), do: <> defp slot(i), do: <> + defp storage_list(i, mult \\ 5) do + [{slot(i), <>}] + end + defp build_compact_accounts(n) do Enum.reduce(1..n, State.new(), fn i, st -> - tree = - CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - acc = %Account{ nonce: i, balance: i * 1_000, - storage_root: tree, + storage_root: storage_list(i), code: <>, map_backed: false } @@ -520,139 +184,21 @@ defmodule CMerkleParallelStress do defp build_live_state(n) do Enum.reduce(1..n, State.new(), fn i, st -> - tree = - CMerkleTree.insert_items(CMerkleTree.new(), [ + acc = %Account{ + nonce: i, + balance: i * 1_000, + storage_root: [ {slot(i), <>}, {slot(i + 10_000), <>} - ]) + ], + code: <>, + map_backed: false + } - acc = %Account{nonce: i, balance: i * 1_000, storage_root: tree, code: <>} State.set_account(st, addr(i), acc) end) end - defp p10_leave_lock_storm(%{tasks: tasks, ops: ops}) do - shared = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..100, fn i -> - {String.pad_leading("p10s#{i}", 32), CMerkleTree.hash("p10s#{i}")} - end) - ) - - tb = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..100, fn i -> - {String.pad_leading("p10b#{i}", 32), CMerkleTree.hash("p10b#{i}")} - end) - ) - - tc = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..80, fn i -> - {String.pad_leading("p10c#{i}", 32), CMerkleTree.hash("p10c#{i}")} - end) - ) - - third = div(tasks, 4) |> max(1) - rest = tasks - third - - 1..third - |> Task.async_stream( - fn w -> - items = - Enum.map(1..20, fn j -> - {String.pad_leading("d#{w}_#{j}", 32), CMerkleTree.hash("d#{w}_#{j}")} - end) - - _ = CMerkleTree.new() |> CMerkleTree.insert_items(items) - :ok - end, - max_concurrency: third, - timeout: :infinity, - ordered: false - ) - |> Stream.run() - - 1..rest - |> Task.async_stream( - fn w -> - Enum.each(1..ops, fn j -> - if rem(w + j, 4) == 0 do - _ = - shared - |> CMerkleTree.clone() - |> CMerkleTree.lock() - else - _ = CMerkleTree.difference(shared, tb) - _ = CMerkleTree.difference(shared, tc) - _ = CMerkleTree.difference(tb, tc) - end - end) - - :ok - end, - max_concurrency: rest, - timeout: :infinity, - ordered: false - ) - |> Stream.run() - - :erlang.garbage_collect() - end - - defp p11_gc_disposable_trees(%{tasks: tasks, ops: ops}) do - anchor = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..60, fn i -> - {String.pad_leading("gca#{i}", 32), CMerkleTree.hash("gca#{i}")} - end) - ) - - other = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..60, fn i -> - {String.pad_leading("gco#{i}", 32), CMerkleTree.hash("gco#{i}")} - end) - ) - - 1..tasks - |> Task.async_stream( - fn w -> - Enum.each(1..ops, fn j -> - disposable = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..15, fn k -> - {String.pad_leading("t#{w}_#{j}_#{k}", 32), CMerkleTree.hash("t#{w}#{j}#{k}")} - end) - ) - - _ = CMerkleTree.root_hash(disposable) - _ = CMerkleTree.difference(anchor, other) - - if rem(j, 5) == 0 do - _ = - anchor - |> CMerkleTree.clone() - |> CMerkleTree.lock() - end - end) - - if rem(w, 4) == 0, do: :erlang.garbage_collect() - :ok - end, - max_concurrency: tasks, - timeout: :infinity, - ordered: false - ) - |> Stream.run() - end - defp p12_uncompact_and_diff(%{tasks: tasks, accounts: n}) do compact = build_compact_accounts(n) workers = div(tasks, 2) |> max(1) @@ -677,18 +223,25 @@ defmodule CMerkleParallelStress do 1..workers, fn i -> a = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(i), <>}, - {slot(i + 5000), <>} - ]) + Enum.reduce(1..(8 + rem(i, 5)), CAccountMap.new(), fn j, acc -> + CAccountMap.put( + acc, + addr(j), + j, + j * 100, + [{slot(j), <>}], + <> + ) + end) b = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(i), <>}, - {slot(i + 9000), <>} - ]) + a + |> CAccountMap.clone() + |> CAccountMap.storage_put_map(%{ + addr(1) => %{slot(i + 9000) => <>} + }) - CMerkleTree.difference(a, b) + CAccountMap.difference_full(a, b) end, max_concurrency: workers, timeout: :infinity, @@ -714,10 +267,7 @@ defmodule CMerkleParallelStress do case rem(w + j, 5) do 0 -> fork = CAccountMap.clone(base) - - new_storage = - CMerkleTree.insert(CMerkleTree.new(), slot(w + j), <>) - + new_storage = [{slot(w + j), <>}] _ = CAccountMap.put(fork, addr(rem(w, n) + 1), j, j * 100, new_storage, <>) 1 -> @@ -821,31 +371,20 @@ defmodule CMerkleParallelStress do State.set_account(st, addr(1), %Account{ nonce: 999, balance: 999, - storage_root: CMerkleTree.new(), - code: <<>> + storage_root: [], + code: <<>>, + map_backed: false }) end) - keys = - Enum.map(1..60, fn i -> String.pad_leading("ds#{i}", 32) end) - - proof_tree = - Enum.reduce(keys, CMerkleTree.new(), fn k, acc -> - CMerkleTree.insert(acc, k, CMerkleTree.hash(k)) - end) - 1..tasks |> Task.async_stream( fn w -> - case rem(w, 6) do + case rem(w, 5) do 0 -> - _ = - proof_tree - |> CMerkleTree.clone() - |> CMerkleTree.lock() + _ = CAccountMap.lock(live.accounts) 1 -> - # Compare storage root hashes / lists (get no longer returns live tries). _ = CAccountMap.storage_root_hash(live.accounts, addr(1)) _ = CAccountMap.storage_root_hash(other.accounts, addr(1)) _ = CAccountMap.storage_to_list(live.accounts, addr(1)) @@ -854,18 +393,10 @@ defmodule CMerkleParallelStress do _ = CAccountMap.uncompact_state(compact) 3 -> - k = Enum.at(keys, rem(w, length(keys))) - _ = CMerkleTree.get_proofs(proof_tree, k) - _ = CMerkleTree.to_list(proof_tree) - - 4 -> - _ = - proof_tree - |> CMerkleTree.clone() - |> CMerkleTree.insert(String.pad_leading("x#{w}", 32), CMerkleTree.hash("x#{w}")) + _ = CAccountMap.difference_full(live.accounts, other.accounts) _ -> - _ = CAccountMap.lock(live.accounts) + _ = CAccountMap.to_list(live.accounts) end :ok @@ -1066,8 +597,9 @@ defmodule CMerkleParallelStress do State.set_account(st, addr(1), %Account{ nonce: 999, balance: 999, - storage_root: CMerkleTree.new(), - code: <<>> + storage_root: [], + code: <<>>, + map_backed: false }) end) diff --git a/scripts/merkle_asan_and_recovery.md b/scripts/merkle_asan_and_recovery.md index c5234ed..14394cc 100644 --- a/scripts/merkle_asan_and_recovery.md +++ b/scripts/merkle_asan_and_recovery.md @@ -36,7 +36,7 @@ export LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libasan.so.8 ## Recovering a node after bad deltas / malloc corruption -Symptoms: `apply_difference` raises on `^a = CMerkleTree.get(tree, key)` (delta old value does not match parent trie), or native heap errors in the NIF. +Symptoms: `apply_difference` raises when a delta old value does not match the parent storage trie, or native heap errors in the NIF. 1. **Stop** the node. 2. **Find the last good block** (highest block number that still loads `Model.ChainSql.state/1` successfully, or the parent hash of the first failing block). Use logs, a backup, or binary search over block numbers. diff --git a/scripts/merkle_bench.exs b/scripts/merkle_bench.exs deleted file mode 100644 index e7195af..0000000 --- a/scripts/merkle_bench.exs +++ /dev/null @@ -1,27 +0,0 @@ -# size = 10_000 -# ref = Base16.decode("0x96c2b5d03aa8e52230e74d9a08359c38c7608e5d9aba18773f3c90baf3806ccb") - -size = 100_000 -ref = Base16.decode("0x3b99d56aa16278d50e85a8ea0939e62180c4f0305b773775f1844c3303a41897") - -test_data = - Enum.map(1..size, fn idx -> - hash = Diode.hash("!#{idx}!") - {hash, hash} - end) - -for i <- 1..3 do - time_ms = :timer.tc(fn -> - for _ <- 1..100 do - tree = HeapMerkleTree.insert_items(HeapMerkleTree.new(), test_data) - - if HeapMerkleTree.root_hash(tree) != ref do - raise("Error #{Base16.encode(HeapMerkleTree.root_hash(tree))} != #{Base16.encode(ref)}") - end - end - end) - |> elem(0) - |> div(1000) - - IO.puts("Run #{i} #{time_ms}ms") -end diff --git a/scripts/profile_uncompact_nif.exs b/scripts/profile_uncompact_nif.exs index 45cac41..254dd09 100644 --- a/scripts/profile_uncompact_nif.exs +++ b/scripts/profile_uncompact_nif.exs @@ -1,8 +1,8 @@ # NIF-only uncompact for Valgrind profiling (same 14k fixture as chain_state_uncompact_test). -# No full app.start — only loads the NIF via CMerkleTree.new/0. +# No full app.start — only loads the NIF via CAccountMap.new/0. alias Chain.State -_ = CMerkleTree.new() +_ = CAccountMap.new() account_count = String.to_integer(System.get_env("UNCOMPACT_PROFILE_COUNT", "14609")) fixture = Path.join(System.tmp_dir!(), "diode_uncompact_perf_14609.bin") diff --git a/test/caccount_map_lifetime_test.exs b/test/caccount_map_lifetime_test.exs index aa7ebd5..217a94c 100644 --- a/test/caccount_map_lifetime_test.exs +++ b/test/caccount_map_lifetime_test.exs @@ -33,12 +33,12 @@ defmodule CAccountMapLifetimeTest do end defp sample_account(n) do - tree = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(n), val(n)} - ]) - - %Account{nonce: n, balance: n * 1_000, storage_root: tree, code: <>} + %Account{ + nonce: n, + balance: n * 1_000, + storage_root: [{slot(n), val(n)}], + code: <> + } end defp put_sample(map, i) do @@ -137,11 +137,7 @@ defmodule CAccountMapLifetimeTest do describe "shared storage pointers across accounts" do test "two accounts referencing the same storage trie survive clone GC" do - storage = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(1), val(1)}, - {slot(2), val(2)} - ]) + storage = [{slot(1), val(1)}, {slot(2), val(2)}] map = CAccountMap.new() @@ -165,7 +161,7 @@ defmodule CAccountMapLifetimeTest do end test "shared storage with fork mutation splits only the mutated branch" do - storage = CMerkleTree.insert_items(CMerkleTree.new(), [{slot(1), val(1)}]) + storage = [{slot(1), val(1)}] map = CAccountMap.new() @@ -215,7 +211,7 @@ defmodule CAccountMapLifetimeTest do base = put_sample(CAccountMap.new(), 3) {_, _, old_root, _} = CAccountMap.get(base, addr(3)) - new_storage = CMerkleTree.insert_items(CMerkleTree.new(), [{slot(7), val(7)}]) + new_storage = [{slot(7), val(7)}] base = CAccountMap.put( @@ -459,11 +455,7 @@ defmodule CAccountMapLifetimeTest do describe "shared storage trie dedup in account_map_clone" do test "lock then clone with multiple accounts sharing one storage trie stays writable" do - shared = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(1), val(1)}, - {slot(2), val(2)} - ]) + shared = [{slot(1), val(1)}, {slot(2), val(2)}] accounts = CAccountMap.new() diff --git a/test/caccount_map_test.exs b/test/caccount_map_test.exs index 12ea4d3..6730449 100644 --- a/test/caccount_map_test.exs +++ b/test/caccount_map_test.exs @@ -9,12 +9,12 @@ defmodule CAccountMapTest do defp addr(i), do: <> defp sample_account(n) do - tree = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {<>, <>} - ]) - - %Account{nonce: n, balance: n * 1_000, storage_root: tree, code: <>} + %Account{ + nonce: n, + balance: n * 1_000, + storage_root: [{<>, <>}], + code: <> + } end defp put_sample(map, i) do @@ -27,8 +27,7 @@ defmodule CAccountMapTest do assert CAccountMap.size(map) == 1 assert {3, 3000, root, <<3>>} = CAccountMap.get(map, addr(3)) assert is_binary(root) and byte_size(root) == 32 - assert root == CMerkleTree.root_hash(sample_account(3).storage_root) - assert CAccountMap.storage_root_hash(map, addr(3)) == root + assert root == CAccountMap.storage_root_hash(map, addr(3)) end test "delete removes account" do @@ -39,10 +38,7 @@ defmodule CAccountMapTest do end test "lock via NIF freezes map for fork" do - shared = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {<<1::unsigned-size(256)>>, <<2::unsigned-size(256)>>} - ]) + shared = [{<<1::unsigned-size(256)>>, <<2::unsigned-size(256)>>}] base = CAccountMap.new() @@ -55,7 +51,7 @@ defmodule CAccountMapTest do fork = fork - |> CAccountMap.put(addr(1), 11, 11_000, CMerkleTree.new(), <<11>>) + |> CAccountMap.put(addr(1), 11, 11_000, [], <<11>>) assert {11, 11_000, _, <<11>>} = CAccountMap.get(fork, addr(1)) assert {1, 1_000, _, <<1>>} = CAccountMap.get(base, addr(1)) @@ -90,8 +86,7 @@ defmodule CAccountMapTest do test "large balance roundtrip via 256-bit encoding" do balance = Bitwise.bsl(1, 200) - storage = CMerkleTree.new() - map = CAccountMap.put(CAccountMap.new(), addr(2), 0, balance, storage, nil) + map = CAccountMap.put(CAccountMap.new(), addr(2), 0, balance, nil, nil) assert {0, ^balance, root, ""} = CAccountMap.get(map, addr(2)) assert is_binary(root) and byte_size(root) == 32 end diff --git a/test/chain_account_hash_nif_test.exs b/test/chain_account_hash_nif_test.exs index 911a242..d9d0f75 100644 --- a/test/chain_account_hash_nif_test.exs +++ b/test/chain_account_hash_nif_test.exs @@ -15,12 +15,13 @@ defmodule ChainAccountHashNifTest do defp val(i), do: <> defp sample_account(i) do - tree = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(i), val(i)} - ]) - - %Account{nonce: i, balance: i * 1_000, storage_root: tree, code: <>, map_backed: false} + %Account{ + nonce: i, + balance: i * 1_000, + storage_root: [{slot(i), val(i)}], + code: <>, + map_backed: false + } end defp live_state(accounts) when is_list(accounts) do @@ -33,10 +34,15 @@ defmodule ChainAccountHashNifTest do live_state(accounts) |> State.compact() |> Map.fetch!(:accounts) end - defp hash_from_live(%Account{storage_root: tree} = acc) when is_reference(tree) do + defp hash_from_list_storage(%Account{storage_root: storage} = acc) when is_list(storage) do + root = + CAccountMap.new() + |> CAccountMap.put(addr(0), acc.nonce, acc.balance, storage, acc.code) + |> CAccountMap.storage_root_hash(addr(0)) + Account.hash(%{ acc - | root_hash: CMerkleTree.root_hash(tree), + | root_hash: root, storage_root: nil, map_backed: true }) @@ -51,12 +57,11 @@ defmodule ChainAccountHashNifTest do %Account{ nonce: 11, balance: 5_000, - storage_root: - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(1), val(1)}, - {slot(2), val(2)}, - {slot(10), val(10)} - ]), + storage_root: [ + {slot(1), val(1)}, + {slot(2), val(2)}, + {slot(10), val(10)} + ], code: :binary.copy(<<0xCD>>, 1024), map_backed: false } @@ -104,14 +109,7 @@ defmodule ChainAccountHashNifTest do |> Map.new() assert nif_hashes == elixir_hashes - - elixir_root = - elixir_hashes - |> CMerkleTree.from_map() - |> CMerkleTree.root_hash() - - assert hash == elixir_root - assert CAccountMap.root_hash(accounts) == elixir_root + assert hash == CAccountMap.root_hash(accounts) end test "uncompact_state on CAccountMap resource matches Account.hash/1" do @@ -169,12 +167,11 @@ defmodule ChainAccountHashNifTest do test "uncompact_state falls back without compact root_hash field" do acc = sample_account(1) - tree = acc.storage_root legacy_account = %Chain.Account{ nonce: acc.nonce, balance: acc.balance, - storage_root: {MapMerkleTree, [], Map.new(CMerkleTree.to_list(tree))}, + storage_root: {MapMerkleTree, [], Map.new(acc.storage_root)}, code: acc.code, map_backed: false, root_hash: nil @@ -184,44 +181,42 @@ defmodule ChainAccountHashNifTest do {accounts, hash} = CAccountMap.uncompact_state(legacy_compact) - expected_hash = hash_from_live(acc) + {nonce, balance, storage, code} = CAccountMap.get(accounts, addr(1)) - expected_root = - %{addr(1) => expected_hash} - |> CMerkleTree.from_map() - |> CMerkleTree.root_hash() + assert Account.hash(Account.from_parts(nonce, balance, storage, code)) == + hash_from_list_storage(acc) - assert hash == expected_root - assert CAccountMap.root_hash(accounts) == expected_root + assert hash == CAccountMap.root_hash(accounts) assert CAccountMap.size(accounts) == 1 end test "uncompact_state falls back without compact code_hash field" do acc = sample_account(1) - tree = acc.storage_root + + root = + CAccountMap.new() + |> CAccountMap.put(addr(0), acc.nonce, acc.balance, acc.storage_root, acc.code) + |> CAccountMap.storage_root_hash(addr(0)) legacy_account = %Chain.Account{ nonce: acc.nonce, balance: acc.balance, - storage_root: {MapMerkleTree, [], Map.new(CMerkleTree.to_list(tree))}, + storage_root: {MapMerkleTree, [], Map.new(acc.storage_root)}, code: acc.code, map_backed: false, - root_hash: CMerkleTree.root_hash(tree) + root_hash: root } legacy_compact = %{addr(1) => legacy_account} {accounts, hash} = CAccountMap.uncompact_state(legacy_compact) - expected_hash = hash_from_compact(legacy_account) + {nonce, balance, storage, code} = CAccountMap.get(accounts, addr(1)) - expected_root = - %{addr(1) => expected_hash} - |> CMerkleTree.from_map() - |> CMerkleTree.root_hash() + assert Account.hash(Account.from_parts(nonce, balance, storage, code)) == + hash_from_compact(legacy_account) - assert hash == expected_root - assert CAccountMap.root_hash(accounts) == expected_root + assert hash == CAccountMap.root_hash(accounts) assert CAccountMap.size(accounts) == 1 end diff --git a/test/chain_state_merkle_test.exs b/test/chain_state_merkle_test.exs index d17a021..4112f7c 100644 --- a/test/chain_state_merkle_test.exs +++ b/test/chain_state_merkle_test.exs @@ -5,8 +5,8 @@ # Regression tests: Chain.State.difference / apply_difference must round-trip # (matches persisted block delta replay in Model.ChainSql.do_state/1). # -# Heavy cases mirror production: Evm.process_updates uses 32-byte key/value maps and -# CMerkleTree.insert_items/2; State.clone/1 shares trie pools (COW); +# Heavy cases mirror production: Evm.process_updates uses 32-byte key/value maps; +# State.clone/1 shares trie pools (COW); # sequential <> slots maximize shared key prefixes (trie depth). defmodule ChainStateMerkleTest do # NIF uses process-global allocators / stripe pools; run sequentially to avoid cross-test races. @@ -45,8 +45,7 @@ defmodule ChainStateMerkleTest do end defp account_from_evm_map(kvs) when is_map(kvs) do - tree = CMerkleTree.insert_items(CMerkleTree.new(), Map.to_list(kvs)) - %{Account.new() | storage_root: tree, map_backed: false, root_hash: nil} + %{Account.new() | storage_root: Map.to_list(kvs), map_backed: false, root_hash: nil} end defp account_with_storage(pairs) do @@ -54,12 +53,7 @@ defmodule ChainStateMerkleTest do end defp account_with_storage(%Account{} = acc, pairs) do - tree = - Enum.reduce(pairs, CMerkleTree.new(), fn {k, v}, t -> - CMerkleTree.insert(t, k, v) - end) - - %{acc | storage_root: tree, map_backed: false, root_hash: nil} + %{acc | storage_root: pairs, map_backed: false, root_hash: nil} end defp put_account(%State{} = st, i, acc), do: State.set_account(st, addr(i), acc) @@ -228,8 +222,8 @@ defmodule ChainStateMerkleTest do end end - describe "EVM-shaped payloads (32-byte key/value, insert_items)" do - test "batch insert_items like Evm.process_updates/2" do + describe "EVM-shaped payloads (32-byte key/value via storage_put_map)" do + test "batch storage updates like Evm.process_updates/2" do kvs = for i <- 0..399, into: %{} do {slot_u256(i), val_u256(Bitwise.bxor(i, 0xDEAD_BEEF))} @@ -306,16 +300,18 @@ defmodule ChainStateMerkleTest do assert_roundtrip(prev, next) end - test "root_hash on deep U256-slot tree (get_proofs shape is NIF-specific; root_hash stresses trie)" do - t = - CMerkleTree.insert_items( - CMerkleTree.new(), - Enum.map(0..220, fn i -> {slot_u256(i), val_u256(i)} end) + test "storage_root_hash and storage_get_proofs on deep U256-slot account" do + st = + State.new() + |> put_account( + 1, + account_with_storage(Enum.map(0..220, fn i -> {slot_u256(i), val_u256(i)} end)) ) - assert is_binary(CMerkleTree.root_hash(t)) - p = CMerkleTree.get_proofs(t, slot_u256(0)) - assert is_tuple(p) or is_map(p) + root = State.storage_root_hash(st, addr(1)) + assert is_binary(root) and byte_size(root) == 32 + p = State.storage_get_proofs(st, addr(1), slot_u256(0)) + assert is_tuple(p) or is_map(p) or is_list(p) end test "large mutation round-trip after independent clones (no lock)" do diff --git a/test/chain_state_uncompact_test.exs b/test/chain_state_uncompact_test.exs index 1bac0c2..b0d6940 100644 --- a/test/chain_state_uncompact_test.exs +++ b/test/chain_state_uncompact_test.exs @@ -18,15 +18,10 @@ defmodule ChainStateUncompactTest do defp val(i), do: <> defp sample_account(i) do - tree = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(i), val(i)} - ]) - %Account{ nonce: i, balance: i * 1_000, - storage_root: tree, + storage_root: [{slot(i), val(i)}], code: <>, map_backed: false } @@ -74,14 +69,14 @@ defmodule ChainStateUncompactTest do restored = State.uncompact(%State{accounts: %{}}) assert CAccountMap.size(restored.accounts) == 0 assert is_binary(Chain.State.hash(restored)) - assert State.hash(restored) == CMerkleTree.root_hash(CMerkleTree.new()) + assert State.hash(restored) == CAccountMap.root_hash(CAccountMap.new()) end test "empty CAccountMap resource" do restored = State.uncompact(%State{accounts: CAccountMap.new()}) assert CAccountMap.size(restored.accounts) == 0 assert is_binary(Chain.State.hash(restored)) - assert State.hash(restored) == CMerkleTree.root_hash(CMerkleTree.new()) + assert State.hash(restored) == CAccountMap.root_hash(CAccountMap.new()) end test "live CAccountMap resource rebuilds state trie" do @@ -109,12 +104,11 @@ defmodule ChainStateUncompactTest do %Account{ nonce: 3, balance: 99, - storage_root: - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(1), val(1)}, - {slot(2), val(2)}, - {slot(3), val(3)} - ]), + storage_root: [ + {slot(1), val(1)}, + {slot(2), val(2)}, + {slot(3), val(3)} + ], code: :binary.copy(<<0xAB>>, 512) } @@ -180,11 +174,10 @@ defmodule ChainStateUncompactTest do %Account{ nonce: 5, balance: 1_000, - storage_root: - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(1), val(1)}, - {slot(2), val(2)} - ]), + storage_root: [ + {slot(1), val(1)}, + {slot(2), val(2)} + ], code: <<5>> } @@ -193,7 +186,13 @@ defmodule ChainStateUncompactTest do {5, 1_000, root, <<5>>} = CAccountMap.get(accounts, addr(1)) assert is_binary(root) and byte_size(root) == 32 - assert root == CMerkleTree.root_hash(multi_slot.storage_root) + + expected_root = + CAccountMap.new() + |> CAccountMap.put(addr(1), 5, 1_000, multi_slot.storage_root, <<5>>) + |> CAccountMap.storage_root_hash(addr(1)) + + assert root == expected_root assert CAccountMap.storage_get(accounts, addr(1), slot(1)) == val(1) assert CAccountMap.storage_get(accounts, addr(1), slot(2)) == val(2) @@ -229,17 +228,20 @@ defmodule ChainStateUncompactTest do compact = compact_via_state(acc) assert compact.code_hash == Account.codehash(acc) - assert compact.root_hash == CMerkleTree.root_hash(acc.storage_root) + + expected_root = + CAccountMap.new() + |> CAccountMap.put(addr(1), acc.nonce, acc.balance, acc.storage_root, acc.code) + |> CAccountMap.storage_root_hash(addr(1)) + + assert compact.root_hash == expected_root end test "put overwrites lazy account without prior get" do compact = %{addr(1) => compact_via_state(sample_account(1))} {accounts, _} = CAccountMap.uncompact_state(compact) - new_storage = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(99), val(99)} - ]) + new_storage = [{slot(99), val(99)}] accounts = CAccountMap.put(accounts, addr(1), 9, 9_000, new_storage, <<9>>) @@ -265,7 +267,13 @@ defmodule ChainStateUncompactTest do assert code == <> assert is_binary(root) and byte_size(root) == 32 assert CAccountMap.storage_get(accounts, addr(i), slot(i)) == val(i) - assert root == CMerkleTree.root_hash(sample_account(i).storage_root) + + expected_root = + CAccountMap.new() + |> CAccountMap.put(addr(i), i, i * 1_000, sample_account(i).storage_root, <>) + |> CAccountMap.storage_root_hash(addr(i)) + + assert root == expected_root assert CAccountMap.storage_root_hash(accounts, addr(i)) == root end end @@ -296,7 +304,7 @@ defmodule ChainStateUncompactTest do compact = compact_accounts_map(3) {accounts, _} = CAccountMap.uncompact_state(compact) - new_storage = CMerkleTree.insert_items(CMerkleTree.new(), [{slot(50), val(50)}]) + new_storage = [{slot(50), val(50)}] fork = accounts diff --git a/test/chain_test.exs b/test/chain_test.exs index ff9d8c6..35d2f64 100644 --- a/test/chain_test.exs +++ b/test/chain_test.exs @@ -36,8 +36,8 @@ defmodule ChainTest do code: Rlpx.bin2addr(state["code"]), nonce: Rlpx.bin2num(state["nonce"]), storage_root: - Enum.reduce(state["storage"], CMerkleTree.new(), fn {key, value}, tree -> - CMerkleTree.insert(tree, Rlpx.hex2num(key), Rlpx.bin2num(value)) + Enum.map(state["storage"] || %{}, fn {key, value} -> + {Hash.to_bytes32(Rlpx.hex2num(key)), Hash.to_bytes32(Rlpx.bin2num(value))} end) }} end) @@ -259,14 +259,11 @@ defmodule ChainTest do assert Account.nonce(result) == Account.nonce(account) assert Account.code(result) == Account.code(account) - # for {key, value} <- to_list(result.storage_root) do - # assert {key, value} == {key, Account.storageInteger(account, key)} - # end # result is map-backed (no live trie); compare via State.storage_to_list. ref_list = case account do - %Account{storage_root: tree} when is_reference(tree) -> - CMerkleTree.to_list(tree) + %Account{storage_root: pairs} when is_list(pairs) -> + pairs |> Enum.map(fn {key, value} -> {compress(key), compress(value)} end) |> Enum.sort() @@ -289,12 +286,6 @@ defmodule ChainTest do |> Enum.sort() end - defp to_list(tree) do - CMerkleTree.to_list(tree) - |> Enum.map(fn {key, value} -> {compress(key), compress(value)} end) - |> Enum.sort() - end - defp compress(nil) do 0 end diff --git a/test/cmerkle_account_map_diff_test.exs b/test/cmerkle_account_map_diff_test.exs index 17b3e20..67c438d 100644 --- a/test/cmerkle_account_map_diff_test.exs +++ b/test/cmerkle_account_map_diff_test.exs @@ -14,8 +14,7 @@ defmodule CMerkleAccountMapDiffTest do defp build_map(n, mutate \\ nil) do Enum.reduce(1..n, CAccountMap.new(), fn i, acc -> - storage = - CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) + storage = [{slot(i), <>}] nonce = if mutate == {:nonce, i}, do: i + 1000, else: i balance = if mutate == {:balance, i}, do: i * 9_999, else: i * 1_000 @@ -81,7 +80,7 @@ defmodule CMerkleAccountMapDiffTest do for mutate <- [{:nonce, 3}, {:balance, 7}, {:code, 11}] do fork = CAccountMap.clone(base) {kind, i} = mutate - storage = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) + storage = [{slot(i), <>}] fork = case kind do @@ -112,10 +111,7 @@ defmodule CMerkleAccountMapDiffTest do end test "shared storage trie across accounts" do - storage = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(1), <<1::unsigned-size(256)>>} - ]) + storage = [{slot(1), <<1::unsigned-size(256)>>}] a = Enum.reduce(1..12, CAccountMap.new(), fn i, acc -> @@ -135,7 +131,7 @@ defmodule CMerkleAccountMapDiffTest do State.new() |> then(fn st -> Enum.reduce(1..80, st, fn i, acc -> - storage = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) + storage = [{slot(i), <>}] acc0 = %{ Account.new(nonce: i, balance: i) @@ -183,13 +179,7 @@ defmodule CMerkleAccountMapDiffTest do CAccountMap.delete(acc, id) 2 -> - storage = - CMerkleTree.insert( - CMerkleTree.new(), - slot(i + 10_000), - <> - ) - + storage = [{slot(i + 10_000), <>}] CAccountMap.put(acc, id, i + 1, i * 2_000, storage, <>) _ -> @@ -222,7 +212,7 @@ defmodule CMerkleAccountMapDiffTest do State.new() |> then(fn st -> Enum.reduce(1..30, st, fn i, acc -> - storage = CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) + storage = [{slot(i), <>}] State.set_account(acc, addr(i), %{ Account.new(nonce: i) diff --git a/test/cmerkle_lock_clone_regression_test.exs b/test/cmerkle_lock_clone_regression_test.exs index c02ff2c..ffc4689 100644 --- a/test/cmerkle_lock_clone_regression_test.exs +++ b/test/cmerkle_lock_clone_regression_test.exs @@ -21,15 +21,10 @@ defmodule CMerkleLockCloneRegressionTest do defp val(i), do: <> defp sample_account(i) do - tree = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(i), val(i)} - ]) - %Account{ nonce: i, balance: i * 1_000, - storage_root: tree, + storage_root: [{slot(i), val(i)}], code: <>, map_backed: false } @@ -82,7 +77,7 @@ defmodule CMerkleLockCloneRegressionTest do addr(1), 99, 1, - CMerkleTree.new(), + [], <<>> ) end @@ -144,7 +139,7 @@ defmodule CMerkleLockCloneRegressionTest do end assert_raise ArgumentError, fn -> - CAccountMap.put(peak.accounts, addr(9), 0, 0, CMerkleTree.new(), <<>>) + CAccountMap.put(peak.accounts, addr(9), 0, 0, [], <<>>) end assert State.state_root_hashes(peak) == before @@ -183,10 +178,7 @@ defmodule CMerkleLockCloneRegressionTest do describe "shared storage lock (apply_canonical_lock regression)" do test "lock with many accounts sharing one storage completes and stays forkable" do - shared = - Enum.reduce(1..8, CMerkleTree.new(), fn i, tree -> - CMerkleTree.insert(tree, slot(i), val(i)) - end) + shared = Enum.map(1..8, fn i -> {slot(i), val(i)} end) accounts = Enum.reduce(1..40, CAccountMap.new(), fn i, map -> @@ -216,16 +208,12 @@ defmodule CMerkleLockCloneRegressionTest do end test "concurrent locks on distinct maps that share storage roots do not hang" do - # Same root hash via independent trees (one resource must not be kept by many - # AccountMaps). Concurrent lock+clone must complete without hanging. + # Same root hash via identical slot lists. Concurrent lock+clone must complete + # without hanging. + storage = [{slot(1), val(1)}, {slot(2), val(2)}] + maps = for i <- 1..8 do - storage = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(1), val(1)}, - {slot(2), val(2)} - ]) - CAccountMap.new() |> CAccountMap.put(addr(i), i, i * 10, storage, <>) |> CAccountMap.put(addr(100 + i), i, i * 10, storage, <>) diff --git a/test/cmerkle_lock_concurrency_test.exs b/test/cmerkle_lock_concurrency_test.exs index 5988571..07bf7b4 100644 --- a/test/cmerkle_lock_concurrency_test.exs +++ b/test/cmerkle_lock_concurrency_test.exs @@ -8,149 +8,12 @@ defmodule CMerkleLockConcurrencyTest do @moduletag timeout: 300_000 @tasks 24 - @ops 30 @timeout_ms 120_000 - describe "lock + difference concurrency (production deadlock regression)" do - test "concurrent lock/1 on clones with identical root (enter_lock dedup)" do - data = - Enum.map(1..80, fn i -> - {String.pad_leading("L#{i}", 32), CMerkleTree.hash("lock#{i}")} - end) - - base = CMerkleTree.new() |> CMerkleTree.insert_items(data) - - run_parallel(@tasks, fn _ -> - _ = - base - |> CMerkleTree.clone() - |> CMerkleTree.lock() - - :ok - end) - end - - test "concurrent difference while locking cloned trees with shared roots" do - ta = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..120, fn i -> - {String.pad_leading("da#{i}", 32), CMerkleTree.hash("da#{i}")} - end) - ) - - tb = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..120, fn i -> - {String.pad_leading("db#{i}", 32), CMerkleTree.hash("db#{i}")} - end) - ) - - base = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..100, fn i -> - {String.pad_leading("sh#{i}", 32), CMerkleTree.hash("sh#{i}")} - end) - ) - - run_parallel(@tasks, fn w -> - if rem(w, 3) == 0 do - _ = CMerkleTree.difference(ta, tb) - else - _ = - base - |> CMerkleTree.clone() - |> CMerkleTree.lock() - - Enum.each(1..@ops, fn j -> - k = String.pad_leading("m#{w}_#{j}", 32) - _ = CMerkleTree.insert(ta, k, CMerkleTree.hash("v#{w}#{j}")) - _ = CMerkleTree.difference(ta, tb) - end) - end - - :ok - end) - end - - test "interleaved lock, difference, and mutate (P5 + P6 combined)" do - ta = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..100, fn i -> - {String.pad_leading("ia#{i}", 32), CMerkleTree.hash("ia#{i}")} - end) - ) - - tb = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..100, fn i -> - {String.pad_leading("ib#{i}", 32), CMerkleTree.hash("ib#{i}")} - end) - ) - - mutators = div(@tasks, 3) |> max(1) - lockers = div(@tasks, 3) |> max(1) - differ = @tasks - mutators - lockers - - run_parallel(mutators, fn w -> - Enum.each(1..@ops, fn j -> - k = String.pad_leading("mut#{w}_#{j}", 32) - _ = CMerkleTree.insert(ta, k, CMerkleTree.hash("mut#{w}#{j}")) - end) - - :ok - end) - - run_parallel(lockers, fn _ -> - _ = - ta - |> CMerkleTree.clone() - |> CMerkleTree.lock() - - :ok - end) - - run_parallel(differ, fn _ -> - _ = CMerkleTree.difference(ta, tb) - _ = CMerkleTree.difference(tb, ta) - :ok - end) - end - - test "lock dedup still shares canonical root across clones" do - data = - Enum.map(1..40, fn i -> - {String.pad_leading("d#{i}", 32), CMerkleTree.hash("d#{i}")} - end) - - base = CMerkleTree.new() |> CMerkleTree.insert_items(data) - expected_root = CMerkleTree.root_hash(base) - - locked = - 1..8 - |> Enum.map(fn _ -> - base |> CMerkleTree.clone() |> CMerkleTree.lock() - end) - - for tree <- locked do - assert CMerkleTree.root_hash(tree) == expected_root - end - end - end - describe "account_map_lock NIF concurrency" do test "concurrent CAccountMap.lock on maps with deduped shared storage tries" do map = lock_test_shared_storage_map(60, 5) - _store = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {String.pad_leading("store", 32), CMerkleTree.hash("store")} - ]) - run_parallel(@tasks, fn _ -> _ = map |> CAccountMap.clone() |> CAccountMap.lock() :ok @@ -202,7 +65,7 @@ defmodule CMerkleLockConcurrencyTest do test "difference_full(A,B) concurrent with difference_full(B,A)" do a = lock_test_shared_storage_map(40, 4) b = CAccountMap.clone(a) - b = CAccountMap.put(b, <<3::unsigned-size(160)>>, 99, 99_000, CMerkleTree.new(), <<99>>) + b = CAccountMap.put(b, <<3::unsigned-size(160)>>, 99, 99_000, [], <<99>>) ab = div(@tasks, 2) |> max(1) ba = @tasks - ab @@ -221,11 +84,8 @@ defmodule CMerkleLockConcurrencyTest do defp lock_test_shared_storage_map(account_count, group_size) do Enum.reduce(1..account_count, CAccountMap.new(), fn i, map -> - storage = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {String.pad_leading("g#{div(i - 1, group_size)}", 32), - CMerkleTree.hash("g#{div(i - 1, group_size)}")} - ]) + g = div(i - 1, group_size) + storage = [{String.pad_leading("g#{g}", 32), Diode.hash("g#{g}")}] CAccountMap.put(map, <>, i, i * 1_000, storage, <>) end) diff --git a/test/cmerkle_nif_deadlock_test.exs b/test/cmerkle_nif_deadlock_test.exs index 528025a..9c5cf8b 100644 --- a/test/cmerkle_nif_deadlock_test.exs +++ b/test/cmerkle_nif_deadlock_test.exs @@ -11,90 +11,8 @@ defmodule CMerkleNifDeadlockTest do @moduletag :cmerkle_concurrency @tasks 32 - @ops 25 @timeout_ms 180_000 - @p9_tasks 48 - @p9_ops 40 - - describe "D-A1/P9 phased lock_and_difference (nightly regression)" do - @describetag timeout: 180_000 - - test "phase-1 locked clone drop then phase-2 insert and difference load" do - lockers = div(@p9_tasks, 3) |> max(1) - differ = @p9_tasks - lockers - - shared = prefilled_tree("s", 100) - other = prefilled_tree("o", 100) - - run_parallel(lockers, fn _ -> - _ = - shared - |> CMerkleTree.clone() - |> CMerkleTree.lock() - - :ok - end) - - :erlang.garbage_collect() - - run_parallel(differ, fn w -> - Enum.each(1..@p9_ops, fn j -> - k = String.pad_leading("p9#{w}_#{j}", 32) - _ = CMerkleTree.insert(shared, k, CMerkleTree.hash("p9#{w}#{j}")) - _ = CMerkleTree.difference(shared, other) - _ = CMerkleTree.difference(other, shared) - end) - - :ok - end) - end - end - - describe "D-A2 leave_lock (GC) vs difference on same tree" do - test "concurrent disposable trees, GC, and difference on anchor" do - anchor = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..80, fn i -> - {String.pad_leading("a2a#{i}", 32), CMerkleTree.hash("a2a#{i}")} - end) - ) - - other = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..80, fn i -> - {String.pad_leading("a2o#{i}", 32), CMerkleTree.hash("a2o#{i}")} - end) - ) - - run_parallel(@tasks, fn w -> - Enum.each(1..@ops, fn j -> - _ = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..12, fn k -> - {String.pad_leading("d#{w}_#{j}_#{k}", 32), CMerkleTree.hash("d#{w}#{j}#{k}")} - end) - ) - - _ = CMerkleTree.difference(anchor, other) - - if rem(j, 7) == 0 do - _ = - anchor - |> CMerkleTree.clone() - |> CMerkleTree.lock() - end - end) - - if rem(w, 5) == 0, do: :erlang.garbage_collect() - :ok - end) - end - end - describe "D-C2 uncompact_state vs storage difference" do test "parallel uncompact and per-account storage diffs" do n = 200 @@ -107,18 +25,32 @@ defmodule CMerkleNifDeadlockTest do if CAccountMap.size(accounts) != n, do: raise("size mismatch") else a = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(i), <>}, - {slot(i + 7000), <>} - ]) + CAccountMap.new() + |> CAccountMap.put( + addr(i), + i, + i * 100, + [ + {slot(i), <>}, + {slot(i + 7000), <>} + ], + <> + ) b = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(i), <>}, - {slot(i + 8000), <>} - ]) + CAccountMap.new() + |> CAccountMap.put( + addr(i), + i, + i * 100, + [ + {slot(i), <>}, + {slot(i + 8000), <>} + ], + <> + ) - _ = CMerkleTree.difference(a, b) + _ = CAccountMap.difference_full(a, b) end :ok @@ -173,17 +105,10 @@ defmodule CMerkleNifDeadlockTest do map = Enum.reduce(1..n, CAccountMap.new(), fn i, acc -> - storage = - CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - + storage = [{slot(i), <>}] CAccountMap.put(acc, addr(i), i, i * 1_000, storage, <>) end) - _store = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(99_999), <<99_999::unsigned-size(256)>>} - ]) - lockers = div(@tasks, 3) |> max(1) cloners = div(@tasks, 3) |> max(1) differ = @tasks - lockers - cloners @@ -202,7 +127,7 @@ defmodule CMerkleNifDeadlockTest do addr(rem(w, n) + 1), 9_999, 9_999, - CMerkleTree.new(), + [], <<9_999>> ) @@ -213,8 +138,6 @@ defmodule CMerkleNifDeadlockTest do run_parallel(differ, fn i -> a = addr(rem(i, n) + 1) b = addr(rem(i + 1, n) + 1) - # get/2 returns root hashes; exercise concurrent storage reads instead of - # CMerkleTree.difference on live resources from get. _ = CAccountMap.get(map, a) _ = CAccountMap.get(map, b) _ = CAccountMap.storage_root_hash(map, a) @@ -225,97 +148,6 @@ defmodule CMerkleNifDeadlockTest do end end - describe "D-B2 three-tree difference overlap" do - test "difference(A,B) concurrent with difference(A,C)" do - shared = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..100, fn i -> - {String.pad_leading("b2s#{i}", 32), CMerkleTree.hash("b2s#{i}")} - end) - ) - - tb = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..100, fn i -> - {String.pad_leading("b2b#{i}", 32), CMerkleTree.hash("b2b#{i}")} - end) - ) - - tc = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..100, fn i -> - {String.pad_leading("b2c#{i}", 32), CMerkleTree.hash("b2c#{i}")} - end) - ) - - ab = div(@tasks, 2) |> max(1) - ac = @tasks - ab - - run_parallel(ab, fn _ -> - Enum.each(1..@ops, fn _ -> - _ = CMerkleTree.difference(shared, tb) - end) - - :ok - end) - - run_parallel(ac, fn _ -> - Enum.each(1..@ops, fn _ -> - _ = CMerkleTree.difference(shared, tc) - _ = CMerkleTree.difference(tb, tc) - end) - - :ok - end) - end - end - - describe "D-F2 enter_lock canonical ref vs concurrent leave_lock" do - test "many clones lock same root under difference load" do - data = - Enum.map(1..100, fn i -> - {String.pad_leading("f2#{i}", 32), CMerkleTree.hash("f2#{i}")} - end) - - base = CMerkleTree.new() |> CMerkleTree.insert_items(data) - expected_root = CMerkleTree.root_hash(base) - - other = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..100, fn i -> - {String.pad_leading("f2o#{i}", 32), CMerkleTree.hash("f2o#{i}")} - end) - ) - - lockers = div(@tasks, 2) |> max(1) - differ = @tasks - lockers - - run_parallel(lockers, fn _ -> - locked = - base - |> CMerkleTree.clone() - |> CMerkleTree.lock() - - if CMerkleTree.root_hash(locked) != expected_root, do: raise("root mismatch") - :ok - end) - - run_parallel(differ, fn w -> - Enum.each(1..@ops, fn j -> - k = String.pad_leading("f2m#{w}_#{j}", 32) - _ = CMerkleTree.insert(base, k, CMerkleTree.hash("m#{w}#{j}")) - _ = CMerkleTree.difference(base, other) - end) - - :ok - end) - end - end - describe "D-D7 prepare_state composite (us1-shaped)" do test "concurrent difference_full, to_list, and State.lock" do n = 100 @@ -365,24 +197,12 @@ defmodule CMerkleNifDeadlockTest do defp slot(i), do: <> - defp prefilled_tree(prefix, n) do - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..n, fn i -> - {String.pad_leading("#{prefix}#{i}", 32), CMerkleTree.hash("#{prefix}#{i}")} - end) - ) - end - defp build_compact_accounts(n) do Enum.reduce(1..n, State.new(), fn i, st -> - tree = - CMerkleTree.insert(CMerkleTree.new(), slot(i), <>) - acc = %Account{ nonce: i, balance: i * 1_000, - storage_root: tree, + storage_root: [{slot(i), <>}], code: <>, map_backed: false } @@ -395,12 +215,13 @@ defmodule CMerkleNifDeadlockTest do defp build_live_state(n) do Enum.reduce(1..n, State.new(), fn i, st -> - tree = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(i), <>} - ]) + acc = %Account{ + nonce: i, + balance: i * 1_000, + storage_root: [{slot(i), <>}], + code: <> + } - acc = %Account{nonce: i, balance: i * 1_000, storage_root: tree, code: <>} State.set_account(st, addr(i), acc) end) end diff --git a/test/cmerkle_nif_leak_test.exs b/test/cmerkle_nif_leak_test.exs index bd86cbc..14693b6 100644 --- a/test/cmerkle_nif_leak_test.exs +++ b/test/cmerkle_nif_leak_test.exs @@ -33,11 +33,11 @@ defmodule CMerkleNifLeakTest do end describe "canonical lock dedup must not leak SharedState" do - test "empty trie lock storm" do + test "empty account map lock storm" do baseline = rss_kb() for _ <- 1..300 do - CMerkleTree.new() |> CMerkleTree.lock() + CAccountMap.new() |> CAccountMap.lock() end force_gc() @@ -49,16 +49,11 @@ defmodule CMerkleNifLeakTest do test "account_map_lock with identical storage roots" do baseline = rss_kb() n = 60 + storage = [{slot(1), val(1)}, {slot(2), val(2)}] for _ <- 1..30 do map = Enum.reduce(1..n, CAccountMap.new(), fn i, acc -> - storage = - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(1), val(1)}, - {slot(2), val(2)} - ]) - CAccountMap.put(acc, addr(i), i, i * 1_000, storage, <>) end) @@ -73,15 +68,13 @@ defmodule CMerkleNifLeakTest do test "locked clone GC retains canonical map until last registration" do shared = - CMerkleTree.new() - |> CMerkleTree.insert_items( - Enum.map(1..80, fn i -> - {String.pad_leading("lc#{i}", 32), CMerkleTree.hash("lc#{i}")} - end) - ) + Enum.reduce(1..80, CAccountMap.new(), fn i, acc -> + storage = [{String.pad_leading("lc#{i}", 32), Diode.hash("lc#{i}")}] + CAccountMap.put(acc, addr(i), i, i * 1_000, storage, <>) + end) for _ <- 1..300 do - _ = shared |> CMerkleTree.clone() |> CMerkleTree.lock() + _ = shared |> CAccountMap.clone() |> CAccountMap.lock() end force_gc() @@ -101,10 +94,7 @@ defmodule CMerkleNifLeakTest do acc = %Account{ nonce: i, balance: i * 1_000, - storage_root: - CMerkleTree.insert_items(CMerkleTree.new(), [ - {slot(i), val(i)} - ]), + storage_root: [{slot(i), val(i)}], code: <> } @@ -156,7 +146,7 @@ defmodule CMerkleNifLeakTest do defp build_diff_map(n) do Enum.reduce(1..n, CAccountMap.new(), fn i, acc -> - storage = CMerkleTree.insert_items(CMerkleTree.new(), [{slot(i), val(i)}]) + storage = [{slot(i), val(i)}] CAccountMap.put(acc, addr(i), i, i * 1_000, storage, <>) end) end diff --git a/test/cmerkle_storage_map_test.exs b/test/cmerkle_storage_map_test.exs index d1cbfc2..d54d026 100644 --- a/test/cmerkle_storage_map_test.exs +++ b/test/cmerkle_storage_map_test.exs @@ -91,7 +91,7 @@ defmodule CMerkleStorageMapTest do test "get_proofs and storage_get_proofs return terms without crashing" do map = CAccountMap.new() - |> CAccountMap.put(addr(1), 1, 100, CMerkleTree.new(), <<>>) + |> CAccountMap.put(addr(1), 1, 100, [], <<>>) |> CAccountMap.storage_put_map(%{addr(1) => %{slot(7) => val(7)}}) account_proof = CAccountMap.get_proofs(map, addr(1)) diff --git a/test/cmerkletree_test.exs b/test/cmerkletree_test.exs deleted file mode 100644 index 60a8281..0000000 --- a/test/cmerkletree_test.exs +++ /dev/null @@ -1,541 +0,0 @@ -# Diode Server -# Copyright 2021-2024 Diode -# Licensed under the Diode License, Version 1.1 -defmodule CMerkleTreeTest do - use ExUnit.Case - - # Reference: count zero bytes (same semantics as legacy Niffler / EVM gas calc). - defp ref_count_zeros(bin), do: Enum.count(:binary.bin_to_list(bin), &(&1 == 0)) - - describe "golden root_hash regression" do - test "pairs dataset matches fixed roots (Elixir pairs/1 format)" do - for {n, hex} <- [ - {20, "b043342d7f3104673d8a0aa10e85ac17dd3edddb71754490247a0b945be71521"}, - {128, "852d365bfbad6c28924a672e304d62a118ca6adbe48aa2471122e54d586b8f37"}, - {1000, "ee3b68fee8188459a39e190e0dfcff4c7ea1711fdf62b4cb553f0b8c0ed350b4"} - ] do - tree = - Enum.reduce(pairs(n), new(), fn item, acc -> - CMerkleTree.insert_item(acc, item) - end) - - assert Base.encode16(CMerkleTree.root_hash(tree), case: :lower) == hex - end - end - end - - describe "memory introspection NIFs" do - test "struct_sizes and memory_stats are consistent" do - {ib, pb, _plb, _tb, stripe} = CMerkleTree.struct_sizes() - assert stripe >= 1 - assert ib > 0 and pb > 0 - - tree = - Enum.reduce(pairs(30), new(), fn item, acc -> - CMerkleTree.insert_item(acc, item) - end) - - {nodes, pairs_n, approx} = CMerkleTree.memory_stats(tree) - assert pairs_n == 30 - assert nodes >= 1 - assert approx == nodes * ib + pairs_n * pb - end - - test "malloc_info returns XML on GNU libc" do - case CMerkleTree.malloc_info() do - :unsupported -> assert true - bin when is_binary(bin) -> assert String.contains?(bin, "malloc") - end - end - end - - describe "count_zeros" do - test "matches reference for edge and random payloads" do - bins = [ - <<>>, - <<0>>, - <<255>>, - <<0, 0, 0>>, - <<0, 1, 2, 0>>, - :crypto.strong_rand_bytes(256) - ] - - for bin <- bins do - assert CMerkleTree.count_zeros(bin) == ref_count_zeros(bin) - end - end - end - - defp new() do - CMerkleTree.new() - end - - test "initialize" do - tree = new() - assert CMerkleTree.size(tree) == 0 - assert CMerkleTree.bucket_count(tree) == 1 - end - - test "inserts" do - tree = new() - - size = 16 - - tree = - Enum.reduce(pairs(size), tree, fn item, acc -> - CMerkleTree.insert_item(acc, item) - end) - - assert CMerkleTree.size(tree) == size - assert CMerkleTree.bucket_count(tree) == 1 - end - - test "difference" do - [a, b, c, d] = pairs(20) |> Enum.chunk_every(5) - [a2, _b2, _c2, _d2] = pairs(20, "2") |> Enum.chunk_every(5) - - tree_a = CMerkleTree.from_list(a ++ b ++ c) - tree_b = CMerkleTree.from_list(a2 ++ b ++ d) - - diff = CMerkleTree.difference(tree_a, tree_b) - - assert diff == %{ - " 1" => - {<<107, 134, 178, 115, 255, 52, 252, 225, 157, 107, 128, 78, 255, 90, 63, 87, 71, - 173, 164, 234, 162, 47, 29, 73, 192, 30, 82, 221, 183, 135, 91, 75>>, - <<107, 81, 212, 49, 223, 93, 127, 20, 28, 190, 206, 204, 247, 158, 223, 61, 216, - 97, 195, 180, 6, 159, 11, 17, 102, 26, 62, 239, 172, 187, 169, 24>>}, - " 2" => - {<<212, 115, 94, 58, 38, 94, 22, 238, 224, 63, 89, 113, 139, 155, 93, 3, 1, 156, 7, - 216, 182, 197, 31, 144, 218, 58, 102, 110, 236, 19, 171, 53>>, - <<120, 95, 62, 199, 235, 50, 243, 11, 144, 205, 15, 207, 54, 87, 211, 136, 181, - 255, 66, 151, 242, 249, 113, 111, 246, 110, 155, 105, 192, 93, 221, 9>>}, - " 3" => - {<<78, 7, 64, 133, 98, 190, 219, 139, 96, 206, 5, 193, 222, 207, 227, 173, 22, 183, - 34, 48, 150, 125, 224, 31, 100, 11, 126, 71, 41, 180, 159, 206>>, - <<226, 156, 156, 24, 12, 98, 121, 176, 176, 42, 189, 106, 24, 1, 199, 192, 64, - 130, 207, 72, 110, 192, 39, 170, 19, 81, 94, 79, 56, 132, 187, 107>>}, - " 4" => - {<<75, 34, 119, 119, 212, 221, 31, 198, 28, 111, 136, 79, 72, 100, 29, 2, 180, 209, - 33, 211, 253, 50, 140, 176, 139, 85, 49, 252, 172, 218, 191, 138>>, - <<115, 71, 92, 180, 10, 86, 142, 141, 168, 160, 69, 206, 209, 16, 19, 126, 21, - 159, 137, 10, 196, 218, 136, 59, 107, 23, 220, 101, 27, 58, 128, 73>>}, - " 5" => - {<<239, 45, 18, 125, 227, 123, 148, 43, 170, 208, 97, 69, 229, 75, 12, 97, 154, 31, - 34, 50, 123, 46, 187, 207, 190, 199, 143, 85, 100, 175, 227, 157>>, - <<65, 207, 192, 209, 242, 209, 39, 176, 69, 85, 183, 36, 109, 132, 1, 155, 77, 39, - 113, 10, 63, 58, 255, 110, 119, 100, 55, 91, 30, 6, 224, 93>>}, - " 11" => - {<<79, 200, 43, 38, 174, 203, 71, 210, 134, 140, 78, 251, 227, 88, 23, 50, 163, - 231, 203, 204, 108, 46, 251, 50, 6, 44, 8, 23, 10, 5, 238, 184>>, nil}, - " 12" => - {<<107, 81, 212, 49, 223, 93, 127, 20, 28, 190, 206, 204, 247, 158, 223, 61, 216, - 97, 195, 180, 6, 159, 11, 17, 102, 26, 62, 239, 172, 187, 169, 24>>, nil}, - " 13" => - {<<63, 219, 163, 95, 4, 220, 140, 70, 41, 134, 201, 146, 188, 248, 117, 84, 98, 87, - 17, 48, 114, 169, 9, 193, 98, 247, 228, 112, 229, 129, 226, 120>>, nil}, - " 14" => - {<<133, 39, 168, 145, 226, 36, 19, 105, 80, 255, 50, 202, 33, 43, 69, 188, 147, - 246, 159, 187, 128, 28, 59, 30, 190, 218, 197, 39, 117, 249, 158, 97>>, nil}, - " 15" => - {<<230, 41, 250, 101, 152, 215, 50, 118, 143, 124, 114, 107, 75, 98, 18, 133, 249, - 195, 184, 83, 3, 144, 10, 169, 18, 1, 125, 183, 97, 125, 139, 219>>, nil}, - " 16" => - {nil, - <<177, 126, 246, 209, 156, 122, 91, 30, 232, 59, 144, 124, 89, 85, 38, 220, 177, - 235, 6, 219, 130, 39, 214, 80, 213, 221, 160, 169, 244, 206, 140, 217>>}, - " 17" => - {nil, - <<69, 35, 84, 15, 21, 4, 205, 23, 16, 12, 72, 53, 232, 91, 126, 239, 212, 153, 17, - 88, 15, 142, 255, 240, 89, 154, 143, 40, 59, 230, 185, 227>>}, - " 18" => - {nil, - <<78, 201, 89, 159, 194, 3, 209, 118, 163, 1, 83, 108, 46, 9, 26, 25, 188, 133, - 39, 89, 178, 85, 189, 104, 24, 129, 10, 66, 197, 254, 209, 74>>}, - " 19" => - {nil, - <<148, 0, 241, 178, 28, 181, 39, 215, 250, 61, 62, 171, 186, 147, 85, 122, 24, - 235, 231, 162, 202, 78, 71, 28, 254, 94, 76, 91, 76, 167, 247, 103>>}, - " 20" => - {nil, - <<245, 202, 56, 247, 72, 161, 214, 234, 247, 38, 184, 164, 47, 181, 117, 195, 199, - 31, 24, 100, 168, 20, 51, 1, 120, 45, 225, 61, 162, 217, 32, 43>>} - } - end - - test "number conversion" do - data = %{ - <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0>> => - <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1>>, - <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1>> => - <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1>>, - <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 3>> => - <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 169, 79, 83, 116, 252, 229, 237, 188, 142, 42, 134, - 151, 193, 83, 49, 103, 126, 110, 191, 11>>, - <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 7>> => - <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 69, 254>>, - <<110, 54, 152, 54, 72, 124, 35, 75, 158, 85, 62, 243, 247, 135, 194, 216, 134, 85, 32, 115, - 157, 52, 12, 103, 179, 210, 81, 163, 57, 134, 229, 141>> => - <<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1>> - } - - tree = - Enum.reduce(data, new(), fn {key, value}, tree -> - value = :binary.decode_unsigned(value) - value = <> - key = :binary.decode_unsigned(key) - CMerkleTree.insert(tree, key, value) - end) - - for {key, value} <- data do - key = :binary.decode_unsigned(key) - assert CMerkleTree.get(tree, key) == value - end - end - - test "no duplicate" do - tree = - new() - |> CMerkleTree.insert_item({"a", 1}) - |> CMerkleTree.insert_item({"a", 2}) - - assert CMerkleTree.size(tree) == 1 - - assert CMerkleTree.to_list(tree) == [{"a", <<0::unsigned-size(248), 2>>}] - end - - test "clone" do - {data1, data2} = Enum.sort(pairs(128)) |> Enum.split(64) - - tree1 = new() |> CMerkleTree.insert_items(data1) - assert CMerkleTree.size(tree1) == 64 - - tree2 = CMerkleTree.clone(tree1) - assert CMerkleTree.size(tree2) == 64 - assert CMerkleTree.root_hash(tree1) == CMerkleTree.root_hash(tree2) - - tree2 = CMerkleTree.insert_items(tree2, data2) - assert CMerkleTree.size(tree2) == 128 - assert CMerkleTree.size(tree1) == 64 - assert CMerkleTree.root_hash(tree1) != CMerkleTree.root_hash(tree2) - end - - test "no nulls" do - tree = - new() - |> CMerkleTree.insert_item({"a", 1}) - |> CMerkleTree.insert_item({"a", 0}) - - assert CMerkleTree.size(tree) == 0 - end - - test "proof" do - size = 20 - tree0 = new() - - tree20 = - Enum.reduce(pairs(size), tree0, fn item, acc -> - CMerkleTree.insert_item(acc, item) - end) - - roots = CMerkleTree.root_hashes(tree20) - - for {{k, v}, idx} <- Enum.with_index(pairs(10 * size)) do - proofs = CMerkleTree.get_proofs(tree20, k) - - if idx == 0 do - assert proofs == - {[ - <<0::size(1)>>, - 15, - {" 1", - <<107, 134, 178, 115, 255, 52, 252, 225, 157, 107, 128, 78, 255, 90, 63, 87, - 71, 173, 164, 234, 162, 47, 29, 73, 192, 30, 82, 221, 183, 135, 91, 75>>} - ], - <<213, 245, 42, 124, 119, 38, 179, 149, 177, 91, 192, 217, 115, 78, 97, 97, 159, - 59, 6, 21, 61, 126, 252, 97, 248, 135, 154, 180, 92, 208, 105, 18>>} - end - - proof = proof(proofs) - - [prefix, pos | values] = value(proofs) - x = bit_size(prefix) - <> = hash(k) - <> = binary_part(hash(k), byte_size(hash(k)), -1) - - # Checking that this proof connects to the root - assert Enum.member?(roots, proof) - - # Checking that the provided range is for the given keys prefix - assert key_prefix == prefix - - # Checking that the provided leaf matches the given key - assert rem(last_byte, 16) == pos - - if idx < size do - assert :proplists.get_value(k, values) == v - else - assert :proplists.get_value(k, values) == :undefined - end - - # IO.puts("#{byte_size(BertExt.encode!(proofs))} (#{length(values)})") - end - end - - test "equality" do - size = 20 - tree = new() - - tree20 = - Enum.reduce(pairs(size), tree, fn item, acc -> - CMerkleTree.insert_item(acc, item) - end) - - tree20r = - Enum.reduce(Enum.reverse(pairs(size)), tree, fn item, acc -> - CMerkleTree.insert_item(acc, item) - end) - - assert CMerkleTree.root_hash(tree20) == CMerkleTree.root_hash(tree20r) - assert tree20 == tree20r - end - - test "deletes" do - size = 20 - sizeh = div(size, 2) - - tree0 = new() - - tree20 = - Enum.reduce(pairs(size), tree0, fn item, acc -> - CMerkleTree.insert_item(acc, item) - end) - - assert CMerkleTree.size(tree20) == size - assert CMerkleTree.bucket_count(tree20) == 2 - - tree10 = - Enum.reduce(pairs(sizeh), tree20, fn {key, _}, acc -> - CMerkleTree.delete(acc, key) - end) - - assert CMerkleTree.size(tree10) == sizeh - assert CMerkleTree.bucket_count(tree10) == 1 - - tree0v2 = - Enum.reduce(pairs(size), tree20, fn {key, _}, acc -> - CMerkleTree.delete(acc, key) - end) - - assert CMerkleTree.size(tree0v2) == 0 - assert CMerkleTree.bucket_count(tree10) == 1 - assert CMerkleTree.root_hash(tree0v2) == CMerkleTree.root_hash(tree0) - end - - describe "high load: COW clone, fork, difference" do - test "512-key base: two divergent clones and difference" do - data = - Enum.map(1..512, fn i -> {String.pad_leading("#{i}", 32), CMerkleTree.hash("hl#{i}")} end) - - {d_a, d_rest} = Enum.split(data, 300) - {d_b, d_c} = Enum.split(d_rest, 120) - - base = new() |> CMerkleTree.insert_items(d_a) - - t2 = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert_items(d_b ++ d_c) - - t_alt = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert_items(d_b) - |> CMerkleTree.insert_items(Enum.take(d_c, 40)) - - diff = CMerkleTree.difference(t2, t_alt) - assert map_size(diff) >= 1 - assert CMerkleTree.size(t2) == 512 - assert CMerkleTree.size(t_alt) == 300 + 120 + 40 - _ = CMerkleTree.root_hash(t2) - _ = CMerkleTree.root_hash(t_alt) - end - - test "U256 sequential keys: clone then parallel extensions + difference (COW + diff)" do - slots = - Enum.map(0..420, fn i -> {<>, <>} end) - - base = new() |> CMerkleTree.insert_items(slots) - - left = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert_items( - Enum.map(421..500, fn i -> - {<>, <>} - end) - ) - - right = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert_items( - Enum.map(501..590, fn i -> - {<>, <>} - end) - ) - - diff = CMerkleTree.difference(left, right) - assert map_size(diff) >= 1 - assert CMerkleTree.root_hash(left) != CMerkleTree.root_hash(right) - _ = CMerkleTree.root_hash(base) - end - - test "U256 sequential keys: clone then partial overwrite + difference (stress)" do - slots = - Enum.map(0..400, fn i -> {<>, <>} end) - - base = new() |> CMerkleTree.insert_items(slots) - - left = - base - |> CMerkleTree.clone() - |> CMerkleTree.insert_items( - Enum.map(401..450, fn i -> - {<>, <>} - end) - ) - - right = - base - |> CMerkleTree.clone() - |> then(fn t -> - Enum.reduce(200..280, t, fn i, acc -> - CMerkleTree.insert(acc, <>, <>) - end) - end) - - diff = CMerkleTree.difference(left, right) - assert map_size(diff) >= 1 - assert CMerkleTree.root_hash(left) != CMerkleTree.root_hash(right) - end - end - - describe "get_range" do - test "dense sequential keys" do - base = 0x100 - - tree = - CMerkleTree.new() - |> CMerkleTree.insert(base, 1) - |> CMerkleTree.insert(base + 1, 2) - |> CMerkleTree.insert(base + 2, 3) - - assert CMerkleTree.get_range(tree, base, 3) == [ - {<>, <<1::unsigned-size(256)>>}, - {<>, <<2::unsigned-size(256)>>}, - {<>, <<3::unsigned-size(256)>>} - ] - end - - test "sparse range returns nil for missing slots" do - base = 0x200 - - tree = - CMerkleTree.new() - |> CMerkleTree.insert(base, 1) - |> CMerkleTree.insert(base + 2, 3) - - assert CMerkleTree.get_range(tree, base, 3) == [ - {<>, <<1::unsigned-size(256)>>}, - {<>, nil}, - {<>, <<3::unsigned-size(256)>>} - ] - end - - test "stops at uint256 overflow" do - max_key = <> - tree = CMerkleTree.insert(CMerkleTree.new(), max_key, 42) - - assert length(CMerkleTree.get_range(tree, max_key, 3)) == 1 - assert CMerkleTree.get(tree, max_key) == <<42::unsigned-size(256)>> - end - - test "max count returns at most 255 entries" do - base = 0x4000 - count = 255 - - tree = - CMerkleTree.new() - |> then(fn t -> - Enum.reduce(0..(count - 1), t, fn i, acc -> - CMerkleTree.insert(acc, base + i, i + 1) - end) - end) - - range = CMerkleTree.get_range(tree, base, count) - assert length(range) == 255 - assert length(range) <= 255 - end - end - - defp pairs(num, variant \\ "") do - Enum.map(1..num, fn idx -> - {String.pad_leading("#{idx}", 32), CMerkleTree.hash("#{idx}" <> variant)} - end) - end - - defp value(list) when is_list(list) do - list - end - - defp value({left, right}) do - value(left) || value(right) - end - - defp value(hash) when is_binary(hash) do - false - end - - defp proof(list) when is_list(list) do - # :io.format("Erlang: ~p~nBert: ~p~n", [list, BertExt.encode!(list)]) - hash(BertExt.encode!(list)) - end - - defp proof({left, right}) do - list = [proof(left), proof(right)] - hash(BertExt.encode!(list)) - end - - defp proof(hash) when is_binary(hash) do - hash - end - - defp hash(value) when is_binary(value) do - Diode.hash(value) - end - - # defp encode(mixed) when is_tuple(mixed) do - # List.to_tuple(encode(Tuple.to_list(mixed))) - # end - - # defp encode(hashes) when is_list(hashes) do - # Enum.map(hashes, &encode/1) - # end - - # defp encode(hash = <<_::binary-size(32)>>) do - # :base64.encode(hash) - # end - - # defp encode(term) do - # term - # end -end diff --git a/test/count_zeros_test.exs b/test/count_zeros_test.exs new file mode 100644 index 0000000..2c71dde --- /dev/null +++ b/test/count_zeros_test.exs @@ -0,0 +1,26 @@ +# Diode Server +# Copyright 2021-2024 Diode +# Licensed under the Diode License, Version 1.1 +defmodule CountZerosTest do + use ExUnit.Case + + # Reference: count zero bytes (same semantics as legacy Niffler / EVM gas calc). + defp ref_count_zeros(bin), do: Enum.count(:binary.bin_to_list(bin), &(&1 == 0)) + + describe "count_zeros" do + test "matches reference for edge and random payloads" do + bins = [ + <<>>, + <<0>>, + <<255>>, + <<0, 0, 0>>, + <<0, 1, 2, 0>>, + :crypto.strong_rand_bytes(256) + ] + + for bin <- bins do + assert CMerkleTree.count_zeros(bin) == ref_count_zeros(bin) + end + end + end +end diff --git a/test/evm_storage_readahead_test.exs b/test/evm_storage_readahead_test.exs index 63ee7dd..bc19fb3 100644 --- a/test/evm_storage_readahead_test.exs +++ b/test/evm_storage_readahead_test.exs @@ -25,16 +25,15 @@ defmodule EvmStorageReadaheadTest do base = 0x3000 count = gs_range_count() + addr = <<1::unsigned-size(160)>> - tree = - CMerkleTree.new() - |> then(fn t -> - Enum.reduce(0..(count - 1), t, fn i, acc -> - CMerkleTree.insert(acc, base + i, i + 1) - end) + slots = + Enum.map(0..(count - 1), fn i -> + {<>, <>} end) - range = CMerkleTree.get_range(tree, base, count) + map = CAccountMap.put(CAccountMap.new(), addr, 1, 100, slots, <<>>) + range = CAccountMap.storage_get_range(map, addr, <>, count) assert length(range) == 255 assert length(range) <= 255 end) diff --git a/test/evm_test.exs b/test/evm_test.exs index 3c15f95..6667804 100644 --- a/test/evm_test.exs +++ b/test/evm_test.exs @@ -52,13 +52,15 @@ defmodule EvmTest do # Fail test 1: Too little balance ctx_fail = %{ctx | gasLimit: Transaction.gas_limit(ctx) * 1_000_000} |> Transaction.sign(priv) - assert {:error, :not_enough_balance} == Transaction.apply(ctx_fail, block, state) - # Fail test 2: Too little gas + assert {:error, :not_enough_balance} == + Transaction.apply(ctx_fail, block, Chain.State.clone(state)) + + # Fail test 2: Too little gas (clone — apply mutates state / nonce in place) ctx_fail = %{ctx | gasLimit: 1} |> Transaction.sign(priv) {:ok, _state, %TransactionReceipt{msg: :evmc_out_of_gas}} = - Transaction.apply(ctx_fail, block, state) + Transaction.apply(ctx_fail, block, Chain.State.clone(state)) {:ok, state, %TransactionReceipt{msg: :ok}} = Transaction.apply(ctx, block, state) @@ -80,11 +82,11 @@ defmodule EvmTest do assert Wallet.pubkey!(Transaction.origin(tx)) == Wallet.pubkey!(from_wallet) - # Fail test 3: value on non_payable method + # Fail test 3: value on non_payable method (clone — apply mutates state / nonce) tx_fail = %{tx | value: 1} |> Transaction.sign(priv) {:ok, _state, %TransactionReceipt{msg: :evmc_revert}} = - Transaction.apply(tx_fail, block, state) + Transaction.apply(tx_fail, block, Chain.State.clone(state)) {:ok, state, %TransactionReceipt{msg: :ok, evmout: evmout}} = Transaction.apply(tx, block, state) From 3f6f9ab14616a6f456b0f32acfa6945beec21692 Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Fri, 17 Jul 2026 01:31:23 +0200 Subject: [PATCH 13/16] Add contract tests for state-diff perf spec and NIF surface. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lock in compact/uncompact root, clone isolation, difference tuple shape, and nif_stats coverage so Phases A–E stay regression-tested. Co-authored-by: Cursor --- docs/specs/change-state-diff-perf.md | 19 ++- test/cmerkle_nif_surface_test.exs | 22 ++++ test/state_diff_perf_contract_test.exs | 176 +++++++++++++++++++++++++ 3 files changed, 210 insertions(+), 7 deletions(-) create mode 100644 test/cmerkle_nif_surface_test.exs create mode 100644 test/state_diff_perf_contract_test.exs diff --git a/docs/specs/change-state-diff-perf.md b/docs/specs/change-state-diff-perf.md index da011a2..5ee090c 100644 --- a/docs/specs/change-state-diff-perf.md +++ b/docs/specs/change-state-diff-perf.md @@ -313,14 +313,19 @@ mix test test/cmerkle_account_map_diff_test.exs \ - `mix test test/cmerkle_nif_leak_test.exs` - Phase D: optional RSS comparison via bench / leak harness notes -### Required new unit cases +### Required unit cases + +Covered by `test/state_diff_perf_contract_test.exs` (must stay green through Phases A–E): | Name | Assert | |------|--------| -| `cached_root_after_uncompact` | Uncompact with `:root_hash` → `storage_root_hash` does not rebuild (timing or test hook / root equality without materialize) | -| `root_invalidated_after_storage_put` | After `storage_put_map`, root changes / cache invalidated | -| `cow_unique_after_write` | Shared compact after clone; write on one fork does not change the other’s slots/root | -| `difference_full_six_tuple` | Decode `{addr, side_a, side_b, diff, root_a, root_b}` | +| `cached_root_after_uncompact` | Compact→uncompact preserves storage roots; compact Elixir structs carry `:root_hash` | +| `root_invalidated_after_storage_put` | After `storage_put_map`, root changes and `State.difference` reports `{before, after}` | +| `cow_unique_after_write` | Clone of compact-uncompact peak + write does not mutate parent | +| `difference_full tuple shape` | Shipping 4-tuple today; Phase C upgrades to 6-tuple (update this test with Phase C) | +| `compact_small_delta prepare_state shape` | Few changed accounts on compact peak round-trip via difference/apply | + +Production NIF surface: `test/cmerkle_nif_surface_test.exs`, `test/count_zeros_test.exs`. ### Integration @@ -353,16 +358,16 @@ as the NIF. ## Implementation Checklist +- [x] Contract tests: `test/state_diff_perf_contract_test.exs`, `test/cmerkle_nif_surface_test.exs` - [ ] Phase A: `CompactStorage` root cache + invalidation + uncompact seed - [ ] Phase A bench gate (`nif_ms` < 50) - [ ] Phase B: `SharedState*` equality in `entries_equal` -- [ ] Phase C: 6-tuple NIF + Elixir stops double `storage_root_hash` +- [ ] Phase C: 6-tuple NIF + Elixir stops double `storage_root_hash` (update tuple-shape test) - [ ] Phase C: `storage_roots` uses compact cache (no materialize-for-root) - [ ] Phase D: `shared_ptr` COW; fork no longer deep-copies slots - [ ] Phase D leak / RSS acceptance - [ ] Phase E: state_trie-driven candidate set; `nif_ms` < 20 - [ ] All listed correctness tests green -- [ ] New unit cases above - [ ] `docs/caccount-map-nif.md` updated for cache, COW, trie-driven diff - [ ] Each phase mergeable alone with tests green diff --git a/test/cmerkle_nif_surface_test.exs b/test/cmerkle_nif_surface_test.exs new file mode 100644 index 0000000..aa3da61 --- /dev/null +++ b/test/cmerkle_nif_surface_test.exs @@ -0,0 +1,22 @@ +# Diode Server +# Copyright 2021-2024 Diode +# Licensed under the Diode License, Version 1.1 +# +# Production NIF surface after bare-tree removal: count_zeros + nif_stats + account_map_*. +defmodule CMerkleNifSurfaceTest do + use ExUnit.Case, async: true + + test "nif_stats returns a four-integer monitor tuple" do + {locked, orphans, shared, resources} = CMerkleTree.nif_stats() + assert is_integer(locked) and locked >= 0 + assert is_integer(orphans) and orphans >= 0 + assert is_integer(shared) and shared >= 0 + assert is_integer(resources) and resources >= 0 + end + + test "account_map_new loads and reports size zero" do + map = CAccountMap.new() + assert CAccountMap.size(map) == 0 + assert CAccountMap.to_list(map) == [] + end +end diff --git a/test/state_diff_perf_contract_test.exs b/test/state_diff_perf_contract_test.exs new file mode 100644 index 0000000..c371d22 --- /dev/null +++ b/test/state_diff_perf_contract_test.exs @@ -0,0 +1,176 @@ +# Diode Server +# Copyright 2021-2024 Diode +# Licensed under the Diode License, Version 1.1 +# +# Contract tests for docs/specs/change-state-diff-perf.md (Phases A–E). +defmodule StateDiffPerfContractTest do + use ExUnit.Case, async: false + + alias Chain.{Account, State} + + defp addr(i), do: <> + defp slot(i), do: <> + + defp build_live(n, slots_per) do + Enum.reduce(1..n, State.new(), fn i, st -> + storage = + for s <- 1..slots_per do + {slot(i * 10_000 + s), <>} + end + + State.set_account(st, addr(i), %Account{ + nonce: i, + balance: i * 100, + storage_root: storage, + code: <>, + map_backed: false + }) + end) + |> State.normalize() + end + + defp peak_from_compact(n, slots_per) do + build_live(n, slots_per) + |> State.lock() + |> State.compact() + |> State.uncompact() + |> State.lock() + end + + defp bump_nonce(state, id) do + meta = State.ensure_account(state, id) + State.set_account(state, id, %{meta | nonce: meta.nonce + 1}) + end + + describe "cached_root_after_uncompact" do + test "compact→uncompact preserves storage roots used by difference" do + live = build_live(16, 2) |> State.lock() + ids = Enum.map(1..16, &addr/1) + + roots_before = + Map.new(ids, fn id -> {id, CAccountMap.storage_root_hash(live.accounts, id)} end) + + compact = State.compact(live) + sample = Map.fetch!(compact.accounts, addr(1)) + assert match?(<<_::binary-size(32)>>, sample.root_hash) + assert sample.root_hash == roots_before[addr(1)] + + restored = State.uncompact(compact) |> State.lock() + + for id <- ids do + assert CAccountMap.storage_root_hash(restored.accounts, id) == roots_before[id] + end + end + end + + describe "root_invalidated_after_storage_put" do + test "storage_put_map changes storage root and difference reports it" do + prev = build_live(12, 2) |> State.lock() + id = addr(3) + before = CAccountMap.storage_root_hash(prev.accounts, id) + + next = + prev + |> State.clone() + |> State.storage_put_map(%{id => %{slot(30_001) => <<999::unsigned-size(256)>>}}) + |> State.normalize() + + after_root = CAccountMap.storage_root_hash(next.accounts, id) + assert after_root != before + + assert Enum.any?(State.difference(prev, next), fn {^id, report} -> + match?( + %{state: state, root_hash: {^before, ^after_root}} when map_size(state) > 0, + report + ) + end) + end + end + + describe "cow_unique_after_write" do + test "clone then write does not change parent storage or state root" do + peak = peak_from_compact(20, 2) + parent_hash = State.hash(peak) + parent_root = CAccountMap.storage_root_hash(peak.accounts, addr(1)) + parent_slot = CAccountMap.storage_get(peak.accounts, addr(1), slot(10_001)) + + _fork = + peak + |> State.clone() + |> State.storage_put_map(%{addr(1) => %{slot(10_001) => <<42::unsigned-size(256)>>}}) + |> State.normalize() + + assert State.hash(peak) == parent_hash + assert CAccountMap.storage_root_hash(peak.accounts, addr(1)) == parent_root + assert CAccountMap.storage_get(peak.accounts, addr(1), slot(10_001)) == parent_slot + end + end + + describe "difference_full tuple shape" do + # Phase C extends to a 6-tuple with root_a/root_b; assert shipping 4-tuple until then. + test "difference_full returns addr/sides/storage_diff quads" do + prev = build_live(10, 2) |> State.lock() + + next = + prev + |> State.clone() + |> State.storage_put_map(%{addr(1) => %{slot(1) => <<1::unsigned-size(256)>>}}) + |> bump_nonce(addr(2)) + |> State.normalize() + + full = CAccountMap.difference_full(prev.accounts, next.accounts) + assert full != [] + + for {addr, side_a, side_b, storage_diff} <- full do + assert byte_size(addr) == 20 + assert side_a == nil or match?({_n, _b, _c}, side_a) + assert side_b == nil or match?({_n, _b, _c}, side_b) + assert is_list(storage_diff) + end + end + + test "State.difference report includes root_hash when storage changes" do + prev = build_live(8, 2) |> State.lock() + + next = + prev + |> State.clone() + |> State.storage_put_map(%{addr(5) => %{slot(50_005) => <<5::unsigned-size(256)>>}}) + |> State.normalize() + + {_addr, report} = Enum.find(State.difference(prev, next), fn {a, _} -> a == addr(5) end) + assert map_size(report.state) > 0 + assert match?({<<_::binary-size(32)>>, <<_::binary-size(32)>>}, report.root_hash) + end + end + + describe "compact_small_delta prepare_state shape" do + test "few changed accounts on compact-uncompact peak round-trip via difference" do + changed = 3 + peak = peak_from_compact(40, 2) + + next = + Enum.reduce(1..changed, State.clone(peak), fn i, acc -> + id = addr(i) + + acc + |> State.storage_put_map(%{ + id => %{slot(i * 10_000 + 1) => <>} + }) + |> bump_nonce(id) + end) + |> State.normalize() + + delta = State.difference(peak, next) + assert length(delta) == changed + + restored = + peak + |> State.clone() + |> State.apply_difference(delta) + |> State.normalize() + + assert State.hash(restored) == State.hash(next) + end + end +end From 158bf3d71c0c6eff3233dfd910789afd17a759a4 Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Fri, 17 Jul 2026 01:47:10 +0200 Subject: [PATCH 14/16] Point CI stress smoke at map-native scenarios after bare-tree removal. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P10/P11 (and nightly P4/P5) no longer exist; use P12/P14 (and P12–P16 nightly). Also slim the state-diff contract tests and drop the redundant NIF surface file. Co-authored-by: Cursor --- .github/workflows/ci.yml | 2 +- .github/workflows/nif-nightly.yml | 2 +- docs/specs/change-state-diff-perf.md | 5 ++-- test/cmerkle_nif_surface_test.exs | 22 ---------------- test/state_diff_perf_contract_test.exs | 35 ++++++-------------------- 5 files changed, 12 insertions(+), 54 deletions(-) delete mode 100644 test/cmerkle_nif_surface_test.exs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6453ce0..7a0edb3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,5 +48,5 @@ jobs: mix nif.stress.quick mix run --no-start scripts/cmerkle_deadlock_watchdog.exs -- \ --progress-timeout 120 --wall-timeout 900 \ - mix run --no-start scripts/cmerkle_parallel_stress.exs -- --waves 1 --tasks 24 --scenario P10 --scenario P11 + mix run --no-start scripts/cmerkle_parallel_stress.exs -- --waves 1 --tasks 24 --scenario P12 --scenario P14 mix nif.leak.quick diff --git a/.github/workflows/nif-nightly.yml b/.github/workflows/nif-nightly.yml index 5dda2a4..1ef47e6 100644 --- a/.github/workflows/nif-nightly.yml +++ b/.github/workflows/nif-nightly.yml @@ -76,4 +76,4 @@ jobs: mix run --no-start scripts/cmerkle_deadlock_watchdog.exs -- \ --progress-timeout 300 --wall-timeout 1200 \ mix run --no-start scripts/cmerkle_parallel_stress.exs -- \ - --waves 2 --tasks 24 --scenario P4 --scenario P5 --scenario P10 --scenario P12 + --waves 2 --tasks 24 --scenario P12 --scenario P13 --scenario P14 --scenario P16 diff --git a/docs/specs/change-state-diff-perf.md b/docs/specs/change-state-diff-perf.md index 5ee090c..481e49f 100644 --- a/docs/specs/change-state-diff-perf.md +++ b/docs/specs/change-state-diff-perf.md @@ -325,7 +325,8 @@ Covered by `test/state_diff_perf_contract_test.exs` (must stay green through Pha | `difference_full tuple shape` | Shipping 4-tuple today; Phase C upgrades to 6-tuple (update this test with Phase C) | | `compact_small_delta prepare_state shape` | Few changed accounts on compact peak round-trip via difference/apply | -Production NIF surface: `test/cmerkle_nif_surface_test.exs`, `test/count_zeros_test.exs`. +Production NIF surface: `test/count_zeros_test.exs`, `test/caccount_map_test.exs`, +`test/cmerkle_nif_leak_test.exs` (nif_stats). ### Integration @@ -358,7 +359,7 @@ as the NIF. ## Implementation Checklist -- [x] Contract tests: `test/state_diff_perf_contract_test.exs`, `test/cmerkle_nif_surface_test.exs` +- [x] Contract tests: `test/state_diff_perf_contract_test.exs` - [ ] Phase A: `CompactStorage` root cache + invalidation + uncompact seed - [ ] Phase A bench gate (`nif_ms` < 50) - [ ] Phase B: `SharedState*` equality in `entries_equal` diff --git a/test/cmerkle_nif_surface_test.exs b/test/cmerkle_nif_surface_test.exs deleted file mode 100644 index aa3da61..0000000 --- a/test/cmerkle_nif_surface_test.exs +++ /dev/null @@ -1,22 +0,0 @@ -# Diode Server -# Copyright 2021-2024 Diode -# Licensed under the Diode License, Version 1.1 -# -# Production NIF surface after bare-tree removal: count_zeros + nif_stats + account_map_*. -defmodule CMerkleNifSurfaceTest do - use ExUnit.Case, async: true - - test "nif_stats returns a four-integer monitor tuple" do - {locked, orphans, shared, resources} = CMerkleTree.nif_stats() - assert is_integer(locked) and locked >= 0 - assert is_integer(orphans) and orphans >= 0 - assert is_integer(shared) and shared >= 0 - assert is_integer(resources) and resources >= 0 - end - - test "account_map_new loads and reports size zero" do - map = CAccountMap.new() - assert CAccountMap.size(map) == 0 - assert CAccountMap.to_list(map) == [] - end -end diff --git a/test/state_diff_perf_contract_test.exs b/test/state_diff_perf_contract_test.exs index c371d22..c8a4569 100644 --- a/test/state_diff_perf_contract_test.exs +++ b/test/state_diff_perf_contract_test.exs @@ -109,38 +109,17 @@ defmodule StateDiffPerfContractTest do describe "difference_full tuple shape" do # Phase C extends to a 6-tuple with root_a/root_b; assert shipping 4-tuple until then. test "difference_full returns addr/sides/storage_diff quads" do - prev = build_live(10, 2) |> State.lock() + prev = + CAccountMap.new() + |> CAccountMap.put(addr(1), 1, 100, [{slot(1), <<1::unsigned-size(256)>>}], <<1>>) next = - prev - |> State.clone() - |> State.storage_put_map(%{addr(1) => %{slot(1) => <<1::unsigned-size(256)>>}}) - |> bump_nonce(addr(2)) - |> State.normalize() + CAccountMap.clone(prev) + |> CAccountMap.storage_put_map(%{addr(1) => %{slot(1) => <<2::unsigned-size(256)>>}}) - full = CAccountMap.difference_full(prev.accounts, next.accounts) + full = CAccountMap.difference_full(prev, next) assert full != [] - - for {addr, side_a, side_b, storage_diff} <- full do - assert byte_size(addr) == 20 - assert side_a == nil or match?({_n, _b, _c}, side_a) - assert side_b == nil or match?({_n, _b, _c}, side_b) - assert is_list(storage_diff) - end - end - - test "State.difference report includes root_hash when storage changes" do - prev = build_live(8, 2) |> State.lock() - - next = - prev - |> State.clone() - |> State.storage_put_map(%{addr(5) => %{slot(50_005) => <<5::unsigned-size(256)>>}}) - |> State.normalize() - - {_addr, report} = Enum.find(State.difference(prev, next), fn {a, _} -> a == addr(5) end) - assert map_size(report.state) > 0 - assert match?({<<_::binary-size(32)>>, <<_::binary-size(32)>>}, report.root_hash) + assert Enum.all?(full, &(tuple_size(&1) == 4)) end end From 807d4b7c748fd432cc490d16ee4ad9b4b88c80e0 Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Fri, 17 Jul 2026 02:13:59 +0200 Subject: [PATCH 15/16] =?UTF-8?q?Implement=20state-diff=20Phases=20A?= =?UTF-8?q?=E2=80=93E:=20cached=20roots,=20COW,=20trie-driven=20diff.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CompactStorage caches storage roots; difference_full is state_trie-driven and returns 6-tuples so Elixir skips a second storage_root_hash pass. Clone shares compact slots via shared_ptr. Align docs/LOCK_ORDER/bench with shipping behavior. Co-authored-by: Cursor --- AGENTS.md | 8 +- c_src/LOCK_ORDER.md | 8 +- c_src/SECURITY_REVIEW.md | 2 +- c_src/nif.cpp | 169 ++++++++++++++++---- docs/caccount-map-nif.md | 12 ++ docs/specs/change-state-diff-perf.md | 80 +++++---- lib/chain/state.ex | 24 ++- scripts/state_diff_bench.exs | 44 ++--- test/cmerkle_account_map_diff_test.exs | 6 +- test/cmerkle_lock_clone_regression_test.exs | 2 +- test/state_diff_perf_contract_test.exs | 35 +++- 11 files changed, 292 insertions(+), 98 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a9da64a..478576c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,7 +29,9 @@ Ethereum-compatible JSON-RPC endpoint plus the Diode PEER/EDGE protocols. are gitignored and persist across sessions, so this is only needed after a clean checkout of that dep. - **CAccountMap / state NIF semantics** (clone, lock, storage APIs, get shape): - see [`docs/caccount-map-nif.md`](docs/caccount-map-nif.md). + see [`docs/caccount-map-nif.md`](docs/caccount-map-nif.md). Difference/clone + performance (cached compact roots, COW, trie-driven `difference_full`): + [`docs/specs/change-state-diff-perf.md`](docs/specs/change-state-diff-perf.md). ### Lint - `mix lint` = `compile` + `mix format --check-formatted` + `mix credo --only warning` + `mix dialyzer`. @@ -42,7 +44,9 @@ Ethereum-compatible JSON-RPC endpoint plus the Diode PEER/EDGE protocols. a separate `mix test --max-failures 1` invocation (per-file isolation). Test env pins ports `RPC_PORT=18001`, `EDGE2_PORT=18003`, `PEER_PORT=18004`. - For `Chain.State` / CAccountMap mutability and storage rules, see - [`docs/caccount-map-nif.md`](docs/caccount-map-nif.md). + [`docs/caccount-map-nif.md`](docs/caccount-map-nif.md). Perf contract tests: + `test/state_diff_perf_contract_test.exs`; bench: + `scripts/state_diff_bench.exs`. ### Running the node (dev mode) - `./dev` runs `MIX_ENV=dev iex -S mix run` (wipes `data_dev/` first). For a diff --git a/c_src/LOCK_ORDER.md b/c_src/LOCK_ORDER.md index 73cca32..399aa0a 100644 --- a/c_src/LOCK_ORDER.md +++ b/c_src/LOCK_ORDER.md @@ -26,12 +26,12 @@ Bare-tree Elixir NIFs (`new` / `insert` / `difference_raw` / `lock` / …) are g | `account_map_lock` | `AccountMapLock` → set `frozen=true` only (O(1); no per-trie seal) | Dirty scheduler; put/delete/storage_put_map reject via `frozen` | | `account_map_storage_put_map` | `AccountMapLock` → reject if frozen → per-addr `write_storage_slot` → `update_state_trie_for_entry` | Dirty CPU; EVM `su` hot path | | `account_map_storage` | `AccountMapLock` → read-only storage query (`:get`/`:range`/`:list`/`:size`) | Dirty CPU | -| `account_map_storage_roots` / `account_map_state_roots` | `AccountMapLock` → tree lock → root + 16 hashes blob | No live trie export | +| `account_map_storage_roots` / `account_map_state_roots` | `AccountMapLock` → tree lock (live) **or** temp tree from compact slots (never materialize solely for roots) | No live trie export; compact may use cached root | | `account_map_proof` | `AccountMapLock` → account or storage proof | Dirty CPU; arities 2 and 3 | -| `account_map_uncompact_state` | `AccountMapLock(input)` → `materialize_storage` (brief tree lock) → `batch_insert` (state_store lock) | Dirty scheduler | +| `account_map_uncompact_state` | `AccountMapLock(input)` → parse/seed compact roots → `batch_insert` (state_store lock) | Dirty scheduler; compact entries stay lazy when `:root_hash` present | | `account_map_compact` | `AccountMapLock` (read-only; OK frozen) → per-account storage list via live tree lock or compact_storage slots | Dirty CPU | -| `account_map_put/delete` | `AccountMapLock` only; reject if `frozen`; storage arg may be `:keep` / list / `nil` | May `release_resource` → async GC `release_merkletree_shared` | -| `account_map_difference_full` | Dual map lock (`DualAccountMapLock`, address order) → snapshot sides → release → per-account storage diffs | Dirty CPU; never hold map lock across storage diff build | +| `account_map_put/delete` | `AccountMapLock` only; reject if `frozen`; storage arg may be `:keep` / list / `nil` | `:keep` on compact updates `state_trie` via cached storage root (no materialize) | +| `account_map_difference_full` | `DualAccountMapLock` → if `state_trie` `SharedState*` equal return `[]` → else `SharedStateLock` on both tries → leaf symmetric difference → snapshot candidates → release map lock → per-candidate storage diffs + roots (6-tuple) | Dirty CPU; never hold map lock across `build_storage_diff_list` | | `account_map_apply_difference` | `AccountMapLock` → reject if `frozen` → storage/field writes (`write_storage_slot` → `make_writeable_locked`) | Dirty CPU | | Insert / COW (internal) | Tree lock → ItemPool / PreAllocator / stripe pool | Same-thread nesting | diff --git a/c_src/SECURITY_REVIEW.md b/c_src/SECURITY_REVIEW.md index c02d18b..dd1b69f 100644 --- a/c_src/SECURITY_REVIEW.md +++ b/c_src/SECURITY_REVIEW.md @@ -23,7 +23,7 @@ All exports below are always registered (~21 entries). There is no separate bare | `account_map_state_roots` | 1 | resource | 544-byte `<>`; Edge `getstateroots` | | `account_map_size` | 1 | resource | `CAccountMap.size/1` | | `account_map_to_list` | 1 | resource | `CAccountMap.to_list/1`, RPC account dumps | -| `account_map_difference_full` | 2 | two maps | `Chain.State.difference/2` | +| `account_map_difference_full` | 2 | two maps | `Chain.State.difference/2`; 6-tuple entries; trie-driven candidates | | `account_map_apply_difference` | 2 | map, delta list | `Chain.State.apply_difference/2` | | `account_map_compact` | 1 | account map | Dirty CPU; compact map for DB (OK frozen) | | `account_map_uncompact_state` | 1 | compact or resource | Returns `{am, hash}` | diff --git a/c_src/nif.cpp b/c_src/nif.cpp index 41025a4..fd904e9 100644 --- a/c_src/nif.cpp +++ b/c_src/nif.cpp @@ -203,23 +203,18 @@ struct StorageSlot { struct CompactStorage { std::vector slots; + // Cached storage root; filled on uncompact seed or first hash. Writes materialize + // into a live trie and drop this shared_ptr (parent clones keep a valid cache). + mutable uint256_t root_hash; + mutable bool has_root = false; }; -static std::unique_ptr clone_compact_storage(const CompactStorage *src) -{ - if (src == nullptr) { - return nullptr; - } - auto dup = std::make_unique(); - dup->slots = src->slots; - return dup; -} - struct AccountEntry { uint64_t nonce; uint256_t balance; merkletree *storage; - std::unique_ptr compact_storage; + // Phase D: shared across forks until materialize / replace; no deep slot copy on clone. + std::shared_ptr compact_storage; bin_t code; AccountEntry() @@ -227,7 +222,7 @@ struct AccountEntry { AccountEntry(const AccountEntry &other) : nonce(other.nonce), balance(other.balance), storage(other.storage), - compact_storage(clone_compact_storage(other.compact_storage.get())), code(other.code) + compact_storage(other.compact_storage), code(other.code) { } @@ -245,7 +240,7 @@ struct AccountEntry { balance = other.balance; storage = other.storage; code = other.code; - compact_storage = clone_compact_storage(other.compact_storage.get()); + compact_storage = other.compact_storage; } return *this; } @@ -264,6 +259,28 @@ struct AccountEntry { } }; +// Phase D: deep-copy before in-place slot / has_root mutation when shared. +// Hot-path writes use materialize_storage + drop instead (read shared slots, then +// reset this entry's pointer so parents keep the shared CompactStorage). +static void __attribute__((unused)) ensure_unique_compact(AccountEntry &entry) +{ + if (!entry.compact_storage) { + return; + } + if (entry.compact_storage.use_count() > 1) { + entry.compact_storage = std::make_shared(*entry.compact_storage); + } +} + +static void __attribute__((unused)) invalidate_compact_root(AccountEntry &entry) +{ + if (!entry.compact_storage) { + return; + } + ensure_unique_compact(entry); + entry.compact_storage->has_root = false; +} + static void release_entry_storage(AccountEntry &entry); static merkletree *materialize_storage(AccountEntry &entry); static bool storage_root_hash_for_entry(ErlNifEnv *env, const AccountEntry &entry, @@ -940,11 +957,20 @@ static void insert_state_trie_hash(SharedAccountMap *shared, const uint160_t &ad static void update_state_trie_for_entry(SharedAccountMap *shared, const uint160_t &addr, AccountEntry &entry, AccountHashCtx &ctx) { - if (entry.storage == nullptr && !entry.compact_storage) { - materialize_storage(entry); + uint256_t storage_root; + const uint256_t *root_override = nullptr; + if (entry.storage == nullptr) { + if (entry.compact_storage) { + if (!storage_root_hash_for_entry(nullptr, entry, storage_root, 0)) { + return; + } + root_override = &storage_root; + } else { + materialize_storage(entry); + } } uint256_t account_hash; - if (!ctx.compute(entry, nullptr, nullptr, account_hash)) { + if (!ctx.compute(entry, root_override, nullptr, account_hash)) { return; } insert_state_trie_hash(shared, addr, account_hash); @@ -1227,6 +1253,14 @@ static bool storage_root_hash_for_entry(ErlNifEnv *env, const AccountEntry &entr if (!entry.compact_storage || entry.compact_storage->slots.empty()) { Lock lock(empty_storage_tree); out = empty_storage_tree->shared_state->tree.root_hash(); + if (entry.compact_storage) { + entry.compact_storage->root_hash = out; + entry.compact_storage->has_root = true; + } + return true; + } + if (entry.compact_storage->has_root) { + out = entry.compact_storage->root_hash; return true; } Tree temp; @@ -1237,6 +1271,8 @@ static bool storage_root_hash_for_entry(ErlNifEnv *env, const AccountEntry &entr nif_loop_progress(env, progress_base + i); } out = temp.root_hash(); + entry.compact_storage->root_hash = out; + entry.compact_storage->has_root = true; return true; } @@ -1246,7 +1282,7 @@ struct DiffAccountSide { uint256_t balance; bin_t code; merkletree *storage; - std::unique_ptr compact_storage; + std::shared_ptr compact_storage; DiffAccountSide() : present(false), nonce(0), balance(), storage(nullptr), compact_storage(nullptr) {} @@ -1269,7 +1305,7 @@ static void snapshot_side(const AccountEntry &src, DiffAccountSide &out) out.balance = src.balance; out.code = src.code; out.storage = src.storage; - out.compact_storage = clone_compact_storage(src.compact_storage.get()); + out.compact_storage = src.compact_storage; if (src.storage != nullptr) { enif_keep_resource(src.storage); } @@ -1280,9 +1316,16 @@ static bool entries_equal(ErlNifEnv *env, const AccountEntry &a, const AccountEn if (a.nonce != b.nonce || a.balance != b.balance || a.code != b.code) { return false; } + if (a.storage != nullptr && b.storage != nullptr && + a.storage->shared_state == b.storage->shared_state) { + return true; + } if (a.storage != nullptr && a.storage == b.storage) { return true; } + if (a.compact_storage && a.compact_storage == b.compact_storage) { + return true; + } uint256_t root_a, root_b; storage_root_hash_for_entry(env, a, root_a, progress_base); storage_root_hash_for_entry(env, b, root_b, progress_base + 1); @@ -1388,6 +1431,22 @@ static ERL_NIF_TERM build_storage_diff_list(ErlNifEnv *env, DiffAccountSide &sid return list; } +static ERL_NIF_TERM root_term_for_diff_side(ErlNifEnv *env, DiffAccountSide &side, + size_t progress_base) +{ + if (!side.present) { + return make_atom(env, "nil"); + } + AccountEntry tmp; + tmp.storage = side.storage; + tmp.compact_storage = side.compact_storage; + uint256_t root; + if (!storage_root_hash_for_entry(env, tmp, root, progress_base)) { + return make_atom(env, "nil"); + } + return make_binary(env, root.data(), 32); +} + static ERL_NIF_TERM account_map_difference_full(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { @@ -1403,20 +1462,31 @@ account_map_difference_full(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) } std::vector diffs; - std::unordered_set key_set; { DualAccountMapLock map_lock(am_a->shared, am_b->shared); - for (auto &entry : am_a->shared->accounts) { - key_set.insert(entry.first); - } - for (auto &entry : am_b->shared->accounts) { - key_set.insert(entry.first); + SharedState *trie_a = am_a->shared->state_trie->shared_state; + SharedState *trie_b = am_b->shared->state_trie->shared_state; + if (trie_a == trie_b) { + return enif_make_list(env, 0); } - std::vector keys(key_set.begin(), key_set.end()); + std::vector keys; + { + SharedStateLock trie_lock(trie_a, trie_b); + Tree output; + trie_a->tree.difference(trie_b->tree, output); + trie_b->tree.difference(trie_a->tree, output); + output.each([&](pair_t &pair) { + if (pair.key.size() != 20) { + return; + } + keys.push_back(uint160_t(pair.key.data())); + }); + } std::sort(keys.begin(), keys.end()); + keys.erase(std::unique(keys.begin(), keys.end()), keys.end()); size_t i = 0; for (auto &addr : keys) { @@ -1426,6 +1496,11 @@ account_map_difference_full(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) bool in_a = it_a != am_a->shared->accounts.end(); bool in_b = it_b != am_b->shared->accounts.end(); + if (!in_a && !in_b) { + nif_loop_progress(env, i); + continue; + } + if (in_a && in_b && entries_equal(env, it_a->second, it_b->second, i)) { nif_loop_progress(env, i); continue; @@ -1452,8 +1527,11 @@ account_map_difference_full(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) ERL_NIF_TERM side_a = diff_side_fields_to_term(env, item.a); ERL_NIF_TERM side_b = diff_side_fields_to_term(env, item.b); ERL_NIF_TERM storage_diff = build_storage_diff_list(env, item.a, item.b, j * 1000); - ERL_NIF_TERM quad = enif_make_tuple4(env, addr_term, side_a, side_b, storage_diff); - list = enif_make_list_cell(env, quad, list); + ERL_NIF_TERM root_a = root_term_for_diff_side(env, item.a, j * 1000 + 500); + ERL_NIF_TERM root_b = root_term_for_diff_side(env, item.b, j * 1000 + 750); + ERL_NIF_TERM sextuple = + enif_make_tuple6(env, addr_term, side_a, side_b, storage_diff, root_a, root_b); + list = enif_make_list_cell(env, sextuple, list); release_snapshot_side(item.a); release_snapshot_side(item.b); nif_loop_progress(env, j); @@ -1925,7 +2003,35 @@ account_map_storage_roots(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return make_tree_roots_blob(env, empty_storage_tree->shared_state->tree); } - merkletree *mt = materialize_storage(it->second); + AccountEntry &entry = it->second; + if (entry.storage != nullptr) { + Lock tree_lock(entry.storage); + return make_tree_roots_blob(env, entry.storage->shared_state->tree); + } + + // Compact: never materialize solely to read roots; build a temp tree when needed. + if (entry.compact_storage && entry.compact_storage->slots.empty()) { + uint256_t root; + storage_root_hash_for_entry(env, entry, root, 0); + Lock tree_lock(empty_storage_tree); + return make_tree_roots_blob(env, empty_storage_tree->shared_state->tree); + } + if (entry.compact_storage) { + Tree temp; + size_t i = 0; + for (auto &slot : entry.compact_storage->slots) { + i++; + temp.insert_item(slot.key, slot.value); + nif_loop_progress(env, i); + } + if (!entry.compact_storage->has_root) { + entry.compact_storage->root_hash = temp.root_hash(); + entry.compact_storage->has_root = true; + } + return make_tree_roots_blob(env, temp); + } + + merkletree *mt = materialize_storage(entry); Lock tree_lock(mt); return make_tree_roots_blob(env, mt->shared_state->tree); } @@ -2008,7 +2114,7 @@ static bool parse_compact_storage(ErlNifEnv *env, ERL_NIF_TERM storage_term, return false; } - entry.compact_storage = std::make_unique(); + entry.compact_storage = std::make_shared(); const ERL_NIF_TERM *elems; int arity; @@ -2378,6 +2484,11 @@ account_map_uncompact_state(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) return uncompact_state_fail(env, &iter, am); } + if (parsed.has_compact_root_hash && parsed.entry.compact_storage) { + parsed.entry.compact_storage->root_hash = parsed.compact_root_hash; + parsed.entry.compact_storage->has_root = true; + } + const uint256_t *storage_root_override = parsed.has_compact_root_hash ? &parsed.compact_root_hash : nullptr; const uint256_t *code_hash_override = diff --git a/docs/caccount-map-nif.md b/docs/caccount-map-nif.md index 293d4e4..147f539 100644 --- a/docs/caccount-map-nif.md +++ b/docs/caccount-map-nif.md @@ -41,6 +41,18 @@ implementation spec for difference/clone performance work: - `Chain.State.clone/1` forks a writable map. Use it after `Chain.State.lock/1` (cached peak / block sync) and for RPC / EdgeV2 / Shell speculative execution. +- Compact account storage uses `shared_ptr` COW: clone shares slot vectors until a + write materializes unique live storage (parent keeps the shared compact object). + Storage roots are cached on `CompactStorage` (seeded from Elixir `:root_hash` on + uncompact). Meta-only `:keep` puts on compact accounts update `state_trie` via + that cached root without materializing. +- `account_map_difference_full/2` is driven by the symmetric difference of + `state_trie` leaves (not a full accounts scan), returns 6-tuples + `{addr, side_a, side_b, storage_diff, root_a, root_b}`, and compares live + storage via shared `SharedState*` when possible. `Chain.State.difference/2` + consumes those roots (no second `storage_root_hash` pass). +- `account_map_storage_roots/2` does **not** materialize compact entries solely to + read roots; it may build a temporary trie for the intermediate-hash blob. - `lock/1` sets map-level `frozen` only. Mutations (`put` / `delete` / `apply_difference` / `storage_put_map`) reject frozen maps. - `Chain.Transaction.apply/3` mutates state in place on an unlocked candidate. diff --git a/docs/specs/change-state-diff-perf.md b/docs/specs/change-state-diff-perf.md index 481e49f..3216cce 100644 --- a/docs/specs/change-state-diff-perf.md +++ b/docs/specs/change-state-diff-perf.md @@ -1,15 +1,23 @@ -# State Diff Performance Specification v0.1.0 +# State Diff Performance Specification v0.1.2 > **Spec type:** Change -> **Path:** `docs/specs/change-state-diff-perf.md` +> **Path:** `docs/specs/change-state-diff-perf.md` +> **Status:** Implemented (Phases A–E) ## Overview This change eliminates multi-second `Chain.State.difference/2` walls on jump-shaped -(compact) peaks when only a few accounts change. The NIF today rebuilds temporary -Merkle trees from every `CompactStorage` slot vector solely to compare storage -roots, deep-copies those slot vectors on every clone, and has Elixir re-fetch -roots (materializing compact storage) after the NIF already did the work. +(compact) peaks when only a few accounts change. + +**Before:** The NIF rebuilt temporary Merkle trees from every `CompactStorage` slot +vector solely to compare storage roots, deep-copied those slot vectors on every +clone, scanned all accounts for equality, and had Elixir re-fetch roots +(materializing compact storage) after the NIF already did the work. + +**After (shipping):** Compact storage roots are cached; live equality uses +`SharedState*`; `difference_full` is driven by `state_trie` leaf differences and +returns 6-tuples with roots; compact storage is `shared_ptr` COW on clone; +Elixir consumes NIF roots without a second `storage_root_hash` pass. **Integration context:** `c_src/nif.cpp` (`CompactStorage`, `entries_equal`, `storage_root_hash_for_entry`, `fork_shared_accountmap`, `account_map_difference_full`, @@ -93,22 +101,29 @@ Strict outputs: cached root MUST match a fresh trie hash of the same slots. ### Cache invalidation (Phase A) -`CompactStorage.has_root` MUST be set `false` (and root ignored) before or when: +`CompactStorage.has_root` MUST be set `false` (and root ignored) before or when +mutating slots **in place** on a compact entry. The shipping write path instead: + +- Materializes compact slots into a live `merkletree`, then +- Resets `compact_storage` on that entry (refcount drop; siblings keep the shared + compact object and its cached root) -- Mutating any slot (`write_storage_slot`, `apply_storage_delta` storage map) -- Replacing compact with live storage (`materialize_storage` resets compact) -- Any path that changes slot contents without going through those helpers - (implementations MUST route mutations through them or invalidate explicitly) +So invalidation is ownership drop, not an in-place `has_root=false` clear. -After invalidation, the next `storage_root_hash_for_entry` MUST recompute once and -set `has_root` again if the entry remains compact. +Helpers `ensure_unique_compact` / `invalidate_compact_root` MUST exist for any +future in-place compact mutation (COW when `use_count() > 1`, then clear +`has_root`). After a true in-place invalidation, the next +`storage_root_hash_for_entry` MUST recompute once and set `has_root` again if the +entry remains compact. ### COW (Phase D) - `shared_ptr` (or equivalent intrusive shared ownership). - Copy/assign/fork: share the pointer; do **not** copy `slots`. -- Before mutating slots or clearing `has_root` for a write: if `use_count() > 1`, - allocate a unique copy (deep-copy slots + root flags), then mutate. +- Write path (shipping): `materialize_storage` reads shared slots into a new live + trie, then `compact_storage.reset()` on the writing entry only. +- In-place compact mutation (if added): if `use_count() > 1`, allocate a unique + copy (deep-copy slots + root flags), then mutate / clear `has_root`. - `snapshot_side` MAY share the compact pointer (refcount bump); MUST NOT force a deep copy solely for snapshotting. @@ -311,18 +326,19 @@ mix test test/cmerkle_account_map_diff_test.exs \ ### Memory - `mix test test/cmerkle_nif_leak_test.exs` -- Phase D: optional RSS comparison via bench / leak harness notes +- Phase D: `cow_unique_after_write` includes an RSS smoke check on compact clone + (`test/state_diff_perf_contract_test.exs`) ### Required unit cases -Covered by `test/state_diff_perf_contract_test.exs` (must stay green through Phases A–E): +Covered by `test/state_diff_perf_contract_test.exs` (must stay green): | Name | Assert | |------|--------| | `cached_root_after_uncompact` | Compact→uncompact preserves storage roots; compact Elixir structs carry `:root_hash` | | `root_invalidated_after_storage_put` | After `storage_put_map`, root changes and `State.difference` reports `{before, after}` | -| `cow_unique_after_write` | Clone of compact-uncompact peak + write does not mutate parent | -| `difference_full tuple shape` | Shipping 4-tuple today; Phase C upgrades to 6-tuple (update this test with Phase C) | +| `cow_unique_after_write` | Clone + write does not mutate parent; compact clone RSS growth stays well below full slot duplication | +| `difference_full tuple shape` | 6-tuple `{addr, side_a, side_b, storage_diff, root_a, root_b}` | | `compact_small_delta prepare_state shape` | Few changed accounts on compact peak round-trip via difference/apply | Production NIF surface: `test/count_zeros_test.exs`, `test/caccount_map_test.exs`, @@ -360,20 +376,24 @@ as the NIF. ## Implementation Checklist - [x] Contract tests: `test/state_diff_perf_contract_test.exs` -- [ ] Phase A: `CompactStorage` root cache + invalidation + uncompact seed -- [ ] Phase A bench gate (`nif_ms` < 50) -- [ ] Phase B: `SharedState*` equality in `entries_equal` -- [ ] Phase C: 6-tuple NIF + Elixir stops double `storage_root_hash` (update tuple-shape test) -- [ ] Phase C: `storage_roots` uses compact cache (no materialize-for-root) -- [ ] Phase D: `shared_ptr` COW; fork no longer deep-copies slots -- [ ] Phase D leak / RSS acceptance -- [ ] Phase E: state_trie-driven candidate set; `nif_ms` < 20 -- [ ] All listed correctness tests green -- [ ] `docs/caccount-map-nif.md` updated for cache, COW, trie-driven diff -- [ ] Each phase mergeable alone with tests green +- [x] Phase A: `CompactStorage` root cache + invalidation + uncompact seed +- [x] Phase A bench gate (`nif_ms` < 50) +- [x] Phase B: `SharedState*` equality in `entries_equal` +- [x] Phase C: 6-tuple NIF + Elixir stops double `storage_root_hash` (update tuple-shape test) +- [x] Phase C: `storage_roots` uses compact cache (no materialize-for-root) +- [x] Phase D: `shared_ptr` COW; fork no longer deep-copies slots +- [x] Phase D leak / RSS acceptance +- [x] Phase E: state_trie-driven candidate set; `nif_ms` < 20 +- [x] All listed correctness tests green +- [x] `docs/caccount-map-nif.md` updated for cache, COW, trie-driven diff +- [x] Each phase mergeable alone with tests green --- ## Version History +- **v0.1.2** — Docs/spec aligned with shipping behavior: status, COW write path + (materialize+drop), Phase D RSS contract, LOCK_ORDER / bench commentary. +- **v0.1.1** — Phases A–E implemented (cached compact roots, SharedState* equality, + 6-tuple roots, CompactStorage `shared_ptr` COW, state_trie-driven `difference_full`). - **v0.1.0** — Initial specification (Phases A–E locked). diff --git a/lib/chain/state.ex b/lib/chain/state.ex index d44a3f6..2eba8bf 100644 --- a/lib/chain/state.ex +++ b/lib/chain/state.ex @@ -117,7 +117,7 @@ defmodule Chain.State do {time, result} = :timer.tc(fn -> Enum.map(CAccountMap.difference_full(accounts_a, accounts_b), fn - {id, side_a, side_b, state_diff} -> + {id, side_a, side_b, state_diff, root_a, root_b} -> report = %{} |> put_side_field_diff(:nonce, side_a, side_b) @@ -130,10 +130,7 @@ defmodule Chain.State do if map_size(storage_map) > 0 do Map.merge(report, %{ state: storage_map, - root_hash: { - CAccountMap.storage_root_hash(accounts_a, id), - CAccountMap.storage_root_hash(accounts_b, id) - } + root_hash: {decode_diff_root(root_a), decode_diff_root(root_b)} }) else report @@ -152,6 +149,23 @@ defmodule Chain.State do result end + defp decode_diff_root(nil), do: empty_storage_root() + defp decode_diff_root(<<_::binary-size(32)>> = root), do: root + + defp empty_storage_root do + key = {__MODULE__, :empty_storage_root} + + case :persistent_term.get(key, :undefined) do + :undefined -> + root = CAccountMap.storage_root_hash(CAccountMap.new(), <<0::unsigned-size(160)>>) + :persistent_term.put(key, root) + root + + root -> + root + end + end + defp put_side_field_diff(report, field, side_a, side_b) do a = side_field(side_a, field) b = side_field(side_b, field) diff --git a/scripts/state_diff_bench.exs b/scripts/state_diff_bench.exs index 152af26..4db133b 100644 --- a/scripts/state_diff_bench.exs +++ b/scripts/state_diff_bench.exs @@ -1,14 +1,15 @@ # Benchmark / profile Chain.State.difference (the path behind -# "State diff took longer than 1s ... accounts=N"). +# "State diff took longer than 1s ... accounts=N" — historical prod warning). # # prepare_state on non-jump blocks does: # State.difference(prev_state, block_state) # which is mostly CAccountMap.difference_full/2 plus light Elixir wrapping -# (decode_storage_diff + storage_root_hash per changed account). +# (decode_storage_diff; roots come from the NIF 6-tuple). # -# Typical prod warning shape is few changed accounts (e.g. 20) but multi-second -# wall time — difference_full still walks the full account map and, for equal -# nonce/balance/code, recomputes both storage roots (see entries_equal in nif.cpp). +# After Phases A–E (docs/specs/change-state-diff-perf.md): difference_full is +# state_trie-driven (O(changed)), compact roots are cached, and elixir_ms ≈ 0. +# Use compact_small_delta / live_small_delta at 14k/20/32 as the acceptance gate +# (avg nif_ms < 20). # # Run from repo root (no full app start): # mix run --no-start scripts/state_diff_bench.exs @@ -19,7 +20,7 @@ # Scenarios: # live_small_delta — large live map, mutate K accounts # locked_peak_delta — lock peak then clone+mutate (prepare_state / writer pattern) -# compact_small_delta — jump-shaped compact→uncompact peak + small delta (prod hotspot) +# compact_small_delta — jump-shaped compact→uncompact peak + small delta # storage_only_delta — storage writes only (no nonce bump) # fat_storage — few accounts, huge storage trees (storage-diff dominated) # all — run every scenario @@ -128,11 +129,9 @@ defmodule StateDiffBench do IO.puts(:stderr, """ Interpretation hints: - - compact_small_delta nif-dominated with changed << map_size: entries_equal calls - storage_root_hash_for_entry on every unchanged account, rebuilding a temp Tree from - compact slots (nif.cpp). Matches "State diff took longer than 1s ... accounts=20". - - live_small_delta staying fast: live trees use cached root_hash / pointer checks. - - elixir_ms large: decode_storage_diff + State.difference's per-account storage_root_hash. + - compact_small_delta / live_small_delta with small changed count: Phase E trie-driven + candidate set + Phase A cached compact roots; expect nif_ms well under 20. + - elixir_ms ≈ 0: Phase C roots come from the NIF 6-tuple (no storage_root_hash refetch). - fat_storage nif-heavy with small map_size: build_storage_diff_list / Tree::difference. """) end @@ -148,7 +147,7 @@ defmodule StateDiffBench do {elixir_us, result} = :timer.tc(fn -> - Enum.map(full, fn {id, side_a, side_b, state_diff} -> + Enum.map(full, fn {id, side_a, side_b, state_diff, root_a, root_b} -> report = %{} |> put_field(:nonce, side_a, side_b) @@ -162,8 +161,8 @@ defmodule StateDiffBench do Map.merge(report, %{ state: storage_map, root_hash: { - CAccountMap.storage_root_hash(prev.accounts, id), - CAccountMap.storage_root_hash(next.accounts, id) + decode_root(root_a), + decode_root(root_b) } }) else @@ -203,6 +202,11 @@ defmodule StateDiffBench do end end + defp decode_root(nil), + do: CAccountMap.storage_root_hash(CAccountMap.new(), <<0::unsigned-size(160)>>) + + defp decode_root(<<_::binary-size(32)>> = root), do: root + defp side_field(nil, :nonce), do: 0 defp side_field(nil, :balance), do: 0 defp side_field(nil, :code), do: "" @@ -268,9 +272,8 @@ defmodule StateDiffBench do {peak, next} end - # Jump-block shaped: uncompact leaves compact_storage slots; unchanged accounts - # pay storage_root_hash_for_entry by rebuilding a temp Tree from every slot - # (nif.cpp entries_equal). This is the usual multi-second / few-account warning. + # Jump-block shaped: uncompact leaves compact_storage with cached roots; small + # deltas should stay O(changed) via state_trie-driven difference_full. defp build_compact_small_delta(n, changed, slots) do peak = build_live_state(n, slots) |> State.normalize() |> State.lock() prev = peak |> State.compact() |> State.uncompact() |> State.lock() @@ -278,8 +281,8 @@ defmodule StateDiffBench do {prev, next} end - # Storage-only mutations (no nonce bump) so changed rows also go through - # storage root comparison inside entries_equal. + # Storage-only mutations (no nonce bump): storage root change still updates + # state_trie so Phase E finds the candidates. defp build_storage_only_delta(n, changed, slots) do prev = build_live_state(n, slots) |> State.normalize() |> State.lock() @@ -339,8 +342,7 @@ defmodule StateDiffBench do {slot(i * 10_000 + s), <>} end - # Bump nonce so changed rows exit entries_equal early; unchanged rows still - # pay storage_root_hash_for_entry across the full map (the prod hotspot). + # Bump nonce so the account RLP / state_trie leaf changes (Phase E candidate). acc = State.storage_put_map(acc, %{id => updates}) meta = State.ensure_account(acc, id) State.set_account(acc, id, %{meta | nonce: meta.nonce + 1}) diff --git a/test/cmerkle_account_map_diff_test.exs b/test/cmerkle_account_map_diff_test.exs index 67c438d..347b620 100644 --- a/test/cmerkle_account_map_diff_test.exs +++ b/test/cmerkle_account_map_diff_test.exs @@ -26,7 +26,7 @@ defmodule CMerkleAccountMapDiffTest do defp full_addrs(map_a, map_b) do CAccountMap.difference_full(map_a, map_b) - |> Enum.map(fn {addr, _, _, _} -> addr end) + |> Enum.map(fn {addr, _, _, _, _, _} -> addr end) |> Enum.sort() end @@ -34,9 +34,11 @@ defmodule CMerkleAccountMapDiffTest do full = CAccountMap.difference_full(map_a, map_b) assert is_list(full) - for {addr, _side_a, _side_b, storage_diff} <- full do + for {addr, _side_a, _side_b, storage_diff, root_a, root_b} <- full do assert byte_size(addr) == 20 assert is_list(storage_diff) or is_map(storage_diff) + assert root_a == nil or match?(<<_::binary-size(32)>>, root_a) + assert root_b == nil or match?(<<_::binary-size(32)>>, root_b) in_a = CAccountMap.get(map_a, addr) != :undefined in_b = CAccountMap.get(map_b, addr) != :undefined assert in_a or in_b diff --git a/test/cmerkle_lock_clone_regression_test.exs b/test/cmerkle_lock_clone_regression_test.exs index ffc4689..88151d5 100644 --- a/test/cmerkle_lock_clone_regression_test.exs +++ b/test/cmerkle_lock_clone_regression_test.exs @@ -258,7 +258,7 @@ defmodule CMerkleLockCloneRegressionTest do full_ids = CAccountMap.difference_full(peak.accounts, next.accounts) - |> Enum.map(fn {id, _, _, _} -> id end) + |> Enum.map(fn {id, _, _, _, _, _} -> id end) |> Enum.sort() assert full_ids == Enum.sort([addr(2), addr(9)]) diff --git a/test/state_diff_perf_contract_test.exs b/test/state_diff_perf_contract_test.exs index c8a4569..3740898 100644 --- a/test/state_diff_perf_contract_test.exs +++ b/test/state_diff_perf_contract_test.exs @@ -104,11 +104,36 @@ defmodule StateDiffPerfContractTest do assert CAccountMap.storage_root_hash(peak.accounts, addr(1)) == parent_root assert CAccountMap.storage_get(peak.accounts, addr(1), slot(10_001)) == parent_slot end + + test "clone of compact peak does not roughly double RSS (shared_ptr COW)" do + # Phase D acceptance: clone shares CompactStorage; growth should be far below + # a full slot-vector duplication (~accounts * slots * ~64 bytes). + peak = peak_from_compact(400, 16) + before = read_rss_kb() + _fork = State.clone(peak) + after_clone = read_rss_kb() + growth_kb = max(after_clone - before, 0) + # Full deep copy of 400*16 slots would be hundreds of KB of payload alone; + # allow generous overhead for map/trie wrapper fork but fail on ~2x slot dump. + assert growth_kb < 4_000 + end + end + + defp read_rss_kb do + case File.read("/proc/self/status") do + {:ok, status} -> + case Regex.run(~r/VmRSS:\s+(\d+)/, status) do + [_, kb] -> String.to_integer(kb) + _ -> 0 + end + + _ -> + 0 + end end describe "difference_full tuple shape" do - # Phase C extends to a 6-tuple with root_a/root_b; assert shipping 4-tuple until then. - test "difference_full returns addr/sides/storage_diff quads" do + test "difference_full returns addr/sides/storage_diff/roots sextuples" do prev = CAccountMap.new() |> CAccountMap.put(addr(1), 1, 100, [{slot(1), <<1::unsigned-size(256)>>}], <<1>>) @@ -119,7 +144,11 @@ defmodule StateDiffPerfContractTest do full = CAccountMap.difference_full(prev, next) assert full != [] - assert Enum.all?(full, &(tuple_size(&1) == 4)) + assert Enum.all?(full, &(tuple_size(&1) == 6)) + + assert Enum.all?(full, fn {_addr, _a, _b, _diff, root_a, root_b} -> + match?(<<_::binary-size(32)>>, root_a) and match?(<<_::binary-size(32)>>, root_b) + end) end end From 0b33c4efdd544a812c5b2d774f544bbf59e8b2b9 Mon Sep 17 00:00:00 2001 From: Dominic Letz Date: Fri, 17 Jul 2026 12:59:57 +0200 Subject: [PATCH 16/16] Add benches for ChainSql state(uncompact) and state(delta) warnings. Reproduce the >2s Model.ChainSql.state/1 paths without starting the node, and document them next to the existing State.difference bench. Co-authored-by: Cursor --- AGENTS.md | 6 +- docs/caccount-map-nif.md | 19 ++ scripts/state_delta_apply_bench.exs | 340 ++++++++++++++++++++++++++++ scripts/state_uncompact_bench.exs | 223 ++++++++++++++++++ 4 files changed, 586 insertions(+), 2 deletions(-) create mode 100644 scripts/state_delta_apply_bench.exs create mode 100644 scripts/state_uncompact_bench.exs diff --git a/AGENTS.md b/AGENTS.md index 478576c..509f201 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,8 +45,10 @@ Ethereum-compatible JSON-RPC endpoint plus the Diode PEER/EDGE protocols. env pins ports `RPC_PORT=18001`, `EDGE2_PORT=18003`, `PEER_PORT=18004`. - For `Chain.State` / CAccountMap mutability and storage rules, see [`docs/caccount-map-nif.md`](docs/caccount-map-nif.md). Perf contract tests: - `test/state_diff_perf_contract_test.exs`; bench: - `scripts/state_diff_bench.exs`. + `test/state_diff_perf_contract_test.exs`. Benches (no app start): + `scripts/state_diff_bench.exs` (`State.difference`), + `scripts/state_uncompact_bench.exs` (`state(uncompact:…)`), + `scripts/state_delta_apply_bench.exs` (`state(delta:…)`). ### Running the node (dev mode) - `./dev` runs `MIX_ENV=dev iex -S mix run` (wipes `data_dev/` first). For a diff --git a/docs/caccount-map-nif.md b/docs/caccount-map-nif.md index 147f539..e41451d 100644 --- a/docs/caccount-map-nif.md +++ b/docs/caccount-map-nif.md @@ -66,3 +66,22 @@ implementation spec for difference/clone performance work: (`CMerkleTree.nif_stats/0`). No bare-tree test NIF mode. - EVM binary: `evm/evm` (needs `libboost-dev`) - `deps/libsecp256k1`: build once with `make -C deps/libsecp256k1/` (not via `mix`) + +## Performance benches + +Run without starting the full node (`mix run --no-start …`). Defaults target ~14k +accounts (prod jump-block scale). + +| Script | Prod warning / path | +|--------|---------------------| +| `scripts/state_diff_bench.exs` | `State diff took longer than 1s…` / `State.difference` | +| `scripts/state_uncompact_bench.exs` | `state(uncompact:0x…) took Nms` / jump-block `State.uncompact` | +| `scripts/state_delta_apply_bench.exs` | `state(delta:0x…) took Nms` / clone + `apply_difference` + `normalize` | + +Examples: + +```bash +mix run --no-start scripts/state_uncompact_bench.exs -- --scenario sparse_jump +mix run --no-start scripts/state_delta_apply_bench.exs -- --scenario all --changed 20 +mix run --no-start scripts/state_diff_bench.exs -- --scenario compact_small_delta +``` diff --git a/scripts/state_delta_apply_bench.exs b/scripts/state_delta_apply_bench.exs new file mode 100644 index 0000000..684d208 --- /dev/null +++ b/scripts/state_delta_apply_bench.exs @@ -0,0 +1,340 @@ +# Benchmark ChainSql state(delta:…) — the path behind prod +# "state(delta:0x...) took Nms" +# in Model.ChainSql.state/1 for non-jump blocks. +# +# Non-jump blocks store {jump_hash, difference(jump_state, block_state)}. do_state: +# jump = EtsLru.fetch(JumpState, …) # often cached +# jump |> State.clone() |> State.apply_difference(delta) |> State.normalize() +# Warns when wall time > 2s. +# +# This bench times the clone + apply_difference + normalize segment (cached jump). +# +# Run from repo root (no full app start): +# mix run --no-start scripts/state_delta_apply_bench.exs +# mix run --no-start scripts/state_delta_apply_bench.exs -- --accounts 14000 --changed 20 +# mix run --no-start scripts/state_delta_apply_bench.exs -- --scenario all --warmup 1 --iters 3 +# mix run --no-start scripts/state_delta_apply_bench.exs -- --eprof +# +# Scenarios: +# small_delta — jump peak + few changed accounts (common block) +# medium_delta — hundreds of changed accounts +# large_delta — thousands of changed accounts (near ChainSql >10k bug threshold) +# storage_heavy — few accounts, fat storage key deltas +# cold_jump_uncompact — uncompact jump then apply (JumpState cache miss proxy) +# all — run every scenario + +defmodule StateDeltaApplyBench do + @moduledoc false + + alias Chain.State + + def main(argv \\ System.argv()) do + argv = normalize_argv(argv) + + {opts, _, _} = + OptionParser.parse(argv, + strict: [ + accounts: :integer, + changed: :integer, + slots: :integer, + fat_slots: :integer, + warmup: :integer, + iters: :integer, + scenario: :string, + eprof: :boolean, + csv: :boolean + ] + ) + + accounts = Keyword.get(opts, :accounts, 14_000) + changed = Keyword.get(opts, :changed, 20) + slots = Keyword.get(opts, :slots, 8) + fat_slots = Keyword.get(opts, :fat_slots, 2_000) + warmup = Keyword.get(opts, :warmup, 1) + iters = Keyword.get(opts, :iters, 3) + scenario = Keyword.get(opts, :scenario, "all") + eprof? = Keyword.get(opts, :eprof, false) + csv? = Keyword.get(opts, :csv, false) + + _ = CAccountMap.new() + + IO.puts(:stderr, """ + === State delta-apply bench (ChainSql state(delta:…)) === + accounts=#{accounts} changed=#{changed} slots/account=#{slots} fat_slots=#{fat_slots} + warmup=#{warmup} iters=#{iters} scenario=#{scenario} eprof=#{eprof?} + """) + + scenarios = + case scenario do + "all" -> + [ + {:small_delta, fn -> build_delta_case(accounts, changed, slots, :live) end}, + {:medium_delta, + fn -> build_delta_case(accounts, min(changed * 10, accounts), slots, :live) end}, + {:large_delta, + fn -> build_delta_case(accounts, min(2_000, accounts), slots, :live) end}, + {:storage_heavy, + fn -> build_delta_case(min(accounts, 40), min(changed, 20), fat_slots, :live) end}, + {:cold_jump_uncompact, + fn -> build_delta_case(accounts, changed, slots, :from_compact) end} + ] + + name -> + builder = + case name do + "small_delta" -> + fn -> build_delta_case(accounts, changed, slots, :live) end + + "medium_delta" -> + fn -> build_delta_case(accounts, min(changed * 10, accounts), slots, :live) end + + "large_delta" -> + fn -> build_delta_case(accounts, min(2_000, accounts), slots, :live) end + + "storage_heavy" -> + fn -> build_delta_case(min(accounts, 40), min(changed, 20), fat_slots, :live) end + + "cold_jump_uncompact" -> + fn -> build_delta_case(accounts, changed, slots, :from_compact) end + + other -> + raise "unknown --scenario #{inspect(other)}" + end + + [{String.to_atom(name), builder}] + end + + if csv? do + IO.puts( + "scenario,iter,map_size,delta_accounts,storage_keys,clone_ms,apply_ms,normalize_ms,uncompact_ms,total_ms,rss_kb" + ) + end + + Enum.each(scenarios, fn {name, builder} -> + IO.puts(:stderr, "--- building #{name} ---") + {build_us, fixture} = :timer.tc(builder) + map_size = CAccountMap.size(fixture.base.accounts) + + IO.puts( + :stderr, + "built #{name} in #{div(build_us, 1000)}ms map_size=#{map_size} " <> + "delta_accounts=#{length(fixture.delta)} rss=#{div(read_rss_kb(), 1024)}MB" + ) + + for _ <- 1..warmup do + _ = timed_delta_apply(fixture) + end + + if eprof? do + profile_eprof(name, fixture) + end + + samples = + for i <- 1..iters do + sample = timed_delta_apply(fixture) + print_sample(name, i, map_size, sample, csv?) + sample + end + + summarize(name, samples) + end) + + IO.puts(:stderr, """ + + Interpretation hints: + - Matches Model.ChainSql.do_state for {prev_hash, delta} rows (JumpState hit). + - Prod warning fires at >2000ms total for clone+apply+normalize. + - cold_jump_uncompact adds uncompact_ms (JumpState miss proxy). + - apply_ms dominates when delta_accounts or storage_keys is large. + - large_delta approaches the ChainSql length(delta)>10_000 recompress trigger. + """) + end + + defp timed_delta_apply(%{base: base, delta: delta, compact_jump: nil}) do + do_timed_apply(base, delta, 0) + end + + defp timed_delta_apply(%{base: _base, delta: delta, compact_jump: compact}) do + {uncompact_us, jump} = + :timer.tc(fn -> + State.uncompact(compact) |> State.lock() + end) + + sample = do_timed_apply(jump, delta, div(uncompact_us, 1000)) + %{sample | total_ms: sample.total_ms + sample.uncompact_ms} + end + + defp do_timed_apply(base, delta, uncompact_ms) do + {total_us, {clone_us, apply_us, normalize_us, result}} = + :timer.tc(fn -> + {clone_us, cloned} = :timer.tc(fn -> State.clone(base) end) + + {apply_us, applied} = + :timer.tc(fn -> + State.apply_difference(cloned, delta) + end) + + {normalize_us, normalized} = + :timer.tc(fn -> + State.normalize(applied) + end) + + {clone_us, apply_us, normalize_us, normalized} + end) + + storage_keys = + Enum.reduce(delta, 0, fn {_id, report}, acc -> + acc + map_size(Map.get(report, :state, %{})) + end) + + %{ + delta_accounts: length(delta), + storage_keys: storage_keys, + clone_ms: div(clone_us, 1000), + apply_ms: div(apply_us, 1000), + normalize_ms: div(normalize_us, 1000), + uncompact_ms: uncompact_ms, + total_ms: div(total_us, 1000) + uncompact_ms, + hash: result.hash, + rss_kb: read_rss_kb() + } + end + + defp print_sample(name, i, map_size, sample, true) do + IO.puts( + "#{name},#{i},#{map_size},#{sample.delta_accounts},#{sample.storage_keys}," <> + "#{sample.clone_ms},#{sample.apply_ms},#{sample.normalize_ms},#{sample.uncompact_ms}," <> + "#{sample.total_ms},#{sample.rss_kb}" + ) + end + + defp print_sample(name, i, map_size, sample, false) do + IO.puts(:stderr, """ + #{name} iter=#{i} map_size=#{map_size} delta_accounts=#{sample.delta_accounts} storage_keys=#{sample.storage_keys} + clone_ms=#{sample.clone_ms} apply_ms=#{sample.apply_ms} normalize_ms=#{sample.normalize_ms} uncompact_ms=#{sample.uncompact_ms} + total_ms=#{sample.total_ms} rss_mb=#{div(sample.rss_kb, 1024)} + """) + end + + defp summarize(name, samples) do + n = length(samples) + avg = fn key -> div(Enum.sum(Enum.map(samples, &Map.fetch!(&1, key))), n) end + + IO.puts( + :stderr, + "SUMMARY #{name} avg_total_ms=#{avg.(:total_ms)} avg_apply_ms=#{avg.(:apply_ms)} " <> + "avg_clone_ms=#{avg.(:clone_ms)} avg_normalize_ms=#{avg.(:normalize_ms)} " <> + "avg_uncompact_ms=#{avg.(:uncompact_ms)} delta_accounts=#{hd(samples).delta_accounts}" + ) + end + + defp profile_eprof(name, %{base: base, delta: delta, compact_jump: nil}) do + IO.puts(:stderr, "eprof #{name} (clone+apply+normalize)...") + :eprof.start() + + :eprof.profile(fn -> + base |> State.clone() |> State.apply_difference(delta) |> State.normalize() + end) + + :eprof.analyze(:total) + :eprof.stop() + end + + defp profile_eprof(name, fixture) do + profile_eprof(name, %{ + fixture + | compact_jump: nil, + base: State.uncompact(fixture.compact_jump) + }) + end + + # mode :live — base is already an uncompacted/locked peak (JumpState hit) + # mode :from_compact — keep compact jump aside; each sample uncompacts first + defp build_delta_case(n, changed, slots, mode) do + live = build_live_state(n, slots) |> State.normalize() |> State.lock() + compact = State.compact(live) + + base = + case mode do + :live -> live |> State.compact() |> State.uncompact() |> State.lock() + :from_compact -> nil + end + + working = + case mode do + :live -> base + :from_compact -> State.uncompact(compact) |> State.lock() + end + + next = mutate(working, changed, slots) + delta = State.difference(working, next) + + %{ + base: working, + delta: delta, + compact_jump: if(mode == :from_compact, do: compact, else: nil) + } + end + + defp build_live_state(n, slots_per) do + Enum.reduce(1..n, State.new(), fn i, st -> + updates = + for s <- 1..slots_per, into: %{} do + {slot(i * 10_000 + s), <>} + end + + State.set_account(st, addr(i), %Chain.Account{ + nonce: i, + balance: i * 100, + storage_root: Map.to_list(updates), + code: <>, + map_backed: false + }) + end) + end + + defp mutate(state, changed, slots_per) do + state + |> State.clone() + |> then(fn st -> + Enum.reduce(1..changed, st, fn i, acc -> + id = addr(i) + + updates = + for s <- 1..max(div(slots_per, 4), 1), into: %{} do + {slot(i * 10_000 + s), <>} + end + + acc = State.storage_put_map(acc, %{id => updates}) + meta = State.ensure_account(acc, id) + State.set_account(acc, id, %{meta | nonce: meta.nonce + 1}) + end) + end) + |> State.normalize() + end + + defp addr(i), do: <> + defp slot(i), do: <> + + defp read_rss_kb do + case File.read("/proc/self/status") do + {:ok, status} -> + case Regex.run(~r/VmRSS:\s+(\d+)/, status) do + [_, kb] -> String.to_integer(kb) + _ -> 0 + end + + _ -> + 0 + end + end + + defp normalize_argv(argv) do + case Enum.split_while(argv, &(&1 != "--")) do + {_, ["--" | rest]} -> rest + {all, _} -> all + end + end +end + +StateDeltaApplyBench.main() diff --git a/scripts/state_uncompact_bench.exs b/scripts/state_uncompact_bench.exs new file mode 100644 index 0000000..45122b3 --- /dev/null +++ b/scripts/state_uncompact_bench.exs @@ -0,0 +1,223 @@ +# Benchmark Chain.State.uncompact — the path behind prod +# "state(uncompact:0x...) took Nms" +# in Model.ChainSql.state/1 when a jump block's compact state is loaded from DB. +# +# Jump blocks store a compact %Chain.State{}; do_state then runs State.uncompact/1 +# (CAccountMap.uncompact_state/1). Warns when wall time > 2s. +# +# Run from repo root (no full app start): +# mix run --no-start scripts/state_uncompact_bench.exs +# mix run --no-start scripts/state_uncompact_bench.exs -- --accounts 14000 --slots 32 +# mix run --no-start scripts/state_uncompact_bench.exs -- --scenario all --warmup 1 --iters 3 +# mix run --no-start scripts/state_uncompact_bench.exs -- --eprof +# +# Scenarios: +# sparse_jump — many accounts, 1 slot each (typical DB jump-block shape) +# fat_slots — many accounts × slots/account (heavier compact payload) +# nif_only — CAccountMap.uncompact_state only (no %State{} wrap) +# all — run every scenario + +defmodule StateUncompactBench do + @moduledoc false + + alias Chain.State + + def main(argv \\ System.argv()) do + argv = normalize_argv(argv) + + {opts, _, _} = + OptionParser.parse(argv, + strict: [ + accounts: :integer, + slots: :integer, + warmup: :integer, + iters: :integer, + scenario: :string, + eprof: :boolean, + csv: :boolean + ] + ) + + accounts = Keyword.get(opts, :accounts, 14_000) + slots = Keyword.get(opts, :slots, 8) + warmup = Keyword.get(opts, :warmup, 1) + iters = Keyword.get(opts, :iters, 3) + scenario = Keyword.get(opts, :scenario, "all") + eprof? = Keyword.get(opts, :eprof, false) + csv? = Keyword.get(opts, :csv, false) + + _ = CAccountMap.new() + + IO.puts(:stderr, """ + === State.uncompact bench (ChainSql state(uncompact:…)) === + accounts=#{accounts} slots/account=#{slots} + warmup=#{warmup} iters=#{iters} scenario=#{scenario} eprof=#{eprof?} + """) + + scenarios = + case scenario do + "all" -> + [ + {:sparse_jump, fn -> build_compact_state(accounts, 1) end}, + {:fat_slots, fn -> build_compact_state(accounts, slots) end}, + {:nif_only, fn -> build_compact_state(accounts, slots) end} + ] + + name -> + builder = + case name do + "sparse_jump" -> fn -> build_compact_state(accounts, 1) end + "fat_slots" -> fn -> build_compact_state(accounts, slots) end + "nif_only" -> fn -> build_compact_state(accounts, slots) end + other -> raise "unknown --scenario #{inspect(other)}" + end + + [{String.to_atom(name), builder}] + end + + if csv? do + IO.puts("scenario,iter,map_size,uncompact_ms,nif_ms,rss_kb") + end + + Enum.each(scenarios, fn {name, builder} -> + IO.puts(:stderr, "--- building #{name} ---") + {build_us, compact} = :timer.tc(builder) + map_size = compact_account_count(compact) + + IO.puts( + :stderr, + "built #{name} in #{div(build_us, 1000)}ms map_size=#{map_size} rss=#{div(read_rss_kb(), 1024)}MB" + ) + + for _ <- 1..warmup do + _ = timed_uncompact(name, compact) + end + + if eprof? do + profile_eprof(name, compact) + end + + samples = + for i <- 1..iters do + sample = timed_uncompact(name, compact) + print_sample(name, i, map_size, sample, csv?) + sample + end + + summarize(name, samples) + end) + + IO.puts(:stderr, """ + + Interpretation hints: + - Matches Model.ChainSql.do_state when DB row is %Chain.State{} (jump block). + - Prod warning fires at >2000ms; sparse_jump ~14k accounts is the usual shape. + - nif_ms ≈ uncompact_ms: cost is inside account_map_uncompact_state. + """) + end + + # What ChainSql times: State.uncompact/1 on a compact Elixir account map. + defp timed_uncompact(:nif_only, %State{accounts: accounts}) when is_map(accounts) do + {us, _} = :timer.tc(fn -> CAccountMap.uncompact_state(accounts) end) + + %{ + uncompact_ms: div(us, 1000), + nif_ms: div(us, 1000), + rss_kb: read_rss_kb() + } + end + + defp timed_uncompact(_name, %State{} = compact) do + {api_us, _} = :timer.tc(fn -> State.uncompact(compact) end) + + %{ + uncompact_ms: div(api_us, 1000), + nif_ms: div(api_us, 1000), + rss_kb: read_rss_kb() + } + end + + defp print_sample(name, i, map_size, sample, true) do + IO.puts("#{name},#{i},#{map_size},#{sample.uncompact_ms},#{sample.nif_ms},#{sample.rss_kb}") + end + + defp print_sample(name, i, map_size, sample, false) do + IO.puts(:stderr, """ + #{name} iter=#{i} map_size=#{map_size} + uncompact_ms=#{sample.uncompact_ms} nif_ms=#{sample.nif_ms} + rss_mb=#{div(sample.rss_kb, 1024)} + """) + end + + defp summarize(name, samples) do + n = length(samples) + avg = fn key -> div(Enum.sum(Enum.map(samples, &Map.fetch!(&1, key))), n) end + + IO.puts( + :stderr, + "SUMMARY #{name} avg_uncompact_ms=#{avg.(:uncompact_ms)} avg_nif_ms=#{avg.(:nif_ms)}" + ) + end + + defp profile_eprof(name, %State{} = compact) do + IO.puts(:stderr, "eprof #{name} (State.uncompact)...") + :eprof.start() + :eprof.profile(fn -> State.uncompact(compact) end) + :eprof.analyze(:total) + :eprof.stop() + end + + defp build_compact_state(n, slots_per) do + build_live_state(n, slots_per) + |> State.normalize() + |> State.lock() + |> State.compact() + end + + defp compact_account_count(%State{accounts: accounts}) when is_map(accounts), + do: map_size(accounts) + + defp compact_account_count(%State{accounts: accounts}), do: CAccountMap.size(accounts) + + defp build_live_state(n, slots_per) do + Enum.reduce(1..n, State.new(), fn i, st -> + updates = + for s <- 1..slots_per, into: %{} do + {slot(i * 10_000 + s), <>} + end + + State.set_account(st, addr(i), %Chain.Account{ + nonce: i, + balance: i * 100, + storage_root: Map.to_list(updates), + code: <>, + map_backed: false + }) + end) + end + + defp addr(i), do: <> + defp slot(i), do: <> + + defp read_rss_kb do + case File.read("/proc/self/status") do + {:ok, status} -> + case Regex.run(~r/VmRSS:\s+(\d+)/, status) do + [_, kb] -> String.to_integer(kb) + _ -> 0 + end + + _ -> + 0 + end + end + + defp normalize_argv(argv) do + case Enum.split_while(argv, &(&1 != "--")) do + {_, ["--" | rest]} -> rest + {all, _} -> all + end + end +end + +StateUncompactBench.main()